diff options
author | 2019-04-02 12:00:55 +0800 | |
---|---|---|
committer | 2019-04-02 12:00:55 +0800 | |
commit | 994900ab47580cbea27b6e908e80964220ef4be3 (patch) | |
tree | 0599bb3c4d5a553a601e85128a99719c5fabee1d /infra/gcb | |
parent | 7bf55a11dfbbd4a720d94b5e523e43f7e72763bb (diff) |
Fix builds status script (#2285)
Get a batch of the last 2000 builds, and filter on those ourselves
instead of calling the cloudbuild API to filter for us. The cloudbuild
API filtering API does not work with our number of builds.
Fixes #2263
Diffstat (limited to 'infra/gcb')
-rwxr-xr-x | infra/gcb/builds_status.py | 67 |
1 files changed, 49 insertions, 18 deletions
diff --git a/infra/gcb/builds_status.py b/infra/gcb/builds_status.py index 730dd0eb..891b5ae6 100755 --- a/infra/gcb/builds_status.py +++ b/infra/gcb/builds_status.py @@ -22,6 +22,8 @@ STATUS_BUCKET = 'oss-fuzz-build-logs' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) RETRY_COUNT = 3 RETRY_WAIT = 5 +MAX_BUILD_RESULTS = 2000 +BUILDS_PAGE_SIZE = 256 def usage(): @@ -58,13 +60,24 @@ def is_build_successful(build): return build['status'] == 'SUCCESS' -def find_last_build(builds): +def find_last_build(builds, project, build_tag_suffix): DELAY_MINUTES = 40 + tag = project + '-' + build_tag_suffix + + builds = builds.get(tag) + if not builds: + return None for build in builds: if build['status'] == 'WORKING': continue + if tag not in build['tags']: + continue + + if not 'finishTime' in build: + continue + finish_time = dateutil.parser.parse(build['finishTime'], ignoretz=True) if (datetime.datetime.utcnow() - finish_time >= datetime.timedelta(minutes=DELAY_MINUTES)): @@ -98,27 +111,44 @@ def execute_with_retries(request): raise +def get_builds(cloudbuild): + """Get a batch of the latest builds (up to MAX_BUILD_RESULTS), grouped by + tag.""" + ungrouped_builds = [] + next_page_token = None + + while True: + page_size = min(BUILDS_PAGE_SIZE, MAX_BUILD_RESULTS - len(ungrouped_builds)) + response = execute_with_retries(cloudbuild.projects().builds().list( + projectId='oss-fuzz', pageSize=page_size, pageToken=next_page_token)) + + if not 'builds' in response: + print >> sys.stderr, 'Invalid response', response + return None + + ungrouped_builds.extend(response['builds']) + if len(ungrouped_builds) >= MAX_BUILD_RESULTS: + break + + next_page_token = response.get('nextPageToken') + + builds = {} + for build in ungrouped_builds: + for tag in build['tags']: + builds.setdefault(tag, []).append(build) + + return builds + + def update_build_status( - cloudbuild, projects, build_tag, status_filename): + builds, projects, build_tag_suffix, status_filename): successes = [] failures = [] + for project in projects: print project - query_filter = ('images="gcr.io/oss-fuzz/{0}" AND tags="{1}"'.format( - project, build_tag)) - try: - response = execute_with_retries(cloudbuild.projects().builds().list( - projectId='oss-fuzz', pageSize=2, filter=query_filter)) - except googleapiclient.errors.HttpError as e: - print >> sys.stderr, 'Failed to list builds for', project, ':', str(e) - continue - - if not 'builds' in response: - continue - - builds = response['builds'] - last_build = find_last_build(builds) + last_build = find_last_build(builds, project, build_tag_suffix) if not last_build: print >> sys.stderr, 'Failed to get build for', project continue @@ -152,9 +182,10 @@ def main(): credentials = GoogleCredentials.get_application_default() cloudbuild = gcb_build('cloudbuild', 'v1', credentials=credentials) - update_build_status(cloudbuild, projects, build_project.FUZZING_BUILD_TAG, + builds = get_builds(cloudbuild) + update_build_status(builds, projects, build_project.FUZZING_BUILD_TAG, status_filename='status.json') - update_build_status(cloudbuild, projects, + update_build_status(builds, projects, build_and_run_coverage.COVERAGE_BUILD_TAG, status_filename='status-coverage.json') |