From 5e3348cdb95d359613cb9931c14165de00e06e0c Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Fri, 31 Jul 2020 10:56:18 +1000 Subject: Create individual schedulers for requesting coverage builds. (#4237) The single function for requesting all coverage builds was timing out. --- infra/build/functions/deploy.sh | 9 ------ infra/build/functions/project_sync.py | 18 +++++++++--- infra/build/functions/project_sync_test.py | 28 ++++++++++++++++-- infra/build/functions/request_build.py | 8 ++---- infra/build/functions/request_coverage_build.py | 38 +++++++++++++------------ infra/build/request_all_builds.sh | 2 +- infra/build/request_build.sh | 12 +++++++- 7 files changed, 75 insertions(+), 40 deletions(-) diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh index d12ed07d..77e004ad 100755 --- a/infra/build/functions/deploy.sh +++ b/infra/build/functions/deploy.sh @@ -23,9 +23,6 @@ BASE_IMAGE_MESSAGE="Start base image build" BUILD_JOB_TOPIC=request-build COVERAGE_BUILD_JOB_TOPIC=request-coverage-build -COVERAGE_BUILD_SCHEDULER_JOB=coverage-build-scheduler -COVERAGE_BUILD_SCHEDULE="0 6 * * *" -COVERAGE_BUILD_MESSAGE="Start coverage report builds" SYNC_JOB_TOPIC=schedule-project-sync SYNC_SCHEDULER_JOB=sync-scheduler @@ -111,12 +108,6 @@ deploy_scheduler $BASE_IMAGE_SCHEDULER_JOB \ "$BASE_IMAGE_MESSAGE" \ $PROJECT_ID -deploy_scheduler $COVERAGE_BUILD_SCHEDULER_JOB \ - "$COVERAGE_BUILD_SCHEDULE" \ - $COVERAGE_BUILD_JOB_TOPIC \ - "$COVERAGE_BUILD_MESSAGE" \ - $PROJECT_ID - deploy_scheduler $UPDATE_BUILD_SCHEDULER_JOB \ "$UPDATE_BUILD_JOB_SCHEDULE" \ $UPDATE_BUILD_JOB_TOPIC \ diff --git a/infra/build/functions/project_sync.py b/infra/build/functions/project_sync.py index 10140839..e695a3a6 100644 --- a/infra/build/functions/project_sync.py +++ b/infra/build/functions/project_sync.py @@ -26,12 +26,17 @@ from google.api_core import exceptions from google.cloud import ndb from google.cloud import scheduler_v1 +import build_and_run_coverage +import build_project from datastore_entities import GithubCreds from datastore_entities import Project VALID_PROJECT_NAME = re.compile(r'^[a-zA-Z0-9_-]+$') DEFAULT_BUILDS_PER_DAY = 1 MAX_BUILDS_PER_DAY = 4 +COVERAGE_SCHEDULE = '0 6 * * *' +FUZZING_BUILD_TOPIC = 'request-build' +COVERAGE_BUILD_TOPIC = 'request-coverage-build' ProjectMetadata = namedtuple( 'ProjectMetadata', 'schedule project_yaml_contents dockerfile_contents') @@ -41,15 +46,16 @@ class ProjectYamlError(Exception): """Error in project.yaml format.""" -def create_scheduler(cloud_scheduler_client, project_name, schedule): +def create_scheduler(cloud_scheduler_client, project_name, schedule, tag, + topic): """Creates schedulers for new projects.""" project_id = os.environ.get('GCP_PROJECT') location_id = os.environ.get('FUNCTION_REGION') parent = cloud_scheduler_client.location_path(project_id, location_id) job = { - 'name': parent + '/jobs/' + project_name + '-scheduler', + 'name': parent + '/jobs/' + project_name + '-scheduler-' + tag, 'pubsub_target': { - 'topic_name': 'projects/' + project_id + '/topics/request-build', + 'topic_name': 'projects/' + project_id + '/topics/' + topic, 'data': project_name.encode() }, 'schedule': schedule @@ -107,7 +113,11 @@ def sync_projects(cloud_scheduler_client, projects): try: create_scheduler(cloud_scheduler_client, project_name, - projects[project_name].schedule) + projects[project_name].schedule, + build_project.FUZZING_BUILD_TAG, FUZZING_BUILD_TOPIC) + create_scheduler(cloud_scheduler_client, project_name, COVERAGE_SCHEDULE, + build_and_run_coverage.COVERAGE_BUILD_TAG, + COVERAGE_BUILD_TOPIC) project_metadata = projects[project_name] Project(name=project_name, schedule=project_metadata.schedule, diff --git a/infra/build/functions/project_sync_test.py b/infra/build/functions/project_sync_test.py index 416029d1..4d7214f4 100644 --- a/infra/build/functions/project_sync_test.py +++ b/infra/build/functions/project_sync_test.py @@ -69,8 +69,7 @@ class CloudSchedulerClient: def create_job(self, parent, job): """Simulate create job.""" del parent - if job['name'] not in self.schedulers: - self.schedulers.append(job) + self.schedulers.append(job) # pylint: disable=no-self-use def job_path(self, project_id, location_id, name): @@ -153,6 +152,31 @@ class TestDataSync(unittest.TestCase): 'test2': '0 7 * * *' }, {project.name: project.schedule for project in projects_query}) + self.assertCountEqual([ + { + 'name': + 'projects/test-project/location/us-central1/jobs/test2-scheduler-fuzzing', + 'pubsub_target': { + 'topic_name': 'projects/test-project/topics/request-build', + 'data': b'test2' + }, + 'schedule': + '0 7 * * *' + }, + { + 'name': + 'projects/test-project/location/us-central1/jobs/test2-scheduler-coverage', + 'pubsub_target': { + 'topic_name': + 'projects/test-project/topics/request-coverage-build', + 'data': + b'test2' + }, + 'schedule': + '0 6 * * *' + }, + ], cloud_scheduler_client.schedulers) + def test_sync_projects_delete(self): """Testing sync_projects() deleting.""" cloud_scheduler_client = CloudSchedulerClient() diff --git a/infra/build/functions/request_build.py b/infra/build/functions/request_build.py index 4f835a10..344734cd 100644 --- a/infra/build/functions/request_build.py +++ b/infra/build/functions/request_build.py @@ -65,11 +65,9 @@ def get_project_data(project_name): def get_build_steps(project_name, image_project, base_images_project): """Retrieve build steps.""" project_yaml_contents, dockerfile_lines = get_project_data(project_name) - build_steps = build_project.get_build_steps(project_name, - project_yaml_contents, - dockerfile_lines, image_project, - base_images_project) - return build_steps + return build_project.get_build_steps(project_name, project_yaml_contents, + dockerfile_lines, image_project, + base_images_project) # pylint: disable=no-member diff --git a/infra/build/functions/request_coverage_build.py b/infra/build/functions/request_coverage_build.py index 0bc57e44..e861875a 100644 --- a/infra/build/functions/request_coverage_build.py +++ b/infra/build/functions/request_coverage_build.py @@ -14,36 +14,38 @@ # ################################################################################ """Cloud function that requests coverage builds.""" +import base64 import google.auth from google.cloud import ndb import build_and_run_coverage -from datastore_entities import Project import request_build BASE_PROJECT = 'oss-fuzz-base' +def get_build_steps(project_name, image_project, base_images_project): + """Retrieve build steps.""" + project_yaml_contents, dockerfile_lines = request_build.get_project_data( + project_name) + return build_and_run_coverage.get_build_steps(project_name, + project_yaml_contents, + dockerfile_lines, image_project, + base_images_project) + + def request_coverage_build(event, context): """Entry point for coverage build cloud function.""" - del event, context #unused + del context #unused + if 'data' in event: + project_name = base64.b64decode(event['data']).decode('utf-8') + else: + raise RuntimeError('Project name missing from payload') with ndb.Client().context(): credentials, image_project = google.auth.default() - for project in Project.query(): - project_name = project.name - project_yaml_contents = project.project_yaml_contents - dockerfile_lines = project.dockerfile_contents.split('\n') - # Catching sys.exit() for a project's build steps to avoid it - # from interferring with other remaining builds. - try: - build_steps = build_and_run_coverage.get_build_steps( - project_name, project_yaml_contents, dockerfile_lines, - image_project, BASE_PROJECT) - except SystemExit: - continue - - request_build.run_build(project_name, image_project, build_steps, - credentials, - build_and_run_coverage.COVERAGE_BUILD_TAG) + build_steps = get_build_steps(project_name, image_project, BASE_PROJECT) + request_build.run_build(project_name, image_project, build_steps, + credentials, + build_and_run_coverage.COVERAGE_BUILD_TAG) diff --git a/infra/build/request_all_builds.sh b/infra/build/request_all_builds.sh index 2c7f39be..15273869 100755 --- a/infra/build/request_all_builds.sh +++ b/infra/build/request_all_builds.sh @@ -20,5 +20,5 @@ for project in ../../projects/*; do continue fi - ./request_build.sh $(basename $project) + ./request_build.sh $(basename $project) $1 done diff --git a/infra/build/request_build.sh b/infra/build/request_build.sh index 58e1de4c..36a9cb47 100755 --- a/infra/build/request_build.sh +++ b/infra/build/request_build.sh @@ -14,4 +14,14 @@ # limitations under the License. # ################################################################################ -gcloud pubsub topics publish request-build --message "$1" --project oss-fuzz + +if [ "$2" = "fuzzing" ]; then + topic=request-build +elif [ "$2" = "coverage" ]; then + topic=request-coverage-build +else + echo "Invalid build type $2." + exit 1 +fi + +gcloud pubsub topics publish $topic --message "$1" --project oss-fuzz -- cgit v1.2.3