diff options
-rwxr-xr-x | infra/base-images/base-builder/compile | 2 | ||||
-rwxr-xr-x | infra/build/functions/build_and_run_coverage.py | 2 | ||||
-rwxr-xr-x | infra/build/functions/deploy.sh | 6 | ||||
-rw-r--r-- | infra/build/functions/project_sync.py | 32 | ||||
-rw-r--r-- | infra/build/functions/project_sync_test.py | 44 |
5 files changed, 67 insertions, 19 deletions
diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index e2c29993..da8fadbe 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -216,7 +216,7 @@ if [ "$SANITIZER" = "introspector" ]; then fi cd $SRC/inspector - python3 $SRC/fuzz-introspector/post-processing/main.py --target_dir=$SRC/inspector --git_repo_url=$GIT_REPO --coverage_url=$COVERAGE_URL + python3 /fuzz-introspector/post-processing/main.py --target_dir=$SRC/inspector --git_repo_url=$GIT_REPO --coverage_url=$COVERAGE_URL cp -rf $SRC/inspector $OUT/inspector fi diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 330886ad..fbe17126 100755 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -34,7 +34,7 @@ ARCHITECTURE = 'x86_64' PLATFORM = 'linux' COVERAGE_BUILD_TYPE = 'coverage' -INTROSPECTOR_BUILD_TYPE = 'fuzz_introspector' +INTROSPECTOR_BUILD_TYPE = 'introspector' # This is needed for ClusterFuzz to pick up the most recent reports data. diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh index 94f0bd8f..e02dbcee 100755 --- a/infra/build/functions/deploy.sh +++ b/infra/build/functions/deploy.sh @@ -106,12 +106,6 @@ deploy_scheduler $BASE_IMAGE_SCHEDULER_JOB \ "$BASE_IMAGE_MESSAGE" \ $PROJECT_ID -deploy_scheduler $UPDATE_BUILD_SCHEDULER_JOB-introspector \ - "$UPDATE_BUILD_JOB_SCHEDULE" \ - $UPDATE_BUILD_JOB_TOPIC \ - "introspector" \ - $PROJECT_ID - deploy_cloud_function sync \ sync \ $SYNC_JOB_TOPIC \ diff --git a/infra/build/functions/project_sync.py b/infra/build/functions/project_sync.py index 7b30cae2..b2531d84 100644 --- a/infra/build/functions/project_sync.py +++ b/infra/build/functions/project_sync.py @@ -37,10 +37,13 @@ MAX_BUILDS_PER_DAY = 4 COVERAGE_SCHEDULE = '0 6 * * *' FUZZING_BUILD_TOPIC = 'request-build' COVERAGE_BUILD_TOPIC = 'request-coverage-build' +INTROSPECTOR_BUILD_TOPIC = 'request-introspector-build' ProjectMetadata = namedtuple( 'ProjectMetadata', 'schedule project_yaml_contents dockerfile_contents') +logging.basicConfig(level=logging.INFO) + class ProjectYamlError(Exception): """Error in project.yaml format.""" @@ -61,7 +64,10 @@ def create_scheduler(cloud_scheduler_client, project_name, schedule, tag, 'schedule': schedule } - cloud_scheduler_client.create_job(parent, job) + try: + cloud_scheduler_client.create_job(parent, job) + except exceptions.AlreadyExists: + return def delete_scheduler(cloud_scheduler_client, project_name, tag): @@ -95,7 +101,8 @@ def delete_project(cloud_scheduler_client, project): """Delete the given project.""" logging.info('Deleting project %s', project.name) for tag in (build_project.FUZZING_BUILD_TYPE, - build_and_run_coverage.COVERAGE_BUILD_TYPE): + build_and_run_coverage.COVERAGE_BUILD_TYPE, + build_and_run_coverage.INTROSPECTOR_BUILD_TYPE): try: delete_scheduler(cloud_scheduler_client, project.name, tag) except exceptions.NotFound: @@ -118,9 +125,6 @@ def sync_projects(cloud_scheduler_client, projects): existing_projects = {project.name for project in Project.query()} for project_name in projects: - if project_name in existing_projects: - continue - try: create_scheduler(cloud_scheduler_client, project_name, projects[project_name].schedule, @@ -128,14 +132,22 @@ def sync_projects(cloud_scheduler_client, projects): create_scheduler(cloud_scheduler_client, project_name, COVERAGE_SCHEDULE, build_and_run_coverage.COVERAGE_BUILD_TYPE, COVERAGE_BUILD_TOPIC) - project_metadata = projects[project_name] - Project(name=project_name, - schedule=project_metadata.schedule, - project_yaml_contents=project_metadata.project_yaml_contents, - dockerfile_contents=project_metadata.dockerfile_contents).put() + create_scheduler(cloud_scheduler_client, project_name, COVERAGE_SCHEDULE, + build_and_run_coverage.INTROSPECTOR_BUILD_TYPE, + INTROSPECTOR_BUILD_TOPIC) except exceptions.GoogleAPICallError as error: logging.error('Scheduler creation for %s failed with %s', project_name, error) + continue + + if project_name in existing_projects: + continue + + project_metadata = projects[project_name] + Project(name=project_name, + schedule=project_metadata.schedule, + project_yaml_contents=project_metadata.project_yaml_contents, + dockerfile_contents=project_metadata.dockerfile_contents).put() for project in Project.query(): if project.name not in projects: diff --git a/infra/build/functions/project_sync_test.py b/infra/build/functions/project_sync_test.py index ad1330ea..37a46b5f 100644 --- a/infra/build/functions/project_sync_test.py +++ b/infra/build/functions/project_sync_test.py @@ -93,7 +93,7 @@ class CloudSchedulerClient: def update_job(self, job, update_mask): """Simulate update jobs.""" for existing_job in self.schedulers: - if existing_job == job: + if existing_job == job and 'schedule' in update_mask: job['schedule'] = update_mask['schedule'] @@ -161,6 +161,37 @@ class TestDataSync(unittest.TestCase): self.assertCountEqual([ { 'name': 'projects/test-project/location/us-central1/jobs/' + 'test1-scheduler-fuzzing', + 'pubsub_target': { + 'topic_name': 'projects/test-project/topics/request-build', + 'data': b'test1' + }, + 'schedule': '0 8 * * *' + }, + { + 'name': 'projects/test-project/location/us-central1/jobs/' + 'test1-scheduler-coverage', + 'pubsub_target': { + 'topic_name': + 'projects/test-project/topics/request-coverage-build', + 'data': + b'test1' + }, + 'schedule': '0 6 * * *' + }, + { + 'name': 'projects/test-project/location/us-central1/jobs/' + 'test1-scheduler-introspector', + 'pubsub_target': { + 'topic_name': + 'projects/test-project/topics/request-introspector-build', + 'data': + b'test1' + }, + 'schedule': '0 6 * * *' + }, + { + 'name': 'projects/test-project/location/us-central1/jobs/' 'test2-scheduler-fuzzing', 'pubsub_target': { 'topic_name': 'projects/test-project/topics/request-build', @@ -179,6 +210,17 @@ class TestDataSync(unittest.TestCase): }, 'schedule': '0 6 * * *' }, + { + 'name': 'projects/test-project/location/us-central1/jobs/' + 'test2-scheduler-introspector', + 'pubsub_target': { + 'topic_name': + 'projects/test-project/topics/request-introspector-build', + 'data': + b'test2' + }, + 'schedule': '0 6 * * *' + }, ], cloud_scheduler_client.schedulers) def test_sync_projects_delete(self): |