diff options
author | Max Moroz <mmoroz@chromium.org> | 2020-07-13 09:47:11 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-07-13 09:47:11 -0700 |
commit | 2fe0d878ae47586ca355efdbe3d8ef6d26c4dab8 (patch) | |
tree | 5b7c2137eaed57cb57cb0aa8efbfb37885c781a6 /infra | |
parent | 9ed73c1cd7bfd775c32a3af807529c2fd6669f35 (diff) |
Revert "Refactoring gcb libraries for external use by Cloud functions (#4103)" (#4117)
This reverts commit 9ed73c1cd7bfd775c32a3af807529c2fd6669f35.
Diffstat (limited to 'infra')
-rw-r--r-- | infra/gcb/build_and_run_coverage.py | 67 | ||||
-rw-r--r-- | infra/gcb/build_lib.py | 27 | ||||
-rw-r--r-- | infra/gcb/build_project.py | 131 |
3 files changed, 72 insertions, 153 deletions
diff --git a/infra/gcb/build_and_run_coverage.py b/infra/gcb/build_and_run_coverage.py index 671fbd46..6357192a 100644 --- a/infra/gcb/build_and_run_coverage.py +++ b/infra/gcb/build_and_run_coverage.py @@ -1,28 +1,15 @@ -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ #!/usr/bin/python2 """Starts and runs coverage build on Google Cloud Builder. + Usage: build_and_run_coverage.py <project_dir> """ import datetime import json import os +import requests import sys -import yaml +import urlparse import build_lib import build_project @@ -58,29 +45,30 @@ def skip_build(message): # Since the script should print build_id, print '0' as a special value. print('0') - sys.exit(0) + exit(0) def usage(): - """Exit with code 1 and display syntax to use this file.""" sys.stderr.write("Usage: " + sys.argv[0] + " <project_dir>\n") - sys.exit(1) + exit(1) -# pylint: disable=too-many-locals -def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, - base_images_project): - """Returns build steps for project.""" - build_project.load_project_yaml(project_name, project_yaml, image_project) +def get_build_steps(project_dir): + project_name = os.path.basename(project_dir) + project_yaml = build_project.load_project_yaml(project_dir) if project_yaml['disabled']: skip_build('Project "%s" is disabled.' % project_name) - if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT: - skip_build(('Project "{project_name}" is written in "{language}", ' - 'coverage is not supported yet.').format( - project_name=project_name, - language=project_yaml['language'])) + build_script_path = os.path.join(project_dir, 'build.sh') + if os.path.exists(build_script_path): + with open(build_script_path) as fh: + if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT: + skip_build(('Project "{project_name}" is written in "{language}", ' + 'coverage is not supported yet.').format( + project_name=project_name, + language=project_yaml['language'])) + dockerfile_path = os.path.join(project_dir, 'Dockerfile') name = project_yaml['name'] image = project_yaml['image'] language = project_yaml['language'] @@ -93,7 +81,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, env.append('OUT=' + out) env.append('FUZZING_LANGUAGE=' + language) - workdir = build_project.workdir_from_dockerfile(dockerfile_lines) + workdir = build_project.workdir_from_dockerfile(dockerfile_path) if not workdir: workdir = '/src' @@ -144,7 +132,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, coverage_env.append('FULL_SUMMARY_PER_TARGET=1') build_steps.append({ - 'name': f'gcr.io/{base_images_project}/base-runner', + 'name': 'gcr.io/oss-fuzz-base/base-runner', 'env': coverage_env, 'args': [ 'bash', '-c', @@ -203,7 +191,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, # Upload the fuzzer logs. Delete the old ones just in case upload_fuzzer_logs_url = UPLOAD_URL_FORMAT.format(project=project_name, type='logs', - date=report_date) + date=report_date), build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url)) build_steps.append({ 'name': @@ -256,25 +244,12 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, def main(): - """Build and run coverage for projects.""" if len(sys.argv) != 2: usage() - image_project = 'oss-fuzz' - base_images_project = 'oss-fuzz-base' project_dir = sys.argv[1].rstrip(os.path.sep) project_name = os.path.basename(project_dir) - dockerfile_path = os.path.join(project_dir, 'Dockerfile') - project_yaml_path = os.path.join(project_dir, 'project.yaml') - - with open(dockerfile_path) as docker_file: - dockerfile_lines = docker_file.readlines() - - with open(project_yaml_path) as project_yaml_file: - project_yaml = yaml.safe_load(project_yaml_file) - - steps = get_build_steps(project_name, project_yaml, dockerfile_lines, - image_project, base_images_project) + steps = get_build_steps(project_dir) build_project.run_build(steps, project_name, COVERAGE_BUILD_TAG) diff --git a/infra/gcb/build_lib.py b/infra/gcb/build_lib.py index 5997f5e2..0fe22bbd 100644 --- a/infra/gcb/build_lib.py +++ b/infra/gcb/build_lib.py @@ -1,27 +1,12 @@ -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ """Utility module for Google Cloud Build scripts.""" import base64 import collections import os +import requests import sys import time - -import six.moves.urllib.parse as urlparse -import requests +import urllib +import urlparse from oauth2client.service_account import ServiceAccountCredentials @@ -70,19 +55,16 @@ ENGINE_INFO = { def get_targets_list_filename(sanitizer): - """Returns target list filename.""" return TARGETS_LIST_BASENAME + '.' + sanitizer def get_targets_list_url(bucket, project, sanitizer): - """Returns target list url.""" filename = get_targets_list_filename(sanitizer) url = GCS_UPLOAD_URL_FORMAT.format(bucket, project, filename) return url def _get_targets_list(project_name): - """Returns target list.""" # libFuzzer ASan is the default configuration, get list of targets from it. url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket, project_name, 'address') @@ -99,7 +81,6 @@ def _get_targets_list(project_name): def get_signed_url(path, method='PUT', content_type=''): - """Returns signed url.""" timestamp = int(time.time() + BUILD_TIMEOUT) blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path) @@ -114,7 +95,7 @@ def get_signed_url(path, method='PUT', content_type=''): } return ('https://storage.googleapis.com{0}?'.format(path) + - urlparse.urlencode(values)) + urllib.urlencode(values)) def download_corpora_steps(project_name): diff --git a/infra/gcb/build_project.py b/infra/gcb/build_project.py index bf4b462c..e9d0480a 100644 --- a/infra/gcb/build_project.py +++ b/infra/gcb/build_project.py @@ -1,18 +1,3 @@ -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ #!/usr/bin/python2 """Starts project build on Google Cloud Builder. @@ -26,8 +11,6 @@ import json import os import re import sys - -import six import yaml from oauth2client.client import GoogleCredentials @@ -60,26 +43,28 @@ LATEST_VERSION_CONTENT_TYPE = 'text/plain' def usage(): - """Exit with code 1 and display syntax to use this file.""" sys.stderr.write('Usage: ' + sys.argv[0] + ' <project_dir>\n') - sys.exit(1) + exit(1) -def set_yaml_defaults(project_name, project_yaml, image_project): - """Set project.yaml's default parameters.""" - project_yaml.setdefault('disabled', False) - project_yaml.setdefault('name', project_name) - project_yaml.setdefault('image', f'gcr.io/{image_project}/' + project_name) - project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES) - project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS) - project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES) - project_yaml.setdefault('run_tests', True) - project_yaml.setdefault('coverage_extra_args', '') - project_yaml.setdefault('labels', {}) +def load_project_yaml(project_dir): + project_name = os.path.basename(project_dir) + project_yaml_path = os.path.join(project_dir, 'project.yaml') + with open(project_yaml_path) as f: + project_yaml = yaml.safe_load(f) + project_yaml.setdefault('disabled', False) + project_yaml.setdefault('name', project_name) + project_yaml.setdefault('image', 'gcr.io/oss-fuzz/' + project_name) + project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES) + project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS) + project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES) + project_yaml.setdefault('run_tests', True) + project_yaml.setdefault('coverage_extra_args', '') + project_yaml.setdefault('labels', {}) + return project_yaml def is_supported_configuration(fuzzing_engine, sanitizer, architecture): - """Check if the given configuration is supported.""" fuzzing_engine_info = build_lib.ENGINE_INFO[fuzzing_engine] if architecture == 'i386' and sanitizer != 'address': return False @@ -88,13 +73,12 @@ def is_supported_configuration(fuzzing_engine, sanitizer, architecture): def get_sanitizers(project_yaml): - """Retrieve sanitizers from project.yaml.""" sanitizers = project_yaml['sanitizers'] assert isinstance(sanitizers, list) processed_sanitizers = [] for sanitizer in sanitizers: - if isinstance(sanitizer, six.string_types): + if isinstance(sanitizer, basestring): processed_sanitizers.append(sanitizer) elif isinstance(sanitizer, dict): for key in sanitizer.iterkeys(): @@ -103,11 +87,15 @@ def get_sanitizers(project_yaml): return processed_sanitizers -def workdir_from_dockerfile(dockerfile_lines): +def workdir_from_dockerfile(dockerfile): """Parse WORKDIR from the Dockerfile.""" - workdir_regex = re.compile(r'\s*WORKDIR\s*([^\s]+)') - for line in dockerfile_lines: - match = re.match(workdir_regex, line) + WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') + + with open(dockerfile) as f: + lines = f.readlines() + + for line in lines: + match = re.match(WORKDIR_REGEX, line) if match: # We need to escape '$' since they're used for subsitutions in Container # Builer builds. @@ -116,28 +104,21 @@ def workdir_from_dockerfile(dockerfile_lines): return None -def load_project_yaml(project_name, project_yaml, image_project): - """Loads project yaml and sets default values.""" - set_yaml_defaults(project_name, project_yaml, image_project) - - -# pylint: disable=too-many-locals -def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, - base_images_project): - """Returns build steps for project.""" - load_project_yaml(project_name, project_yaml, image_project) +def get_build_steps(project_dir): + project_yaml = load_project_yaml(project_dir) + dockerfile_path = os.path.join(project_dir, 'Dockerfile') name = project_yaml['name'] image = project_yaml['image'] language = project_yaml['language'] run_tests = project_yaml['run_tests'] - time_stamp = datetime.datetime.now().strftime('%Y%m%d%H%M') + ts = datetime.datetime.now().strftime('%Y%m%d%H%M') build_steps = build_lib.project_image_steps(name, image, language) # Copy over MSan instrumented libraries. build_steps.append({ - 'name': f'gcr.io/{base_images_project}/msan-builder', + 'name': 'gcr.io/oss-fuzz-base/msan-builder', 'args': [ 'bash', '-c', @@ -155,7 +136,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, env = CONFIGURATIONS['engine-' + fuzzing_engine][:] env.extend(CONFIGURATIONS['sanitizer-' + sanitizer]) out = '/workspace/out/' + sanitizer - stamped_name = '-'.join([name, sanitizer, time_stamp]) + stamped_name = '-'.join([name, sanitizer, ts]) latest_version_file = '-'.join( [name, sanitizer, LATEST_VERSION_FILENAME]) zip_file = stamped_name + '.zip' @@ -182,7 +163,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, env.append('ARCHITECTURE=' + architecture) env.append('FUZZING_LANGUAGE=' + language) - workdir = workdir_from_dockerfile(dockerfile_lines) + workdir = workdir_from_dockerfile(dockerfile_path) if not workdir: workdir = '/src' @@ -221,7 +202,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, # Patch dynamic libraries to use instrumented ones. build_steps.append({ 'name': - f'gcr.io/{base_images_project}/msan-builder', + 'gcr.io/oss-fuzz-base/msan-builder', 'args': [ 'bash', '-c', @@ -250,7 +231,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, # test binaries { 'name': - f'gcr.io/{base_images_project}/base-runner', + 'gcr.io/oss-fuzz-base/base-runner', 'env': env, 'args': [ @@ -274,8 +255,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, }) if sanitizer == 'dataflow' and fuzzing_engine == 'dataflow': - dataflow_steps = dataflow_post_build_steps(name, env, - base_images_project) + dataflow_steps = dataflow_post_build_steps(name, env) if dataflow_steps: build_steps.extend(dataflow_steps) else: @@ -285,7 +265,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, # generate targets list { 'name': - f'gcr.io/{base_images_project}/base-runner', + 'gcr.io/oss-fuzz-base/base-runner', 'env': env, 'args': [ @@ -307,7 +287,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, }, # upload srcmap { - 'name': f'gcr.io/{base_images_project}/uploader', + 'name': 'gcr.io/oss-fuzz-base/uploader', 'args': [ '/workspace/srcmap.json', srcmap_url, @@ -315,7 +295,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, }, # upload binaries { - 'name': f'gcr.io/{base_images_project}/uploader', + 'name': 'gcr.io/oss-fuzz-base/uploader', 'args': [ os.path.join(out, zip_file), upload_url, @@ -324,7 +304,7 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, # upload targets list { 'name': - f'gcr.io/{base_images_project}/uploader', + 'gcr.io/oss-fuzz-base/uploader', 'args': [ '/workspace/{0}'.format(targets_list_filename), targets_list_url, @@ -347,15 +327,14 @@ def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project, return build_steps -def dataflow_post_build_steps(project_name, env, base_images_project): - """Appends dataflow post build steps.""" +def dataflow_post_build_steps(project_name, env): steps = build_lib.download_corpora_steps(project_name) if not steps: return None steps.append({ 'name': - f'gcr.io/{base_images_project}/base-runner', + 'gcr.io/oss-fuzz-base/base-runner', 'env': env + [ 'COLLECT_DFT_TIMEOUT=2h', @@ -376,16 +355,13 @@ def dataflow_post_build_steps(project_name, env, base_images_project): return steps -def get_logs_url(build_id, image_project='oss-fuzz'): - """Returns url where logs are displayed for the build.""" - url_format = ('https://console.developers.google.com/logs/viewer?' - 'resource=build%2Fbuild_id%2F{0}&project={1}') - return url_format.format(build_id, image_project) +def get_logs_url(build_id): + URL_FORMAT = ('https://console.developers.google.com/logs/viewer?' + 'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz') + return URL_FORMAT.format(build_id) -# pylint: disable=no-member def run_build(build_steps, project_name, tag): - """Run the build for given steps on cloud build.""" options = {} if 'GCB_OPTIONS' in os.environ: options = yaml.safe_load(os.environ['GCB_OPTIONS']) @@ -409,26 +385,13 @@ def run_build(build_steps, project_name, tag): def main(): - """Build and run projects.""" if len(sys.argv) != 2: usage() - image_project = 'oss-fuzz' - base_images_project = 'oss-fuzz-base' project_dir = sys.argv[1].rstrip(os.path.sep) - dockerfile_path = os.path.join(project_dir, 'Dockerfile') - project_yaml_path = os.path.join(project_dir, 'project.yaml') - project_name = os.path.basename(project_dir) - - with open(dockerfile_path) as dockerfile: - dockerfile_lines = dockerfile.readlines() - - with open(project_yaml_path) as project_yaml_file: - project_yaml = yaml.safe_load(project_yaml_file) - - steps = get_build_steps(project_name, project_yaml, dockerfile_lines, - image_project, base_images_project) + steps = get_build_steps(project_dir) + project_name = os.path.basename(project_dir) run_build(steps, project_name, FUZZING_BUILD_TAG) |