Refactoring gcb libraries for external use by Cloud functions (#4103)

* Refactoring gcb libraries for external use

* Few changes done, a couple more left

* Fixed linting/formatting issues + other changes requested

* Fixing import order

* Fixing import order

* license header change

* Undo

Co-authored-by: Kabeer Seth <kabeerseth@google.com>

3 files changed, 153 insertions, 72 deletions

diff --git a/infra/gcb/build_and_run_coverage.py b/infra/gcb/build_and_run_coverage.py
index 6357192a..671fbd46 100644
--- a/infra/gcb/build_and_run_coverage.py
+++ b/infra/gcb/build_and_run_coverage.py
@@ -1,15 +1,28 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
 #!/usr/bin/python2
 """Starts and runs coverage build on Google Cloud Builder.
-
 Usage: build_and_run_coverage.py <project_dir>
 """
 
 import datetime
 import json
 import os
-import requests
 import sys
-import urlparse
+import yaml
 
 import build_lib
 import build_project
@@ -45,30 +58,29 @@ def skip_build(message):
 
   # Since the script should print build_id, print '0' as a special value.
   print('0')
-  exit(0)
+  sys.exit(0)
 
 
 def usage():
+  """Exit with code 1 and display syntax to use this file."""
   sys.stderr.write("Usage: " + sys.argv[0] + " <project_dir>\n")
-  exit(1)
+  sys.exit(1)
 
 
-def get_build_steps(project_dir):
-  project_name = os.path.basename(project_dir)
-  project_yaml = build_project.load_project_yaml(project_dir)
+# pylint: disable=too-many-locals
+def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project,
+                    base_images_project):
+  """Returns build steps for project."""
+  build_project.load_project_yaml(project_name, project_yaml, image_project)
   if project_yaml['disabled']:
     skip_build('Project "%s" is disabled.' % project_name)
 
-  build_script_path = os.path.join(project_dir, 'build.sh')
-  if os.path.exists(build_script_path):
-    with open(build_script_path) as fh:
-      if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT:
-        skip_build(('Project "{project_name}" is written in "{language}", '
-                    'coverage is not supported yet.').format(
-                        project_name=project_name,
-                        language=project_yaml['language']))
+  if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT:
+    skip_build(('Project "{project_name}" is written in "{language}", '
+                'coverage is not supported yet.').format(
+                    project_name=project_name,
+                    language=project_yaml['language']))
 
-  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
   name = project_yaml['name']
   image = project_yaml['image']
   language = project_yaml['language']
@@ -81,7 +93,7 @@ def get_build_steps(project_dir):
   env.append('OUT=' + out)
   env.append('FUZZING_LANGUAGE=' + language)
 
-  workdir = build_project.workdir_from_dockerfile(dockerfile_path)
+  workdir = build_project.workdir_from_dockerfile(dockerfile_lines)
   if not workdir:
     workdir = '/src'
 
@@ -132,7 +144,7 @@ def get_build_steps(project_dir):
     coverage_env.append('FULL_SUMMARY_PER_TARGET=1')
 
   build_steps.append({
-      'name': 'gcr.io/oss-fuzz-base/base-runner',
+      'name': f'gcr.io/{base_images_project}/base-runner',
       'env': coverage_env,
       'args': [
           'bash', '-c',
@@ -191,7 +203,7 @@ def get_build_steps(project_dir):
   # Upload the fuzzer logs. Delete the old ones just in case
   upload_fuzzer_logs_url = UPLOAD_URL_FORMAT.format(project=project_name,
                                                     type='logs',
-                                                    date=report_date),
+                                                    date=report_date)
   build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url))
   build_steps.append({
       'name':
@@ -244,12 +256,25 @@ def get_build_steps(project_dir):
 
 
 def main():
+  """Build and run coverage for projects."""
   if len(sys.argv) != 2:
     usage()
 
+  image_project = 'oss-fuzz'
+  base_images_project = 'oss-fuzz-base'
   project_dir = sys.argv[1].rstrip(os.path.sep)
   project_name = os.path.basename(project_dir)
-  steps = get_build_steps(project_dir)
+  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
+  project_yaml_path = os.path.join(project_dir, 'project.yaml')
+
+  with open(dockerfile_path) as docker_file:
+    dockerfile_lines = docker_file.readlines()
+
+  with open(project_yaml_path) as project_yaml_file:
+    project_yaml = yaml.safe_load(project_yaml_file)
+
+  steps = get_build_steps(project_name, project_yaml, dockerfile_lines,
+                          image_project, base_images_project)
   build_project.run_build(steps, project_name, COVERAGE_BUILD_TAG)
 
 
diff --git a/infra/gcb/build_lib.py b/infra/gcb/build_lib.py
index 0fe22bbd..5997f5e2 100644
--- a/infra/gcb/build_lib.py
+++ b/infra/gcb/build_lib.py
@@ -1,12 +1,27 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
 """Utility module for Google Cloud Build scripts."""
 import base64
 import collections
 import os
-import requests
 import sys
 import time
-import urllib
-import urlparse
+
+import six.moves.urllib.parse as urlparse
+import requests
 
 from oauth2client.service_account import ServiceAccountCredentials
 
@@ -55,16 +70,19 @@ ENGINE_INFO = {
 
 
 def get_targets_list_filename(sanitizer):
+  """Returns target list filename."""
   return TARGETS_LIST_BASENAME + '.' + sanitizer
 
 
 def get_targets_list_url(bucket, project, sanitizer):
+  """Returns target list url."""
   filename = get_targets_list_filename(sanitizer)
   url = GCS_UPLOAD_URL_FORMAT.format(bucket, project, filename)
   return url
 
 
 def _get_targets_list(project_name):
+  """Returns target list."""
   # libFuzzer ASan is the default configuration, get list of targets from it.
   url = get_targets_list_url(ENGINE_INFO['libfuzzer'].upload_bucket,
                              project_name, 'address')
@@ -81,6 +99,7 @@ def _get_targets_list(project_name):
 
 
 def get_signed_url(path, method='PUT', content_type=''):
+  """Returns signed url."""
   timestamp = int(time.time() + BUILD_TIMEOUT)
   blob = '{0}\n\n{1}\n{2}\n{3}'.format(method, content_type, timestamp, path)
 
@@ -95,7 +114,7 @@ def get_signed_url(path, method='PUT', content_type=''):
   }
 
   return ('https://storage.googleapis.com{0}?'.format(path) +
-          urllib.urlencode(values))
+          urlparse.urlencode(values))
 
 
 def download_corpora_steps(project_name):
diff --git a/infra/gcb/build_project.py b/infra/gcb/build_project.py
index e9d0480a..bf4b462c 100644
--- a/infra/gcb/build_project.py
+++ b/infra/gcb/build_project.py
@@ -1,3 +1,18 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
 #!/usr/bin/python2
 """Starts project build on Google Cloud Builder.
 
@@ -11,6 +26,8 @@ import json
 import os
 import re
 import sys
+
+import six
 import yaml
 
 from oauth2client.client import GoogleCredentials
@@ -43,28 +60,26 @@ LATEST_VERSION_CONTENT_TYPE = 'text/plain'
 
 
 def usage():
+  """Exit with code 1 and display syntax to use this file."""
   sys.stderr.write('Usage: ' + sys.argv[0] + ' <project_dir>\n')
-  exit(1)
+  sys.exit(1)
 
 
-def load_project_yaml(project_dir):
-  project_name = os.path.basename(project_dir)
-  project_yaml_path = os.path.join(project_dir, 'project.yaml')
-  with open(project_yaml_path) as f:
-    project_yaml = yaml.safe_load(f)
-    project_yaml.setdefault('disabled', False)
-    project_yaml.setdefault('name', project_name)
-    project_yaml.setdefault('image', 'gcr.io/oss-fuzz/' + project_name)
-    project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES)
-    project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
-    project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
-    project_yaml.setdefault('run_tests', True)
-    project_yaml.setdefault('coverage_extra_args', '')
-    project_yaml.setdefault('labels', {})
-    return project_yaml
+def set_yaml_defaults(project_name, project_yaml, image_project):
+  """Set project.yaml's default parameters."""
+  project_yaml.setdefault('disabled', False)
+  project_yaml.setdefault('name', project_name)
+  project_yaml.setdefault('image', f'gcr.io/{image_project}/' + project_name)
+  project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES)
+  project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
+  project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
+  project_yaml.setdefault('run_tests', True)
+  project_yaml.setdefault('coverage_extra_args', '')
+  project_yaml.setdefault('labels', {})
 
 
 def is_supported_configuration(fuzzing_engine, sanitizer, architecture):
+  """Check if the given configuration is supported."""
   fuzzing_engine_info = build_lib.ENGINE_INFO[fuzzing_engine]
   if architecture == 'i386' and sanitizer != 'address':
     return False
@@ -73,12 +88,13 @@ def is_supported_configuration(fuzzing_engine, sanitizer, architecture):
 
 
 def get_sanitizers(project_yaml):
+  """Retrieve sanitizers from project.yaml."""
   sanitizers = project_yaml['sanitizers']
   assert isinstance(sanitizers, list)
 
   processed_sanitizers = []
   for sanitizer in sanitizers:
-    if isinstance(sanitizer, basestring):
+    if isinstance(sanitizer, six.string_types):
       processed_sanitizers.append(sanitizer)
     elif isinstance(sanitizer, dict):
       for key in sanitizer.iterkeys():
@@ -87,15 +103,11 @@ def get_sanitizers(project_yaml):
   return processed_sanitizers
 
 
-def workdir_from_dockerfile(dockerfile):
+def workdir_from_dockerfile(dockerfile_lines):
   """Parse WORKDIR from the Dockerfile."""
-  WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)')
-
-  with open(dockerfile) as f:
-    lines = f.readlines()
-
-  for line in lines:
-    match = re.match(WORKDIR_REGEX, line)
+  workdir_regex = re.compile(r'\s*WORKDIR\s*([^\s]+)')
+  for line in dockerfile_lines:
+    match = re.match(workdir_regex, line)
     if match:
       # We need to escape '$' since they're used for subsitutions in Container
       # Builer builds.
@@ -104,21 +116,28 @@ def workdir_from_dockerfile(dockerfile):
   return None
 
 
-def get_build_steps(project_dir):
-  project_yaml = load_project_yaml(project_dir)
-  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
+def load_project_yaml(project_name, project_yaml, image_project):
+  """Loads project yaml and sets default values."""
+  set_yaml_defaults(project_name, project_yaml, image_project)
+
+
+# pylint: disable=too-many-locals
+def get_build_steps(project_name, project_yaml, dockerfile_lines, image_project,
+                    base_images_project):
+  """Returns build steps for project."""
+  load_project_yaml(project_name, project_yaml, image_project)
   name = project_yaml['name']
   image = project_yaml['image']
   language = project_yaml['language']
   run_tests = project_yaml['run_tests']
 
-  ts = datetime.datetime.now().strftime('%Y%m%d%H%M')
+  time_stamp = datetime.datetime.now().strftime('%Y%m%d%H%M')
 
   build_steps = build_lib.project_image_steps(name, image, language)
 
   # Copy over MSan instrumented libraries.
   build_steps.append({
-      'name': 'gcr.io/oss-fuzz-base/msan-builder',
+      'name': f'gcr.io/{base_images_project}/msan-builder',
       'args': [
           'bash',
           '-c',
@@ -136,7 +155,7 @@ def get_build_steps(project_dir):
         env = CONFIGURATIONS['engine-' + fuzzing_engine][:]
         env.extend(CONFIGURATIONS['sanitizer-' + sanitizer])
         out = '/workspace/out/' + sanitizer
-        stamped_name = '-'.join([name, sanitizer, ts])
+        stamped_name = '-'.join([name, sanitizer, time_stamp])
         latest_version_file = '-'.join(
             [name, sanitizer, LATEST_VERSION_FILENAME])
         zip_file = stamped_name + '.zip'
@@ -163,7 +182,7 @@ def get_build_steps(project_dir):
         env.append('ARCHITECTURE=' + architecture)
         env.append('FUZZING_LANGUAGE=' + language)
 
-        workdir = workdir_from_dockerfile(dockerfile_path)
+        workdir = workdir_from_dockerfile(dockerfile_lines)
         if not workdir:
           workdir = '/src'
 
@@ -202,7 +221,7 @@ def get_build_steps(project_dir):
           # Patch dynamic libraries to use instrumented ones.
           build_steps.append({
               'name':
-                  'gcr.io/oss-fuzz-base/msan-builder',
+                  f'gcr.io/{base_images_project}/msan-builder',
               'args': [
                   'bash',
                   '-c',
@@ -231,7 +250,7 @@ def get_build_steps(project_dir):
               # test binaries
               {
                   'name':
-                      'gcr.io/oss-fuzz-base/base-runner',
+                      f'gcr.io/{base_images_project}/base-runner',
                   'env':
                       env,
                   'args': [
@@ -255,7 +274,8 @@ def get_build_steps(project_dir):
           })
 
         if sanitizer == 'dataflow' and fuzzing_engine == 'dataflow':
-          dataflow_steps = dataflow_post_build_steps(name, env)
+          dataflow_steps = dataflow_post_build_steps(name, env,
+                                                     base_images_project)
           if dataflow_steps:
             build_steps.extend(dataflow_steps)
           else:
@@ -265,7 +285,7 @@ def get_build_steps(project_dir):
             # generate targets list
             {
                 'name':
-                    'gcr.io/oss-fuzz-base/base-runner',
+                    f'gcr.io/{base_images_project}/base-runner',
                 'env':
                     env,
                 'args': [
@@ -287,7 +307,7 @@ def get_build_steps(project_dir):
             },
             # upload srcmap
             {
-                'name': 'gcr.io/oss-fuzz-base/uploader',
+                'name': f'gcr.io/{base_images_project}/uploader',
                 'args': [
                     '/workspace/srcmap.json',
                     srcmap_url,
@@ -295,7 +315,7 @@ def get_build_steps(project_dir):
             },
             # upload binaries
             {
-                'name': 'gcr.io/oss-fuzz-base/uploader',
+                'name': f'gcr.io/{base_images_project}/uploader',
                 'args': [
                     os.path.join(out, zip_file),
                     upload_url,
@@ -304,7 +324,7 @@ def get_build_steps(project_dir):
             # upload targets list
             {
                 'name':
-                    'gcr.io/oss-fuzz-base/uploader',
+                    f'gcr.io/{base_images_project}/uploader',
                 'args': [
                     '/workspace/{0}'.format(targets_list_filename),
                     targets_list_url,
@@ -327,14 +347,15 @@ def get_build_steps(project_dir):
   return build_steps
 
 
-def dataflow_post_build_steps(project_name, env):
+def dataflow_post_build_steps(project_name, env, base_images_project):
+  """Appends dataflow post build steps."""
   steps = build_lib.download_corpora_steps(project_name)
   if not steps:
     return None
 
   steps.append({
       'name':
-          'gcr.io/oss-fuzz-base/base-runner',
+          f'gcr.io/{base_images_project}/base-runner',
       'env':
           env + [
               'COLLECT_DFT_TIMEOUT=2h',
@@ -355,13 +376,16 @@ def dataflow_post_build_steps(project_name, env):
   return steps
 
 
-def get_logs_url(build_id):
-  URL_FORMAT = ('https://console.developers.google.com/logs/viewer?'
-                'resource=build%2Fbuild_id%2F{0}&project=oss-fuzz')
-  return URL_FORMAT.format(build_id)
+def get_logs_url(build_id, image_project='oss-fuzz'):
+  """Returns url where logs are displayed for the build."""
+  url_format = ('https://console.developers.google.com/logs/viewer?'
+                'resource=build%2Fbuild_id%2F{0}&project={1}')
+  return url_format.format(build_id, image_project)
 
 
+# pylint: disable=no-member
 def run_build(build_steps, project_name, tag):
+  """Run the build for given steps on cloud build."""
   options = {}
   if 'GCB_OPTIONS' in os.environ:
     options = yaml.safe_load(os.environ['GCB_OPTIONS'])
@@ -385,13 +409,26 @@ def run_build(build_steps, project_name, tag):
 
 
 def main():
+  """Build and run projects."""
   if len(sys.argv) != 2:
     usage()
 
+  image_project = 'oss-fuzz'
+  base_images_project = 'oss-fuzz-base'
   project_dir = sys.argv[1].rstrip(os.path.sep)
-  steps = get_build_steps(project_dir)
-
+  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
+  project_yaml_path = os.path.join(project_dir, 'project.yaml')
   project_name = os.path.basename(project_dir)
+
+  with open(dockerfile_path) as dockerfile:
+    dockerfile_lines = dockerfile.readlines()
+
+  with open(project_yaml_path) as project_yaml_file:
+    project_yaml = yaml.safe_load(project_yaml_file)
+
+  steps = get_build_steps(project_name, project_yaml, dockerfile_lines,
+                          image_project, base_images_project)
+
   run_build(steps, project_name, FUZZING_BUILD_TAG)