aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--infra/build/functions/base_images.py11
-rw-r--r--infra/build/functions/datastore_entities.py2
-rwxr-xr-xinfra/build/functions/deploy.sh18
-rw-r--r--infra/build/functions/request_build.py31
-rw-r--r--infra/build/functions/request_build_test.py7
-rw-r--r--infra/build/functions/request_coverage_build.py4
-rw-r--r--infra/build/functions/update_build_status.py7
-rwxr-xr-xinfra/build/request_all_builds.sh24
-rwxr-xr-xinfra/build/request_build.sh17
9 files changed, 84 insertions, 37 deletions
diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py
index f3f13224..a56e9ffb 100644
--- a/infra/build/functions/base_images.py
+++ b/infra/build/functions/base_images.py
@@ -22,13 +22,15 @@ from googleapiclient.discovery import build
import build_base_images
+BASE_PROJECT = 'oss-fuzz-base'
+
# pylint: disable=no-member
def base_builder(event, context):
"""Cloud function to build base images."""
del event, context
- credentials, project_id = google.auth.default()
- tag_prefix = f'gcr.io/{project_id}/'
+ credentials, _ = google.auth.default()
+ tag_prefix = f'gcr.io/{BASE_PROJECT}/'
build_body = {
'steps':
build_base_images.get_steps(build_base_images.BASE_IMAGES,
@@ -47,8 +49,9 @@ def base_builder(event, context):
'v1',
credentials=credentials,
cache_discovery=False)
- build_info = cloudbuild.projects().builds().create(projectId=project_id,
+ build_info = cloudbuild.projects().builds().create(projectId=BASE_PROJECT,
body=build_body).execute()
build_id = build_info['metadata']['build']['id']
logging.info('Build ID: %s', build_id)
- logging.info('Logs: %s', build_base_images.get_logs_url(build_id, project_id))
+ logging.info('Logs: %s',
+ build_base_images.get_logs_url(build_id, BASE_PROJECT))
diff --git a/infra/build/functions/datastore_entities.py b/infra/build/functions/datastore_entities.py
index 1946cb52..8683803c 100644
--- a/infra/build/functions/datastore_entities.py
+++ b/infra/build/functions/datastore_entities.py
@@ -36,6 +36,6 @@ class GithubCreds(ndb.Model):
# pylint: disable=too-few-public-methods
class BuildsHistory(ndb.Model):
"""Container for build history of projects."""
- build_tag_suffix = ndb.StringProperty()
+ build_tag = ndb.StringProperty()
project = ndb.StringProperty()
build_ids = ndb.StringProperty(repeated=True)
diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh
index 0c361a3d..26f99cbb 100755
--- a/infra/build/functions/deploy.sh
+++ b/infra/build/functions/deploy.sh
@@ -83,19 +83,19 @@ function deploy_cloud_function {
--trigger-topic $topic \
--runtime python37 \
--project $project \
- --timeout 540
+ --timeout 540 \
+ --max-instances 1
}
-if [ $# == 2 ]; then
+if [ $# == 1 ]; then
PROJECT_ID=$1
- BASE_PROJECT_ID=$2
else
- echo -e "\n Usage ./deploy.sh <project-name> <base-project-name>"; exit;
+ echo -e "\n Usage ./deploy.sh <project-name>"; exit;
fi
deploy_pubsub_topic $BUILD_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $SYNC_JOB_TOPIC $PROJECT_ID
-deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $BASE_PROJECT_ID
+deploy_pubsub_topic $BASE_IMAGE_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $COVERAGE_BUILD_JOB_TOPIC $PROJECT_ID
deploy_pubsub_topic $UPDATE_BUILD_JOB_TOPIC $PROJECT_ID
@@ -106,10 +106,10 @@ deploy_scheduler $SYNC_SCHEDULER_JOB \
$PROJECT_ID
deploy_scheduler $BASE_IMAGE_SCHEDULER_JOB \
- "$BASE_IMAGE SCHEDULE" \
+ "$BASE_IMAGE_SCHEDULE" \
$BASE_IMAGE_JOB_TOPIC \
"$BASE_IMAGE_MESSAGE" \
- $BASE_PROJECT_ID
+ $PROJECT_ID
deploy_scheduler $COVERAGE_BUILD_SCHEDULER_JOB \
"$COVERAGE_BUILD_SCHEDULE" \
@@ -121,7 +121,7 @@ deploy_scheduler $UPDATE_BUILD_SCHEDULER_JOB \
"$UPDATE_BUILD_JOB_SCHEDULE" \
$UPDATE_BUILD_JOB_TOPIC \
"$UPDATE_BUILD_MESSAGE" \
- $PROJECT_ID
+ $PROJECT_ID
deploy_cloud_function sync \
@@ -132,7 +132,7 @@ deploy_cloud_function sync \
deploy_cloud_function base-image-build \
build_base_images \
$BASE_IMAGE_JOB_TOPIC \
- $BASE_PROJECT_ID
+ $PROJECT_ID
deploy_cloud_function request-build \
build_project \
diff --git a/infra/build/functions/request_build.py b/infra/build/functions/request_build.py
index c50c6e3d..b7046c7e 100644
--- a/infra/build/functions/request_build.py
+++ b/infra/build/functions/request_build.py
@@ -30,14 +30,14 @@ BASE_PROJECT = 'oss-fuzz-base'
MAX_BUILD_HISTORY_LENGTH = 64
-def update_build_history(project_name, build_id, build_tag_suffix):
+def update_build_history(project_name, build_id, build_tag):
"""Update build history of project."""
- project_key = ndb.Key(BuildsHistory, project_name + build_tag_suffix)
+ project_key = ndb.Key(BuildsHistory, project_name + build_tag)
project = project_key.get()
if not project:
- project = BuildsHistory(id=project_name + '-' + build_tag_suffix,
- build_tag_suffix=build_tag_suffix,
+ project = BuildsHistory(id=project_name + '-' + build_tag,
+ build_tag=build_tag,
project=project_name,
build_ids=[])
@@ -50,14 +50,13 @@ def update_build_history(project_name, build_id, build_tag_suffix):
def get_project_data(project_name):
"""Retrieve project metadata from datastore."""
- with ndb.Client().context():
- query = Project.query(Project.name == project_name)
- project = query.get()
- if not project:
- raise RuntimeError(
- 'Project {0} not available in cloud datastore'.format(project_name))
- project_yaml_contents = project.project_yaml_contents
- dockerfile_lines = project.dockerfile_contents.split('\n')
+ query = Project.query(Project.name == project_name)
+ project = query.get()
+ if not project:
+ raise RuntimeError(
+ 'Project {0} not available in cloud datastore'.format(project_name))
+ project_yaml_contents = project.project_yaml_contents
+ dockerfile_lines = project.dockerfile_contents.split('\n')
return (project_yaml_contents, dockerfile_lines)
@@ -107,6 +106,8 @@ def request_build(event, context):
else:
raise RuntimeError('Project name missing from payload')
- credentials, image_project = google.auth.default()
- build_steps = get_build_steps(project_name, image_project, BASE_PROJECT)
- run_build(project_name, image_project, build_steps, credentials, '-fuzzing')
+ with ndb.Client().context():
+ credentials, image_project = google.auth.default()
+ build_steps = get_build_steps(project_name, image_project, BASE_PROJECT)
+ run_build(project_name, image_project, build_steps, credentials,
+ build_project.FUZZING_BUILD_TAG)
diff --git a/infra/build/functions/request_build_test.py b/infra/build/functions/request_build_test.py
index 57ff3fe4..9589c09b 100644
--- a/infra/build/functions/request_build_test.py
+++ b/infra/build/functions/request_build_test.py
@@ -65,9 +65,8 @@ class TestRequestBuilds(unittest.TestCase):
Project(name='test-project',
project_yaml_contents=project_yaml_contents,
dockerfile_contents='test line').put()
-
- build_steps = get_build_steps('test-project', image_project,
- base_images_project)
+ build_steps = get_build_steps('test-project', image_project,
+ base_images_project)
self.assertEqual(build_steps, expected_build_steps)
def test_get_build_steps_no_project(self):
@@ -80,7 +79,7 @@ class TestRequestBuilds(unittest.TestCase):
"""Testing build history."""
with ndb.Client().context():
BuildsHistory(id='test-project-fuzzing',
- build_tag_suffix='fuzzing',
+ build_tag='fuzzing',
project='test-project',
build_ids=[str(i) for i in range(1, 65)]).put()
update_build_history('test-project', '65', '-fuzzing')
diff --git a/infra/build/functions/request_coverage_build.py b/infra/build/functions/request_coverage_build.py
index d874f0ea..0bc57e44 100644
--- a/infra/build/functions/request_coverage_build.py
+++ b/infra/build/functions/request_coverage_build.py
@@ -43,5 +43,7 @@ def request_coverage_build(event, context):
image_project, BASE_PROJECT)
except SystemExit:
continue
+
request_build.run_build(project_name, image_project, build_steps,
- credentials, '-coverage')
+ credentials,
+ build_and_run_coverage.COVERAGE_BUILD_TAG)
diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py
index 490efab9..618a2a58 100644
--- a/infra/build/functions/update_build_status.py
+++ b/infra/build/functions/update_build_status.py
@@ -58,13 +58,13 @@ def get_last_build(build_ids):
return None
-def update_build_status(build_tag_suffix, status_filename):
+def update_build_status(build_tag, status_filename):
"""Update build statuses."""
statuses = {}
successes = []
failures = []
for project_build in BuildsHistory.query(
- BuildsHistory.build_tag_suffix == build_tag_suffix):
+ BuildsHistory.build_tag == build_tag):
last_build = get_last_build(project_build.build_ids)
if not last_build:
logging.error('Failed to get last build for project %s',
@@ -136,7 +136,8 @@ def update_status(event, context):
status_filename='status-coverage.json')
for project in Project.query():
- if project.name not in project_build_statuses or project.name not in coverage_build_statuses:
+ if (project.name not in project_build_statuses or
+ project.name not in coverage_build_statuses):
continue
update_build_badges(project.name, project_build_statuses[project.name],
diff --git a/infra/build/request_all_builds.sh b/infra/build/request_all_builds.sh
new file mode 100755
index 00000000..2c7f39be
--- /dev/null
+++ b/infra/build/request_all_builds.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -ex
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+for project in ../../projects/*; do
+ if [[ ! -f $project/Dockerfile ]]; then
+ continue
+ fi
+
+ ./request_build.sh $(basename $project)
+done
diff --git a/infra/build/request_build.sh b/infra/build/request_build.sh
new file mode 100755
index 00000000..58e1de4c
--- /dev/null
+++ b/infra/build/request_build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash -ex
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+gcloud pubsub topics publish request-build --message "$1" --project oss-fuzz
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-27 16:31:41 +0000