1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
|
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementations for various CI systems."""
import os
import collections
import sys
import logging
# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import build_specified_commit
import docker
import helper
import repo_manager
import retry
import utils
import workspace_utils
# pylint: disable=too-few-public-methods
BuildPreparationResult = collections.namedtuple(
'BuildPreparationResult', ['success', 'image_repo_path', 'repo_manager'])
_IMAGE_BUILD_TRIES = 3
_IMAGE_BUILD_BACKOFF = 2
def fix_git_repo_for_diff(repo_manager_obj):
"""Fixes git repos cloned by the "checkout" action so that diffing works on
them."""
command = [
'git', 'symbolic-ref', 'refs/remotes/origin/HEAD',
'refs/remotes/origin/master'
]
return utils.execute(command, location=repo_manager_obj.repo_dir)
class BaseCi:
"""Class representing common CI functionality."""
def __init__(self, config):
self.config = config
self.workspace = workspace_utils.Workspace(config)
self._repo_dir = None
@property
def repo_dir(self):
"""Returns the source repo path, if it has been checked out. None is
returned otherwise."""
raise NotImplementedError('Child class must implement method.')
def prepare_for_fuzzer_build(self):
"""Builds the fuzzer builder image and gets the source code we need to
fuzz."""
raise NotImplementedError('Child class must implement method.')
def get_diff_base(self):
"""Returns the base to diff against with git to get the change under
test."""
if self.config.base_ref:
logging.debug('Diffing against base_ref: %s.', self.config.base_ref)
return self.config.base_ref
if self.config.base_commit:
logging.debug('Diffing against base_commit: %s.', self.config.base_commit)
return self.config.base_commit
# TODO(metzman): Do we want this at all? What purpose does it serve? I guess
# it is a decent fallback when there is no base_commit or base_ref.
logging.debug('Diffing against origin.')
return 'origin'
def get_changed_code_under_test(self, repo_manager_obj):
"""Returns the changed files that need to be tested."""
if self.config.base_ref:
repo_manager_obj.fetch_branch(self.config.base_ref)
fix_git_repo_for_diff(repo_manager_obj)
base = self.get_diff_base()
logging.info('Diffing against %s.', base)
# git diff <commit>... is equivalent to
# git diff $(git merge-base <commit> HEAD)
return repo_manager_obj.get_git_diff(base + '...')
def get_build_command(self, host_repo_path, image_repo_path):
"""Returns the command for building the project that is run inside the
project builder container."""
raise NotImplementedError('Child class must implement method.')
def _build_external_project_docker_image(self, manager):
"""Helper for child classes that builds an external project's docker image.
Returns a BuildPreparationResult indicating failure or success."""
build_integration_abs_path = os.path.join(
manager.repo_dir, self.config.build_integration_path)
if not build_external_project_docker_image(manager.repo_dir,
build_integration_abs_path):
logging.error('Failed to build external project.')
return get_build_preparation_failure()
image_repo_path = os.path.join('/src', self.config.project_repo_name)
return BuildPreparationResult(success=True,
image_repo_path=image_repo_path,
repo_manager=manager)
def _make_repo_storage_dir(self):
os.makedirs(self.workspace.repo_storage, exist_ok=True)
def _clone_repo_and_checkout(self, repo_url, repo_name):
"""Helper for child classes that clones the git repo specified by |repo_url|
to |repo_name|, checks out the specified commit, and returns the
|manager|."""
self._make_repo_storage_dir()
# Checkout project's repo in the shared volume.
manager = repo_manager.clone_repo_and_get_manager(
repo_url,
self.workspace.repo_storage,
repo_name=repo_name,
username=self.config.actor,
password=self.config.token)
self._checkout_specified_commit(manager)
return manager
def _checkout_specified_commit(self, manager):
checkout_specified_commit(manager, self.config.pr_ref, self.config.git_sha)
def _detect_main_repo(self):
"""Helper for child classes that detects the main repo and returns a tuple
containing the inffered url and path to the repo in the image."""
inferred_url, image_repo_path = build_specified_commit.detect_main_repo(
self.config.oss_fuzz_project_name,
repo_name=self.config.project_repo_name)
if not inferred_url or not image_repo_path:
logging.error('Could not detect repo.')
return inferred_url, image_repo_path
def _create_repo_manager_for_project_src_path(self):
"""Returns a repo manager for |project_src_path|."""
return repo_manager.RepoManager(self.config.project_src_path)
def get_build_command():
"""Returns the command to build the project inside the project builder
container."""
return 'compile'
def get_replace_repo_and_build_command(host_repo_path, image_repo_path):
"""Returns the command to replace the repo located at |image_repo_path| with
|host_repo_path| and build the project inside the project builder
container."""
rm_path = os.path.join(image_repo_path, '*')
image_src_path = os.path.dirname(image_repo_path)
build_command = get_build_command()
command = (f'cd / && rm -rf {rm_path} && cp -r {host_repo_path} '
f'{image_src_path} && cd - && {build_command}')
return command
def get_ci(config):
"""Determines what kind of CI is being used and returns the object
representing that system."""
if config.platform == config.Platform.EXTERNAL_GENERIC_CI:
# Non-OSS-Fuzz projects must bring their own source and their own build
# integration (which is relative to that source).
return ExternalGeneric(config)
if config.platform == config.Platform.EXTERNAL_GITHUB:
# Non-OSS-Fuzz projects must bring their own source and their own build
# integration (which is relative to that source).
return ExternalGithub(config)
if config.platform == config.Platform.INTERNAL_GENERIC_CI:
# Builds of OSS-Fuzz projects not hosted on Github must bring their own
# source since the checkout logic CIFuzz implements is github-specific.
# TODO(metzman): Consider moving Github-actions builds of OSS-Fuzz projects
# to this system to reduce implementation complexity.
return InternalGeneric(config)
return InternalGithub(config)
def checkout_specified_commit(repo_manager_obj, pr_ref, git_sha):
"""Checks out the specified commit or pull request using
|repo_manager_obj|."""
try:
if pr_ref:
repo_manager_obj.checkout_pr(pr_ref)
else:
repo_manager_obj.checkout_commit(git_sha)
except (RuntimeError, ValueError):
logging.error(
'Can not check out requested state %s. '
'Using current repo state.', pr_ref or git_sha)
class GithubCiMixin:
"""Mixin for Github based CI systems."""
def __init__(self, config):
super().__init__(config)
# Unlike in other classes, here _repo_dir is the parent directory of the
# repo, not its actual directory.
self._repo_dir = self.workspace.repo_storage
@property
def repo_dir(self):
"""Returns the source repo path, if it has been checked out. None is
returned otherwise."""
if not os.path.exists(self._repo_dir):
logging.warning('Repo dir: %s does not exist.', self._repo_dir)
return None
# Note: this assumes there is only one repo checked out here.
listing = os.listdir(self._repo_dir)
if len(listing) != 1:
raise RuntimeError('Invalid repo directory.')
repo_path = os.path.join(self._repo_dir, listing[0])
if not os.path.isdir(repo_path):
raise RuntimeError('Repo is not a directory.')
return repo_path
class InternalGithub(GithubCiMixin, BaseCi):
"""Class representing CI for an OSS-Fuzz project on Github Actions."""
def _copy_repo_from_image(self, image_repo_path):
self._make_repo_storage_dir()
repo_name = os.path.basename(image_repo_path)
host_repo_path = os.path.join(self._repo_dir, repo_name)
bash_command = f'cp -r {image_repo_path} {host_repo_path}'
docker_args, _ = docker.get_base_docker_run_args(
self.workspace, self.config.sanitizer, self.config.language,
self.config.docker_in_docker)
docker_args.extend([
docker.get_project_image_name(self.config.oss_fuzz_project_name),
'/bin/bash', '-c', bash_command
])
if not helper.docker_run(docker_args):
raise RuntimeError('Failed to copy repo.')
return repo_manager.RepoManager(host_repo_path)
def prepare_for_fuzzer_build(self):
"""Builds the fuzzer builder image, checks out the pull request/commit and
returns the BuildPreparationResult."""
logging.info('InternalGithub: preparing for fuzzer build.')
assert self.config.pr_ref or self.config.git_sha
# _detect_main_repo builds the image as a side effect.
_, image_repo_path = self._detect_main_repo()
if not image_repo_path:
return get_build_preparation_failure()
# Use the same name used in the docker image so we can overwrite it.
manager = self._copy_repo_from_image(image_repo_path)
self._checkout_specified_commit(manager)
return BuildPreparationResult(success=True,
image_repo_path=image_repo_path,
repo_manager=manager)
def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use
"""Returns the command for building the project that is run inside the
project builder container. Command also replaces |image_repo_path| with
|host_repo_path|."""
return get_replace_repo_and_build_command(host_repo_path, image_repo_path)
def get_build_preparation_failure():
"""Returns a BuildPreparationResult indicating failure."""
return BuildPreparationResult(success=False,
image_repo_path=None,
repo_manager=None)
class InternalGeneric(BaseCi):
"""Class representing CI for an OSS-Fuzz project on a CI other than Github
actions."""
def __init__(self, config):
super().__init__(config)
self._repo_dir = config.project_src_path
@property
def repo_dir(self):
"""Returns the source repo path, if it has been checked out. None is
returned otherwise."""
return self._repo_dir
def prepare_for_fuzzer_build(self):
"""Builds the project builder image for an OSS-Fuzz project outside of
GitHub actions. Returns the repo_manager. Does not checkout source code
since external projects are expected to bring their own source code to
CIFuzz."""
logging.info('InternalGeneric: preparing for fuzzer build.')
# detect_main_repo builds the image as a side effect.
_, image_repo_path = self._detect_main_repo()
if not image_repo_path:
return get_build_preparation_failure()
manager = self._create_repo_manager_for_project_src_path()
return BuildPreparationResult(success=True,
image_repo_path=image_repo_path,
repo_manager=manager)
def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use
"""Returns the command for building the project that is run inside the
project builder container. Command also replaces |image_repo_path| with
|host_repo_path|."""
return get_replace_repo_and_build_command(host_repo_path, image_repo_path)
@retry.wrap(_IMAGE_BUILD_TRIES, _IMAGE_BUILD_BACKOFF)
def build_external_project_docker_image(project_src, build_integration_path):
"""Builds the project builder image for an external (non-OSS-Fuzz) project.
Returns True on success."""
dockerfile_path = os.path.join(build_integration_path, 'Dockerfile')
command = [
'-t', docker.EXTERNAL_PROJECT_IMAGE, '-f', dockerfile_path, project_src
]
return helper.docker_build(command)
class ExternalGeneric(BaseCi):
"""CI implementation for generic CI for external (non-OSS-Fuzz) projects."""
def __init__(self, config):
super().__init__(config)
self._repo_dir = config.project_src_path
@property
def repo_dir(self):
"""Returns the source repo path, if it has been checked out. None is
returned otherwise."""
return self._repo_dir
def prepare_for_fuzzer_build(self):
logging.info('ExternalGeneric: preparing for fuzzer build.')
manager = self._create_repo_manager_for_project_src_path()
return self._build_external_project_docker_image(manager)
def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use
"""Returns the command for building the project that is run inside the
project builder container."""
return get_build_command()
class ExternalGithub(GithubCiMixin, BaseCi):
"""Class representing CI for a non-OSS-Fuzz project on Github Actions."""
def prepare_for_fuzzer_build(self):
"""Builds the project builder image for a non-OSS-Fuzz project on GitHub
actions. Sets the repo manager. Does not checkout source code since external
projects are expected to bring their own source code to CIFuzz. Returns True
on success."""
logging.info('ExternalGithub: preparing for fuzzer build.')
# Checkout before building, so we don't need to rely on copying the source
# from the image.
# TODO(metzman): Figure out if we want second copy at all.
manager = self._clone_repo_and_checkout(self.config.git_url,
self.config.project_repo_name)
return self._build_external_project_docker_image(manager)
def get_build_command(self, host_repo_path, image_repo_path): # pylint: disable=no-self-use
"""Returns the command for building the project that is run inside the
project builder container."""
return get_build_command()
|