aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra/cifuzz
diff options
context:
space:
mode:
authorGravatar jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com>2021-08-10 11:10:10 -0700
committerGravatar GitHub <noreply@github.com>2021-08-10 11:10:10 -0700
commit94cfc4fe2f0c9743ab7acd19910d6961f2317eeb (patch)
treea4b0b54cd2c6ed7556bad15b31dc7e1f3c046086 /infra/cifuzz
parente407f54e61a134de43019d69947a72a471bdd7c6 (diff)
[cifuzz] Add pruning task (#6188)
Fixes: #6064
Diffstat (limited to 'infra/cifuzz')
-rw-r--r--infra/cifuzz/clusterfuzz_deployment.py69
-rw-r--r--infra/cifuzz/clusterfuzz_deployment_test.py42
-rw-r--r--infra/cifuzz/config_utils.py3
-rw-r--r--infra/cifuzz/fuzz_target.py54
-rw-r--r--infra/cifuzz/fuzz_target_test.py18
-rw-r--r--infra/cifuzz/generate_coverage_report.py7
-rw-r--r--infra/cifuzz/generate_coverage_report_test.py6
-rw-r--r--infra/cifuzz/run_fuzzers.py43
-rw-r--r--infra/cifuzz/workspace_utils.py5
9 files changed, 157 insertions, 90 deletions
diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py
index eb20dc4d..a2be1ff9 100644
--- a/infra/cifuzz/clusterfuzz_deployment.py
+++ b/infra/cifuzz/clusterfuzz_deployment.py
@@ -18,6 +18,7 @@ import sys
import urllib.error
import urllib.request
+import config_utils
import filestore
import filestore_utils
import http_utils
@@ -50,8 +51,8 @@ class BaseClusterFuzzDeployment:
"""
raise NotImplementedError('Child class must implement method.')
- def download_corpus(self, target_name):
- """Downloads the corpus for |target_name| from ClusterFuzz.
+ def download_corpus(self, target_name, corpus_dir):
+ """Downloads the corpus for |target_name| from ClusterFuzz to |corpus_dir|.
Returns:
A path to where the OSS-Fuzz build was stored, or None if it wasn't.
@@ -62,11 +63,7 @@ class BaseClusterFuzzDeployment:
"""Uploads crashes in |crashes_dir| to filestore."""
raise NotImplementedError('Child class must implement method.')
- def get_target_corpus_dir(self, target_name):
- """Returns the path to the corpus dir for |target_name|."""
- return os.path.join(self.workspace.corpora, target_name)
-
- def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument
+ def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument
"""Uploads the corpus for |target_name| to filestore."""
raise NotImplementedError('Child class must implement method.')
@@ -78,12 +75,10 @@ class BaseClusterFuzzDeployment:
"""Returns the project coverage object for the project."""
raise NotImplementedError('Child class must implement method.')
- def make_empty_corpus_dir(self, target_name):
- """Makes an empty corpus directory for |target_name| in |parent_dir| and
- returns the path to the directory."""
- corpus_dir = self.get_target_corpus_dir(target_name)
- os.makedirs(corpus_dir, exist_ok=True)
- return corpus_dir
+
+def _make_empty_dir_if_nonexistent(path):
+ """Makes an empty directory at |path| if it does not exist."""
+ os.makedirs(path, exist_ok=True)
class ClusterFuzzLite(BaseClusterFuzzDeployment):
@@ -104,7 +99,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
# called if multiple bugs are found.
return self.workspace.clusterfuzz_build
- os.makedirs(self.workspace.clusterfuzz_build, exist_ok=True)
+ _make_empty_dir_if_nonexistent(self.workspace.clusterfuzz_build)
build_name = self._get_build_name()
try:
@@ -118,8 +113,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
return None
- def download_corpus(self, target_name):
- corpus_dir = self.make_empty_corpus_dir(target_name)
+ def download_corpus(self, target_name, corpus_dir):
+ _make_empty_dir_if_nonexistent(corpus_dir)
logging.info('Downloading corpus for %s to %s.', target_name, corpus_dir)
corpus_name = self._get_corpus_name(target_name)
try:
@@ -142,9 +137,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment):
"""Returns the name of the crashes artifact."""
return 'current'
- def upload_corpus(self, target_name):
+ def upload_corpus(self, target_name, corpus_dir):
"""Upload the corpus produced by |target_name|."""
- corpus_dir = self.get_target_corpus_dir(target_name)
logging.info('Uploading corpus in %s for %s.', corpus_dir, target_name)
name = self._get_corpus_name(target_name)
try:
@@ -241,7 +235,7 @@ class OSSFuzz(BaseClusterFuzzDeployment):
# again.
return self.workspace.clusterfuzz_build
- os.makedirs(self.workspace.clusterfuzz_build, exist_ok=True)
+ _make_empty_dir_if_nonexistent(self.workspace.clusterfuzz_build)
latest_build_name = self.get_latest_build_name()
if not latest_build_name:
@@ -263,7 +257,7 @@ class OSSFuzz(BaseClusterFuzzDeployment):
"""Noop Implementation of upload_latest_build."""
logging.info('Not uploading latest build because on OSS-Fuzz.')
- def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument
+ def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument
"""Noop Implementation of upload_corpus."""
logging.info('Not uploading corpus because on OSS-Fuzz.')
@@ -271,13 +265,13 @@ class OSSFuzz(BaseClusterFuzzDeployment):
"""Noop Implementation of upload_crashes."""
logging.info('Not uploading crashes because on OSS-Fuzz.')
- def download_corpus(self, target_name):
+ def download_corpus(self, target_name, corpus_dir):
"""Downloads the latest OSS-Fuzz corpus for the target.
Returns:
The local path to to corpus or None if download failed.
"""
- corpus_dir = self.make_empty_corpus_dir(target_name)
+ _make_empty_dir_if_nonexistent(corpus_dir)
project_qualified_fuzz_target_name = target_name
qualified_name_prefix = self.config.oss_fuzz_project_name + '_'
if not target_name.startswith(qualified_name_prefix):
@@ -314,7 +308,7 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment):
logging.info('Not uploading latest build because no ClusterFuzz '
'deployment.')
- def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument
+ def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument
"""Noop Implementation of upload_corpus."""
logging.info('Not uploading corpus because no ClusterFuzz deployment.')
@@ -322,10 +316,10 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment):
"""Noop Implementation of upload_crashes."""
logging.info('Not uploading crashes because no ClusterFuzz deployment.')
- def download_corpus(self, target_name):
+ def download_corpus(self, target_name, corpus_dir):
"""Noop Implementation of download_corpus."""
logging.info('Not downloading corpus because no ClusterFuzz deployment.')
- return self.make_empty_corpus_dir(target_name)
+ return _make_empty_dir_if_nonexistent(corpus_dir)
def download_latest_build(self): # pylint: disable=no-self-use
"""Noop Implementation of download_latest_build."""
@@ -343,14 +337,21 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment):
'Not getting project coverage because no ClusterFuzz deployment.')
+_PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING = {
+ config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI:
+ OSSFuzz,
+ config_utils.BaseConfig.Platform.INTERNAL_GITHUB:
+ OSSFuzz,
+ config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI:
+ NoClusterFuzzDeployment,
+ config_utils.BaseConfig.Platform.EXTERNAL_GITHUB:
+ ClusterFuzzLite,
+}
+
+
def get_clusterfuzz_deployment(config, workspace):
"""Returns object reprsenting deployment of ClusterFuzz used by |config|."""
- if (config.platform == config.Platform.INTERNAL_GENERIC_CI or
- config.platform == config.Platform.INTERNAL_GITHUB):
- logging.info('Using OSS-Fuzz as ClusterFuzz deployment.')
- return OSSFuzz(config, workspace)
- if config.platform == config.Platform.EXTERNAL_GENERIC_CI:
- logging.info('Not using a ClusterFuzz deployment.')
- return NoClusterFuzzDeployment(config, workspace)
- logging.info('Using ClusterFuzzLite as ClusterFuzz deployment.')
- return ClusterFuzzLite(config, workspace)
+ deployment_cls = _PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING[config.platform]
+ result = deployment_cls(config, workspace)
+ logging.info('ClusterFuzzDeployment: %s.', result)
+ return result
diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py
index 873ef8e3..3d3f0fe2 100644
--- a/infra/cifuzz/clusterfuzz_deployment_test.py
+++ b/infra/cifuzz/clusterfuzz_deployment_test.py
@@ -64,28 +64,26 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase):
def setUp(self):
self.setUpPyfakefs()
self.deployment = _create_deployment()
+ self.corpus_dir = os.path.join(self.deployment.workspace.corpora,
+ EXAMPLE_FUZZER)
@mock.patch('http_utils.download_and_unpack_zip', return_value=True)
def test_download_corpus(self, mocked_download_and_unpack_zip):
"""Tests that we can download a corpus for a valid project."""
- result = self.deployment.download_corpus(EXAMPLE_FUZZER)
- self.assertIsNotNone(result)
- expected_corpus_dir = os.path.join(self.deployment.workspace.corpora,
- EXAMPLE_FUZZER)
+ self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir)
expected_url = ('https://storage.googleapis.com/example-backup.'
'clusterfuzz-external.appspot.com/corpus/libFuzzer/'
'example_crash_fuzzer/public.zip')
call_args, _ = mocked_download_and_unpack_zip.call_args
- self.assertEqual(call_args, (expected_url, expected_corpus_dir))
+ self.assertEqual(call_args, (expected_url, self.corpus_dir))
+ self.assertTrue(os.path.exists(self.corpus_dir))
@mock.patch('http_utils.download_and_unpack_zip', return_value=False)
def test_download_corpus_fail(self, _):
"""Tests that when downloading fails, an empty corpus directory is still
returned."""
- corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER)
- self.assertEqual(corpus_path,
- '/workspace/cifuzz-corpus/example_crash_fuzzer')
- self.assertEqual(os.listdir(corpus_path), [])
+ self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir)
+ self.assertEqual(os.listdir(self.corpus_dir), [])
def test_get_latest_build_name(self):
"""Tests that the latest build name can be retrieved from GCS."""
@@ -96,7 +94,7 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase):
@parameterized.parameterized.expand([
('upload_latest_build', tuple(),
'Not uploading latest build because on OSS-Fuzz.'),
- ('upload_corpus', ('target',),
+ ('upload_corpus', ('target', 'corpus-dir'),
'Not uploading corpus because on OSS-Fuzz.'),
('upload_crashes', tuple(), 'Not uploading crashes because on OSS-Fuzz.'),
])
@@ -133,27 +131,25 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase):
self.deployment = _create_deployment(run_fuzzers_mode='batch',
oss_fuzz_project_name='',
is_github=True)
+ self.corpus_dir = os.path.join(self.deployment.workspace.corpora,
+ EXAMPLE_FUZZER)
@mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus',
return_value=True)
def test_download_corpus(self, mocked_download_corpus):
"""Tests that download_corpus works for a valid project."""
- result = self.deployment.download_corpus(EXAMPLE_FUZZER)
- expected_corpus_dir = os.path.join(WORKSPACE, 'cifuzz-corpus',
- EXAMPLE_FUZZER)
- self.assertEqual(result, expected_corpus_dir)
+ self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir)
mocked_download_corpus.assert_called_with('example_crash_fuzzer',
- expected_corpus_dir)
+ self.corpus_dir)
+ self.assertTrue(os.path.exists(self.corpus_dir))
@mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus',
side_effect=Exception)
def test_download_corpus_fail(self, _):
"""Tests that when downloading fails, an empty corpus directory is still
returned."""
- corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER)
- self.assertEqual(corpus_path,
- '/workspace/cifuzz-corpus/example_crash_fuzzer')
- self.assertEqual(os.listdir(corpus_path), [])
+ self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir)
+ self.assertEqual(os.listdir(self.corpus_dir), [])
@mock.patch('filestore.github_actions.GithubActionsFilestore.download_build',
return_value=True)
@@ -191,21 +187,21 @@ class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase):
workspace = workspace_utils.Workspace(config)
self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment(
config, workspace)
+ self.corpus_dir = os.path.join(workspace.corpora, EXAMPLE_FUZZER)
@mock.patch('logging.info')
def test_download_corpus(self, mocked_info):
"""Tests that download corpus returns the path to the empty corpus
directory."""
- corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER)
- self.assertEqual(corpus_path,
- '/workspace/cifuzz-corpus/example_crash_fuzzer')
+ self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir)
mocked_info.assert_called_with(
'Not downloading corpus because no ClusterFuzz deployment.')
+ self.assertTrue(os.path.exists(self.corpus_dir))
@parameterized.parameterized.expand([
('upload_latest_build', tuple(),
'Not uploading latest build because no ClusterFuzz deployment.'),
- ('upload_corpus', ('target',),
+ ('upload_corpus', ('target', 'corpus-dir'),
'Not uploading corpus because no ClusterFuzz deployment.'),
('upload_crashes', tuple(),
'Not uploading crashes because no ClusterFuzz deployment.'),
diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py
index 8ca3eec1..b8029bf4 100644
--- a/infra/cifuzz/config_utils.py
+++ b/infra/cifuzz/config_utils.py
@@ -26,7 +26,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import constants
-RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage']
+RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage', 'prune']
SANITIZERS = ['address', 'memory', 'undefined', 'coverage']
# TODO(metzman): Set these on config objects so there's one source of truth.
@@ -277,6 +277,7 @@ class RunFuzzersConfig(BaseConfig):
def __init__(self):
super().__init__()
+ # TODO(metzman): Pick a better default for pruning.
self.fuzz_seconds = int(os.environ.get('FUZZ_SECONDS', 600))
self.run_fuzzers_mode = os.environ.get('RUN_FUZZERS_MODE', 'ci').lower()
if self.is_coverage:
diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py
index 14a63a1e..7fd43a69 100644
--- a/infra/cifuzz/fuzz_target.py
+++ b/infra/cifuzz/fuzz_target.py
@@ -53,6 +53,17 @@ class ReproduceError(Exception):
"""Error for when we can't attempt to reproduce a crash."""
+def get_fuzz_target_corpus_dir(workspace, target_name):
+ """Returns the directory for storing |target_name|'s corpus in |workspace|."""
+ return os.path.join(workspace.corpora, target_name)
+
+
+def get_fuzz_target_pruned_corpus_dir(workspace, target_name):
+ """Returns the directory for storing |target_name|'s puned corpus in
+ |workspace|."""
+ return os.path.join(workspace.pruned_corpora, target_name)
+
+
class FuzzTarget: # pylint: disable=too-many-instance-attributes
"""A class to manage a single fuzz target.
@@ -82,29 +93,54 @@ class FuzzTarget: # pylint: disable=too-many-instance-attributes
self.workspace = workspace
self.clusterfuzz_deployment = clusterfuzz_deployment
self.config = config
- self.latest_corpus_path = None
+ self.latest_corpus_path = get_fuzz_target_corpus_dir(
+ self.workspace, self.target_name)
+ os.makedirs(self.latest_corpus_path, exist_ok=True)
+ self.pruned_corpus_path = get_fuzz_target_pruned_corpus_dir(
+ self.workspace, self.target_name)
+ os.makedirs(self.pruned_corpus_path, exist_ok=True)
+
+ def _download_corpus(self):
+ """Downloads the corpus for the target from ClusterFuzz and returns the path
+ to the corpus. An empty directory is provided if the corpus can't be
+ downloaded or is empty."""
+ self.clusterfuzz_deployment.download_corpus(self.target_name,
+ self.latest_corpus_path)
+ return self.latest_corpus_path
+
+ def prune(self):
+ """Prunes the corpus and returns the result."""
+ self._download_corpus()
+ prune_options = [
+ '-merge=1', self.pruned_corpus_path, self.latest_corpus_path
+ ]
+ result = self.fuzz(use_corpus=False, extra_libfuzzer_options=prune_options)
+ return FuzzResult(result.testcase, result.stacktrace,
+ self.pruned_corpus_path)
- def fuzz(self):
+ def fuzz(self, use_corpus=True, extra_libfuzzer_options=None):
"""Starts the fuzz target run for the length of time specified by duration.
Returns:
FuzzResult namedtuple with stacktrace and testcase if applicable.
"""
logging.info('Running fuzzer: %s.', self.target_name)
+ if extra_libfuzzer_options is None:
+ extra_libfuzzer_options = []
env = base_runner_utils.get_env(self.config, self.workspace)
# TODO(metzman): Is this needed?
env['RUN_FUZZER_MODE'] = 'interactive'
- # If corpus can be downloaded, use it for fuzzing.
- self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus(
- self.target_name)
- env['CORPUS_DIR'] = self.latest_corpus_path
+ if use_corpus:
+ # If corpus can be downloaded, use it for fuzzing.
+ self._download_corpus()
+ env['CORPUS_DIR'] = self.latest_corpus_path
options = LIBFUZZER_OPTIONS.copy() + [
f'-max_total_time={self.duration}',
# Make sure libFuzzer artifact files don't pollute $OUT.
f'-artifact_prefix={self.workspace.artifacts}/'
- ]
+ ] + extra_libfuzzer_options
command = ['run_fuzzer', self.target_name] + options
logging.info('Running command: %s', command)
@@ -151,10 +187,10 @@ class FuzzTarget: # pylint: disable=too-many-instance-attributes
self.target_name)
# Delete the seed corpus, corpus, and fuzz target.
- if self.latest_corpus_path and os.path.exists(self.latest_corpus_path):
+ for corpus_path in [self.latest_corpus_path, self.pruned_corpus_path]:
# Use ignore_errors=True to fix
# https://github.com/google/oss-fuzz/issues/5383.
- shutil.rmtree(self.latest_corpus_path, ignore_errors=True)
+ shutil.rmtree(corpus_path, ignore_errors=True)
target_seed_corpus_path = self.target_path + '_seed_corpus.zip'
if os.path.exists(target_seed_corpus_path):
diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py
index 268d3a64..ee262eda 100644
--- a/infra/cifuzz/fuzz_target_test.py
+++ b/infra/cifuzz/fuzz_target_test.py
@@ -75,7 +75,11 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
self.workspace = deployment.workspace
self.fuzz_target_path = os.path.join(self.workspace.out,
self.fuzz_target_name)
+ self.setUpPyfakefs()
+ self.fs.create_file(self.fuzz_target_path)
self.testcase_path = '/testcase'
+ self.fs.create_file(self.testcase_path)
+
self.target = fuzz_target.FuzzTarget(self.fuzz_target_path,
fuzz_target.REPRODUCE_ATTEMPTS,
self.workspace, deployment,
@@ -86,7 +90,6 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
def test_reproducible(self, _):
"""Tests that is_reproducible returns True if crash is detected and that
is_reproducible uses the correct command to reproduce a crash."""
- self._set_up_fakefs()
all_repro = [EXECUTE_FAILURE_RETVAL] * fuzz_target.REPRODUCE_ATTEMPTS
with mock.patch('utils.execute', side_effect=all_repro) as mocked_execute:
result = self.target.is_reproducible(self.testcase_path,
@@ -106,17 +109,9 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
self.assertTrue(result)
self.assertEqual(1, mocked_execute.call_count)
- def _set_up_fakefs(self):
- """Helper to setup pyfakefs and add important files to the fake
- filesystem."""
- self.setUpPyfakefs()
- self.fs.create_file(self.fuzz_target_path)
- self.fs.create_file(self.testcase_path)
-
def test_flaky(self, _):
"""Tests that is_reproducible returns True if crash is detected on the last
attempt."""
- self._set_up_fakefs()
last_time_repro = [EXECUTE_SUCCESS_RETVAL] * 9 + [EXECUTE_FAILURE_RETVAL]
with mock.patch('utils.execute',
side_effect=last_time_repro) as mocked_execute:
@@ -136,7 +131,6 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase):
"""Tests that is_reproducible returns False for a crash that did not
reproduce."""
all_unrepro = [EXECUTE_SUCCESS_RETVAL] * fuzz_target.REPRODUCE_ATTEMPTS
- self._set_up_fakefs()
with mock.patch('utils.execute', side_effect=all_unrepro):
result = self.target.is_reproducible(self.testcase_path,
self.fuzz_target_path)
@@ -172,13 +166,13 @@ class IsCrashReportableTest(fake_filesystem_unittest.TestCase):
def setUp(self):
"""Sets up example fuzz target to test is_crash_reportable method."""
+ self.setUpPyfakefs()
self.fuzz_target_path = '/example/do_stuff_fuzzer'
deployment = _create_deployment()
self.target = fuzz_target.FuzzTarget(self.fuzz_target_path, 100,
- '/example/outdir', deployment,
+ deployment.workspace, deployment,
deployment.config)
self.oss_fuzz_build_path = '/oss-fuzz-build'
- self.setUpPyfakefs()
self.fs.create_file(self.fuzz_target_path)
self.oss_fuzz_target_path = os.path.join(
self.oss_fuzz_build_path, os.path.basename(self.fuzz_target_path))
diff --git a/infra/cifuzz/generate_coverage_report.py b/infra/cifuzz/generate_coverage_report.py
index 2bfbe51d..9901c452 100644
--- a/infra/cifuzz/generate_coverage_report.py
+++ b/infra/cifuzz/generate_coverage_report.py
@@ -15,6 +15,7 @@
import os
import base_runner_utils
+import fuzz_target
import utils
@@ -33,8 +34,10 @@ def download_corpora(fuzz_target_paths, clusterfuzz_deployment):
"""Downloads corpora for fuzz targets in |fuzz_target_paths| using
|clusterfuzz_deployment| to download corpora from ClusterFuzz/OSS-Fuzz."""
for target_path in fuzz_target_paths:
- target = os.path.basename(target_path)
- clusterfuzz_deployment.download_corpus(target)
+ target_name = os.path.basename(target_path)
+ corpus_dir = fuzz_target.get_fuzz_target_corpus_dir(
+ clusterfuzz_deployment.workspace, target_name)
+ clusterfuzz_deployment.download_corpus(target_name, corpus_dir)
def generate_coverage_report(fuzz_target_paths, workspace,
diff --git a/infra/cifuzz/generate_coverage_report_test.py b/infra/cifuzz/generate_coverage_report_test.py
index bed49430..b24eb3fe 100644
--- a/infra/cifuzz/generate_coverage_report_test.py
+++ b/infra/cifuzz/generate_coverage_report_test.py
@@ -60,8 +60,12 @@ class DownloadCorporaTest(unittest.TestCase):
def test_download_corpora(self): # pylint: disable=no-self-use
"""Tests that download_corpora works as intended."""
clusterfuzz_deployment = mock.Mock()
+ clusterfuzz_deployment.workspace = test_helpers.create_workspace()
fuzz_target_paths = ['/path/to/fuzzer1', '/path/to/fuzzer2']
- expected_calls = [mock.call('fuzzer1'), mock.call('fuzzer2')]
+ expected_calls = [
+ mock.call('fuzzer1', '/workspace/cifuzz-corpus/fuzzer1'),
+ mock.call('fuzzer2', '/workspace/cifuzz-corpus/fuzzer2')
+ ]
generate_coverage_report.download_corpora(fuzz_target_paths,
clusterfuzz_deployment)
clusterfuzz_deployment.download_corpus.assert_has_calls(expected_calls)
diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py
index 93a30a37..79cf432c 100644
--- a/infra/cifuzz/run_fuzzers.py
+++ b/infra/cifuzz/run_fuzzers.py
@@ -166,6 +166,26 @@ class BaseFuzzTargetRunner:
return bug_found
+class PruneTargetRunner(BaseFuzzTargetRunner):
+ """Runner that prunes corpora."""
+
+ @property
+ def quit_on_bug_found(self):
+ return False
+
+ def run_fuzz_target(self, fuzz_target_obj):
+ """Prunes with |fuzz_target_obj| and returns the result."""
+ result = fuzz_target_obj.prune()
+ logging.debug('Corpus path contents: %s.', os.listdir(result.corpus_path))
+ self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name,
+ result.corpus_path)
+ return result
+
+ def cleanup_after_fuzz_target_run(self, fuzz_target_obj): # pylint: disable=no-self-use
+ """Cleans up after pruning with |fuzz_target_obj|."""
+ fuzz_target_obj.free_disk_if_needed()
+
+
class CoverageTargetRunner(BaseFuzzTargetRunner):
"""Runner that runs the 'coverage' command."""
@@ -224,8 +244,9 @@ class BatchFuzzTargetRunner(BaseFuzzTargetRunner):
def run_fuzz_target(self, fuzz_target_obj):
"""Fuzzes with |fuzz_target_obj| and returns the result."""
result = fuzz_target_obj.fuzz()
- logging.debug('corpus_path: %s', os.listdir(result.corpus_path))
- self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name)
+ logging.debug('Corpus path contents: %s.', os.listdir(result.corpus_path))
+ self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name,
+ result.corpus_path)
return result
def cleanup_after_fuzz_target_run(self, fuzz_target_obj):
@@ -255,15 +276,21 @@ class BatchFuzzTargetRunner(BaseFuzzTargetRunner):
return result
+_RUN_FUZZERS_MODE_RUNNER_MAPPING = {
+ 'batch': BatchFuzzTargetRunner,
+ 'coverage': CoverageTargetRunner,
+ 'prune': PruneTargetRunner,
+ 'ci': CiFuzzTargetRunner,
+}
+
+
def get_fuzz_target_runner(config):
"""Returns a fuzz target runner object based on the run_fuzzers_mode of
|config|."""
- logging.info('RUN_FUZZERS_MODE is: %s', config.run_fuzzers_mode)
- if config.run_fuzzers_mode == 'batch':
- return BatchFuzzTargetRunner(config)
- if config.run_fuzzers_mode == 'coverage':
- return CoverageTargetRunner(config)
- return CiFuzzTargetRunner(config)
+ runner = _RUN_FUZZERS_MODE_RUNNER_MAPPING[config.run_fuzzers_mode](config)
+ logging.info('RUN_FUZZERS_MODE is: %s. Runner: %s.', config.run_fuzzers_mode,
+ runner)
+ return runner
def run_fuzzers(config): # pylint: disable=too-many-locals
diff --git a/infra/cifuzz/workspace_utils.py b/infra/cifuzz/workspace_utils.py
index d6a68665..d3b9596d 100644
--- a/infra/cifuzz/workspace_utils.py
+++ b/infra/cifuzz/workspace_utils.py
@@ -63,3 +63,8 @@ class Workspace:
def corpora(self):
"""The directory where corpora from ClusterFuzz are stored."""
return os.path.join(self.workspace, 'cifuzz-corpus')
+
+ @property
+ def pruned_corpora(self):
+ """The directory where pruned corpora are stored."""
+ return os.path.join(self.workspace, 'cifuzz-pruned-corpus')