diff options
author | jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> | 2021-08-10 11:10:10 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-10 11:10:10 -0700 |
commit | 94cfc4fe2f0c9743ab7acd19910d6961f2317eeb (patch) | |
tree | a4b0b54cd2c6ed7556bad15b31dc7e1f3c046086 /infra/cifuzz | |
parent | e407f54e61a134de43019d69947a72a471bdd7c6 (diff) |
[cifuzz] Add pruning task (#6188)
Fixes: #6064
Diffstat (limited to 'infra/cifuzz')
-rw-r--r-- | infra/cifuzz/clusterfuzz_deployment.py | 69 | ||||
-rw-r--r-- | infra/cifuzz/clusterfuzz_deployment_test.py | 42 | ||||
-rw-r--r-- | infra/cifuzz/config_utils.py | 3 | ||||
-rw-r--r-- | infra/cifuzz/fuzz_target.py | 54 | ||||
-rw-r--r-- | infra/cifuzz/fuzz_target_test.py | 18 | ||||
-rw-r--r-- | infra/cifuzz/generate_coverage_report.py | 7 | ||||
-rw-r--r-- | infra/cifuzz/generate_coverage_report_test.py | 6 | ||||
-rw-r--r-- | infra/cifuzz/run_fuzzers.py | 43 | ||||
-rw-r--r-- | infra/cifuzz/workspace_utils.py | 5 |
9 files changed, 157 insertions, 90 deletions
diff --git a/infra/cifuzz/clusterfuzz_deployment.py b/infra/cifuzz/clusterfuzz_deployment.py index eb20dc4d..a2be1ff9 100644 --- a/infra/cifuzz/clusterfuzz_deployment.py +++ b/infra/cifuzz/clusterfuzz_deployment.py @@ -18,6 +18,7 @@ import sys import urllib.error import urllib.request +import config_utils import filestore import filestore_utils import http_utils @@ -50,8 +51,8 @@ class BaseClusterFuzzDeployment: """ raise NotImplementedError('Child class must implement method.') - def download_corpus(self, target_name): - """Downloads the corpus for |target_name| from ClusterFuzz. + def download_corpus(self, target_name, corpus_dir): + """Downloads the corpus for |target_name| from ClusterFuzz to |corpus_dir|. Returns: A path to where the OSS-Fuzz build was stored, or None if it wasn't. @@ -62,11 +63,7 @@ class BaseClusterFuzzDeployment: """Uploads crashes in |crashes_dir| to filestore.""" raise NotImplementedError('Child class must implement method.') - def get_target_corpus_dir(self, target_name): - """Returns the path to the corpus dir for |target_name|.""" - return os.path.join(self.workspace.corpora, target_name) - - def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument + def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument """Uploads the corpus for |target_name| to filestore.""" raise NotImplementedError('Child class must implement method.') @@ -78,12 +75,10 @@ class BaseClusterFuzzDeployment: """Returns the project coverage object for the project.""" raise NotImplementedError('Child class must implement method.') - def make_empty_corpus_dir(self, target_name): - """Makes an empty corpus directory for |target_name| in |parent_dir| and - returns the path to the directory.""" - corpus_dir = self.get_target_corpus_dir(target_name) - os.makedirs(corpus_dir, exist_ok=True) - return corpus_dir + +def _make_empty_dir_if_nonexistent(path): + """Makes an empty directory at |path| if it does not exist.""" + os.makedirs(path, exist_ok=True) class ClusterFuzzLite(BaseClusterFuzzDeployment): @@ -104,7 +99,7 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): # called if multiple bugs are found. return self.workspace.clusterfuzz_build - os.makedirs(self.workspace.clusterfuzz_build, exist_ok=True) + _make_empty_dir_if_nonexistent(self.workspace.clusterfuzz_build) build_name = self._get_build_name() try: @@ -118,8 +113,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): return None - def download_corpus(self, target_name): - corpus_dir = self.make_empty_corpus_dir(target_name) + def download_corpus(self, target_name, corpus_dir): + _make_empty_dir_if_nonexistent(corpus_dir) logging.info('Downloading corpus for %s to %s.', target_name, corpus_dir) corpus_name = self._get_corpus_name(target_name) try: @@ -142,9 +137,8 @@ class ClusterFuzzLite(BaseClusterFuzzDeployment): """Returns the name of the crashes artifact.""" return 'current' - def upload_corpus(self, target_name): + def upload_corpus(self, target_name, corpus_dir): """Upload the corpus produced by |target_name|.""" - corpus_dir = self.get_target_corpus_dir(target_name) logging.info('Uploading corpus in %s for %s.', corpus_dir, target_name) name = self._get_corpus_name(target_name) try: @@ -241,7 +235,7 @@ class OSSFuzz(BaseClusterFuzzDeployment): # again. return self.workspace.clusterfuzz_build - os.makedirs(self.workspace.clusterfuzz_build, exist_ok=True) + _make_empty_dir_if_nonexistent(self.workspace.clusterfuzz_build) latest_build_name = self.get_latest_build_name() if not latest_build_name: @@ -263,7 +257,7 @@ class OSSFuzz(BaseClusterFuzzDeployment): """Noop Implementation of upload_latest_build.""" logging.info('Not uploading latest build because on OSS-Fuzz.') - def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument + def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument """Noop Implementation of upload_corpus.""" logging.info('Not uploading corpus because on OSS-Fuzz.') @@ -271,13 +265,13 @@ class OSSFuzz(BaseClusterFuzzDeployment): """Noop Implementation of upload_crashes.""" logging.info('Not uploading crashes because on OSS-Fuzz.') - def download_corpus(self, target_name): + def download_corpus(self, target_name, corpus_dir): """Downloads the latest OSS-Fuzz corpus for the target. Returns: The local path to to corpus or None if download failed. """ - corpus_dir = self.make_empty_corpus_dir(target_name) + _make_empty_dir_if_nonexistent(corpus_dir) project_qualified_fuzz_target_name = target_name qualified_name_prefix = self.config.oss_fuzz_project_name + '_' if not target_name.startswith(qualified_name_prefix): @@ -314,7 +308,7 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment): logging.info('Not uploading latest build because no ClusterFuzz ' 'deployment.') - def upload_corpus(self, target_name): # pylint: disable=no-self-use,unused-argument + def upload_corpus(self, target_name, corpus_dir): # pylint: disable=no-self-use,unused-argument """Noop Implementation of upload_corpus.""" logging.info('Not uploading corpus because no ClusterFuzz deployment.') @@ -322,10 +316,10 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment): """Noop Implementation of upload_crashes.""" logging.info('Not uploading crashes because no ClusterFuzz deployment.') - def download_corpus(self, target_name): + def download_corpus(self, target_name, corpus_dir): """Noop Implementation of download_corpus.""" logging.info('Not downloading corpus because no ClusterFuzz deployment.') - return self.make_empty_corpus_dir(target_name) + return _make_empty_dir_if_nonexistent(corpus_dir) def download_latest_build(self): # pylint: disable=no-self-use """Noop Implementation of download_latest_build.""" @@ -343,14 +337,21 @@ class NoClusterFuzzDeployment(BaseClusterFuzzDeployment): 'Not getting project coverage because no ClusterFuzz deployment.') +_PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING = { + config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI: + OSSFuzz, + config_utils.BaseConfig.Platform.INTERNAL_GITHUB: + OSSFuzz, + config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI: + NoClusterFuzzDeployment, + config_utils.BaseConfig.Platform.EXTERNAL_GITHUB: + ClusterFuzzLite, +} + + def get_clusterfuzz_deployment(config, workspace): """Returns object reprsenting deployment of ClusterFuzz used by |config|.""" - if (config.platform == config.Platform.INTERNAL_GENERIC_CI or - config.platform == config.Platform.INTERNAL_GITHUB): - logging.info('Using OSS-Fuzz as ClusterFuzz deployment.') - return OSSFuzz(config, workspace) - if config.platform == config.Platform.EXTERNAL_GENERIC_CI: - logging.info('Not using a ClusterFuzz deployment.') - return NoClusterFuzzDeployment(config, workspace) - logging.info('Using ClusterFuzzLite as ClusterFuzz deployment.') - return ClusterFuzzLite(config, workspace) + deployment_cls = _PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING[config.platform] + result = deployment_cls(config, workspace) + logging.info('ClusterFuzzDeployment: %s.', result) + return result diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py index 873ef8e3..3d3f0fe2 100644 --- a/infra/cifuzz/clusterfuzz_deployment_test.py +++ b/infra/cifuzz/clusterfuzz_deployment_test.py @@ -64,28 +64,26 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() self.deployment = _create_deployment() + self.corpus_dir = os.path.join(self.deployment.workspace.corpora, + EXAMPLE_FUZZER) @mock.patch('http_utils.download_and_unpack_zip', return_value=True) def test_download_corpus(self, mocked_download_and_unpack_zip): """Tests that we can download a corpus for a valid project.""" - result = self.deployment.download_corpus(EXAMPLE_FUZZER) - self.assertIsNotNone(result) - expected_corpus_dir = os.path.join(self.deployment.workspace.corpora, - EXAMPLE_FUZZER) + self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir) expected_url = ('https://storage.googleapis.com/example-backup.' 'clusterfuzz-external.appspot.com/corpus/libFuzzer/' 'example_crash_fuzzer/public.zip') call_args, _ = mocked_download_and_unpack_zip.call_args - self.assertEqual(call_args, (expected_url, expected_corpus_dir)) + self.assertEqual(call_args, (expected_url, self.corpus_dir)) + self.assertTrue(os.path.exists(self.corpus_dir)) @mock.patch('http_utils.download_and_unpack_zip', return_value=False) def test_download_corpus_fail(self, _): """Tests that when downloading fails, an empty corpus directory is still returned.""" - corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER) - self.assertEqual(corpus_path, - '/workspace/cifuzz-corpus/example_crash_fuzzer') - self.assertEqual(os.listdir(corpus_path), []) + self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir) + self.assertEqual(os.listdir(self.corpus_dir), []) def test_get_latest_build_name(self): """Tests that the latest build name can be retrieved from GCS.""" @@ -96,7 +94,7 @@ class OSSFuzzTest(fake_filesystem_unittest.TestCase): @parameterized.parameterized.expand([ ('upload_latest_build', tuple(), 'Not uploading latest build because on OSS-Fuzz.'), - ('upload_corpus', ('target',), + ('upload_corpus', ('target', 'corpus-dir'), 'Not uploading corpus because on OSS-Fuzz.'), ('upload_crashes', tuple(), 'Not uploading crashes because on OSS-Fuzz.'), ]) @@ -133,27 +131,25 @@ class ClusterFuzzLiteTest(fake_filesystem_unittest.TestCase): self.deployment = _create_deployment(run_fuzzers_mode='batch', oss_fuzz_project_name='', is_github=True) + self.corpus_dir = os.path.join(self.deployment.workspace.corpora, + EXAMPLE_FUZZER) @mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus', return_value=True) def test_download_corpus(self, mocked_download_corpus): """Tests that download_corpus works for a valid project.""" - result = self.deployment.download_corpus(EXAMPLE_FUZZER) - expected_corpus_dir = os.path.join(WORKSPACE, 'cifuzz-corpus', - EXAMPLE_FUZZER) - self.assertEqual(result, expected_corpus_dir) + self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir) mocked_download_corpus.assert_called_with('example_crash_fuzzer', - expected_corpus_dir) + self.corpus_dir) + self.assertTrue(os.path.exists(self.corpus_dir)) @mock.patch('filestore.github_actions.GithubActionsFilestore.download_corpus', side_effect=Exception) def test_download_corpus_fail(self, _): """Tests that when downloading fails, an empty corpus directory is still returned.""" - corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER) - self.assertEqual(corpus_path, - '/workspace/cifuzz-corpus/example_crash_fuzzer') - self.assertEqual(os.listdir(corpus_path), []) + self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir) + self.assertEqual(os.listdir(self.corpus_dir), []) @mock.patch('filestore.github_actions.GithubActionsFilestore.download_build', return_value=True) @@ -191,21 +187,21 @@ class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase): workspace = workspace_utils.Workspace(config) self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment( config, workspace) + self.corpus_dir = os.path.join(workspace.corpora, EXAMPLE_FUZZER) @mock.patch('logging.info') def test_download_corpus(self, mocked_info): """Tests that download corpus returns the path to the empty corpus directory.""" - corpus_path = self.deployment.download_corpus(EXAMPLE_FUZZER) - self.assertEqual(corpus_path, - '/workspace/cifuzz-corpus/example_crash_fuzzer') + self.deployment.download_corpus(EXAMPLE_FUZZER, self.corpus_dir) mocked_info.assert_called_with( 'Not downloading corpus because no ClusterFuzz deployment.') + self.assertTrue(os.path.exists(self.corpus_dir)) @parameterized.parameterized.expand([ ('upload_latest_build', tuple(), 'Not uploading latest build because no ClusterFuzz deployment.'), - ('upload_corpus', ('target',), + ('upload_corpus', ('target', 'corpus-dir'), 'Not uploading corpus because no ClusterFuzz deployment.'), ('upload_crashes', tuple(), 'Not uploading crashes because no ClusterFuzz deployment.'), diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index 8ca3eec1..b8029bf4 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -26,7 +26,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import constants -RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage'] +RUN_FUZZERS_MODES = ['batch', 'ci', 'coverage', 'prune'] SANITIZERS = ['address', 'memory', 'undefined', 'coverage'] # TODO(metzman): Set these on config objects so there's one source of truth. @@ -277,6 +277,7 @@ class RunFuzzersConfig(BaseConfig): def __init__(self): super().__init__() + # TODO(metzman): Pick a better default for pruning. self.fuzz_seconds = int(os.environ.get('FUZZ_SECONDS', 600)) self.run_fuzzers_mode = os.environ.get('RUN_FUZZERS_MODE', 'ci').lower() if self.is_coverage: diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index 14a63a1e..7fd43a69 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -53,6 +53,17 @@ class ReproduceError(Exception): """Error for when we can't attempt to reproduce a crash.""" +def get_fuzz_target_corpus_dir(workspace, target_name): + """Returns the directory for storing |target_name|'s corpus in |workspace|.""" + return os.path.join(workspace.corpora, target_name) + + +def get_fuzz_target_pruned_corpus_dir(workspace, target_name): + """Returns the directory for storing |target_name|'s puned corpus in + |workspace|.""" + return os.path.join(workspace.pruned_corpora, target_name) + + class FuzzTarget: # pylint: disable=too-many-instance-attributes """A class to manage a single fuzz target. @@ -82,29 +93,54 @@ class FuzzTarget: # pylint: disable=too-many-instance-attributes self.workspace = workspace self.clusterfuzz_deployment = clusterfuzz_deployment self.config = config - self.latest_corpus_path = None + self.latest_corpus_path = get_fuzz_target_corpus_dir( + self.workspace, self.target_name) + os.makedirs(self.latest_corpus_path, exist_ok=True) + self.pruned_corpus_path = get_fuzz_target_pruned_corpus_dir( + self.workspace, self.target_name) + os.makedirs(self.pruned_corpus_path, exist_ok=True) + + def _download_corpus(self): + """Downloads the corpus for the target from ClusterFuzz and returns the path + to the corpus. An empty directory is provided if the corpus can't be + downloaded or is empty.""" + self.clusterfuzz_deployment.download_corpus(self.target_name, + self.latest_corpus_path) + return self.latest_corpus_path + + def prune(self): + """Prunes the corpus and returns the result.""" + self._download_corpus() + prune_options = [ + '-merge=1', self.pruned_corpus_path, self.latest_corpus_path + ] + result = self.fuzz(use_corpus=False, extra_libfuzzer_options=prune_options) + return FuzzResult(result.testcase, result.stacktrace, + self.pruned_corpus_path) - def fuzz(self): + def fuzz(self, use_corpus=True, extra_libfuzzer_options=None): """Starts the fuzz target run for the length of time specified by duration. Returns: FuzzResult namedtuple with stacktrace and testcase if applicable. """ logging.info('Running fuzzer: %s.', self.target_name) + if extra_libfuzzer_options is None: + extra_libfuzzer_options = [] env = base_runner_utils.get_env(self.config, self.workspace) # TODO(metzman): Is this needed? env['RUN_FUZZER_MODE'] = 'interactive' - # If corpus can be downloaded, use it for fuzzing. - self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus( - self.target_name) - env['CORPUS_DIR'] = self.latest_corpus_path + if use_corpus: + # If corpus can be downloaded, use it for fuzzing. + self._download_corpus() + env['CORPUS_DIR'] = self.latest_corpus_path options = LIBFUZZER_OPTIONS.copy() + [ f'-max_total_time={self.duration}', # Make sure libFuzzer artifact files don't pollute $OUT. f'-artifact_prefix={self.workspace.artifacts}/' - ] + ] + extra_libfuzzer_options command = ['run_fuzzer', self.target_name] + options logging.info('Running command: %s', command) @@ -151,10 +187,10 @@ class FuzzTarget: # pylint: disable=too-many-instance-attributes self.target_name) # Delete the seed corpus, corpus, and fuzz target. - if self.latest_corpus_path and os.path.exists(self.latest_corpus_path): + for corpus_path in [self.latest_corpus_path, self.pruned_corpus_path]: # Use ignore_errors=True to fix # https://github.com/google/oss-fuzz/issues/5383. - shutil.rmtree(self.latest_corpus_path, ignore_errors=True) + shutil.rmtree(corpus_path, ignore_errors=True) target_seed_corpus_path = self.target_path + '_seed_corpus.zip' if os.path.exists(target_seed_corpus_path): diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py index 268d3a64..ee262eda 100644 --- a/infra/cifuzz/fuzz_target_test.py +++ b/infra/cifuzz/fuzz_target_test.py @@ -75,7 +75,11 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase): self.workspace = deployment.workspace self.fuzz_target_path = os.path.join(self.workspace.out, self.fuzz_target_name) + self.setUpPyfakefs() + self.fs.create_file(self.fuzz_target_path) self.testcase_path = '/testcase' + self.fs.create_file(self.testcase_path) + self.target = fuzz_target.FuzzTarget(self.fuzz_target_path, fuzz_target.REPRODUCE_ATTEMPTS, self.workspace, deployment, @@ -86,7 +90,6 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase): def test_reproducible(self, _): """Tests that is_reproducible returns True if crash is detected and that is_reproducible uses the correct command to reproduce a crash.""" - self._set_up_fakefs() all_repro = [EXECUTE_FAILURE_RETVAL] * fuzz_target.REPRODUCE_ATTEMPTS with mock.patch('utils.execute', side_effect=all_repro) as mocked_execute: result = self.target.is_reproducible(self.testcase_path, @@ -106,17 +109,9 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase): self.assertTrue(result) self.assertEqual(1, mocked_execute.call_count) - def _set_up_fakefs(self): - """Helper to setup pyfakefs and add important files to the fake - filesystem.""" - self.setUpPyfakefs() - self.fs.create_file(self.fuzz_target_path) - self.fs.create_file(self.testcase_path) - def test_flaky(self, _): """Tests that is_reproducible returns True if crash is detected on the last attempt.""" - self._set_up_fakefs() last_time_repro = [EXECUTE_SUCCESS_RETVAL] * 9 + [EXECUTE_FAILURE_RETVAL] with mock.patch('utils.execute', side_effect=last_time_repro) as mocked_execute: @@ -136,7 +131,6 @@ class IsReproducibleTest(fake_filesystem_unittest.TestCase): """Tests that is_reproducible returns False for a crash that did not reproduce.""" all_unrepro = [EXECUTE_SUCCESS_RETVAL] * fuzz_target.REPRODUCE_ATTEMPTS - self._set_up_fakefs() with mock.patch('utils.execute', side_effect=all_unrepro): result = self.target.is_reproducible(self.testcase_path, self.fuzz_target_path) @@ -172,13 +166,13 @@ class IsCrashReportableTest(fake_filesystem_unittest.TestCase): def setUp(self): """Sets up example fuzz target to test is_crash_reportable method.""" + self.setUpPyfakefs() self.fuzz_target_path = '/example/do_stuff_fuzzer' deployment = _create_deployment() self.target = fuzz_target.FuzzTarget(self.fuzz_target_path, 100, - '/example/outdir', deployment, + deployment.workspace, deployment, deployment.config) self.oss_fuzz_build_path = '/oss-fuzz-build' - self.setUpPyfakefs() self.fs.create_file(self.fuzz_target_path) self.oss_fuzz_target_path = os.path.join( self.oss_fuzz_build_path, os.path.basename(self.fuzz_target_path)) diff --git a/infra/cifuzz/generate_coverage_report.py b/infra/cifuzz/generate_coverage_report.py index 2bfbe51d..9901c452 100644 --- a/infra/cifuzz/generate_coverage_report.py +++ b/infra/cifuzz/generate_coverage_report.py @@ -15,6 +15,7 @@ import os import base_runner_utils +import fuzz_target import utils @@ -33,8 +34,10 @@ def download_corpora(fuzz_target_paths, clusterfuzz_deployment): """Downloads corpora for fuzz targets in |fuzz_target_paths| using |clusterfuzz_deployment| to download corpora from ClusterFuzz/OSS-Fuzz.""" for target_path in fuzz_target_paths: - target = os.path.basename(target_path) - clusterfuzz_deployment.download_corpus(target) + target_name = os.path.basename(target_path) + corpus_dir = fuzz_target.get_fuzz_target_corpus_dir( + clusterfuzz_deployment.workspace, target_name) + clusterfuzz_deployment.download_corpus(target_name, corpus_dir) def generate_coverage_report(fuzz_target_paths, workspace, diff --git a/infra/cifuzz/generate_coverage_report_test.py b/infra/cifuzz/generate_coverage_report_test.py index bed49430..b24eb3fe 100644 --- a/infra/cifuzz/generate_coverage_report_test.py +++ b/infra/cifuzz/generate_coverage_report_test.py @@ -60,8 +60,12 @@ class DownloadCorporaTest(unittest.TestCase): def test_download_corpora(self): # pylint: disable=no-self-use """Tests that download_corpora works as intended.""" clusterfuzz_deployment = mock.Mock() + clusterfuzz_deployment.workspace = test_helpers.create_workspace() fuzz_target_paths = ['/path/to/fuzzer1', '/path/to/fuzzer2'] - expected_calls = [mock.call('fuzzer1'), mock.call('fuzzer2')] + expected_calls = [ + mock.call('fuzzer1', '/workspace/cifuzz-corpus/fuzzer1'), + mock.call('fuzzer2', '/workspace/cifuzz-corpus/fuzzer2') + ] generate_coverage_report.download_corpora(fuzz_target_paths, clusterfuzz_deployment) clusterfuzz_deployment.download_corpus.assert_has_calls(expected_calls) diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py index 93a30a37..79cf432c 100644 --- a/infra/cifuzz/run_fuzzers.py +++ b/infra/cifuzz/run_fuzzers.py @@ -166,6 +166,26 @@ class BaseFuzzTargetRunner: return bug_found +class PruneTargetRunner(BaseFuzzTargetRunner): + """Runner that prunes corpora.""" + + @property + def quit_on_bug_found(self): + return False + + def run_fuzz_target(self, fuzz_target_obj): + """Prunes with |fuzz_target_obj| and returns the result.""" + result = fuzz_target_obj.prune() + logging.debug('Corpus path contents: %s.', os.listdir(result.corpus_path)) + self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name, + result.corpus_path) + return result + + def cleanup_after_fuzz_target_run(self, fuzz_target_obj): # pylint: disable=no-self-use + """Cleans up after pruning with |fuzz_target_obj|.""" + fuzz_target_obj.free_disk_if_needed() + + class CoverageTargetRunner(BaseFuzzTargetRunner): """Runner that runs the 'coverage' command.""" @@ -224,8 +244,9 @@ class BatchFuzzTargetRunner(BaseFuzzTargetRunner): def run_fuzz_target(self, fuzz_target_obj): """Fuzzes with |fuzz_target_obj| and returns the result.""" result = fuzz_target_obj.fuzz() - logging.debug('corpus_path: %s', os.listdir(result.corpus_path)) - self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name) + logging.debug('Corpus path contents: %s.', os.listdir(result.corpus_path)) + self.clusterfuzz_deployment.upload_corpus(fuzz_target_obj.target_name, + result.corpus_path) return result def cleanup_after_fuzz_target_run(self, fuzz_target_obj): @@ -255,15 +276,21 @@ class BatchFuzzTargetRunner(BaseFuzzTargetRunner): return result +_RUN_FUZZERS_MODE_RUNNER_MAPPING = { + 'batch': BatchFuzzTargetRunner, + 'coverage': CoverageTargetRunner, + 'prune': PruneTargetRunner, + 'ci': CiFuzzTargetRunner, +} + + def get_fuzz_target_runner(config): """Returns a fuzz target runner object based on the run_fuzzers_mode of |config|.""" - logging.info('RUN_FUZZERS_MODE is: %s', config.run_fuzzers_mode) - if config.run_fuzzers_mode == 'batch': - return BatchFuzzTargetRunner(config) - if config.run_fuzzers_mode == 'coverage': - return CoverageTargetRunner(config) - return CiFuzzTargetRunner(config) + runner = _RUN_FUZZERS_MODE_RUNNER_MAPPING[config.run_fuzzers_mode](config) + logging.info('RUN_FUZZERS_MODE is: %s. Runner: %s.', config.run_fuzzers_mode, + runner) + return runner def run_fuzzers(config): # pylint: disable=too-many-locals diff --git a/infra/cifuzz/workspace_utils.py b/infra/cifuzz/workspace_utils.py index d6a68665..d3b9596d 100644 --- a/infra/cifuzz/workspace_utils.py +++ b/infra/cifuzz/workspace_utils.py @@ -63,3 +63,8 @@ class Workspace: def corpora(self): """The directory where corpora from ClusterFuzz are stored.""" return os.path.join(self.workspace, 'cifuzz-corpus') + + @property + def pruned_corpora(self): + """The directory where pruned corpora are stored.""" + return os.path.join(self.workspace, 'cifuzz-pruned-corpus') |