From 6f0aa82cd149910cb60e0d7d5ab49e47453f8c87 Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Tue, 27 Jul 2021 12:11:53 -0700 Subject: [NFC][cifuzz] Move Workspace from docker into config_utils. (#6117) It doesn't really fit into docker. --- infra/cifuzz/affected_fuzz_targets_test.py | 4 +-- infra/cifuzz/build_fuzzers.py | 3 +- infra/cifuzz/build_fuzzers_entrypoint.py | 3 +- infra/cifuzz/clusterfuzz_deployment_test.py | 7 ++--- infra/cifuzz/config_utils.py | 49 +++++++++++++++++++++++++++++ infra/cifuzz/docker.py | 49 ----------------------------- infra/cifuzz/docker_test.py | 2 +- infra/cifuzz/filestore/git/__init__.py | 4 +-- infra/cifuzz/fuzz_target_test.py | 4 +-- infra/cifuzz/run_fuzzers.py | 4 +-- infra/cifuzz/test_helpers.py | 3 +- 11 files changed, 65 insertions(+), 67 deletions(-) (limited to 'infra/cifuzz') diff --git a/infra/cifuzz/affected_fuzz_targets_test.py b/infra/cifuzz/affected_fuzz_targets_test.py index d5e948a6..34e51b97 100644 --- a/infra/cifuzz/affected_fuzz_targets_test.py +++ b/infra/cifuzz/affected_fuzz_targets_test.py @@ -22,7 +22,7 @@ import parameterized import affected_fuzz_targets import clusterfuzz_deployment -import docker +import config_utils import test_helpers # pylint: disable=protected-access @@ -64,7 +64,7 @@ class RemoveUnaffectedFuzzTargets(unittest.TestCase): is_github=True, oss_fuzz_project_name=EXAMPLE_PROJECT, workspace='/workspace') - workspace = docker.Workspace(config) + workspace = config_utils.Workspace(config) deployment = clusterfuzz_deployment.get_clusterfuzz_deployment( config, workspace) # We can't use fakefs in this test because this test executes diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py index ef70c90d..7831e0bf 100644 --- a/infra/cifuzz/build_fuzzers.py +++ b/infra/cifuzz/build_fuzzers.py @@ -20,6 +20,7 @@ import sys import affected_fuzz_targets import clusterfuzz_deployment +import config_utils import continuous_integration import docker @@ -51,7 +52,7 @@ class Builder: # pylint: disable=too-many-instance-attributes def __init__(self, config, ci_system): self.config = config self.ci_system = ci_system - self.workspace = docker.Workspace(config) + self.workspace = config_utils.Workspace(config) self.workspace.initialize_dir(self.workspace.out) self.workspace.initialize_dir(self.workspace.work) self.clusterfuzz_deployment = ( diff --git a/infra/cifuzz/build_fuzzers_entrypoint.py b/infra/cifuzz/build_fuzzers_entrypoint.py index 505a3741..b391afb9 100644 --- a/infra/cifuzz/build_fuzzers_entrypoint.py +++ b/infra/cifuzz/build_fuzzers_entrypoint.py @@ -17,7 +17,6 @@ import sys import build_fuzzers import config_utils -import docker # pylint: disable=c-extension-no-member # pylint gets confused because of the relative import of cifuzz. @@ -54,7 +53,7 @@ def build_fuzzers_entrypoint(): returncode = 0 # yapf: disable elif build_fuzzers.check_fuzzer_build( - docker.Workspace(config), + config_utils.Workspace(config), config.sanitizer, config.language, allowed_broken_targets_percentage=config.allowed_broken_targets_percentage diff --git a/infra/cifuzz/clusterfuzz_deployment_test.py b/infra/cifuzz/clusterfuzz_deployment_test.py index c896d86e..83237960 100644 --- a/infra/cifuzz/clusterfuzz_deployment_test.py +++ b/infra/cifuzz/clusterfuzz_deployment_test.py @@ -22,7 +22,6 @@ from pyfakefs import fake_filesystem_unittest import clusterfuzz_deployment import config_utils -import docker import test_helpers # NOTE: This integration test relies on @@ -54,7 +53,7 @@ def _create_config(**kwargs): def _create_deployment(**kwargs): config = _create_config(**kwargs) - workspace = docker.Workspace(config) + workspace = config_utils.Workspace(config) return clusterfuzz_deployment.get_clusterfuzz_deployment(config, workspace) @@ -190,7 +189,7 @@ class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase): config = test_helpers.create_run_config(build_integration_path='/', workspace=WORKSPACE, is_github=False) - workspace = docker.Workspace(config) + workspace = config_utils.Workspace(config) self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment( config, workspace) @@ -242,7 +241,7 @@ class GetClusterFuzzDeploymentTest(unittest.TestCase): new_callable=mock.PropertyMock): with mock.patch('filestore_utils.get_filestore', return_value=None): config = _create_config() - workspace = docker.Workspace(config) + workspace = config_utils.Workspace(config) self.assertIsInstance( clusterfuzz_deployment.get_clusterfuzz_deployment( diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index 6493d7f9..12139033 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -195,3 +195,52 @@ class BuildFuzzersConfig(BaseConfig): # var is set to '0'? self.keep_unaffected_fuzz_targets = bool( os.getenv('KEEP_UNAFFECTED_FUZZERS')) + + +class Workspace: + """Class representing the workspace directory.""" + + def __init__(self, config): + self.workspace = config.workspace + + def initialize_dir(self, directory): # pylint: disable=no-self-use + """Creates directory if it doesn't already exist, otherwise does nothing.""" + os.makedirs(directory, exist_ok=True) + + @property + def out(self): + """The out directory used for storing the fuzzer build built by + build_fuzzers.""" + # Don't use 'out' because it needs to be used by artifacts. + return os.path.join(self.workspace, 'build-out') + + @property + def work(self): + """The directory used as the work directory for the fuzzer build/run.""" + return os.path.join(self.workspace, 'work') + + @property + def artifacts(self): + """The directory used to store artifacts for download by CI-system users.""" + # This is hardcoded by a lot of clients, so we need to use this. + return os.path.join(self.workspace, 'out', 'artifacts') + + @property + def clusterfuzz_build(self): + """The directory where builds from ClusterFuzz are stored.""" + return os.path.join(self.workspace, 'cifuzz-prev-build') + + @property + def clusterfuzz_coverage(self): + """The directory where builds from ClusterFuzz are stored.""" + return os.path.join(self.workspace, 'cifuzz-prev-coverage') + + @property + def coverage_report(self): + """The directory where coverage reports generated by cifuzz are put.""" + return os.path.join(self.workspace, 'cifuzz-coverage') + + @property + def corpora(self): + """The directory where corpora from ClusterFuzz are stored.""" + return os.path.join(self.workspace, 'cifuzz-corpus') diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py index 9621c799..ab6c34df 100644 --- a/infra/cifuzz/docker.py +++ b/infra/cifuzz/docker.py @@ -96,52 +96,3 @@ def _get_args_mapping_host_path_to_container(host_path, container_path=None): # --volumes-from) is used for mapping volumes. It will break production. container_path = host_path if container_path is None else container_path return ['-v', f'{host_path}:{container_path}'] - - -class Workspace: - """Class representing the workspace directory.""" - - def __init__(self, config): - self.workspace = config.workspace - - def initialize_dir(self, directory): # pylint: disable=no-self-use - """Creates directory if it doesn't already exist, otherwise does nothing.""" - os.makedirs(directory, exist_ok=True) - - @property - def out(self): - """The out directory used for storing the fuzzer build built by - build_fuzzers.""" - # Don't use 'out' because it needs to be used by artifacts. - return os.path.join(self.workspace, 'build-out') - - @property - def work(self): - """The directory used as the work directory for the fuzzer build/run.""" - return os.path.join(self.workspace, 'work') - - @property - def artifacts(self): - """The directory used to store artifacts for download by CI-system users.""" - # This is hardcoded by a lot of clients, so we need to use this. - return os.path.join(self.workspace, 'out', 'artifacts') - - @property - def clusterfuzz_build(self): - """The directory where builds from ClusterFuzz are stored.""" - return os.path.join(self.workspace, 'cifuzz-prev-build') - - @property - def clusterfuzz_coverage(self): - """The directory where builds from ClusterFuzz are stored.""" - return os.path.join(self.workspace, 'cifuzz-prev-coverage') - - @property - def coverage_report(self): - """The directory where coverage reports generated by cifuzz are put.""" - return os.path.join(self.workspace, 'cifuzz-coverage') - - @property - def corpora(self): - """The directory where corpora from ClusterFuzz are stored.""" - return os.path.join(self.workspace, 'cifuzz-corpus') diff --git a/infra/cifuzz/docker_test.py b/infra/cifuzz/docker_test.py index 7146c8c8..68effb5a 100644 --- a/infra/cifuzz/docker_test.py +++ b/infra/cifuzz/docker_test.py @@ -21,7 +21,7 @@ import docker CONTAINER_NAME = 'example-container' config = config_utils.RunFuzzersConfig() config.workspace = '/workspace' -WORKSPACE = docker.Workspace(config) +WORKSPACE = config_utils.Workspace(config) SANITIZER = 'example-sanitizer' LANGUAGE = 'example-language' diff --git a/infra/cifuzz/filestore/git/__init__.py b/infra/cifuzz/filestore/git/__init__.py index 5281c10a..f1ac0c2c 100644 --- a/infra/cifuzz/filestore/git/__init__.py +++ b/infra/cifuzz/filestore/git/__init__.py @@ -83,8 +83,8 @@ class GitFilestore(filestore.BaseFilestore): upload_path, local_path, replace=False): - """Uploads a directory to git. If `replace` is True, then existing contents in - the upload_path is deleted.""" + """Uploads a directory to git. If `replace` is True, then existing contents + in the upload_path is deleted.""" self._reset_git(branch) full_repo_path = os.path.join(self.repo_path, upload_path) diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py index 5e65d02a..350a8030 100644 --- a/infra/cifuzz/fuzz_target_test.py +++ b/infra/cifuzz/fuzz_target_test.py @@ -23,7 +23,7 @@ import parameterized from pyfakefs import fake_filesystem_unittest import clusterfuzz_deployment -import docker +import config_utils import fuzz_target import test_helpers @@ -59,7 +59,7 @@ def _create_config(**kwargs): def _create_deployment(**kwargs): config = _create_config(**kwargs) - workspace = docker.Workspace(config) + workspace = config_utils.Workspace(config) return clusterfuzz_deployment.get_clusterfuzz_deployment(config, workspace) diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py index 0a8df2ce..ce2dfb85 100644 --- a/infra/cifuzz/run_fuzzers.py +++ b/infra/cifuzz/run_fuzzers.py @@ -20,7 +20,7 @@ import sys import time import clusterfuzz_deployment -import docker +import config_utils import fuzz_target import generate_coverage_report import stack_parser @@ -43,7 +43,7 @@ class BaseFuzzTargetRunner: def __init__(self, config): self.config = config - self.workspace = docker.Workspace(config) + self.workspace = config_utils.Workspace(config) self.clusterfuzz_deployment = ( clusterfuzz_deployment.get_clusterfuzz_deployment( self.config, self.workspace)) diff --git a/infra/cifuzz/test_helpers.py b/infra/cifuzz/test_helpers.py index c5eea2de..3e87f7a7 100644 --- a/infra/cifuzz/test_helpers.py +++ b/infra/cifuzz/test_helpers.py @@ -20,7 +20,6 @@ import tempfile from unittest import mock import config_utils -import docker def _create_config(config_cls, **kwargs): @@ -53,7 +52,7 @@ def create_workspace(workspace_path='/workspace'): """Returns a workspace located at |workspace_path| ('/workspace' by default).""" config = create_run_config(workspace=workspace_path) - return docker.Workspace(config) + return config_utils.Workspace(config) def patch_environ(testcase_obj, env=None): -- cgit v1.2.3