diff options
| -rw-r--r-- | infra/bisector.py | 55 | ||||
| -rw-r--r-- | infra/build_specified_commit.py | 58 | ||||
| -rw-r--r-- | infra/repo_manager.py | 149 |
3 files changed, 170 insertions, 92 deletions
diff --git a/infra/bisector.py b/infra/bisector.py index 1b06997d..ee13be3d 100644 --- a/infra/bisector.py +++ b/infra/bisector.py @@ -94,35 +94,23 @@ def main(): return 0 -def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals - """From a commit range, this function caluclates which introduced a - specific error from a fuzz test_case_path. - - Args: - old_commit: The oldest commit in the error regression range. - new_commit: The newest commit in the error regression range. - test_case_path: The file path of the test case that triggers the error - fuzz_target: The name of the fuzzer to be tested. - build_data: a class holding all of the input parameters for bisection. - - Returns: - A Result. - - Raises: - ValueError: when a repo url can't be determine from the project. - """ +def _bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals + """Perform the bisect.""" with tempfile.TemporaryDirectory() as tmp_dir: repo_url, repo_path = build_specified_commit.detect_main_repo( build_data.project_name, commit=new_commit) if not repo_url or not repo_path: raise ValueError('Main git repo can not be determined.') + # Copy /src from the built Docker container to ensure all dependencies + # exist. This will be mounted when running them. host_src_dir = build_specified_commit.copy_src_from_docker( build_data.project_name, tmp_dir) - bisect_repo_manager = repo_manager.RepoManager( - repo_url, host_src_dir, repo_name=os.path.basename(repo_path)) + bisect_repo_manager = repo_manager.BaseRepoManager( + os.path.join(host_src_dir, os.path.basename(repo_path))) commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) + old_idx = len(commit_list) - 1 new_idx = 0 logging.info('Testing against new_commit (%s)', commit_list[new_idx]) @@ -166,5 +154,34 @@ def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # return Result(repo_url, commit_list[new_idx]) +def bisect(old_commit, new_commit, test_case_path, fuzz_target, build_data): # pylint: disable=too-many-locals + """From a commit range, this function caluclates which introduced a + specific error from a fuzz test_case_path. + + Args: + old_commit: The oldest commit in the error regression range. + new_commit: The newest commit in the error regression range. + test_case_path: The file path of the test case that triggers the error + fuzz_target: The name of the fuzzer to be tested. + build_data: a class holding all of the input parameters for bisection. + + Returns: + The commit SHA that introduced the error or None. + + Raises: + ValueError: when a repo url can't be determine from the project. + """ + result = _bisect(old_commit, new_commit, test_case_path, fuzz_target, + build_data) + + # Clean up projects/ as _bisect may have modified it. + oss_fuzz_repo_manager = repo_manager.BaseRepoManager(helper.OSS_FUZZ_DIR) + oss_fuzz_repo_manager.git(['reset', 'projects']) + oss_fuzz_repo_manager.git(['checkout', 'projects']) + oss_fuzz_repo_manager.git(['clean', '-fxd', 'projects']) + + return result + + if __name__ == '__main__': main() diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index 5a012344..23b47eaa 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -21,8 +21,10 @@ import os import collections import logging import re +import shutil import helper +import repo_manager import utils BuildData = collections.namedtuple( @@ -49,7 +51,7 @@ def copy_src_from_docker(project_name, host_dir): def build_fuzzers_from_commit(commit, build_repo_manager, host_src_path, build_data): - """Builds a OSS-Fuzz fuzzer at a specific commit SHA. + """Builds a OSS-Fuzz fuzzer at a specific commit SHA. Args: commit: The commit SHA to build the fuzzers at. @@ -58,15 +60,51 @@ def build_fuzzers_from_commit(commit, build_repo_manager, host_src_path, Returns: 0 on successful build or error code on failure. """ - build_repo_manager.checkout_commit(commit, clean=False) - result = helper.build_fuzzers_impl(project_name=build_data.project_name, - clean=True, - engine=build_data.engine, - sanitizer=build_data.sanitizer, - architecture=build_data.architecture, - env_to_add=None, - source_path=host_src_path, - mount_location=os.path.join('/src')) + oss_fuzz_repo_manager = repo_manager.BaseRepoManager(helper.OSS_FUZZ_DIR) + num_retry = 1 + + for i in range(num_retry + 1): + build_repo_manager.checkout_commit(commit, clean=False) + result = helper.build_fuzzers_impl(project_name=build_data.project_name, + clean=True, + engine=build_data.engine, + sanitizer=build_data.sanitizer, + architecture=build_data.architecture, + env_to_add=None, + source_path=host_src_path, + mount_location='/src') + if result == 0 or i == num_retry: + break + + # Retry with an OSS-Fuzz builder container that's closer to the project + # commit date. + commit_date = build_repo_manager.commit_date(commit) + projects_dir = os.path.join('projects', build_data.project_name) + + # Find first change in the projects/<PROJECT> directory before the project + # commit date. + oss_fuzz_commit, _, _ = oss_fuzz_repo_manager.git([ + 'log', '--before=' + commit_date.isoformat(), '-n1', '--format=%H', + projects_dir + ], + check_result=True) + oss_fuzz_commit = oss_fuzz_commit.strip() + + logging.info('Build failed. Retrying on earlier OSS-Fuzz commit %s.', + oss_fuzz_commit) + + # Check out projects/<PROJECT> dir to the commit that was found. + oss_fuzz_repo_manager.git(['checkout', oss_fuzz_commit, projects_dir], + check_result=True) + + # Rebuild image and re-copy src dir since things in /src could have changed. + if not helper.build_image_impl(build_data.project_name): + raise RuntimeError('Failed to rebuild image.') + + shutil.rmtree(host_src_path, ignore_errors=True) + copy_src_from_docker(build_data.project_name, + os.path.dirname(host_src_path)) + return result == 0 diff --git a/infra/repo_manager.py b/infra/repo_manager.py index 71c5ba00..238cb3e7 100644 --- a/infra/repo_manager.py +++ b/infra/repo_manager.py @@ -21,6 +21,7 @@ a python API and manage the current state of the git repo. r_man = RepoManager('https://github.com/google/oss-fuzz.git') r_man.checkout('5668cc422c2c92d38a370545d3591039fb5bb8d4') """ +import datetime import logging import os import shutil @@ -28,48 +29,11 @@ import shutil import utils -class RepoManager: - """Class to manage git repos from python. - - Attributes: - repo_url: The location of the git repo. - base_dir: The location of where the repo clone is stored locally. - repo_name: The name of the GitHub project. - repo_dir: The location of the main repo. - """ - - def __init__(self, repo_url, base_dir, repo_name=None): - """Constructs a repo manager class. +class BaseRepoManager: + """Base repo manager.""" - Args: - repo_url: The github url needed to clone. - base_dir: The full file-path where the git repo is located. - repo_name: The name of the directory the repo is cloned to. - """ - self.repo_url = repo_url - self.base_dir = base_dir - if repo_name: - self.repo_name = repo_name - else: - self.repo_name = os.path.basename(self.repo_url).replace('.git', '') - self.repo_dir = os.path.join(self.base_dir, self.repo_name) - - if not os.path.exists(self.repo_dir): - self._clone() - - def _clone(self): - """Creates a clone of the repo in the specified directory. - - Raises: - ValueError: when the repo is not able to be cloned. - """ - if not os.path.exists(self.base_dir): - os.makedirs(self.base_dir) - self.remove_repo() - out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name], - location=self.base_dir) - if not self._is_git_repo(): - raise ValueError('%s is not a git repo' % self.repo_url) + def __init__(self, repo_dir): + self.repo_dir = repo_dir def _is_git_repo(self): """Test if the current repo dir is a git repo or not. @@ -80,6 +44,20 @@ class RepoManager: git_path = os.path.join(self.repo_dir, '.git') return os.path.isdir(git_path) + def git(self, cmd, check_result=False): + """Run a git command. + + Args: + command: The git command as a list to be run. + check_result: Should an exception be thrown on failed command. + + Returns: + stdout, stderr, error code. + """ + return utils.execute(['git'] + cmd, + location=self.repo_dir, + check_result=check_result) + def commit_exists(self, commit): """Checks to see if a commit exists in the project repo. @@ -92,10 +70,22 @@ class RepoManager: if not commit.rstrip(): return False - _, _, err_code = utils.execute(['git', 'cat-file', '-e', commit], - self.repo_dir) + _, _, err_code = self.git(['cat-file', '-e', commit]) return not err_code + def commit_date(self, commit): + """Get the date of a commit. + + Args: + commit: The commit hash. + + Returns: + A datetime representing the date of the commit. + """ + out, _, _ = self.git(['show', '-s', '--format=%ct', commit], + check_result=True) + return datetime.datetime.fromtimestamp(int(out)) + def get_git_diff(self): """Gets a list of files that have changed from the repo head. @@ -103,8 +93,7 @@ class RepoManager: A list of changed file paths or None on Error. """ self.fetch_unshallow() - out, err_msg, err_code = utils.execute( - ['git', 'diff', '--name-only', 'origin...'], self.repo_dir) + out, err_msg, err_code = self.git(['diff', '--name-only', 'origin...']) if err_code: logging.error('Git diff failed with error message %s.', err_msg) return None @@ -119,9 +108,7 @@ class RepoManager: Returns: The current active commit SHA. """ - out, _, _ = utils.execute(['git', 'rev-parse', 'HEAD'], - self.repo_dir, - check_result=True) + out, _, _ = self.git(['rev-parse', 'HEAD'], check_result=True) return out.strip('\n') def get_commit_list(self, newest_commit, oldest_commit=None): @@ -151,8 +138,7 @@ class RepoManager: else: commit_range = newest_commit - out, _, err_code = utils.execute(['git', 'rev-list', commit_range], - self.repo_dir) + out, _, err_code = self.git(['rev-list', commit_range]) commits = out.split('\n') commits = [commit for commit in commits if commit] if err_code or not commits: @@ -168,9 +154,7 @@ class RepoManager: """Gets the current git repository history.""" shallow_file = os.path.join(self.repo_dir, '.git', 'shallow') if os.path.exists(shallow_file): - utils.execute(['git', 'fetch', '--unshallow'], - self.repo_dir, - check_result=True) + self.git(['fetch', '--unshallow'], check_result=True) def checkout_pr(self, pr_ref): """Checks out a remote pull request. @@ -179,12 +163,8 @@ class RepoManager: pr_ref: The pull request reference to be checked out. """ self.fetch_unshallow() - utils.execute(['git', 'fetch', 'origin', pr_ref], - self.repo_dir, - check_result=True) - utils.execute(['git', 'checkout', '-f', 'FETCH_HEAD'], - self.repo_dir, - check_result=True) + self.git(['fetch', 'origin', pr_ref], check_result=True) + self.git(['checkout', '-f', 'FETCH_HEAD'], check_result=True) def checkout_commit(self, commit, clean=True): """Checks out a specific commit from the repo. @@ -199,11 +179,9 @@ class RepoManager: self.fetch_unshallow() if not self.commit_exists(commit): raise ValueError('Commit %s does not exist in current branch' % commit) - utils.execute(['git', 'checkout', '-f', commit], - self.repo_dir, - check_result=True) + self.git(['checkout', '-f', commit], check_result=True) if clean: - utils.execute(['git', 'clean', '-fxd'], self.repo_dir, check_result=True) + self.git(['clean', '-fxd'], check_result=True) if self.get_current_commit() != commit: raise RuntimeError('Error checking out commit %s' % commit) @@ -211,3 +189,48 @@ class RepoManager: """Attempts to remove the git repo. """ if os.path.isdir(self.repo_dir): shutil.rmtree(self.repo_dir) + + +class RepoManager(BaseRepoManager): + """Class to manage git repos from python. + + Attributes: + repo_url: The location of the git repo. + base_dir: The location of where the repo clone is stored locally. + repo_name: The name of the GitHub project. + repo_dir: The location of the main repo. + """ + + def __init__(self, repo_url, base_dir, repo_name=None): + """Constructs a repo manager class. + + Args: + repo_url: The github url needed to clone. + base_dir: The full file-path where the git repo is located. + repo_name: The name of the directory the repo is cloned to. + """ + self.repo_url = repo_url + self.base_dir = base_dir + if repo_name: + self.repo_name = repo_name + else: + self.repo_name = os.path.basename(self.repo_url).replace('.git', '') + repo_dir = os.path.join(self.base_dir, self.repo_name) + super(RepoManager, self).__init__(repo_dir) + + if not os.path.exists(self.repo_dir): + self._clone() + + def _clone(self): + """Creates a clone of the repo in the specified directory. + + Raises: + ValueError: when the repo is not able to be cloned. + """ + if not os.path.exists(self.base_dir): + os.makedirs(self.base_dir) + self.remove_repo() + out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name], + location=self.base_dir) + if not self._is_git_repo(): + raise ValueError('%s is not a git repo' % self.repo_url) |