From ed292c0b4036c4dd84d230be64d8504623070c47 Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Tue, 21 Apr 2020 10:11:29 +1000 Subject: Bisector: Be a bit smarter about picking which OSS-Fuzz commit to build with. (#3665) When the build fails against HEAD OSS-Fuzz, we find the date of the commit for the project, and use the latest revision of OSS-Fuzz before it to rebuild the project builder container. Subsequent runs will use the last built container, and if that fails that will again find the closest revision of OSS-Fuzz. Also factor BaseRepoManager out of RepoManager to provide a generic repo manager class for dealing with existing checkouts (which don't need a clone). --- infra/repo_manager.py | 149 +++++++++++++++++++++++++++++--------------------- 1 file changed, 86 insertions(+), 63 deletions(-) (limited to 'infra/repo_manager.py') diff --git a/infra/repo_manager.py b/infra/repo_manager.py index 71c5ba00..238cb3e7 100644 --- a/infra/repo_manager.py +++ b/infra/repo_manager.py @@ -21,6 +21,7 @@ a python API and manage the current state of the git repo. r_man = RepoManager('https://github.com/google/oss-fuzz.git') r_man.checkout('5668cc422c2c92d38a370545d3591039fb5bb8d4') """ +import datetime import logging import os import shutil @@ -28,48 +29,11 @@ import shutil import utils -class RepoManager: - """Class to manage git repos from python. - - Attributes: - repo_url: The location of the git repo. - base_dir: The location of where the repo clone is stored locally. - repo_name: The name of the GitHub project. - repo_dir: The location of the main repo. - """ - - def __init__(self, repo_url, base_dir, repo_name=None): - """Constructs a repo manager class. +class BaseRepoManager: + """Base repo manager.""" - Args: - repo_url: The github url needed to clone. - base_dir: The full file-path where the git repo is located. - repo_name: The name of the directory the repo is cloned to. - """ - self.repo_url = repo_url - self.base_dir = base_dir - if repo_name: - self.repo_name = repo_name - else: - self.repo_name = os.path.basename(self.repo_url).replace('.git', '') - self.repo_dir = os.path.join(self.base_dir, self.repo_name) - - if not os.path.exists(self.repo_dir): - self._clone() - - def _clone(self): - """Creates a clone of the repo in the specified directory. - - Raises: - ValueError: when the repo is not able to be cloned. - """ - if not os.path.exists(self.base_dir): - os.makedirs(self.base_dir) - self.remove_repo() - out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name], - location=self.base_dir) - if not self._is_git_repo(): - raise ValueError('%s is not a git repo' % self.repo_url) + def __init__(self, repo_dir): + self.repo_dir = repo_dir def _is_git_repo(self): """Test if the current repo dir is a git repo or not. @@ -80,6 +44,20 @@ class RepoManager: git_path = os.path.join(self.repo_dir, '.git') return os.path.isdir(git_path) + def git(self, cmd, check_result=False): + """Run a git command. + + Args: + command: The git command as a list to be run. + check_result: Should an exception be thrown on failed command. + + Returns: + stdout, stderr, error code. + """ + return utils.execute(['git'] + cmd, + location=self.repo_dir, + check_result=check_result) + def commit_exists(self, commit): """Checks to see if a commit exists in the project repo. @@ -92,10 +70,22 @@ class RepoManager: if not commit.rstrip(): return False - _, _, err_code = utils.execute(['git', 'cat-file', '-e', commit], - self.repo_dir) + _, _, err_code = self.git(['cat-file', '-e', commit]) return not err_code + def commit_date(self, commit): + """Get the date of a commit. + + Args: + commit: The commit hash. + + Returns: + A datetime representing the date of the commit. + """ + out, _, _ = self.git(['show', '-s', '--format=%ct', commit], + check_result=True) + return datetime.datetime.fromtimestamp(int(out)) + def get_git_diff(self): """Gets a list of files that have changed from the repo head. @@ -103,8 +93,7 @@ class RepoManager: A list of changed file paths or None on Error. """ self.fetch_unshallow() - out, err_msg, err_code = utils.execute( - ['git', 'diff', '--name-only', 'origin...'], self.repo_dir) + out, err_msg, err_code = self.git(['diff', '--name-only', 'origin...']) if err_code: logging.error('Git diff failed with error message %s.', err_msg) return None @@ -119,9 +108,7 @@ class RepoManager: Returns: The current active commit SHA. """ - out, _, _ = utils.execute(['git', 'rev-parse', 'HEAD'], - self.repo_dir, - check_result=True) + out, _, _ = self.git(['rev-parse', 'HEAD'], check_result=True) return out.strip('\n') def get_commit_list(self, newest_commit, oldest_commit=None): @@ -151,8 +138,7 @@ class RepoManager: else: commit_range = newest_commit - out, _, err_code = utils.execute(['git', 'rev-list', commit_range], - self.repo_dir) + out, _, err_code = self.git(['rev-list', commit_range]) commits = out.split('\n') commits = [commit for commit in commits if commit] if err_code or not commits: @@ -168,9 +154,7 @@ class RepoManager: """Gets the current git repository history.""" shallow_file = os.path.join(self.repo_dir, '.git', 'shallow') if os.path.exists(shallow_file): - utils.execute(['git', 'fetch', '--unshallow'], - self.repo_dir, - check_result=True) + self.git(['fetch', '--unshallow'], check_result=True) def checkout_pr(self, pr_ref): """Checks out a remote pull request. @@ -179,12 +163,8 @@ class RepoManager: pr_ref: The pull request reference to be checked out. """ self.fetch_unshallow() - utils.execute(['git', 'fetch', 'origin', pr_ref], - self.repo_dir, - check_result=True) - utils.execute(['git', 'checkout', '-f', 'FETCH_HEAD'], - self.repo_dir, - check_result=True) + self.git(['fetch', 'origin', pr_ref], check_result=True) + self.git(['checkout', '-f', 'FETCH_HEAD'], check_result=True) def checkout_commit(self, commit, clean=True): """Checks out a specific commit from the repo. @@ -199,11 +179,9 @@ class RepoManager: self.fetch_unshallow() if not self.commit_exists(commit): raise ValueError('Commit %s does not exist in current branch' % commit) - utils.execute(['git', 'checkout', '-f', commit], - self.repo_dir, - check_result=True) + self.git(['checkout', '-f', commit], check_result=True) if clean: - utils.execute(['git', 'clean', '-fxd'], self.repo_dir, check_result=True) + self.git(['clean', '-fxd'], check_result=True) if self.get_current_commit() != commit: raise RuntimeError('Error checking out commit %s' % commit) @@ -211,3 +189,48 @@ class RepoManager: """Attempts to remove the git repo. """ if os.path.isdir(self.repo_dir): shutil.rmtree(self.repo_dir) + + +class RepoManager(BaseRepoManager): + """Class to manage git repos from python. + + Attributes: + repo_url: The location of the git repo. + base_dir: The location of where the repo clone is stored locally. + repo_name: The name of the GitHub project. + repo_dir: The location of the main repo. + """ + + def __init__(self, repo_url, base_dir, repo_name=None): + """Constructs a repo manager class. + + Args: + repo_url: The github url needed to clone. + base_dir: The full file-path where the git repo is located. + repo_name: The name of the directory the repo is cloned to. + """ + self.repo_url = repo_url + self.base_dir = base_dir + if repo_name: + self.repo_name = repo_name + else: + self.repo_name = os.path.basename(self.repo_url).replace('.git', '') + repo_dir = os.path.join(self.base_dir, self.repo_name) + super(RepoManager, self).__init__(repo_dir) + + if not os.path.exists(self.repo_dir): + self._clone() + + def _clone(self): + """Creates a clone of the repo in the specified directory. + + Raises: + ValueError: when the repo is not able to be cloned. + """ + if not os.path.exists(self.base_dir): + os.makedirs(self.base_dir) + self.remove_repo() + out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name], + location=self.base_dir) + if not self._is_git_repo(): + raise ValueError('%s is not a git repo' % self.repo_url) -- cgit v1.2.3