Bisector: Be a bit smarter about picking which OSS-Fuzz commit to build with. (#3665)

When the build fails against HEAD OSS-Fuzz, we find the date of the
commit for the project, and use the latest revision of OSS-Fuzz before
it to rebuild the project builder container.

Subsequent runs will use the last built container, and if that fails
that will again find the closest revision of OSS-Fuzz.

Also factor BaseRepoManager out of RepoManager to provide a generic repo
manager class for dealing with existing checkouts (which don't need a
clone).

1 files changed, 86 insertions, 63 deletions

diff --git a/infra/repo_manager.py b/infra/repo_manager.py
index 71c5ba00..238cb3e7 100644
--- a/infra/repo_manager.py
+++ b/infra/repo_manager.py
@@ -21,6 +21,7 @@ a python API and manage the current state of the git repo.
     r_man =  RepoManager('https://github.com/google/oss-fuzz.git')
     r_man.checkout('5668cc422c2c92d38a370545d3591039fb5bb8d4')
 """
+import datetime
 import logging
 import os
 import shutil
@@ -28,48 +29,11 @@ import shutil
 import utils
 
 
-class RepoManager:
-  """Class to manage git repos from python.
-
-  Attributes:
-    repo_url: The location of the git repo.
-    base_dir: The location of where the repo clone is stored locally.
-    repo_name: The name of the GitHub project.
-    repo_dir: The location of the main repo.
-  """
-
-  def __init__(self, repo_url, base_dir, repo_name=None):
-    """Constructs a repo manager class.
+class BaseRepoManager:
+  """Base repo manager."""
 
-    Args:
-      repo_url: The github url needed to clone.
-      base_dir: The full file-path where the git repo is located.
-      repo_name: The name of the directory the repo is cloned to.
-    """
-    self.repo_url = repo_url
-    self.base_dir = base_dir
-    if repo_name:
-      self.repo_name = repo_name
-    else:
-      self.repo_name = os.path.basename(self.repo_url).replace('.git', '')
-    self.repo_dir = os.path.join(self.base_dir, self.repo_name)
-
-    if not os.path.exists(self.repo_dir):
-      self._clone()
-
-  def _clone(self):
-    """Creates a clone of the repo in the specified directory.
-
-      Raises:
-        ValueError: when the repo is not able to be cloned.
-    """
-    if not os.path.exists(self.base_dir):
-      os.makedirs(self.base_dir)
-    self.remove_repo()
-    out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name],
-                              location=self.base_dir)
-    if not self._is_git_repo():
-      raise ValueError('%s is not a git repo' % self.repo_url)
+  def __init__(self, repo_dir):
+    self.repo_dir = repo_dir
 
   def _is_git_repo(self):
     """Test if the current repo dir is a git repo or not.
@@ -80,6 +44,20 @@ class RepoManager:
     git_path = os.path.join(self.repo_dir, '.git')
     return os.path.isdir(git_path)
 
+  def git(self, cmd, check_result=False):
+    """Run a git command.
+
+    Args:
+      command: The git command as a list to be run.
+      check_result: Should an exception be thrown on failed command.
+
+    Returns:
+      stdout, stderr, error code.
+    """
+    return utils.execute(['git'] + cmd,
+                         location=self.repo_dir,
+                         check_result=check_result)
+
   def commit_exists(self, commit):
     """Checks to see if a commit exists in the project repo.
 
@@ -92,10 +70,22 @@ class RepoManager:
     if not commit.rstrip():
       return False
 
-    _, _, err_code = utils.execute(['git', 'cat-file', '-e', commit],
-                                   self.repo_dir)
+    _, _, err_code = self.git(['cat-file', '-e', commit])
     return not err_code
 
+  def commit_date(self, commit):
+    """Get the date of a commit.
+
+    Args:
+      commit: The commit hash.
+
+    Returns:
+      A datetime representing the date of the commit.
+    """
+    out, _, _ = self.git(['show', '-s', '--format=%ct', commit],
+                         check_result=True)
+    return datetime.datetime.fromtimestamp(int(out))
+
   def get_git_diff(self):
     """Gets a list of files that have changed from the repo head.
 
@@ -103,8 +93,7 @@ class RepoManager:
       A list of changed file paths or None on Error.
     """
     self.fetch_unshallow()
-    out, err_msg, err_code = utils.execute(
-        ['git', 'diff', '--name-only', 'origin...'], self.repo_dir)
+    out, err_msg, err_code = self.git(['diff', '--name-only', 'origin...'])
     if err_code:
       logging.error('Git diff failed with error message %s.', err_msg)
       return None
@@ -119,9 +108,7 @@ class RepoManager:
     Returns:
       The current active commit SHA.
     """
-    out, _, _ = utils.execute(['git', 'rev-parse', 'HEAD'],
-                              self.repo_dir,
-                              check_result=True)
+    out, _, _ = self.git(['rev-parse', 'HEAD'], check_result=True)
     return out.strip('\n')
 
   def get_commit_list(self, newest_commit, oldest_commit=None):
@@ -151,8 +138,7 @@ class RepoManager:
     else:
       commit_range = newest_commit
 
-    out, _, err_code = utils.execute(['git', 'rev-list', commit_range],
-                                     self.repo_dir)
+    out, _, err_code = self.git(['rev-list', commit_range])
     commits = out.split('\n')
     commits = [commit for commit in commits if commit]
     if err_code or not commits:
@@ -168,9 +154,7 @@ class RepoManager:
     """Gets the current git repository history."""
     shallow_file = os.path.join(self.repo_dir, '.git', 'shallow')
     if os.path.exists(shallow_file):
-      utils.execute(['git', 'fetch', '--unshallow'],
-                    self.repo_dir,
-                    check_result=True)
+      self.git(['fetch', '--unshallow'], check_result=True)
 
   def checkout_pr(self, pr_ref):
     """Checks out a remote pull request.
@@ -179,12 +163,8 @@ class RepoManager:
       pr_ref: The pull request reference to be checked out.
     """
     self.fetch_unshallow()
-    utils.execute(['git', 'fetch', 'origin', pr_ref],
-                  self.repo_dir,
-                  check_result=True)
-    utils.execute(['git', 'checkout', '-f', 'FETCH_HEAD'],
-                  self.repo_dir,
-                  check_result=True)
+    self.git(['fetch', 'origin', pr_ref], check_result=True)
+    self.git(['checkout', '-f', 'FETCH_HEAD'], check_result=True)
 
   def checkout_commit(self, commit, clean=True):
     """Checks out a specific commit from the repo.
@@ -199,11 +179,9 @@ class RepoManager:
     self.fetch_unshallow()
     if not self.commit_exists(commit):
       raise ValueError('Commit %s does not exist in current branch' % commit)
-    utils.execute(['git', 'checkout', '-f', commit],
-                  self.repo_dir,
-                  check_result=True)
+    self.git(['checkout', '-f', commit], check_result=True)
     if clean:
-      utils.execute(['git', 'clean', '-fxd'], self.repo_dir, check_result=True)
+      self.git(['clean', '-fxd'], check_result=True)
     if self.get_current_commit() != commit:
       raise RuntimeError('Error checking out commit %s' % commit)
 
@@ -211,3 +189,48 @@ class RepoManager:
     """Attempts to remove the git repo. """
     if os.path.isdir(self.repo_dir):
       shutil.rmtree(self.repo_dir)
+
+
+class RepoManager(BaseRepoManager):
+  """Class to manage git repos from python.
+
+  Attributes:
+    repo_url: The location of the git repo.
+    base_dir: The location of where the repo clone is stored locally.
+    repo_name: The name of the GitHub project.
+    repo_dir: The location of the main repo.
+  """
+
+  def __init__(self, repo_url, base_dir, repo_name=None):
+    """Constructs a repo manager class.
+
+    Args:
+      repo_url: The github url needed to clone.
+      base_dir: The full file-path where the git repo is located.
+      repo_name: The name of the directory the repo is cloned to.
+    """
+    self.repo_url = repo_url
+    self.base_dir = base_dir
+    if repo_name:
+      self.repo_name = repo_name
+    else:
+      self.repo_name = os.path.basename(self.repo_url).replace('.git', '')
+    repo_dir = os.path.join(self.base_dir, self.repo_name)
+    super(RepoManager, self).__init__(repo_dir)
+
+    if not os.path.exists(self.repo_dir):
+      self._clone()
+
+  def _clone(self):
+    """Creates a clone of the repo in the specified directory.
+
+      Raises:
+        ValueError: when the repo is not able to be cloned.
+    """
+    if not os.path.exists(self.base_dir):
+      os.makedirs(self.base_dir)
+    self.remove_repo()
+    out, _, _ = utils.execute(['git', 'clone', self.repo_url, self.repo_name],
+                              location=self.base_dir)
+    if not self._is_git_repo():
+      raise ValueError('%s is not a git repo' % self.repo_url)