rebaseline_server: allow user to specify which builders to process

BUG=skia:1543,skia:1915
NOTRY=True
R=borenet@google.com

Author: epoger@google.com

Review URL: https://codereview.chromium.org/232103002

git-svn-id: http://skia.googlecode.com/svn/trunk@14131 2bbb7eff-a529-9590-31e7-b0007b416f81

6 files changed, 108 insertions, 37 deletions

diff --git a/gm/rebaseline_server/compare_configs.py b/gm/rebaseline_server/compare_configs.py
index 8f92551559..ba256cab60 100755
--- a/gm/rebaseline_server/compare_configs.py
+++ b/gm/rebaseline_server/compare_configs.py
@@ -47,12 +47,12 @@ class ConfigComparisons(results.BaseComparisons):
   """Loads results from two different configurations into an ImagePairSet.
 
   Loads actual and expected results from all builders, except for those skipped
-  by BaseComparisons._ignore_builder().
+  by _ignore_builder().
   """
 
   def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
                generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
-               diff_base_url=None):
+               diff_base_url=None, builder_regex_list=None):
     """
     Args:
       configs: (string, string) tuple; pair of configs to compare
@@ -62,8 +62,12 @@ class ConfigComparisons(results.BaseComparisons):
       diff_base_url: base URL within which the client should look for diff
           images; if not specified, defaults to a "file:///" URL representation
           of generated_images_root
+      builder_regex_list: List of regular expressions specifying which builders
+          we will process. If None, process all builders.
     """
     time_start = int(time.time())
+    if builder_regex_list != None:
+      self.set_match_builders_pattern_list(builder_regex_list)
     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
     self._diff_base_url = (
         diff_base_url or
@@ -84,8 +88,7 @@ class ConfigComparisons(results.BaseComparisons):
     """
     logging.info('Reading actual-results JSON files from %s...' %
                  self._actuals_root)
-    actual_builder_dicts = ConfigComparisons._read_dicts_from_root(
-        self._actuals_root)
+    actual_builder_dicts = self._read_dicts_from_root(self._actuals_root)
     configA, configB = configs
     logging.info('Comparing configs %s and %s...' % (configA, configB))
 
diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py
index 14b1fb1d80..80a42e51f9 100755
--- a/gm/rebaseline_server/compare_rendered_pictures.py
+++ b/gm/rebaseline_server/compare_rendered_pictures.py
@@ -97,9 +97,9 @@ class RenderedPicturesComparisons(results.BaseComparisons):
         'Reading actual-results JSON files from %s subdirs within %s...' % (
             subdirs, actuals_root))
     subdirA, subdirB = subdirs
-    subdirA_builder_dicts = results.BaseComparisons._read_dicts_from_root(
+    subdirA_builder_dicts = self._read_dicts_from_root(
         os.path.join(actuals_root, subdirA))
-    subdirB_builder_dicts = results.BaseComparisons._read_dicts_from_root(
+    subdirB_builder_dicts = self._read_dicts_from_root(
         os.path.join(actuals_root, subdirB))
     logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
 
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index c8510d6595..2389b61dad 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -65,7 +65,7 @@ class ExpectationComparisons(results.BaseComparisons):
   def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
                expected_root=DEFAULT_EXPECTATIONS_DIR,
                generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
-               diff_base_url=None):
+               diff_base_url=None, builder_regex_list=None):
     """
     Args:
       actuals_root: root directory containing all actual-results.json files
@@ -75,8 +75,12 @@ class ExpectationComparisons(results.BaseComparisons):
       diff_base_url: base URL within which the client should look for diff
           images; if not specified, defaults to a "file:///" URL representation
           of generated_images_root
+      builder_regex_list: List of regular expressions specifying which builders
+          we will process. If None, process all builders.
     """
     time_start = int(time.time())
+    if builder_regex_list != None:
+      self.set_match_builders_pattern_list(builder_regex_list)
     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
     self._diff_base_url = (
         diff_base_url or
@@ -117,8 +121,7 @@ class ExpectationComparisons(results.BaseComparisons):
          ]
 
     """
-    expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
-        self._expected_root)
+    expected_builder_dicts = self._read_dicts_from_root(self._expected_root)
     for mod in modifications:
       image_name = results.IMAGE_FILENAME_FORMATTER % (
           mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
@@ -174,8 +177,6 @@ class ExpectationComparisons(results.BaseComparisons):
     for dirpath, dirnames, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
-        if ExpectationComparisons._ignore_builder(builder):
-          continue
         per_builder_dict = meta_dict.get(builder)
         if per_builder_dict is not None:
           fullpath = os.path.join(dirpath, matching_filename)
@@ -199,12 +200,10 @@ class ExpectationComparisons(results.BaseComparisons):
     """
     logging.info('Reading actual-results JSON files from %s...' %
                  self._actuals_root)
-    actual_builder_dicts = ExpectationComparisons._read_dicts_from_root(
-        self._actuals_root)
+    actual_builder_dicts = self._read_dicts_from_root(self._actuals_root)
     logging.info('Reading expected-results JSON files from %s...' %
                  self._expected_root)
-    expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
-        self._expected_root)
+    expected_builder_dicts = self._read_dicts_from_root(self._expected_root)
 
     all_image_pairs = imagepairset.ImagePairSet(
         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index 49e32519a7..255dfa31e2 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -59,26 +59,24 @@ KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
 IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
 
-# Ignore expectations/actuals for builders matching any of these patterns.
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
+    PARENT_DIRECTORY, '.generated-images')
+
+# Define the default set of builders we will process expectations/actuals for.
 # This allows us to ignore builders for which we don't maintain expectations
 # (trybots, Valgrind, ASAN, TSAN), and avoid problems like
 # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
 # produces error when trying to add baselines for ASAN/TSAN builders')
-SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [
-    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']]
-
-DEFAULT_ACTUALS_DIR = '.gm-actuals'
-DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
-    PARENT_DIRECTORY, '.generated-images')
+DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
+DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
+    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']
 
 
 class BaseComparisons(object):
   """Base class for generating summary of comparisons between two image sets.
   """
 
-  def __init__(self):
-    raise NotImplementedError('cannot instantiate the abstract base class')
-
   def get_results_of_type(self, results_type):
     """Return results of some/all tests (depending on 'results_type' parameter).
 
@@ -138,9 +136,45 @@ class BaseComparisons(object):
     """
     return self._timestamp
 
-  @staticmethod
-  def _ignore_builder(builder):
-    """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST.
+  _match_builders_pattern_list = [
+      re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
+  _skip_builders_pattern_list = [
+      re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]
+
+  def set_match_builders_pattern_list(self, pattern_list):
+    """Override the default set of builders we should process.
+
+    The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .
+
+    Note that skip_builders_pattern_list overrides this; regardless of whether a
+    builder is in the "match" list, if it's in the "skip" list, we will skip it.
+
+    Args:
+      pattern_list: list of regex patterns; process builders that match any
+          entry within this list
+    """
+    if pattern_list == None:
+      pattern_list = []
+    self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]
+
+  def set_skip_builders_pattern_list(self, pattern_list):
+    """Override the default set of builders we should skip while processing.
+
+    The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .
+
+    This overrides match_builders_pattern_list; regardless of whether a
+    builder is in the "match" list, if it's in the "skip" list, we will skip it.
+
+    Args:
+      pattern_list: list of regex patterns; skip builders that match any
+          entry within this list
+    """
+    if pattern_list == None:
+      pattern_list = []
+    self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]
+
+  def _ignore_builder(self, builder):
+    """Returns True if we should skip processing this builder.
 
     Args:
       builder: name of this builder, as a string
@@ -148,13 +182,15 @@ class BaseComparisons(object):
     Returns:
       True if we should ignore expectations and actuals for this builder.
     """
-    for pattern in SKIP_BUILDERS_PATTERN_LIST:
+    for pattern in self._skip_builders_pattern_list:
       if pattern.match(builder):
         return True
-    return False
+    for pattern in self._match_builders_pattern_list:
+      if pattern.match(builder):
+        return False
+    return True
 
-  @staticmethod
-  def _read_dicts_from_root(root, pattern='*.json'):
+  def _read_dicts_from_root(self, root, pattern='*.json'):
     """Read all JSON dictionaries within a directory tree.
 
     Args:
@@ -174,7 +210,7 @@ class BaseComparisons(object):
     for dirpath, dirnames, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
-        if BaseComparisons._ignore_builder(builder):
+        if self._ignore_builder(builder):
           continue
         fullpath = os.path.join(dirpath, matching_filename)
         meta_dict[builder] = gm_json.LoadFromFile(fullpath)
diff --git a/gm/rebaseline_server/results_test.py b/gm/rebaseline_server/results_test.py
index a2f4073dcf..f22e833fe3 100755
--- a/gm/rebaseline_server/results_test.py
+++ b/gm/rebaseline_server/results_test.py
@@ -17,6 +17,29 @@ import results
 
 class ResultsTest(base_unittest.TestCase):
 
+  def test_ignore_builder(self):
+    """Test _ignore_builder()."""
+    results_obj = results.BaseComparisons()
+    self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), True)
+    self.assertEqual(results_obj._ignore_builder('Something-Trybot'), True)
+    self.assertEqual(results_obj._ignore_builder(
+        'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), False)
+    results_obj.set_skip_builders_pattern_list(['.*TSAN.*', '.*GTX660.*'])
+    self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), True)
+    self.assertEqual(results_obj._ignore_builder('Something-Trybot'), False)
+    self.assertEqual(results_obj._ignore_builder(
+        'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), True)
+    results_obj.set_skip_builders_pattern_list(None)
+    self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), False)
+    self.assertEqual(results_obj._ignore_builder('Something-Trybot'), False)
+    self.assertEqual(results_obj._ignore_builder(
+        'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), False)
+    results_obj.set_match_builders_pattern_list(['.*TSAN'])
+    self.assertEqual(results_obj._ignore_builder('SomethingTSAN'), False)
+    self.assertEqual(results_obj._ignore_builder('Something-Trybot'), True)
+    self.assertEqual(results_obj._ignore_builder(
+        'Test-Ubuntu12-ShuttleA-GTX660-x86-Release'), True)
+
   def test_combine_subdicts_typical(self):
     """Test combine_subdicts() with no merge conflicts. """
     input_dict = {
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index 04620f51ca..73cfbef21e 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -216,7 +216,7 @@ class Server(object):
                actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION,
                actuals_repo_url=DEFAULT_ACTUALS_REPO_URL,
                port=DEFAULT_PORT, export=False, editable=True,
-               reload_seconds=0, config_pairs=None):
+               reload_seconds=0, config_pairs=None, builder_regex_list=None):
     """
     Args:
       actuals_dir: directory under which we will check out the latest actual
@@ -233,6 +233,8 @@ class Server(object):
       config_pairs: List of (string, string) tuples; for each tuple, compare
           actual results of these two configs.  If None or empty,
           don't compare configs at all.
+      builder_regex_list: List of regular expressions specifying which builders
+          we will process. If None, process all builders.
     """
     self._actuals_dir = actuals_dir
     self._actuals_repo_revision = actuals_repo_revision
@@ -242,6 +244,7 @@ class Server(object):
     self._editable = editable
     self._reload_seconds = reload_seconds
     self._config_pairs = config_pairs or []
+    self._builder_regex_list = builder_regex_list
     _create_index(
         file_path=os.path.join(
             PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
@@ -329,7 +332,8 @@ class Server(object):
               PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
               GENERATED_IMAGES_SUBDIR),
           diff_base_url=posixpath.join(
-              os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR))
+              os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR),
+          builder_regex_list=self._builder_regex_list)
 
       json_dir = os.path.join(
           PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_JSON_SUBDIR)
@@ -344,7 +348,8 @@ class Server(object):
                 PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
                 GENERATED_IMAGES_SUBDIR),
             diff_base_url=posixpath.join(
-                os.pardir, GENERATED_IMAGES_SUBDIR))
+                os.pardir, GENERATED_IMAGES_SUBDIR),
+            builder_regex_list=self._builder_regex_list)
         for summary_type in SUMMARY_TYPES:
           gm_json.WriteToFile(
               config_comparisons.get_packaged_results_of_type(
@@ -627,6 +632,10 @@ def main():
                           'argument in conjunction with --editable; you '
                           'probably only want to edit results at HEAD.'),
                     default=DEFAULT_ACTUALS_REPO_REVISION)
+  parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+',
+                      help=('Only process builders matching these regular '
+                            'expressions.  If unspecified, process all '
+                            'builders.'))
   parser.add_argument('--compare-configs', action='store_true',
                       help=('In addition to generating differences between '
                             'expectations and actuals, also generate '
@@ -663,7 +672,8 @@ def main():
                    actuals_repo_revision=args.actuals_revision,
                    actuals_repo_url=args.actuals_repo,
                    port=args.port, export=args.export, editable=args.editable,
-                   reload_seconds=args.reload, config_pairs=config_pairs)
+                   reload_seconds=args.reload, config_pairs=config_pairs,
+                   builder_regex_list=args.builders)
   _SERVER.run()