rebaseline_server: serve JSON from static file, rather than generating it live

BUG=skia:1455, skia:2230
NOTREECHECKS=True
NOTRY=True
R=borenet@google.com

Author: epoger@google.com

Review URL: https://codereview.chromium.org/197213033

git-svn-id: http://skia.googlecode.com/svn/trunk@13876 2bbb7eff-a529-9590-31e7-b0007b416f81

6 files changed, 68 insertions, 75 deletions

diff --git a/gm/rebaseline_server/imagepairset.py b/gm/rebaseline_server/imagepairset.py
index 48c3064718..04aea90342 100644
--- a/gm/rebaseline_server/imagepairset.py
+++ b/gm/rebaseline_server/imagepairset.py
@@ -28,7 +28,6 @@ KEY__IMAGESETS__SET__IMAGE_B = 'imageB'
 KEY__IMAGESETS__SET__WHITEDIFFS = 'whiteDiffs'
 
 DEFAULT_DESCRIPTIONS = ('setA', 'setB')
-DIFF_BASE_URL = '/static/generated-images'
 
 
 class ImagePairSet(object):
@@ -40,9 +39,10 @@ class ImagePairSet(object):
   - or any other pairwise set of images.
   """
 
-  def __init__(self, descriptions=None):
+  def __init__(self, diff_base_url, descriptions=None):
     """
     Args:
+      diff_base_url: base URL indicating where diff images can be loaded from
       descriptions: a (string, string) tuple describing the two image sets.
           If not specified, DEFAULT_DESCRIPTIONS will be used.
     """
@@ -52,7 +52,7 @@ class ImagePairSet(object):
                                      #                -> instances_per_value
     self._image_pair_dicts = []
     self._image_base_url = None
-    self._diff_base_url = DIFF_BASE_URL
+    self._diff_base_url = diff_base_url
 
   def add_image_pair(self, image_pair):
     """Adds an ImagePair; this may be repeated any number of times."""
diff --git a/gm/rebaseline_server/imagepairset_test.py b/gm/rebaseline_server/imagepairset_test.py
index 2fc6d72bd4..c046ec7c7c 100755
--- a/gm/rebaseline_server/imagepairset_test.py
+++ b/gm/rebaseline_server/imagepairset_test.py
@@ -20,6 +20,7 @@ import imagepairset
 
 BASE_URL_1 = 'http://base/url/1'
 BASE_URL_2 = 'http://base/url/2'
+DIFF_BASE_URL = 'http://diff/base/url'
 IMAGEPAIR_1_AS_DICT = {
     imagepair.KEY__EXTRA_COLUMN_VALUES: {
         'builder': 'MyBuilder',
@@ -116,18 +117,19 @@ class ImagePairSetTest(unittest.TestCase):
                 'description': SET_B_DESCRIPTION,
             },
             'diffs': {
-                'baseUrl': '/static/generated-images/diffs',
+                'baseUrl': DIFF_BASE_URL + '/diffs',
                 'description': 'color difference per channel',
             },
             'whiteDiffs': {
-                'baseUrl': '/static/generated-images/whitediffs',
+                'baseUrl': DIFF_BASE_URL + '/whitediffs',
                 'description': 'differing pixels in white',
             },
         },
     }
 
     image_pair_set = imagepairset.ImagePairSet(
-        descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION))
+        descriptions=(SET_A_DESCRIPTION, SET_B_DESCRIPTION),
+        diff_base_url=DIFF_BASE_URL)
     for image_pair in image_pairs:
       image_pair_set.add_image_pair(image_pair)
     # The 'builder' column header uses the default settings,
@@ -144,7 +146,8 @@ class ImagePairSetTest(unittest.TestCase):
 
   def test_mismatched_base_url(self):
     """Confirms that mismatched base_urls will cause an exception."""
-    image_pair_set = imagepairset.ImagePairSet()
+    image_pair_set = imagepairset.ImagePairSet(
+        diff_base_url=DIFF_BASE_URL)
     image_pair_set.add_image_pair(
         MockImagePair(base_url=BASE_URL_1, dict_to_return=IMAGEPAIR_1_AS_DICT))
     image_pair_set.add_image_pair(
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index 0a53136a25..66105a4921 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -21,6 +21,10 @@ import time
 
 # Imports from within Skia
 #
+# TODO(epoger): Once we move the create_filepath_url() function out of
+# download_actuals into a shared utility module, we won't need to import
+# download_actuals anymore.
+#
 # We need to add the 'gm' directory, so that we can import gm_json.py within
 # that directory.  That script allows us to parse the actual-results.json file
 # written out by the GM tool.
@@ -31,6 +35,7 @@ GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
+import download_actuals
 import gm_json
 import imagediffdb
 import imagepair
@@ -76,8 +81,8 @@ IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
 DEFAULT_ACTUALS_DIR = '.gm-actuals'
 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
-DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(PARENT_DIRECTORY, 'static',
-                                             'generated-images')
+DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
+    PARENT_DIRECTORY, '.generated-images')
 
 
 class Results(object):
@@ -92,16 +97,23 @@ class Results(object):
 
   def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR,
                expected_root=DEFAULT_EXPECTATIONS_DIR,
-               generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT):
+               generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT,
+               diff_base_url=None):
     """
     Args:
       actuals_root: root directory containing all actual-results.json files
       expected_root: root directory containing all expected-results.json files
       generated_images_root: directory within which to create all pixel diffs;
           if this directory does not yet exist, it will be created
+      diff_base_url: base URL within which the client should look for diff
+          images; if not specified, defaults to a "file:///" URL representation
+          of generated_images_root
     """
     time_start = int(time.time())
     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
+    self._diff_base_url = (
+        diff_base_url or
+        download_actuals.create_filepath_url(generated_images_root))
     self._actuals_root = actuals_root
     self._expected_root = expected_root
     self._load_actual_and_expected()
@@ -343,8 +355,12 @@ class Results(object):
                  self._expected_root)
     expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
 
-    all_image_pairs = imagepairset.ImagePairSet(IMAGEPAIR_SET_DESCRIPTIONS)
-    failing_image_pairs = imagepairset.ImagePairSet(IMAGEPAIR_SET_DESCRIPTIONS)
+    all_image_pairs = imagepairset.ImagePairSet(
+        descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
+        diff_base_url=self._diff_base_url)
+    failing_image_pairs = imagepairset.ImagePairSet(
+        descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
+        diff_base_url=self._diff_base_url)
 
     all_image_pairs.ensure_extra_column_values_in_summary(
         column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[
diff --git a/gm/rebaseline_server/results_test.py b/gm/rebaseline_server/results_test.py
index 7cbddefe23..6d7b05fcd3 100755
--- a/gm/rebaseline_server/results_test.py
+++ b/gm/rebaseline_server/results_test.py
@@ -34,7 +34,8 @@ class ResultsTest(base_unittest.TestCase):
     results_obj = results.Results(
         actuals_root=os.path.join(self._input_dir, 'gm-actuals'),
         expected_root=os.path.join(self._input_dir, 'gm-expectations'),
-        generated_images_root=self._temp_dir)
+        generated_images_root=self._temp_dir,
+        diff_base_url='/static/generated-images')
     results_obj.get_timestamp = mock_get_timestamp
     gm_json.WriteToFile(
         results_obj.get_packaged_results_of_type(
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index 396f42d8e4..1ca5757b1a 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -28,16 +28,20 @@ import urlparse
 
 # Imports from within Skia
 #
-# We need to add the 'tools' directory, so that we can import svn.py within
-# that directory.
-# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
+# We need to add the 'tools' directory for svn.py, and the 'gm' directory for
+# gm_json.py .
+# Make sure that these dirs are in the PYTHONPATH, but add them at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
-TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY))
+GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
+TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
 TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
 if TOOLS_DIRECTORY not in sys.path:
   sys.path.append(TOOLS_DIRECTORY)
 import svn
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
 
 # Imports from local dir
 #
@@ -71,6 +75,11 @@ DEFAULT_ACTUALS_REPO_REVISION = 'HEAD'
 DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
 DEFAULT_PORT = 8888
 
+# Directory within which the server will serve out static files.
+STATIC_CONTENTS_DIR = os.path.realpath(os.path.join(PARENT_DIRECTORY, 'static'))
+GENERATED_IMAGES_DIR = os.path.join(STATIC_CONTENTS_DIR, 'generated-images')
+GENERATED_JSON_DIR = os.path.join(STATIC_CONTENTS_DIR, 'generated-json')
+
 # How often (in seconds) clients should reload while waiting for initial
 # results to load.
 RELOAD_INTERVAL_UNTIL_READY = 10
@@ -232,7 +241,20 @@ class Server(object):
             results_mod.DEFAULT_EXPECTATIONS_DIR)
         _run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
 
-      self._results = results_mod.Results(actuals_root=self._actuals_dir)
+      new_results = results_mod.Results(
+          actuals_root=self._actuals_dir,
+          generated_images_root=GENERATED_IMAGES_DIR,
+          diff_base_url=os.path.relpath(
+              GENERATED_IMAGES_DIR, GENERATED_JSON_DIR))
+      if not os.path.isdir(GENERATED_JSON_DIR):
+        os.makedirs(GENERATED_JSON_DIR)
+      for summary_type in [results_mod.KEY__HEADER__RESULTS_ALL,
+                           results_mod.KEY__HEADER__RESULTS_FAILURES]:
+        gm_json.WriteToFile(
+            new_results.get_packaged_results_of_type(results_type=summary_type),
+            os.path.join(GENERATED_JSON_DIR, '%s.json' % summary_type))
+
+      self._results = new_results
 
   def _result_loader(self, reload_seconds=0):
     """ Call self.update_results(), either once or periodically.
@@ -296,7 +318,6 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       normpath = posixpath.normpath(self.path)
       (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
       dispatchers = {
-          'results': self.do_GET_results,
           'static': self.do_GET_static,
       }
       dispatcher = dispatchers[dispatcher_name]
@@ -305,41 +326,6 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       self.send_error(404)
       raise
 
-  def do_GET_results(self, results_type):
-    """ Handle a GET request for GM results.
-
-    Args:
-      results_type: string indicating which set of results to return;
-            must be one of the results_mod.RESULTS_* constants
-    """
-    logging.debug('do_GET_results: sending results of type "%s"' % results_type)
-    # Since we must make multiple calls to the Results object, grab a
-    # reference to it in case it is updated to point at a new Results
-    # object within another thread.
-    #
-    # TODO(epoger): Rather than using a global variable for the handler
-    # to refer to the Server object, make Server a subclass of
-    # HTTPServer, and then it could be available to the handler via
-    # the handler's .server instance variable.
-    results_obj = _SERVER.results
-    if results_obj:
-      response_dict = results_obj.get_packaged_results_of_type(
-          results_type=results_type, reload_seconds=_SERVER.reload_seconds,
-          is_editable=_SERVER.is_editable, is_exported=_SERVER.is_exported)
-    else:
-      now = int(time.time())
-      response_dict = {
-          results_mod.KEY__HEADER: {
-              results_mod.KEY__HEADER__SCHEMA_VERSION: (
-                  results_mod.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
-              results_mod.KEY__HEADER__IS_STILL_LOADING: True,
-              results_mod.KEY__HEADER__TIME_UPDATED: now,
-              results_mod.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
-                  now + RELOAD_INTERVAL_UNTIL_READY),
-          },
-      }
-    self.send_json_dict(response_dict)
-
   def do_GET_static(self, path):
     """ Handle a GET request for a file under the 'static' directory.
     Only allow serving of files within the 'static' directory that is a
@@ -352,14 +338,13 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     path = urlparse.urlparse(path).path
 
     logging.debug('do_GET_static: sending file "%s"' % path)
-    static_dir = os.path.realpath(os.path.join(PARENT_DIRECTORY, 'static'))
-    full_path = os.path.realpath(os.path.join(static_dir, path))
-    if full_path.startswith(static_dir):
+    full_path = os.path.realpath(os.path.join(STATIC_CONTENTS_DIR, path))
+    if full_path.startswith(STATIC_CONTENTS_DIR):
       self.send_file(full_path)
     else:
       logging.error(
           'Attempted do_GET_static() of path [%s] outside of static dir [%s]'
-          % (full_path, static_dir))
+          % (full_path, STATIC_CONTENTS_DIR))
       self.send_error(404)
 
   def do_POST(self):
@@ -471,18 +456,6 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
     else:
       self.send_error(404)
 
-  def send_json_dict(self, json_dict):
-    """ Send the contents of this dictionary in JSON format, with a JSON
-        mimetype.
-
-    Args:
-      json_dict: dictionary to send
-    """
-    self.send_response(200)
-    self.send_header('Content-type', 'application/json')
-    self.end_headers()
-    json.dump(json_dict, self.wfile)
-
 
 def main():
   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
diff --git a/gm/rebaseline_server/static/index.html b/gm/rebaseline_server/static/index.html
index df9bb0e757..419b7b59d8 100644
--- a/gm/rebaseline_server/static/index.html
+++ b/gm/rebaseline_server/static/index.html
@@ -9,16 +9,16 @@
     Here are links to the result pages:
     <ul>
       <li>
-        <a href="/static/view.html#/view.html?resultsToLoad=/results/failures">
-          failures only
+        <a href="/static/view.html#/view.html?resultsToLoad=generated-json/failures.json">
+          failures
         </a>
-        (loads faster)
+        (includes failed, failure-ignored, and no-comparison)
       </li>
       <li>
-        <a href="/static/view.html#/view.html?resultsToLoad=/results/all">
+        <a href="/static/view.html#/view.html?resultsToLoad=generated-json/all.json">
           all results
         </a>
-        (includes successful test results)
+        (includes successful test results also, but takes longer to load)
       </li>
     </ul>
     Instructions, roadmap, etc. are at