Create HTTP-based GM results viewer.

For now, it only allows VIEWING results... next, it will allow the user to
rebaseline GM results via the web interface.

R=borenet@google.com

Review URL: https://codereview.chromium.org/24274003

git-svn-id: http://skia.googlecode.com/svn/trunk@11500 2bbb7eff-a529-9590-31e7-b0007b416f81

4 files changed, 527 insertions, 0 deletions

diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
new file mode 100755
index 0000000000..0c50d26397
--- /dev/null
+++ b/gm/rebaseline_server/results.py
@@ -0,0 +1,178 @@
+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory.  That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+  """ Loads actual and expected results from all builders, supplying combined
+  reports as requested. """
+
+  def __init__(self, actuals_root, expected_root):
+    """
+    params:
+      actuals_root: root directory containing all actual-results.json files
+      expected_root: root directory containing all expected-results.json files
+    """
+    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+    self._all_results = self._Combine()
+
+  def GetAll(self):
+    """Return results of all tests, as a list in this form:
+
+       [
+         {
+           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+           "test": "bigmatrix",
+           "config": "8888",
+           "resultType": "failed",
+           "expectedHashType": "bitmap-64bitMD5",
+           "expectedHashDigest": "10894408024079689926",
+           "actualHashType": "bitmap-64bitMD5",
+           "actualHashDigest": "2409857384569",
+         },
+         ...
+       ]
+    """
+    return self._all_results
+
+  @staticmethod
+  def _GetDictsFromRoot(root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree, returning them within
+    a meta-dictionary (keyed by the builder name for each dictionary).
+
+    params:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+    """
+    meta_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        builder = os.path.basename(dirpath)
+        if builder.endswith('-Trybot'):
+          continue
+        fullpath = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+    return meta_dict
+
+  def _Combine(self):
+    """Returns a list of all tests, across all builders, based on the
+    contents of self._actual_builder_dicts and self._expected_builder_dicts .
+    Returns the list in the same form needed for GetAllResults().
+    """
+    all_tests = []
+    for builder in sorted(self._actual_builder_dicts.keys()):
+      actual_results_for_this_builder = (
+          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+      for result_type in sorted(actual_results_for_this_builder.keys()):
+        results_of_this_type = actual_results_for_this_builder[result_type]
+        if not results_of_this_type:
+          continue
+        for image_name in sorted(results_of_this_type.keys()):
+          actual_image = results_of_this_type[image_name]
+          try:
+            # TODO(epoger): assumes a single allowed digest per test
+            expected_image = (
+                self._expected_builder_dicts
+                    [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
+                    [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
+                    [0])
+          except (KeyError, TypeError):
+            # There are several cases in which we would expect to find
+            # no expectations for a given test:
+            #
+            # 1. result_type == NOCOMPARISON
+            #   There are no expectations for this test yet!
+            #
+            # 2. ignore-tests.txt
+            #   If a test has been listed in ignore-tests.txt, then its status
+            #   may show as FAILUREIGNORED even if it doesn't have any
+            #   expectations yet.
+            #
+            # 3. alternate rendering mode failures (e.g. serialized)
+            #   In cases like
+            #   https://code.google.com/p/skia/issues/detail?id=1684
+            #   ('tileimagefilter GM test failing in serialized render mode'),
+            #   the gm-actuals will list a failure for the alternate
+            #   rendering mode even though we don't have explicit expectations
+            #   for the test (the implicit expectation is that it must
+            #   render the same in all rendering modes).
+            #
+            # Don't log types 1 or 2, because they are common.
+            # Log other types, because they are rare and we should know about
+            # them, but don't throw an exception, because we need to keep our
+            # tools working in the meanwhile!
+            if result_type not in [
+                gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
+                gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
+              print 'WARNING: No expectations found for test: %s' % {
+                  'builder': builder,
+                  'image_name': image_name,
+                  'result_type': result_type,
+                  }
+            expected_image = [None, None]
+
+          # If this test was recently rebaselined, it will remain in
+          # the "failed" set of actuals until all the bots have
+          # cycled (although the expectations have indeed been set
+          # from the most recent actuals).  Treat these as successes
+          # instead of failures.
+          #
+          # TODO(epoger): Do we need to do something similar in
+          # other cases, such as when we have recently marked a test
+          # as ignoreFailure but it still shows up in the "failed"
+          # category?  Maybe we should not rely on the result_type
+          # categories recorded within the gm_actuals AT ALL, and
+          # instead evaluate the result_type ourselves based on what
+          # we see in expectations vs actual checksum?
+          if expected_image == actual_image:
+            updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
+          else:
+            updated_result_type = result_type
+
+          # TODO(epoger): For now, don't include succeeded results.
+          # There are so many of them that they make the client too slow.
+          if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+            continue
+
+          (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+          all_tests.append({
+              "builder": builder,
+              "test": test,
+              "config": config,
+              "resultType": updated_result_type,
+              "actualHashType": actual_image[0],
+              "actualHashDigest": str(actual_image[1]),
+              "expectedHashType": expected_image[0],
+              "expectedHashDigest": str(expected_image[1]),
+          })
+    return all_tests
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
new file mode 100755
index 0000000000..5b81d8c2e2
--- /dev/null
+++ b/gm/rebaseline_server/server.py
@@ -0,0 +1,228 @@
+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+HTTP server for our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import argparse
+import BaseHTTPServer
+import json
+import os
+import posixpath
+import re
+import shutil
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'tools' directory, so that we can import svn.py within
+# that directory.
+# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.realpath(__file__))))
+TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
+if TOOLS_DIRECTORY not in sys.path:
+  sys.path.append(TOOLS_DIRECTORY)
+import svn
+
+# Imports from local dir
+import results
+
+ACTUALS_SVN_REPO = 'http://skia-autogen.googlecode.com/svn/gm-actual'
+PATHSPLIT_RE = re.compile('/([^/]+)/(.+)')
+TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.realpath(__file__))))
+
+# A simple dictionary of file name extensions to MIME types. The empty string
+# entry is used as the default when no extension was given or if the extension
+# has no entry in this dictionary.
+MIME_TYPE_MAP = {'': 'application/octet-stream',
+                 'html': 'text/html',
+                 'css': 'text/css',
+                 'png': 'image/png',
+                 'js': 'application/javascript',
+                 'json': 'application/json'
+                 }
+
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_PORT = 8888
+
+_SERVER = None   # This gets filled in by main()
+
+class Server(object):
+  """ HTTP server for our HTML rebaseline viewer.
+
+  params:
+    actuals_dir: directory under which we will check out the latest actual
+                 GM results
+    expectations_dir: directory under which to find GM expectations (they
+                      must already be in that directory)
+    port: which TCP port to listen on for HTTP requests
+    export: whether to allow HTTP clients on other hosts to access this server
+  """
+  def __init__(self,
+               actuals_dir=DEFAULT_ACTUALS_DIR,
+               expectations_dir=DEFAULT_EXPECTATIONS_DIR,
+               port=DEFAULT_PORT, export=False):
+    self._actuals_dir = actuals_dir
+    self._expectations_dir = expectations_dir
+    self._port = port
+    self._export = export
+
+  def fetch_results(self):
+    """ Create self.results, based on the expectations in
+    self._expectations_dir and the latest actuals from skia-autogen.
+
+    TODO(epoger): Add a new --browseonly mode setting.  In that mode,
+    the gm-actuals and expectations will automatically be updated every few
+    minutes.  See discussion in https://codereview.chromium.org/24274003/ .
+    """
+    print 'Checking out latest actual GM results from %s into %s ...' % (
+        ACTUALS_SVN_REPO, self._actuals_dir)
+    actuals_repo = svn.Svn(self._actuals_dir)
+    if not os.path.isdir(self._actuals_dir):
+      os.makedirs(self._actuals_dir)
+      actuals_repo.Checkout(ACTUALS_SVN_REPO, '.')
+    else:
+      actuals_repo.Update('.')
+    print 'Parsing results from actuals in %s and expectations in %s ...' % (
+        self._actuals_dir, self._expectations_dir)
+    self.results = results.Results(
+      actuals_root=self._actuals_dir,
+      expected_root=self._expectations_dir)
+
+  def run(self):
+    self.fetch_results()
+    if self._export:
+      server_address = ('', self._port)
+      print ('WARNING: Running in "export" mode. Users on other machines will '
+             'be able to modify your GM expectations!')
+    else:
+      server_address = ('127.0.0.1', self._port)
+    http_server = BaseHTTPServer.HTTPServer(server_address, HTTPRequestHandler)
+    print 'Ready for requests on http://%s:%d' % (
+        http_server.server_name, http_server.server_port)
+    http_server.serve_forever()
+
+
+class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+  """ HTTP request handlers for various types of queries this server knows
+      how to handle (static HTML and Javascript, expected/actual results, etc.)
+  """
+  def do_GET(self):
+    """ Handles all GET requests, forwarding them to the appropriate
+        do_GET_* dispatcher. """
+    if self.path == '' or self.path == '/' or self.path == '/index.html' :
+      self.redirect_to('/static/view.html')
+      return
+    if self.path == '/favicon.ico' :
+      self.redirect_to('/static/favicon.ico')
+      return
+
+    # All requests must be of this form:
+    #   /dispatcher/remainder
+    # where "dispatcher" indicates which do_GET_* dispatcher to run
+    # and "remainder" is the remaining path sent to that dispatcher.
+    normpath = posixpath.normpath(self.path)
+    (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
+    dispatchers = {
+      'results': self.do_GET_results,
+      'static': self.do_GET_static,
+    }
+    dispatcher = dispatchers[dispatcher_name]
+    dispatcher(remainder)
+
+  def do_GET_results(self, result_type):
+    """ Handle a GET request for GM results.
+    For now, we ignore the remaining path info, because we only know how to
+    return all results.
+
+    TODO(epoger): Unless we start making use of result_type, remove that
+    parameter."""
+    print 'do_GET_results: sending results of type "%s"' % result_type
+    response_dict = _SERVER.results.GetAll()
+    if response_dict:
+      self.send_json_dict(response_dict)
+    else:
+      self.send_error(404)
+
+  def do_GET_static(self, path):
+    """ Handle a GET request for a file under the 'static' directory. """
+    print 'do_GET_static: sending file "%s"' % path
+    self.send_file(posixpath.join('static', path))
+
+  def redirect_to(self, url):
+    """ Redirect the HTTP client to a different url. """
+    self.send_response(301)
+    self.send_header('Location', url)
+    self.end_headers()
+
+  def send_file(self, path):
+    """ Send the contents of the file at this path, with a mimetype based
+        on the filename extension. """
+    # Grab the extension if there is one
+    extension = os.path.splitext(path)[1]
+    if len(extension) >= 1:
+      extension = extension[1:]
+
+    # Determine the MIME type of the file from its extension
+    mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP[''])
+
+    # Open the file and send it over HTTP
+    if os.path.isfile(path):
+      with open(path, 'rb') as sending_file:
+        self.send_response(200)
+        self.send_header('Content-type', mime_type)
+        self.end_headers()
+        self.wfile.write(sending_file.read())
+    else:
+      self.send_error(404)
+
+  def send_json_dict(self, json_dict):
+    """ Send the contents of this dictionary in JSON format, with a JSON
+        mimetype. """
+    self.send_response(200)
+    self.send_header('Content-type', 'application/json')
+    self.end_headers()
+    json.dump(json_dict, self.wfile)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--actuals-dir',
+                    help=('Directory into which we will check out the latest '
+                          'actual GM results. If this directory does not '
+                          'exist, it will be created. Defaults to %(default)s'),
+                    default=DEFAULT_ACTUALS_DIR)
+  parser.add_argument('--expectations-dir',
+                    help=('Directory under which to find GM expectations; '
+                          'defaults to %(default)s'),
+                    default=DEFAULT_EXPECTATIONS_DIR)
+  parser.add_argument('--export', action='store_true',
+                      help=('Instead of only allowing access from HTTP clients '
+                            'on localhost, allow HTTP clients on other hosts '
+                            'to access this server.  WARNING: doing so will '
+                            'allow users on other hosts to modify your '
+                            'GM expectations!'))
+  parser.add_argument('--port',
+                    help=('Which TCP port to listen on for HTTP requests; '
+                          'defaults to %(default)s'),
+                    default=DEFAULT_PORT)
+  args = parser.parse_args()
+  global _SERVER
+  _SERVER = Server(expectations_dir=args.expectations_dir,
+                   port=args.port, export=args.export)
+  _SERVER.run()
+
+if __name__ == '__main__':
+  main()
diff --git a/gm/rebaseline_server/static/loader.js b/gm/rebaseline_server/static/loader.js
new file mode 100644
index 0000000000..68da73afda
--- /dev/null
+++ b/gm/rebaseline_server/static/loader.js
@@ -0,0 +1,20 @@
+/*
+ * Loader:
+ * Reads GM result reports written out by results_loader.py, and imports
+ * their data into $scope.results .
+ */
+var Loader = angular.module(
+    'Loader',
+    []
+);
+Loader.controller(
+  'Loader.Controller',
+  function($scope, $http) {
+    $http.get("/results/all").then(
+      function(response) {
+        $scope.results = response.data;
+        $scope.sortColumn = 'test';
+      }
+    );
+  }
+);
diff --git a/gm/rebaseline_server/static/view.html b/gm/rebaseline_server/static/view.html
new file mode 100644
index 0000000000..89ef538271
--- /dev/null
+++ b/gm/rebaseline_server/static/view.html
@@ -0,0 +1,101 @@
+<!DOCTYPE html>
+
+<html ng-app="Loader">
+
+<head>
+  <title>Current GM Results</title>
+  <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.1.5/angular.js"></script>
+  <script src="loader.js"></script>
+</head>
+
+<body>
+  <div ng-controller="Loader.Controller">
+
+  <!-- TODO(epoger): Add a warning banner if the server is running in
+  --export mode
+  -->
+
+ Settings:
+  <ul>
+    <!-- TODO(epoger): Now that we get multiple result types in a single
+    fetch, modify the UI: add a column showing resultType, and allow
+    the user to sort/filter on that column just like all the
+    others. -->
+    <li>show results of type
+      <select ng-model="showResultsOfType"
+              ng-init="showResultsOfType='failed'">
+        <option>failed</option>
+        <option>failure-ignored</option>
+        <!--
+        <option>no-comparison</option>
+
+        TODO(epoger): For now, I have disabled viewing the
+        no-comparison results because there are so many of them, and
+        the browser takes forever to download all the images.  Maybe
+        we should use some sort of lazy-loading technique
+        (e.g. http://www.appelsiini.net/projects/lazyload ), so that
+        the images are only loaded as they become viewable...
+        -->
+        <!--
+        <option>succeeded</option>
+
+        TODO(epoger): See results.py: for now, I have disabled
+        returning succeeded tests as part of the JSON, because it
+        makes the returned JSON too big (and slows down the client).
+        -->
+      </select>
+    </li>
+    <li>image size
+      <input type="text" ng-model="imageSize" ng-init="imageSize=100"
+             maxlength="4"/>
+    </li>
+  </ul>
+
+    <p>
+      <!-- TODO(epoger): Show some sort of "loading" message, instead of
+           an empty table, while the data is loading.  Otherwise, if there are
+           a lot of failures and it takes a long time to load them, the user
+           might think there are NO failures and leave the page! -->
+      <table border="1">
+        <tr>
+          <th ng:click="sortColumn='builder'">Builder</th>
+          <th ng:click="sortColumn='test'">Test</th>
+          <th ng:click="sortColumn='config'">Config</th>
+          <th ng:click="sortColumn='expectedHashDigest'">Expected Image</th>
+          <th ng:click="sortColumn='actualHashDigest'">Actual Image</th>
+          <!-- TODO(epoger): Add more columns, such as...
+               pixel diff
+               notes/bugs
+               ignoreFailure boolean
+          -->
+        </tr>
+        <!-- TODO(epoger): improve the column sorting, as per
+             http://jsfiddle.net/vojtajina/js64b/14/ -->
+        <tr ng-repeat="result in results | filter: { resultType: showResultsOfType } | orderBy: sortColumn">
+          <td>{{result.builder}}</td>
+          <td>{{result.test}}</td>
+          <td>{{result.config}}</td>
+          <td>
+	    <a target="_blank" href="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.expectedHashType}}/{{result.test}}/{{result.expectedHashDigest}}.png">
+              <img width="{{imageSize}}" src="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.expectedHashType}}/{{result.test}}/{{result.expectedHashDigest}}.png"/>
+            </a>
+          </td>
+          <td>
+	    <a target="_blank" href="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.actualHashType}}/{{result.test}}/{{result.actualHashDigest}}.png">
+	      <img width="{{imageSize}}" src="http://chromium-skia-gm.commondatastorage.googleapis.com/gm/{{result.actualHashType}}/{{result.test}}/{{result.actualHashDigest}}.png"/>
+            </a>
+          </td>
+        </tr>
+      </table>
+  </div>
+
+  <!-- TODO(epoger): Can we get the base URLs (commondatastorage and
+       issues list) from
+       http://skia.googlecode.com/svn/buildbot/site_config/global_variables.json
+       ?  I tried importing the
+       http://skia.googlecode.com/svn/buildbot/skia_tools.js script and using
+       that to do so, but I got Access-Control-Allow-Origin errors.
+    -->
+
+</body>
+</html>