diff options
Diffstat (limited to 'gm/rebaseline_server/server.py')
-rwxr-xr-x | gm/rebaseline_server/server.py | 103 |
1 files changed, 85 insertions, 18 deletions
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 50367f6f35..6271f97dd2 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -40,6 +40,7 @@ import gm_json # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44 # pylint: enable=C0301 import compare_configs +import compare_rendered_pictures import compare_to_expectations import download_actuals import imagediffdb @@ -73,8 +74,11 @@ DEFAULT_PORT = 8888 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY)) # Directory, relative to PARENT_DIRECTORY, within which the server will serve -# out live results (not static files). -RESULTS_SUBDIR = 'results' +# out image diff data from within the precomputed _SERVER.results . +PRECOMPUTED_RESULTS_SUBDIR = 'results' +# Directory, relative to PARENT_DIRECTORY, within which the server will serve +# out live-generated image diff data. +LIVE_RESULTS_SUBDIR = 'live-results' # Directory, relative to PARENT_DIRECTORY, within which the server will serve # out static files. STATIC_CONTENTS_SUBDIR = 'static' @@ -83,6 +87,10 @@ GENERATED_HTML_SUBDIR = 'generated-html' GENERATED_IMAGES_SUBDIR = 'generated-images' GENERATED_JSON_SUBDIR = 'generated-json' +# Parameters we use within do_GET_live_results() +LIVE_PARAM__SET_A_DIR = 'setADir' +LIVE_PARAM__SET_B_DIR = 'setBDir' + # How often (in seconds) clients should reload while waiting for initial # results to load. RELOAD_INTERVAL_UNTIL_READY = 10 @@ -165,7 +173,7 @@ def _create_index(file_path, config_pairs): '<li><a href="/{static_subdir}/view.html#/view.html?' 'resultsToLoad=/{results_subdir}/{summary_type}">' '{summary_type}</a></li>'.format( - results_subdir=RESULTS_SUBDIR, + results_subdir=PRECOMPUTED_RESULTS_SUBDIR, static_subdir=STATIC_CONTENTS_SUBDIR, summary_type=summary_type)) file_handle.write('</ul>') @@ -193,7 +201,9 @@ class Server(object): json_filename=DEFAULT_JSON_FILENAME, gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET, port=DEFAULT_PORT, export=False, editable=True, - reload_seconds=0, config_pairs=None, builder_regex_list=None): + reload_seconds=0, config_pairs=None, builder_regex_list=None, + boto_file_path=None, + imagediffdb_threads=imagediffdb.DEFAULT_NUM_WORKER_THREADS): """ Args: actuals_dir: directory under which we will check out the latest actual @@ -212,6 +222,10 @@ class Server(object): don't compare configs at all. builder_regex_list: List of regular expressions specifying which builders we will process. If None, process all builders. + boto_file_path: Path to .boto file giving us credentials to access + Google Storage buckets; if None, we will only be able to access + public GS buckets. + imagediffdb_threads: How many threads to spin up within imagediffdb. """ self._actuals_dir = actuals_dir self._json_filename = json_filename @@ -222,7 +236,13 @@ class Server(object): self._reload_seconds = reload_seconds self._config_pairs = config_pairs or [] self._builder_regex_list = builder_regex_list - self._gs = gs_utils.GSUtils() + self.truncate_results = False + + if boto_file_path: + self._gs = gs_utils.GSUtils(boto_file_path=boto_file_path) + else: + self._gs = gs_utils.GSUtils() + _create_index( file_path=os.path.join( PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR, @@ -234,9 +254,16 @@ class Server(object): # 2. the expected or actual results on local disk self.results_rlock = threading.RLock() - # These will be filled in by calls to update_results() + # Create a single ImageDiffDB instance that is used by all our differs. + self._image_diff_db = imagediffdb.ImageDiffDB( + gs=self._gs, + storage_root=os.path.join( + PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, + GENERATED_IMAGES_SUBDIR), + num_worker_threads=imagediffdb_threads) + + # This will be filled in by calls to update_results() self._results = None - self._image_diff_db = None @property def results(self): @@ -245,6 +272,16 @@ class Server(object): return self._results @property + def image_diff_db(self): + """ Returns reference to our ImageDiffDB object.""" + return self._image_diff_db + + @property + def gs(self): + """ Returns reference to our GSUtils object.""" + return self._gs + + @property def is_exported(self): """ Returns true iff HTTP clients on other hosts are allowed to access this server. """ @@ -343,12 +380,6 @@ class Server(object): compare_to_expectations.DEFAULT_EXPECTATIONS_DIR) _run_command(['gclient', 'sync'], TRUNK_DIRECTORY) - if not self._image_diff_db: - self._image_diff_db = imagediffdb.ImageDiffDB( - storage_root=os.path.join( - PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, - GENERATED_IMAGES_SUBDIR)) - self._results = compare_to_expectations.ExpectationComparisons( image_diff_db=self._image_diff_db, actuals_root=self._actuals_dir, @@ -443,7 +474,8 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): normpath = posixpath.normpath(self.path) (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups() dispatchers = { - RESULTS_SUBDIR: self.do_GET_results, + PRECOMPUTED_RESULTS_SUBDIR: self.do_GET_precomputed_results, + LIVE_RESULTS_SUBDIR: self.do_GET_live_results, STATIC_CONTENTS_SUBDIR: self.do_GET_static, } dispatcher = dispatchers[dispatcher_name] @@ -452,14 +484,15 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.send_error(404) raise - def do_GET_results(self, results_type): - """ Handle a GET request for GM results. + def do_GET_precomputed_results(self, results_type): + """ Handle a GET request for part of the precomputed _SERVER.results object. Args: results_type: string indicating which set of results to return; must be one of the results_mod.RESULTS_* constants """ - logging.debug('do_GET_results: sending results of type "%s"' % results_type) + logging.debug('do_GET_precomputed_results: sending results of type "%s"' % + results_type) # Since we must make multiple calls to the ExpectationComparisons object, # grab a reference to it in case it is updated to point at a new # ExpectationComparisons object within another thread. @@ -487,6 +520,23 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): } self.send_json_dict(response_dict) + def do_GET_live_results(self, url_remainder): + """ Handle a GET request for live-generated image diff data. + + Args: + url_remainder: string indicating which image diffs to generate + """ + logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) + param_dict = urlparse.parse_qs(url_remainder) + results_obj = compare_rendered_pictures.RenderedPicturesComparisons( + setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], + setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], + image_diff_db=_SERVER.image_diff_db, + diff_base_url='/static/generated-images', + gs=_SERVER.gs, truncate_results=_SERVER.truncate_results) + self.send_json_dict(results_obj.get_packaged_results_of_type( + results_mod.KEY__HEADER__RESULTS_ALL)) + def do_GET_static(self, path): """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a @@ -643,6 +693,12 @@ def main(): 'actual GM results. If this directory does not ' 'exist, it will be created. Defaults to %(default)s'), default=DEFAULT_ACTUALS_DIR) + parser.add_argument('--boto', + help=('Path to .boto file giving us credentials to access ' + 'Google Storage buckets. If not specified, we will ' + 'only be able to access public GS buckets (and thus ' + 'won\'t be able to download SKP images).'), + default='') # TODO(epoger): Before https://codereview.chromium.org/310093003 , # when this tool downloaded the JSON summaries from skia-autogen, # it had an --actuals-revision the caller could specify to download @@ -688,6 +744,14 @@ def main(): 'By default, we do not reload at all, and you ' 'must restart the server to pick up new data.'), default=0) + parser.add_argument('--threads', type=int, + help=('How many parallel threads we use to download ' + 'images and generate diffs; defaults to ' + '%(default)s'), + default=imagediffdb.DEFAULT_NUM_WORKER_THREADS) + parser.add_argument('--truncate', action='store_true', + help=('FOR TESTING ONLY: truncate the set of images we ' + 'process, to speed up testing.')) args = parser.parse_args() if args.compare_configs: config_pairs = CONFIG_PAIRS_TO_COMPARE @@ -700,7 +764,10 @@ def main(): gm_summaries_bucket=args.gm_summaries_bucket, port=args.port, export=args.export, editable=args.editable, reload_seconds=args.reload, config_pairs=config_pairs, - builder_regex_list=args.builders) + builder_regex_list=args.builders, boto_file_path=args.boto, + imagediffdb_threads=args.threads) + if args.truncate: + _SERVER.truncate_results = True _SERVER.run() |