#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Archives or replays webpages and creates SKPs in a Google Storage location. To archive webpages and store SKP files (archives should be rarely updated): cd ../buildbot/slave/skia_slave_scripts python webpages_playback.py --dest_gsbase=gs://rmistry --record \ --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \ --browser_executable=/tmp/chromium/out/Release/chrome To replay archived webpages and re-generate SKP files (should be run whenever SkPicture.PICTURE_VERSION changes): cd ../buildbot/slave/skia_slave_scripts python webpages_playback.py --dest_gsbase=gs://rmistry \ --page_sets=all --skia_tools=/home/default/trunk/out/Debug/ \ --browser_executable=/tmp/chromium/out/Release/chrome Specify the --page_sets flag (default value is 'all') to pick a list of which webpages should be archived and/or replayed. Eg: --page_sets=page_sets/skia_yahooanswers_desktop.json,\ page_sets/skia_wikipedia_galaxynexus.json The --browser_executable flag should point to the browser binary you want to use to capture archives and/or capture SKP files. Majority of the time it should be a newly built chrome binary. The --upload_to_gs flag controls whether generated artifacts will be uploaded to Google Storage (default value is False if not specified). The --non-interactive flag controls whether the script will prompt the user (default value is False if not specified). The --skia_tools flag if specified will allow this script to run debugger, render_pictures, and render_pdfs on the captured SKP(s). The tools are run after all SKPs are succesfully captured to make sure they can be added to the buildbots with no breakages. To preview the captured SKP before proceeding to the next page_set specify both --skia_tools and --view_debugger_output. """ import glob import optparse import os import posixpath import shutil import subprocess import sys import tempfile import time import traceback sys.path.insert(0, os.getcwd()) from common.py.utils import gs_utils from common.py.utils import shell_utils ROOT_PLAYBACK_DIR_NAME = 'playback' SKPICTURES_DIR_NAME = 'skps' # Local archive and SKP directories. LOCAL_PLAYBACK_ROOT_DIR = os.path.join( tempfile.gettempdir(), ROOT_PLAYBACK_DIR_NAME) LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data') TMP_SKP_DIR = tempfile.mkdtemp() # Location of the credentials.json file and the string that represents missing # passwords. CREDENTIALS_FILE_PATH = os.path.join( os.path.abspath(os.path.dirname(__file__)), 'page_sets', 'data', 'credentials.json' ) # Name of the SKP benchmark SKP_BENCHMARK = 'skpicture_printer' # The max base name length of Skp files. MAX_SKP_BASE_NAME_LEN = 31 # Dictionary of device to platform prefixes for SKP files. DEVICE_TO_PLATFORM_PREFIX = { 'desktop': 'desk', 'galaxynexus': 'mobi', 'nexus10': 'tabl' } # How many times the record_wpr binary should be retried. RETRY_RECORD_WPR_COUNT = 5 # How many times the run_benchmark binary should be retried. RETRY_RUN_MEASUREMENT_COUNT = 5 # Location of the credentials.json file in Google Storage. CREDENTIALS_GS_PATH = '/playback/credentials/credentials.json' X11_DISPLAY = os.getenv('DISPLAY', ':0') GS_PREDEFINED_ACL = gs_utils.GSUtils.PredefinedACL.PRIVATE GS_FINE_GRAINED_ACL_LIST = [ (gs_utils.GSUtils.IdType.GROUP_BY_DOMAIN, 'google.com', gs_utils.GSUtils.Permission.READ), ] class SkPicturePlayback(object): """Class that archives or replays webpages and creates SKPs.""" def __init__(self, parse_options): """Constructs a SkPicturePlayback BuildStep instance.""" assert parse_options.browser_executable, 'Must specify --browser_executable' self._browser_executable = parse_options.browser_executable self._all_page_sets_specified = parse_options.page_sets == 'all' self._page_sets = self._ParsePageSets(parse_options.page_sets) self._dest_gsbase = parse_options.dest_gsbase self._record = parse_options.record self._skia_tools = parse_options.skia_tools self._non_interactive = parse_options.non_interactive self._upload_to_gs = parse_options.upload_to_gs self._alternate_upload_dir = parse_options.alternate_upload_dir self._skip_all_gs_access = parse_options.skip_all_gs_access self._telemetry_binaries_dir = os.path.join(parse_options.chrome_src_path, 'tools', 'perf') self._local_skp_dir = os.path.join( parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, SKPICTURES_DIR_NAME) self._local_record_webpages_archive_dir = os.path.join( parse_options.output_dir, ROOT_PLAYBACK_DIR_NAME, 'webpages_archive') # List of SKP files generated by this script. self._skp_files = [] def _ParsePageSets(self, page_sets): if not page_sets: raise ValueError('Must specify at least one page_set!') elif self._all_page_sets_specified: # Get everything from the page_sets directory. page_sets_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'page_sets') ps = [os.path.join(page_sets_dir, page_set) for page_set in os.listdir(page_sets_dir) if not os.path.isdir(os.path.join(page_sets_dir, page_set)) and page_set.endswith('.py')] elif '*' in page_sets: # Explode and return the glob. ps = glob.glob(page_sets) else: ps = page_sets.split(',') ps.sort() return ps def Run(self): """Run the SkPicturePlayback BuildStep.""" # Download the credentials file if it was not previously downloaded. if self._skip_all_gs_access: print """\n\nPlease create a %s file that contains: { "google": { "username": "google_testing_account_username", "password": "google_testing_account_password" }, "facebook": { "username": "facebook_testing_account_username", "password": "facebook_testing_account_password" } }\n\n""" % CREDENTIALS_FILE_PATH raw_input("Please press a key when you are ready to proceed...") elif not os.path.isfile(CREDENTIALS_FILE_PATH): # Download the credentials.json file from Google Storage. gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) gs_utils.GSUtils().download_file(gs_bucket, CREDENTIALS_GS_PATH, CREDENTIALS_FILE_PATH) # Delete any left over data files in the data directory. for archive_file in glob.glob( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, 'skia_*')): os.remove(archive_file) # Delete the local root directory if it already exists. if os.path.exists(LOCAL_PLAYBACK_ROOT_DIR): shutil.rmtree(LOCAL_PLAYBACK_ROOT_DIR) # Create the required local storage directories. self._CreateLocalStorageDirs() # Start the timer. start_time = time.time() # Loop through all page_sets. for page_set in self._page_sets: page_set_basename = os.path.basename(page_set).split('.')[0] page_set_json_name = page_set_basename + '.json' wpr_data_file = page_set.split(os.path.sep)[-1].split('.')[0] + '_000.wpr' page_set_dir = os.path.dirname(page_set) if self._record: # Create an archive of the specified webpages if '--record=True' is # specified. record_wpr_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, os.path.join(self._telemetry_binaries_dir, 'record_wpr'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, '%s_page_set' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir ) for _ in range(RETRY_RECORD_WPR_COUNT): try: shell_utils.run(' '.join(record_wpr_cmd), shell=True) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() else: # If we get here then record_wpr did not succeed and thus did not # break out of the loop. raise Exception('record_wpr failed for page_set: %s' % page_set) else: if not self._skip_all_gs_access: # Get the webpages archive so that it can be replayed. self._DownloadWebpagesArchive(wpr_data_file, page_set_json_name) run_benchmark_cmd = ( 'PYTHONPATH=%s:$PYTHONPATH' % page_set_dir, 'DISPLAY=%s' % X11_DISPLAY, 'timeout', '300', os.path.join(self._telemetry_binaries_dir, 'run_benchmark'), '--extra-browser-args=--disable-setuid-sandbox', '--browser=exact', '--browser-executable=%s' % self._browser_executable, SKP_BENCHMARK, '--page-set-name=%s' % page_set_basename, '--page-set-base-dir=%s' % page_set_dir, '--skp-outdir=%s' % TMP_SKP_DIR, '--also-run-disabled-tests' ) for _ in range(RETRY_RUN_MEASUREMENT_COUNT): try: print '\n\n=======Capturing SKP of %s=======\n\n' % page_set shell_utils.run(' '.join(run_benchmark_cmd), shell=True) except shell_utils.CommandFailedException: # skpicture_printer sometimes fails with AssertionError but the # captured SKP is still valid. This is a known issue. pass if self._record: # Move over the created archive into the local webpages archive # directory. shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file), self._local_record_webpages_archive_dir) shutil.move( os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, page_set_json_name), self._local_record_webpages_archive_dir) # Rename generated SKP files into more descriptive names. try: self._RenameSkpFiles(page_set) # Break out of the retry loop since there were no errors. break except Exception: # There was a failure continue with the loop. traceback.print_exc() print '\n\n=======Retrying %s=======\n\n' % page_set time.sleep(10) else: # If we get here then run_benchmark did not succeed and thus did not # break out of the loop. raise Exception('run_benchmark failed for page_set: %s' % page_set) print '\n\n=======Capturing SKP files took %s seconds=======\n\n' % ( time.time() - start_time) if self._skia_tools: render_pictures_cmd = [ os.path.join(self._skia_tools, 'render_pictures'), '-r', self._local_skp_dir ] render_pdfs_cmd = [ os.path.join(self._skia_tools, 'render_pdfs'), self._local_skp_dir ] for tools_cmd in (render_pictures_cmd, render_pdfs_cmd): print '\n\n=======Running %s=======' % ' '.join(tools_cmd) proc = subprocess.Popen(tools_cmd) (code, _) = shell_utils.log_process_after_completion(proc, echo=False) if code != 0: raise Exception('%s failed!' % ' '.join(tools_cmd)) if not self._non_interactive: print '\n\n=======Running debugger=======' os.system('%s %s' % (os.path.join(self._skia_tools, 'debugger'), os.path.join(self._local_skp_dir, '*'))) print '\n\n' if not self._skip_all_gs_access and self._upload_to_gs: print '\n\n=======Uploading to Google Storage=======\n\n' # Copy the directory structure in the root directory into Google Storage. dest_dir_name = ROOT_PLAYBACK_DIR_NAME if self._alternate_upload_dir: dest_dir_name = self._alternate_upload_dir gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) gs_utils.GSUtils().upload_dir_contents( LOCAL_PLAYBACK_ROOT_DIR, gs_bucket, dest_dir_name, upload_if=gs_utils.GSUtils.UploadIf.IF_MODIFIED, predefined_acl=GS_PREDEFINED_ACL, fine_grained_acl_list=GS_FINE_GRAINED_ACL_LIST) print '\n\n=======New SKPs have been uploaded to %s =======\n\n' % ( posixpath.join(self._dest_gsbase, dest_dir_name, SKPICTURES_DIR_NAME)) else: print '\n\n=======Not Uploading to Google Storage=======\n\n' print 'Generated resources are available in %s\n\n' % ( LOCAL_PLAYBACK_ROOT_DIR) return 0 def _RenameSkpFiles(self, page_set): """Rename generated SKP files into more descriptive names. Look into the subdirectory of TMP_SKP_DIR and find the most interesting .skp in there to be this page_set's representative .skp. """ # Here's where we're assuming there's one page per pageset. # If there were more than one, we'd overwrite filename below. # /path/to/skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop.json _, ps_filename = os.path.split(page_set) # skia_yahooanswers_desktop.json -> skia_yahooanswers_desktop ps_basename, _ = os.path.splitext(ps_filename) # skia_yahooanswers_desktop -> skia, yahooanswers, desktop _, page_name, device = ps_basename.split('_') basename = '%s_%s' % (DEVICE_TO_PLATFORM_PREFIX[device], page_name) filename = basename[:MAX_SKP_BASE_NAME_LEN] + '.skp' subdirs = glob.glob(os.path.join(TMP_SKP_DIR, '*')) assert len(subdirs) == 1 for site in subdirs: # We choose the largest .skp as the most likely to be interesting. largest_skp = max(glob.glob(os.path.join(site, '*.skp')), key=lambda path: os.stat(path).st_size) dest = os.path.join(self._local_skp_dir, filename) print 'Moving', largest_skp, 'to', dest shutil.move(largest_skp, dest) self._skp_files.append(filename) shutil.rmtree(site) def _CreateLocalStorageDirs(self): """Creates required local storage directories for this script.""" for d in (self._local_record_webpages_archive_dir, self._local_skp_dir): if os.path.exists(d): shutil.rmtree(d) os.makedirs(d) def _DownloadWebpagesArchive(self, wpr_data_file, page_set_json_name): """Downloads the webpages archive and its required page set from GS.""" wpr_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', wpr_data_file) page_set_source = posixpath.join(ROOT_PLAYBACK_DIR_NAME, 'webpages_archive', page_set_json_name) gs = gs_utils.GSUtils() gs_bucket = self._dest_gsbase.lstrip(gs_utils.GS_PREFIX) if (gs.does_storage_object_exist(gs_bucket, wpr_source) and gs.does_storage_object_exist(gs_bucket, page_set_source)): gs.download_file(gs_bucket, wpr_source, os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, wpr_data_file)) gs.download_file(gs_bucket, page_set_source, os.path.join(LOCAL_REPLAY_WEBPAGES_ARCHIVE_DIR, page_set_json_name)) else: raise Exception('%s and %s do not exist in Google Storage!' % ( wpr_source, page_set_source)) if '__main__' == __name__: option_parser = optparse.OptionParser() option_parser.add_option( '', '--page_sets', help='Specifies the page sets to use to archive. Supports globs.', default='all') option_parser.add_option( '', '--skip_all_gs_access', action='store_true', help='All Google Storage interactions will be skipped if this flag is ' 'specified. This is useful for cases where the user does not have ' 'the required .boto file but would like to generate webpage ' 'archives and SKPs from the Skia page sets.', default=False) option_parser.add_option( '', '--record', action='store_true', help='Specifies whether a new website archive should be created.', default=False) option_parser.add_option( '', '--dest_gsbase', help='gs:// bucket_name, the bucket to upload the file to.', default='gs://chromium-skia-gm') option_parser.add_option( '', '--skia_tools', help=('Path to compiled Skia executable tools. ' 'render_pictures/render_pdfs is run on the set ' 'after all SKPs are captured. If the script is run without ' '--non-interactive then the debugger is also run at the end. Debug ' 'builds are recommended because they seem to catch more failures ' 'than Release builds.'), default=None) option_parser.add_option( '', '--upload_to_gs', action='store_true', help='Does not upload to Google Storage if this is False.', default=False) option_parser.add_option( '', '--alternate_upload_dir', help='Uploads to a different directory in Google Storage if this flag is ' 'specified', default=None) option_parser.add_option( '', '--output_dir', help='Directory where SKPs and webpage archives will be outputted to.', default=tempfile.gettempdir()) option_parser.add_option( '', '--browser_executable', help='The exact browser executable to run.', default=None) option_parser.add_option( '', '--chrome_src_path', help='Path to the chromium src directory.', default=None) option_parser.add_option( '', '--non-interactive', action='store_true', help='Runs the script without any prompts. If this flag is specified and ' '--skia_tools is specified then the debugger is not run.', default=False) options, unused_args = option_parser.parse_args() playback = SkPicturePlayback(options) sys.exit(playback.Run())