path: root/experimental/benchtools/rebase.py

#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""rebase.py: standalone script to batch update bench expectations.

    Requires gsutil to access gs://chromium-skia-gm and Rietveld credentials.

    Usage:
      Copy script to a separate dir outside Skia repo. The script will create a
          skia dir on the first run to host the repo, and will create/delete
          temp dirs as needed.
      ./rebase.py --githash <githash prefix to use for getting bench data>
"""


import argparse
import filecmp
import os
import re
import shutil
import subprocess
import time
import urllib2


# googlesource url that has most recent Skia git hash info.
SKIA_GIT_HEAD_URL = 'https://skia.googlesource.com/skia/+log/HEAD'

# Google Storage bench file prefix.
GS_PREFIX = 'gs://chromium-skia-gm/perfdata'

# Regular expression for matching githash data.
HA_RE = '<a href="/skia/\+/([0-9a-f]+)">'
HA_RE_COMPILED = re.compile(HA_RE)


def get_git_hashes():
  print 'Getting recent git hashes...'
  hashes = HA_RE_COMPILED.findall(
      urllib2.urlopen(SKIA_GIT_HEAD_URL).read())

  return hashes

def filter_file(f):
  if f.find('_msaa') > 0 or f.find('_record') > 0:
    return True

  return False

def clean_dir(d):
  if os.path.exists(d):
    shutil.rmtree(d)
  os.makedirs(d)

def get_gs_filelist(p, h):
  print 'Looking up for the closest bench files in Google Storage...'
  proc = subprocess.Popen(['gsutil', 'ls',
      '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*'])],
          stdout=subprocess.PIPE)
  out, err = proc.communicate()
  if err or not out:
    return []
  return [i for i in out.strip().split('\n') if not filter_file(i)]

def download_gs_files(p, h, gs_dir):
  print 'Downloading raw bench files from Google Storage...'
  proc = subprocess.Popen(['gsutil', 'cp',
      '/'.join([GS_PREFIX, p, 'bench_' + h + '_data_skp_*']),
          '%s/%s' % (gs_dir, p)],
          stdout=subprocess.PIPE)
  out, err = proc.communicate()
  if err:
    clean_dir(gs_dir)
    return False
  files = 0
  for f in os.listdir(os.path.join(gs_dir, p)):
    if filter_file(f):
      os.remove(os.path.join(gs_dir, p, f))
    else:
      files += 1
  if files:
    return True
  return False

def get_expectations_dict(f):
  """Given an expectations file f, returns a dictionary of data."""
  # maps row_key to (expected, lower_bound, upper_bound) float tuple.
  dic = {}
  for l in open(f).readlines():
    line_parts = l.strip().split(',')
    if line_parts[0].startswith('#') or len(line_parts) != 5:
      continue
    dic[','.join(line_parts[:2])] = (float(line_parts[2]), float(line_parts[3]),
                                     float(line_parts[4]))

  return dic

def calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir, extra_hash):
  exp_filename = 'bench_expectations_%s.txt' % p
  exp_fullname = os.path.join(exp_dir, exp_filename)
  proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py',
      '-r', h, '-b', p, '-d', os.path.join(gs_dir, p), '-o', exp_fullname],
              stdout=subprocess.PIPE)
  out, err = proc.communicate()
  if err:
    print 'ERR_CALCULATING_EXPECTATIONS: ' + err
    return False
  print 'CALCULATED_EXPECTATIONS: ' + out
  if extra_dir:  # Adjust data with the ones in extra_dir
    print 'USE_EXTRA_DATA_FOR_ADJUSTMENT.'
    proc = subprocess.Popen(['python', 'skia/bench/gen_bench_expectations.py',
        '-r', extra_hash, '-b', p, '-d', os.path.join(extra_dir, p), '-o',
            os.path.join(extra_dir, exp_filename)],
                stdout=subprocess.PIPE)
    out, err = proc.communicate()
    if err:
      print 'ERR_CALCULATING_EXTRA_EXPECTATIONS: ' + err
      return False
    extra_dic = get_expectations_dict(os.path.join(extra_dir, exp_filename))
    output_lines = []
    for l in open(exp_fullname).readlines():
      parts = l.strip().split(',')
      if parts[0].startswith('#') or len(parts) != 5:
        output_lines.append(l.strip())
        continue
      key = ','.join(parts[:2])
      if key in extra_dic:
        exp, lb, ub = (float(parts[2]), float(parts[3]), float(parts[4]))
        alt, _, _ = extra_dic[key]
        avg = (exp + alt) / 2
        # Keeps the extra range in lower/upper bounds from two actual values.
        new_lb = min(exp, alt) - (exp - lb)
        new_ub = max(exp, alt) + (ub - exp)
        output_lines.append('%s,%.2f,%.2f,%.2f' % (key, avg, new_lb, new_ub))
      else:
        output_lines.append(l.strip())
    with open(exp_fullname, 'w') as f:
      f.write('\n'.join(output_lines))

  repo_file = os.path.join(repo_dir, 'expectations', 'bench', exp_filename)
  if (os.path.isfile(repo_file) and
      filecmp.cmp(repo_file, os.path.join(exp_dir, exp_filename))):
      print 'NO CHANGE ON %s' % repo_file
      return False
  return True

def checkout_or_update_skia(repo_dir):
  status = True
  old_cwd = os.getcwd()
  os.chdir(repo_dir)
  print 'CHECK SKIA REPO...'
  if subprocess.call(['git', 'pull'],
                     stderr=subprocess.PIPE):
    print 'Checking out Skia from git, please be patient...'
    os.chdir(old_cwd)
    clean_dir(repo_dir)
    os.chdir(repo_dir)
    if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch',
                        'https://skia.googlesource.com/skia.git', '.']):
      status = False
  subprocess.call(['git', 'checkout', 'master'])
  subprocess.call(['git', 'pull'])
  os.chdir(old_cwd)
  return status

def git_commit_expectations(repo_dir, exp_dir, update_li, h, commit,
                            extra_hash):
  if extra_hash:
    extra_hash = ', adjusted with ' + extra_hash
  commit_msg = """manual bench rebase after %s%s

TBR=robertphillips@google.com

Bypassing trybots:
NOTRY=true""" % (h, extra_hash)
  old_cwd = os.getcwd()
  os.chdir(repo_dir)
  upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks',
            '--bypass-watchlists', '-m', commit_msg]
  branch = exp_dir.split('/')[-1]
  if commit:
    upload.append('--use-commit-queue')
  cmds = ([['git', 'checkout', 'master'],
           ['git', 'pull'],
           ['git', 'checkout', '-b', branch, '-t', 'origin/master']] +
          [['cp', '%s/%s' % (exp_dir, f), 'expectations/bench'] for f in
           update_li] +
          [['git', 'add'] + ['expectations/bench/%s' % i for i in update_li],
           ['git', 'commit', '-m', commit_msg],
           upload,
           ['git', 'checkout', 'master'],
           ['git', 'branch', '-D', branch],
          ])
  status = True
  for cmd in cmds:
    print 'Running ' + ' '.join(cmd)
    if subprocess.call(cmd):
      print 'FAILED. Please check if skia git repo is present.'
      subprocess.call(['git', 'checkout', 'master'])
      status = False
      break
  os.chdir(old_cwd)
  return status

def delete_dirs(li):
  for d in li:
    print 'Deleting directory %s' % d
    shutil.rmtree(d)


def main():
  d = os.path.dirname(os.path.abspath(__file__))
  os.chdir(d)
  if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE):
    print 'Please copy script to a separate dir outside git repos to use.'
    return
  parser = argparse.ArgumentParser()
  parser.add_argument('--githash',
                      help=('Githash prefix (7+ chars) to rebaseline to. If '
                            'a second one is supplied after comma, and it has '
                            'corresponding bench data, will shift the range '
                            'center to the average of two expected values.'))
  parser.add_argument('--bots',
                      help=('Comma-separated list of bots to work on. If no '
                            'matching bots are found in the list, will default '
                            'to processing all bots.'))
  parser.add_argument('--commit', action='store_true',
                      help='Whether to commit changes automatically.')
  args = parser.parse_args()

  repo_dir = os.path.join(d, 'skia')
  if not os.path.exists(repo_dir):
    os.makedirs(repo_dir)
  if not checkout_or_update_skia(repo_dir):
    print 'ERROR setting up Skia repo at %s' % repo_dir
    return 1

  file_in_repo = os.path.join(d, 'skia/experimental/benchtools/rebase.py')
  if not filecmp.cmp(__file__, file_in_repo):
    shutil.copy(file_in_repo, __file__)
    print 'Updated this script from repo; please run again.'
    return

  all_platforms = []  # Find existing list of platforms with expectations.
  for item in os.listdir(os.path.join(d, 'skia/expectations/bench')):
    all_platforms.append(
        item.replace('bench_expectations_', '').replace('.txt', ''))

  platforms = []
  # If at least one given bot is in all_platforms, use list of valid args.bots.
  if args.bots:
    bots = args.bots.strip().split(',')
    for bot in bots:
      if bot in all_platforms:  # Filters platforms with given bot list.
        platforms.append(bot)
  if not platforms:  # Include all existing platforms with expectations.
    platforms = all_platforms

  if not args.githash or len(args.githash) < 7:
    raise Exception('Please provide --githash with a longer prefix (7+).')
  githashes = args.githash.strip().split(',')
  if len(githashes[0]) < 7:
    raise Exception('Please provide --githash with longer prefixes (7+).')
  commit = False
  if args.commit:
    commit = True
  rebase_hash = githashes[0][:7]
  extra_hash = ''
  if len(githashes) == 2:
    extra_hash = githashes[1][:7]
  hashes = get_git_hashes()
  short_hashes = [h[:7] for h in hashes]
  if (rebase_hash not in short_hashes or
      (extra_hash and extra_hash not in short_hashes) or
      rebase_hash == extra_hash):
    raise Exception('Provided --githashes not found, or identical!')
  if extra_hash:
    extra_hash = hashes[short_hashes.index(extra_hash)]
  hashes = hashes[:short_hashes.index(rebase_hash) + 1]
  update_li = []

  ts_str = '%s' % time.time()
  gs_dir = os.path.join(d, 'gs' + ts_str)
  exp_dir = os.path.join(d, 'exp' + ts_str)
  extra_dir = os.path.join(d, 'extra' + ts_str)
  clean_dir(gs_dir)
  clean_dir(exp_dir)
  clean_dir(extra_dir)
  for p in platforms:
    clean_dir(os.path.join(gs_dir, p))
    clean_dir(os.path.join(extra_dir, p))
    hash_to_use = ''
    for h in reversed(hashes):
      li = get_gs_filelist(p, h)
      if not len(li):  # no data
        continue
      if download_gs_files(p, h, gs_dir):
        print 'Copied %s/%s' % (p, h)
        hash_to_use = h
        break
      else:
        print 'DOWNLOAD BENCH FAILED %s/%s' % (p, h)
        break
    if hash_to_use:
      if extra_hash and download_gs_files(p, extra_hash, extra_dir):
        print 'Copied extra data %s/%s' % (p, extra_hash)
        if calc_expectations(p, h, gs_dir, exp_dir, repo_dir, extra_dir,
                             extra_hash):
          update_li.append('bench_expectations_%s.txt' % p)
      elif calc_expectations(p, h, gs_dir, exp_dir, repo_dir, '', ''):
        update_li.append('bench_expectations_%s.txt' % p)
  if not update_li:
    print 'No bench data to update after %s!' % args.githash
  elif not git_commit_expectations(
      repo_dir, exp_dir, update_li, rebase_hash, commit, extra_hash):
    print 'ERROR uploading expectations using git.'
  elif not commit:
    print 'CL created. Please take a look at the link above.'
  else:
    print 'New bench baselines should be in CQ now.'
  delete_dirs([gs_dir, exp_dir, extra_dir])


if __name__ == "__main__":
  main()