diff options
author | csmartdalton <csmartdalton@google.com> | 2016-09-22 05:10:02 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2016-09-22 05:10:03 -0700 |
commit | d7a9db644496785100c4e61add1c9f8ed0494408 (patch) | |
tree | 0ccc969ee1969077f7a7147c470ded84201b16cf /tools | |
parent | 50537e46e4f0999df0a4707b227000cfa8c800ff (diff) |
Add hardware monitoring to skpbench
Adds a Hardware class with hooks for entering and exiting
"benchmarking" mode (e.g. locking clocks, etc.) as well as periodic
polling of hardware to verify the environment is stable.
Adds a partial implementation for generic Android hardware, but
ultimately we will need to write specific classes tailored to each
unique platform we need to test.
BUG=skia:
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2360473002
Review-Url: https://codereview.chromium.org/2360473002
Diffstat (limited to 'tools')
-rw-r--r-- | tools/skpbench/_adb.py | 46 | ||||
-rw-r--r-- | tools/skpbench/_adb_path.py | 17 | ||||
-rw-r--r-- | tools/skpbench/_hardware.py | 29 | ||||
-rw-r--r-- | tools/skpbench/_hardware_android.py | 93 | ||||
-rwxr-xr-x | tools/skpbench/parseskpbench.py | 24 | ||||
-rw-r--r-- | tools/skpbench/skpbench.cpp | 11 | ||||
-rwxr-xr-x | tools/skpbench/skpbench.py | 220 |
7 files changed, 333 insertions, 107 deletions
diff --git a/tools/skpbench/_adb.py b/tools/skpbench/_adb.py index 6125c8d538..1769f58e57 100644 --- a/tools/skpbench/_adb.py +++ b/tools/skpbench/_adb.py @@ -3,17 +3,39 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +import re import subprocess -def shell(cmd, device_serial=None): - if device_serial is None: - subprocess.call(['adb', 'shell', cmd]) - else: - subprocess.call(['adb', '-s', device_serial, 'shell', cmd]) - -def check(cmd, device_serial=None): - if device_serial is None: - out = subprocess.check_output(['adb', 'shell', cmd]) - else: - out = subprocess.check_output(['adb', '-s', device_serial, 'shell', cmd]) - return out.rstrip() +class Adb: + def __init__(self, device_serial=None): + self.__invocation = ['adb'] + if device_serial: + self.__invocation.extend(['-s', device_serial]) + + def shell(self, cmd): + subprocess.call(self.__invocation + ['shell', cmd]) + + def check(self, cmd): + result = subprocess.check_output(self.__invocation + ['shell', cmd]) + return result.rstrip() + + def check_lines(self, cmd): + result = self.check(cmd) + return re.split('[\r\n]+', result) + + def get_device_model(self): + result = self.check('getprop | grep ro.product.model') + result = re.match(r'\[ro.product.model\]:\s*\[(.*)\]', result) + return result.group(1) if result else 'unknown_product' + + def is_root(self): + return self.check('echo $USER') == 'root' + + def attempt_root(self): + if self.is_root(): + return True + subprocess.call(self.__invocation + ['root']) + return self.is_root() + + def remount(self): + subprocess.call(self.__invocation + ['remount']) diff --git a/tools/skpbench/_adb_path.py b/tools/skpbench/_adb_path.py index 377ba12490..47eb7de17e 100644 --- a/tools/skpbench/_adb_path.py +++ b/tools/skpbench/_adb_path.py @@ -3,15 +3,15 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import _adb +from _adb import Adb import re import subprocess -__ADB_DEVICE_SERIAL = None +__ADB = None -def set_device_serial(device_serial): - global __ADB_DEVICE_SERIAL - __ADB_DEVICE_SERIAL = device_serial +def init(device_serial): + global __ADB + __ADB = Adb(device_serial) def join(*pathnames): return '/'.join(pathnames) @@ -20,14 +20,13 @@ def basename(pathname): return pathname.rsplit('/', maxsplit=1)[-1] def find_skps(skps): - escapedskps = [re.sub(r'([^a-zA-Z0-9_\*\?\[\!\]])', r'\\\1', x) # Keep globs. + escapedskps = [re.sub(r'([^a-zA-Z0-9_/\.\*\?\[\!\]])', r'\\\1', x) for x in skps] - pathnames = _adb.check(''' + return __ADB.check_lines('''\ for PATHNAME in %s; do if [ -d "$PATHNAME" ]; then ls "$PATHNAME"/*.skp else echo "$PATHNAME" fi - done''' % ' '.join(escapedskps), device_serial=__ADB_DEVICE_SERIAL) - return re.split('[\r\n]+', pathnames) + done''' % ' '.join(escapedskps)) diff --git a/tools/skpbench/_hardware.py b/tools/skpbench/_hardware.py new file mode 100644 index 0000000000..23cfc827bc --- /dev/null +++ b/tools/skpbench/_hardware.py @@ -0,0 +1,29 @@ +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import time + +class HardwareException(Exception): + def __init__(self, message, sleeptime=60): + Exception.__init__(self, message) + self.sleeptime = sleeptime + +class Hardware: + def __init__(self): + self.kick_in_time = 0 + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + pass + + def sanity_check(self): + '''Raises a HardwareException if any hardware state is not as expected.''' + pass + + def sleep(self, sleeptime): + '''Puts the hardware into a resting state for a fixed amount of time.''' + time.sleep(sleeptime) diff --git a/tools/skpbench/_hardware_android.py b/tools/skpbench/_hardware_android.py new file mode 100644 index 0000000000..a752ff5d93 --- /dev/null +++ b/tools/skpbench/_hardware_android.py @@ -0,0 +1,93 @@ +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import print_function +from _hardware import Hardware +import sys +import time + +class HardwareAndroid(Hardware): + def __init__(self, adb): + Hardware.__init__(self) + self.kick_in_time = 5 + self._adb = adb + self._is_root = self._adb.attempt_root() + if self._is_root: + self._adb.remount() + self._initial_airplane_mode = None + self._initial_location_providers = None + self._initial_ASLR = None + + def __enter__(self): + # turn on airplane mode. + self._initial_airplane_mode = \ + self._adb.check('settings get global airplane_mode_on') + self._adb.shell('settings put global airplane_mode_on 1') + + # disable GPS. + self._initial_location_providers = \ + self._adb.check('settings get secure location_providers_allowed') + self._initial_location_providers = \ + self._initial_location_providers.replace(',', ' ') + self._adb.shell('''\ + for PROVIDER in %s; do + settings put secure location_providers_allowed -$PROVIDER + done''' % self._initial_location_providers) + + if self._is_root: + # disable bluetooth, wifi, and mobile data. + # TODO: can we query these initial values? + self._adb.shell('''\ + service call bluetooth_manager 8 && + svc wifi disable && + svc data disable''') + + # kill the gui. + self._adb.shell('''\ + setprop ctl.stop media && + setprop ctl.stop zygote && + setprop ctl.stop surfaceflinger && + setprop ctl.stop drm''') + + # disable ASLR. + self._initial_ASLR = \ + self._adb.check('cat /proc/sys/kernel/randomize_va_space') + self._adb.shell('echo 0 > /proc/sys/kernel/randomize_va_space') + else: + print("WARNING: no adb root access; results may be unreliable.", + file=sys.stderr) + + return Hardware.__enter__(self) + + def __exit__(self, exception_type, exception_value, traceback): + Hardware.__exit__(self, exception_type, exception_value, traceback) + + if self._is_root: + # restore ASLR. + self._adb.shell('echo %s > /proc/sys/kernel/randomize_va_space' % + self._initial_ASLR) + + # revive the gui. + self._adb.shell('''\ + setprop ctl.start drm && + setprop ctl.start surfaceflinger && + setprop ctl.start zygote && + setprop ctl.start media''') + + # restore GPS (doesn't seem to work if we killed the gui). + self._adb.shell('''\ + for PROVIDER in %s; do + settings put secure location_providers_allowed +$PROVIDER + done''' % self._initial_location_providers) + + # restore airplane mode (doesn't seem to work if we killed the gui). + self._adb.shell('settings put global airplane_mode_on %s' % + self._initial_airplane_mode) + + def sanity_check(self): + Hardware.sanity_check(self) + + def sleep(self, sleeptime): + Hardware.sleep(self, sleeptime) diff --git a/tools/skpbench/parseskpbench.py b/tools/skpbench/parseskpbench.py index 21f46632df..2481e1d7e1 100755 --- a/tools/skpbench/parseskpbench.py +++ b/tools/skpbench/parseskpbench.py @@ -18,7 +18,7 @@ import urllib import urlparse import webbrowser -__argparse = ArgumentParser(description=""" +__argparse = ArgumentParser(description=''' Parses output files from skpbench.py into csv. @@ -31,21 +31,21 @@ This script can also be used to generate a Google sheet: (3) Run parseskpbench.py with the --open flag. -""") +''') __argparse.add_argument('-r', '--result', choices=['median', 'accum', 'max', 'min'], default='median', - help='result to use for cell values') + help="result to use for cell values") __argparse.add_argument('-f', '--force', action='store_true', help='silently ignore warnings') __argparse.add_argument('-o', '--open', action='store_true', - help='generate a temp file and open it (theoretically in a web browser)') + help="generate a temp file and open it (theoretically in a web browser)") __argparse.add_argument('-n', '--name', default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), - help='if using --open, a name for the temp file') + help="if using --open, a name for the temp file") __argparse.add_argument('sources', - nargs='+', help='source files with skpbench results ("-" for stdin)') + nargs='+', help="source files with skpbench results ('-' for stdin)") FLAGS = __argparse.parse_args() @@ -67,18 +67,18 @@ class Parser: if self.metric is None: self.metric = match.metric elif match.metric != self.metric: - raise ValueError('results have mismatched metrics (%s and %s)' % + raise ValueError("results have mismatched metrics (%s and %s)" % (self.metric, match.metric)) if self.samples is None: self.samples = match.samples elif not FLAGS.force and match.samples != self.samples: - raise ValueError('results have mismatched number of samples. ' - '(use --force to ignore)') + raise ValueError("results have mismatched number of samples. " + "(use --force to ignore)") if self.sample_ms is None: self.sample_ms = match.sample_ms elif not FLAGS.force and match.sample_ms != self.sample_ms: - raise ValueError('results have mismatched sampling times. ' - '(use --force to ignore)') + raise ValueError("results have mismatched sampling times. " + "(use --force to ignore)") if not match.config in self.configs: self.configs.append(match.config) self.rows[match.bench][match.config] = match.get_string(FLAGS.result) @@ -102,7 +102,7 @@ class Parser: elif FLAGS.force: outfile.write(',') else: - raise ValueError('%s: missing value for %s. (use --force to ignore)' % + raise ValueError("%s: missing value for %s. (use --force to ignore)" % (bench, config)) outfile.write('\n') diff --git a/tools/skpbench/skpbench.cpp b/tools/skpbench/skpbench.cpp index afe44b5c7b..063261a448 100644 --- a/tools/skpbench/skpbench.cpp +++ b/tools/skpbench/skpbench.cpp @@ -186,13 +186,13 @@ int main(int argc, char** argv) { SkCommandLineConfigArray configs; ParseConfigs(FLAGS_config, &configs); if (configs.count() != 1 || !(config = configs[0]->asConfigGpu())) { - exitf(ExitErr::kUsage, "invalid config %s; must specify one (and only one) GPU config", + exitf(ExitErr::kUsage, "invalid config %s, must specify one (and only one) GPU config", join(FLAGS_config).c_str()); } // Parse the skp. if (FLAGS_skp.count() != 1) { - exitf(ExitErr::kUsage, "invalid skp \"%s\"; one (and only one) skp must be specified.", + exitf(ExitErr::kUsage, "invalid skp %s, must specify (and only one) skp path name.", join(FLAGS_skp).c_str()); } const char* skpfile = FLAGS_skp[0]; @@ -206,10 +206,11 @@ int main(int argc, char** argv) { } int width = SkTMin(SkScalarCeilToInt(skp->cullRect().width()), 2048), height = SkTMin(SkScalarCeilToInt(skp->cullRect().height()), 2048); - if (FLAGS_verbosity >= 2 && + if (FLAGS_verbosity >= 3 && (width != skp->cullRect().width() || height != skp->cullRect().height())) { - fprintf(stderr, "NOTE: %s is too large (%ix%i); cropping to %ix%i.\n", - skpfile, SkScalarCeilToInt(skp->cullRect().width()), + fprintf(stderr, "%s is too large (%ix%i), cropping to %ix%i.\n", + SkOSPath::Basename(skpfile).c_str(), + SkScalarCeilToInt(skp->cullRect().width()), SkScalarCeilToInt(skp->cullRect().height()), width, height); } diff --git a/tools/skpbench/skpbench.py b/tools/skpbench/skpbench.py index 94d68b28e4..b84b0703a0 100755 --- a/tools/skpbench/skpbench.py +++ b/tools/skpbench/skpbench.py @@ -6,57 +6,60 @@ # found in the LICENSE file. from __future__ import print_function +from _adb import Adb from _benchresult import BenchResult +from _hardware import HardwareException, Hardware from argparse import ArgumentParser from queue import Queue -from threading import Thread +from threading import Thread, Timer import collections import glob import math import re import subprocess import sys +import time -__argparse = ArgumentParser(description=""" +__argparse = ArgumentParser(description=''' Executes the skpbench binary with various configs and skps. Also monitors the output in order to filter out and re-run results that have an unacceptable stddev. -""") +''') __argparse.add_argument('--adb', - action='store_true', help='execute skpbench over adb') + action='store_true', help="execute skpbench over adb") __argparse.add_argument('-s', '--device-serial', - help='if using adb, id of the specific device to target') + help="if using adb, id of the specific device to target") __argparse.add_argument('-p', '--path', - help='directory to execute ./skpbench from') + help="directory to execute ./skpbench from") __argparse.add_argument('-m', '--max-stddev', type=float, default=4, - help='initial max allowable relative standard deviation') + help="initial max allowable relative standard deviation") __argparse.add_argument('-x', '--suffix', - help='suffix to append on config (e.g. "_before", "_after")') + help="suffix to append on config (e.g. '_before', '_after')") __argparse.add_argument('-w','--write-path', - help='directory to save .png proofs to disk.') + help="directory to save .png proofs to disk.") __argparse.add_argument('-v','--verbosity', - type=int, default=0, help='level of verbosity (0=none to 5=debug)') + type=int, default=1, help="level of verbosity (0=none to 5=debug)") __argparse.add_argument('-n', '--samples', - type=int, help='number of samples to collect for each bench') + type=int, help="number of samples to collect for each bench") __argparse.add_argument('-d', '--sample-ms', - type=int, help='duration of each sample') + type=int, help="duration of each sample") __argparse.add_argument('--fps', - action='store_true', help='use fps instead of ms') + action='store_true', help="use fps instead of ms") __argparse.add_argument('-c', '--config', - default='gpu', help='comma- or space-separated list of GPU configs') + default='gpu', help="comma- or space-separated list of GPU configs") __argparse.add_argument('skps', nargs='+', - help='.skp files or directories to expand for .skp files') + help=".skp files or directories to expand for .skp files") FLAGS = __argparse.parse_args() if FLAGS.adb: import _adb_path as _path - _path.set_device_serial(FLAGS.device_serial) + _path.init(FLAGS.device_serial) else: import _os_path as _path @@ -66,12 +69,25 @@ class StddevException(Exception): class Message: READLINE = 0, - EXIT = 1 + POLL_HARDWARE = 1, + EXIT = 2 def __init__(self, message, value=None): self.message = message self.value = value -class SKPBench(Thread): +class SubprocessMonitor(Thread): + def __init__(self, queue, proc): + self._queue = queue + self._proc = proc + Thread.__init__(self) + + def run(self): + '''Runs on the background thread.''' + for line in iter(self._proc.stdout.readline, b''): + self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) + self._queue.put(Message(Message.EXIT)) + +class SKPBench: ARGV = ['skpbench', '--verbosity', str(FLAGS.verbosity)] if FLAGS.samples: ARGV.extend(['--samples', str(FLAGS.samples)]) @@ -97,86 +113,152 @@ class SKPBench(Thread): self.max_stddev = max_stddev self.best_result = best_result self._queue = Queue() - Thread.__init__(self) + self._proc = None + self._monitor = None + self._hw_poll_timer = None + + def __enter__(self): + return self + + def __exit__(self, exception_type, exception_value, traceback): + if self._proc: + self.terminate() + if self._hw_poll_timer: + self._hw_poll_timer.cancel() + + def execute(self, hardware): + hardware.sanity_check() + self._schedule_hardware_poll() + + commandline = self.ARGV + ['--config', self.config, + '--skp', self.skp, + '--suppressHeader', 'true'] + if FLAGS.write_path: + pngfile = _path.join(FLAGS.write_path, self.config, + _path.basename(self.skp) + '.png') + commandline.extend(['--png', pngfile]) + if (FLAGS.verbosity >= 4): + quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline] + print(' '.join(quoted), file=sys.stderr) + self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) + self._monitor = SubprocessMonitor(self._queue, self._proc) + self._monitor.start() - def execute(self): - self.start() while True: message = self._queue.get() if message.message == Message.READLINE: result = BenchResult.match(message.value) if result: - self.__process_result(result) + hardware.sanity_check() + self._process_result(result) else: print(message.value) sys.stdout.flush() continue + if message.message == Message.POLL_HARDWARE: + hardware.sanity_check() + self._schedule_hardware_poll() + continue if message.message == Message.EXIT: - self.join() + self._monitor.join() + self._proc.wait() + if self._proc.returncode != 0: + raise Exception("skpbench exited with nonzero exit code %i" % + self._proc.returncode) + self._proc = None break - def __process_result(self, result): + def _schedule_hardware_poll(self): + if self._hw_poll_timer: + self._hw_poll_timer.cancel() + self._hw_poll_timer = \ + Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE))) + self._hw_poll_timer.start() + + def _process_result(self, result): if not self.best_result or result.stddev <= self.best_result.stddev: self.best_result = result - elif FLAGS.verbosity >= 1: - print('NOTE: reusing previous result for %s/%s with lower stddev ' - '(%s%% instead of %s%%).' % + elif FLAGS.verbosity >= 2: + print("reusing previous result for %s/%s with lower stddev " + "(%s%% instead of %s%%)." % (result.config, result.bench, self.best_result.stddev, result.stddev), file=sys.stderr) if self.max_stddev and self.best_result.stddev > self.max_stddev: raise StddevException() - self.best_result.print_values(config_suffix=FLAGS.suffix) - def run(self): - """Called on the background thread. - - Launches and reads output from an skpbench process. - - """ - commandline = self.ARGV + ['--config', self.config, - '--skp', self.skp, - '--suppressHeader', 'true'] - if (FLAGS.write_path): - pngfile = _path.join(FLAGS.write_path, self.config, - _path.basename(self.skp) + '.png') - commandline.extend(['--png', pngfile]) - if (FLAGS.verbosity >= 3): - print(' '.join(commandline), file=sys.stderr) - proc = subprocess.Popen(commandline, stdout=subprocess.PIPE) - for line in iter(proc.stdout.readline, b''): - self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) - proc.wait() - self._queue.put(Message(Message.EXIT, proc.returncode)) + def terminate(self): + if self._proc: + self._proc.kill() + self._monitor.join() + self._proc.wait() + self._proc = None -def main(): +def run_benchmarks(configs, skps, hardware): SKPBench.print_header() - # Delimiter is "," or " ", skip if nested inside parens (e.g. gpu(a=b,c=d)). - DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' - configs = re.split(DELIMITER, FLAGS.config) - skps = _path.find_skps(FLAGS.skps) - benches = collections.deque([(skp, config, FLAGS.max_stddev) for skp in skps for config in configs]) while benches: benchargs = benches.popleft() - skpbench = SKPBench(*benchargs) - try: - skpbench.execute() - - except StddevException: - retry_max_stddev = skpbench.max_stddev * math.sqrt(2) - if FLAGS.verbosity >= 1: - print('NOTE: stddev too high for %s/%s (%s%%; max=%.2f%%). ' - 'Re-queuing with max=%.2f%%.' % - (skpbench.best_result.config, skpbench.best_result.bench, - skpbench.best_result.stddev, skpbench.max_stddev, - retry_max_stddev), - file=sys.stderr) - benches.append((skpbench.skp, skpbench.config, retry_max_stddev, - skpbench.best_result)) + with SKPBench(*benchargs) as skpbench: + try: + skpbench.execute(hardware) + if skpbench.best_result: + skpbench.best_result.print_values(config_suffix=FLAGS.suffix) + else: + print("WARNING: no result for %s with config %s" % + (skpbench.skp, skpbench.config), file=sys.stderr) + + except StddevException: + retry_max_stddev = skpbench.max_stddev * math.sqrt(2) + if FLAGS.verbosity >= 2: + print("stddev is too high for %s/%s (%s%%, max=%.2f%%), " + "re-queuing with max=%.2f%%." % + (skpbench.best_result.config, skpbench.best_result.bench, + skpbench.best_result.stddev, skpbench.max_stddev, + retry_max_stddev), + file=sys.stderr) + benches.append((skpbench.skp, skpbench.config, retry_max_stddev, + skpbench.best_result)) + + except HardwareException as exception: + skpbench.terminate() + naptime = max(hardware.kick_in_time, exception.sleeptime) + if FLAGS.verbosity >= 1: + print("%s; taking a %i second nap..." % + (exception.message, naptime), file=sys.stderr) + benches.appendleft(benchargs) # retry the same bench next time. + hardware.sleep(naptime - hardware.kick_in_time) + time.sleep(hardware.kick_in_time) + + +def main(): + # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)). + DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' + configs = re.split(DELIMITER, FLAGS.config) + skps = _path.find_skps(FLAGS.skps) + + if FLAGS.adb: + adb = Adb(FLAGS.device_serial) + model = adb.get_device_model() + if False: + pass # TODO: unique subclasses tailored to individual platforms. + else: + from _hardware_android import HardwareAndroid + print("WARNING: %s: don't know how to monitor this hardware; results " + "may be unreliable." % model, file=sys.stderr) + hardware = HardwareAndroid(adb) + else: + hardware = Hardware() + + with hardware: + if hardware.kick_in_time: + print("sleeping %i seconds to allow hardware settings to kick in..." % + hardware.kick_in_time, file=sys.stderr) + time.sleep(hardware.kick_in_time) + run_benchmarks(configs, skps, hardware) if __name__ == '__main__': |