aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/skpbench/skpbench.py
blob: ef44577c5328c6ed885ed77f8288be84bdb231cd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
#!/usr/bin/env python

# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function
from _adb import Adb
from _benchresult import BenchResult
from _hardware import HardwareException, Hardware
from argparse import ArgumentParser
from multiprocessing import Queue
from threading import Thread, Timer
import collections
import glob
import math
import re
import subprocess
import sys
import time

__argparse = ArgumentParser(description="""

Executes the skpbench binary with various configs and skps.

Also monitors the output in order to filter out and re-run results that have an
unacceptable stddev.

""")

__argparse.add_argument('skpbench',
  help="path to the skpbench binary")
__argparse.add_argument('--adb',
  action='store_true', help="execute skpbench over adb")
__argparse.add_argument('-s', '--device-serial',
  help="if using adb, ID of the specific device to target "
       "(only required if more than 1 device is attached)")
__argparse.add_argument('-m', '--max-stddev',
  type=float, default=4,
  help="initial max allowable relative standard deviation")
__argparse.add_argument('-x', '--suffix',
  help="suffix to append on config (e.g. '_before', '_after')")
__argparse.add_argument('-w','--write-path',
  help="directory to save .png proofs to disk.")
__argparse.add_argument('-v','--verbosity',
  type=int, default=1, help="level of verbosity (0=none to 5=debug)")
__argparse.add_argument('-d', '--duration',
  type=int, help="number of milliseconds to run each benchmark")
__argparse.add_argument('-l', '--sample-ms',
  type=int, help="duration of a sample (minimum)")
__argparse.add_argument('--gpu',
  action='store_true',
  help="perform timing on the gpu clock instead of cpu (gpu work only)")
__argparse.add_argument('--fps',
  action='store_true', help="use fps instead of ms")
__argparse.add_argument('-c', '--config',
  default='gl', help="comma- or space-separated list of GPU configs")
__argparse.add_argument('-a', '--resultsfile',
  help="optional file to append results into")
__argparse.add_argument('skps',
  nargs='+',
  help=".skp files or directories to expand for .skp files")

FLAGS = __argparse.parse_args()
if FLAGS.adb:
  import _adb_path as _path
  _path.init(FLAGS.device_serial)
else:
  import _os_path as _path

def dump_commandline_if_verbose(commandline):
  if FLAGS.verbosity >= 5:
    quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline]
    print(' '.join(quoted), file=sys.stderr)


class StddevException(Exception):
  pass

class Message:
  READLINE = 0,
  POLL_HARDWARE = 1,
  EXIT = 2
  def __init__(self, message, value=None):
    self.message = message
    self.value = value

class SubprocessMonitor(Thread):
  def __init__(self, queue, proc):
    self._queue = queue
    self._proc = proc
    Thread.__init__(self)

  def run(self):
    """Runs on the background thread."""
    for line in iter(self._proc.stdout.readline, b''):
      self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip()))
    self._queue.put(Message(Message.EXIT))

class SKPBench:
  ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)]
  if FLAGS.duration:
    ARGV.extend(['--duration', str(FLAGS.duration)])
  if FLAGS.sample_ms:
    ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)])
  if FLAGS.gpu:
    ARGV.extend(['--gpuClock', 'true'])
  if FLAGS.fps:
    ARGV.extend(['--fps', 'true'])
  if FLAGS.adb:
    if FLAGS.device_serial is None:
      ARGV[:0] = ['adb', 'shell']
    else:
      ARGV[:0] = ['adb', '-s', FLAGS.device_serial, 'shell']

  @classmethod
  def get_header(cls, outfile=sys.stdout):
    commandline = cls.ARGV + ['--duration', '0']
    dump_commandline_if_verbose(commandline)
    out = subprocess.check_output(commandline, stderr=subprocess.STDOUT)
    return out.rstrip()

  @classmethod
  def run_warmup(cls, warmup_time, config):
    if not warmup_time:
      return
    print('running %i second warmup...' % warmup_time, file=sys.stderr)
    commandline = cls.ARGV + ['--duration', str(warmup_time * 1000),
                              '--config', config,
                              '--skp', 'warmup']
    dump_commandline_if_verbose(commandline)
    output = subprocess.check_output(commandline, stderr=subprocess.STDOUT)

    # validate the warmup run output.
    for line in output.decode('utf-8').split('\n'):
      match = BenchResult.match(line.rstrip())
      if match and match.bench == 'warmup':
        return
    raise Exception('Invalid warmup output:\n%s' % output)

  def __init__(self, skp, config, max_stddev, best_result=None):
    self.skp = skp
    self.config = config
    self.max_stddev = max_stddev
    self.best_result = best_result
    self._queue = Queue()
    self._proc = None
    self._monitor = None
    self._hw_poll_timer = None

  def __enter__(self):
    return self

  def __exit__(self, exception_type, exception_value, traceback):
    if self._proc:
      self.terminate()
    if self._hw_poll_timer:
      self._hw_poll_timer.cancel()

  def execute(self, hardware):
    hardware.sanity_check()
    self._schedule_hardware_poll()

    commandline = self.ARGV + ['--config', self.config,
                               '--skp', self.skp,
                               '--suppressHeader', 'true']
    if FLAGS.write_path:
      pngfile = _path.join(FLAGS.write_path, self.config,
                           _path.basename(self.skp) + '.png')
      commandline.extend(['--png', pngfile])
    dump_commandline_if_verbose(commandline)
    self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
    self._monitor = SubprocessMonitor(self._queue, self._proc)
    self._monitor.start()

    while True:
      message = self._queue.get()
      if message.message == Message.READLINE:
        result = BenchResult.match(message.value)
        if result:
          hardware.sanity_check()
          self._process_result(result)
        elif hardware.filter_line(message.value):
          print(message.value, file=sys.stderr)
        continue
      if message.message == Message.POLL_HARDWARE:
        hardware.sanity_check()
        self._schedule_hardware_poll()
        continue
      if message.message == Message.EXIT:
        self._monitor.join()
        self._proc.wait()
        if self._proc.returncode != 0:
          raise Exception("skpbench exited with nonzero exit code %i" %
                          self._proc.returncode)
        self._proc = None
        break

  def _schedule_hardware_poll(self):
    if self._hw_poll_timer:
      self._hw_poll_timer.cancel()
    self._hw_poll_timer = \
      Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE)))
    self._hw_poll_timer.start()

  def _process_result(self, result):
    if not self.best_result or result.stddev <= self.best_result.stddev:
      self.best_result = result
    elif FLAGS.verbosity >= 2:
      print("reusing previous result for %s/%s with lower stddev "
            "(%s%% instead of %s%%)." %
            (result.config, result.bench, self.best_result.stddev,
             result.stddev), file=sys.stderr)
    if self.max_stddev and self.best_result.stddev > self.max_stddev:
      raise StddevException()

  def terminate(self):
    if self._proc:
      self._proc.terminate()
      self._monitor.join()
      self._proc.wait()
      self._proc = None

def emit_result(line, resultsfile=None):
  print(line)
  sys.stdout.flush()
  if resultsfile:
    print(line, file=resultsfile)
    resultsfile.flush()

def run_benchmarks(configs, skps, hardware, resultsfile=None):
  emit_result(SKPBench.get_header(), resultsfile)
  benches = collections.deque([(skp, config, FLAGS.max_stddev)
                               for skp in skps
                               for config in configs])
  while benches:
    benchargs = benches.popleft()
    with SKPBench(*benchargs) as skpbench:
      try:
        skpbench.execute(hardware)
        if skpbench.best_result:
          emit_result(skpbench.best_result.format(FLAGS.suffix), resultsfile)
        else:
          print("WARNING: no result for %s with config %s" %
                (skpbench.skp, skpbench.config), file=sys.stderr)

      except StddevException:
        retry_max_stddev = skpbench.max_stddev * math.sqrt(2)
        if FLAGS.verbosity >= 1:
          print("stddev is too high for %s/%s (%s%%, max=%.2f%%), "
                "re-queuing with max=%.2f%%." %
                (skpbench.best_result.config, skpbench.best_result.bench,
                 skpbench.best_result.stddev, skpbench.max_stddev,
                 retry_max_stddev),
                file=sys.stderr)
        benches.append((skpbench.skp, skpbench.config, retry_max_stddev,
                        skpbench.best_result))

      except HardwareException as exception:
        skpbench.terminate()
        if FLAGS.verbosity >= 4:
          hardware.print_debug_diagnostics()
        if FLAGS.verbosity >= 1:
          print("%s; taking a %i second nap..." %
                (exception.message, exception.sleeptime), file=sys.stderr)
        benches.appendleft(benchargs) # retry the same bench next time.
        hardware.sleep(exception.sleeptime)
        if FLAGS.verbosity >= 4:
          hardware.print_debug_diagnostics()
        SKPBench.run_warmup(hardware.warmup_time, configs[0])

def main():
  # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)).
  DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))'
  configs = re.split(DELIMITER, FLAGS.config)
  skps = _path.find_skps(FLAGS.skps)

  if FLAGS.adb:
    adb = Adb(FLAGS.device_serial, echo=(FLAGS.verbosity >= 5))
    model = adb.check('getprop ro.product.model').strip()
    if model == 'Pixel C':
      from _hardware_pixel_c import HardwarePixelC
      hardware = HardwarePixelC(adb)
    elif model == 'Nexus 6P':
      from _hardware_nexus_6p import HardwareNexus6P
      hardware = HardwareNexus6P(adb)
    else:
      from _hardware_android import HardwareAndroid
      print("WARNING: %s: don't know how to monitor this hardware; results "
            "may be unreliable." % model, file=sys.stderr)
      hardware = HardwareAndroid(adb)
  else:
    hardware = Hardware()

  with hardware:
    SKPBench.run_warmup(hardware.warmup_time, configs[0])
    if FLAGS.resultsfile:
      with open(FLAGS.resultsfile, mode='a+') as resultsfile:
        run_benchmarks(configs, skps, hardware, resultsfile=resultsfile)
    else:
      run_benchmarks(configs, skps, hardware)


if __name__ == '__main__':
  main()