diff options
Diffstat (limited to 'tools/run_tests/jobset.py')
| -rwxr-xr-x | tools/run_tests/jobset.py | 56 |
1 files changed, 45 insertions, 11 deletions
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 88d95027e2..48afbaf4cb 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -43,11 +43,23 @@ import time _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() _MAX_RESULT_SIZE = 8192 +def platform_string(): + if platform.system() == 'Windows': + return 'windows' + elif platform.system()[:7] == 'MSYS_NT': + return 'windows' + elif platform.system() == 'Darwin': + return 'mac' + elif platform.system() == 'Linux': + return 'linux' + else: + return 'posix' + # setup a signal handler so that signal.pause registers 'something' # when a child finishes # not using futures and threading to avoid a dependency on subprocess32 -if platform.system() == 'Windows': +if platform_string() == 'windows': pass else: have_alarm = False @@ -99,7 +111,7 @@ def message(tag, msg, explanatory_text=None, do_newline=False): message.old_tag = tag message.old_msg = msg try: - if platform.system() == 'Windows' or not sys.stdout.isatty(): + if platform_string() == 'windows' or not sys.stdout.isatty(): if explanatory_text: print explanatory_text print '%s: %s' % (tag, msg) @@ -166,6 +178,9 @@ class JobSpec(object): def __cmp__(self, other): return self.identity() == other.identity() + + def __repr__(self): + return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline) class JobResult(object): @@ -258,7 +273,7 @@ class Job(object): update_cache.finished(self._spec.identity(), self._bin_hash) elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds: if self._timeout_retries < self._spec.timeout_retries: - message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True) + message('TIMEOUT_FLAKE', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout, do_newline=True) self._timeout_retries += 1 self.result.num_failures += 1 self.result.retries = self._timeout_retries + self._retries @@ -267,7 +282,7 @@ class Job(object): self._process.terminate() self.start() else: - message('TIMEOUT', self._spec.shortname, stdout, do_newline=True) + message('TIMEOUT', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout, do_newline=True) self.kill() self.result.state = 'TIMEOUT' self.result.num_failures += 1 @@ -302,9 +317,13 @@ class Jobset(object): self._hashes = {} self._add_env = add_env self.resultset = {} - + self._remaining = None + + def set_remaining(self, remaining): + self._remaining = remaining + def get_num_failures(self): - return self._failures + return self._failures def start(self, spec): """Start a job. Return True on success, False on failure.""" @@ -357,9 +376,10 @@ class Jobset(object): self._running.remove(job) if dead: return if (not self._travis): - message('WAITING', '%d jobs running, %d complete, %d failed' % ( - len(self._running), self._completed, self._failures)) - if platform.system() == 'Windows': + rstr = '' if self._remaining is None else '%d queued, ' % self._remaining + message('WAITING', '%s%d jobs running, %d complete, %d failed' % ( + rstr, len(self._running), self._completed, self._failures)) + if platform_string() == 'windows': time.sleep(0.1) else: global have_alarm @@ -397,6 +417,17 @@ class NoCache(object): pass +def tag_remaining(xs): + staging = [] + for x in xs: + staging.append(x) + if len(staging) > 1000: + yield (staging.pop(0), None) + n = len(staging) + for i, x in enumerate(staging): + yield (x, n - i - 1) + + def run(cmdlines, check_cancelled=_never_cancelled, maxjobs=None, @@ -410,8 +441,11 @@ def run(cmdlines, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, cache if cache is not None else NoCache()) - for cmdline in cmdlines: + for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break - js.finish() + if remaining is not None: + js.set_remaining(remaining) + js.finish() return js.get_num_failures(), js.resultset + |