diff options
| -rwxr-xr-x | tools/run_tests/jobset.py | 23 |
1 files changed, 17 insertions, 6 deletions
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 0c4d1b8143..88d95027e2 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -203,12 +203,23 @@ class Job(object): env.update(self._spec.environ) env.update(self._add_env) self._start = time.time() - self._process = subprocess.Popen(args=self._spec.cmdline, - stderr=subprocess.STDOUT, - stdout=self._tempfile, - cwd=self._spec.cwd, - shell=self._spec.shell, - env=env) + try_start = lambda: subprocess.Popen(args=self._spec.cmdline, + stderr=subprocess.STDOUT, + stdout=self._tempfile, + cwd=self._spec.cwd, + shell=self._spec.shell, + env=env) + delay = 0.3 + for i in range(0, 4): + try: + self._process = try_start() + break + except OSError: + message('WARNING', 'Failed to start %s, retrying in %f seconds' % (self._spec.shortname, delay)) + time.sleep(delay) + delay *= 2 + else: + self._process = try_start() self._state = _RUNNING def state(self, update_cache): |