Merge pull request #3487 from ctiller/flaky-timeouts

Add support for flaky timeouts

2 files changed, 21 insertions, 9 deletions

diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index fb3632e4ad..b9ec020f9e 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -83,6 +83,7 @@ _CLEAR_LINE = '\x1b[2K'
 _TAG_COLOR = {
     'FAILED': 'red',
     'FLAKE': 'purple',
+    'TIMEOUT_FLAKE': 'purple',
     'WARNING': 'yellow',
     'TIMEOUT': 'red',
     'PASSED': 'green',
@@ -133,7 +134,8 @@ class JobSpec(object):
   """Specifies what to run for a job."""
 
   def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
-               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0):
+               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0,
+               timeout_retries=0):
     """
     Arguments:
       cmdline: a list of arguments to pass as the command line
@@ -153,6 +155,7 @@ class JobSpec(object):
     self.shell = shell
     self.timeout_seconds = timeout_seconds
     self.flake_retries = flake_retries
+    self.timeout_retries = timeout_retries
 
   def identity(self):
     return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
@@ -176,6 +179,7 @@ class Job(object):
     self._xml_test = ET.SubElement(xml_report, 'testcase',
                                    name=self._spec.shortname) if xml_report is not None else None
     self._retries = 0
+    self._timeout_retries = 0
     message('START', spec.shortname, do_newline=self._travis)
     self.start()
 
@@ -223,19 +227,26 @@ class Job(object):
             ET.SubElement(self._xml_test, 'failure', message='Failure').text
       else:
         self._state = _SUCCESS
-        message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries),
-                do_newline=self._newline_on_success or self._travis)
+        message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
+                    self._spec.shortname, elapsed, self._retries, self._timeout_retries),
+            do_newline=self._newline_on_success or self._travis)
         if self._bin_hash:
           update_cache.finished(self._spec.identity(), self._bin_hash)
     elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
       self._tempfile.seek(0)
       stdout = self._tempfile.read()
       filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
-      message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
-      self.kill()
-      if self._xml_test is not None:
-        ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
-        ET.SubElement(self._xml_test, 'error', message='Timeout')
+      if self._timeout_retries < self._spec.timeout_retries:
+        message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True)
+        self._timeout_retries += 1
+        self._process.terminate()
+        self.start()
+      else:
+        message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
+        self.kill()
+        if self._xml_test is not None:
+          ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
+          ET.SubElement(self._xml_test, 'error', message='Timeout')
     return self._state
 
   def kill(self):
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index cbbb58d7e7..73b2b3640b 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -120,7 +120,8 @@ class ValgrindConfig(object):
                           self.args + cmdline,
                           shortname='valgrind %s' % cmdline[0],
                           hash_targets=None,
-                          flake_retries=5 if args.allow_flakes else 0)
+                          flake_retries=5 if args.allow_flakes else 0,
+                          timeout_retries=2 if args.allow_flakes else 0)
 
 
 def get_c_tests(travis, test_lang) :