Merge remote-tracking branch 'upstream/master' into cares_buildin

8 files changed, 1249 insertions, 0 deletions

diff --git a/tools/run_tests/python_utils/__init__.py b/tools/run_tests/python_utils/__init__.py
new file mode 100644
index 0000000000..100a624dc9
--- /dev/null
+++ b/tools/run_tests/python_utils/__init__.py
@@ -0,0 +1,28 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/tools/run_tests/python_utils/antagonist.py b/tools/run_tests/python_utils/antagonist.py
new file mode 100755
index 0000000000..857addfb38
--- /dev/null
+++ b/tools/run_tests/python_utils/antagonist.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python2.7
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""This is used by run_tests.py to create cpu load on a machine"""
+
+while True:
+	pass
diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py
new file mode 100755
index 0000000000..0869c5cee9
--- /dev/null
+++ b/tools/run_tests/python_utils/dockerjob.py
@@ -0,0 +1,124 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Helpers to run docker instances as jobs."""
+
+from __future__ import print_function
+
+import tempfile
+import time
+import uuid
+import os
+import subprocess
+
+import jobset
+
+_DEVNULL = open(os.devnull, 'w')
+
+
+def random_name(base_name):
+  """Randomizes given base name."""
+  return '%s_%s' % (base_name, uuid.uuid4())
+
+
+def docker_kill(cid):
+  """Kills a docker container. Returns True if successful."""
+  return subprocess.call(['docker','kill', str(cid)],
+                         stdin=subprocess.PIPE,
+                         stdout=_DEVNULL,
+                         stderr=subprocess.STDOUT) == 0
+
+
+def docker_mapped_port(cid, port, timeout_seconds=15):
+  """Get port mapped to internal given internal port for given container."""
+  started = time.time()
+  while time.time() - started < timeout_seconds:
+    try:
+      output = subprocess.check_output('docker port %s %s' % (cid, port),
+                                       stderr=_DEVNULL,
+                                       shell=True)
+      return int(output.split(':', 2)[1])
+    except subprocess.CalledProcessError as e:
+      pass
+  raise Exception('Failed to get exposed port %s for container %s.' %
+                  (port, cid))
+
+
+def finish_jobs(jobs):
+  """Kills given docker containers and waits for corresponding jobs to finish"""
+  for job in jobs:
+    job.kill(suppress_failure=True)
+
+  while any(job.is_running() for job in jobs):
+    time.sleep(1)
+
+
+def image_exists(image):
+  """Returns True if given docker image exists."""
+  return subprocess.call(['docker','inspect', image],
+                         stdin=subprocess.PIPE,
+                         stdout=_DEVNULL,
+                         stderr=subprocess.STDOUT) == 0
+
+
+def remove_image(image, skip_nonexistent=False, max_retries=10):
+  """Attempts to remove docker image with retries."""
+  if skip_nonexistent and not image_exists(image):
+    return True
+  for attempt in range(0, max_retries):
+    if subprocess.call(['docker','rmi', '-f', image],
+                       stdin=subprocess.PIPE,
+                       stdout=_DEVNULL,
+                       stderr=subprocess.STDOUT) == 0:
+      return True
+    time.sleep(2)
+  print('Failed to remove docker image %s' % image)
+  return False
+
+
+class DockerJob:
+  """Encapsulates a job"""
+
+  def __init__(self, spec):
+    self._spec = spec
+    self._job = jobset.Job(spec, newline_on_success=True, travis=True, add_env={})
+    self._container_name = spec.container_name
+
+  def mapped_port(self, port):
+    return docker_mapped_port(self._container_name, port)
+
+  def kill(self, suppress_failure=False):
+    """Sends kill signal to the container."""
+    if suppress_failure:
+      self._job.suppress_failure_message()
+    return docker_kill(self._container_name)
+
+  def is_running(self):
+    """Polls a job and returns True if given job is still running."""
+    return self._job.state() == jobset._RUNNING
diff --git a/tools/run_tests/python_utils/filter_pull_request_tests.py b/tools/run_tests/python_utils/filter_pull_request_tests.py
new file mode 100644
index 0000000000..ca1d6d4eb5
--- /dev/null
+++ b/tools/run_tests/python_utils/filter_pull_request_tests.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python2.7
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Filter out tests based on file differences compared to merge target branch"""
+
+import re
+from subprocess import check_output
+
+
+class TestSuite:
+  """
+  Contains label to identify job as belonging to this test suite and
+  triggers to identify if changed files are relevant
+  """
+  def __init__(self, labels):
+    """
+    Build TestSuite to group tests based on labeling
+    :param label: strings that should match a jobs's platform, config, language, or test group
+    """
+    self.triggers = []
+    self.labels = labels
+
+  def add_trigger(self, trigger):
+    """
+    Add a regex to list of triggers that determine if a changed file should run tests
+    :param trigger: regex matching file relevant to tests
+    """
+    self.triggers.append(trigger)
+
+
+# Create test suites
+_CORE_TEST_SUITE = TestSuite(['c'])
+_CPP_TEST_SUITE = TestSuite(['c++'])
+_CSHARP_TEST_SUITE = TestSuite(['csharp'])
+_NODE_TEST_SUITE = TestSuite(['node'])
+_OBJC_TEST_SUITE = TestSuite(['objc'])
+_PHP_TEST_SUITE = TestSuite(['php', 'php7'])
+_PYTHON_TEST_SUITE = TestSuite(['python'])
+_RUBY_TEST_SUITE = TestSuite(['ruby'])
+_LINUX_TEST_SUITE = TestSuite(['linux'])
+_WINDOWS_TEST_SUITE = TestSuite(['windows'])
+_MACOS_TEST_SUITE = TestSuite(['macos'])
+_ALL_TEST_SUITES = [_CORE_TEST_SUITE, _CPP_TEST_SUITE, _CSHARP_TEST_SUITE,
+                    _NODE_TEST_SUITE, _OBJC_TEST_SUITE, _PHP_TEST_SUITE,
+                    _PYTHON_TEST_SUITE, _RUBY_TEST_SUITE, _LINUX_TEST_SUITE,
+                    _WINDOWS_TEST_SUITE, _MACOS_TEST_SUITE]
+
+# Dictionary of whitelistable files where the key is a regex matching changed files
+# and the value is a list of tests that should be run. An empty list means that
+# the changed files should not trigger any tests. Any changed file that does not
+# match any of these regexes will trigger all tests
+# DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING (be careful even if you do)
+_WHITELIST_DICT = {
+  '^doc/': [],
+  '^examples/': [],
+  '^include/grpc\+\+/': [_CPP_TEST_SUITE],
+  '^summerofcode/': [],
+  '^src/cpp/': [_CPP_TEST_SUITE],
+  '^src/csharp/': [_CSHARP_TEST_SUITE],
+  '^src/node/': [_NODE_TEST_SUITE],
+  '^src/objective\-c/': [_OBJC_TEST_SUITE],
+  '^src/php/': [_PHP_TEST_SUITE],
+  '^src/python/': [_PYTHON_TEST_SUITE],
+  '^src/ruby/': [_RUBY_TEST_SUITE],
+  '^templates/': [],
+  '^test/core/': [_CORE_TEST_SUITE],
+  '^test/cpp/': [_CPP_TEST_SUITE],
+  '^test/distrib/cpp/': [_CPP_TEST_SUITE],
+  '^test/distrib/csharp/': [_CSHARP_TEST_SUITE],
+  '^test/distrib/node/': [_NODE_TEST_SUITE],
+  '^test/distrib/php/': [_PHP_TEST_SUITE],
+  '^test/distrib/python/': [_PYTHON_TEST_SUITE],
+  '^test/distrib/ruby/': [_RUBY_TEST_SUITE],
+  '^vsprojects/': [_WINDOWS_TEST_SUITE],
+  'binding\.gyp$': [_NODE_TEST_SUITE],
+  'composer\.json$': [_PHP_TEST_SUITE],
+  'config\.m4$': [_PHP_TEST_SUITE],
+  'CONTRIBUTING\.md$': [],
+  'Gemfile$': [_RUBY_TEST_SUITE],
+  'grpc\.def$': [_WINDOWS_TEST_SUITE],
+  'grpc\.gemspec$': [_RUBY_TEST_SUITE],
+  'gRPC\.podspec$': [_OBJC_TEST_SUITE],
+  'gRPC\-Core\.podspec$': [_OBJC_TEST_SUITE],
+  'gRPC\-ProtoRPC\.podspec$': [_OBJC_TEST_SUITE],
+  'gRPC\-RxLibrary\.podspec$': [_OBJC_TEST_SUITE],
+  'INSTALL\.md$': [],
+  'LICENSE$': [],
+  'MANIFEST\.md$': [],
+  'package\.json$': [_PHP_TEST_SUITE],
+  'package\.xml$': [_PHP_TEST_SUITE],
+  'PATENTS$': [],
+  'PYTHON\-MANIFEST\.in$': [_PYTHON_TEST_SUITE],
+  'README\.md$': [],
+  'requirements\.txt$': [_PYTHON_TEST_SUITE],
+  'setup\.cfg$': [_PYTHON_TEST_SUITE],
+  'setup\.py$': [_PYTHON_TEST_SUITE]
+}
+
+# Add all triggers to their respective test suites
+for trigger, test_suites in _WHITELIST_DICT.iteritems():
+  for test_suite in test_suites:
+    test_suite.add_trigger(trigger)
+
+
+def _get_changed_files(base_branch):
+  """
+  Get list of changed files between current branch and base of target merge branch
+  """
+  # Get file changes between branch and merge-base of specified branch
+  # Not combined to be Windows friendly
+  base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip()
+  return check_output(["git", "diff", base_commit, "--name-only"]).splitlines()
+
+
+def _can_skip_tests(file_names, triggers):
+  """
+  Determines if tests are skippable based on if all files do not match list of regexes
+  :param file_names: list of changed files generated by _get_changed_files()
+  :param triggers: list of regexes matching file name that indicates tests should be run
+  :return: safe to skip tests
+  """
+  for file_name in file_names:
+    if any(re.match(trigger, file_name) for trigger in triggers):
+      return False
+  return True
+
+
+def _remove_irrelevant_tests(tests, skippable_labels):
+  """
+  Filters out tests by config or language - will not remove sanitizer tests
+  :param tests: list of all tests generated by run_tests_matrix.py
+  :param skippable_labels: list of languages and platforms with skippable tests
+  :return: list of relevant tests
+  """
+  # test.labels[0] is platform and test.labels[2] is language
+  # We skip a test if both are considered safe to skip
+  return [test for test in tests if test.labels[0] not in skippable_labels or \
+          test.labels[2] not in skippable_labels]
+
+
+def filter_tests(tests, base_branch):
+  """
+  Filters out tests that are safe to ignore
+  :param tests: list of all tests generated by run_tests_matrix.py
+  :return: list of relevant tests
+  """
+  print('Finding file differences between gRPC %s branch and pull request...\n' % base_branch)
+  changed_files = _get_changed_files(base_branch)
+  for changed_file in changed_files:
+    print('  %s' % changed_file)
+  print('')
+
+  # Regex that combines all keys in _WHITELIST_DICT
+  all_triggers = "(" + ")|(".join(_WHITELIST_DICT.keys()) + ")"
+  # Check if all tests have to be run
+  for changed_file in changed_files:
+    if not re.match(all_triggers, changed_file):
+      return(tests)
+  # Figure out which language and platform tests to run
+  skippable_labels = []
+  for test_suite in _ALL_TEST_SUITES:
+    if _can_skip_tests(changed_files, test_suite.triggers):
+      for label in test_suite.labels:
+        print('  %s tests safe to skip' % label)
+        skippable_labels.append(label)
+  tests = _remove_irrelevant_tests(tests, skippable_labels)
+  return tests
+
diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py
new file mode 100755
index 0000000000..7b2c62d1a2
--- /dev/null
+++ b/tools/run_tests/python_utils/jobset.py
@@ -0,0 +1,493 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Run a group of subprocesses and then finish."""
+
+from __future__ import print_function
+
+import multiprocessing
+import os
+import platform
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+
+
+# cpu cost measurement
+measure_cpu_costs = False
+
+
+_DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count()
+_MAX_RESULT_SIZE = 8192
+
+
+# NOTE: If you change this, please make sure to test reviewing the
+# github PR with http://reviewable.io, which is known to add UTF-8
+# characters to the PR description, which leak into the environment here
+# and cause failures.
+def strip_non_ascii_chars(s):
+  return ''.join(c for c in s if ord(c) < 128)
+
+
+def sanitized_environment(env):
+  sanitized = {}
+  for key, value in env.items():
+    sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value)
+  return sanitized
+
+
+def platform_string():
+  if platform.system() == 'Windows':
+    return 'windows'
+  elif platform.system()[:7] == 'MSYS_NT':
+    return 'windows'
+  elif platform.system() == 'Darwin':
+    return 'mac'
+  elif platform.system() == 'Linux':
+    return 'linux'
+  else:
+    return 'posix'
+
+
+# setup a signal handler so that signal.pause registers 'something'
+# when a child finishes
+# not using futures and threading to avoid a dependency on subprocess32
+if platform_string() == 'windows':
+  pass
+else:
+  have_alarm = False
+  def alarm_handler(unused_signum, unused_frame):
+    global have_alarm
+    have_alarm = False
+
+  signal.signal(signal.SIGCHLD, lambda unused_signum, unused_frame: None)
+  signal.signal(signal.SIGALRM, alarm_handler)
+
+
+_SUCCESS = object()
+_FAILURE = object()
+_RUNNING = object()
+_KILLED = object()
+
+
+_COLORS = {
+    'red': [ 31, 0 ],
+    'green': [ 32, 0 ],
+    'yellow': [ 33, 0 ],
+    'lightgray': [ 37, 0],
+    'gray': [ 30, 1 ],
+    'purple': [ 35, 0 ],
+    'cyan': [ 36, 0 ]
+    }
+
+
+_BEGINNING_OF_LINE = '\x1b[0G'
+_CLEAR_LINE = '\x1b[2K'
+
+
+_TAG_COLOR = {
+    'FAILED': 'red',
+    'FLAKE': 'purple',
+    'TIMEOUT_FLAKE': 'purple',
+    'WARNING': 'yellow',
+    'TIMEOUT': 'red',
+    'PASSED': 'green',
+    'START': 'gray',
+    'WAITING': 'yellow',
+    'SUCCESS': 'green',
+    'IDLE': 'gray',
+    'SKIPPED': 'cyan'
+    }
+
+
+def message(tag, msg, explanatory_text=None, do_newline=False):
+  if message.old_tag == tag and message.old_msg == msg and not explanatory_text:
+    return
+  message.old_tag = tag
+  message.old_msg = msg
+  try:
+    if platform_string() == 'windows' or not sys.stdout.isatty():
+      if explanatory_text:
+        print(explanatory_text)
+      print('%s: %s' % (tag, msg))
+    else:
+      sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % (
+          _BEGINNING_OF_LINE,
+          _CLEAR_LINE,
+          '\n%s' % explanatory_text if explanatory_text is not None else '',
+          _COLORS[_TAG_COLOR[tag]][1],
+          _COLORS[_TAG_COLOR[tag]][0],
+          tag,
+          msg,
+          '\n' if do_newline or explanatory_text is not None else ''))
+    sys.stdout.flush()
+  except:
+    pass
+
+message.old_tag = ''
+message.old_msg = ''
+
+def which(filename):
+  if '/' in filename:
+    return filename
+  for path in os.environ['PATH'].split(os.pathsep):
+    if os.path.exists(os.path.join(path, filename)):
+      return os.path.join(path, filename)
+  raise Exception('%s not found' % filename)
+
+
+class JobSpec(object):
+  """Specifies what to run for a job."""
+
+  def __init__(self, cmdline, shortname=None, environ=None,
+               cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0,
+               timeout_retries=0, kill_handler=None, cpu_cost=1.0,
+               verbose_success=False):
+    """
+    Arguments:
+      cmdline: a list of arguments to pass as the command line
+      environ: a dictionary of environment variables to set in the child process
+      kill_handler: a handler that will be called whenever job.kill() is invoked
+      cpu_cost: number of cores per second this job needs
+    """
+    if environ is None:
+      environ = {}
+    self.cmdline = cmdline
+    self.environ = environ
+    self.shortname = cmdline[0] if shortname is None else shortname
+    self.cwd = cwd
+    self.shell = shell
+    self.timeout_seconds = timeout_seconds
+    self.flake_retries = flake_retries
+    self.timeout_retries = timeout_retries
+    self.kill_handler = kill_handler
+    self.cpu_cost = cpu_cost
+    self.verbose_success = verbose_success
+
+  def identity(self):
+    return '%r %r' % (self.cmdline, self.environ)
+
+  def __hash__(self):
+    return hash(self.identity())
+
+  def __cmp__(self, other):
+    return self.identity() == other.identity()
+
+  def __repr__(self):
+    return 'JobSpec(shortname=%s, cmdline=%s)' % (self.shortname, self.cmdline)
+
+
+class JobResult(object):
+  def __init__(self):
+    self.state = 'UNKNOWN'
+    self.returncode = -1
+    self.elapsed_time = 0
+    self.num_failures = 0
+    self.retries = 0
+    self.message = ''
+
+
+class Job(object):
+  """Manages one job."""
+
+  def __init__(self, spec, newline_on_success, travis, add_env,
+               quiet_success=False):
+    self._spec = spec
+    self._newline_on_success = newline_on_success
+    self._travis = travis
+    self._add_env = add_env.copy()
+    self._retries = 0
+    self._timeout_retries = 0
+    self._suppress_failure_message = False
+    self._quiet_success = quiet_success
+    if not self._quiet_success:
+      message('START', spec.shortname, do_newline=self._travis)
+    self.result = JobResult()
+    self.start()
+
+  def GetSpec(self):
+    return self._spec
+
+  def start(self):
+    self._tempfile = tempfile.TemporaryFile()
+    env = dict(os.environ)
+    env.update(self._spec.environ)
+    env.update(self._add_env)
+    env = sanitized_environment(env)
+    self._start = time.time()
+    cmdline = self._spec.cmdline
+    if measure_cpu_costs:
+      cmdline = ['time', '--portability'] + cmdline
+    try_start = lambda: subprocess.Popen(args=cmdline,
+                                         stderr=subprocess.STDOUT,
+                                         stdout=self._tempfile,
+                                         cwd=self._spec.cwd,
+                                         shell=self._spec.shell,
+                                         env=env)
+    delay = 0.3
+    for i in range(0, 4):
+      try:
+        self._process = try_start()
+        break
+      except OSError:
+        message('WARNING', 'Failed to start %s, retrying in %f seconds' % (self._spec.shortname, delay))
+        time.sleep(delay)
+        delay *= 2
+    else:
+      self._process = try_start()
+    self._state = _RUNNING
+
+  def state(self):
+    """Poll current state of the job. Prints messages at completion."""
+    def stdout(self=self):
+      self._tempfile.seek(0)
+      stdout = self._tempfile.read()
+      self.result.message = stdout[-_MAX_RESULT_SIZE:]
+      return stdout
+    if self._state == _RUNNING and self._process.poll() is not None:
+      elapsed = time.time() - self._start
+      self.result.elapsed_time = elapsed
+      if self._process.returncode != 0:
+        if self._retries < self._spec.flake_retries:
+          message('FLAKE', '%s [ret=%d, pid=%d]' % (
+            self._spec.shortname, self._process.returncode, self._process.pid),
+            stdout(), do_newline=True)
+          self._retries += 1
+          self.result.num_failures += 1
+          self.result.retries = self._timeout_retries + self._retries
+          self.start()
+        else:
+          self._state = _FAILURE
+          if not self._suppress_failure_message:
+            message('FAILED', '%s [ret=%d, pid=%d]' % (
+                self._spec.shortname, self._process.returncode, self._process.pid),
+                stdout(), do_newline=True)
+          self.result.state = 'FAILED'
+          self.result.num_failures += 1
+          self.result.returncode = self._process.returncode
+      else:
+        self._state = _SUCCESS
+        measurement = ''
+        if measure_cpu_costs:
+          m = re.search(r'real ([0-9.]+)\nuser ([0-9.]+)\nsys ([0-9.]+)', stdout())
+          real = float(m.group(1))
+          user = float(m.group(2))
+          sys = float(m.group(3))
+          if real > 0.5:
+            cores = (user + sys) / real
+            measurement = '; cpu_cost=%.01f; estimated=%.01f' % (cores, self._spec.cpu_cost)
+        if not self._quiet_success:
+          message('PASSED', '%s [time=%.1fsec; retries=%d:%d%s]' % (
+              self._spec.shortname, elapsed, self._retries, self._timeout_retries, measurement),
+              stdout() if self._spec.verbose_success else None,
+              do_newline=self._newline_on_success or self._travis)
+        self.result.state = 'PASSED'
+    elif (self._state == _RUNNING and
+          self._spec.timeout_seconds is not None and
+          time.time() - self._start > self._spec.timeout_seconds):
+      if self._timeout_retries < self._spec.timeout_retries:
+        message('TIMEOUT_FLAKE', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout(), do_newline=True)
+        self._timeout_retries += 1
+        self.result.num_failures += 1
+        self.result.retries = self._timeout_retries + self._retries
+        if self._spec.kill_handler:
+          self._spec.kill_handler(self)
+        self._process.terminate()
+        self.start()
+      else:
+        message('TIMEOUT', '%s [pid=%d]' % (self._spec.shortname, self._process.pid), stdout(), do_newline=True)
+        self.kill()
+        self.result.state = 'TIMEOUT'
+        self.result.num_failures += 1
+    return self._state
+
+  def kill(self):
+    if self._state == _RUNNING:
+      self._state = _KILLED
+      if self._spec.kill_handler:
+        self._spec.kill_handler(self)
+      self._process.terminate()
+
+  def suppress_failure_message(self):
+    self._suppress_failure_message = True
+
+
+class Jobset(object):
+  """Manages one run of jobs."""
+
+  def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
+               stop_on_failure, add_env, quiet_success):
+    self._running = set()
+    self._check_cancelled = check_cancelled
+    self._cancelled = False
+    self._failures = 0
+    self._completed = 0
+    self._maxjobs = maxjobs
+    self._newline_on_success = newline_on_success
+    self._travis = travis
+    self._stop_on_failure = stop_on_failure
+    self._add_env = add_env
+    self._quiet_success = quiet_success
+    self.resultset = {}
+    self._remaining = None
+    self._start_time = time.time()
+
+  def set_remaining(self, remaining):
+    self._remaining = remaining
+
+  def get_num_failures(self):
+    return self._failures
+
+  def cpu_cost(self):
+    c = 0
+    for job in self._running:
+      c += job._spec.cpu_cost
+    return c
+
+  def start(self, spec):
+    """Start a job. Return True on success, False on failure."""
+    while True:
+      if self.cancelled(): return False
+      current_cpu_cost = self.cpu_cost()
+      if current_cpu_cost == 0: break
+      if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break
+      self.reap()
+    if self.cancelled(): return False
+    job = Job(spec,
+              self._newline_on_success,
+              self._travis,
+              self._add_env,
+              self._quiet_success)
+    self._running.add(job)
+    if job.GetSpec().shortname not in self.resultset:
+      self.resultset[job.GetSpec().shortname] = []
+    return True
+
+  def reap(self):
+    """Collect the dead jobs."""
+    while self._running:
+      dead = set()
+      for job in self._running:
+        st = job.state()
+        if st == _RUNNING: continue
+        if st == _FAILURE or st == _KILLED:
+          self._failures += 1
+          if self._stop_on_failure:
+            self._cancelled = True
+            for job in self._running:
+              job.kill()
+        dead.add(job)
+        break
+      for job in dead:
+        self._completed += 1
+        if not self._quiet_success or job.result.state != 'PASSED':
+          self.resultset[job.GetSpec().shortname].append(job.result)
+        self._running.remove(job)
+      if dead: return
+      if not self._travis and platform_string() != 'windows':
+        rstr = '' if self._remaining is None else '%d queued, ' % self._remaining
+        if self._remaining is not None and self._completed > 0:
+          now = time.time()
+          sofar = now - self._start_time
+          remaining = sofar / self._completed * (self._remaining + len(self._running))
+          rstr = 'ETA %.1f sec; %s' % (remaining, rstr)
+        message('WAITING', '%s%d jobs running, %d complete, %d failed' % (
+            rstr, len(self._running), self._completed, self._failures))
+      if platform_string() == 'windows':
+        time.sleep(0.1)
+      else:
+        global have_alarm
+        if not have_alarm:
+          have_alarm = True
+          signal.alarm(10)
+        signal.pause()
+
+  def cancelled(self):
+    """Poll for cancellation."""
+    if self._cancelled: return True
+    if not self._check_cancelled(): return False
+    for job in self._running:
+      job.kill()
+    self._cancelled = True
+    return True
+
+  def finish(self):
+    while self._running:
+      if self.cancelled(): pass  # poll cancellation
+      self.reap()
+    return not self.cancelled() and self._failures == 0
+
+
+def _never_cancelled():
+  return False
+
+
+def tag_remaining(xs):
+  staging = []
+  for x in xs:
+    staging.append(x)
+    if len(staging) > 5000:
+      yield (staging.pop(0), None)
+  n = len(staging)
+  for i, x in enumerate(staging):
+    yield (x, n - i - 1)
+
+
+def run(cmdlines,
+        check_cancelled=_never_cancelled,
+        maxjobs=None,
+        newline_on_success=False,
+        travis=False,
+        infinite_runs=False,
+        stop_on_failure=False,
+        add_env={},
+        skip_jobs=False,
+        quiet_success=False):
+  if skip_jobs:
+    results = {}
+    skipped_job_result = JobResult()
+    skipped_job_result.state = 'SKIPPED'
+    for job in cmdlines:
+      message('SKIPPED', job.shortname, do_newline=True)
+      results[job.shortname] = [skipped_job_result]
+    return results
+  js = Jobset(check_cancelled,
+              maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
+              newline_on_success, travis, stop_on_failure, add_env,
+              quiet_success)
+  for cmdline, remaining in tag_remaining(cmdlines):
+    if not js.start(cmdline):
+      break
+    if remaining is not None:
+      js.set_remaining(remaining)
+  js.finish()
+  return js.get_num_failures(), js.resultset
diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py
new file mode 100755
index 0000000000..e9b3f7ff79
--- /dev/null
+++ b/tools/run_tests/python_utils/port_server.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python2.7
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Manage TCP ports for unit tests; started by run_tests.py"""
+
+from __future__ import print_function
+
+import argparse
+from six.moves import BaseHTTPServer
+import hashlib
+import os
+import socket
+import sys
+import time
+
+
+# increment this number whenever making a change to ensure that
+# the changes are picked up by running CI servers
+# note that all changes must be backwards compatible
+_MY_VERSION = 9
+
+
+if len(sys.argv) == 2 and sys.argv[1] == 'dump_version':
+  print(_MY_VERSION)
+  sys.exit(0)
+
+
+argp = argparse.ArgumentParser(description='Server for httpcli_test')
+argp.add_argument('-p', '--port', default=12345, type=int)
+argp.add_argument('-l', '--logfile', default=None, type=str)
+args = argp.parse_args()
+
+if args.logfile is not None:
+  sys.stdin.close()
+  sys.stderr.close()
+  sys.stdout.close()
+  sys.stderr = open(args.logfile, 'w')
+  sys.stdout = sys.stderr
+
+print('port server running on port %d' % args.port)
+
+pool = []
+in_use = {}
+
+
+def refill_pool(max_timeout, req):
+  """Scan for ports not marked for being in use"""
+  for i in range(1025, 32766):
+    if len(pool) > 100: break
+    if i in in_use:
+      age = time.time() - in_use[i]
+      if age < max_timeout:
+        continue
+      req.log_message("kill old request %d" % i)
+      del in_use[i]
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    try:
+      s.bind(('localhost', i))
+      req.log_message("found available port %d" % i)
+      pool.append(i)
+    except:
+      pass # we really don't care about failures
+    finally:
+      s.close()
+
+
+def allocate_port(req):
+  global pool
+  global in_use
+  max_timeout = 600
+  while not pool:
+    refill_pool(max_timeout, req)
+    if not pool:
+      req.log_message("failed to find ports: retrying soon")
+      time.sleep(1)
+      max_timeout /= 2
+  port = pool[0]
+  pool = pool[1:]
+  in_use[port] = time.time()
+  return port
+
+
+keep_running = True
+
+
+class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+  
+  def setup(self):
+    # If the client is unreachable for 5 seconds, close the connection
+    self.timeout = 5
+    BaseHTTPServer.BaseHTTPRequestHandler.setup(self)
+
+  def do_GET(self):
+    global keep_running
+    if self.path == '/get':
+      # allocate a new port, it will stay bound for ten minutes and until
+      # it's unused
+      self.send_response(200)
+      self.send_header('Content-Type', 'text/plain')
+      self.end_headers()
+      p = allocate_port(self)
+      self.log_message('allocated port %d' % p)
+      self.wfile.write('%d' % p)
+    elif self.path[0:6] == '/drop/':
+      self.send_response(200)
+      self.send_header('Content-Type', 'text/plain')
+      self.end_headers()
+      p = int(self.path[6:])
+      if p in in_use:
+        del in_use[p]
+        pool.append(p)
+        self.log_message('drop known port %d' % p)
+      else:
+        self.log_message('drop unknown port %d' % p)
+    elif self.path == '/version_number':
+      # fetch a version string and the current process pid
+      self.send_response(200)
+      self.send_header('Content-Type', 'text/plain')
+      self.end_headers()
+      self.wfile.write(_MY_VERSION)
+    elif self.path == '/dump':
+      # yaml module is not installed on Macs and Windows machines by default
+      # so we import it lazily (/dump action is only used for debugging)
+      import yaml
+      self.send_response(200)
+      self.send_header('Content-Type', 'text/plain')
+      self.end_headers()
+      now = time.time()
+      self.wfile.write(yaml.dump({'pool': pool, 'in_use': dict((k, now - v) for k, v in in_use.items())}))
+    elif self.path == '/quitquitquit':
+      self.send_response(200)
+      self.end_headers()
+      keep_running = False
+
+
+httpd = BaseHTTPServer.HTTPServer(('', args.port), Handler)
+while keep_running:
+  httpd.handle_request()
+  sys.stderr.flush()
+
+print('done')
diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py
new file mode 100644
index 0000000000..352cf7abe7
--- /dev/null
+++ b/tools/run_tests/python_utils/report_utils.py
@@ -0,0 +1,131 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Generate XML and HTML test reports."""
+
+from __future__ import print_function
+
+try:
+  from mako.runtime import Context
+  from mako.template import Template
+  from mako import exceptions
+except (ImportError):
+  pass  # Mako not installed but it is ok.
+import os
+import string
+import xml.etree.cElementTree as ET
+
+
+def _filter_msg(msg, output_format):
+  """Filters out nonprintable and illegal characters from the message."""
+  if output_format in ['XML', 'HTML']:
+    # keep whitespaces but remove formfeed and vertical tab characters
+    # that make XML report unparseable.
+    filtered_msg = filter(
+        lambda x: x in string.printable and x != '\f' and x != '\v',
+        msg.decode('UTF-8', 'ignore'))
+    if output_format == 'HTML':
+      filtered_msg = filtered_msg.replace('"', '&quot;')
+    return filtered_msg
+  else:
+    return msg
+
+
+def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
+                            suite_name='tests'):
+  """Generate JUnit-like XML report."""
+  root = ET.Element('testsuites')
+  testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package,
+                            name=suite_name)
+  for shortname, results in resultset.iteritems():
+    for result in results:
+      xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
+      if result.elapsed_time:
+        xml_test.set('time', str(result.elapsed_time))
+      ET.SubElement(xml_test, 'system-out').text = _filter_msg(result.message,
+                                                               'XML')
+      if result.state == 'FAILED':
+        ET.SubElement(xml_test, 'failure', message='Failure')
+      elif result.state == 'TIMEOUT':
+        ET.SubElement(xml_test, 'error', message='Timeout')
+      elif result.state == 'SKIPPED':
+        ET.SubElement(xml_test, 'skipped', message='Skipped')
+  tree = ET.ElementTree(root)
+  tree.write(xml_report, encoding='UTF-8')
+
+
+def render_interop_html_report(
+  client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
+  resultset, num_failures, cloud_to_prod, prod_servers, http2_interop):
+  """Generate HTML report for interop tests."""
+  template_file = 'tools/run_tests/interop/interop_html_report.template'
+  try:
+    mytemplate = Template(filename=template_file, format_exceptions=True)
+  except NameError:
+    print('Mako template is not installed. Skipping HTML report generation.')
+    return
+  except IOError as e:
+    print('Failed to find the template %s: %s' % (template_file, e))
+    return
+
+  sorted_test_cases = sorted(test_cases)
+  sorted_auth_test_cases = sorted(auth_test_cases)
+  sorted_http2_cases = sorted(http2_cases)
+  sorted_client_langs = sorted(client_langs)
+  sorted_server_langs = sorted(server_langs)
+  sorted_prod_servers = sorted(prod_servers)
+
+  args = {'client_langs': sorted_client_langs,
+          'server_langs': sorted_server_langs,
+          'test_cases': sorted_test_cases,
+          'auth_test_cases': sorted_auth_test_cases,
+          'http2_cases': sorted_http2_cases,
+          'resultset': resultset,
+          'num_failures': num_failures,
+          'cloud_to_prod': cloud_to_prod,
+          'prod_servers': sorted_prod_servers,
+          'http2_interop': http2_interop}
+
+  html_report_out_dir = 'reports'
+  if not os.path.exists(html_report_out_dir):
+    os.mkdir(html_report_out_dir)
+  html_file_path = os.path.join(html_report_out_dir, 'index.html')
+  try:
+    with open(html_file_path, 'w') as output_file:
+      mytemplate.render_context(Context(output_file, **args))
+  except:
+    print(exceptions.text_error_template().render())
+    raise
+
+def render_perf_profiling_results(output_filepath, profile_names):
+  with open(output_filepath, 'w') as output_file:
+    output_file.write('<ul>\n')
+    for name in profile_names:
+      output_file.write('<li><a href=%s>%s</a></li>\n' % (name, name))
+    output_file.write('</ul>\n')
diff --git a/tools/run_tests/python_utils/watch_dirs.py b/tools/run_tests/python_utils/watch_dirs.py
new file mode 100755
index 0000000000..21ef23e158
--- /dev/null
+++ b/tools/run_tests/python_utils/watch_dirs.py
@@ -0,0 +1,75 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Helper to watch a (set) of directories for modifications."""
+
+import os
+import time
+
+
+class DirWatcher(object):
+  """Helper to watch a (set) of directories for modifications."""
+
+  def __init__(self, paths):
+    if isinstance(paths, basestring):
+      paths = [paths]
+    self._done = False
+    self.paths = list(paths)
+    self.lastrun = time.time()
+    self._cache = self._calculate()
+
+  def _calculate(self):
+    """Walk over all subscribed paths, check most recent mtime."""
+    most_recent_change = None
+    for path in self.paths:
+      if not os.path.exists(path):
+        continue
+      if not os.path.isdir(path):
+        continue
+      for root, _, files in os.walk(path):
+        for f in files:
+          if f and f[0] == '.': continue
+          try:
+            st = os.stat(os.path.join(root, f))
+          except OSError as e:
+            if e.errno == os.errno.ENOENT:
+              continue
+            raise
+          if most_recent_change is None:
+            most_recent_change = st.st_mtime
+          else:
+            most_recent_change = max(most_recent_change, st.st_mtime)
+    return most_recent_change
+
+  def most_recent_change(self):
+    if time.time() - self.lastrun > 1:
+      self._cache = self._calculate()
+      self.lastrun = time.time()
+    return self._cache
+