Merge pull request #10967 from matt-kwong/upload_results2

Add option to upload Jenkins test result to BQ

22 files changed, 209 insertions, 6 deletions

diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
index f9e709dccb..0b21a22226 100644
--- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
index 4bb97c7aa9..d9dc272a1b 100644
--- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
index c4b710b5df..11ef52d1c0 100644
--- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
+++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
index bd742dff34..41d3b2b520 100644
--- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
index bc46b3055a..23d6fb8c41 100644
--- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
index f7d7f542c1..1848e5e5c8 100644
--- a/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
+++ b/tools/dockerfile/test/cxx_wheezy_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile
index b398b70b64..4200ba0b26 100644
--- a/tools/dockerfile/test/fuzzer/Dockerfile
+++ b/tools/dockerfile/test/fuzzer/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
index c1cce0a141..9b50d85e2f 100644
--- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #================
 # C# dependencies
 
diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile
index 4595aa6bea..deef892952 100644
--- a/tools/dockerfile/test/node_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 
 # Install Electron apt dependencies
 RUN apt-get update && apt-get install -y \
diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
index 0e2c103afd..6057c2d6eb 100644
--- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
@@ -75,6 +75,10 @@ RUN cd /var/local/git/php-src \
   && make \
   && make install
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile
index c6f3dde39a..1510c3649c 100644
--- a/tools/dockerfile/test/php_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile
index 94c17078d3..cc69f4b5cd 100644
--- a/tools/dockerfile/test/python_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
index 435a9fdc97..a105d334da 100644
--- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile
+++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
index 679c8ff47a..0a5c9a633d 100644
--- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 0da2a1914a..76923303ea 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -63,6 +63,10 @@ RUN apt-get update && apt-get install -y \
 # Build profiling
 RUN apt-get update && apt-get install -y time && apt-get clean
 
+# Google Cloud platform API libraries
+RUN apt-get update && apt-get install -y python-pip && apt-get clean
+RUN pip install --upgrade google-api-python-client
+
 #====================
 # Python dependencies
 
diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh
index d552343bde..f795980aa8 100755
--- a/tools/gce/linux_worker_init.sh
+++ b/tools/gce/linux_worker_init.sh
@@ -61,6 +61,10 @@ sudo usermod -aG docker jenkins
 # see https://github.com/grpc/grpc/issues/4988
 printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json
 
+# Install pip and Google API library to enable using GCP services
+sudo apt-get install -y python-pip
+sudo pip install google-api-python-client
+
 # Install RVM
 # TODO(jtattermusch): why is RVM needed?
 gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
diff --git a/tools/run_tests/dockerize/build_docker_and_run_tests.sh b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
index f10916d192..731bb02d21 100755
--- a/tools/run_tests/dockerize/build_docker_and_run_tests.sh
+++ b/tools/run_tests/dockerize/build_docker_and_run_tests.sh
@@ -79,7 +79,10 @@ docker run \
   -e HOST_GIT_ROOT=$git_root \
   -e LOCAL_GIT_ROOT=$docker_instance_git_root \
   -e "BUILD_ID=$BUILD_ID" \
+  -e "BUILD_URL=$BUILD_URL" \
+  -e "JOB_BASE_NAME=$JOB_BASE_NAME" \
   -i $TTY_FLAG \
+  -v ~/.config/gcloud:/root/.config/gcloud \
   -v "$git_root:$docker_instance_git_root" \
   -v /tmp/ccache:/tmp/ccache \
   -v /tmp/npm-cache:/tmp/npm-cache \
diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py
index d01e82a76b..4e97128d46 100755
--- a/tools/run_tests/python_utils/jobset.py
+++ b/tools/run_tests/python_utils/jobset.py
@@ -222,7 +222,8 @@ class JobResult(object):
     self.num_failures = 0
     self.retries = 0
     self.message = ''
-
+    self.cpu_estimated = 1
+    self.cpu_measured = 0
 
 class Job(object):
   """Manages one job."""
@@ -312,7 +313,9 @@ class Job(object):
           sys = float(m.group(3))
           if real > 0.5:
             cores = (user + sys) / real
-            measurement = '; cpu_cost=%.01f; estimated=%.01f' % (cores, self._spec.cpu_cost)
+            self.result.cpu_measured = float('%.01f' % cores)
+            self.result.cpu_estimated = float('%.01f' % self._spec.cpu_cost)
+            measurement = '; cpu_cost=%.01f; estimated=%.01f' % (self.result.cpu_measured, self.result.cpu_estimated)
         if not self._quiet_success:
           message('PASSED', '%s [time=%.1fsec; retries=%d:%d%s]' % (
               self._spec.shortname, elapsed, self._retries, self._timeout_retries, measurement),
diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py
index 502efc31f4..b8dd67e060 100644
--- a/tools/run_tests/python_utils/report_utils.py
+++ b/tools/run_tests/python_utils/report_utils.py
@@ -89,6 +89,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
   tree = ET.ElementTree(root)
   tree.write(xml_report, encoding='UTF-8')
 
+
 def render_interop_html_report(
   client_langs, server_langs, test_cases, auth_test_cases, http2_cases,
   http2_server_cases, resultset,
diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py
new file mode 100644
index 0000000000..d076d1e5a2
--- /dev/null
+++ b/tools/run_tests/python_utils/upload_test_results.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Helper to upload Jenkins test results to BQ"""
+
+from __future__ import print_function
+
+import os
+import six
+import sys
+import time
+import uuid
+
+gcp_utils_dir = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '../../gcp/utils'))
+sys.path.append(gcp_utils_dir)
+import big_query_utils
+
+_DATASET_ID = 'jenkins_test_results'
+_DESCRIPTION = 'Test results from master job run on Jenkins'
+_PROJECT_ID = 'grpc-testing'
+_RESULTS_SCHEMA = [
+  ('job_name', 'STRING', 'Name of Jenkins job'),
+  ('build_id', 'INTEGER', 'Build ID of Jenkins job'),
+  ('build_url', 'STRING', 'URL of Jenkins job'),
+  ('test_name', 'STRING', 'Individual test name'),
+  ('language', 'STRING', 'Language of test'),
+  ('platform', 'STRING', 'Platform used for test'),
+  ('config', 'STRING', 'Config used for test'),
+  ('compiler', 'STRING', 'Compiler used for test'),
+  ('iomgr_platform', 'STRING', 'Iomgr used for test'),
+  ('result', 'STRING', 'Test result: PASSED, TIMEOUT, FAILED, or SKIPPED'),
+  ('timestamp', 'TIMESTAMP', 'Timestamp of test run'),
+  ('elapsed_time', 'FLOAT', 'How long test took to run'),
+  ('cpu_estimated', 'FLOAT', 'Estimated CPU usage of test'),
+  ('cpu_measured', 'FLOAT', 'Actual CPU usage of test'),
+]
+
+
+def _get_build_metadata(test_results):
+  """Add Jenkins build metadata to test_results based on environment variables set by Jenkins."""
+  build_id = os.getenv('BUILD_ID')
+  build_url = os.getenv('BUILD_URL')
+  job_name = os.getenv('JOB_BASE_NAME')
+
+  if build_id:
+    test_results['build_id'] = build_id
+  if build_url:
+    test_results['build_url'] = build_url
+  if job_name:
+    test_results['job_name'] = job_name
+
+def upload_results_to_bq(resultset, bq_table, args, platform):
+  """Upload test results to a BQ table.
+
+  Args:
+      resultset: dictionary generated by jobset.run
+      bq_table: string name of table to create/upload results to in BQ
+      args: args in run_tests.py, generated by argparse
+      platform: string name of platform tests were run on
+  """
+  bq = big_query_utils.create_big_query()
+  big_query_utils.create_table(bq, _PROJECT_ID, _DATASET_ID, bq_table, _RESULTS_SCHEMA, _DESCRIPTION)
+
+  for shortname, results in six.iteritems(resultset):
+    for result in results:
+      test_results = {}
+      _get_build_metadata(test_results)
+      test_results['compiler'] = args.compiler
+      test_results['config'] = args.config
+      test_results['cpu_estimated'] = result.cpu_estimated
+      test_results['cpu_measured'] = result.cpu_measured
+      test_results['elapsed_time'] = '%.2f' % result.elapsed_time
+      test_results['iomgr_platform'] = args.iomgr_platform
+      # args.language is a list, but will always have one element in the contexts
+      # this function is used.
+      test_results['language'] = args.language[0]
+      test_results['platform'] = platform
+      test_results['result'] = result.state
+      test_results['test_name'] = shortname
+      test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S')
+
+      row = big_query_utils.make_row(str(uuid.uuid4()), test_results)
+      if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, bq_table, [row]):
+        print('Error uploading result to bigquery.')
+        sys.exit(1)
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 62e92c3d1d..22891c5c98 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -60,7 +60,10 @@ import python_utils.jobset as jobset
 import python_utils.report_utils as report_utils
 import python_utils.watch_dirs as watch_dirs
 import python_utils.start_port_server as start_port_server
-
+try:
+  from python_utils.upload_test_results import upload_results_to_bq
+except (ImportError):
+  pass # It's ok to not import because this is only necessary to upload results to BQ.
 
 _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
 os.chdir(_ROOT)
@@ -1185,7 +1188,7 @@ argp.add_argument('--build_only',
                   default=False,
                   action='store_const',
                   const=True,
-                  help='Perform all the build steps but dont run any tests.')
+                  help='Perform all the build steps but don\'t run any tests.')
 argp.add_argument('--measure_cpu_costs', default=False, action='store_const', const=True,
                   help='Measure the cpu costs of tests')
 argp.add_argument('--update_submodules', default=[], nargs='*',
@@ -1200,11 +1203,16 @@ argp.add_argument('--quiet_success',
                   default=False,
                   action='store_const',
                   const=True,
-                  help='Dont print anything when a test passes. Passing tests also will not be reported in XML report. ' +
+                  help='Don\'t print anything when a test passes. Passing tests also will not be reported in XML report. ' +
                        'Useful when running many iterations of each test (argument -n).')
 argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
-                  help='Dont try to iterate over many polling strategies when they exist')
+                  help='Don\'t try to iterate over many polling strategies when they exist')
 argp.add_argument('--max_time', default=-1, type=int, help='Maximum test runtime in seconds')
+argp.add_argument('--bq_result_table',
+                  default='',
+                  type=str,
+                  nargs='?',
+                  help='Upload test results to a specified BQ table.')
 args = argp.parse_args()
 
 if args.force_default_poller:
@@ -1503,6 +1511,8 @@ def _build_and_run(
   finally:
     for antagonist in antagonists:
       antagonist.kill()
+    if args.bq_result_table and resultset:
+      upload_results_to_bq(resultset, args.bq_result_table, args, platform_string())
     if xml_report and resultset:
       report_utils.render_junit_xml_report(resultset, xml_report,
                                            suite_name=args.report_suite_name)
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 76e83ef22f..884d626469 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -396,6 +396,11 @@ if __name__ == "__main__":
                     const=True,
                     help='Put reports into subdirectories to improve presentation of '
                     'results by Internal CI.')
+  argp.add_argument('--bq_result_table',
+                    default='',
+                    type=str,
+                    nargs='?',
+                    help='Upload test results to a specified BQ table.')
   args = argp.parse_args()
 
   if args.internal_ci:
@@ -412,6 +417,10 @@ if __name__ == "__main__":
     extra_args.append('--quiet_success')
   if args.max_time > 0:
     extra_args.extend(('--max_time', '%d' % args.max_time))
+  if args.bq_result_table:
+    extra_args.append('--bq_result_table')
+    extra_args.append('%s' % args.bq_result_table)
+    extra_args.append('--measure_cpu_costs')
 
   all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
              _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)