aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/interop_matrix
diff options
context:
space:
mode:
Diffstat (limited to 'tools/interop_matrix')
-rw-r--r--tools/interop_matrix/README.md21
-rw-r--r--tools/interop_matrix/client_matrix.py6
-rwxr-xr-xtools/interop_matrix/create_matrix_images.py2
-rwxr-xr-xtools/interop_matrix/run_interop_matrix_tests.py22
4 files changed, 30 insertions, 21 deletions
diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md
index c0e9a33c5e..a3717a5a70 100644
--- a/tools/interop_matrix/README.md
+++ b/tools/interop_matrix/README.md
@@ -5,32 +5,21 @@ This directory contains scripts that facilitate building and running gRPC tests
The setup builds gRPC docker images for each language/runtime and upload it to Google Container Registry (GCR). These images, encapsulating gRPC stack
from specific releases/tag, are used to test version compatiblity between gRPC release versions.
-## Step-by-step instructions for adding a new release to compatibility test
+## Step-by-step instructions for adding a GCR image for a new release for compatibility test
We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are:
-- Add (or update) an entry in ./client_matrix.py file to reference the github tag for the release.
+- Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release.
- Build new client docker image(s). For example, for java release `v1.9.9`, do
- `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language=java`
- Verify that the new docker image was built successfully and uploaded to GCR. For example,
- - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8`
- - should show an image entry with tag `v1.9.9`.
+ - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos.
+ - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`.
- Verify the just-created docker client image would pass backward compatibility test (it should). For example,
- `gcloud docker -- pull gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9` followed by
- - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 ./testcases/java__master`
+ - `docker_image=gcr.io/grpc-testing/grpc_interop_java_oracle8:v1.9.9 tools/interop_matrix/testcases/java__master`
- git commit the change and merge it to upstream/master.
- (Optional) clean up the tmp directory to where grpc source is cloned at `/export/hda3/tmp/grpc_matrix/`.
For more details on each step, refer to sections below.
-## Instructions for creating GCR images
-- Edit `./client_matrix.py` to include desired gRPC release.
-- Run `tools/interop_matrix/create_matrix_images.py`. Useful options:
- - `--git_checkout` enables git checkout grpc release branch/tag.
- - `--release` specifies a git release tag. Make sure it is a valid tag in the grpc github rep.
- - `--language` specifies a language.
- For example, To build all languages for all gRPC releases across all runtimes, do `tools/interop_matrix/create_matrix_images.py --git_checkout --release=all`.
-- Verify the newly created docker images are uploaded to GCR. For example:
- - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos.
- - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_go1.7` show tags for a image repo.
-
## Instructions for adding new language/runtimes*
- Create new `Dockerfile.template`, `build_interop.sh.template` for the language/runtime under `template/tools/dockerfile/`.
- Run `tools/buildgen/generate_projects.sh` to create corresponding files under `tools/dockerfile/`.
diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py
index 4d1b5f0169..23c5ae00c1 100644
--- a/tools/interop_matrix/client_matrix.py
+++ b/tools/interop_matrix/client_matrix.py
@@ -38,12 +38,16 @@ LANG_RELEASE_MATRIX = {
'v1.2.5',
'v1.3.9',
'v1.4.2',
+ 'v1.6.6',
],
'go': [
'v1.0.5',
'v1.2.1',
'v1.3.0',
'v1.4.2',
+ 'v1.5.2',
+ 'v1.6.0',
+ 'v1.7.0',
],
'java': [
'v1.0.3',
@@ -52,5 +56,7 @@ LANG_RELEASE_MATRIX = {
'v1.3.1',
'v1.4.0',
'v1.5.0',
+ 'v1.6.1',
+ 'v1.7.0',
],
}
diff --git a/tools/interop_matrix/create_matrix_images.py b/tools/interop_matrix/create_matrix_images.py
index 119cbd2e88..67355e7fbf 100755
--- a/tools/interop_matrix/create_matrix_images.py
+++ b/tools/interop_matrix/create_matrix_images.py
@@ -174,7 +174,7 @@ def build_all_images_for_release(lang, release):
# If we not using current tree or the sibling for grpc stack, do checkout.
if args.git_checkout:
stack_base = checkout_grpc_stack(lang, release)
- var ={'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT'}.get(lang, 'GRPC_ROOT')
+ var ={'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT'}.get(lang, 'GRPC_ROOT')
env[var] = stack_base
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py
index 48c918d25d..d037e13921 100755
--- a/tools/interop_matrix/run_interop_matrix_tests.py
+++ b/tools/interop_matrix/run_interop_matrix_tests.py
@@ -36,6 +36,7 @@ sys.path.append(python_util_dir)
import dockerjob
import jobset
import report_utils
+import upload_test_results
_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
# All gRPC release tags, flattened, deduped and sorted.
@@ -74,6 +75,11 @@ argp.add_argument('--allow_flakes',
const=True,
help=('Allow flaky tests to show as passing (re-runs failed '
'tests up to five times)'))
+argp.add_argument('--bq_result_table',
+ default='',
+ type=str,
+ nargs='?',
+ help='Upload test results to a specified BQ table.')
args = argp.parse_args()
@@ -117,7 +123,7 @@ def find_all_images_for_lang(lang):
# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime.
_loaded_testcases = {}
-def find_test_cases(lang, release):
+def find_test_cases(lang, release, suite_name):
"""Returns the list of test cases from testcase files per lang/release."""
file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s')
if not os.path.exists(file_tmpl % (lang, release)):
@@ -134,8 +140,12 @@ def find_test_cases(lang, release):
if line.startswith('docker run'):
m = re.search('--test_case=(.*)"', line)
shortname = m.group(1) if m else 'unknown_test'
+ m = re.search('--server_host_override=(.*).sandbox.googleapis.com',
+ line)
+ server = m.group(1) if m else 'unknown_server'
spec = jobset.JobSpec(cmdline=line,
- shortname=shortname,
+ shortname='%s:%s:%s:%s' % (suite_name, lang,
+ server, shortname),
timeout_seconds=_TEST_TIMEOUT,
shell=True,
flake_retries=5 if args.allow_flakes else 0)
@@ -163,11 +173,15 @@ def run_tests_for_lang(lang, runtime, images):
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
_docker_images_cleanup.append(image)
- job_spec_list = find_test_cases(lang,release)
+ suite_name = '%s__%s_%s' % (lang, runtime, release)
+ job_spec_list = find_test_cases(lang, release, suite_name)
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image':image},
maxjobs=args.jobs)
+ if args.bq_result_table and resultset:
+ upload_test_results.upload_interop_results_to_bq(
+ resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
@@ -178,7 +192,7 @@ def run_tests_for_lang(lang, runtime, images):
_xml_report_tree,
resultset,
'grpc_interop_matrix',
- '%s__%s %s'%(lang,runtime,release),
+ suite_name,
str(uuid.uuid4()))
return total_num_failures
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-10 15:25:16 +0000