diff options
author | Jan Tattermusch <jtattermusch@google.com> | 2018-06-11 11:53:49 -0400 |
---|---|---|
committer | Jan Tattermusch <jtattermusch@google.com> | 2018-06-11 11:53:49 -0400 |
commit | cc42e6dba8035ec8dd77c59d6c13f9d510e6cf14 (patch) | |
tree | 37c8b6715a39953750b453e9fcce32d092f3c9b1 /tools/run_tests/python_utils | |
parent | 55178622eb9d33a6a2dec1fe770a52a70ff54c5c (diff) |
upload bq test results in batches
Diffstat (limited to 'tools/run_tests/python_utils')
-rw-r--r-- | tools/run_tests/python_utils/upload_test_results.py | 11 |
1 files changed, 8 insertions, 3 deletions
diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index 09dcd57ad4..63fa38b678 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -163,6 +163,7 @@ def upload_interop_results_to_bq(resultset, bq_table, args): expiration_ms=_EXPIRATION_MS) for shortname, results in six.iteritems(resultset): + bq_rows = [] for result in results: test_results = {} _get_build_metadata(test_results) @@ -175,11 +176,15 @@ def upload_interop_results_to_bq(resultset, bq_table, args): test_results['test_case'] = shortname.split(':')[3] test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') row = big_query_utils.make_row(str(uuid.uuid4()), test_results) - # TODO(jtattermusch): rows are inserted one by one, very inefficient + bq_rows.append(row) + + # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time. + for i in range((len(bq_rows) / 1000) + 1): max_retries = 3 for attempt in range(max_retries): - if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, - bq_table, [row]): + if big_query_utils.insert_rows( + bq, _PROJECT_ID, _DATASET_ID, bq_table, + bq_rows[i * 1000:(i + 1) * 1000]): break else: if attempt < max_retries - 1: |