diff options
| author |  Jan Tattermusch <jtattermusch@users.noreply.github.com> | 2018-06-12 11:03:13 +0200 |
|---|---|---|
| committer |  GitHub <noreply@github.com> | 2018-06-12 11:03:13 +0200 |
| commit | aa1aa4329f2cef0e27272abcd5c056f63feb9117 (patch) | |
| tree | d82d10961a4cd388ec40ccbfaf3ce253deefa783 /tools/run_tests/python_utils | |
| parent | 8775eadee86724c38a5fee68f26415320abc2003 (diff) | |
| parent | cc42e6dba8035ec8dd77c59d6c13f9d510e6cf14 (diff) | |
Merge pull request #15705 from jtattermusch/bq_batch_upload
Upload test results to BQ in batches
Diffstat (limited to 'tools/run_tests/python_utils')
| -rw-r--r-- | tools/run_tests/python_utils/upload_rbe_results.py | 2 | ||||
| -rw-r--r-- | tools/run_tests/python_utils/upload_test_results.py | 11 |
2 files changed, 9 insertions, 4 deletions
diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index a2dd0bc39d..7236227f7c 100644 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -125,7 +125,7 @@ def _get_resultstore_data(api_key, invocation_id): if __name__ == "__main__": - # Arguments are necessary if running in a non-Kokoro envrionment. + # Arguments are necessary if running in a non-Kokoro environment. argp = argparse.ArgumentParser(description='Upload RBE results.') argp.add_argument('--api_key', default='', type=str) argp.add_argument('--invocation_id', default='', type=str) diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index 09dcd57ad4..63fa38b678 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -163,6 +163,7 @@ def upload_interop_results_to_bq(resultset, bq_table, args): expiration_ms=_EXPIRATION_MS) for shortname, results in six.iteritems(resultset): + bq_rows = [] for result in results: test_results = {} _get_build_metadata(test_results) @@ -175,11 +176,15 @@ def upload_interop_results_to_bq(resultset, bq_table, args): test_results['test_case'] = shortname.split(':')[3] test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') row = big_query_utils.make_row(str(uuid.uuid4()), test_results) - # TODO(jtattermusch): rows are inserted one by one, very inefficient + bq_rows.append(row) + + # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time. + for i in range((len(bq_rows) / 1000) + 1): max_retries = 3 for attempt in range(max_retries): - if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, - bq_table, [row]): + if big_query_utils.insert_rows( + bq, _PROJECT_ID, _DATASET_ID, bq_table, + bq_rows[i * 1000:(i + 1) * 1000]): break else: if attempt < max_retries - 1: |