From d14c0ea1f58c1383361abbbc177d09818a9def2f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Fri, 19 May 2017 14:25:01 -0700 Subject: Upload Jenkins tests results to a partitioned BQ table --- tools/gcp/utils/big_query_utils.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'tools/gcp') diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index 9dbc69c5d6..8efca4b710 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -37,6 +37,8 @@ from apiclient import discovery from apiclient.errors import HttpError from oauth2client.client import GoogleCredentials +# 30 days in milliseconds +_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000 NUM_RETRIES = 3 @@ -79,8 +81,21 @@ def create_table(big_query, project_id, dataset_id, table_id, table_schema, fields, description) +def create_partitioned_table(big_query, project_id, dataset_id, table_id, table_schema, + description, partition_type='DAY', expiration_ms=_EXPIRATION_MS): + """Creates a partitioned table. By default, a date-paritioned table is created with + each partition lasting 30 days after it was last modified. + """ + fields = [{'name': field_name, + 'type': field_type, + 'description': field_description + } for (field_name, field_type, field_description) in table_schema] + return create_table2(big_query, project_id, dataset_id, table_id, + fields, description, partition_type, expiration_ms) + + def create_table2(big_query, project_id, dataset_id, table_id, fields_schema, - description): + description, partition_type=None, expiration_ms=None): is_success = True body = { @@ -95,6 +110,12 @@ def create_table2(big_query, project_id, dataset_id, table_id, fields_schema, } } + if partition_type and expiration_ms: + body["timePartitioning"] = { + "type": partition_type, + "expirationMs": expiration_ms + } + try: table_req = big_query.tables().insert(projectId=project_id, datasetId=dataset_id, -- cgit v1.2.3