diff options
author | 2020-07-27 13:19:44 +1000 | |
---|---|---|
committer | 2020-07-27 13:19:44 +1000 | |
commit | bc7d478d4a64e93e1f0eb7022100ef3af75829d4 (patch) | |
tree | 4ba7f543684cb85d77b110ceadcb54ecf79d6646 /infra/build | |
parent | 7f9866ee90ac8be46c1e4673342b6d81cf0d81b4 (diff) |
build: Project sync fixes. (#4194)
- Add some more logging.
- Use GitHub client ID/secret rather than personal access token.
- Fix function deploy wrt "--project" argument.
Diffstat (limited to 'infra/build')
-rw-r--r-- | infra/build/functions/datastore_entities.py | 7 | ||||
-rwxr-xr-x | infra/build/functions/deploy.sh | 9 | ||||
-rw-r--r-- | infra/build/functions/project_sync.py | 21 | ||||
-rw-r--r-- | infra/build/functions/project_sync_test.py | 8 |
4 files changed, 26 insertions, 19 deletions
diff --git a/infra/build/functions/datastore_entities.py b/infra/build/functions/datastore_entities.py index d2ebb647..1946cb52 100644 --- a/infra/build/functions/datastore_entities.py +++ b/infra/build/functions/datastore_entities.py @@ -27,9 +27,10 @@ class Project(ndb.Model): # pylint: disable=too-few-public-methods -class GitAuth(ndb.Model): - """Represents Github access token entity.""" - access_token = ndb.StringProperty() +class GithubCreds(ndb.Model): + """Represents GitHub credentials.""" + client_id = ndb.StringProperty() + client_secret = ndb.StringProperty() # pylint: disable=too-few-public-methods diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh index 08810bb8..0c361a3d 100755 --- a/infra/build/functions/deploy.sh +++ b/infra/build/functions/deploy.sh @@ -1,3 +1,4 @@ +#!/bin/bash -ex # Copyright 2020 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -58,16 +59,16 @@ function deploy_scheduler { if gcloud scheduler jobs describe $scheduler_name --project $project ; then gcloud scheduler jobs update pubsub $scheduler_name \ + --project $project \ --schedule "$schedule" \ --topic $topic \ - --message-body "$message" \ - --project $project + --message-body "$message" else gcloud scheduler jobs create pubsub $scheduler_name \ + --project $project \ --schedule "$schedule" \ --topic $topic \ - --message-body "$message" \ - --project $project + --message-body "$message" fi } diff --git a/infra/build/functions/project_sync.py b/infra/build/functions/project_sync.py index 8ceabe68..10140839 100644 --- a/infra/build/functions/project_sync.py +++ b/infra/build/functions/project_sync.py @@ -26,7 +26,7 @@ from google.api_core import exceptions from google.cloud import ndb from google.cloud import scheduler_v1 -from datastore_entities import GitAuth +from datastore_entities import GithubCreds from datastore_entities import Project VALID_PROJECT_NAME = re.compile(r'^[a-zA-Z0-9_-]+$') @@ -92,6 +92,7 @@ def sync_projects(cloud_scheduler_client, projects): if project.name in projects: continue + logging.info('Deleting project %s', project.name) try: delete_scheduler(cloud_scheduler_client, project.name) project.key.delete() @@ -119,10 +120,13 @@ def sync_projects(cloud_scheduler_client, projects): for project in Project.query(): if project.name not in projects: continue + + logging.info('Setting up project %s', project.name) project_metadata = projects[project.name] project_changed = False if project.schedule != project_metadata.schedule: try: + logging.info('Schedule changed.') update_scheduler(cloud_scheduler_client, project, projects[project.name].schedule) project.schedule = project_metadata.schedule @@ -197,12 +201,12 @@ def get_projects(repo): return projects -def get_access_token(): - """Retrieves Github's Access token from Cloud Datastore.""" - token = GitAuth.query().get() - if token is None: - raise RuntimeError('No access token available') - return token.access_token +def get_github_creds(): + """Retrieves GitHub client credentials.""" + git_creds = GithubCreds.query().get() + if git_creds is None: + raise RuntimeError('Git credentials not available.') + return git_creds def sync(event, context): @@ -210,7 +214,8 @@ def sync(event, context): del event, context #unused with ndb.Client().context(): - github_client = Github(get_access_token()) + git_creds = get_github_creds() + github_client = Github(git_creds.client_id, git_creds.client_secret) repo = github_client.get_repo('google/oss-fuzz') projects = get_projects(repo) cloud_scheduler_client = scheduler_v1.CloudSchedulerClient() diff --git a/infra/build/functions/project_sync_test.py b/infra/build/functions/project_sync_test.py index 34fd674c..416029d1 100644 --- a/infra/build/functions/project_sync_test.py +++ b/infra/build/functions/project_sync_test.py @@ -21,7 +21,7 @@ import unittest from google.cloud import ndb from datastore_entities import Project -from project_sync import get_access_token +from project_sync import get_github_creds from project_sync import get_projects from project_sync import ProjectMetadata from project_sync import sync_projects @@ -276,10 +276,10 @@ class TestDataSync(unittest.TestCase): self.assertEqual(get_projects(repo), {}) - def test_get_access_token(self): - """Testing get_access_token().""" + def test_get_github_creds(self): + """Testing get_github_creds().""" with ndb.Client().context(): - self.assertRaises(RuntimeError, get_access_token) + self.assertRaises(RuntimeError, get_github_creds) @classmethod def tearDownClass(cls): |