Display historical logs ui change (#4197)

* Initial UI Change to display historical logs

* Minor changes

* More formatting changes

* Adding license header

* Changes, take a look Oliver

* Changes for handling empty build history/ no last successful build

* Reverting the logs folder path from /logs/log to /log

* Merged to latest master and other chnages to template

* Changing backend to reflect build history

* Fixing order of query

* Removed last build status

* More changes

* Adding unit tests and fixes found by unit testing

* Fixing lint errors

* Added more unit tests and fixed typos and formatting

* Fixing comment caps

* Minor formatting issue

Co-authored-by: Oliver Chang <oliverchang@users.noreply.github.com>

5 files changed, 414 insertions, 32 deletions

diff --git a/infra/build/functions/datastore_entities.py b/infra/build/functions/datastore_entities.py
index 8683803c..550ab82f 100644
--- a/infra/build/functions/datastore_entities.py
+++ b/infra/build/functions/datastore_entities.py
@@ -39,3 +39,11 @@ class BuildsHistory(ndb.Model):
   build_tag = ndb.StringProperty()
   project = ndb.StringProperty()
   build_ids = ndb.StringProperty(repeated=True)
+
+
+class LastSuccessfulBuild(ndb.Model):
+  """Container for storing last successful build of project."""
+  build_tag = ndb.StringProperty()
+  project = ndb.StringProperty()
+  build_id = ndb.StringProperty()
+  finish_time = ndb.StringProperty()
diff --git a/infra/build/functions/deploy.sh b/infra/build/functions/deploy.sh
index 26f99cbb..d12ed07d 100755
--- a/infra/build/functions/deploy.sh
+++ b/infra/build/functions/deploy.sh
@@ -148,3 +148,5 @@ deploy_cloud_function update-builds \
 					  builds_status \
 					  $UPDATE_BUILD_JOB_TOPIC \
 					  $PROJECT_ID
+
+gcloud datastore indexes create index.yaml --project $PROJECT_ID
diff --git a/infra/build/functions/index.yaml b/infra/build/functions/index.yaml
new file mode 100644
index 00000000..260bee27
--- /dev/null
+++ b/infra/build/functions/index.yaml
@@ -0,0 +1,5 @@
+indexes:
+  - kind: BuildsHistory
+    properties:
+      - name: build_tag
+      - name: project
diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py
index 566f600e..3d0ed93c 100644
--- a/infra/build/functions/update_build_status.py
+++ b/infra/build/functions/update_build_status.py
@@ -14,7 +14,7 @@
 #
 ################################################################################
 """Cloud function to request builds."""
-import logging
+import json
 
 import google.auth
 from googleapiclient.discovery import build
@@ -24,18 +24,79 @@ import build_and_run_coverage
 import build_project
 import builds_status
 from datastore_entities import BuildsHistory
+from datastore_entities import LastSuccessfulBuild
 from datastore_entities import Project
 
 BADGE_DIR = 'badge_images'
 DESTINATION_BADGE_DIR = 'badges'
+MAX_BUILD_LOGS = 7
 
 
 class MissingBuildLogError(Exception):
   """Missing build log file in cloud storage."""
 
 
+def upload_status(data, status_filename):
+  """Upload json file to cloud storage."""
+  bucket = builds_status.get_storage_client().get_bucket(
+      builds_status.STATUS_BUCKET)
+  blob = bucket.blob(status_filename)
+  blob.cache_control = 'no-cache'
+  blob.upload_from_string(json.dumps(data), content_type='application/json')
+
+
+def sort_projects(projects):
+  """Sort projects in order Failures, Successes, Not yet built."""
+
+  def key_func(project):
+    if not project['history']:
+      return 2  # Order projects without history last.
+
+    if project['history'][0]['success']:
+      # Successful builds come second.
+      return 1
+
+    # Build failures come first.
+    return 0
+
+  projects.sort(key=key_func)
+
+
+def get_build(cloudbuild, image_project, build_id):
+  """Get build object from cloudbuild."""
+  return cloudbuild.projects().builds().get(projectId=image_project,
+                                            id=build_id).execute()
+
+
+def update_last_successful_build(project, build_tag):
+  """Update last successful build."""
+  last_successful_build = ndb.Key(LastSuccessfulBuild,
+                                  project['name'] + '-' + build_tag).get()
+  if not last_successful_build and 'last_successful_build' not in project:
+    return
+
+  if 'last_successful_build' not in project:
+    project['last_successful_build'] = {
+        'build_id': last_successful_build.build_id,
+        'finish_time': last_successful_build.finish_time
+    }
+  else:
+    if last_successful_build:
+      last_successful_build.build_id = project['last_successful_build'][
+          'build_id']
+      last_successful_build.finish_time = project['last_successful_build'][
+          'finish_time']
+    else:
+      last_successful_build = LastSuccessfulBuild(
+          id=project['name'] + '-' + build_tag,
+          project=project['name'],
+          build_id=project['last_successful_build']['build_id'],
+          finish_time=project['last_successful_build']['finish_time'])
+    last_successful_build.put()
+
+
 # pylint: disable=no-member
-def get_last_build(build_ids):
+def get_build_history(build_ids):
   """Returns build object for the last finished build of project."""
   credentials, image_project = google.auth.default()
   cloudbuild = build('cloudbuild',
@@ -43,52 +104,60 @@ def get_last_build(build_ids):
                      credentials=credentials,
                      cache_discovery=False)
 
+  history = []
+  last_successful_build = None
+
   for build_id in reversed(build_ids):
-    project_build = cloudbuild.projects().builds().get(projectId=image_project,
-                                                       id=build_id).execute()
+    project_build = get_build(cloudbuild, image_project, build_id)
     if project_build['status'] not in ('SUCCESS', 'FAILURE', 'TIMEOUT'):
       continue
 
+    if (not last_successful_build and
+        builds_status.is_build_successful(project_build)):
+      last_successful_build = {
+          'build_id': build_id,
+          'finish_time': project_build['finishTime'],
+      }
+
     if not builds_status.upload_log(build_id):
       log_name = 'log-{0}'.format(build_id)
       raise MissingBuildLogError('Missing build log file {0}'.format(log_name))
 
-    return project_build
+    history.append({
+        'build_id': build_id,
+        'finish_time': project_build['finishTime'],
+        'success': builds_status.is_build_successful(project_build)
+    })
+
+    if len(history) == MAX_BUILD_LOGS:
+      break
 
-  return None
+  project = {'history': history}
+  if last_successful_build:
+    project['last_successful_build'] = last_successful_build
+  return project
 
 
+# pylint: disable=too-many-locals
 def update_build_status(build_tag, status_filename):
   """Update build statuses."""
+  projects = []
   statuses = {}
-  successes = []
-  failures = []
   for project_build in BuildsHistory.query(
-      BuildsHistory.build_tag == build_tag):
-    last_build = get_last_build(project_build.build_ids)
-    if not last_build:
-      logging.error('Failed to get last build for project %s',
-                    project_build.project)
-      continue
+      BuildsHistory.build_tag == build_tag).order('project'):
+
+    project = get_build_history(project_build.build_ids)
+    project['name'] = project_build.project
+    projects.append(project)
+    if project['history']:
+      statuses[project_build.project] = project['history'][0]['success']
+
+    update_last_successful_build(project, build_tag)
+
+  sort_projects(projects)
+  data = {'projects': projects}
+  upload_status(data, status_filename)
 
-    if last_build['status'] == 'SUCCESS':
-      statuses[project_build.project] = True
-      successes.append({
-          'name': project_build.project,
-          'build_id': last_build['id'],
-          'finish_time': last_build['finishTime'],
-          'success': True,
-      })
-    else:
-      statuses[project_build.project] = False
-      failures.append({
-          'name': project_build.project,
-          'build_id': last_build['id'],
-          'finish_time': last_build['finishTime'],
-          'success': False,
-      })
-
-  builds_status.upload_status(successes, failures, status_filename)
   return statuses
 
 
diff --git a/infra/build/functions/update_build_status_test.py b/infra/build/functions/update_build_status_test.py
new file mode 100644
index 00000000..a65553b0
--- /dev/null
+++ b/infra/build/functions/update_build_status_test.py
@@ -0,0 +1,298 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+"""Unit tests for Cloud Function update builds status."""
+import unittest
+from unittest import mock
+from unittest.mock import MagicMock
+
+from google.cloud import ndb
+
+from datastore_entities import BuildsHistory
+from datastore_entities import LastSuccessfulBuild
+import test_utils
+import update_build_status
+
+
+# pylint: disable=too-few-public-methods
+class MockGetBuild:
+  """Spoofing get_builds function."""
+
+  def __init__(self, builds):
+    self.builds = builds
+
+  def get_build(self, cloudbuild, image_project, build_id):
+    """Mimic build object retrieval."""
+    del cloudbuild, image_project
+    for build in self.builds:
+      if build['build_id'] == build_id:
+        return build
+
+    return None
+
+
+@mock.patch('google.auth.default', return_value=['temp', 'temp'])
+@mock.patch('update_build_status.build', return_value='cloudbuild')
+@mock.patch('builds_status.upload_log')
+class TestGetBuildHistory(unittest.TestCase):
+  """Unit tests for get_build_history."""
+
+  def test_get_build_history(self, mocked_upload_log, mocked_cloud_build,
+                             mocked_google_auth):
+    """Test for get_build_steps."""
+    del mocked_cloud_build, mocked_google_auth
+    mocked_upload_log.return_value = True
+    builds = [{'build_id': '1', 'finishTime': 'test_time', 'status': 'SUCCESS'}]
+    mocked_get_build = MockGetBuild(builds)
+    update_build_status.get_build = mocked_get_build.get_build
+
+    expected_projects = {
+        'history': [{
+            'build_id': '1',
+            'finish_time': 'test_time',
+            'success': True
+        }],
+        'last_successful_build': {
+            'build_id': '1',
+            'finish_time': 'test_time'
+        }
+    }
+    self.assertDictEqual(update_build_status.get_build_history(['1']),
+                         expected_projects)
+
+  def test_get_build_history_missing_log(self, mocked_upload_log,
+                                         mocked_cloud_build,
+                                         mocked_google_auth):
+    """Test for missing build log file."""
+    del mocked_cloud_build, mocked_google_auth
+    builds = [{'build_id': '1', 'finishTime': 'test_time', 'status': 'SUCCESS'}]
+    mocked_get_build = MockGetBuild(builds)
+    update_build_status.get_build = mocked_get_build.get_build
+    mocked_upload_log.return_value = False
+    self.assertRaises(update_build_status.MissingBuildLogError,
+                      update_build_status.get_build_history, ['1'])
+
+  def test_get_build_history_no_last_success(self, mocked_upload_log,
+                                             mocked_cloud_build,
+                                             mocked_google_auth):
+    """Test when there is no last successful build."""
+    del mocked_cloud_build, mocked_google_auth
+    builds = [{'build_id': '1', 'finishTime': 'test_time', 'status': 'FAILURE'}]
+    mocked_get_build = MockGetBuild(builds)
+    update_build_status.get_build = mocked_get_build.get_build
+    mocked_upload_log.return_value = True
+
+    expected_projects = {
+        'history': [{
+            'build_id': '1',
+            'finish_time': 'test_time',
+            'success': False
+        }]
+    }
+    self.assertDictEqual(update_build_status.get_build_history(['1']),
+                         expected_projects)
+
+
+class TestSortProjects(unittest.TestCase):
+  """Unit tests for testing sorting functionality."""
+
+  def test_sort_projects(self):
+    """Test sorting functionality."""
+    projects = [{
+        'name': '1',
+        'history': []
+    }, {
+        'name': '2',
+        'history': [{
+            'success': True
+        }]
+    }, {
+        'name': '3',
+        'history': [{
+            'success': False
+        }]
+    }]
+    expected_order = ['3', '2', '1']
+    update_build_status.sort_projects(projects)
+    self.assertEqual(expected_order, [project['name'] for project in projects])
+
+
+class TestUpdateLastSuccessfulBuild(unittest.TestCase):
+  """Unit tests for updating last successful build."""
+
+  @classmethod
+  def setUpClass(cls):
+    cls.ds_emulator = test_utils.start_datastore_emulator()
+    test_utils.wait_for_emulator_ready(cls.ds_emulator, 'datastore',
+                                       test_utils.DATASTORE_READY_INDICATOR)
+    test_utils.set_gcp_environment()
+
+  def setUp(self):
+    test_utils.reset_ds_emulator()
+
+  def test_update_last_successful_build_new(self):
+    """When last successful build isn't available in datastore."""
+    with ndb.Client().context():
+      project = {
+          'name': 'test-project',
+          'last_successful_build': {
+              'build_id': '1',
+              'finish_time': 'test_time'
+          }
+      }
+      update_build_status.update_last_successful_build(project, 'fuzzing')
+      expected_build_id = '1'
+      self.assertEqual(
+          expected_build_id,
+          ndb.Key(LastSuccessfulBuild, 'test-project-fuzzing').get().build_id)
+
+  def test_update_last_successful_build_datastore(self):
+    """When last successful build is only available in datastore."""
+    with ndb.Client().context():
+      project = {'name': 'test-project'}
+      LastSuccessfulBuild(id='test-project-fuzzing',
+                          build_tag='fuzzing',
+                          project='test-project',
+                          build_id='1',
+                          finish_time='test_time').put()
+
+      update_build_status.update_last_successful_build(project, 'fuzzing')
+      expected_project = {
+          'name': 'test-project',
+          'last_successful_build': {
+              'build_id': '1',
+              'finish_time': 'test_time'
+          }
+      }
+      self.assertDictEqual(project, expected_project)
+
+  def test_update_last_successful_build(self):
+    """When last successful build is available at both places."""
+    with ndb.Client().context():
+      project = {
+          'name': 'test-project',
+          'last_successful_build': {
+              'build_id': '2',
+              'finish_time': 'test_time'
+          }
+      }
+      LastSuccessfulBuild(id='test-project-fuzzing',
+                          build_tag='fuzzing',
+                          project='test-project',
+                          build_id='1',
+                          finish_time='test_time').put()
+
+      update_build_status.update_last_successful_build(project, 'fuzzing')
+      expected_build_id = '2'
+      self.assertEqual(
+          expected_build_id,
+          ndb.Key(LastSuccessfulBuild, 'test-project-fuzzing').get().build_id)
+
+  @classmethod
+  def tearDownClass(cls):
+    test_utils.cleanup_emulator(cls.ds_emulator)
+
+
+class TestUpdateBuildStatus(unittest.TestCase):
+  """Unit test for update build status."""
+
+  @classmethod
+  def setUpClass(cls):
+    cls.ds_emulator = test_utils.start_datastore_emulator()
+    test_utils.wait_for_emulator_ready(cls.ds_emulator, 'datastore',
+                                       test_utils.DATASTORE_READY_INDICATOR)
+    test_utils.set_gcp_environment()
+
+  def setUp(self):
+    test_utils.reset_ds_emulator()
+
+  # pylint: disable=no-self-use
+  @mock.patch('google.auth.default', return_value=['temp', 'temp'])
+  @mock.patch('update_build_status.build', return_value='cloudbuild')
+  @mock.patch('builds_status.upload_log')
+  def test_update_build_status(self, mocked_upload_log, mocked_cloud_build,
+                               mocked_google_auth):
+    """Testing update build status as a whole."""
+    del self, mocked_cloud_build, mocked_google_auth
+    update_build_status.upload_status = MagicMock()
+    mocked_upload_log.return_value = True
+    status_filename = 'status.json'
+    with ndb.Client().context():
+      BuildsHistory(id='test-project-1-fuzzing',
+                    build_tag='fuzzing',
+                    project='test-project-1',
+                    build_ids=['1']).put()
+
+      BuildsHistory(id='test-project-2-fuzzing',
+                    build_tag='fuzzing',
+                    project='test-project-2',
+                    build_ids=['2']).put()
+
+      BuildsHistory(id='test-project-3-fuzzing',
+                    build_tag='fuzzing',
+                    project='test-project-3',
+                    build_ids=['3']).put()
+
+      builds = [{
+          'build_id': '1',
+          'finishTime': 'test_time',
+          'status': 'SUCCESS'
+      }, {
+          'build_id': '2',
+          'finishTime': 'test_time',
+          'status': 'FAILURE'
+      }, {
+          'build_id': '3',
+          'status': 'WORKING'
+      }]
+      mocked_get_build = MockGetBuild(builds)
+      update_build_status.get_build = mocked_get_build.get_build
+
+      expected_data = {
+          'projects': [{
+              'history': [{
+                  'build_id': '2',
+                  'finish_time': 'test_time',
+                  'success': False
+              }],
+              'name': 'test-project-2'
+          }, {
+              'history': [{
+                  'build_id': '1',
+                  'finish_time': 'test_time',
+                  'success': True
+              }],
+              'last_successful_build': {
+                  'build_id': '1',
+                  'finish_time': 'test_time'
+              },
+              'name': 'test-project-1'
+          }, {
+              'history': [],
+              'name': 'test-project-3'
+          }]
+      }
+
+      update_build_status.update_build_status('fuzzing', 'status.json')
+      update_build_status.upload_status.assert_called_with(
+          expected_data, status_filename)
+
+  @classmethod
+  def tearDownClass(cls):
+    test_utils.cleanup_emulator(cls.ds_emulator)
+
+
+if __name__ == '__main__':
+  unittest.main(exit=False)