Test SKQP on Firebase Testlab and Upload

- adds building the testlab driver (run_testlab) as a separate step
- adds gcloud isolate necessary to run testlab
- adds Testlab support and uploading a verified AKP to GCS (with
  meta data attached).

Bug: skia:
Change-Id: I1bf265f46c99360eb3a9eb684886f93de48085fe
Reviewed-on: https://skia-review.googlesource.com/111603
Reviewed-by: Eric Boren <borenet@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Commit-Queue: Stephan Altmueller <stephana@google.com>

2 files changed, 312 insertions, 96 deletions

diff --git a/infra/cts/run_testlab.go b/infra/cts/run_testlab.go
index 6a7f76591c..b764bb825a 100644
--- a/infra/cts/run_testlab.go
+++ b/infra/cts/run_testlab.go
@@ -9,9 +9,12 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
@@ -21,51 +24,16 @@ import (
 	"syscall"
 	"time"
 
+	"go.skia.org/infra/go/gcs"
+
+	"cloud.google.com/go/storage"
+	"google.golang.org/api/option"
 	gstorage "google.golang.org/api/storage/v1"
 
 	"go.skia.org/infra/go/auth"
 	"go.skia.org/infra/go/common"
 	"go.skia.org/infra/go/sklog"
 	"go.skia.org/infra/go/util"
-	"go.skia.org/infra/golden/go/tsuite"
-)
-
-// TODO(stephana): Convert the hard coded whitelist to a command line flag that
-// loads a file with the whitelisted devices and versions. Make sure to include
-// human readable names for the devices.
-
-var (
-	// WHITELIST_DEV_IDS contains a mapping from the device id to the list of
-	// Android API versions that we should run agains. Usually this will be the
-	// latest version. To see available devices and version run with
-	// --dryrun flag or run '$ gcloud firebase test android models list'
-
-	WHITELIST_DEV_IDS = map[string][]string{
-		"A0001": {"22"},
-		// "E5803":       {"22"},    deprecated
-		// "F5121":       {"23"},    deprecated
-		"G8142":      {"25"},
-		"HWMHA":      {"24"},
-		"SH-04H":     {"23"},
-		"athene":     {"23"},
-		"athene_f":   {"23"},
-		"hammerhead": {"23"},
-		"harpia":     {"23"},
-		"hero2lte":   {"23"},
-		"herolte":    {"24"},
-		"j1acevelte": {"22"},
-		"j5lte":      {"23"},
-		"j7xelte":    {"23"},
-		"lucye":      {"24"},
-		// "mako":        {"22"},   deprecated
-		"osprey_umts": {"22"},
-		// "p1":          {"22"},   deprecated
-		"sailfish": {"26"},
-		"shamu":    {"23"},
-		"trelte":   {"22"},
-		"zeroflte": {"22"},
-		"zerolte":  {"22"},
-	}
 )
 
 const (
@@ -74,10 +42,14 @@ const (
 
 // Command line flags.
 var (
-	serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
+	devicesFile        = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.")
 	dryRun             = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
-	minAPIVersion      = flag.Int("min_api", 22, "Minimum API version required by device.")
-	maxAPIVersion      = flag.Int("max_api", 23, "Maximum API version required by device.")
+	dumpDevFile        = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.")
+	minAPIVersion      = flag.Int("min_api", 0, "Minimum API version required by device.")
+	maxAPIVersion      = flag.Int("max_api", 99, "Maximum API version required by device.")
+	properties         = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.")
+	serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
+	uploadGCSPath      = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).")
 )
 
 const (
@@ -90,88 +62,145 @@ const (
 	--timeout 30m
 	%s
 `
-	MODEL_VERSION_TMPL   = "--device model=%s,version=%s,orientation=portrait"
-	RESULT_BUCKET        = "skia-firebase-test-lab"
-	RESULT_DIR_TMPL      = "testruns/%s/%s"
-	RUN_ID_TMPL          = "testrun-%d"
-	CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json"
+	MODEL_VERSION_TMPL    = "--device model=%s,version=%s,orientation=portrait"
+	RESULT_BUCKET         = "skia-firebase-test-lab"
+	RESULT_DIR_TMPL       = "testruns/%s/%s"
+	RUN_ID_TMPL           = "testrun-%d"
+	CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json"
 )
 
 func main() {
 	common.Init()
 
-	// Get the apk.
-	args := flag.Args()
-	apk_path := args[0]
+	// Get the path to the APK. It can be empty if we are dumping the device list.
+	apkPath := flag.Arg(0)
+	if *dumpDevFile == "" && apkPath == "" {
+		sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.")
+		os.Exit(1)
+	}
 
-	// Make sure we can get the service account client.
-	client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
+	// Get the available devices.
+	fbDevices, deviceList, err := getAvailableDevices()
 	if err != nil {
-		sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
+		sklog.Fatalf("Error retrieving available devices: %s", err)
 	}
 
-	// Get list of all available devices.
-	devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS, *minAPIVersion, *maxAPIVersion)
+	// Dump the device list and exit.
+	if *dumpDevFile != "" {
+		if err := writeDeviceList(*dumpDevFile, deviceList); err != nil {
+			sklog.Fatalf("Unable to write devices: %s", err)
+		}
+		return
+	}
+
+	// If no devices are explicitly listed. Use all of them.
+	whiteList := deviceList
+	if *devicesFile != "" {
+		whiteList, err = readDeviceList(*devicesFile)
+		if err != nil {
+			sklog.Fatalf("Error reading device file: %s", err)
+		}
+	}
+
+	// Make sure we can authenticate locally and in the cloud.
+	client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
 	if err != nil {
-		sklog.Fatalf("Unable to retrieve available devices: %s", err)
+		sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
 	}
-	sklog.Infof("---")
-	sklog.Infof("Selected devices:")
+
+	// Filter the devices according the white list and other parameters.
+	devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion)
+	sklog.Infof("---\nSelected devices:")
 	logDevices(devices)
 
-	if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil {
-		sklog.Fatalf("Error triggering tests on Firebase: %s", err)
+	if len(devices) == 0 {
+		sklog.Errorf("No devices selected. Not running tests.")
+		os.Exit(1)
+	}
+
+	if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil {
+		sklog.Fatalf("Error running tests on Firebase: %s", err)
+	}
+
+	if !*dryRun && (*uploadGCSPath != "") && (*properties != "") {
+		if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil {
+			sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err)
+		}
 	}
 }
 
-// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all
-// available devices and then returns a list of devices to be tested and the list
-// of ignored devices.
-func getAvailableDevices(whiteList map[string][]string, minAPIVersion, maxAPIVersion int) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) {
+// getAvailableDevices queries Firebase Testlab for all physical devices that
+// are not deprecated. It returns two lists with the same information.
+// The first contains all device information as returned by Firebase while
+// the second contains the information necessary to use in a whitelist.
+func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) {
 	// Get the list of all devices in JSON format from Firebase testlab.
 	var buf bytes.Buffer
-	cmd := parseCommand(CMD_AVAILABE_DEVICES)
+	var errBuf bytes.Buffer
+	cmd := parseCommand(CMD_AVAILABLE_DEVICES)
 	cmd.Stdout = &buf
-	cmd.Stderr = os.Stdout
+	cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
 	if err := cmd.Run(); err != nil {
-		return nil, nil, err
+		return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf)
 	}
 
 	// Unmarshal the result.
-	foundDevices := []*tsuite.FirebaseDevice{}
+	foundDevices := []*DeviceVersions{}
 	bufBytes := buf.Bytes()
 	if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
 		return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes))
 	}
 
+	// Filter the devices and copy them to device list.
+	devList := DeviceList{}
+	ret := make([]*DeviceVersions, 0, len(foundDevices))
+	for _, foundDev := range foundDevices {
+		// Only consider physical devices and devices that are not deprecated.
+		if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) {
+			ret = append(ret, foundDev)
+			devList = append(devList, &DevInfo{
+				ID:          foundDev.ID,
+				Name:        foundDev.Name,
+				RunVersions: foundDev.VersionIDs,
+			})
+		}
+	}
+	return foundDevices, devList, nil
+}
+
+// filterDevices filters the given devices by ensuring that they are in the white list
+// and within the given api version range.
+// It returns two lists: (accepted_devices, ignored_devices)
+func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) {
 	// iterate over the available devices and partition them.
-	allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
-	ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
-	ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
+	allDevices := make([]*DeviceVersions, 0, len(foundDevices))
+	ret := make([]*DeviceVersions, 0, len(foundDevices))
+	ignored := make([]*DeviceVersions, 0, len(foundDevices))
 	for _, dev := range foundDevices {
-		// Filter out all the virtual devices.
-		if dev.Form == "PHYSICAL" {
-			// Only include devices that are on the whitelist and have versions defined.
-			if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) {
-				versionSet := util.NewStringSet(dev.VersionIDs)
-				reqVersions := util.NewStringSet(filterVersions(foundVersions, minAPIVersion, maxAPIVersion))
-				whiteListVersions := versionSet.Intersect(reqVersions).Keys()
-				ignoredVersions := versionSet.Complement(reqVersions).Keys()
-				sort.Strings(whiteListVersions)
-				sort.Strings(ignoredVersions)
-				ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions})
-				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions})
-			} else {
-				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
+		// Only include devices that are on the whitelist and have versions defined.
+		if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) {
+			versionSet := util.NewStringSet(dev.VersionIDs)
+			reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion))
+			whiteListVersions := versionSet.Intersect(reqVersions).Keys()
+			ignoredVersions := versionSet.Complement(reqVersions).Keys()
+			sort.Strings(whiteListVersions)
+			sort.Strings(ignoredVersions)
+			if len(whiteListVersions) > 0 {
+				ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions})
+			}
+			if len(ignoredVersions) > 0 {
+				ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions})
 			}
-			allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
+		} else {
+			ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
 		}
+		allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
 	}
 
 	sklog.Infof("All devices:")
 	logDevices(allDevices)
 
-	return ret, ignored, nil
+	return ret, ignored
 }
 
 // filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion.
@@ -190,12 +219,12 @@ func filterVersions(versionIDs []string, minVersion, maxVersion int) []string {
 }
 
 // runTests runs the given apk on the given list of devices.
-func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error {
+func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error {
 	// Get the model-version we want to test. Assume on average each model has 5 supported versions.
 	modelSelectors := make([]string, 0, len(devices)*5)
 	for _, devRec := range devices {
-		for _, version := range devRec.Versions {
-			modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version))
+		for _, version := range devRec.RunVersions {
+			modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version))
 		}
 	}
 
@@ -207,9 +236,10 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
 	cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))
 
 	// Run the command.
+	var errBuf bytes.Buffer
 	cmd := parseCommand(cmdStr)
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stdout
+	cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
 	exitCode := 0
 
 	if dryRun {
@@ -223,18 +253,19 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
 			ws := exitError.Sys().(syscall.WaitStatus)
 			exitCode = ws.ExitStatus()
 		}
+
 		sklog.Errorf("Error running tests: %s", err)
 		sklog.Errorf("Exit code: %d", exitCode)
 
 		// Exit code 10 means triggering on Testlab succeeded, but but some of the
 		// runs on devices failed. We consider it a success for this script.
 		if exitCode != 10 {
-			return err
+			return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf)
 		}
 	}
 
 	// Store the result in a meta json file.
-	meta := &tsuite.TestRunMeta{
+	meta := &TestRunMeta{
 		ID:             runID,
 		TS:             nowMs,
 		Devices:        devices,
@@ -242,15 +273,56 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
 		ExitCode:       exitCode,
 	}
 
-	meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client)
+	targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME)
+	if err := meta.writeToGCS(targetPath, client); err != nil {
+		return err
+	}
+	sklog.Infof("Meta data written to gs://%s", targetPath)
+	return nil
+}
+
+// uploadAPK uploads the APK at the given path to the bucket/path in gcsPath.
+// The key-value pairs in propStr are set as custom meta data of the APK.
+func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error {
+	properties, err := splitProperties(propStr)
+	if err != nil {
+		return err
+	}
+	apkFile, err := os.Open(apkPath)
+	if err != nil {
+		return err
+	}
+	defer util.Close(apkFile)
+
+	if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil {
+		return err
+	}
+
+	sklog.Infof("APK uploaded to gs://%s", gcsPath)
 	return nil
 }
 
+// splitProperties receives a comma separated list of 'key=value' pairs and
+// returnes them as a map.
+func splitProperties(propStr string) (map[string]string, error) {
+	splitProps := strings.Split(propStr, ",")
+	properties := make(map[string]string, len(splitProps))
+	for _, oneProp := range splitProps {
+		kv := strings.Split(oneProp, "=")
+		if len(kv) != 2 {
+			return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr)
+		}
+		properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
+	}
+	return properties, nil
+}
+
 // logDevices logs the given list of devices.
-func logDevices(devices []*tsuite.DeviceVersions) {
+func logDevices(devices []*DeviceVersions) {
 	sklog.Infof("Found %d devices.", len(devices))
 	for _, dev := range devices {
-		sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions)
+		fbDev := dev.FirebaseDevice
+		sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions)
 	}
 }
 
@@ -262,3 +334,138 @@ func parseCommand(cmdStr string) *exec.Cmd {
 	}
 	return exec.Command(cmdArgs[0], cmdArgs[1:]...)
 }
+
+// DeviceList is a simple list of devices, primarily used to define the
+// whitelist of devices we want to run on.
+type DeviceList []*DevInfo
+
+type DevInfo struct {
+	ID          string   `json:"id"`
+	Name        string   `json:"name"`
+	RunVersions []string `json:"runVersions"`
+}
+
+func (d DeviceList) find(id string) *DevInfo {
+	for _, devInfo := range d {
+		if devInfo.ID == id {
+			return devInfo
+		}
+	}
+	return nil
+}
+
+func writeDeviceList(fileName string, devList DeviceList) error {
+	jsonBytes, err := json.MarshalIndent(devList, "", "  ")
+	if err != nil {
+		return sklog.FmtErrorf("Unable to encode JSON: %s", err)
+	}
+
+	if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil {
+		sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err)
+	}
+	return nil
+}
+
+func readDeviceList(fileName string) (DeviceList, error) {
+	inFile, err := os.Open(fileName)
+	if err != nil {
+		return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err)
+	}
+	defer util.Close(inFile)
+
+	var devList DeviceList
+	if err := json.NewDecoder(inFile).Decode(&devList); err != nil {
+		return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err)
+	}
+	return devList, nil
+}
+
+// FirebaseDevice contains the information and JSON tags for device information
+// returned by firebase.
+type FirebaseDevice struct {
+	Brand        string   `json:"brand"`
+	Form         string   `json:"form"`
+	ID           string   `json:"id"`
+	Manufacturer string   `json:"manufacturer"`
+	Name         string   `json:"name"`
+	VersionIDs   []string `json:"supportedVersionIds"`
+	Tags         []string `json:"tags"`
+}
+
+// DeviceVersions combines device information from Firebase Testlab with
+// a selected list of versions. This is used to define a subset of versions
+// used by a devices.
+type DeviceVersions struct {
+	*FirebaseDevice
+
+	// RunVersions contains the version ids of interest contained in Device.
+	RunVersions []string
+}
+
+// TestRunMeta contains the meta data of a complete testrun on firebase.
+type TestRunMeta struct {
+	ID             string            `json:"id"`
+	TS             int64             `json:"timeStamp"`
+	Devices        []*DeviceVersions `json:"devices"`
+	IgnoredDevices []*DeviceVersions `json:"ignoredDevices"`
+	ExitCode       int               `json:"exitCode"`
+}
+
+// writeToGCS writes the meta data as JSON to the given bucket and path in
+// GCS. It assumes that the provided client has permissions to write to the
+// specified location in GCS.
+func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error {
+	jsonBytes, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+	return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true)
+}
+
+// TODO(stephana): Merge copyReaderToGCS into the go/gcs in
+// the infra repository.
+
+// copyReaderToGCS reads all available content from the given reader and writes
+// it to the given path in GCS.
+func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error {
+	storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
+	if err != nil {
+		return err
+	}
+	bucket, path := gcs.SplitGSPath(gcsPath)
+	w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background())
+
+	// Set the content if requested.
+	if contentType != "" {
+		w.ObjectAttrs.ContentType = contentType
+	}
+
+	// Set the meta data if requested
+	if metaData != nil {
+		w.Metadata = metaData
+	}
+
+	// Make the object public if requested.
+	if public {
+		w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
+	}
+
+	// Write the everything the reader can provide to the GCS object. Either
+	// gzip'ed or plain.
+	if gzip {
+		w.ObjectAttrs.ContentEncoding = "gzip"
+		err = util.WithGzipWriter(w, func(w io.Writer) error {
+			_, err := io.Copy(w, reader)
+			return err
+		})
+	} else {
+		_, err = io.Copy(w, reader)
+	}
+
+	// Make sure we return an error when we close the remote object.
+	if err != nil {
+		_ = w.CloseWithError(err)
+		return err
+	}
+	return w.Close()
+}
diff --git a/infra/cts/whitelist_devices.json b/infra/cts/whitelist_devices.json
new file mode 100644
index 0000000000..86838c6f86
--- /dev/null
+++ b/infra/cts/whitelist_devices.json
@@ -0,0 +1,9 @@
+[
+  {
+    "id": "sailfish",
+    "name": "Pixel",
+    "runVersions": [
+      "26"
+    ]
+  }
+]
\ No newline at end of file