aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra/cts
diff options
context:
space:
mode:
Diffstat (limited to 'infra/cts')
-rw-r--r--infra/cts/run_testlab.go399
-rw-r--r--infra/cts/whitelist_devices.json9
2 files changed, 312 insertions, 96 deletions
diff --git a/infra/cts/run_testlab.go b/infra/cts/run_testlab.go
index 6a7f76591c..b764bb825a 100644
--- a/infra/cts/run_testlab.go
+++ b/infra/cts/run_testlab.go
@@ -9,9 +9,12 @@ package main
import (
"bytes"
+ "context"
"encoding/json"
"flag"
"fmt"
+ "io"
+ "io/ioutil"
"net/http"
"os"
"os/exec"
@@ -21,51 +24,16 @@ import (
"syscall"
"time"
+ "go.skia.org/infra/go/gcs"
+
+ "cloud.google.com/go/storage"
+ "google.golang.org/api/option"
gstorage "google.golang.org/api/storage/v1"
"go.skia.org/infra/go/auth"
"go.skia.org/infra/go/common"
"go.skia.org/infra/go/sklog"
"go.skia.org/infra/go/util"
- "go.skia.org/infra/golden/go/tsuite"
-)
-
-// TODO(stephana): Convert the hard coded whitelist to a command line flag that
-// loads a file with the whitelisted devices and versions. Make sure to include
-// human readable names for the devices.
-
-var (
- // WHITELIST_DEV_IDS contains a mapping from the device id to the list of
- // Android API versions that we should run agains. Usually this will be the
- // latest version. To see available devices and version run with
- // --dryrun flag or run '$ gcloud firebase test android models list'
-
- WHITELIST_DEV_IDS = map[string][]string{
- "A0001": {"22"},
- // "E5803": {"22"}, deprecated
- // "F5121": {"23"}, deprecated
- "G8142": {"25"},
- "HWMHA": {"24"},
- "SH-04H": {"23"},
- "athene": {"23"},
- "athene_f": {"23"},
- "hammerhead": {"23"},
- "harpia": {"23"},
- "hero2lte": {"23"},
- "herolte": {"24"},
- "j1acevelte": {"22"},
- "j5lte": {"23"},
- "j7xelte": {"23"},
- "lucye": {"24"},
- // "mako": {"22"}, deprecated
- "osprey_umts": {"22"},
- // "p1": {"22"}, deprecated
- "sailfish": {"26"},
- "shamu": {"23"},
- "trelte": {"22"},
- "zeroflte": {"22"},
- "zerolte": {"22"},
- }
)
const (
@@ -74,10 +42,14 @@ const (
// Command line flags.
var (
- serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
+ devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.")
dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
- minAPIVersion = flag.Int("min_api", 22, "Minimum API version required by device.")
- maxAPIVersion = flag.Int("max_api", 23, "Maximum API version required by device.")
+ dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.")
+ minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.")
+ maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.")
+ properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.")
+ serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
+ uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).")
)
const (
@@ -90,88 +62,145 @@ const (
--timeout 30m
%s
`
- MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait"
- RESULT_BUCKET = "skia-firebase-test-lab"
- RESULT_DIR_TMPL = "testruns/%s/%s"
- RUN_ID_TMPL = "testrun-%d"
- CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json"
+ MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait"
+ RESULT_BUCKET = "skia-firebase-test-lab"
+ RESULT_DIR_TMPL = "testruns/%s/%s"
+ RUN_ID_TMPL = "testrun-%d"
+ CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json"
)
func main() {
common.Init()
- // Get the apk.
- args := flag.Args()
- apk_path := args[0]
+ // Get the path to the APK. It can be empty if we are dumping the device list.
+ apkPath := flag.Arg(0)
+ if *dumpDevFile == "" && apkPath == "" {
+ sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.")
+ os.Exit(1)
+ }
- // Make sure we can get the service account client.
- client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
+ // Get the available devices.
+ fbDevices, deviceList, err := getAvailableDevices()
if err != nil {
- sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
+ sklog.Fatalf("Error retrieving available devices: %s", err)
}
- // Get list of all available devices.
- devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS, *minAPIVersion, *maxAPIVersion)
+ // Dump the device list and exit.
+ if *dumpDevFile != "" {
+ if err := writeDeviceList(*dumpDevFile, deviceList); err != nil {
+ sklog.Fatalf("Unable to write devices: %s", err)
+ }
+ return
+ }
+
+ // If no devices are explicitly listed. Use all of them.
+ whiteList := deviceList
+ if *devicesFile != "" {
+ whiteList, err = readDeviceList(*devicesFile)
+ if err != nil {
+ sklog.Fatalf("Error reading device file: %s", err)
+ }
+ }
+
+ // Make sure we can authenticate locally and in the cloud.
+ client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
if err != nil {
- sklog.Fatalf("Unable to retrieve available devices: %s", err)
+ sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
}
- sklog.Infof("---")
- sklog.Infof("Selected devices:")
+
+ // Filter the devices according the white list and other parameters.
+ devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion)
+ sklog.Infof("---\nSelected devices:")
logDevices(devices)
- if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil {
- sklog.Fatalf("Error triggering tests on Firebase: %s", err)
+ if len(devices) == 0 {
+ sklog.Errorf("No devices selected. Not running tests.")
+ os.Exit(1)
+ }
+
+ if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil {
+ sklog.Fatalf("Error running tests on Firebase: %s", err)
+ }
+
+ if !*dryRun && (*uploadGCSPath != "") && (*properties != "") {
+ if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil {
+ sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err)
+ }
}
}
-// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all
-// available devices and then returns a list of devices to be tested and the list
-// of ignored devices.
-func getAvailableDevices(whiteList map[string][]string, minAPIVersion, maxAPIVersion int) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) {
+// getAvailableDevices queries Firebase Testlab for all physical devices that
+// are not deprecated. It returns two lists with the same information.
+// The first contains all device information as returned by Firebase while
+// the second contains the information necessary to use in a whitelist.
+func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) {
// Get the list of all devices in JSON format from Firebase testlab.
var buf bytes.Buffer
- cmd := parseCommand(CMD_AVAILABE_DEVICES)
+ var errBuf bytes.Buffer
+ cmd := parseCommand(CMD_AVAILABLE_DEVICES)
cmd.Stdout = &buf
- cmd.Stderr = os.Stdout
+ cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
if err := cmd.Run(); err != nil {
- return nil, nil, err
+ return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf)
}
// Unmarshal the result.
- foundDevices := []*tsuite.FirebaseDevice{}
+ foundDevices := []*DeviceVersions{}
bufBytes := buf.Bytes()
if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes))
}
+ // Filter the devices and copy them to device list.
+ devList := DeviceList{}
+ ret := make([]*DeviceVersions, 0, len(foundDevices))
+ for _, foundDev := range foundDevices {
+ // Only consider physical devices and devices that are not deprecated.
+ if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) {
+ ret = append(ret, foundDev)
+ devList = append(devList, &DevInfo{
+ ID: foundDev.ID,
+ Name: foundDev.Name,
+ RunVersions: foundDev.VersionIDs,
+ })
+ }
+ }
+ return foundDevices, devList, nil
+}
+
+// filterDevices filters the given devices by ensuring that they are in the white list
+// and within the given api version range.
+// It returns two lists: (accepted_devices, ignored_devices)
+func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) {
// iterate over the available devices and partition them.
- allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
- ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
- ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
+ allDevices := make([]*DeviceVersions, 0, len(foundDevices))
+ ret := make([]*DeviceVersions, 0, len(foundDevices))
+ ignored := make([]*DeviceVersions, 0, len(foundDevices))
for _, dev := range foundDevices {
- // Filter out all the virtual devices.
- if dev.Form == "PHYSICAL" {
- // Only include devices that are on the whitelist and have versions defined.
- if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) {
- versionSet := util.NewStringSet(dev.VersionIDs)
- reqVersions := util.NewStringSet(filterVersions(foundVersions, minAPIVersion, maxAPIVersion))
- whiteListVersions := versionSet.Intersect(reqVersions).Keys()
- ignoredVersions := versionSet.Complement(reqVersions).Keys()
- sort.Strings(whiteListVersions)
- sort.Strings(ignoredVersions)
- ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions})
- ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions})
- } else {
- ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
+ // Only include devices that are on the whitelist and have versions defined.
+ if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) {
+ versionSet := util.NewStringSet(dev.VersionIDs)
+ reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion))
+ whiteListVersions := versionSet.Intersect(reqVersions).Keys()
+ ignoredVersions := versionSet.Complement(reqVersions).Keys()
+ sort.Strings(whiteListVersions)
+ sort.Strings(ignoredVersions)
+ if len(whiteListVersions) > 0 {
+ ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions})
+ }
+ if len(ignoredVersions) > 0 {
+ ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions})
}
- allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
+ } else {
+ ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
}
+ allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs})
}
sklog.Infof("All devices:")
logDevices(allDevices)
- return ret, ignored, nil
+ return ret, ignored
}
// filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion.
@@ -190,12 +219,12 @@ func filterVersions(versionIDs []string, minVersion, maxVersion int) []string {
}
// runTests runs the given apk on the given list of devices.
-func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error {
+func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error {
// Get the model-version we want to test. Assume on average each model has 5 supported versions.
modelSelectors := make([]string, 0, len(devices)*5)
for _, devRec := range devices {
- for _, version := range devRec.Versions {
- modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version))
+ for _, version := range devRec.RunVersions {
+ modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version))
}
}
@@ -207,9 +236,10 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))
// Run the command.
+ var errBuf bytes.Buffer
cmd := parseCommand(cmdStr)
cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stdout
+ cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf)
exitCode := 0
if dryRun {
@@ -223,18 +253,19 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
ws := exitError.Sys().(syscall.WaitStatus)
exitCode = ws.ExitStatus()
}
+
sklog.Errorf("Error running tests: %s", err)
sklog.Errorf("Exit code: %d", exitCode)
// Exit code 10 means triggering on Testlab succeeded, but but some of the
// runs on devices failed. We consider it a success for this script.
if exitCode != 10 {
- return err
+ return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf)
}
}
// Store the result in a meta json file.
- meta := &tsuite.TestRunMeta{
+ meta := &TestRunMeta{
ID: runID,
TS: nowMs,
Devices: devices,
@@ -242,15 +273,56 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions,
ExitCode: exitCode,
}
- meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client)
+ targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME)
+ if err := meta.writeToGCS(targetPath, client); err != nil {
+ return err
+ }
+ sklog.Infof("Meta data written to gs://%s", targetPath)
+ return nil
+}
+
+// uploadAPK uploads the APK at the given path to the bucket/path in gcsPath.
+// The key-value pairs in propStr are set as custom meta data of the APK.
+func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error {
+ properties, err := splitProperties(propStr)
+ if err != nil {
+ return err
+ }
+ apkFile, err := os.Open(apkPath)
+ if err != nil {
+ return err
+ }
+ defer util.Close(apkFile)
+
+ if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil {
+ return err
+ }
+
+ sklog.Infof("APK uploaded to gs://%s", gcsPath)
return nil
}
+// splitProperties receives a comma separated list of 'key=value' pairs and
+// returnes them as a map.
+func splitProperties(propStr string) (map[string]string, error) {
+ splitProps := strings.Split(propStr, ",")
+ properties := make(map[string]string, len(splitProps))
+ for _, oneProp := range splitProps {
+ kv := strings.Split(oneProp, "=")
+ if len(kv) != 2 {
+ return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr)
+ }
+ properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
+ }
+ return properties, nil
+}
+
// logDevices logs the given list of devices.
-func logDevices(devices []*tsuite.DeviceVersions) {
+func logDevices(devices []*DeviceVersions) {
sklog.Infof("Found %d devices.", len(devices))
for _, dev := range devices {
- sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions)
+ fbDev := dev.FirebaseDevice
+ sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions)
}
}
@@ -262,3 +334,138 @@ func parseCommand(cmdStr string) *exec.Cmd {
}
return exec.Command(cmdArgs[0], cmdArgs[1:]...)
}
+
+// DeviceList is a simple list of devices, primarily used to define the
+// whitelist of devices we want to run on.
+type DeviceList []*DevInfo
+
+type DevInfo struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ RunVersions []string `json:"runVersions"`
+}
+
+func (d DeviceList) find(id string) *DevInfo {
+ for _, devInfo := range d {
+ if devInfo.ID == id {
+ return devInfo
+ }
+ }
+ return nil
+}
+
+func writeDeviceList(fileName string, devList DeviceList) error {
+ jsonBytes, err := json.MarshalIndent(devList, "", " ")
+ if err != nil {
+ return sklog.FmtErrorf("Unable to encode JSON: %s", err)
+ }
+
+ if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil {
+ sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err)
+ }
+ return nil
+}
+
+func readDeviceList(fileName string) (DeviceList, error) {
+ inFile, err := os.Open(fileName)
+ if err != nil {
+ return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err)
+ }
+ defer util.Close(inFile)
+
+ var devList DeviceList
+ if err := json.NewDecoder(inFile).Decode(&devList); err != nil {
+ return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err)
+ }
+ return devList, nil
+}
+
+// FirebaseDevice contains the information and JSON tags for device information
+// returned by firebase.
+type FirebaseDevice struct {
+ Brand string `json:"brand"`
+ Form string `json:"form"`
+ ID string `json:"id"`
+ Manufacturer string `json:"manufacturer"`
+ Name string `json:"name"`
+ VersionIDs []string `json:"supportedVersionIds"`
+ Tags []string `json:"tags"`
+}
+
+// DeviceVersions combines device information from Firebase Testlab with
+// a selected list of versions. This is used to define a subset of versions
+// used by a devices.
+type DeviceVersions struct {
+ *FirebaseDevice
+
+ // RunVersions contains the version ids of interest contained in Device.
+ RunVersions []string
+}
+
+// TestRunMeta contains the meta data of a complete testrun on firebase.
+type TestRunMeta struct {
+ ID string `json:"id"`
+ TS int64 `json:"timeStamp"`
+ Devices []*DeviceVersions `json:"devices"`
+ IgnoredDevices []*DeviceVersions `json:"ignoredDevices"`
+ ExitCode int `json:"exitCode"`
+}
+
+// writeToGCS writes the meta data as JSON to the given bucket and path in
+// GCS. It assumes that the provided client has permissions to write to the
+// specified location in GCS.
+func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error {
+ jsonBytes, err := json.Marshal(t)
+ if err != nil {
+ return err
+ }
+ return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true)
+}
+
+// TODO(stephana): Merge copyReaderToGCS into the go/gcs in
+// the infra repository.
+
+// copyReaderToGCS reads all available content from the given reader and writes
+// it to the given path in GCS.
+func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error {
+ storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client))
+ if err != nil {
+ return err
+ }
+ bucket, path := gcs.SplitGSPath(gcsPath)
+ w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background())
+
+ // Set the content if requested.
+ if contentType != "" {
+ w.ObjectAttrs.ContentType = contentType
+ }
+
+ // Set the meta data if requested
+ if metaData != nil {
+ w.Metadata = metaData
+ }
+
+ // Make the object public if requested.
+ if public {
+ w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
+ }
+
+ // Write the everything the reader can provide to the GCS object. Either
+ // gzip'ed or plain.
+ if gzip {
+ w.ObjectAttrs.ContentEncoding = "gzip"
+ err = util.WithGzipWriter(w, func(w io.Writer) error {
+ _, err := io.Copy(w, reader)
+ return err
+ })
+ } else {
+ _, err = io.Copy(w, reader)
+ }
+
+ // Make sure we return an error when we close the remote object.
+ if err != nil {
+ _ = w.CloseWithError(err)
+ return err
+ }
+ return w.Close()
+}
diff --git a/infra/cts/whitelist_devices.json b/infra/cts/whitelist_devices.json
new file mode 100644
index 0000000000..86838c6f86
--- /dev/null
+++ b/infra/cts/whitelist_devices.json
@@ -0,0 +1,9 @@
+[
+ {
+ "id": "sailfish",
+ "name": "Pixel",
+ "runVersions": [
+ "26"
+ ]
+ }
+] \ No newline at end of file