diff options
Diffstat (limited to 'infra/cts')
| -rw-r--r-- | infra/cts/run_testlab.go | 399 | ||||
| -rw-r--r-- | infra/cts/whitelist_devices.json | 9 |
2 files changed, 312 insertions, 96 deletions
diff --git a/infra/cts/run_testlab.go b/infra/cts/run_testlab.go index 6a7f76591c..b764bb825a 100644 --- a/infra/cts/run_testlab.go +++ b/infra/cts/run_testlab.go @@ -9,9 +9,12 @@ package main import ( "bytes" + "context" "encoding/json" "flag" "fmt" + "io" + "io/ioutil" "net/http" "os" "os/exec" @@ -21,51 +24,16 @@ import ( "syscall" "time" + "go.skia.org/infra/go/gcs" + + "cloud.google.com/go/storage" + "google.golang.org/api/option" gstorage "google.golang.org/api/storage/v1" "go.skia.org/infra/go/auth" "go.skia.org/infra/go/common" "go.skia.org/infra/go/sklog" "go.skia.org/infra/go/util" - "go.skia.org/infra/golden/go/tsuite" -) - -// TODO(stephana): Convert the hard coded whitelist to a command line flag that -// loads a file with the whitelisted devices and versions. Make sure to include -// human readable names for the devices. - -var ( - // WHITELIST_DEV_IDS contains a mapping from the device id to the list of - // Android API versions that we should run agains. Usually this will be the - // latest version. To see available devices and version run with - // --dryrun flag or run '$ gcloud firebase test android models list' - - WHITELIST_DEV_IDS = map[string][]string{ - "A0001": {"22"}, - // "E5803": {"22"}, deprecated - // "F5121": {"23"}, deprecated - "G8142": {"25"}, - "HWMHA": {"24"}, - "SH-04H": {"23"}, - "athene": {"23"}, - "athene_f": {"23"}, - "hammerhead": {"23"}, - "harpia": {"23"}, - "hero2lte": {"23"}, - "herolte": {"24"}, - "j1acevelte": {"22"}, - "j5lte": {"23"}, - "j7xelte": {"23"}, - "lucye": {"24"}, - // "mako": {"22"}, deprecated - "osprey_umts": {"22"}, - // "p1": {"22"}, deprecated - "sailfish": {"26"}, - "shamu": {"23"}, - "trelte": {"22"}, - "zeroflte": {"22"}, - "zerolte": {"22"}, - } ) const ( @@ -74,10 +42,14 @@ const ( // Command line flags. var ( - serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") + devicesFile = flag.String("devices", "", "JSON file that maps device ids to versions to run on. Same format as produced by the dump_devices flag.") dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.") - minAPIVersion = flag.Int("min_api", 22, "Minimum API version required by device.") - maxAPIVersion = flag.Int("max_api", 23, "Maximum API version required by device.") + dumpDevFile = flag.String("dump_devices", "", "Creates a JSON file with all physical devices that are not deprecated.") + minAPIVersion = flag.Int("min_api", 0, "Minimum API version required by device.") + maxAPIVersion = flag.Int("max_api", 99, "Maximum API version required by device.") + properties = flag.String("properties", "", "Custom meta data to be added to the uploaded APK. Comma separated list of key=value pairs, i.e. 'k1=v1,k2=v2,k3=v3.") + serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") + uploadGCSPath = flag.String("upload_path", "", "GCS path (bucket/path) to where the APK should be uploaded to. It's assume to a full path (not a directory).") ) const ( @@ -90,88 +62,145 @@ const ( --timeout 30m %s ` - MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" - RESULT_BUCKET = "skia-firebase-test-lab" - RESULT_DIR_TMPL = "testruns/%s/%s" - RUN_ID_TMPL = "testrun-%d" - CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json" + MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" + RESULT_BUCKET = "skia-firebase-test-lab" + RESULT_DIR_TMPL = "testruns/%s/%s" + RUN_ID_TMPL = "testrun-%d" + CMD_AVAILABLE_DEVICES = "gcloud firebase test android models list --format json" ) func main() { common.Init() - // Get the apk. - args := flag.Args() - apk_path := args[0] + // Get the path to the APK. It can be empty if we are dumping the device list. + apkPath := flag.Arg(0) + if *dumpDevFile == "" && apkPath == "" { + sklog.Errorf("Missing APK. The APK file needs to be passed as the positional argument.") + os.Exit(1) + } - // Make sure we can get the service account client. - client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") + // Get the available devices. + fbDevices, deviceList, err := getAvailableDevices() if err != nil { - sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) + sklog.Fatalf("Error retrieving available devices: %s", err) } - // Get list of all available devices. - devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS, *minAPIVersion, *maxAPIVersion) + // Dump the device list and exit. + if *dumpDevFile != "" { + if err := writeDeviceList(*dumpDevFile, deviceList); err != nil { + sklog.Fatalf("Unable to write devices: %s", err) + } + return + } + + // If no devices are explicitly listed. Use all of them. + whiteList := deviceList + if *devicesFile != "" { + whiteList, err = readDeviceList(*devicesFile) + if err != nil { + sklog.Fatalf("Error reading device file: %s", err) + } + } + + // Make sure we can authenticate locally and in the cloud. + client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") if err != nil { - sklog.Fatalf("Unable to retrieve available devices: %s", err) + sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) } - sklog.Infof("---") - sklog.Infof("Selected devices:") + + // Filter the devices according the white list and other parameters. + devices, ignoredDevices := filterDevices(fbDevices, whiteList, *minAPIVersion, *maxAPIVersion) + sklog.Infof("---\nSelected devices:") logDevices(devices) - if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil { - sklog.Fatalf("Error triggering tests on Firebase: %s", err) + if len(devices) == 0 { + sklog.Errorf("No devices selected. Not running tests.") + os.Exit(1) + } + + if err := runTests(apkPath, devices, ignoredDevices, client, *dryRun); err != nil { + sklog.Fatalf("Error running tests on Firebase: %s", err) + } + + if !*dryRun && (*uploadGCSPath != "") && (*properties != "") { + if err := uploadAPK(apkPath, *uploadGCSPath, *properties, client); err != nil { + sklog.Fatalf("Error uploading APK to '%s': %s", *uploadGCSPath, err) + } } } -// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all -// available devices and then returns a list of devices to be tested and the list -// of ignored devices. -func getAvailableDevices(whiteList map[string][]string, minAPIVersion, maxAPIVersion int) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) { +// getAvailableDevices queries Firebase Testlab for all physical devices that +// are not deprecated. It returns two lists with the same information. +// The first contains all device information as returned by Firebase while +// the second contains the information necessary to use in a whitelist. +func getAvailableDevices() ([]*DeviceVersions, DeviceList, error) { // Get the list of all devices in JSON format from Firebase testlab. var buf bytes.Buffer - cmd := parseCommand(CMD_AVAILABE_DEVICES) + var errBuf bytes.Buffer + cmd := parseCommand(CMD_AVAILABLE_DEVICES) cmd.Stdout = &buf - cmd.Stderr = os.Stdout + cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) if err := cmd.Run(); err != nil { - return nil, nil, err + return nil, nil, sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", CMD_AVAILABLE_DEVICES, err, errBuf) } // Unmarshal the result. - foundDevices := []*tsuite.FirebaseDevice{} + foundDevices := []*DeviceVersions{} bufBytes := buf.Bytes() if err := json.Unmarshal(bufBytes, &foundDevices); err != nil { return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes)) } + // Filter the devices and copy them to device list. + devList := DeviceList{} + ret := make([]*DeviceVersions, 0, len(foundDevices)) + for _, foundDev := range foundDevices { + // Only consider physical devices and devices that are not deprecated. + if (foundDev.Form == "PHYSICAL") && !util.In("deprecated", foundDev.Tags) { + ret = append(ret, foundDev) + devList = append(devList, &DevInfo{ + ID: foundDev.ID, + Name: foundDev.Name, + RunVersions: foundDev.VersionIDs, + }) + } + } + return foundDevices, devList, nil +} + +// filterDevices filters the given devices by ensuring that they are in the white list +// and within the given api version range. +// It returns two lists: (accepted_devices, ignored_devices) +func filterDevices(foundDevices []*DeviceVersions, whiteList DeviceList, minAPIVersion, maxAPIVersion int) ([]*DeviceVersions, []*DeviceVersions) { // iterate over the available devices and partition them. - allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) - ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) - ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) + allDevices := make([]*DeviceVersions, 0, len(foundDevices)) + ret := make([]*DeviceVersions, 0, len(foundDevices)) + ignored := make([]*DeviceVersions, 0, len(foundDevices)) for _, dev := range foundDevices { - // Filter out all the virtual devices. - if dev.Form == "PHYSICAL" { - // Only include devices that are on the whitelist and have versions defined. - if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) { - versionSet := util.NewStringSet(dev.VersionIDs) - reqVersions := util.NewStringSet(filterVersions(foundVersions, minAPIVersion, maxAPIVersion)) - whiteListVersions := versionSet.Intersect(reqVersions).Keys() - ignoredVersions := versionSet.Complement(reqVersions).Keys() - sort.Strings(whiteListVersions) - sort.Strings(ignoredVersions) - ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions}) - ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions}) - } else { - ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) + // Only include devices that are on the whitelist and have versions defined. + if targetDev := whiteList.find(dev.ID); targetDev != nil && (len(targetDev.RunVersions) > 0) { + versionSet := util.NewStringSet(dev.VersionIDs) + reqVersions := util.NewStringSet(filterVersions(targetDev.RunVersions, minAPIVersion, maxAPIVersion)) + whiteListVersions := versionSet.Intersect(reqVersions).Keys() + ignoredVersions := versionSet.Complement(reqVersions).Keys() + sort.Strings(whiteListVersions) + sort.Strings(ignoredVersions) + if len(whiteListVersions) > 0 { + ret = append(ret, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: whiteListVersions}) + } + if len(ignoredVersions) > 0 { + ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: ignoredVersions}) } - allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) + } else { + ignored = append(ignored, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) } + allDevices = append(allDevices, &DeviceVersions{FirebaseDevice: dev.FirebaseDevice, RunVersions: dev.VersionIDs}) } sklog.Infof("All devices:") logDevices(allDevices) - return ret, ignored, nil + return ret, ignored } // filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion. @@ -190,12 +219,12 @@ func filterVersions(versionIDs []string, minVersion, maxVersion int) []string { } // runTests runs the given apk on the given list of devices. -func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error { +func runTests(apk_path string, devices, ignoredDevices []*DeviceVersions, client *http.Client, dryRun bool) error { // Get the model-version we want to test. Assume on average each model has 5 supported versions. modelSelectors := make([]string, 0, len(devices)*5) for _, devRec := range devices { - for _, version := range devRec.Versions { - modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version)) + for _, version := range devRec.RunVersions { + modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.FirebaseDevice.ID, version)) } } @@ -207,9 +236,10 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1)) // Run the command. + var errBuf bytes.Buffer cmd := parseCommand(cmdStr) cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout + cmd.Stderr = io.MultiWriter(os.Stdout, &errBuf) exitCode := 0 if dryRun { @@ -223,18 +253,19 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, ws := exitError.Sys().(syscall.WaitStatus) exitCode = ws.ExitStatus() } + sklog.Errorf("Error running tests: %s", err) sklog.Errorf("Exit code: %d", exitCode) // Exit code 10 means triggering on Testlab succeeded, but but some of the // runs on devices failed. We consider it a success for this script. if exitCode != 10 { - return err + return sklog.FmtErrorf("Error running: %s\nError:%s\nStdErr:%s", cmdStr, err, errBuf) } } // Store the result in a meta json file. - meta := &tsuite.TestRunMeta{ + meta := &TestRunMeta{ ID: runID, TS: nowMs, Devices: devices, @@ -242,15 +273,56 @@ func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, ExitCode: exitCode, } - meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client) + targetPath := fmt.Sprintf("%s/%s/%s", RESULT_BUCKET, resultsDir, META_DATA_FILENAME) + if err := meta.writeToGCS(targetPath, client); err != nil { + return err + } + sklog.Infof("Meta data written to gs://%s", targetPath) + return nil +} + +// uploadAPK uploads the APK at the given path to the bucket/path in gcsPath. +// The key-value pairs in propStr are set as custom meta data of the APK. +func uploadAPK(apkPath, gcsPath, propStr string, client *http.Client) error { + properties, err := splitProperties(propStr) + if err != nil { + return err + } + apkFile, err := os.Open(apkPath) + if err != nil { + return err + } + defer util.Close(apkFile) + + if err := copyReaderToGCS(gcsPath, apkFile, client, "application/vnd.android.package-archive", properties, true, false); err != nil { + return err + } + + sklog.Infof("APK uploaded to gs://%s", gcsPath) return nil } +// splitProperties receives a comma separated list of 'key=value' pairs and +// returnes them as a map. +func splitProperties(propStr string) (map[string]string, error) { + splitProps := strings.Split(propStr, ",") + properties := make(map[string]string, len(splitProps)) + for _, oneProp := range splitProps { + kv := strings.Split(oneProp, "=") + if len(kv) != 2 { + return nil, sklog.FmtErrorf("Inavlid porperties format. Unable to parse '%s'", propStr) + } + properties[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + return properties, nil +} + // logDevices logs the given list of devices. -func logDevices(devices []*tsuite.DeviceVersions) { +func logDevices(devices []*DeviceVersions) { sklog.Infof("Found %d devices.", len(devices)) for _, dev := range devices { - sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions) + fbDev := dev.FirebaseDevice + sklog.Infof("%-15s %-30s %v / %v", fbDev.ID, fbDev.Name, fbDev.VersionIDs, dev.RunVersions) } } @@ -262,3 +334,138 @@ func parseCommand(cmdStr string) *exec.Cmd { } return exec.Command(cmdArgs[0], cmdArgs[1:]...) } + +// DeviceList is a simple list of devices, primarily used to define the +// whitelist of devices we want to run on. +type DeviceList []*DevInfo + +type DevInfo struct { + ID string `json:"id"` + Name string `json:"name"` + RunVersions []string `json:"runVersions"` +} + +func (d DeviceList) find(id string) *DevInfo { + for _, devInfo := range d { + if devInfo.ID == id { + return devInfo + } + } + return nil +} + +func writeDeviceList(fileName string, devList DeviceList) error { + jsonBytes, err := json.MarshalIndent(devList, "", " ") + if err != nil { + return sklog.FmtErrorf("Unable to encode JSON: %s", err) + } + + if err := ioutil.WriteFile(fileName, jsonBytes, 0644); err != nil { + sklog.FmtErrorf("Unable to write file '%s': %s", fileName, err) + } + return nil +} + +func readDeviceList(fileName string) (DeviceList, error) { + inFile, err := os.Open(fileName) + if err != nil { + return nil, sklog.FmtErrorf("Unable to open file '%s': %s", fileName, err) + } + defer util.Close(inFile) + + var devList DeviceList + if err := json.NewDecoder(inFile).Decode(&devList); err != nil { + return nil, sklog.FmtErrorf("Unable to decode JSON from '%s': %s", fileName, err) + } + return devList, nil +} + +// FirebaseDevice contains the information and JSON tags for device information +// returned by firebase. +type FirebaseDevice struct { + Brand string `json:"brand"` + Form string `json:"form"` + ID string `json:"id"` + Manufacturer string `json:"manufacturer"` + Name string `json:"name"` + VersionIDs []string `json:"supportedVersionIds"` + Tags []string `json:"tags"` +} + +// DeviceVersions combines device information from Firebase Testlab with +// a selected list of versions. This is used to define a subset of versions +// used by a devices. +type DeviceVersions struct { + *FirebaseDevice + + // RunVersions contains the version ids of interest contained in Device. + RunVersions []string +} + +// TestRunMeta contains the meta data of a complete testrun on firebase. +type TestRunMeta struct { + ID string `json:"id"` + TS int64 `json:"timeStamp"` + Devices []*DeviceVersions `json:"devices"` + IgnoredDevices []*DeviceVersions `json:"ignoredDevices"` + ExitCode int `json:"exitCode"` +} + +// writeToGCS writes the meta data as JSON to the given bucket and path in +// GCS. It assumes that the provided client has permissions to write to the +// specified location in GCS. +func (t *TestRunMeta) writeToGCS(gcsPath string, client *http.Client) error { + jsonBytes, err := json.Marshal(t) + if err != nil { + return err + } + return copyReaderToGCS(gcsPath, bytes.NewReader(jsonBytes), client, "", nil, false, true) +} + +// TODO(stephana): Merge copyReaderToGCS into the go/gcs in +// the infra repository. + +// copyReaderToGCS reads all available content from the given reader and writes +// it to the given path in GCS. +func copyReaderToGCS(gcsPath string, reader io.Reader, client *http.Client, contentType string, metaData map[string]string, public bool, gzip bool) error { + storageClient, err := storage.NewClient(context.Background(), option.WithHTTPClient(client)) + if err != nil { + return err + } + bucket, path := gcs.SplitGSPath(gcsPath) + w := storageClient.Bucket(bucket).Object(path).NewWriter(context.Background()) + + // Set the content if requested. + if contentType != "" { + w.ObjectAttrs.ContentType = contentType + } + + // Set the meta data if requested + if metaData != nil { + w.Metadata = metaData + } + + // Make the object public if requested. + if public { + w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} + } + + // Write the everything the reader can provide to the GCS object. Either + // gzip'ed or plain. + if gzip { + w.ObjectAttrs.ContentEncoding = "gzip" + err = util.WithGzipWriter(w, func(w io.Writer) error { + _, err := io.Copy(w, reader) + return err + }) + } else { + _, err = io.Copy(w, reader) + } + + // Make sure we return an error when we close the remote object. + if err != nil { + _ = w.CloseWithError(err) + return err + } + return w.Close() +} diff --git a/infra/cts/whitelist_devices.json b/infra/cts/whitelist_devices.json new file mode 100644 index 0000000000..86838c6f86 --- /dev/null +++ b/infra/cts/whitelist_devices.json @@ -0,0 +1,9 @@ +[ + { + "id": "sailfish", + "name": "Pixel", + "runVersions": [ + "26" + ] + } +]
\ No newline at end of file |