From c35959f3cb855c6500f279b893eea07ce5a2573b Mon Sep 17 00:00:00 2001 From: Stephan Altmueller Date: Mon, 8 Jan 2018 15:53:37 -0500 Subject: First version of SkQP app to run on Firebase Testlab Adds activities to the skqp app so it can run as an Android app (as opposed to just instrumentation tests). A user can trigger the tests via a button. Adds the an intent receiver so the tests can be triggered on Firebase Testlab via the gameloop option. It adds the run_testlab.go script to run an apk across devices on Firebase Testlab. Bug: skia: Change-Id: I3ff5c37d743fa47913a916a0fa1e7db3c2cc79c7 Reviewed-on: https://skia-review.googlesource.com/89163 Commit-Queue: Stephan Altmueller Reviewed-by: Kevin Lubick Reviewed-by: Derek Sollenberger --- infra/cts/README.md | 12 +++ infra/cts/run_testlab.go | 245 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 infra/cts/README.md create mode 100644 infra/cts/run_testlab.go (limited to 'infra/cts') diff --git a/infra/cts/README.md b/infra/cts/README.md new file mode 100644 index 0000000000..d4e8c2c0a2 --- /dev/null +++ b/infra/cts/README.md @@ -0,0 +1,12 @@ +Running CTS on Firebase Testlab +=============================== + +The run_testlab.go script uploads a given apk to Firebase Testlab and +runs them on the list of devices whitelisted in the script. +See the WHITELIST\_DEV\_IDS variable. + +To run 'skqpapp.apk' on Testlab run the following command: + +``` + $ go run run_testlab.go --logtostderr --service_account_file=service-account.json skqpapp.apk +``` diff --git a/infra/cts/run_testlab.go b/infra/cts/run_testlab.go new file mode 100644 index 0000000000..cf88c30cc9 --- /dev/null +++ b/infra/cts/run_testlab.go @@ -0,0 +1,245 @@ +/* + * Copyright 2018 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/http" + "os" + "os/exec" + "sort" + "strings" + "syscall" + "time" + + gstorage "google.golang.org/api/storage/v1" + + "go.skia.org/infra/go/auth" + "go.skia.org/infra/go/common" + "go.skia.org/infra/go/sklog" + "go.skia.org/infra/go/util" + "go.skia.org/infra/golden/go/tsuite" +) + +// TODO(stephana): Convert the hard coded whitelist to a command line flag that +// loads a file with the whitelisted devices and versions. Make sure to include +// human readable names for the devices. + +var ( + // WHITELIST_DEV_IDS contains a mapping from the device id to the list of + // Android API versions that we should run agains. Usually this will be the + // latest version. To see available devices and version run with + // --dryrun flag or run '$ gcloud firebase test android models list' + + WHITELIST_DEV_IDS = map[string][]string{ + "A0001": {"22"}, + // "E5803": {"22"}, deprecated + // "F5121": {"23"}, deprecated + "G8142": {"25"}, + "HWMHA": {"24"}, + "SH-04H": {"23"}, + "athene": {"23"}, + "athene_f": {"23"}, + "hammerhead": {"23"}, + "harpia": {"23"}, + "hero2lte": {"23"}, + "herolte": {"24v"}, + "j1acevelte": {"22"}, + "j5lte": {"23"}, + "j7xelte": {"23"}, + "lucye": {"24"}, + "mako": {"22"}, + "osprey_umts": {"22"}, + "p1": {"22"}, + "sailfish": {"26"}, + "shamu": {"23"}, + "trelte": {"22"}, + "zeroflte": {"22"}, + "zerolte": {"22"}, + } +) + +const ( + META_DATA_FILENAME = "meta.json" +) + +// Command line flags. +var ( + serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.") + dryRun = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.") +) + +const ( + RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run + --type=game-loop + --app=%s + --results-bucket=%s + --results-dir=%s + --directories-to-pull=/sdcard/Android/data/org.skia.skqpapp + %s +` + MODEL_VERSION_TMPL = "--device model=%s,version=%s,orientation=portrait" + RESULT_BUCKET = "skia-firebase-test-lab" + RESULT_DIR_TMPL = "testruns/%s/%s" + RUN_ID_TMPL = "testrun-%d" + CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json" +) + +func main() { + common.Init() + + // Get the apk. + args := flag.Args() + apk_path := args[0] + + // Make sure we can get the service account client. + client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email") + if err != nil { + sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err) + } + + // Get list of all available devices. + devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS) + if err != nil { + sklog.Fatalf("Unable to retrieve available devices: %s", err) + } + sklog.Infof("---") + sklog.Infof("Selected devices:") + logDevices(devices) + + if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil { + sklog.Fatalf("Error triggering tests on Firebase: %s", err) + } +} + +// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all +// available devices and then returns a list of devices to be tested and the list +// of ignored devices. +func getAvailableDevices(whiteList map[string][]string) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) { + // Get the list of all devices in JSON format from Firebase testlab. + var buf bytes.Buffer + cmd := parseCommand(CMD_AVAILABE_DEVICES) + cmd.Stdout = &buf + cmd.Stderr = os.Stdout + if err := cmd.Run(); err != nil { + return nil, nil, err + } + + // Unmarshal the result. + foundDevices := []*tsuite.FirebaseDevice{} + bufBytes := buf.Bytes() + if err := json.Unmarshal(bufBytes, &foundDevices); err != nil { + return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes)) + } + + // iterate over the available devices and partition them. + allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) + ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) + ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices)) + for _, dev := range foundDevices { + // Filter out all the virtual devices. + if dev.Form == "PHYSICAL" { + // Only include devices that are on the whitelist and have versions defined. + if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) { + versionSet := util.NewStringSet(dev.VersionIDs) + reqVersions := util.NewStringSet(foundVersions) + whiteListVersions := versionSet.Intersect(reqVersions).Keys() + ignoredVersions := versionSet.Complement(reqVersions).Keys() + sort.Strings(whiteListVersions) + sort.Strings(ignoredVersions) + ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions}) + ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions}) + } else { + ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) + } + allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs}) + } + } + + sklog.Infof("All devices:") + logDevices(allDevices) + + return ret, ignored, nil +} + +// runTests runs the given apk on the given list of devices. +func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error { + // Get the model-version we want to test. Assume on average each model has 5 supported versions. + modelSelectors := make([]string, 0, len(devices)*5) + for _, devRec := range devices { + for _, version := range devRec.Versions { + modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version)) + } + } + + now := time.Now() + nowMs := now.UnixNano() / int64(time.Millisecond) + runID := fmt.Sprintf(RUN_ID_TMPL, nowMs) + resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID) + cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n")) + cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1)) + + // Run the command. + cmd := parseCommand(cmdStr) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + exitCode := 0 + + if dryRun { + fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr) + return nil + } + + if err := cmd.Run(); err != nil { + // Get the exit code. + if exitError, ok := err.(*exec.ExitError); ok { + ws := exitError.Sys().(syscall.WaitStatus) + exitCode = ws.ExitStatus() + } + sklog.Errorf("Error running tests: %s", err) + sklog.Errorf("Exit code: %d", exitCode) + + // Exit code 10 means triggering on Testlab succeeded, but but some of the + // runs on devices failed. We consider it a success for this script. + if exitCode != 10 { + return err + } + } + + // Store the result in a meta json file. + meta := &tsuite.TestRunMeta{ + ID: runID, + TS: nowMs, + Devices: devices, + IgnoredDevices: ignoredDevices, + ExitCode: exitCode, + } + + meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client) + return nil +} + +// logDevices logs the given list of devices. +func logDevices(devices []*tsuite.DeviceVersions) { + sklog.Infof("Found %d devices.", len(devices)) + for _, dev := range devices { + sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions) + } +} + +// parseCommad parses a command line and wraps it in an exec.Command instance. +func parseCommand(cmdStr string) *exec.Cmd { + cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ") + for idx := range cmdArgs { + cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx]) + } + return exec.Command(cmdArgs[0], cmdArgs[1:]...) +} -- cgit v1.2.3