From c35959f3cb855c6500f279b893eea07ce5a2573b Mon Sep 17 00:00:00 2001 From: Stephan Altmueller Date: Mon, 8 Jan 2018 15:53:37 -0500 Subject: First version of SkQP app to run on Firebase Testlab Adds activities to the skqp app so it can run as an Android app (as opposed to just instrumentation tests). A user can trigger the tests via a button. Adds the an intent receiver so the tests can be triggered on Firebase Testlab via the gameloop option. It adds the run_testlab.go script to run an apk across devices on Firebase Testlab. Bug: skia: Change-Id: I3ff5c37d743fa47913a916a0fa1e7db3c2cc79c7 Reviewed-on: https://skia-review.googlesource.com/89163 Commit-Queue: Stephan Altmueller Reviewed-by: Kevin Lubick Reviewed-by: Derek Sollenberger --- platform_tools/android/apps/skqp/build.gradle | 6 +- .../android/apps/skqp/src/main/AndroidManifest.xml | 25 +++++- .../src/main/java/org/skia/skqp/MainActivity.java | 21 ++++++ .../skqp/src/main/java/org/skia/skqp/SkQP.java | 88 +++++++++++++++++++++- .../src/main/java/org/skia/skqp/SkQPActivity.java | 35 +++++++++ .../src/main/java/org/skia/skqp/SkQPRunner.java | 73 ++++++------------ .../skqp/src/main/res/layout/activity_main.xml | 37 +++++++++ .../skqp/src/main/res/layout/activity_skqp.xml | 25 ++++++ .../apps/skqp/src/main/res/layout/content_skqp.xml | 19 +++++ .../apps/skqp/src/main/res/values/colors.xml | 6 ++ .../apps/skqp/src/main/res/values/strings.xml | 4 + .../apps/skqp/src/main/res/values/styles.xml | 20 +++++ 12 files changed, 301 insertions(+), 58 deletions(-) create mode 100644 platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/MainActivity.java create mode 100644 platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPActivity.java create mode 100644 platform_tools/android/apps/skqp/src/main/res/layout/activity_main.xml create mode 100644 platform_tools/android/apps/skqp/src/main/res/layout/activity_skqp.xml create mode 100644 platform_tools/android/apps/skqp/src/main/res/layout/content_skqp.xml create mode 100644 platform_tools/android/apps/skqp/src/main/res/values/colors.xml create mode 100644 platform_tools/android/apps/skqp/src/main/res/values/strings.xml create mode 100644 platform_tools/android/apps/skqp/src/main/res/values/styles.xml (limited to 'platform_tools') diff --git a/platform_tools/android/apps/skqp/build.gradle b/platform_tools/android/apps/skqp/build.gradle index 0a883ca869..67a89e52af 100644 --- a/platform_tools/android/apps/skqp/build.gradle +++ b/platform_tools/android/apps/skqp/build.gradle @@ -7,13 +7,13 @@ apply plugin: 'com.android.application' dependencies { - compile 'com.android.support:support-annotations:24.0.0' + compile 'com.android.support.constraint:constraint-layout:1.0.2' + compile 'com.android.support:design:26.+' compile 'com.android.support.test:runner:0.5' - compile group: 'junit', name: 'junit', version: '4.+' } android { - compileSdkVersion 23 + compileSdkVersion 26 buildToolsVersion "22.0.1" defaultConfig { applicationId "org.skia.skqp" diff --git a/platform_tools/android/apps/skqp/src/main/AndroidManifest.xml b/platform_tools/android/apps/skqp/src/main/AndroidManifest.xml index 30028fb436..b080286f2f 100644 --- a/platform_tools/android/apps/skqp/src/main/AndroidManifest.xml +++ b/platform_tools/android/apps/skqp/src/main/AndroidManifest.xml @@ -3,7 +3,30 @@ package="org.skia.skqp" android:versionCode="1" android:versionName="1.0"> - + + + + + + + + + + + + + + + + + diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/MainActivity.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/MainActivity.java new file mode 100644 index 0000000000..43320077ce --- /dev/null +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/MainActivity.java @@ -0,0 +1,21 @@ +package org.skia.skqp; + +import android.content.Intent; +import android.support.v7.app.AppCompatActivity; +import android.os.Bundle; +import android.view.View; +import android.widget.EditText; + +public class MainActivity extends AppCompatActivity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + } + + public void startTests(View view) { + Intent intent = new Intent(this, SkQPActivity.class); + startActivity(intent); + } +} diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java index c5843f013e..8ced43dbe2 100644 --- a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java @@ -7,8 +7,90 @@ package org.skia.skqp; -import org.junit.runner.RunWith; +import android.content.Context; +import android.content.res.AssetManager; +import android.util.Log; +import java.io.File; +import java.io.IOException; -@RunWith(SkQPRunner.class) -public class SkQP {} +public class SkQP { + protected native void nInit(AssetManager assetManager, String dataDir); + protected native float nExecuteGM(int gm, int backend) throws SkQPException; + protected native String[] nExecuteUnitTest(int test); + protected native void nMakeReport(); + + protected String[] mGMs; + protected String[] mBackends; + protected String[] mUnitTests; + + protected static final String kSkiaGM = "SkiaGM_"; + protected static final String kSkiaUnitTests = "Skia_UnitTests"; + protected static final String LOG_PREFIX = "org.skis.skqp"; + + static { + System.loadLibrary("skqp_app"); + } + + protected void runTests(Context context, String outputDirPath) { + Log.w(LOG_PREFIX, "Output Dir: " + outputDirPath); + File outputDir = new File(outputDirPath); + if (outputDir.exists()) { + try { + deleteDirectoryContents(outputDir); + } catch (IOException e) { + Log.w(LOG_PREFIX, "DeleteDirectoryContents: " + e.getMessage()); + } + } + + // Note: nInit will initialize the mGMs, mBackends and mUnitTests fields. + AssetManager assetManager = context.getResources().getAssets(); + this.nInit(assetManager, outputDirPath); + + for (int backend = 0; backend < mBackends.length; backend++) { + String classname = kSkiaGM + mBackends[backend]; + for (int gm = 0; gm < mGMs.length; gm++) { + String testName = kSkiaGM + mBackends[backend] + "_" +mGMs[gm]; + float value = java.lang.Float.MAX_VALUE; + String error = null; + Log.w(LOG_PREFIX, "Running: " + testName); + try { + value = this.nExecuteGM(gm, backend); + } catch (SkQPException exept) { + error = exept.getMessage(); + } + if (error != null) { + // Record error message and carry on. + } else if (value != 0) { + // Record failure and carry on. + // SkQPRunner.Fail(desc, notifier, String.format( + // "Image mismatch: max channel diff = %f", value)); + } else { + // Record success for this test. + } + } + } + for (int unitTest = 0; unitTest < mUnitTests.length; unitTest++) { + String testName = kSkiaUnitTests + "_" + mUnitTests[unitTest]; + Log.w(LOG_PREFIX, "Running: " + testName); + String[] errors = this.nExecuteUnitTest(unitTest); + if (errors != null && errors.length > 0) { + for (String error : errors) { + // Record unit test failures. + } + } else { + // Record success. + } + } + nMakeReport(); + } + + protected static void deleteDirectoryContents(File f) throws IOException { + for (File s : f.listFiles()) { + if (s.isDirectory()) { + deleteDirectoryContents(s); + } + s.delete(); + } + } +} diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPActivity.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPActivity.java new file mode 100644 index 0000000000..ab7ce168b9 --- /dev/null +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPActivity.java @@ -0,0 +1,35 @@ +package org.skia.skqp; + +import android.content.Context; +import android.os.Bundle; +import android.support.v7.app.AppCompatActivity; +import android.support.v7.widget.Toolbar; +import android.util.Log; + +public class SkQPActivity extends AppCompatActivity implements Runnable { + private SkQP testRunner = new SkQP(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_skqp); + Toolbar toolbar = (Toolbar) findViewById(R.id.toolbar); + setSupportActionBar(toolbar); + + // Start the tests. + run(); + } + + // run implements the Runnable interface. + public void run() { + // Note: /sdcard/Android/data/ is a location an app is allowed to write to. + // When running tests on Firebase it expects any result files to have a '/sdcard + // prefix or it won't trigger tests from the CLI. + + Context context = getApplicationContext(); + String outputDirPath = "/sdcard/Android/data/" + context.getPackageName(); + testRunner.runTests(context, outputDirPath); + finish(); + } +} + diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java index 464c9e2e71..dc9aea3f19 100644 --- a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java @@ -21,30 +21,8 @@ import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunNotifier; public class SkQPRunner extends Runner { - private native void nInit(AssetManager assetManager, String dataDir); - private native float nExecuteGM(int gm, int backend) throws SkQPException; - private native String[] nExecuteUnitTest(int test); - private native void nMakeReport(); - - private AssetManager mAssetManager; - private String[] mGMs; - private String[] mBackends; - private String[] mUnitTests; - - private static boolean sOnceFlag = false; - private static final String kSkiaGM = "SkiaGM_"; - private static final String kSkiaUnitTests = "Skia_UnitTests"; - private Description mDescription; - - private static void DeleteDirectoryContents(File f) throws IOException { - for (File s : f.listFiles()) { - if (s.isDirectory()) { - SkQPRunner.DeleteDirectoryContents(s); - } - s.delete(); - } - } + private SkQP impl; private static void Fail(Description desc, RunNotifier notifier, String failure) { notifier.fireTestFailure(new Failure(desc, new Throwable(failure))); @@ -53,37 +31,30 @@ public class SkQPRunner extends Runner { //////////////////////////////////////////////////////////////////////////// public SkQPRunner(Class testClass) { - synchronized (SkQPRunner.class) { - if (sOnceFlag) { - throw new IllegalStateException("Error multiple SkQPs defined"); - } - sOnceFlag = true; - } - System.loadLibrary("skqp_app"); - + impl = new SkQP(); Context context = InstrumentationRegistry.getTargetContext(); File filesDir = context.getFilesDir(); try { - SkQPRunner.DeleteDirectoryContents(filesDir); + SkQP.deleteDirectoryContents(filesDir); } catch (IOException e) { Log.w("org.skis.skqp", "DeleteDirectoryContents: " + e.getMessage()); } Resources resources = context.getResources(); - mAssetManager = resources.getAssets(); - this.nInit(mAssetManager, filesDir.getAbsolutePath()); + AssetManager mAssetManager = resources.getAssets(); + impl.nInit(mAssetManager, filesDir.getAbsolutePath()); mDescription = Description.createSuiteDescription(testClass); Annotation annots[] = new Annotation[0]; - for (int backend = 0; backend < mBackends.length; backend++) { - String classname = kSkiaGM + mBackends[backend]; - for (int gm = 0; gm < mGMs.length; gm++) { - mDescription.addChild(Description.createTestDescription(classname, mGMs[gm], annots)); + for (int backend = 0; backend < impl.mBackends.length; backend++) { + String classname = SkQP.kSkiaGM + impl.mBackends[backend]; + for (int gm = 0; gm < impl.mGMs.length; gm++) { + mDescription.addChild(Description.createTestDescription(classname, impl.mGMs[gm], annots)); } } - for (int unitTest = 0; unitTest < mUnitTests.length; unitTest++) { - mDescription.addChild(Description.createTestDescription(kSkiaUnitTests, - mUnitTests[unitTest], annots)); + for (int unitTest = 0; unitTest < impl.mUnitTests.length; unitTest++) { + mDescription.addChild(Description.createTestDescription(SkQP.kSkiaUnitTests, + impl.mUnitTests[unitTest], annots)); } } @@ -91,20 +62,20 @@ public class SkQPRunner extends Runner { public Description getDescription() { return mDescription; } @Override - public int testCount() { return mUnitTests.length + mGMs.length * mBackends.length; } + public int testCount() { return impl.mUnitTests.length + impl.mGMs.length * impl.mBackends.length; } @Override public void run(RunNotifier notifier) { Annotation annots[] = new Annotation[0]; - for (int backend = 0; backend < mBackends.length; backend++) { - String classname = kSkiaGM + mBackends[backend]; - for (int gm = 0; gm < mGMs.length; gm++) { - Description desc = Description.createTestDescription(classname, mGMs[gm], annots); + for (int backend = 0; backend < impl.mBackends.length; backend++) { + String classname = SkQP.kSkiaGM + impl.mBackends[backend]; + for (int gm = 0; gm < impl.mGMs.length; gm++) { + Description desc = Description.createTestDescription(classname, impl.mGMs[gm], annots); notifier.fireTestStarted(desc); float value = java.lang.Float.MAX_VALUE; String error = null; try { - value = this.nExecuteGM(gm, backend); + value = impl.nExecuteGM(gm, backend); } catch (SkQPException exept) { error = exept.getMessage(); } @@ -117,11 +88,11 @@ public class SkQPRunner extends Runner { notifier.fireTestFinished(desc); } } - for (int unitTest = 0; unitTest < mUnitTests.length; unitTest++) { + for (int unitTest = 0; unitTest < impl.mUnitTests.length; unitTest++) { Description desc = Description.createTestDescription( - kSkiaUnitTests, mUnitTests[unitTest], annots); + SkQP.kSkiaUnitTests, impl.mUnitTests[unitTest], annots); notifier.fireTestStarted(desc); - String[] errors = this.nExecuteUnitTest(unitTest); + String[] errors = impl.nExecuteUnitTest(unitTest); if (errors != null && errors.length > 0) { for (String error : errors) { SkQPRunner.Fail(desc, notifier, error); @@ -129,7 +100,7 @@ public class SkQPRunner extends Runner { } notifier.fireTestFinished(desc); } - this.nMakeReport(); + impl.nMakeReport(); } } diff --git a/platform_tools/android/apps/skqp/src/main/res/layout/activity_main.xml b/platform_tools/android/apps/skqp/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000000..45b9c77f05 --- /dev/null +++ b/platform_tools/android/apps/skqp/src/main/res/layout/activity_main.xml @@ -0,0 +1,37 @@ + + + + + + + +