From 281bf5249c2930ff4e8a500c8c26deb8e3253b11 Mon Sep 17 00:00:00 2001 From: hcm Date: Mon, 2 Mar 2015 11:25:25 -0800 Subject: Add testing section of docs. Consolidated trybot/buildbot section BUG=skia: Preview link: http://skiadocs.com:8000/dev/testing/?cl=954523004 Review URL: https://codereview.chromium.org/954523004 --- site/dev/testing/buildbot.md | 70 +++++++++++++++++ site/dev/testing/fonts.md | 118 ++++++++++++++++++++++++++++ site/dev/testing/index.md | 16 ++++ site/dev/testing/testing.md | 181 +++++++++++++++++++++++++++++++++++++++++++ site/dev/testing/tests.md | 65 ++++++++++++++++ 5 files changed, 450 insertions(+) create mode 100644 site/dev/testing/buildbot.md create mode 100644 site/dev/testing/fonts.md create mode 100644 site/dev/testing/index.md create mode 100644 site/dev/testing/testing.md create mode 100644 site/dev/testing/tests.md (limited to 'site/dev/testing') diff --git a/site/dev/testing/buildbot.md b/site/dev/testing/buildbot.md new file mode 100644 index 0000000000..7d9035db14 --- /dev/null +++ b/site/dev/testing/buildbot.md @@ -0,0 +1,70 @@ +Skia Buildbots +============== + +Overview +-------- + +Like the Chromium team, the Skia team uses [buildbot](http://trac.buildbot.net/) +to run continuous builds and tests. + +Here is a link to our main status page: https://status.skia.org/ + +There are also Skia client, compile, Android, and FYI console pages for a detailed +view of those results: + + Externally-facing: http://build.chromium.org/p/client.skia/console + + Internally-facing: http://chromegw.corp.google.com/i/client.skia/console + http://chromegw.corp.google.com/i/client.skia.internal/console + \(only visible internally\) + +Architecture +------------ + +The buildbot system consists of these elements: \(see +http://buildbot.net/buildbot/docs/current/manual/introduction.html#system-architecture +for more detail\) + +* builder + + * one repeatable build and/or test configuration on a given platform. + * each builder maintains its own local checkout of the Skia repo + * only one builder is running at any given time on any single buildslave; otherwise, + different builders could interfere with each other's performance numbers + +* buildbot master + + * watches for new commits to land in the Skia repository + \(https://skia.googlesource.com/skia\) + * whenever a new commit lands, it tells buildbot slaves to start building and + testing the latest revision + * serves up status pages whenever anybody requests them + +* buildslave \(or "buildbot slave"\) + + * a process on a machine that builds and runs code as directed by the buildbot + master + * one or more builders run on each buildslave + +* build + + * one run of a particular builder, at a particular code revision + + +Status View +------------ + +The status view shows a table with builders, grouped by test type and platform, +on the X-axis and commits on the Y-axis. The cells are colored according to +the status of the build for each commit: green indicates success, red indicates +failure, light orange indicates an in-progress build, and white indicates that +no build has started yet for a given revision. Commits are listed by author, and +the branch on which the commit was made is shown on the very left. + +For more detail, you can click on an individual cell to get a summary of the +steps which ran for that build. You can also click one of the white bars at +the top of each column to see a summary of recent builds for a given builder. + + + + diff --git a/site/dev/testing/fonts.md b/site/dev/testing/fonts.md new file mode 100644 index 0000000000..317ee9f1fa --- /dev/null +++ b/site/dev/testing/fonts.md @@ -0,0 +1,118 @@ +Fonts and GM Tests +================== + +Overview +-------- + +Each test in the gm directory draws a reference image. Their primary purpose is +to detect when images change unexpectedly, indicating that a rendering bug has +been introduced. + +The gm tests have a secondary purpose: they detect when rendering is different +across platforms and configurations. + +The dm \(Diamond Master\) tool supports flags that minimize or eliminate the +differences introduced by the font scaler native to each platform. + + +Portable fonts +-------------- + +The most portable font format uses Skia to draw characters directly from paths, +and contains a idealized set of font metrics. This does not exercise platform +specific fonts at all, but does support specifying the font name, font size, +font style, and attributes like fakeBold. The paths are generated on a reference +platform \(currently a Mac\) and are stored as data in +'tools/test_font_data.cpp' . + +To use portable fonts, pass '\-\-portableFonts' to dm. + + +Resource fonts +-------------- + +The '\-\-resourceFonts' flag directs dm to use font files present in the resources +directory. By using the same font set on all buildbots, the generated gm images +become more uniform across platforms. + +Today, the set of fonts used by gm, and present in my resources directory, +include: + + * Courier New Bold Italic.ttf + * Courier New Bold.ttf + * Courier New Italic.ttf + * Courier New.ttf + * LiberationSans-Bold.ttf + * LiberationSans-BoldItalic.ttf + * LiberationSans-Italic.ttf + * LiberationSans-Regular.ttf + * Papyrus.ttc + * Pro W4.otf + * Times New Roman Bold Italic.ttf + * Times New Roman Bold.ttf + * Times New Roman Italic.ttf + * Times New Roman.ttf + + +System fonts +------------ + +If neither '\-\-portableFonts' nor '\-\-resourceFonts' is specified, dm uses the fonts +present on the system. Also, if '\-\-portableFonts' or '\-\-resourceFonts' is specified +and the desired font is not available, the native font lookup algorithm is +invoked. + + +GM font selection +----------------- + +Each gm specifies the typeface to use when drawing text. For now, to set the +portable typeface on the paint, call: + +~~~~ +sk_tool_utils::set_portable_typeface(SkPaint* , const char* name = NULL, +SkTypeface::Style style = SkTypeface::kNormal ); +~~~~ + +To create a portable typeface, use: + +~~~~ +SkTypeface* typeface = sk_tool_utils::create_portable_typeface(const char* name, +SkTypeface::Style style); +~~~~ + +Eventually, both 'set_portable_typeface()' and 'create_portable_typeface()' will be +removed. Instead, a test-wide 'SkFontMgr' will be selected to choose portable +fonts or resource fonts. + + +Adding new fonts and glyphs to a GM +----------------------------------- + +If a font is missing from the portable data or the resource directory, the +system font is used instead. If a glyph is missing from the portable data, the +first character, usually a space, is drawn instead. + +Running dm with '\-\-portableFonts' and '\-\-reportUsedChars' generates +'tools/test_font_data_chars.cpp', which describes the fonts and characters used by +all gm tests. Subsequently running the 'create_test_font' tool generates new paths +and writes them into 'tools/test_font_data.cpp' . + + +Future work +----------- + +The font set used by gm tests today is arbitrary and not intended to be +cross-platform. By choosing fonts without licensing issues, all bots can freely +contain the same fonts. By narrowing the font selection, the size of the test +font data will be more manageable. + +Adding support for selecting from multiple font managers at runtime permits +removing manual typeface selection in the gm tests. Today, options to dm like +'\-\-pipe' fail with '\-\-portableFonts' because we're hard-coded to using the default +font manage when pictures are serialized. + +Some gm tests explicitly always want to use system fonts and system metrics; +other gm tests use text only to label the drawing; yet other gm tests use text +to generate paths for testing. Additional discrimination is needed to +distinguish these cases. diff --git a/site/dev/testing/index.md b/site/dev/testing/index.md new file mode 100644 index 0000000000..fb90ec106f --- /dev/null +++ b/site/dev/testing/index.md @@ -0,0 +1,16 @@ +Testing +======= + +Skia relies heavily on our suite of unit and Golden Master \(GM\) tests, which +are served by our Diamond Master \(DM\) test tool, for correctness testing. +Tests are executed by our trybots, for every commit, across most of our +supported platforms and configurations. +Skia [Gold](https://gold.skia.org) is a web interface for triaging these results. + +We also have a robust set of performance tests, served by the nanobench tool and +accessible via the Skia [Perf](https://perf.skia.org) web interface. + +Cluster Telemetry is a powerful framework that helps us capture and benchmark +SKP files, a binary format for draw commands, across up to one million websites. + +See the individual subpages for more details on our various test tools. diff --git a/site/dev/testing/testing.md b/site/dev/testing/testing.md new file mode 100644 index 0000000000..1ed56235bf --- /dev/null +++ b/site/dev/testing/testing.md @@ -0,0 +1,181 @@ +Correctness Testing +=================== + +Skia correctness testing is primarily served by a tool named DM. +This is a quickstart to building and running DM. + +~~~ +$ ./gyp_skia +$ ninja -C out/Debug dm +$ out/Debug/dm -v -w dm_output +~~~ + +When you run this, you may notice your CPU peg to 100% for a while, then taper +off to 1 or 2 active cores as the run finishes. This is intentional. DM is +very multithreaded, but some of the work, particularly GPU-backed work, is +still forced to run on a single thread. You can use `--threads N` to limit DM to +N threads if you like. This can sometimes be helpful on machines that have +relatively more CPU available than RAM. + +As DM runs, you ought to see a giant spew of output that looks something like this. +~~~ +Skipping nonrendering: Don't understand 'nonrendering'. +Skipping angle: Don't understand 'angle'. +Skipping nvprmsaa4: Could not create a surface. +492 srcs * 3 sinks + 382 tests == 1858 tasks + +( 25MB 1857) 1.36ms 8888 image mandrill_132x132_12x12.astc-5-subsets +( 25MB 1856) 1.41ms 8888 image mandrill_132x132_6x6.astc-5-subsets +( 25MB 1855) 1.35ms 8888 image mandrill_132x130_6x5.astc-5-subsets +( 25MB 1854) 1.41ms 8888 image mandrill_132x130_12x10.astc-5-subsets +( 25MB 1853) 151µs 8888 image mandrill_130x132_10x6.astc-5-subsets +( 25MB 1852) 154µs 8888 image mandrill_130x130_5x5.astc-5-subsets + ... +( 748MB 5) 9.43ms unit test GLInterfaceValidation +( 748MB 4) 30.3ms unit test HalfFloatTextureTest +( 748MB 3) 31.2ms unit test FloatingPointTextureTest +( 748MB 2) 32.9ms unit test DeferredCanvas_GPU +( 748MB 1) 49.4ms unit test ClipCache +( 748MB 0) 37.2ms unit test Blur +~~~ +Do not panic. + +As you become more familiar with DM, this spew may be a bit annoying. If you +remove -v from the command line, DM will spin its progress on a single line +rather than print a new line for each status update. + +Don't worry about the "Skipping something: Here's why." lines at startup. DM +supports many test configurations, which are not all appropriate for all +machines. These lines are a sort of FYI, mostly in case DM can't run some +configuration you might be expecting it to run. + +The next line is an overview of the work DM is about to do. +~~~ +492 srcs * 3 sinks + 382 tests == 1858 tasks +~~~ + +DM has found 382 unit tests (code linked in from tests/), and 492 other drawing +sources. These drawing sources may be GM integration tests (code linked in +from gm/), image files (from `--images`, which defaults to "resources") or .skp +files (from `--skps`, which defaults to "skps"). You can control the types of +sources DM will use with `--src` (default, "tests gm image skp"). + +DM has found 3 usable ways to draw those 492 sources. This is controlled by +`--config`, which today defaults to "565 8888 gpu nonrendering angle nvprmsaa4". +DM has skipped nonrendering, angle, and nvprmssa4, leaving three usable configs: +565, 8888, and gpu. These three name different ways to draw using Skia: + + - 565: draw using the software backend into a 16-bit RGB bitmap + - 8888: draw using the software backend into a 32-bit RGBA bitmap + - gpu: draw using the GPU backend (Ganesh) into a 32-bit RGBA bitmap + +Sometimes DM calls these configs, sometimes sinks. Sorry. There are many +possible configs but generally we pay most attention to 8888 and gpu. + +DM always tries to draw all sources into all sinks, which is why we multiply +492 by 3. The unit tests don't really fit into this source-sink model, so they +stand alone. A couple thousand tasks is pretty normal. Let's look at the +status line for one of those tasks. +~~~ +( 25MB 1857) 1.36ms 8888 image mandrill_132x132_12x12.astc-5-subsets +~~~ + +This status line tells us several things. + +First, it tells us that at the time we wrote the status line, the maximum +amount of memory DM had ever used was 25MB. Note this is a high water mark, +not the current memory usage. This is mostly useful for us to track on our +buildbots, some of which run perilously close to the system memory limit. + +Next, the status line tells us that there are 1857 unfinished tasks, either +currently running or waiting to run. We generally run one task per hardware +thread available, so on a typical laptop there are probably 4 or 8 running at +once. Sometimes the counts appear to show up out of order, particularly at DM +startup; it's harmless, and doesn't affect the correctness of the run. + +Next, we see this task took 1.36 milliseconds to run. Generally, the precision +of this timer is around 1 microsecond. The time is purely there for +informational purposes, to make it easier for us to find slow tests. + +Finally we see the configuration and name of the test we ran. We drew the test +"mandrill_132x132_12x12.astc-5-subsets", which is an "image" source, into an +"8888" sink. + +When DM finishes running, you should find a directory with file named dm.json, +and some nested directories filled with lots of images. +~~~ +$ ls dm_output +565 8888 dm.json gpu + +$ find dm_output -name '*.png' +dm_output/565/gm/3x3bitmaprect.png +dm_output/565/gm/aaclip.png +dm_output/565/gm/aarectmodes.png +dm_output/565/gm/alphagradients.png +dm_output/565/gm/arcofzorro.png +dm_output/565/gm/arithmode.png +dm_output/565/gm/astcbitmap.png +dm_output/565/gm/bezier_conic_effects.png +dm_output/565/gm/bezier_cubic_effects.png +dm_output/565/gm/bezier_quad_effects.png + ... +~~~ + +The directories are nested first by sink type (`--config`), then by source type (`--src`). +The image from the task we just looked at, "8888 image mandrill_132x132_12x12.astc-5-subsets", +can be found at dm_output/8888/image/mandrill_132x132_12x12.astc-5-subsets.png. + +dm.json is used by our automated testing system, so you can ignore it if you +like. It contains a listing of each test run and a checksum of the image +generated for that run. (Boring technical detail: it is not a checksum of the +.png file, but rather a checksum of the raw pixels used to create that .png.) + +Unit tests don't generally output anything but a status update when they pass. +If a test fails, DM will print out its assertion failures, both at the time +they happen and then again all together after everything is done running. +These failures are also included in the dm.json file. + +DM has a simple facility to compare against the results of a previous run: +~~~ +$ ./gyp_skia +$ ninja -C out/Debug dm +$ out/Debug/dm -w good + + (do some work) + +$ ./gyp_skia +$ ninja -C out/Debug dm +$ out/Debug/dm -r good -w bad +~~~ +When using `-r`, DM will display a failure for any test that didn't produce the +same image as the `good` run. + +For anything fancier, I suggest using skdiff: +~~~ +$ ./gyp_skia +$ ninja -C out/Debug dm +$ out/Debug/dm -w good + + (do some work) + +$ ./gyp_skia +$ ninja -C out/Debug dm +$ out/Debug/dm -w bad + +$ ninja -C out/Debug skdiff +$ mkdir diff +$ out/Debug/skdiff good bad diff + + (open diff/index.html in your web browser) +~~~ + +That's the basics of DM. DM supports many other modes and flags. Here are a +few examples you might find handy. +~~~ +$ out/Debug/dm --help # Print all flags, their defaults, and a brief explanation of each. +$ out/Debug/dm --src tests # Run only unit tests. +$ out/Debug/dm --nocpu # Test only GPU-backed work. +$ out/Debug/dm --nogpu # Test only CPU-backed work. +$ out/Debug/dm --match blur # Run only work with "blur" in its name. +$ out/Debug/dm --dryRun # Don't really do anything, just print out what we'd do. +~~~ diff --git a/site/dev/testing/tests.md b/site/dev/testing/tests.md new file mode 100644 index 0000000000..fbe84dfeff --- /dev/null +++ b/site/dev/testing/tests.md @@ -0,0 +1,65 @@ +Writing Unit and Rendering Tests +================================ + +Writing a Unit Test +------------------- + +1. Add a file `tests/NewUnitTest.cpp`: + + + + /* + * Copyright ........ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file. + */ + #include "Test.h" + DEF_TEST(NewUnitTest, reporter) { + if (1 + 1 != 2) { + ERRORF(reporter, "%d + %d != %d", 1, 1, 2); + } + bool lifeIsGood = true; + REPORTER_ASSERT(reporter, lifeIsGood); + } + +2. Add a line to `gyp/tests.gypi`: + + '../tests/NewUnitTest.cpp', + +3. Recompile and run test: + + ./gyp_skia + ninja -C out/Debug dm + out/Debug/dm --match NewUnitTest + +Writing a Rendering Test +------------------------ + +1. Add a file `gm/newgmtest.cpp`: + + + + /* + * Copyright ........ + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file. + */ + #include "gm.h" + DEF_SIMPLE_GM(newgmtest, canvas, 128, 128) { + canvas->clear(SK_ColorWHITE); + SkPaint p; + p.setStrokeWidth(2); + canvas->drawLine(16, 16, 112, 112, p); + } + +2. Add a line to `gyp/gmslides.gypi`: + + '../gm/newgmtest.cpp', + +3. Recompile and run test: + + ./gyp_skia + ninja -C out/Debug dm + out/Debug/dm --match newgmtest -- cgit v1.2.3