From 0ee2627026370bbb25e4efda490e14e468ac86fb Mon Sep 17 00:00:00 2001
From: scroggo <scroggo@google.com>
Date: Fri, 7 Nov 2014 06:07:32 -0800
Subject: When running DM, write test failures to json.

Add skiatest::Failure to keep track of data about a test failure.

Reporter::reportFailed and ::onReportFailed now take Failure as a
parameter. This allows the implementation to treat the failure as it
wishes. Provide a helper to format the failure the same as prior to
the change.

Update the macros for calling reportFailed (REPORTER_ASSERT etc) to
create a Failure object.

Convert a direct call to reportFailed to the macro ERRORF.

Write Failures to Json.
Sample output when running dm on the dummy test crrev.com/705723004:
{
   "test_results" : {
      "failures" : [
         {
            "condition" : "0 > 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 10,
            "message" : ""
         },
         {
            "condition" : "false",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 4,
            "message" : ""
         },
         {
            "condition" : "1 == 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 5,
            "message" : "I can too count!"
         },
         {
            "condition" : "",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 6,
            "message" : "seven is 7"
         },
         {
            "condition" : "1 == 3",
            "file_name" : "../../tests/DummyTest.cpp",
            "line_no" : 14,
            "message" : "I can too count!"
         }
      ]
   }
}

Report all of the failures from one test.
Previously, if one test had multiple failures, only one was reportered.
e.g:

Failures:
  test Dummy: ../../tests/DummyTest.cpp:6   seven is 7
  test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
  test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.

Now, we get all the messages:

Failures:
  test Dummy: ../../tests/DummyTest.cpp:4   false
        ../../tests/DummyTest.cpp:5 I can too count!: 1 == 3
        ../../tests/DummyTest.cpp:6 seven is 7
  test Dummy2: ../../tests/DummyTest.cpp:10 0 > 3
  test Dummy3: ../../tests/DummyTest.cpp:14 I can too count!: 1 == 3
3 failures.

(Note that we still state "3 failures" because 3 DM::Tasks failed.)

BUG=skia:3082
BUG=skia:2454

Review URL: https://codereview.chromium.org/694703005
---
 dm/DMJsonWriter.cpp | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

(limited to 'dm/DMJsonWriter.cpp')

diff --git a/dm/DMJsonWriter.cpp b/dm/DMJsonWriter.cpp
index f86264715b..a2de38bb31 100644
--- a/dm/DMJsonWriter.cpp
+++ b/dm/DMJsonWriter.cpp
@@ -24,6 +24,14 @@ void JsonWriter::AddBitmapResult(const BitmapResult& result) {
     gBitmapResults.push_back(result);
 }
 
+SkTArray<skiatest::Failure> gFailures;
+SK_DECLARE_STATIC_MUTEX(gFailureLock);
+
+void JsonWriter::AddTestFailure(const skiatest::Failure& failure) {
+    SkAutoMutexAcquire lock(gFailureLock);
+    gFailures.push_back(failure);
+}
+
 void JsonWriter::DumpJson() {
     if (FLAGS_writePath.isEmpty()) {
         return;
@@ -52,6 +60,19 @@ void JsonWriter::DumpJson() {
         }
     }
 
+    {
+        SkAutoMutexAcquire lock(gFailureLock);
+        for (int i = 0; i < gFailures.count(); i++) {
+            Json::Value result;
+            result["file_name"]     = gFailures[i].fileName;
+            result["line_no"]       = gFailures[i].lineNo;
+            result["condition"]     = gFailures[i].condition;
+            result["message"]       = gFailures[i].message.c_str();
+
+            root["test_results"]["failures"].append(result);
+        }
+    }
+
     SkString path = SkOSPath::Join(FLAGS_writePath[0], "dm.json");
     SkFILEWStream stream(path.c_str());
     stream.writeText(Json::StyledWriter().write(root).c_str());
-- 
cgit v1.2.3