Split generated_examples test into multiple test targets

PiperOrigin-RevId: 197490872

3 files changed, 152 insertions, 179 deletions

diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl
index 8521677682..9bfc0a0fbe 100644
--- a/tensorflow/contrib/lite/build_def.bzl
+++ b/tensorflow/contrib/lite/build_def.bzl
@@ -1,4 +1,8 @@
 """Generate Flatbuffer binary from json."""
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+)
 
 def tflite_copts():
   """Defines compile time flags."""
@@ -185,32 +189,102 @@ def json_to_tflite(name, src, out):
       tools = [flatc],
   )
 
-def gen_zipped_test_files(name, files):
+# This is the master list of generated examples that will be made into tests. A
+# function called make_XXX_tests() must also appear in generate_examples.py.
+# Disable a test by commenting it out. If you do, add a link to a bug or issue.
+def generated_test_models():
+    return [
+        "add",
+        "arg_max",
+        "avg_pool",
+        "batch_to_space_nd",
+        "concat",
+        "constant",
+        "control_dep",
+        "conv",
+        "depthwiseconv",
+        "div",
+        "exp",
+        "floor",
+        "fully_connected",
+        "fused_batch_norm",
+        "gather",
+        "global_batch_norm",
+        "greater",
+        "greater_equal",
+        "l2_pool",
+        "l2norm",
+        "less",
+        "less_equal",
+        "local_response_norm",
+        "log_softmax",
+        "max_pool",
+        "maximum",
+        "mean",
+        "minimum",
+        "mul",
+        "neg",
+        "pad",
+        "padv2",
+        # "prelu",
+        "relu",
+        "relu1",
+        "relu6",
+        "reshape",
+        "resize_bilinear",
+        "sigmoid",
+        "sin",
+        "slice",
+        "softmax",
+        "space_to_batch_nd",
+        "space_to_depth",
+        "split",
+        "squeeze",
+        "strided_slice",
+        "strided_slice_1d_exhaustive",
+        "sub",
+        "topk",
+        "transpose",
+        "transpose_conv",
+        "where",
+    ]
+
+def gen_zip_test(name, test_name, **kwargs):
+  """Generate a zipped-example test and its dependent zip files.
+
+  Args:
+    name: Resulting cc_test target name
+    test_name: Test targets this model. Comes from the list above.
+    **kwargs: tf_cc_test kwargs.
+  """
+  gen_zipped_test_file(
+      name = "zip_%s" % test_name,
+      file = "%s.zip" % test_name,
+  )
+  tf_cc_test(name, **kwargs)
+
+def gen_zipped_test_file(name, file):
   """Generate a zip file of tests by using :generate_examples.
 
   Args:
-    name: Name of output. We will produce "`name`_files" as a target.
-    files: A list of zip file basenames.
+    name: Name of output. We will produce "`file`.files" as a target.
+    file: The name of one of the generated_examples targets, e.g. "transpose"
   """
   toco = "//tensorflow/contrib/lite/toco:toco"
-  out_files = []
-  for f in files:
-    out_file = name + "/" + f
-    out_files.append(out_file)
-    native.genrule(
-        name = name + "_" + f + ".files",
-        cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco
-               + " --zip_to_output " + f + " $(@D)"),
-        outs = [out_file],
-        tools = [
-            ":generate_examples",
-            toco,
-        ],
-    )
+  native.genrule(
+      name = file + ".files",
+      cmd = ("$(locations :generate_examples) --toco $(locations %s) " % toco
+             + " --zip_to_output " + file + " $(@D)"),
+      outs = [file],
+      tools = [
+          ":generate_examples",
+          toco,
+      ],
+  )
 
   native.filegroup(
       name = name,
-      srcs = out_files,
+      srcs = [file],
   )
 
 def gen_selected_ops(name, model):
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 480685f4e9..74fc32a12b 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -6,7 +6,8 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow/contrib/lite:build_def.bzl",
-    "gen_zipped_test_files",
+    "gen_zip_test",
+    "generated_test_models",
 )
 load("//tensorflow/contrib/lite:special_rules.bzl", "tflite_portable_test_suite")
 load(
@@ -14,61 +15,52 @@ load(
     "tf_cc_test",
 )
 
-gen_zipped_test_files(
-    name = "optest",
-    files = [
-        "add.zip",
-        "arg_max.zip",
-        "avg_pool.zip",
-        "batch_to_space_nd.zip",
-        "concat.zip",
-        "constant.zip",
-        "control_dep.zip",
-        "conv.zip",
-        "depthwiseconv.zip",
-        "div.zip",
-        "exp.zip",
-        "floor.zip",
-        "fully_connected.zip",
-        "fused_batch_norm.zip",
-        "gather.zip",
-        "global_batch_norm.zip",
-        "greater.zip",
-        "greater_equal.zip",
-        "l2_pool.zip",
-        "l2norm.zip",
-        "less.zip",
-        "less_equal.zip",
-        "local_response_norm.zip",
-        "log_softmax.zip",
-        "max_pool.zip",
-        "maximum.zip",
-        "mean.zip",
-        "minimum.zip",
-        "mul.zip",
-        "neg.zip",
-        "pad.zip",
-        "padv2.zip",
-        "relu.zip",
-        "relu1.zip",
-        "relu6.zip",
-        "reshape.zip",
-        "resize_bilinear.zip",
-        "sigmoid.zip",
-        "sin.zip",
-        "slice.zip",
-        "softmax.zip",
-        "space_to_batch_nd.zip",
-        "space_to_depth.zip",
-        "split.zip",
-        "squeeze.zip",
-        "strided_slice.zip",
-        "strided_slice_1d_exhaustive.zip",
-        "sub.zip",
-        "topk.zip",
-        "transpose.zip",
-        "transpose_conv.zip",
-        "where.zip",
+[gen_zip_test(
+    name = "zip_test_%s" % test_name,
+    size = "large",
+    srcs = ["generated_examples_zip_test.cc"],
+    args = [
+        "--zip_file_path=$(location :zip_%s)" % test_name,
+        # TODO(angerson) We may be able to add an external unzip binary instead
+        # of relying on an existing one for OSS builds.
+        "--unzip_binary_path=/usr/bin/unzip",
+    ],
+    data = [
+        ":zip_%s" % test_name,
+    ],
+    shard_count = 20,
+    tags = [
+        "gen_zip_test",
+        "no_oss",
+        "tflite_not_portable",
+    ],
+    test_name = test_name,
+    deps = [
+        ":parse_testdata_lib",
+        ":tflite_driver",
+        ":util",
+        "@com_google_googletest//:gtest",
+        "@com_googlesource_code_re2//:re2",
+        "//tensorflow/contrib/lite:builtin_op_data",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:builtin_ops",
+    ] + select({
+        "//conditions:default": [
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+            "//tensorflow/core:test",
+        ],
+        "//tensorflow:android": [
+            "//tensorflow/core:android_tensorflow_lib",
+            "//tensorflow/core:android_tensorflow_test_lib",
+        ],
+    }),
+) for test_name in generated_test_models()]
+
+test_suite(
+    name = "generated_zip_tests",
+    tags = [
+        "gen_zip_test",
     ],
 )
 
@@ -353,42 +345,4 @@ cc_binary(
     ],
 )
 
-tf_cc_test(
-    name = "generated_examples_zip_test",
-    size = "large",
-    srcs = ["generated_examples_zip_test.cc"],
-    args = [
-        "--zip_files_dir=tensorflow/contrib/lite/testing/optest",
-        # TODO(angerson) We may be able to add an external unzip binary instead
-        # of relying on an existing one for OSS builds.
-        "--unzip_binary_path=/usr/bin/unzip",
-    ],
-    data = [":optest"],
-    shard_count = 20,
-    tags = [
-        "no_oss",
-        "tflite_not_portable",
-    ],
-    deps = [
-        ":parse_testdata_lib",
-        ":tflite_driver",
-        ":util",
-        "@com_google_googletest//:gtest",
-        "@com_googlesource_code_re2//:re2",
-        "//tensorflow/contrib/lite:builtin_op_data",
-        "//tensorflow/contrib/lite:framework",
-        "//tensorflow/contrib/lite/kernels:builtin_ops",
-    ] + select({
-        "//conditions:default": [
-            "//tensorflow/core:framework_internal",
-            "//tensorflow/core:lib",
-            "//tensorflow/core:test",
-        ],
-        "//tensorflow:android": [
-            "//tensorflow/core:android_tensorflow_lib",
-            "//tensorflow/core:android_tensorflow_test_lib",
-        ],
-    }),
-)
-
 tflite_portable_test_suite()
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 581535db39..2f069ff8e7 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -35,7 +35,7 @@ namespace {
 bool FLAGS_ignore_known_bugs = true;
 // TODO(b/71769302) zip_files_dir should have a more accurate default, if
 // possible
-string* FLAGS_zip_files_dir = new string("./");
+string* FLAGS_zip_file_path = new string("./");
 string* FLAGS_unzip_binary_path = new string("/usr/bin/unzip");
 }  // namespace
 
@@ -137,7 +137,10 @@ class ZipEnvironment : public ::testing::Environment {
       *out_dir = dir;
       return tensorflow::Status::OK();
     } else {
-      return tensorflow::Status(tensorflow::error::UNKNOWN, "unzip failed");
+      return tensorflow::Status(tensorflow::error::UNKNOWN,
+                                "unzip failed. "
+                                "stdout:\n" +
+                                    out + "\nstderr:\n" + err);
     }
   }
 
@@ -191,8 +194,7 @@ tensorflow::Status ReadManifest(const string& original_file, const string& dir,
 }
 
 // Get a list of tests from a zip file `zip_file_name`.
-std::vector<string> UnarchiveZipAndFindTestNames(const string& zip_file_name) {
-  string zip_file = *FLAGS_zip_files_dir + "/" + zip_file_name;
+std::vector<string> UnarchiveZipAndFindTestNames(const string& zip_file) {
   string decompress_tmp_dir;
   TF_CHECK_OK(zip_environment()->UnZip(zip_file, &decompress_tmp_dir));
   std::vector<string> stuff;
@@ -251,67 +253,10 @@ struct ZipPathParamName {
   }
 };
 
-// Instantiate a test. This assumes `zip_base`.zip is a declared data file
-// of this test.
-#define INSTANTIATE_TESTS(zip_base)                                        \
-  INSTANTIATE_TEST_CASE_P(                                                 \
-      zip_base, OpsTest,                                                   \
-      ::testing::ValuesIn(UnarchiveZipAndFindTestNames(#zip_base ".zip")), \
-      ZipPathParamName());
-
-INSTANTIATE_TESTS(add)
-INSTANTIATE_TESTS(arg_max)
-INSTANTIATE_TESTS(avg_pool)
-INSTANTIATE_TESTS(batch_to_space_nd)
-INSTANTIATE_TESTS(concat)
-INSTANTIATE_TESTS(constant)
-INSTANTIATE_TESTS(control_dep)
-INSTANTIATE_TESTS(conv)
-INSTANTIATE_TESTS(depthwiseconv)
-INSTANTIATE_TESTS(div)
-INSTANTIATE_TESTS(exp)
-INSTANTIATE_TESTS(floor)
-INSTANTIATE_TESTS(fully_connected)
-INSTANTIATE_TESTS(fused_batch_norm)
-INSTANTIATE_TESTS(gather)
-INSTANTIATE_TESTS(global_batch_norm)
-INSTANTIATE_TESTS(greater)
-INSTANTIATE_TESTS(greater_equal)
-INSTANTIATE_TESTS(l2_pool)
-INSTANTIATE_TESTS(l2norm)
-INSTANTIATE_TESTS(less)
-INSTANTIATE_TESTS(less_equal)
-INSTANTIATE_TESTS(local_response_norm)
-INSTANTIATE_TESTS(log_softmax)
-INSTANTIATE_TESTS(max_pool)
-INSTANTIATE_TESTS(maximum)
-INSTANTIATE_TESTS(mean)
-INSTANTIATE_TESTS(minimum)
-INSTANTIATE_TESTS(mul)
-INSTANTIATE_TESTS(neg)
-INSTANTIATE_TESTS(pad)
-INSTANTIATE_TESTS(padv2)
-// INSTANTIATE_TESTS(prelu)
-INSTANTIATE_TESTS(relu)
-INSTANTIATE_TESTS(relu1)
-INSTANTIATE_TESTS(relu6)
-INSTANTIATE_TESTS(reshape)
-INSTANTIATE_TESTS(resize_bilinear)
-INSTANTIATE_TESTS(sigmoid)
-INSTANTIATE_TESTS(sin)
-INSTANTIATE_TESTS(slice)
-INSTANTIATE_TESTS(softmax)
-INSTANTIATE_TESTS(space_to_batch_nd)
-INSTANTIATE_TESTS(space_to_depth)
-INSTANTIATE_TESTS(split)
-INSTANTIATE_TESTS(squeeze)
-INSTANTIATE_TESTS(strided_slice)
-INSTANTIATE_TESTS(strided_slice_1d_exhaustive)
-INSTANTIATE_TESTS(sub)
-INSTANTIATE_TESTS(topk)
-INSTANTIATE_TESTS(transpose)
-INSTANTIATE_TESTS(transpose_conv)
-INSTANTIATE_TESTS(where)
+INSTANTIATE_TEST_CASE_P(
+    tests, OpsTest,
+    ::testing::ValuesIn(UnarchiveZipAndFindTestNames(*FLAGS_zip_file_path)),
+    ZipPathParamName());
 
 }  // namespace testing
 }  // namespace tflite
@@ -324,8 +269,8 @@ int main(int argc, char** argv) {
           "ignore_known_bugs", &tflite::testing::FLAGS_ignore_known_bugs,
           "If a particular model is affected by a known bug, the "
           "corresponding test should expect the outputs to not match."),
-      tensorflow::Flag("zip_files_dir", tflite::testing::FLAGS_zip_files_dir,
-                       "Required: Location of the test zips."),
+      tensorflow::Flag("zip_file_path", tflite::testing::FLAGS_zip_file_path,
+                       "Required: Location of the test zip file."),
       tensorflow::Flag("unzip_binary_path",
                        tflite::testing::FLAGS_unzip_binary_path,
                        "Required: Location of a suitable unzip binary.")};