Make the srcs test be a Bazel test

./compile.sh srcs has been broken at several time because this
test cannot run on our internal presubmit (more precisely, don't see
most breakage on our internal presubmit). Making this test a Bazel
test should fix that at the same time reducing the scope of ./compile.sh
(with the other move of the determinism test, this make compile.sh
being only the bootstrapping script).

To do so, we fetch the list of sources in the repository through
a skylark repository and use genquery to interrogate the list of
sources in //:srcs filegroup.

Change-Id: If83af28a366cc0c52a7327ce9ed2489c7e88532f
PiperOrigin-RevId: 166064682

6 files changed, 112 insertions, 2 deletions

diff --git a/BUILD b/BUILD
index 3c506215c8..9d65300389 100644
--- a/BUILD
+++ b/BUILD
@@ -22,7 +22,7 @@ filegroup(
         "//tools:srcs",
         "//third_party:srcs",
     ],
-    visibility = ["//visibility:private"],
+    visibility = ["//src/test/shell/bazel:__pkg__"],
 )
 
 filegroup(
diff --git a/WORKSPACE b/WORKSPACE
index e9665ae9a2..20dac815cc 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -39,6 +39,10 @@ docker_repository()
 load("//src/test/docker:flavours.bzl", "pull_images_for_docker_tests")
 pull_images_for_docker_tests()
 
+# For src/test/shell/bazel:test_srcs
+load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository")
+list_source_repository(name = "local_bazel_source_list")
+
 # To run the Android integration tests in //src/test/shell/bazel/android:all or
 # build the Android sample app in //examples/android/java/bazel:hello_world
 #
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 62f0c55e90..8203e3d36c 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -481,6 +481,23 @@ sh_test(
     data = [":test-deps"],
 )
 
+genquery(
+    name = "srcs_list",
+    expression = "kind(\"source file\", deps(//:srcs))",
+    scope = ["//:srcs"],
+)
+
+sh_test(
+    name = "srcs_test",
+    size = "small",
+    srcs = ["srcs_test.sh"],
+    data = [
+        ":srcs_list",
+        "@local_bazel_source_list//:sources",
+    ],
+    tags = ["no_windows"],
+)
+
 test_suite(
     name = "all_tests",
     visibility = ["//visibility:public"],
diff --git a/src/test/shell/bazel/list_source_repository.bzl b/src/test/shell/bazel/list_source_repository.bzl
new file mode 100644
index 0000000000..f0900ab3ae
--- /dev/null
+++ b/src/test/shell/bazel/list_source_repository.bzl
@@ -0,0 +1,48 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Create a repository that produce the list of sources of Bazel in order to test
+# that all sources in Bazel are contained in the //:srcs filegroup. CI systems
+# can set the SRCS_EXCLUDES environment variable to exclude certain files from
+# being considered as sources.
+"""A repository definition to fetch all sources in Bazel."""
+
+def _impl(rctx):
+  workspace = rctx.path(Label("//:BUILD")).dirname
+  srcs_excludes = "XXXXXXXXXXXXXX1268778dfsdf4"
+  if "SRCS_EXCLUDES" in rctx.os.environ:
+    srcs_excludes = rctx.os.environ["SRCS_EXCLUDES"]
+  r = rctx.execute(["find", str(workspace), "-type", "f"])
+  rctx.file("find.result.raw", r.stdout.replace(str(workspace) + "/", ""))
+  rctx.file("BUILD", """
+genrule(
+  name = "sources",
+  outs = ["sources.txt"],
+  srcs = ["find.result.raw"],
+  visibility = ["//visibility:public"],
+  cmd = " | ".join([
+    "cat $<",
+    "grep -Ev '^(\\\\.git|out/|output/|bazel-|derived|tools/defaults/BUILD)'",
+    "grep -Ev '%s'",
+    "sort -u > $@",
+  ]),
+)
+""" % srcs_excludes)
+
+list_source_repository = repository_rule(
+    implementation = _impl,
+    environ = ["SRCS_EXCLUDES"])
+"""Create a //:sources target containing the list of sources of Bazel.
+
+SRCS_EXCLUDES give a regex of files to excludes in the list."""
diff --git a/src/test/shell/bazel/srcs_test.sh b/src/test/shell/bazel/srcs_test.sh
new file mode 100644
index 0000000000..422b58a82b
--- /dev/null
+++ b/src/test/shell/bazel/srcs_test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Test that all sources in Bazel are contained in the //:srcs filegroup
+# Actually this test just compares the two input (the file list and the
+# //:srcs filegroup and show the diff)
+
+LIST_SRCS="${TEST_SRCDIR}/local_bazel_source_list/sources.txt"
+SRCS_QUERY="$(mktemp)"
+
+cat "${TEST_SRCDIR}/io_bazel/src/test/shell/bazel/srcs_list" \
+  | grep -v '^@' \
+  | sed -e 's|^//||' | sed -e 's|^:||' | sed -e 's|:|/|' \
+  | sort -u >"${SRCS_QUERY}"
+
+res="$(diff -U 0 "${LIST_SRCS}" "${SRCS_QUERY}" | sed -e 's|^-||' \
+  | grep -Ev '^(@@|\+\+|--)' || true)"
+
+if [ -n "${res}" ]; then
+  echo "//:srcs filegroup do not contains all the sources, missing:
+${res}"
+  exit 1
+fi
diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh
index 45a23c4132..7791a91c4e 100755
--- a/src/test/shell/testenv.sh
+++ b/src/test/shell/testenv.sh
@@ -162,7 +162,7 @@ fi
 
 
 function use_bazel_workspace_file() {
-  mkdir -p src/test/docker
+  mkdir -p src/test/{shell/bazel,docker}
   cat >src/test/docker/docker_repository.bzl <<EOF
 def docker_repository():
   pass
@@ -172,6 +172,11 @@ def pull_images_for_docker_tests():
   pass
 EOF
   touch src/test/docker/BUILD
+  cat >src/test/shell/bazel/list_source_repository.bzl <<EOF
+def list_source_repository(name):
+  pass
+EOF
+  touch src/test/shell/bazel/BUILD
   rm -f WORKSPACE
   ln -sf ${workspace_file} WORKSPACE
 }