From 0baff0f51ecbc7956086d21e5c01201bc6672df9 Mon Sep 17 00:00:00 2001 From: Damien Martin-Guillerez Date: Tue, 22 Aug 2017 17:40:37 +0200 Subject: Make the srcs test be a Bazel test ./compile.sh srcs has been broken at several time because this test cannot run on our internal presubmit (more precisely, don't see most breakage on our internal presubmit). Making this test a Bazel test should fix that at the same time reducing the scope of ./compile.sh (with the other move of the determinism test, this make compile.sh being only the bootstrapping script). To do so, we fetch the list of sources in the repository through a skylark repository and use genquery to interrogate the list of sources in //:srcs filegroup. Change-Id: If83af28a366cc0c52a7327ce9ed2489c7e88532f PiperOrigin-RevId: 166064682 --- BUILD | 2 +- WORKSPACE | 4 +++ src/test/shell/bazel/BUILD | 17 +++++++++ src/test/shell/bazel/list_source_repository.bzl | 48 +++++++++++++++++++++++++ src/test/shell/bazel/srcs_test.sh | 36 +++++++++++++++++++ src/test/shell/testenv.sh | 7 +++- 6 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 src/test/shell/bazel/list_source_repository.bzl create mode 100644 src/test/shell/bazel/srcs_test.sh diff --git a/BUILD b/BUILD index 3c506215c8..9d65300389 100644 --- a/BUILD +++ b/BUILD @@ -22,7 +22,7 @@ filegroup( "//tools:srcs", "//third_party:srcs", ], - visibility = ["//visibility:private"], + visibility = ["//src/test/shell/bazel:__pkg__"], ) filegroup( diff --git a/WORKSPACE b/WORKSPACE index e9665ae9a2..20dac815cc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -39,6 +39,10 @@ docker_repository() load("//src/test/docker:flavours.bzl", "pull_images_for_docker_tests") pull_images_for_docker_tests() +# For src/test/shell/bazel:test_srcs +load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository") +list_source_repository(name = "local_bazel_source_list") + # To run the Android integration tests in //src/test/shell/bazel/android:all or # build the Android sample app in //examples/android/java/bazel:hello_world # diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 62f0c55e90..8203e3d36c 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -481,6 +481,23 @@ sh_test( data = [":test-deps"], ) +genquery( + name = "srcs_list", + expression = "kind(\"source file\", deps(//:srcs))", + scope = ["//:srcs"], +) + +sh_test( + name = "srcs_test", + size = "small", + srcs = ["srcs_test.sh"], + data = [ + ":srcs_list", + "@local_bazel_source_list//:sources", + ], + tags = ["no_windows"], +) + test_suite( name = "all_tests", visibility = ["//visibility:public"], diff --git a/src/test/shell/bazel/list_source_repository.bzl b/src/test/shell/bazel/list_source_repository.bzl new file mode 100644 index 0000000000..f0900ab3ae --- /dev/null +++ b/src/test/shell/bazel/list_source_repository.bzl @@ -0,0 +1,48 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Create a repository that produce the list of sources of Bazel in order to test +# that all sources in Bazel are contained in the //:srcs filegroup. CI systems +# can set the SRCS_EXCLUDES environment variable to exclude certain files from +# being considered as sources. +"""A repository definition to fetch all sources in Bazel.""" + +def _impl(rctx): + workspace = rctx.path(Label("//:BUILD")).dirname + srcs_excludes = "XXXXXXXXXXXXXX1268778dfsdf4" + if "SRCS_EXCLUDES" in rctx.os.environ: + srcs_excludes = rctx.os.environ["SRCS_EXCLUDES"] + r = rctx.execute(["find", str(workspace), "-type", "f"]) + rctx.file("find.result.raw", r.stdout.replace(str(workspace) + "/", "")) + rctx.file("BUILD", """ +genrule( + name = "sources", + outs = ["sources.txt"], + srcs = ["find.result.raw"], + visibility = ["//visibility:public"], + cmd = " | ".join([ + "cat $<", + "grep -Ev '^(\\\\.git|out/|output/|bazel-|derived|tools/defaults/BUILD)'", + "grep -Ev '%s'", + "sort -u > $@", + ]), +) +""" % srcs_excludes) + +list_source_repository = repository_rule( + implementation = _impl, + environ = ["SRCS_EXCLUDES"]) +"""Create a //:sources target containing the list of sources of Bazel. + +SRCS_EXCLUDES give a regex of files to excludes in the list.""" diff --git a/src/test/shell/bazel/srcs_test.sh b/src/test/shell/bazel/srcs_test.sh new file mode 100644 index 0000000000..422b58a82b --- /dev/null +++ b/src/test/shell/bazel/srcs_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Test that all sources in Bazel are contained in the //:srcs filegroup +# Actually this test just compares the two input (the file list and the +# //:srcs filegroup and show the diff) + +LIST_SRCS="${TEST_SRCDIR}/local_bazel_source_list/sources.txt" +SRCS_QUERY="$(mktemp)" + +cat "${TEST_SRCDIR}/io_bazel/src/test/shell/bazel/srcs_list" \ + | grep -v '^@' \ + | sed -e 's|^//||' | sed -e 's|^:||' | sed -e 's|:|/|' \ + | sort -u >"${SRCS_QUERY}" + +res="$(diff -U 0 "${LIST_SRCS}" "${SRCS_QUERY}" | sed -e 's|^-||' \ + | grep -Ev '^(@@|\+\+|--)' || true)" + +if [ -n "${res}" ]; then + echo "//:srcs filegroup do not contains all the sources, missing: +${res}" + exit 1 +fi diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh index 45a23c4132..7791a91c4e 100755 --- a/src/test/shell/testenv.sh +++ b/src/test/shell/testenv.sh @@ -162,7 +162,7 @@ fi function use_bazel_workspace_file() { - mkdir -p src/test/docker + mkdir -p src/test/{shell/bazel,docker} cat >src/test/docker/docker_repository.bzl <src/test/shell/bazel/list_source_repository.bzl <