diff options
| -rw-r--r-- | BUILD | 1 | ||||
| -rw-r--r-- | WORKSPACE | 6 | ||||
| -rw-r--r-- | scripts/ci/bazel-slow-tests.json | 23 | ||||
| -rw-r--r-- | scripts/ij.bazelproject | 1 | ||||
| -rw-r--r-- | src/BUILD | 1 | ||||
| -rw-r--r-- | src/test/docker/BUILD | 68 | ||||
| -rw-r--r-- | src/test/docker/Dockerfile.centos6.9 | 16 | ||||
| -rw-r--r-- | src/test/docker/Dockerfile.debian-stretch | 7 | ||||
| -rw-r--r-- | src/test/docker/docker_pull.bzl | 80 | ||||
| -rw-r--r-- | src/test/docker/docker_repository.bzl | 64 | ||||
| -rw-r--r-- | src/test/docker/docker_test.py | 158 | ||||
| -rw-r--r-- | src/test/docker/flavours.bzl | 34 | ||||
| -rwxr-xr-x | src/test/shell/testenv.sh | 11 |
13 files changed, 2 insertions, 468 deletions
@@ -42,7 +42,6 @@ filegroup( name = "workspace-file", srcs = [":WORKSPACE"], visibility = [ - "//src/test/docker:__pkg__", "//src/test/shell/bazel:__subpackages__", ], ) @@ -40,12 +40,6 @@ http_archive( strip_prefix = "j2objc-2.0.3", ) -# For src/test/docker/... -load("//src/test/docker:docker_repository.bzl", "docker_repository") -docker_repository() -load("//src/test/docker:flavours.bzl", "pull_images_for_docker_tests") -pull_images_for_docker_tests() - # For src/test/shell/bazel:test_srcs load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository") list_source_repository(name = "local_bazel_source_list") diff --git a/scripts/ci/bazel-slow-tests.json b/scripts/ci/bazel-slow-tests.json index 43f6a469a1..9e648ad873 100644 --- a/scripts/ci/bazel-slow-tests.json +++ b/scripts/ci/bazel-slow-tests.json @@ -1,26 +1,5 @@ [ { - "node": "docker", - "configurations": [ - { - "shard": 1, - "parameters": {"test_tag_filters": ["shard-1"]} - }, - { - "shard": 2, - "parameters": {"test_tag_filters": ["shard-2"]} - }, - { - "shard": 3, - "parameters": {"test_tag_filters": ["shard-3"]} - } - ], - "parameters": { - "targets": [], - "tests": ["//src/test/docker/..."] - } - }, - { "configurations": [ {"node": "linux-x86_64"}, {"node": "ubuntu_16.04-x86_64"}, @@ -32,7 +11,7 @@ "parameters": { "targets": [], "tests": ["//src/test/..."], - "test_tag_filters": ["-docker", "+slow"] + "test_tag_filters": ["+slow"] } } ] diff --git a/scripts/ij.bazelproject b/scripts/ij.bazelproject index e90868242d..4cdd223ef1 100644 --- a/scripts/ij.bazelproject +++ b/scripts/ij.bazelproject @@ -18,5 +18,4 @@ targets: //src/java_tools/junitrunner/javatests/... //src/java_tools/singlejar:SingleJar //src/test/... - -//src/test/docker/... //src/tools/remote/... @@ -336,7 +336,6 @@ filegroup( "//src/objc_tools/bundlemerge:srcs", "//src/objc_tools/plmerge:srcs", "//src/test/cpp:srcs", - "//src/test/docker:srcs", "//src/test/gen:srcs", "//src/test/native:srcs", "//src/test/skylark:srcs", diff --git a/src/test/docker/BUILD b/src/test/docker/BUILD deleted file mode 100644 index da0e7ecb06..0000000000 --- a/src/test/docker/BUILD +++ /dev/null @@ -1,68 +0,0 @@ -load("//tools/build_defs/docker:docker.bzl", "docker_build") -load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar") -load("//src/test/docker:flavours.bzl", "FLAVOURS") - -pkg_tar( - name = "bazel_cc_configure", - package_dir = "/opt/workspace", - strip_prefix = "/", - deps = ["//:bazel-distfile-tar"], -) - -[ - [docker_build( - name = "bazel_cc_configure-%s-%s" % (flavour, mode), - base = "@docker-%s//:image" % flavour, - entrypoint = "/opt/workspace/compile.sh && ./output/bazel test ${EXTRA_BAZEL_ARGS} //examples/cpp:hello-success_test", - env = { - "EXTRA_BAZEL_ARGS": "--spawn_strategy=standalone --genrule_strategy=standalone -c %s" % mode, - }, - tars = [":bazel_cc_configure"], - workdir = "/opt/workspace", - ) for mode in [ - "dbg", - "opt", - "fastbuild", - ]] - for flavour in FLAVOURS -] - -# Shard the docker test by compilation mode -SHARD_NUMBER = { - "opt": 1, - "fastbuild": 2, - "dbg": 3, -} - -[ - [py_test( - name = "test_cc_configure-%s-%s" % (flavour, mode), - size = "large", - srcs = ["docker_test.py"], - args = [ - "--main='$(location :bazel_cc_configure-%s-%s)'" % (flavour, mode), - "--docker='../docker/docker'", - ], - data = [ - ":bazel_cc_configure-%s-%s" % (flavour, mode), - "@docker//:docker", - ], - local = 1, - main = "docker_test.py", - tags = [ - "docker", - "exclusive", - "local", - "slow", - "shard-%s" % SHARD_NUMBER[mode], - ], - deps = ["//third_party/py/gflags"], - ) for mode in SHARD_NUMBER] - for flavour in FLAVOURS -] - -filegroup( - name = "srcs", - srcs = glob(["**"]), - visibility = ["//src:__pkg__"], -) diff --git a/src/test/docker/Dockerfile.centos6.9 b/src/test/docker/Dockerfile.centos6.9 deleted file mode 100644 index 90177871de..0000000000 --- a/src/test/docker/Dockerfile.centos6.9 +++ /dev/null @@ -1,16 +0,0 @@ -FROM centos:centos6.9 - -RUN yum -y upgrade - -RUN yum -y install \ - wget git findutils binutils \ - zip unzip tar gzip zlib-devel \ - clang gcc gcc-c++ \ - java java-devel java-1.8.0-openjdk-devel \ - python; yum clean all - -RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo -RUN yum -y install devtoolset-2-gcc devtoolset-2-gcc-c++ devtoolset-2-binutils; yum clean all - -ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 -ENV CC /opt/rh/devtoolset-2/root/usr/bin/gcc diff --git a/src/test/docker/Dockerfile.debian-stretch b/src/test/docker/Dockerfile.debian-stretch deleted file mode 100644 index a1f54fbe89..0000000000 --- a/src/test/docker/Dockerfile.debian-stretch +++ /dev/null @@ -1,7 +0,0 @@ -FROM debian:stretch -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl ca-certificates ca-certificates-java \ - git pkg-config build-essential \ - zip unzip zlib1g-dev libarchive-dev \ - g++ gcc openjdk-8-jdk python && \ - apt-get clean diff --git a/src/test/docker/docker_pull.bzl b/src/test/docker/docker_pull.bzl deleted file mode 100644 index 61b395900b..0000000000 --- a/src/test/docker/docker_pull.bzl +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Quick and not really nice docker_pull rules based on the docker daemon.""" - -def _impl(repository_ctx): - docker = repository_ctx.which("docker") - if docker == None and repository_ctx.attr.optional: - repository_ctx.file("BUILD", """ -load("@io_bazel//tools/build_defs/docker:docker.bzl", "docker_build") - -# an empty image to still allow building despite not having the base -# image. -docker_build( - name = "image", - visibility = ['//visibility:public'], -) -""") - repository_ctx.file("image.tar") - return - - repository_ctx.file("BUILD", """ -load("@io_bazel//tools/build_defs/docker:docker.bzl", "docker_build") - -docker_build( - name = "image", - base = ":base.tar", - visibility = ["//visibility:public"], -) -""") - tag = repository_ctx.attr.tag - cmd = "pull" - if repository_ctx.attr.dockerfile: - dockerfile = repository_ctx.path(repository_ctx.attr.dockerfile) - cmd = "build" - print("Running `docker build`") - result = repository_ctx.execute([ - docker, - "build", - "-q", - "-t", - tag, - "-f", - dockerfile, - dockerfile.dirname, - ], quiet=False, timeout=3600) - else: - print("Running `docker pull`") - result = repository_ctx.execute([docker, "pull", tag], quiet=False, timeout=3600) - if result.return_code: - fail("docker %s failed with error code %s:\n%s" % ( - cmd, - result.return_code, - result.stderr)) - result = repository_ctx.execute([ - docker, "save", "-o", repository_ctx.path("base.tar"), tag]) - if result.return_code: - fail("docker save failed with error code %s:\n%s" % ( - result.return_code, - result.stderr)) - -docker_pull = repository_rule( - implementation = _impl, - attrs = { - "tag": attr.string(mandatory=True), - "dockerfile": attr.label(default=None), - "optional": attr.bool(default=False), - }, -) diff --git a/src/test/docker/docker_repository.bzl b/src/test/docker/docker_repository.bzl deleted file mode 100644 index 1b49d1a9ed..0000000000 --- a/src/test/docker/docker_repository.bzl +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Rule for importing the docker binary for tests (experimental).""" - -def _impl(ctx): - docker = ctx.which("docker") - if docker == None: - # We cannot find docker, we won't be able to run tests depending - # on it, silently ignoring. - ctx.file("BUILD", - "\n".join([ - "sh_binary(", - " name = 'docker',", - " srcs = ['docker.sh'],", - " visibility = ['//visibility:public'],", - ")" - ])) - ctx.file("docker.sh", "\n".join([ - "#!/bin/bash", - "echo 'ERROR: docker is not installed' >&2", - "exit 1" - ])) - else: - exports = [] - for k in ctx.os.environ: - # DOCKER* environment variable are used by the docker client - # to know how to talk to the docker daemon. - if k.startswith("DOCKER"): - exports.append("export %s='%s'" % (k, ctx.os.environ[k])) - ctx.symlink(docker, "docker-bin") - ctx.file("docker.sh", "\n".join([ - "#!/bin/bash", - "\n".join(exports), -"""BIN="$0" -while [ -L "${BIN}" ]; do - BIN="$(readlink "${BIN}")" -done -exec "${BIN%%.sh}-bin" "$@" -"""])) - ctx.file("BUILD", "\n".join([ - "sh_binary(", - " name = 'docker',", - " srcs = ['docker.sh'],", - " data = [':docker-bin'],", - " visibility = ['//visibility:public'],", - ")"])) - -docker_repository_ = repository_rule(_impl) - -def docker_repository(): - """Declare a @docker repository that provide a docker binary.""" - docker_repository_(name = "docker") - diff --git a/src/test/docker/docker_test.py b/src/test/docker/docker_test.py deleted file mode 100644 index a185e27775..0000000000 --- a/src/test/docker/docker_test.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A simple test runner for docker (experimental).""" - -import copy -import os -import os.path -import shlex -import StringIO -import subprocess -import sys -import threading - -from third_party.py import gflags - -gflags.DEFINE_multistring( - "image", [], - "The list of additional docker image to load (path to a docker_build " - "target), optional.") - -gflags.DEFINE_string( - "main", None, - "The main image to run (path to a docker_build target), mandatory.") -gflags.MarkFlagAsRequired("main") - -gflags.DEFINE_string( - "cmd", None, - "A command to provide to the docker image, optional (default: use the " - "entrypoint).") - -gflags.DEFINE_string("docker", "docker", "Path to the docker client binary.") - -gflags.DEFINE_boolean("verbose", True, "Be verbose.") - -FLAGS = gflags.FLAGS - -LOCAL_IMAGE_PREFIX = "bazel/" - - -def _copy_stream(in_stream, out_stream): - for c in iter(lambda: in_stream.read(1), ""): - out_stream.write(c) - out_stream.flush() - - -def execute(command, stdout=sys.stdout, stderr=sys.stderr, env=None): - """Execute a command while redirecting its output streams.""" - if FLAGS.verbose: - print "Executing '%s'" % " ".join(command) - p = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - t1 = threading.Thread(target=_copy_stream, args=[p.stdout, stdout]) - t2 = threading.Thread(target=_copy_stream, args=[p.stderr, stderr]) - t1.daemon = True - t2.daemon = True - t1.start() - t2.start() - p.wait() - t1.join() - t2.join() - return p.returncode - - -def load_image(image): - """Load a docker image using the runner provided by docker_build.""" - tag = LOCAL_IMAGE_PREFIX + ":".join(image.rsplit("/", 1)) - err = StringIO.StringIO() - env = copy.deepcopy(os.environ) - env["DOCKER"] = FLAGS.docker - ret = execute([image], stderr=err, env=env) - if ret != 0: - sys.stderr.write("Error loading image %s (return code: %s):\n" % - (image, ret)) - sys.stderr.write(err.getvalue()) - return None - return tag - - -def load_images(images): - """Load a series of docker images using the docker_build's runner.""" - print "### Image loading ###" - return [load_image(image) for image in images] - - -def cleanup_images(tags): - """Remove docker tags and images previously loaded.""" - print "### Image cleanup ###" - for tag in tags: - if isinstance(tag, basestring): - execute([FLAGS.docker, "rmi", tag]) - - -def run_image(tag): - """Run a docker image, in background.""" - print "Running " + tag - out = StringIO.StringIO() - err = StringIO.StringIO() - process = execute([FLAGS.docker, "run", "--rm", tag], out, err) - if process.wait() != 0: - sys.stderr.write("Error running docker run on %s:\n" % tag) - sys.stderr.write(err.getvalue()) - return None - else: - return out.getvalue().strip() - - -def run_images(tags): - """Run a list of docker images, in background.""" - print "### Running images ###" - return [run_image(tag) for tag in tags] - - -def cleanup_containers(containers): - """Kill containers.""" - print "### Containers cleanup ###" - for c in containers: - if isinstance(c, basestring): - execute([FLAGS.docker, "kill", c]) - - -def main(unused_argv): - tags = load_images([FLAGS.main] + FLAGS.image) - if None in tags: - cleanup_images(tags) - return -1 - try: - containers = run_images(tags[1:]) - ret = -1 - if None not in containers: - print "### Running main container ###" - ret = execute([ - FLAGS.docker, - "run", - "--rm", - "--attach=STDOUT", - "--attach=STDERR", tags[0] - ] + ([] if FLAGS.cmd is None else shlex.split(FLAGS.cmd))) - finally: - cleanup_containers(containers) - cleanup_images(tags) - return ret - - -if __name__ == "__main__": - sys.exit(main(FLAGS(sys.argv))) diff --git a/src/test/docker/flavours.bzl b/src/test/docker/flavours.bzl deleted file mode 100644 index 9ba4ddff13..0000000000 --- a/src/test/docker/flavours.bzl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Create base images for docker tests.""" -# TODO(dmarting): Right now we use a custom docker_pull that can build -# docker images, which is not reproducible and as a high cost, ideally -# we would switch to the docker_pull from bazelbuild/rules_docker but -# we do not have an easy mean to create and maintain the images we need -# for those tests. -load("//src/test/docker:docker_pull.bzl", "docker_pull") - -FLAVOURS = [ - "centos6.9", - "debian-stretch", -] - -def pull_images_for_docker_tests(): - for flavour in FLAVOURS: - docker_pull( - name = "docker-" + flavour, - tag = "bazel_tools_cpp_test:" + flavour, - dockerfile = "//src/test/docker:Dockerfile." + flavour, - optional = True, - ) diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh index e095d4dfae..6a7476cf9d 100755 --- a/src/test/shell/testenv.sh +++ b/src/test/shell/testenv.sh @@ -161,16 +161,7 @@ fi function use_bazel_workspace_file() { - mkdir -p src/test/{shell/bazel,docker} - cat >src/test/docker/docker_repository.bzl <<EOF -def docker_repository(): - pass -EOF - cat >src/test/docker/flavours.bzl <<EOF -def pull_images_for_docker_tests(): - pass -EOF - touch src/test/docker/BUILD + mkdir -p src/test/shell/bazel cat >src/test/shell/bazel/list_source_repository.bzl <<EOF def list_source_repository(name): pass |