diff options
| -rw-r--r-- | BUILD | 6 | ||||
| -rw-r--r-- | WORKSPACE | 4 | ||||
| -rw-r--r-- | tools/cpp/BUILD | 2 | ||||
| -rw-r--r-- | tools/cpp/test/BUILD | 120 | ||||
| -rw-r--r-- | tools/cpp/test/Dockerfile.fedora23 | 8 | ||||
| -rw-r--r-- | tools/cpp/test/Dockerfile.ubuntu-15.04 | 8 | ||||
| -rw-r--r-- | tools/cpp/test/Dockerfile.ubuntu-15.10 | 8 | ||||
| -rw-r--r-- | tools/cpp/test/docker_repository.bzl | 57 | ||||
| -rw-r--r-- | tools/cpp/test/docker_test.py | 158 |
9 files changed, 370 insertions, 1 deletions
@@ -11,6 +11,12 @@ filegroup( ) filegroup( + name = "workspace-file", + srcs = [":WORKSPACE"], + visibility = ["//tools/cpp/test:__pkg__"], +) + +filegroup( name = "srcs", srcs = glob( ["**"], @@ -23,6 +23,10 @@ bind( actual = "//:dummy", ) +# For tools/cpp/test/... +load("//tools/cpp/test:docker_repository.bzl", "docker_repository") +docker_repository() + # In order to run the Android integration tests, run # scripts/workspace_user.sh and uncomment the next two lines. # load("/WORKSPACE.user", "android_repositories") diff --git a/tools/cpp/BUILD b/tools/cpp/BUILD index 6e243e8cdc..cd671e1f12 100644 --- a/tools/cpp/BUILD +++ b/tools/cpp/BUILD @@ -127,5 +127,5 @@ cc_toolchain( filegroup( name = "srcs", - srcs = glob(["**"]), + srcs = glob(["**"]) + ["//tools/cpp/test:srcs"], ) diff --git a/tools/cpp/test/BUILD b/tools/cpp/test/BUILD new file mode 100644 index 0000000000..e1030223ad --- /dev/null +++ b/tools/cpp/test/BUILD @@ -0,0 +1,120 @@ +load("//tools/build_defs/docker:docker.bzl", "docker_build") +load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar") + +# This is totally non hermetic, we should really replace that by a +# docker_pull rule. +FLAVOURS = [f[f.find(".") + 1:] for f in glob(["Dockerfile.*"])] + +[ + # This is totally non hermetic. + genrule( + name = "docker-" + flavour, + srcs = ["Dockerfile." + flavour], + outs = ["docker-%s.tar" % flavour], + cmd = "\n".join([ + "DIR=\"$$(dirname $<)\"", + "IMG=\"$$(basename $<)\"", + "DOCKER=\"$${PWD}/$(location @docker//:docker)\"", + "(", + " cd $$DIR", + " $$DOCKER build -f $$IMG -t bazel_tools_cpp_test:%s ." % flavour, + ")", + "$$DOCKER save bazel_tools_cpp_test:%s > $@" % flavour, + ]), + tags = ["local"], + # Docker needs to knows how to contact the daemon from the environment. + # @docker//:docker point to a docker wrapper that copy the environment + # of the user. + tools = ["@docker//:docker"], + ) + for flavour in FLAVOURS +] + +# Just to avoid re-reading docker images all the time +[ + docker_build( + name = "base-" + flavour, + base = "docker-" + flavour, + ) + for flavour in FLAVOURS +] + +genrule( + name = "gen_workspace", + srcs = ["//:workspace-file"], + outs = ["WORKSPACE"], + cmd = """ + cat <<EOF >$@ +load("//tools/cpp:cc_configure.bzl", "cc_configure") +cc_configure() +EOF + cat $(location //:workspace-file) >>$@ +""", +) + +pkg_tar( + name = "cc_configure_ws", + files = [":WORKSPACE"], + package_dir = "/opt/workspace", + strip_prefix = ".", +) + +pkg_tar( + name = "bazel_cc_configure", + package_dir = "/opt/workspace", + strip_prefix = "/", + deps = [ + # Order matters. + ":cc_configure_ws", + "//:bazel-srcs", + ], +) + +[ + [docker_build( + name = "bazel_cc_configure-%s-%s" % (flavour, mode), + base = ":base-" + flavour, + entrypoint = "/opt/workspace/compile.sh", + env = { + "EXTRA_BAZEL_ARGS": "--spawn_strategy=standalone --genrule_strategy=standalone -c %s" % mode, + }, + tars = [":bazel_cc_configure"], + workdir = "/opt/workspace", + ) for mode in [ + "dbg", + "opt", + "fastbuild", + ]] + for flavour in FLAVOURS +] + +[ + [py_test( + name = "test_cc_configure-%s-%s" % (flavour, mode), + size = "large", + srcs = ["docker_test.py"], + args = [ + "--main='$(location :bazel_cc_configure-%s-%s)'" % (flavour, mode), + "--docker='$(location @docker//:docker)'", + ], + data = [ + ":bazel_cc_configure-%s-%s" % (flavour, mode), + "@docker//:docker", + ], + local = 1, + main = "docker_test.py", + tags = ["local"], + deps = ["//third_party/py/gflags"], + ) for mode in [ + "dbg", + "opt", + "fastbuild", + ]] + for flavour in FLAVOURS +] + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//tools/cpp:__pkg__"], +) diff --git a/tools/cpp/test/Dockerfile.fedora23 b/tools/cpp/test/Dockerfile.fedora23 new file mode 100644 index 0000000000..396e1c66f2 --- /dev/null +++ b/tools/cpp/test/Dockerfile.fedora23 @@ -0,0 +1,8 @@ +FROM fedora:23 + +RUN dnf -y update && dnf clean all +RUN dnf -y install \ + which findutils binutils gcc tar gzip \ + zip unzip java java-devel git clang zlib-devel \ + && dnf clean all +ENV JAVA_HOME /usr/lib/jvm/java-openjdk diff --git a/tools/cpp/test/Dockerfile.ubuntu-15.04 b/tools/cpp/test/Dockerfile.ubuntu-15.04 new file mode 100644 index 0000000000..57aab547d6 --- /dev/null +++ b/tools/cpp/test/Dockerfile.ubuntu-15.04 @@ -0,0 +1,8 @@ +FROM ubuntu:15.04 +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl ca-certificates \ + git pkg-config zip unzip \ + g++ gcc openjdk-8-jdk \ + zlib1g-dev libarchive-dev \ + ca-certificates-java && \ + apt-get clean diff --git a/tools/cpp/test/Dockerfile.ubuntu-15.10 b/tools/cpp/test/Dockerfile.ubuntu-15.10 new file mode 100644 index 0000000000..8c30fa7e00 --- /dev/null +++ b/tools/cpp/test/Dockerfile.ubuntu-15.10 @@ -0,0 +1,8 @@ +FROM ubuntu:15.10 +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl ca-certificates \ + git pkg-config zip unzip \ + g++ gcc openjdk-8-jdk \ + zlib1g-dev libarchive-dev \ + ca-certificates-java && \ + apt-get clean diff --git a/tools/cpp/test/docker_repository.bzl b/tools/cpp/test/docker_repository.bzl new file mode 100644 index 0000000000..4f96170176 --- /dev/null +++ b/tools/cpp/test/docker_repository.bzl @@ -0,0 +1,57 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule for importing the docker binary for tests (experimental).""" + +def _impl(ctx): + docker = ctx.which("docker") + if docker == None: + # We cannot find docker, we won't be able to run tests depending + # on it, silently ignoring. + ctx.file("BUILD", + "\n".join([ + "filegroup(", + " name = 'docker',", + " visibility = ['//visibility:public'],", + ")" + ])) + else: + exports = [] + for k in ctx.os.environ: + # DOCKER* environment variable are used by the docker client + # to know how to talk to the docker daemon. + if k.startswith("DOCKER"): + exports.append("export %s='%s'" % (k, ctx.os.environ[k])) + ctx.symlink(docker, "docker-bin") + ctx.file("docker.sh", "\n".join([ + "#!/bin/bash", + "\n".join(exports), +"""BIN="$0" +while [ -L "${BIN}" ]; do + BIN="$(readlink "${BIN}")" +done +exec "${BIN%%.sh}-bin" "$@" +"""])) + ctx.file("BUILD", "\n".join([ + "sh_binary(", + " name = 'docker',", + " srcs = ['docker.sh'],", + " data = [':docker-bin'],", + " visibility = ['//visibility:public'],", + ")"])) + +docker_repository_ = repository_rule(_impl) + +def docker_repository(): + """Declare a @docker repository that provide a docker binary.""" + docker_repository_(name = "docker") diff --git a/tools/cpp/test/docker_test.py b/tools/cpp/test/docker_test.py new file mode 100644 index 0000000000..8d804cdeb4 --- /dev/null +++ b/tools/cpp/test/docker_test.py @@ -0,0 +1,158 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A simple test runner for docker (experimental).""" + +import copy +import os +import os.path +import shlex +import StringIO +import subprocess +import sys +import threading + +from third_party.py import gflags + +gflags.DEFINE_multistring( + "image", [], + "The list of additional docker image to load (path to a docker_build " + "target), optional.") + +gflags.DEFINE_string( + "main", None, + "The main image to run (path to a docker_build target), mandatory.") +gflags.MarkFlagAsRequired("main") + +gflags.DEFINE_string( + "cmd", None, + "A command to provide to the docker image, optional (default: use the " + "entrypoint).") + +gflags.DEFINE_string("docker", "docker", "Path to the docker client binary.") + +gflags.DEFINE_boolean("verbose", True, "Be verbose.") + +FLAGS = gflags.FLAGS + +LOCAL_IMAGE_PREFIX = "bazel/docker_test:" + + +def _copy_stream(in_stream, out_stream): + for c in iter(lambda: in_stream.read(1), ""): + out_stream.write(c) + out_stream.flush() + + +def execute(command, stdout=sys.stdout, stderr=sys.stderr, env=None): + """Execute a command while redirecting its output streams.""" + if FLAGS.verbose: + print "Executing '%s'" % " ".join(command) + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + t1 = threading.Thread(target=_copy_stream, args=[p.stdout, stdout]) + t2 = threading.Thread(target=_copy_stream, args=[p.stderr, stderr]) + t1.daemon = True + t2.daemon = True + t1.start() + t2.start() + p.wait() + t1.join() + t2.join() + return p.returncode + + +def load_image(image): + """Load a docker image using the runner provided by docker_build.""" + tag = LOCAL_IMAGE_PREFIX + image.replace("/", "_") + err = StringIO.StringIO() + env = copy.deepcopy(os.environ) + env["DOCKER"] = FLAGS.docker + ret = execute([image, tag], stderr=err, env=env) + if ret != 0: + sys.stderr.write("Error loading image %s (return code: %s):\n" % + (image, ret)) + sys.stderr.write(err.getvalue()) + return None + return tag + + +def load_images(images): + """Load a series of docker images using the docker_build's runner.""" + print "### Image loading ###" + return [load_image(image) for image in images] + + +def cleanup_images(tags): + """Remove docker tags and images previously loaded.""" + print "### Image cleanup ###" + for tag in tags: + if isinstance(tag, basestring): + execute([FLAGS.docker, "rmi", tag]) + + +def run_image(tag): + """Run a docker image, in background.""" + print "Running " + tag + out = StringIO.StringIO() + err = StringIO.StringIO() + process = execute([FLAGS.docker, "run", "--rm", tag], out, err) + if process.wait() != 0: + sys.stderr.write("Error running docker run on %s:\n" % tag) + sys.stderr.write(err.getvalue()) + return None + else: + return out.getvalue().strip() + + +def run_images(tags): + """Run a list of docker images, in background.""" + print "### Running images ###" + return [run_image(tag) for tag in tags] + + +def cleanup_containers(containers): + """Kill containers.""" + print "### Containers cleanup ###" + for c in containers: + if isinstance(c, basestring): + execute([FLAGS.docker, "kill", c]) + + +def main(unused_argv): + tags = load_images([FLAGS.main] + FLAGS.image) + if None in tags: + cleanup_images(tags) + return -1 + try: + containers = run_images(tags[1:]) + ret = -1 + if None not in containers: + print "### Running main container ###" + ret = execute([ + FLAGS.docker, + "run", + "--rm", + "--attach=STDOUT", + "--attach=STDERR", tags[0] + ] + ([] if FLAGS.cmd is None else shlex.split(FLAGS.cmd))) + finally: + cleanup_containers(containers) + cleanup_images(tags) + return ret + + +if __name__ == "__main__": + sys.exit(main(FLAGS(sys.argv))) |