diff options
29 files changed, 2241 insertions, 0 deletions
diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000000..7cee610e35 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,4 @@ +buildgen: contains the template renderer for our build system. +dockerfile: contains all of the docker files to test gRPC. +gce_setup: contains boilerplate for running the docker files under GCE. +run_tests: contains python scripts to properly run the tests in parallel. diff --git a/tools/buildgen/bunch.py b/tools/buildgen/bunch.py new file mode 100755 index 0000000000..e859d53388 --- /dev/null +++ b/tools/buildgen/bunch.py @@ -0,0 +1,37 @@ +"""Allows dot-accessible dictionaries.""" + + +class Bunch(dict): + + def __init__(self, d): + dict.__init__(self, d) + self.__dict__.update(d) + + +# Converts any kind of variable to a Bunch +def to_bunch(var): + if isinstance(var, list): + return [to_bunch(i) for i in var] + if isinstance(var, dict): + ret = {} + for k, v in var.items(): + if isinstance(v, (list, dict)): + v = to_bunch(v) + ret[k] = v + return Bunch(ret) + else: + return var + + +# Merges JSON 'add' into JSON 'dst' +def merge_json(dst, add): + if isinstance(dst, dict) and isinstance(add, dict): + for k, v in add.items(): + if k in dst: + merge_json(dst[k], v) + else: + dst[k] = v + elif isinstance(dst, list) and isinstance(add, list): + dst.extend(add) + else: + raise Exception('Tried to merge incompatible objects %r, %r' % (dst, add)) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh new file mode 100755 index 0000000000..f4e367d620 --- /dev/null +++ b/tools/buildgen/generate_projects.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -ex + +if [ "x$TEST" == "x" ] ; then + TEST=false +fi + + +cd `dirname $0`/.. +mako_renderer=tools/buildgen/mako_renderer.py +gen_build_json=test/core/end2end/gen_build_json.py + +end2end_test_build=`mktemp` +$gen_build_json > $end2end_test_build + +global_plugins=`find ./tools/buildgen/plugins -name '*.py' | + sort | grep -v __init__ | + while read p ; do echo -n "-p $p " ; done` + +for dir in . ; do + local_plugins=`find $dir/templates -name '*.py' | + sort | grep -v __init__ | + while read p ; do echo -n "-p $p " ; done` + + plugins="$global_plugins $local_plugins" + + find -L $dir/templates -type f -and -name *.template | while read file ; do + out=${dir}/${file#$dir/templates/} # strip templates dir prefix + out=${out%.*} # strip template extension + json_files="build.json $end2end_test_build" + data=`for i in $json_files; do echo -n "-d $i "; done` + if [ $TEST == true ] ; then + actual_out=$out + out=`mktemp` + else + g4 open $out || true + fi + $mako_renderer $plugins $data -o $out $file + if [ $TEST == true ] ; then + diff -q $out $actual_out + rm $out + fi + done +done + +rm $end2end_test_build + diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py new file mode 100755 index 0000000000..29c7cf0307 --- /dev/null +++ b/tools/buildgen/mako_renderer.py @@ -0,0 +1,105 @@ +#!/usr/bin/python + +"""Simple Mako renderer. + +Just a wrapper around the mako rendering library. + +""" + +import getopt +import imp +import os +import sys + + +from mako.lookup import TemplateLookup +from mako.runtime import Context +from mako.template import Template +import simplejson +import bunch + + +# Imports a plugin +def import_plugin(name): + _, base_ex = os.path.split(name) + base, _ = os.path.splitext(base_ex) + + with open(name, 'r') as plugin_file: + plugin_code = plugin_file.read() + plugin_module = imp.new_module(base) + exec plugin_code in plugin_module.__dict__ + return plugin_module + + +def out(msg): + print >> sys.stderr, msg + + +def showhelp(): + out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template') + + +def main(argv): + got_input = False + module_directory = None + dictionary = {} + json_dict = {} + got_output = False + output_file = sys.stdout + plugins = [] + + try: + opts, args = getopt.getopt(argv, 'hm:d:o:p:') + except getopt.GetoptError: + out('Unknown option') + showhelp() + sys.exit(2) + + for opt, arg in opts: + if opt == '-h': + out('Displaying showhelp') + showhelp() + sys.exit() + elif opt == '-o': + if got_output: + out('Got more than one output') + showhelp() + sys.exit(3) + got_output = True + output_file = open(arg, 'w') + elif opt == '-m': + if module_directory is not None: + out('Got more than one cache directory') + showhelp() + sys.exit(4) + module_directory = arg + elif opt == '-d': + dict_file = open(arg, 'r') + bunch.merge_json(json_dict, simplejson.loads(dict_file.read())) + dict_file.close() + elif opt == '-p': + plugins.append(import_plugin(arg)) + + for plugin in plugins: + plugin.mako_plugin(json_dict) + + for k, v in json_dict.items(): + dictionary[k] = bunch.to_bunch(v) + + ctx = Context(output_file, **dictionary) + + for arg in args: + got_input = True + template = Template(filename=arg, + module_directory=module_directory, + lookup=TemplateLookup(directories=['.'])) + template.render_context(ctx) + + if not got_input: + out('Got nothing to do') + showhelp() + + output_file.close() + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tools/buildgen/plugins/expand_filegroups.py b/tools/buildgen/plugins/expand_filegroups.py new file mode 100755 index 0000000000..108debefd5 --- /dev/null +++ b/tools/buildgen/plugins/expand_filegroups.py @@ -0,0 +1,45 @@ +"""Buildgen expand filegroups plugin. + +This takes the list of libs from our json dictionary, +and expands any and all filegroup. + +""" + + +def excluded(filename, exclude_res): + for r in exclude_res: + if r.search(filename): + return True + return False + + +def mako_plugin(dictionary): + """The exported plugin code for expand_filegroups. + + The list of libs in the build.json file can contain "filegroups" tags. + These refer to the filegroups in the root object. We will expand and + merge filegroups on the src, headers and public_headers properties. + + """ + libs = dictionary.get('libs') + filegroups_list = dictionary.get('filegroups') + filegroups = {} + + for fg in filegroups_list: + filegroups[fg['name']] = fg + + for lib in libs: + for fg_name in lib.get('filegroups', []): + fg = filegroups[fg_name] + + src = lib.get('src', []) + src.extend(fg.get('src', [])) + lib['src'] = src + + headers = lib.get('headers', []) + headers.extend(fg.get('headers', [])) + lib['headers'] = headers + + public_headers = lib.get('public_headers', []) + public_headers.extend(fg.get('public_headers', [])) + lib['public_headers'] = public_headers diff --git a/tools/buildgen/plugins/list_protos.py b/tools/buildgen/plugins/list_protos.py new file mode 100755 index 0000000000..c5a09dd4d0 --- /dev/null +++ b/tools/buildgen/plugins/list_protos.py @@ -0,0 +1,41 @@ +"""Buildgen .proto files list plugin. + +This parses the list of targets from the json build file, and creates +a list called "protos" that contains all of the proto file names. + +""" + + +import re + + +def mako_plugin(dictionary): + """The exported plugin code for list_protos. + + Some projects generators may want to get the full list of unique .proto files + that are being included in a project. This code extracts all files referenced + in any library or target that ends in .proto, and builds and exports that as + a list called "protos". + + """ + + libs = dictionary.get('libs', []) + targets = dictionary.get('targets', []) + + proto_re = re.compile('(.*)\\.proto') + + protos = set() + for lib in libs: + for src in lib.get('src', []): + m = proto_re.match(src) + if m: + protos.add(m.group(1)) + for tgt in targets: + for src in tgt.get('src', []): + m = proto_re.match(src) + if m: + protos.add(m.group(1)) + + protos = sorted(protos) + + dictionary['protos'] = protos diff --git a/tools/dockerfile/grpc_base/Dockerfile b/tools/dockerfile/grpc_base/Dockerfile new file mode 100644 index 0000000000..76e585a7d0 --- /dev/null +++ b/tools/dockerfile/grpc_base/Dockerfile @@ -0,0 +1,57 @@ +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Install Git. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libevent-dev \ + libtool \ + make \ + strace \ + python-dev \ + python-setuptools \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +# Install useful useful python modules +RUN easy_install -U pip +RUN pip install -U crcmod # makes downloads from cloud storage faster + +# Install GCloud +RUN wget https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.zip \ + && unzip google-cloud-sdk.zip && rm google-cloud-sdk.zip +ENV CLOUD_SDK /google-cloud-sdk +RUN $CLOUD_SDK/install.sh --usage-reporting=true --path-update=true --bash-completion=true --rc-path=/.bashrc --disable-installation-options +ENV PATH $CLOUD_SDK/bin:$PATH + +# Install gcompute-tools to allow access to private git-on-borg repos +RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools + +# Start the daemon that allows access to private git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +# Install the grpc-tools scripts dir from git +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-tools /var/local/git/grpc-tools + +# Install the grpc-protobuf dir that has the protoc patch +RUN git clone https://team.googlesource.com/one-platform-grpc-team/protobuf /var/local/git/protobuf + +# Install the patched version of protoc +RUN cd /var/local/git/protobuf && \ + ./autogen.sh && \ + ./configure --prefix=/usr && \ + make && make check && make install && make clean + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/grpc_base/README.md b/tools/dockerfile/grpc_base/README.md new file mode 100644 index 0000000000..4745141fc4 --- /dev/null +++ b/tools/dockerfile/grpc_base/README.md @@ -0,0 +1,11 @@ +Base GRPC Dockerfile +==================== + +Dockerfile for creating the base gRPC development Docker instance. +For now, this assumes that the development will be done on GCE instances, with source code on Git-on-Borg. + +As of 2014/09/29, it includes +- git +- some useful tools like curl, emacs, strace, telnet etc +- downloads the gerrit-compute-tools and installs the script that allows access to gerrit when on git-on-borg +- a patched version of protoc, to allow protos with stream tags to work diff --git a/tools/dockerfile/grpc_cxx/Dockerfile b/tools/dockerfile/grpc_cxx/Dockerfile new file mode 100644 index 0000000000..cf38e976b1 --- /dev/null +++ b/tools/dockerfile/grpc_cxx/Dockerfile @@ -0,0 +1,15 @@ +# Dockerfile for gRPC C++ +FROM grpc/base + +# Start the daemon that allows access to the protected git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc +RUN cd /var/local/git/grpc \ + && git pull --recurse-submodules \ + && git submodule update --init --recursive + +RUN make install -C /var/local/git/grpc + +# Define the default command. +CMD ["bash"]
\ No newline at end of file diff --git a/tools/dockerfile/grpc_java/Dockerfile b/tools/dockerfile/grpc_java/Dockerfile new file mode 100644 index 0000000000..f7156c7f38 --- /dev/null +++ b/tools/dockerfile/grpc_java/Dockerfile @@ -0,0 +1,16 @@ +# Dockerfile for the gRPC Java dev image +FROM grpc/java_base + +# Start the daemon that allows access to private git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +RUN cd /var/local/git/grpc-java/lib/okhttp && \ + mvn -pl okhttp -am install +RUN cd /var/local/git/grpc-java/lib/netty && \ + mvn -pl codec-http2 -am -DskipTests install +RUN cd /var/local/git/grpc-java && \ + protoc --version>ver.txt && \ + mvn install + +# Specify the default command such that the interop server runs on its known testing port +CMD ["/var/local/git/grpc-java/run-test-server.sh", "--transport=NETTY_TLS", "--grpc_version=2", "--port=8030"] diff --git a/tools/dockerfile/grpc_java/README.md b/tools/dockerfile/grpc_java/README.md new file mode 100644 index 0000000000..2da2393bef --- /dev/null +++ b/tools/dockerfile/grpc_java/README.md @@ -0,0 +1,9 @@ +GRPC Java Dockerfile +==================== + +Dockerfile for creating the Java development image + +As of 2014/12 this + - is based on the gRPC Java base + - pulls from gRPC Java on git-on-borg + - installs it and runs the tests
\ No newline at end of file diff --git a/tools/dockerfile/grpc_java_base/Dockerfile b/tools/dockerfile/grpc_java_base/Dockerfile new file mode 100644 index 0000000000..fe8edeb698 --- /dev/null +++ b/tools/dockerfile/grpc_java_base/Dockerfile @@ -0,0 +1,25 @@ +# Base Dockerfile for the gRPC Java dev image +FROM grpc/base + +RUN apt-get update && apt-get -y install java7-jdk + +# Install maven +RUN wget http://mirror.olnevhost.net/pub/apache/maven/binaries/apache-maven-3.2.1-bin.tar.gz && \ + tar xvf apache-maven-3.2.1-bin.tar.gz -C /var/local + +ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64 +ENV M2_HOME /var/local/apache-maven-3.2.1 +ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin +ENV LD_LIBRARY_PATH /usr/local/lib + +# Start the daemon that allows access to the protected git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +RUN git clone --recursive https://team.googlesource.com/one-platform-grpc-team/grpc-java /var/local/git/grpc-java + +RUN cd /var/local/git/grpc-java/lib/okhttp && \ + mvn -pl okhttp -am validate +RUN cd /var/local/git/grpc-java/lib/netty && \ + mvn -pl codec-http2 -am validate +RUN cd /var/local/git/grpc-java && \ + mvn validate
\ No newline at end of file diff --git a/tools/dockerfile/grpc_java_base/README.md b/tools/dockerfile/grpc_java_base/README.md new file mode 100644 index 0000000000..446287fc7c --- /dev/null +++ b/tools/dockerfile/grpc_java_base/README.md @@ -0,0 +1,9 @@ +GRPC Java Base Dockerfile +========================= + +Dockerfile for creating the Java gRPC development Docker instance. + +As of 2014/12 this + - installs tools and dependencies needed to build gRPC Java + - does not install gRPC Java itself; a separate Dockerfile that depends on + this one will do that.
\ No newline at end of file diff --git a/tools/dockerfile/grpc_php/Dockerfile b/tools/dockerfile/grpc_php/Dockerfile new file mode 100644 index 0000000000..0e50af70a2 --- /dev/null +++ b/tools/dockerfile/grpc_php/Dockerfile @@ -0,0 +1,18 @@ +# Dockerfile for gRPC PHP +FROM grpc/php_base + +# Start the daemon that allows access to the protected git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +RUN cd /var/local/git/grpc \ + && git pull --recurse-submodules \ + && git submodule update --init --recursive + +RUN make install_c -j12 -C /var/local/git/grpc + +RUN cd /var/local/git/grpc/src/php/ext/grpc && git pull && phpize + +# Build the grpc PHP extension +RUN cd /var/local/git/grpc/src/php/ext/grpc \ + && ./configure \ + && make
\ No newline at end of file diff --git a/tools/dockerfile/grpc_php/README.md b/tools/dockerfile/grpc_php/README.md new file mode 100644 index 0000000000..a37389ff60 --- /dev/null +++ b/tools/dockerfile/grpc_php/README.md @@ -0,0 +1,10 @@ +GRPC PHP Dockerfile +=================== + +Dockerfile for creating the PHP development instances + +As of 2014/10 this +- is based on the GRPC PHP base +- adds a pull of the HEAD GRPC PHP source from git-on-borg +- it builds it +- runs the tests, i.e, the image won't be created if the tests don't pass diff --git a/tools/dockerfile/grpc_php_base/Dockerfile b/tools/dockerfile/grpc_php_base/Dockerfile new file mode 100644 index 0000000000..8ec90f48b8 --- /dev/null +++ b/tools/dockerfile/grpc_php_base/Dockerfile @@ -0,0 +1,85 @@ +# Base Dockerfile for gRPC PHP. +# +# Includes PHP installation dependencies, things that are unlikely to vary. +FROM grpc/base + +# Install RVM dependencies and other packages +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + bison \ + curl \ + g++ \ + gawk \ + gcc \ + groff \ + libc6-dev \ + libffi-dev \ + libgdbm-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libyaml-dev \ + make \ + patch \ + procps \ +# TODO(mlumish): Uncomment these lines when building against them works +# php5-common \ +# php5-cli \ +# php5-dev \ +# php-pear \ + pkg-config \ + procps \ + sqlite3 \ + zlib1g-dev + +# Install the version of PHP gRPC is tested against +ENV DEBIAN_FRONTEND noniteractive +RUN apt-get update && apt-get install -y libxml2 libxml2-dev # used by PHP +RUN cd /var/local \ + && curl -o php-5.5.17.tar.gz http://php.net/distributions/php-5.5.17.tar.gz \ + && tar -xf php-5.5.17.tar.gz \ + && cd php-5.5.17 \ + && ./configure --with-zlib=/usr --with-libxml-dir=ext/libxml \ + && make && make install + +# Start the daemon that allows access to the protected git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +# Download the patched PHP protobuf so that PHP gRPC clients can be generated +# from proto3 schemas. +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-php-protobuf-php /var/local/git/protobuf-php + +# Install ruby (via RVM) as ruby tools are dependencies for building Protobuf +# PHP extensions. +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 # Needed for RVM +RUN \curl -sSL https://get.rvm.io | bash -s stable --ruby +ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# ronn: a ruby tool used to convert markdown to man pages, used during the +# install of Protobuf extensions +# +# rake: a ruby version of make used to build the PHP Protobuf extension +RUN rvm all do gem install ronn rake + +# Install the patched PHP protobuf so that PHP gRPC clients can be generated +# from proto3 schemas. +RUN cd /var/local/git/protobuf-php \ + && rvm all do rake pear:package version=1.0 \ + && pear install Protobuf-1.0.tgz + +# Install PHPUnit, used to run the PHP unit tests +RUN wget https://phar.phpunit.de/phpunit.phar \ + && chmod +x phpunit.phar \ + && mv phpunit.phar /usr/local/bin/phpunit + +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc +RUN cd /var/local/git/grpc \ + && git submodule update --init --recursive + +RUN make static_c shared_c -j12 -C /var/local/git/grpc + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/grpc_php_base/README.md b/tools/dockerfile/grpc_php_base/README.md new file mode 100644 index 0000000000..d93778e5ec --- /dev/null +++ b/tools/dockerfile/grpc_php_base/README.md @@ -0,0 +1,9 @@ +GRPC PHP Base Dockerfile +======================== + +Dockerfile for creating the PHP gRPC development Docker instance. + +As of 2014/10 this +- it installs tools and dependencies needed to build gRPC PHP +- it does not install gRPC PHP itself; a separate Dockerfile that depends on + this one will do that diff --git a/tools/dockerfile/grpc_ruby/Dockerfile b/tools/dockerfile/grpc_ruby/Dockerfile new file mode 100644 index 0000000000..209257144b --- /dev/null +++ b/tools/dockerfile/grpc_ruby/Dockerfile @@ -0,0 +1,21 @@ +# Dockerfile for gRPC Ruby +FROM grpc/ruby_base + +RUN cd /var/local/git/grpc \ + && git pull --recurse-submodules \ + && git submodule update --init --recursive + +RUN make install_c -C /var/local/git/grpc + +# Install the grpc gem locally with its dependencies and build the extension. +RUN /bin/bash -l -c 'cd /var/local/git/beefcake && bundle && gem build beefcake.gemspec && gem install beefcake' +RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake compile:grpc && gem build grpc.gemspec && gem install grpc' + +# TODO add a command to run the unittest tests when the bug below is fixed +# - the tests fail due to an error in the C threading library: +# they fail with 'ruby: __pthread_mutex_cond_lock_adjust for unknown reasons' at the end of a testcase +# - however, the interop server and client run OK, so this bug can be investigated +# RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake' + +# Specify the default command such that the interop server runs on its known testing port +CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --port 8060"] diff --git a/tools/dockerfile/grpc_ruby/README.md b/tools/dockerfile/grpc_ruby/README.md new file mode 100644 index 0000000000..51fb2f5970 --- /dev/null +++ b/tools/dockerfile/grpc_ruby/README.md @@ -0,0 +1,10 @@ +GRPC Ruby Dockerfile +==================== + +Dockerfile for creating the Ruby development instances + +As of 2014/10 this +- is based on the GRPC Ruby base +- adds a pull of the HEAD gRPC Ruby source from git-on-borg +- it builds it +- runs the tests, i.e, the image won't be created if the tests don't pass diff --git a/tools/dockerfile/grpc_ruby_base/Dockerfile b/tools/dockerfile/grpc_ruby_base/Dockerfile new file mode 100644 index 0000000000..ad14e43ec7 --- /dev/null +++ b/tools/dockerfile/grpc_ruby_base/Dockerfile @@ -0,0 +1,59 @@ +# Base Dockerfile for gRPC Ruby. +# +# Includes Ruby installation dependencies, things that are unlikely to vary. +FROM grpc/base + +# Allows 'source' to work +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# Install RVM dependencies +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + bison \ + curl \ + g++ \ + gawk \ + gcc \ + libc6-dev \ + libffi-dev \ + libgdbm-dev \ + libncurses5-dev \ + libreadline6-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libyaml-dev \ + make \ + patch \ + pkg-config \ + procps \ + sqlite3 \ + zlib1g-dev + + +# Start the daemon that allows access to the protected git-on-borg repos +RUN /var/local/git/gcompute-tools/git-cookie-authdaemon + +# Download the patched Ruby protobuf (beefcake) so that Ruby gRPC clients can +# be generated from proto3 schemas. +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-ruby-beefcake \ + /var/local/git/beefcake + +# Install RVM, use this to install ruby +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 # Needed for RVM +RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable" + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'source /home/grpc_ruby/.rvm/scripts/rvm' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc +RUN cd /var/local/git/grpc \ + && git submodule update --init --recursive + +RUN make static_c shared_c -C /var/local/git/grpc
\ No newline at end of file diff --git a/tools/dockerfile/grpc_ruby_base/README.md b/tools/dockerfile/grpc_ruby_base/README.md new file mode 100644 index 0000000000..acf1e50dbe --- /dev/null +++ b/tools/dockerfile/grpc_ruby_base/README.md @@ -0,0 +1,9 @@ +GRPC RUBY Base Dockerfile +======================== + +Dockerfile for creating the Ruby gRPC development Docker instance. + +As of 2014/10 this +- it installs tools and dependencies needed to build gRPC Ruby +- it does not install gRPC Ruby itself; a separate Dockerfile that depends on + this one will do that diff --git a/tools/gce_setup/README.md b/tools/gce_setup/README.md new file mode 100644 index 0000000000..253e94daa9 --- /dev/null +++ b/tools/gce_setup/README.md @@ -0,0 +1,48 @@ +GCE images for GRPC +=================== + +This directory contains a number of shell files used for setting up GCE images +and instances for developing and testing gRPC. + + + +Goal +---- + +- provides a script to create a GCE image that has everything needed to try +out gRPC on GCE. +- provide another script that creates a new GCE instance from the latest image + +- additional scripts may be added in the future + + +Usage +------ + +# Minimal usage (see the scripts themselves for options) + +$ create_grpc_dev_image.sh # creates a grpc GCE image +$ ... +$ new_grpc_dev_instance.sh # creates an instance using the latest grpc GCE image + + +Requirements +------------ + +Install [Google Cloud SDK](https://developers.google.com/cloud/sdk/) + +Contents +-------- + +Library scripts that contain bash functions used in the other scripts: +- shared_setup_funcs.sh # funcs used in create_grpc_dev_image and new_grpc_dev_instance +- gcutil_extras.sh # wrappers for common tasks that us gcutil +- build_grpc_dist.sh # funcs building the GRPC library and tests into a debian dist + +GCE [startup scripts](https://developers.google.com/compute/docs/howtos/startupscript) +- *_on_startup.sh + +Main scripts (as of 2014/09/04) +- create_grpc_dev_instance.sh +- new_grpc_dev_instance.sh + diff --git a/tools/gce_setup/compute_extras.sh b/tools/gce_setup/compute_extras.sh new file mode 100755 index 0000000000..e0def1a743 --- /dev/null +++ b/tools/gce_setup/compute_extras.sh @@ -0,0 +1,255 @@ +#!/bin/bash + +# Bash funcs shared that combine common gcutil actions into single commands + +# remove_instance removes a named instance +# +# remove_instance <project> <instance_name> [<zone>="us-central1-b"] +remove_instance() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local an_instance=$2 + [[ -n $an_instance ]] || { + echo "$FUNCNAME: missing arg: an_instance" 1>&2 + return 1 + } + local zone=$3 + [[ -n $zone ]] || zone="us-central1-b" + + gcloud --project $project --quiet \ + compute instances delete $an_instance --zone=$zone +} + +# has_instance checks if a project contains a named instance +# +# has_instance <project> <instance_name> +has_instance() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local checked_instance=$2 + [[ -n $checked_instance ]] || { + echo "$FUNCNAME: missing arg: checked_instance" 1>&2 + return 1 + } + + instances=$(gcloud --project $project compute instances list \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 1) + for i in $instances + do + if [[ $i == $checked_instance ]] + then + return 0 + fi + done + + return 1 +} + +# find_network_ip finds the ip address of a instance if it is present in the project. +# +# find_network_ip <project> <instance_name> +find_network_ip() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local checked_instance=$2 + [[ -n $checked_instance ]] || { + echo "$FUNCNAME: missing arg: checked_instance" 1>&2 + return 1 + } + + has_instance $project $checked_instance || return 1 + gcloud --project $project compute instances list \ + | grep -e "$checked_instance\s" | sed -e 's/ \+/ /g' | cut -d' ' -f 4 +} + +# delete_disks deletes a bunch of disks matching a pattern +# +# delete_disks <project> <disk_pattern> +delete_disks() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local disk_pattern=$2 + [[ -n $disk_pattern ]] || { + echo "$FUNCNAME: missing arg: disk_pattern" 1>&2 + return 1 + } + + trash_disks=$(gcloud --project=$project compute disks list \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 1 | grep $disk_pattern) + [[ -n $trash_disks ]] && gcloud --project $project \ + --quiet compute disks delete $trash_disks +} + +# has_firewall checks if a project contains a named firewall +# +# has_firewall <project> <checked_firewall> +has_firewall() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local checked_firewall=$2 + [[ -n $checked_firewall ]] || { + echo "$FUNCNAME: missing arg: checked_firewall" 1>&2 + return 1 + } + + instances=$(gcloud --project $project compute firewall-rules list \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 1) + for i in $instances + do + if [[ $i == $checked_firewall ]] + then + return 0 + fi + done + + return 1 +} + +# remove_firewall removes a named firewall from a project. +# +# remove_firewall <project> <checked_firewall> +remove_firewall() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local a_firewall=$2 + [[ -n $a_firewall ]] || { + echo "$FUNCNAME: missing arg: a_firewall" 1>&2 + return 1 + } + + gcloud --project $project --quiet compute firewall-rules delete $a_firewall +} + +# has_network checks if a project contains a named network +# +# has_network <project> <checked_network> +has_network() { + local project=$1 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + local checked_network=$2 + [[ -n $checked_network ]] || { + echo "$FUNCNAME: missing arg: checked_network" 1>&2 + return 1 + } + + instances=$(gcloud --project $project compute networks list \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 1) + for i in $instances + do + if [[ $i == $checked_network ]] + then + return 0 + fi + done + + return 1 +} + +# maybe_setup_dev_network adds a network with the given name with firewalls +# useful to development +# +# - All machines can accessed internally and externally over SSH (port 22) +# - All machines can access one another other the internal network +# - All machines can be accessed externally via port 80, 443, 8080 and 8443 +maybe_setup_dev_network() { + local name=$1 + [[ -n $name ]] || { + echo "$FUNCNAME: missing arg: network name" 1>&2 + return 1 + } + + local project=$2 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + + has_network $project $name || { + echo "creating network '$name'" 1>&2 + gcloud compute --project $project networks create $name || return 1 + } + + # allow instances on the network to connect to each other internally + has_firewall $project "$name-ssh" || { + echo "adding firewall '$name-ssh'" 1>&2 + gcloud compute --project $project firewall-rules create "$name-ssh" \ + --network $name \ + --allow tcp:22 || return 1; + } + + # allow instances on the network to connect to each other internally + has_firewall $project "$name-internal" || { + echo "adding firewall '$name-internal'" 1>&2 + gcloud compute --project $project firewall-rules create "$name-internal" \ + --network $name \ + --source-ranges 10.0.0.0/16 --allow tcp udp icmp || return 1; + } + + # allow instances on the network to be connected to from external ips on + # specific ports + has_firewall $project "$name-external" || { + echo "adding firewall '$name-external'" 1>&2 + gcloud compute --project $project firewall-rules create "$name-external" \ + --network $name \ + --allow tcp:80 tcp:8080 tcp:443 tcp:8443 || return 1; + } +} + +# maybe_remove_dev_network removes a network set up by maybe_setup_dev_network +maybe_remove_dev_network() { + local name=$1 + [[ -n $name ]] || { + echo "$FUNCNAME: missing arg: network name" 1>&2 + return 1 + } + + local project=$2 + [[ -n $project ]] || { + echo "$FUNCNAME: missing arg: project" 1>&2 + return 1 + } + + has_network $project $name || { + echo "network $name is not present" + return 0 + } + for i in $(gcloud compute firewall-rules list \ + | grep "$name-" | cut -d' ' -f 1) + do + gcloud compute --quiet firewall-rules delete $i || return 1; + done + gcloud compute --quiet networks delete $name +} + +# find_named_ip finds the external ip address for a given name. +# +# find_named_ip <named-ip-address> +find_named_ip() { + local name=$1 + [[ -n $name ]] || { echo "$FUNCNAME: missing arg: name" 1>&2; return 1; } + [[ $name == 'none' ]] && return 0; + + gcloud compute addresses list | sed -e 's/ \+/ /g' \ + | grep $name | cut -d' ' -f 3 +} diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh new file mode 100755 index 0000000000..094b97bf3c --- /dev/null +++ b/tools/gce_setup/grpc_docker.sh @@ -0,0 +1,503 @@ +#!/bin/bash +# +# Contains funcs that help maintain GRPC's Docker images. +# +# Most funcs rely on the special-purpose GCE instance to build the docker +# instances and store them in a GCS-backed docker repository. +# +# The GCE instance +# - should be based on the container-optimized GCE instance +# [https://cloud.google.com/compute/docs/containers]. +# - should be running google/docker-registry image +# [https://registry.hub.docker.com/u/google/docker-registry/], so that images +# can be saved to GCS +# - should have the GCE support scripts from this directory install on it. +# +# The expected workflow is +# - start a grpc docker GCE instance +# * on startup, some of the docker images will be regenerated automatically +# - used grpc_update_image to update images via that instance + +# Pushes a dockerfile dir to cloud storage. +# +# dockerfile is expected to the parent directory to a nunber of directoies each +# of which specifies a Dockerfiles. +# +# grpc_push_dockerfiles path/to/docker_parent_dir gs://bucket/path/to/gcs/parent +grpc_push_dockerfiles() { + local docker_dir=$1 + [[ -n $docker_dir ]] || { + echo "$FUNCNAME: missing arg: docker_dir" 1>&2 + return 1 + } + + local gs_root_uri=$2 + [[ -n $gs_root_uri ]] || { + echo "$FUNCNAME: missing arg: gs_root_uri" 1>&2 + return 1 + } + + find $docker_dir -name '*~' -o -name '#*#' -exec rm -fv {} \; || { + echo "$FUNCNAME: failed: cleanup of tmp files in $docker_dir" 1>&2 + return 1 + } + gsutil cp -R $docker_dir $gs_root_uri || { + echo "$FUNCNAME: failed: cp $docker_dir -> $gs_root_uri" 1>&2 + return 1 + } +} + +# Adds the user to docker group on a GCE instance, and restarts the docker +# daemon +grpc_add_docker_user() { + local host=$1 + [[ -n $host ]] || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + + local project=$2 + local project_opt='' + [[ -n $project ]] && project_opt=" --project $project" + + local zone=$3 + local zone_opt='' + [[ -n $zone ]] && zone_opt=" --zone $zone" + + + local func_lib="/var/local/startup_scripts/shared_startup_funcs.sh" + local ssh_cmd="source $func_lib && grpc_docker_add_docker_group" + gcloud compute $project_opt ssh $zone_opt $host --command "$ssh_cmd" +} + +# Updates a docker image specified in a local dockerfile via the docker +# container GCE instance. +# +# the docker container GCE instance +# - should have been setup using ./new_grpc_docker_instance +# - so will have /var/local/startup_scripts/shared_startup_funcs.sh, a copy of +# ./shared_startup_funcs.sh +# +# grpc_update_image gs://bucket/path/to/dockerfile parent \. +# image_label path/to/docker_dir docker_gce_instance [project] [zone] +grpc_update_image() { + local gs_root_uri=$1 + [[ -n $gs_root_uri ]] || { + echo "$FUNCNAME: missing arg: gs_root_uri" 1>&2 + return 1 + } + + local image_label=$2 + [[ -n $image_label ]] || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + + local docker_dir=$3 + [[ -n $docker_dir ]] || { + echo "$FUNCNAME: missing arg: docker_dir" 1>&2 + return 1 + } + [[ -d $docker_dir ]] || { + echo "could find directory $docker_dir" 1>&2 + return 1 + } + local docker_parent_dir=$(dirname $docker_dir) + local gce_docker_dir="/var/local/dockerfile/$(basename $docker_dir)" + + local host=$4 + [[ -n $host ]] || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + + local project=$5 + local project_opt='' + [[ -n $project ]] && project_opt=" --project $project" + + local zone=$6 + local zone_opt='' + [[ -n $zone ]] && zone_opt=" --zone $zone" + + local func_lib="/var/local/startup_scripts/shared_startup_funcs.sh" + local ssh_cmd="source $func_lib" + local ssh_cmd+=" && grpc_dockerfile_refresh $image_label $gce_docker_dir" + + grpc_push_dockerfiles $docker_parent_dir $gs_root_uri || return 1 + gcloud compute $project_opt ssh $zone_opt $host --command "$ssh_cmd" +} + +# gce_has_instance checks if a project contains a named instance +# +# gce_has_instance <project> <instance_name> +gce_has_instance() { + local project=$1 + [[ -n $project ]] || { echo "$FUNCNAME: missing arg: project" 1>&2; return 1; } + local checked_instance=$2 + [[ -n $checked_instance ]] || { + echo "$FUNCNAME: missing arg: checked_instance" 1>&2 + return 1 + } + + instances=$(gcloud --project $project compute instances list \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 1) + for i in $instances + do + if [[ $i == $checked_instance ]] + then + return 0 + fi + done + + echo "instance '$checked_instance' not found in compute project $project" 1>&2 + return 1 +} + +# gce_find_internal_ip finds the ip address of a instance if it is present in +# the project. +# +# gce_find_internal_ip <project> <instance_name> +gce_find_internal_ip() { + local project=$1 + [[ -n $project ]] || { echo "$FUNCNAME: missing arg: project" 1>&2; return 1; } + local checked_instance=$2 + [[ -n $checked_instance ]] || { + echo "$FUNCNAME: missing arg: checked_instance" 1>&2 + return 1 + } + + gce_has_instance $project $checked_instance || return 1 + gcloud --project $project compute instances list \ + | grep -e "$checked_instance\s" \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 4 +} + +# sets the vars grpc_zone and grpc_project +# +# to be used in funcs that want to set the zone and project and potential +# override them with +# +# grpc_zone +# - is set to the value gcloud config value for compute/zone if that's present +# - it defaults to asia-east1-a +# - it can be overridden by passing -z <other value> +# +# grpc_project +# - is set to the value gcloud config value for project if that's present +# - it defaults to stoked-keyword-656 (the grpc cloud testing project) +# - it can be overridden by passing -p <other value> +grpc_set_project_and_zone() { + dry_run=0 + grpc_zone=$(gcloud config list compute/zone --format text \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 2) + # pick a known zone as a default + [[ $grpc_zone == 'None' ]] && grpc_zone='asia-east1-a' + + grpc_project=$(gcloud config list project --format text \ + | sed -e 's/ \+/ /g' | cut -d' ' -f 2) + # pick an known zone as a default + [[ $grpc_project == 'None' ]] && grpc_project='stoked-keyword-656' + + # see if -p or -z is used to override the the project or zone + local OPTIND + local OPTARG + local arg_func + while getopts :p:z:f:n name + do + case $name in + f) declare -F $OPTARG >> /dev/null && { + arg_func=$OPTARG; + } || { + echo "-f: arg_func value: $OPTARG is not defined" + return 2 + } + ;; + n) dry_run=1 ;; + p) grpc_project=$OPTARG ;; + z) grpc_zone=$OPTARG ;; + :) [[ $OPT_ARG == 'f' ]] && { + echo "-f: arg_func provided" 1>&2 + return 2 + } || { + # ignore -p or -z without args, just use the defaults + continue + } + ;; + \?) echo "-$OPTARG: unknown flag; it's ignored" 1>&2; continue ;; + esac + done + shift $((OPTIND-1)) + [[ -n $arg_func ]] && $arg_func "$@" +} + +# construct the flags to be passed to the binary running the test client +# +# call-seq: +# flags=$(grpc_interop_test_flags <server_ip> <server_port> <test_case>) +# [[ -n flags ]] || return 1 +grpc_interop_test_flags() { + [[ -n $1 ]] && { # server_ip + local server_ip=$1 + shift + } || { + echo "$FUNCNAME: missing arg: server_ip" 1>&2 + return 1 + } + [[ -n $1 ]] && { # port + local port=$1 + shift + } || { + echo "$FUNCNAME: missing arg: port" 1>&2 + return 1 + } + [[ -n $1 ]] && { # test_case + local test_case=$1 + shift + } || { + echo "$FUNCNAME: missing arg: test_case" 1>&2 + return 1 + } + echo "--server_host=$server_ip --server_port=$port --test_case=$test_case" +} + +# checks the positional args and assigns them to variables visible in the caller +# +# these are the positional args passed to grpc_interop_test after option flags +# are removed +# +# five args are expected, in order +# - test_case +# - host <the gce docker instance on which to run the test> +# - client to run +# - server_host <the gce docker instance on which the test server is running> +# - server type +grpc_interop_test_args() { + [[ -n $1 ]] && { # test_case + test_case=$1 + shift + } || { + echo "$FUNCNAME: missing arg: test_case" 1>&2 + return 1 + } + + [[ -n $1 ]] && { # host + host=$1 + shift + } || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + + [[ -n $1 ]] && { # client_type + case $1 in + cxx|go|java|nodejs|php|python|ruby) + grpc_gen_test_cmd="grpc_interop_gen_$1_cmd" + declare -F $grpc_gen_test_cmd >> /dev/null || { + echo "-f: test_func for $1 => $grpc_gen_test_cmd is not defined" 1>&2 + return 2 + } + shift + ;; + *) + echo "bad client_type: $1" 1>&2 + return 1 + ;; + esac + } || { + echo "$FUNCNAME: missing arg: client_type" 1>&2 + return 1 + } + + [[ -n $1 ]] && { # grpc_server + grpc_server=$1 + shift + } || { + echo "$FUNCNAME: missing arg: grpc_server" 1>&2 + return 1 + } + + [[ -n $1 ]] && { # server_type + case $1 in + cxx) grpc_port=8010 ;; + go) grpc_port=8020 ;; + java) grpc_port=8030 ;; + nodejs) grpc_port=8040 ;; + python) grpc_port=8050 ;; + ruby) grpc_port=8060 ;; + *) echo "bad server_type: $1" 1>&2; return 1 ;; + esac + shift + } || { + echo "$FUNCNAME: missing arg: server_type" 1>&2 + return 1 + } +} + +grpc_launch_server_args() { + [[ -n $1 ]] && { # host + host=$1 + shift + } || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + + [[ -n $1 ]] && { # server_type + case $1 in + cxx) grpc_port=8010 ;; + go) grpc_port=8020 ;; + java) grpc_port=8030 ;; + nodejs) grpc_port=8040 ;; + python) grpc_port=8050 ;; + ruby) grpc_port=8060 ;; + *) echo "bad server_type: $1" 1>&2; return 1 ;; + esac + docker_label="grpc/$1" + docker_name="grpc_interop_$1" + shift + } || { + echo "$FUNCNAME: missing arg: server_type" 1>&2 + return 1 + } +} + +# Launches a server on a docker instance. +# +# call-seq; +# grpc_launch_server <server_name> <server_type> +# +# Runs the server_type on a GCE instance running docker with server_name +grpc_launch_server() { + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + local grpc_zone grpc_project dry_run # set by grpc_set_project_and_zone + # set by grpc_launch_server_args + local docker_label docker_name host grpc_port + + # set the project zone and check that all necessary args are provided + grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1 + gce_has_instance $grpc_project $host || return 1; + + cmd="sudo docker run -d --name $docker_name" + cmd+=" -p $grpc_port:$grpc_port $docker_label" + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" +} + +# Runs a test command on a docker instance. +# +# call-seq: +# grpc_interop_test <test_name> <host> <client_type> \ +# <server_host> <server_type> +# +# N.B: server_name defaults to 'grpc-docker-server' +# +# requirements: +# host is a GCE instance running docker with access to the gRPC docker images +# server_name is a GCE docker instance running the gRPC server in docker +# test_name is one of the named gRPC tests [http://go/grpc_interop_tests] +# client_type is one of [cxx,go,java,php,python,ruby] +# server_type is one of [cxx,go,java,python,ruby] +# +# it assumes: +# that each grpc-imp has a docker image named grpc/<imp>, e.g, grpc/java +# a test is run using $ docker run 'path/to/interop_test_bin --flags' +# the required images are available on <host> +# +# server_name [default:grpc-docker-server] is an instance that runs the +# <server_type> server on the standard test port for the <server_type> +# +# each server_type runs it tests on a standard test port as follows: +# cxx: 8010 +# go: 8020 +# java: 8030 +# nodejs: 8040 +# python: 8050 +# ruby: 8060 +# +# each client_type should have an associated bash func: +# grpc_interop_gen_<client_type>_cmd +# the func provides the dockerized commmand for running client_type's test. +# If no such func is available, tests for that client type cannot be run. +# +# the flags for running a test are the same: +# +# --server_host=<svr_addr> --server_port=<svr_port> --test_case=<...> +grpc_interop_test() { + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + + local grpc_zone grpc_project dry_run # set by grpc_set_project_and_zone + # grpc_interop_test_args + local test_case host grpc_gen_test_cmd grpc_server grpc_port + + # set the project zone and check that all necessary args are provided + grpc_set_project_and_zone -f grpc_interop_test_args "$@" || return 1 + gce_has_instance $grpc_project $host || return 1; + + local addr=$(gce_find_internal_ip $grpc_project $grpc_server) + [[ -n $addr ]] || return 1 + local flags=$(grpc_interop_test_flags $addr $grpc_port $test_case) + [[ -n $flags ]] || return 1 + cmd=$($grpc_gen_test_cmd $flags) + [[ -n $cmd ]] || return 1 + + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" +} + +# constructs the full dockerized ruby interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_ruby_cmd() { + local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c" + local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb" + local the_cmd="$cmd_prefix 'ruby $test_script $@'" + echo $the_cmd +} + +# constructs the full dockerized java interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_java_cmd() { + local cmd_prefix="sudo docker run grpc/java"; + local test_script="/var/local/git/grpc-java/run-test-client.sh"; + local test_script+=" --transport=NETTY_TLS --grpc_version=2" + local the_cmd="$cmd_prefix $test_script $@"; + echo $the_cmd +} + +# constructs the full dockerized php interop test cmd. +# +# TODO(mlumish): update this to use the script once that's on git-on-borg +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_php_cmd() { + local cmd_prefix="sudo docker run grpc/php bin/bash -l -c"; + local test_script="cd /var/local/git/grpc/src/php/tests/interop"; + local test_script+=" && php -d extension_dir=../../ext/grpc/modules/"; + local test_script+=" -d extension=grpc.so interop_client.php"; + local the_cmd="$cmd_prefix '$test_script $@ 1>&2'"; + echo $the_cmd +} + + +# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|cxx|nodejs|go diff --git a/tools/gce_setup/new_grpc_docker_builder.sh b/tools/gce_setup/new_grpc_docker_builder.sh new file mode 100755 index 0000000000..9a3988f343 --- /dev/null +++ b/tools/gce_setup/new_grpc_docker_builder.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Triggers the build of a GCE 'grpc-docker' instance. +# +# Usage: +# /path/to/new_grpc_docker_builder.sh \ +# [--project <cloud-project-id> | -p<cloud-project-id>] \ +# [--instance <instance-to-create> | -i<instance-to-create>] \ +# [--address <named_cloud_static_ip> | -a<named_cloud_static_ip>] +# +# To run a new docker builder instance. +# $ /path/to/new_grpc_docker_builder.sh -pmy-project -imy-instance -amy-ip +# +# See main() for the full list of flags + +function this_dir() { + SCRIPT_PATH="${BASH_SOURCE[0]}"; + if ([ -h "${SCRIPT_PATH}" ]) then + while([ -h "${SCRIPT_PATH}" ]) do SCRIPT_PATH=`readlink "${SCRIPT_PATH}"`; done + fi + pushd . > /dev/null + cd `dirname ${SCRIPT_PATH}` > /dev/null + SCRIPT_PATH=`pwd`; + popd > /dev/null + echo $SCRIPT_PATH +} + +source $(this_dir)/compute_extras.sh +source $(this_dir)/grpc_docker.sh + +cp_startup_script() { + local script_dir=$1 + [[ -n $script_dir ]] || { echo "missing arg: script_dir" 1>&2; return 1; } + + local gs_script_root=$2 + [[ -n $gs_script_root ]] || { echo "missing arg: gs_script_root" 1>&2; return 1; } + + local script_path=$3 + [[ -n $script_path ]] || { echo "missing arg: script_name" 1>&2; return 1; } + + local startup_script=$script_dir/$script_path + local gs_startup_uri=$gs_script_root/$script_path + gsutil cp $startup_script $gs_startup_uri +} + +# add_instance adds a generic instance that runs +# new_grpc_docker_builder_on_startup.sh on startup +add_instance() { + local project=$1 + [[ -n $project ]] || { echo "missing arg: project" 1>&2; return 1; } + local gs_admin_root=$2 + [[ -n $gs_admin_root ]] || { echo "missing arg: gs_admin_root" 1>&2; return 1; } + local instance=$3 + [[ -n $instance ]] || { echo "missing arg: instance" 1>&2; return 1; } + local zone=$4 + [[ -n $zone ]] || { echo "missing arg: zone" 1>&2; return 1; } + local address=$5 + [[ -n $address ]] || { echo "missing arg: address" 1>&2; return 1; } + + local script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + local gs_script_root="$gs_admin_root/startup" + + local on_startup=new_grpc_docker_builder_on_startup.sh + local gs_on_startup=$gs_script_root/$on_startup + cp_startup_script $script_dir $gs_script_root $on_startup || { + echo "Could not save script to $gs_on_startup" 1>&2 + return 1 + } + startup_md="startup-script-url=$gs_on_startup" + + local shared_startup=shared_startup_funcs.sh + local gs_shared_startup=$gs_script_root/$shared_startup + cp_startup_script $script_dir $gs_script_root $shared_startup || { + echo "Could not save script to $gs_shared_startup" 1>&2 + return 1 + } + startup_md+=" shared_startup_script_url=$gs_shared_startup" + + local docker_dir=$(this_dir)/../dockerfile + grpc_push_dockerfiles $docker_dir $gs_admin_root || return 1; + startup_md+=" gs_dockerfile_root=$gs_admin_root/dockerfile" + startup_md+=" gs_docker_reg=$gs_admin_root/docker_images" + + local address_flag="" + local the_address=$(find_named_ip $address) + [[ -n $the_address ]] && address_flag="--address $the_address" + local the_image='container-vm-v20140925' + local scopes='compute-rw storage-full' + scopes+=' https://www.googleapis.com/auth/gerritcodereview' + gcloud --project $project compute instances create $instance \ + $address_flag \ + --image $the_image \ + --image-project google-containers \ + --metadata $startup_md \ + --machine-type='n1-standard-1' \ + --scopes $scopes \ + --tags grpc testing \ + --zone $zone \ + --boot-disk-size 500GB +} + +main() { + local INSTANCE_NAME="grpc-docker-builder" + local PROJECT="stoked-keyword-656" + local GS_ADMIN_ROOT="gs://tmp-grpc-dev/admin" + local ZONE='asia-east1-a' + local ADDRESS_NAME='grpc-php-dev-static-1' # use 'none' if no static ip is needed + + # Parse the options + opts=`getopt -o a::p::g::i::z:: --long address_name::,project::,gs_admin_root::,instance_name::,zone:: -n $0 -- "$@"` + eval set -- "$opts" + while true ; do + case "$1" in + -p|--project) + case "$2" in + "") shift 2 ;; + *) PROJECT=$2; shift 2 ;; + esac ;; + -a|--address_name) + case $2 in + "") shift 2 ;; + *) ADDRESS_NAME=$2; shift 2 ;; + esac ;; + -g|--gs_admin_root) + case "$2" in + "") shift 2 ;; + *) GS_ADMIN_ROOT=$2; shift 2 ;; + esac ;; + -i|--instance_name) + case "$2" in + "") shift 2 ;; + *) INSTANCE_NAME=$2; shift 2 ;; + esac ;; + -z|--zone) + case "$2" in + "") shift 2 ;; + *) ZONE=$2; shift 2 ;; + esac ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac + done + + # verify that the instance does not currently exist + has_instance $PROJECT $INSTANCE_NAME && remove_instance $PROJECT $INSTANCE_NAME $ZONE + has_instance $PROJECT $INSTANCE_NAME && { echo "$INSTANCE_NAME already exists" 1>&2; return 1; } + + # N.B the quotes around are necessary to allow cmds with spaces + add_instance $PROJECT $GS_ADMIN_ROOT $INSTANCE_NAME $ZONE $ADDRESS_NAME +} + +set -x +main "$@" diff --git a/tools/gce_setup/new_grpc_docker_builder_on_startup.sh b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh new file mode 100755 index 0000000000..87e8aac1e7 --- /dev/null +++ b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Startup script that initializes a grpc-dev GCE machine. +# +# A grpc-docker GCE machine is based on docker container image. +# +# On startup, it copies the grpc dockerfiles to a local directory, and update its address +# so that the docker containers within it have git-on-borg-access. + +# _load_metadata curls a metadata url +_load_metadata() { + local metadata_root=http://metadata/computeMetadata/v1 + local uri=$1 + [[ -n $uri ]] || { echo "missing arg: uri" >&2; return 1; } + + if [[ $uri =~ ^'attributes/' ]] + then + for a in $(curl -H "X-Google-Metadata-Request: True" $metadata_root/instance/attributes/) + do + [[ $uri =~ "/$a"$ ]] && { curl $metadata_root/instance/$uri -H "X-Google-Metadata-Request: True"; return; } + done + fi + + # if the uri is a full request uri + [[ $uri =~ ^$metadata_root ]] && { curl $uri -H "X-Google-Metadata-Request: True"; return; } +} + +_source_gs_script() { + local script_attr=$1 + [[ -n $script_attr ]] || { echo "missing arg: script_attr" >&2; return 1; } + + local gs_uri=$(_load_metadata "attributes/$script_attr") + [[ -n $gs_uri ]] || { echo "missing metadata: $script_attr" >&2; return 1; } + + local out_dir='/var/local/startup_scripts' + local script_path=$out_dir/$(basename $gs_uri) + mkdir -p $out_dir + gsutil cp $gs_uri $script_path || { + echo "could not cp $gs_uri -> $script_path" + return 1 + } + chmod a+rwx $out_dir $script_path + source $script_path +} + +main() { + local script_attr='shared_startup_script_url' + _source_gs_script $script_attr || { + echo "halting, script 'attributes/$script_attr' could not be sourced" + return 1 + } + grpc_dockerfile_pull + chmod -R a+rw /var/local/dockerfile + + # Install git and emacs + apt-get update && apt-get install -y git emacs || return 1 + + # Enable access to git repos on git-on-borg + local git_root='/var/local/git' + install_gob_daemon $git_root/gerrit-gcompute-tools || return 1 + + # Startup the docker registry + grpc_docker_launch_registry && grpc_docker_pull_known + + # Add a sentinel file to indicate that startup has completed. + local sentinel_file=/var/log/GRPC_DOCKER_IS_UP + touch $sentinel_file +} + +set -x +main "$@" diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh new file mode 100755 index 0000000000..8bd62e213e --- /dev/null +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -0,0 +1,432 @@ +#!/bin/bash +# Contains common funcs shared by instance startup scripts. +# +# The funcs assume that the code is being run on a GCE instance during instance +# startup. + +function die() { + local msg="$0 failed" + if [[ -n $1 ]] + then + msg=$1 + fi + echo $msg + exit 1 +} + +# umount_by_disk_id umounts a disk given its disk_id. +umount_by_disk_id() { + local disk_id=$1 + [[ -n $disk_id ]] || { echo "missing arg: disk_id" >&2; return 1; } + + # Unmount the disk first + sudo umount /dev/disk/by-id/google-$disk_id || { echo "Could not unmount /mnt/disk-by-id/google-$disk_id" >&2; return 1; } +} + +# check_metadata confirms that the result of curling a metadata url does not +# contain 'Error 404' +check_metadata() { + local curl_output=$1 + [[ -n $curl_output ]] || { echo "missing arg: curl_output" >&2; return 1; } + + if [[ $curl_output =~ "Error 404" ]] + then + return 1 + fi + + return 0 +} + +# name_this_instance determines the current instance name. +name_this_instance() { + local the_full_host_name + the_full_host_name=$(load_metadata "http://metadata/computeMetadata/v1/instance/hostname") + check_metadata $the_full_host_name || return 1 + local the_instance + the_instance=$(echo $the_full_host_name | cut -d . -f 1 -) || { + echo "could not get the instance name from $the_full_host_name" >&2 + return 1 + } + + echo $the_instance +} + +# delete_this_instance deletes this GCE instance. (it will shutdown as a result +# of running this cmd) +delete_this_instance() { + local the_full_zone + the_full_zone=$(load_metadata "http://metadata/computeMetadata/v1/instance/zone") + check_metadata $the_full_zone || return 1 + local the_zone + the_zone=$(echo $the_full_zone | cut -d / -f 4 -) || { echo "could not get zone from $the_full_zone" >&2; return 1; } + + local the_full_host_name + the_full_host_name=$(load_metadata "http://metadata/computeMetadata/v1/instance/hostname") + check_metadata $the_full_host_name || return 1 + local the_instance + the_instance=$(echo $the_full_host_name | cut -d . -f 1 -) || { echo "could not get zone from $the_full_host_name" >&2; return 1; } + + echo "using gcloud compute instances delete to remove: ${the_instance}" + gcloud compute --quiet instances delete --delete-disks boot --zone $the_zone $the_instance +} + +# save_image_info updates the 'images' release info file on GCS. +save_image_info() { + local image_id=$1 + [[ -n $image_id ]] || { echo "missing arg: image_id" >&2; return 1; } + + local repo_gs_uri=$2 + [[ -n $repo_gs_uri ]] || { echo "missing arg: repo_gs_uri" >&2; return 1; } + + local sentinel="/tmp/$image_id.txt" + echo $image_id > $sentinel || { echo "could not create /tmp/$image_id.txt" >&2; return 1; } + + local gs_sentinel="$repo_gs_uri/images/info/LATEST" + gsutil cp $sentinel $gs_sentinel || { echo "failed to update $gs_sentinel" >&2; return 1; } +} + +# creates an image, getting the name and cloud storage uri from the supplied +# instance metadata. +create_image() { + local image_id + image_id=$(load_metadata "attributes/image_id") + [[ -n $image_id ]] || { echo "missing metadata: image_id" >&2; return 1; } + + local repo_gs_uri + repo_gs_uri=$(load_metadata "attributes/repo_gs_uri") + [[ -n $repo_gs_uri ]] || { echo "missing metadata: repo_gs_uri" >&2; return 1; } + + local the_project + the_project=$(load_metadata "http://metadata/computeMetadata/v1/project/project-id") + check_metadata $the_project || return 1 + + sudo gcimagebundle -d /dev/sda -o /tmp/ --log_file=/tmp/$image_id.log || { echo "image creation failed" >&2; return 1; } + image_path=$(ls /tmp/*.tar.gz) + image_gs_uri="$repo_gs_uri/images/$image_id.tar.gz" + + # copy the image to cloud storage + gsutil cp $image_path $image_gs_uri || { echo "failed to save image to $repo_gs_uri/$image_path " >&2; return 1; } + gcloud compute --project=$the_project images create \ + $image_id --source-uri $image_gs_uri || { echo "failed to register $image_gs_uri as $image_id" >&2; return 1; } + + save_image_info $image_id $repo_gs_uri +} + +# load_metadata curls a metadata url +load_metadata() { + local metadata_root=http://metadata/computeMetadata/v1 + local uri=$1 + [[ -n $uri ]] || { echo "missing arg: uri" >&2; return 1; } + + if [[ $uri =~ ^'attributes/' ]] + then + for a in $(curl -H "X-Google-Metadata-Request: True" $metadata_root/instance/attributes/) + do + [[ $uri =~ "/$a"$ ]] && { curl $metadata_root/instance/$uri -H "X-Google-Metadata-Request: True"; return; } + done + fi + + # if the uri is a full request uri + [[ $uri =~ ^$metadata_root ]] && { curl $uri -H "X-Google-Metadata-Request: True"; return; } +} + +install_python_module() { + local mod=$1 + [[ -z $mod ]] && { echo "missing arg: mod" >&2; return 1; } + + echo '------------------------------------' + echo 'Installing: $mod' + echo '------------------------------------' + echo + install_with_apt_get gcc python-dev python-setuptools + sudo apt-get install -y gcc python-dev python-setuptools + sudo easy_install -U pip + sudo pip uninstall -y $mod + sudo pip install -U $mod +} + +install_with_apt_get() { + local pkgs=$@ + echo '---------------------------' + echo 'Installing: $pkgs' + echo '---------------------------' + echo + sudo apt-get install -y $pkgs +} + +# pulls code from a git repo @HEAD to a local directory, removing the current version if present. +setup_git_dir() { + local git_http_repo=$1 + [[ -n $git_http_repo ]] || { echo "missing arg: git_http_repo" >&2; return 1; } + + local git_dir=$2 + [[ -n $git_dir ]] || { echo "missing arg: git_dir" >&2; return 1; } + + if [[ -e $git_dir ]] + then + rm -fR $git_dir || { echo "could not remove existing repo at $git_dir" >&2; return 1; } + fi + + local git_user + git_user=$(load_metadata "http://metadata/computeMetadata/v1/instance/service-accounts/default/email") + check_metadata $git_user || return 1 + urlsafe_git_user=$(echo $git_user | sed -e s/@/%40/g) || return 1 + + local access_token=$(load_metadata "http://metadata/computeMetadata/v1/instance/service-accounts/default/token?alt=text") + check_metadata $access_token || return 1 + local git_pwd=$(echo $access_token | cut -d' ' -f 2) || return 1 + + git clone https://$urlsafe_git_user:$git_pwd@$git_http_repo $git_dir +} + +# network_copy copies a file to another gce instance. +network_copy() { + local the_node=$1 + [[ -n $the_node ]] || { echo "missing arg: the_node" >&2; return 1; } + + local src=$2 + [[ -n $src ]] || { echo "missing arg: src" >&2; return 1; } + + local dst=$3 + [[ -n $dst ]] || { echo "missing arg: dst" >&2; return 1; } + + gcloud compute copy-files --zone=us-central1-b $src $node:$dst +} + +# gcs_copy copies a file to a location beneath a root gcs object path. +gcs_copy() { + local gce_root=$1 + [[ -n $gce_root ]] || { echo "missing arg: gce_root" >&2; return 1; } + + local src=$2 + [[ -n $src ]] || { echo "missing arg: src" >&2; return 1; } + + local dst=$3 + [[ -n $dst ]] || { echo "missing arg: dst" >&2; return 1; } + + gsutil cp $src $gce_root/$dst +} + +# find_named_ip finds the external ip address for a given name. +find_named_ip() { + local name=$1 + [[ -n $name ]] || { echo "missing arg: name" >&2; return 1; } + + gcloud compute addresses list | sed -e 's/ \+/ /g' | grep $name | cut -d' ' -f 3 +} + +# update_address_to updates this instances ip address to the reserved ip address with a given name +update_address_to() { + local name=$1 + [[ -n $name ]] || { echo "missing arg: name" >&2; return 1; } + + named_ip=$(find_named_ip $name) + [[ -n $named_ip ]] || { echo "did not find an address corresponding to $name" >&2; return 1; } + + local the_full_zone + the_full_zone=$(load_metadata "http://metadata/computeMetadata/v1/instance/zone") + check_metadata $the_full_zone || return 1 + local the_zone + the_zone=$(echo $the_full_zone | cut -d / -f 4 -) || { + echo "could not get zone from $the_full_zone" >&2 + return 1 + } + + local the_full_host_name + the_full_host_name=$(load_metadata "http://metadata/computeMetadata/v1/instance/hostname") + check_metadata $the_full_host_name || return 1 + local the_instance + the_instance=$(echo $the_full_host_name | cut -d . -f 1 -) || { + echo "could not determine the instance from $the_full_host_name" >&2 + return 1 + } + + gcloud compute instances delete-access-config --zone $the_zone $the_instance || { + echo "could not delete the access config for $the_instance" >&2 + return 1 + } + gcloud compute instances add-access-config --zone $the_zone $the_instance --address $named_ip || { + echo "could not update the access config for $the_instance to $named_ip" >&2 + return 1 + } +} + +# Allows instances to checkout repos on git-on-borg. +# +install_gob_daemon() { + local gob_dir=$1 + [[ -n $gob_dir ]] || { echo "missing args: gob_dir" >&2; return 1; } + + local gob_repo=$2 + [[ -n $gob_repo ]] || gob_repo='https://gerrit.googlesource.com/gcompute-tools/' + + if [[ -e $gob_dir ]] + then + rm -fv $gob_dir || { + echo "could not remove existing git repo at $gob_dir" >&2 + return 1 + } + fi + + git clone $gob_repo $gob_dir || { echo "failed to pull gerrit cookie repo" >&2; return 1; } + local startup_script=/etc/profile.d/gob_cookie_daemon.sh + + cat <<EOF >> $startup_script +#!/bin/bash + +$gob_dir/git-cookie-authdaemon + +EOF + + chmod 755 $startup_script + $startup_script +} + +# grpc_docker_add_docker_group +# +# Adds a docker group, restarts docker, relaunches the docker registry +grpc_docker_add_docker_group() { + [[ -f /var/log/GRPC_DOCKER_IS_UP ]] || { + echo "missing file /var/log/GRPC_DOCKER_IS_UP; either wrong machine or still starting up" >&2; + return 1 + } + sudo groupadd docker + + local user=$(id -un) + [[ -n ${user} ]] || { echo 'could not determine the user' >&2; return 1; } + sudo gpasswd -a ${user} docker + sudo service docker restart || return 1; + grpc_docker_launch_registry +} + +# grpc_dockerfile_pull <local_docker_parent_dir> +# +# requires: attributes/gs_dockerfile_root is set to cloud storage directory +# containing the dockerfile directory +grpc_dockerfile_pull() { + local dockerfile_parent=$1 + [[ -n $dockerfile_parent ]] || dockerfile_parent='/var/local' + + local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root") + [[ -n $gs_dockerfile_root ]] || { echo "missing metadata: gs_dockerfile_root" >&2; return 1; } + + mkdir -p $dockerfile_parent + gsutil cp -R $gs_dockerfile_root $dockerfile_parent || { + echo "Did not copy docker files from $gs_dockerfile_root -> $dockerfile_parent" + return 1 + } + } + +# grpc_docker_launch_registry +# +# requires: attributes/gs_docker_reg is set to the cloud storage directory to +# use to store docker images +grpc_docker_launch_registry() { + local gs_docker_reg=$(load_metadata "attributes/gs_docker_reg") + [[ -n $gs_docker_reg ]] || { echo "missing metadata: gs_docker_reg" >&2; return 1; } + + local gs_bucket=$(echo $gs_docker_reg | sed -r 's|gs://([^/]*?).*|\1|g') + [[ -n $gs_bucket ]] || { + echo "could not determine cloud storage bucket from $gs_bucket" >&2; + return 1 + } + + local storage_path_env='' + local image_path=$(echo $gs_docker_reg | sed -r 's|gs://[^/]*(.*)|\1|g' | sed -e 's:/$::g') + [[ -n $image_path ]] && { + storage_path_env="-e STORAGE_PATH=$image_path" + } + + sudo docker run -d -e GCS_BUCKET=$gs_bucket $storage_path_env -p 5000:5000 google/docker-registry + # wait a couple of minutes max, for the registry to come up + local is_up=0 + for i in {1..24} + do + local secs=`expr $i \* 5` + echo "is docker registry up? waited for $secs secs ..." + wget -q localhost:5000 && { + echo 'docker registry is up!' + is_up=1 + break + } + sleep 5 + done + + [[ $is_up == 0 ]] && { + echo "docker registry not available after 120 seconds"; return 1; + } || return 0 +} + +# grpc_docker_pull_known +# +# This pulls a set of known docker images from a private docker registry to +# the local image cache. It re-labels the images so that FROM in dockerfiles +# used in dockerfiles running on the docker instance can find the images OK. +# +# optional: address of a grpc docker registry, the default is 0.0.0.0:5000 +grpc_docker_pull_known() { + local addr=$1 + [[ -n $addr ]] || addr="0.0.0.0:5000" + local known="base cxx php_base php ruby_base ruby java_base java" + echo "... pulling docker images for '$known'" + for i in $known + do + sudo docker pull ${addr}/grpc/$i \ + && sudo docker tag ${addr}/grpc/$i grpc/$i || { + # log and continue + echo "docker op error: could not pull ${addr}/grpc/$i" + } + done +} + +# grpc_dockerfile_build_install +# +# requires: $1 is the label to apply to the docker image +# requires: $2 is a local directory containing a Dockerfile +# requires: there is a docker registry running on 5000, e.g, grpc_docker_launch_registry was run +# +# grpc_dockerfile_install "grpc/image" /var/local/dockerfile/grpc_image +grpc_dockerfile_install() { + local image_label=$1 + [[ -n $image_label ]] || { echo "missing arg: image_label" >&2; return 1; } + local docker_img_url=0.0.0.0:5000/$image_label + + local dockerfile_dir=$2 + [[ -n $dockerfile_dir ]] || { echo "missing arg: dockerfile_dir" >&2; return 1; } + + local cache_opt='--no-cache' + local cache=$3 + [[ $cache == "cache=yes" ]] && { cache_opt=''; } + [[ $cache == "cache=1" ]] && { cache_opt=''; } + [[ $cache == "cache=true" ]] && { cache_opt=''; } + + [[ -d $dockerfile_dir ]] || { echo "not a valid dir: $dockerfile_dir"; return 1; } + + # TODO(temiola): maybe make cache/no-cache a func option? + sudo docker build $cache_opt -t $image_label $dockerfile_dir || { + echo "docker op error: build of $image_label <- $dockerfile_dir" + return 1 + } + sudo docker tag $image_label $docker_img_url || { + echo "docker op error: tag of $docker_img_url" + return 1 + } + sudo docker push $docker_img_url || { + echo "docker op error: push of $docker_img_url" + return 1 + } +} + +# grpc_dockerfile_refresh +# +# requires: $1 is the label to apply to the docker image +# requires: $2 is a local directory containing a Dockerfile +# requires: there is a docker registry running on 5000, e.g, grpc_docker_launch_registry was run +# +# invokes pull_dockerfiles to refresh them all from cloud storage, then grpc_dockerfile_install +# +# grpc_dockerfile_refresh "grpc/mylabel" /var/local/dockerfile/dir_containing_my_dockerfile +grpc_dockerfile_refresh() { + grpc_dockerfile_pull || return 1 + grpc_dockerfile_install "$@" +} diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py new file mode 100755 index 0000000000..0890cc5d99 --- /dev/null +++ b/tools/run_tests/jobset.py @@ -0,0 +1,88 @@ +"""Run a group of subprocesses and then finish.""" + +import multiprocessing +import random +import subprocess +import sys +import threading + +# multiplicative factor to over subscribe CPU cores +# (many tests sleep for a long time) +_OVERSUBSCRIBE = 32 +_active_jobs = threading.Semaphore( + multiprocessing.cpu_count() * _OVERSUBSCRIBE) +_output_lock = threading.Lock() + + +def shuffle_iteratable(it): + """Return an iterable that randomly walks it""" + # take a random sampling from the passed in iterable + # we take an element with probablity 1/p and rapidly increase + # p as we take elements - this gives us a somewhat random set of values before + # we've seen all the values, but starts producing values without having to + # compute ALL of them at once, allowing tests to start a little earlier + nextit = [] + p = 1 + for val in it: + if random.randint(0, p) == 0: + p *= 2 + yield val + else: + nextit.append(val) + # after taking a random sampling, we shuffle the rest of the elements and + # yield them + random.shuffle(nextit) + for val in nextit: + yield val + + +class Jobset(object): + """Manages one run of jobs.""" + + def __init__(self, cmdlines): + self._cmdlines = shuffle_iteratable(cmdlines) + self._failures = 0 + + def _run_thread(self, cmdline): + try: + # start the process + p = subprocess.Popen(args=cmdline, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + stdout, _ = p.communicate() + # log output (under a lock) + _output_lock.acquire() + try: + if p.returncode != 0: + sys.stdout.write('\x1b[0G\x1b[2K\x1b[31mFAILED\x1b[0m: %s' + ' [ret=%d]\n' + '%s\n' % ( + ' '.join(cmdline), p.returncode, + stdout)) + self._failures += 1 + else: + sys.stdout.write('\x1b[0G\x1b[2K\x1b[32mPASSED\x1b[0m: %s' % + ' '.join(cmdline)) + sys.stdout.flush() + finally: + _output_lock.release() + finally: + _active_jobs.release() + + def run(self): + threads = [] + for cmdline in self._cmdlines: + # cap number of active jobs - release in _run_thread + _active_jobs.acquire() + t = threading.Thread(target=self._run_thread, + args=[cmdline]) + t.start() + threads.append(t) + for thread in threads: + thread.join() + return self._failures == 0 + + +def run(cmdlines): + return Jobset(cmdlines).run() + diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py new file mode 100755 index 0000000000..ee61f33484 --- /dev/null +++ b/tools/run_tests/run_tests.py @@ -0,0 +1,49 @@ +#!/usr/bin/python +"""Run tests in parallel.""" + +import argparse +import glob +import itertools +import multiprocessing +import sys + +import jobset + +# flags required for make for each configuration +_CONFIGS = ['dbg', 'opt', 'tsan', 'msan', 'asan'] + +# parse command line +argp = argparse.ArgumentParser(description='Run grpc tests.') +argp.add_argument('-c', '--config', + choices=['all'] + _CONFIGS, + nargs='+', + default=['all']) +argp.add_argument('-t', '--test-filter', nargs='*', default=['*']) +argp.add_argument('-n', '--runs_per_test', default=1, type=int) +args = argp.parse_args() + +# grab config +configs = [cfg + for cfg in itertools.chain.from_iterable( + _CONFIGS if x == 'all' else [x] + for x in args.config)] +filters = args.test_filter +runs_per_test = args.runs_per_test + +# build latest, sharing cpu between the various makes +if not jobset.run( + ['make', + '-j', '%d' % max(multiprocessing.cpu_count() / len(configs), 1), + 'buildtests_c', + 'CONFIG=%s' % cfg] + for cfg in configs): + sys.exit(1) + +# run all the tests +jobset.run([x] + for x in itertools.chain.from_iterable( + itertools.chain.from_iterable(itertools.repeat( + glob.glob('bins/%s/%s_test' % (config, filt)), + runs_per_test)) + for config in configs + for filt in filters)) |