diff options
| -rw-r--r-- | tools/dockerfile/grpc_base/Dockerfile | 25 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_cxx/Dockerfile | 19 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_php/Dockerfile | 5 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_php_base/Dockerfile | 20 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_ruby/Dockerfile | 9 | ||||
| -rw-r--r-- | tools/dockerfile/grpc_ruby_base/Dockerfile | 26 | ||||
| -rwxr-xr-x | tools/gce_setup/shared_startup_funcs.sh | 44 |
7 files changed, 89 insertions, 59 deletions
diff --git a/tools/dockerfile/grpc_base/Dockerfile b/tools/dockerfile/grpc_base/Dockerfile index 45be172593..be1b69b0dc 100644 --- a/tools/dockerfile/grpc_base/Dockerfile +++ b/tools/dockerfile/grpc_base/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y \ libc6 \ libc6-dbg \ libc6-dev \ + libgtest-dev \ libtool \ make \ strace \ @@ -34,23 +35,13 @@ ENV CLOUD_SDK /google-cloud-sdk RUN $CLOUD_SDK/install.sh --usage-reporting=true --path-update=true --bash-completion=true --rc-path=/.bashrc --disable-installation-options ENV PATH $CLOUD_SDK/bin:$PATH -# Install gcompute-tools to allow access to private git-on-borg repos -RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools - -# Start the daemon that allows access to private git-on-borg repos -RUN /var/local/git/gcompute-tools/git-cookie-authdaemon - -# Install the grpc-tools scripts dir from git -RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-tools /var/local/git/grpc-tools - -# Install the grpc-protobuf dir that has the protoc patch -RUN git clone https://team.googlesource.com/one-platform-grpc-team/protobuf /var/local/git/protobuf - -# Install the patched version of protoc -RUN cd /var/local/git/protobuf && \ - ./autogen.sh && \ - ./configure --prefix=/usr && \ - make && make check && make install && make clean +# Install a GitHub SSH service credential that gives access to the GitHub repo while it's private +# TODO: remove this once the repo is public +ADD .ssh .ssh +RUN chmod 600 .ssh/github.rsa +RUN mkdir -p $HOME/.ssh && echo 'Host github.com' > $HOME/.ssh/config +RUN echo " IdentityFile /.ssh/github.rsa" >> $HOME/.ssh/config +RUN echo 'StrictHostKeyChecking no' >> $HOME/.ssh/config # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/grpc_cxx/Dockerfile b/tools/dockerfile/grpc_cxx/Dockerfile index cf38e976b1..ea3a1dba8f 100644 --- a/tools/dockerfile/grpc_cxx/Dockerfile +++ b/tools/dockerfile/grpc_cxx/Dockerfile @@ -1,15 +1,18 @@ # Dockerfile for gRPC C++ FROM grpc/base -# Start the daemon that allows access to the protected git-on-borg repos -RUN /var/local/git/gcompute-tools/git-cookie-authdaemon - -RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc -RUN cd /var/local/git/grpc \ - && git pull --recurse-submodules \ - && git submodule update --init --recursive +# Get the source from GitHub +RUN git clone git@github.com:google/grpc.git /var/local/git/grpc +RUN cd /var/local/git/grpc && \ + git pull --recurse-submodules && \ + git submodule update --init --recursive +# Build the protobuf library; then the C core. +RUN cd /var/local/git/grpc/third_party/protobuf && \ + ./autogen.sh && \ + ./configure --prefix=/usr && \ + make -j12 && make check && make install && make clean RUN make install -C /var/local/git/grpc # Define the default command. -CMD ["bash"]
\ No newline at end of file +CMD ["bash"] diff --git a/tools/dockerfile/grpc_php/Dockerfile b/tools/dockerfile/grpc_php/Dockerfile index 0e50af70a2..177587669c 100644 --- a/tools/dockerfile/grpc_php/Dockerfile +++ b/tools/dockerfile/grpc_php/Dockerfile @@ -1,9 +1,6 @@ # Dockerfile for gRPC PHP FROM grpc/php_base -# Start the daemon that allows access to the protected git-on-borg repos -RUN /var/local/git/gcompute-tools/git-cookie-authdaemon - RUN cd /var/local/git/grpc \ && git pull --recurse-submodules \ && git submodule update --init --recursive @@ -15,4 +12,4 @@ RUN cd /var/local/git/grpc/src/php/ext/grpc && git pull && phpize # Build the grpc PHP extension RUN cd /var/local/git/grpc/src/php/ext/grpc \ && ./configure \ - && make
\ No newline at end of file + && make diff --git a/tools/dockerfile/grpc_php_base/Dockerfile b/tools/dockerfile/grpc_php_base/Dockerfile index 8ec90f48b8..47266a310e 100644 --- a/tools/dockerfile/grpc_php_base/Dockerfile +++ b/tools/dockerfile/grpc_php_base/Dockerfile @@ -43,9 +43,10 @@ RUN cd /var/local \ && tar -xf php-5.5.17.tar.gz \ && cd php-5.5.17 \ && ./configure --with-zlib=/usr --with-libxml-dir=ext/libxml \ - && make && make install + && make -j12 && make install # Start the daemon that allows access to the protected git-on-borg repos +RUN git clone https://gerrit.googlesource.com/gcompute-tools /var/local/git/gcompute-tools RUN /var/local/git/gcompute-tools/git-cookie-authdaemon # Download the patched PHP protobuf so that PHP gRPC clients can be generated @@ -64,6 +65,18 @@ ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/s # rake: a ruby version of make used to build the PHP Protobuf extension RUN rvm all do gem install ronn rake +# Get the source from GitHub, this gets the protobuf library as well +RUN git clone git@github.com:google/grpc.git /var/local/git/grpc +RUN cd /var/local/git/grpc && \ + git pull --recurse-submodules && \ + git submodule update --init --recursive + +# Build and install the protobuf library +RUN cd /var/local/git/grpc/third_party/protobuf && \ + ./autogen.sh && \ + ./configure --prefix=/usr && \ + make -j12 && make check && make install && make clean + # Install the patched PHP protobuf so that PHP gRPC clients can be generated # from proto3 schemas. RUN cd /var/local/git/protobuf-php \ @@ -75,10 +88,7 @@ RUN wget https://phar.phpunit.de/phpunit.phar \ && chmod +x phpunit.phar \ && mv phpunit.phar /usr/local/bin/phpunit -RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc -RUN cd /var/local/git/grpc \ - && git submodule update --init --recursive - +# Build the C core RUN make static_c shared_c -j12 -C /var/local/git/grpc # Define the default command. diff --git a/tools/dockerfile/grpc_ruby/Dockerfile b/tools/dockerfile/grpc_ruby/Dockerfile index 9aa34bfcc9..43ec0183ca 100644 --- a/tools/dockerfile/grpc_ruby/Dockerfile +++ b/tools/dockerfile/grpc_ruby/Dockerfile @@ -1,19 +1,14 @@ # Dockerfile for gRPC Ruby FROM grpc/ruby_base -# Start the daemon that allows access to the protected git-on-borg repos -RUN /var/local/git/gcompute-tools/git-cookie-authdaemon - +# Build the C libary RUN cd /var/local/git/grpc \ && git pull --recurse-submodules \ && git submodule update --init --recursive +# Build the C core. RUN make install_c -C /var/local/git/grpc -# Install the grpc gem locally with its dependencies and build the extension. -RUN /bin/bash -l -c 'cd /var/local/git/beefcake && bundle && gem build beefcake.gemspec && gem install beefcake' -RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake compile:grpc && gem build grpc.gemspec && gem install grpc' - # TODO add a command to run the unittest tests when the bug below is fixed # - the tests fail due to an error in the C threading library: # they fail with 'ruby: __pthread_mutex_cond_lock_adjust for unknown reasons' at the end of a testcase diff --git a/tools/dockerfile/grpc_ruby_base/Dockerfile b/tools/dockerfile/grpc_ruby_base/Dockerfile index ad14e43ec7..b2af9d7160 100644 --- a/tools/dockerfile/grpc_ruby_base/Dockerfile +++ b/tools/dockerfile/grpc_ruby_base/Dockerfile @@ -31,15 +31,6 @@ RUN apt-get update && apt-get install -y \ sqlite3 \ zlib1g-dev - -# Start the daemon that allows access to the protected git-on-borg repos -RUN /var/local/git/gcompute-tools/git-cookie-authdaemon - -# Download the patched Ruby protobuf (beefcake) so that Ruby gRPC clients can -# be generated from proto3 schemas. -RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc-ruby-beefcake \ - /var/local/git/beefcake - # Install RVM, use this to install ruby RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys D39DC0E3 # Needed for RVM RUN /bin/bash -l -c "curl -L get.rvm.io | bash -s stable" @@ -52,8 +43,17 @@ RUN /bin/bash -l -c "echo 'source /home/grpc_ruby/.rvm/scripts/rvm' >> ~/.bashrc RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" -RUN git clone https://team.googlesource.com/one-platform-grpc-team/grpc /var/local/git/grpc -RUN cd /var/local/git/grpc \ - && git submodule update --init --recursive +# Get the source from GitHub +RUN git clone git@github.com:google/grpc.git /var/local/git/grpc +RUN cd /var/local/git/grpc && \ + git pull --recurse-submodules && \ + git submodule update --init --recursive + +# Build and install the protobuf library +RUN cd /var/local/git/grpc/third_party/protobuf && \ + ./autogen.sh && \ + ./configure --prefix=/usr && \ + make -j12 && make check && make install && make clean -RUN make static_c shared_c -C /var/local/git/grpc
\ No newline at end of file +# Build the C core +RUN make static_c shared_c -j12 -C /var/local/git/grpc diff --git a/tools/gce_setup/shared_startup_funcs.sh b/tools/gce_setup/shared_startup_funcs.sh index 9ea6eca461..9c747466a9 100755 --- a/tools/gce_setup/shared_startup_funcs.sh +++ b/tools/gce_setup/shared_startup_funcs.sh @@ -388,7 +388,7 @@ grpc_docker_pull_known() { # grpc_dockerfile_install "grpc/image" /var/local/dockerfile/grpc_image grpc_dockerfile_install() { local image_label=$1 - [[ -n $image_label ]] || { echo "missing arg: image_label" >&2; return 1; } + [[ -n $image_label ]] || { echo "$FUNCNAME: missing arg: image_label" >&2; return 1; } local docker_img_url=0.0.0.0:5000/$image_label local dockerfile_dir=$2 @@ -400,19 +400,25 @@ grpc_dockerfile_install() { [[ $cache == "cache=1" ]] && { cache_opt=''; } [[ $cache == "cache=true" ]] && { cache_opt=''; } - [[ -d $dockerfile_dir ]] || { echo "not a valid dir: $dockerfile_dir"; return 1; } + [[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; } + + # For grpc/base, sync the ssh key into the .ssh dir in the dockerfile context + + [[ $image_label == "grpc/base" ]] && { + grpc_docker_sync_github_key $dockerfile_dir/.ssh || return 1; + } # TODO(temiola): maybe make cache/no-cache a func option? sudo docker build $cache_opt -t $image_label $dockerfile_dir || { - echo "docker op error: build of $image_label <- $dockerfile_dir" + echo "$FUNCNAME:: build of $image_label <- $dockerfile_dir" return 1 } sudo docker tag $image_label $docker_img_url || { - echo "docker op error: tag of $docker_img_url" + echo "$FUNCNAME: failed to tag $docker_img_url as $image_label" return 1 } sudo docker push $docker_img_url || { - echo "docker op error: push of $docker_img_url" + echo "$FUNCNAME: failed to push $docker_img_url" return 1 } } @@ -428,3 +434,31 @@ grpc_dockerfile_install() { grpc_dockerfile_refresh() { grpc_dockerfile_install "$@" } + +# grpc_docker_sync_github_key. +# +# Copies the docker github key from GCS to the target dir +# +# call-seq: +# grpc_docker_sync_github_key <target_dir> +grpc_docker_sync_github_key() { + local target_dir=$1 + [[ -n $target_dir ]] || { echo "$FUNCNAME: missing arg: target_dir" >&2; return 1; } + + # determine the admin root; the parent of the dockerfile root, + local gs_dockerfile_root=$(load_metadata "attributes/gs_dockerfile_root") + [[ -n $gs_dockerfile_root ]] || { + echo "$FUNCNAME: missing metadata: gs_dockerfile_root" >&2 + return 1 + } + local gcs_admin_root=$(dirname $gs_dockerfile_root) + + # cp the file from gsutil to a known local area + local gcs_key_path=$gcs_admin_root/github/ssh_key + local local_key_path=$target_dir/github.rsa + mkdir -p $target_dir || { + echo "$FUNCNAME: could not create dir: $target_dir" 1>&2 + return 1 + } + gsutil cp $src $gcs_key_path $local_key_path +} |