diff options
Diffstat (limited to 'kokoro')
56 files changed, 1350 insertions, 19 deletions
diff --git a/kokoro/linux/32-bit/Dockerfile b/kokoro/linux/32-bit/Dockerfile new file mode 100644 index 00000000..1278889f --- /dev/null +++ b/kokoro/linux/32-bit/Dockerfile @@ -0,0 +1,143 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM 32bit/debian:latest + +# Apt source for php +RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07 + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + unzip \ + # -- For python -- + python-setuptools \ + python-pip \ + python-dev \ + # -- For C++ benchmarks -- + cmake \ + # -- For PHP -- + php5.5 \ + php5.5-dev \ + php5.5-xml \ + php5.6 \ + php5.6-dev \ + php5.6-xml \ + php7.0 \ + php7.0-dev \ + php7.0-xml \ + phpunit \ + valgrind \ + libxml2-dev \ + && apt-get clean + +################## +# PHP dependencies. +RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror +RUN mv mirror php-5.5.38.tar.bz2 +RUN tar -xvf php-5.5.38.tar.bz2 +RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \ + make && make install && make clean && cd .. +RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \ + make && make install && make clean && cd .. + +RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror +RUN mv mirror php-5.6.30.tar.bz2 +RUN tar -xvf php-5.6.30.tar.bz2 +RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \ + make && make install && cd .. +RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror +RUN mv mirror php-7.0.18.tar.bz2 +RUN tar -xvf php-7.0.18.tar.bz2 +RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \ + make && make install && cd .. +RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror +RUN mv mirror php-7.1.4.tar.bz2 +RUN tar -xvf php-7.1.4.tar.bz2 +RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \ + make && make install && cd .. +RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \ + make && make install && cd .. + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" +RUN php composer-setup.php +RUN mv composer.phar /usr/bin/composer +RUN php -r "unlink('composer-setup.php');" +RUN composer config -g -- disable-tls true +RUN composer config -g -- secure-http false +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf/php && \ + git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \ + ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.5 && \ + ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.6 && \ + ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.0 && \ + ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.1 + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/kokoro/linux/32-bit/build.sh b/kokoro/linux/32-bit/build.sh new file mode 100755 index 00000000..04383a57 --- /dev/null +++ b/kokoro/linux/32-bit/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request 32" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/32-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="php_all_32" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/32-bit/continuous.cfg b/kokoro/linux/32-bit/continuous.cfg new file mode 100644 index 00000000..28b66c07 --- /dev/null +++ b/kokoro/linux/32-bit/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/32-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/32-bit/presubmit.cfg b/kokoro/linux/32-bit/presubmit.cfg new file mode 100644 index 00000000..28b66c07 --- /dev/null +++ b/kokoro/linux/32-bit/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/32-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/64-bit/Dockerfile b/kokoro/linux/64-bit/Dockerfile new file mode 100644 index 00000000..3a279e66 --- /dev/null +++ b/kokoro/linux/64-bit/Dockerfile @@ -0,0 +1,244 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Apt source for old Python versions. +RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C + +# Apt source for Oracle Java. +RUN echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections + +# Apt source for Mono +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ + echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +# Apt source for php +RUN echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu trusty main" | tee /etc/apt/sources.list.d/various-php.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F4FCBB07 + +# Install dotnet SDK based on https://www.microsoft.com/net/core#debian +# (Ubuntu instructions need apt to support https) +RUN apt-get update && apt-get install -y --force-yes curl libunwind8 gettext && \ + curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=847105 && \ + mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet && \ + ln -s /opt/dotnet/dotnet /usr/local/bin + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + # -- For csharp -- + mono-devel \ + referenceassemblies-pcl \ + nunit \ + # -- For all Java builds -- \ + maven \ + # -- For java_jdk6 -- \ + # oops! not in jessie. too old? openjdk-6-jdk \ + # -- For java_jdk7 -- \ + openjdk-7-jdk \ + # -- For java_oracle7 -- \ + oracle-java7-installer \ + # -- For python / python_cpp -- \ + python-setuptools \ + python-pip \ + python-dev \ + python2.6-dev \ + python3.3-dev \ + python3.4-dev \ + # -- For Ruby -- + ruby \ + # -- For C++ benchmarks -- + cmake \ + # -- For PHP -- + php5.6 \ + php5.6-dev \ + php5.6-xml \ + php7.0 \ + php7.0-dev \ + php7.0-xml \ + phpunit \ + valgrind \ + libxml2-dev \ + && apt-get clean + +################## +# C# dependencies + +RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1, Ruby 2.2 and JRuby 1.7 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm install ruby-2.2" +RUN /bin/bash -l -c "rvm install jruby-1.7" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Java dependencies + +# This step requires compiling protoc. :( + +ENV MAVEN_REPO /var/maven_local_repository +ENV MVN mvn --batch-mode + +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + git reset --hard 129a6e2aca95dcfb6c3e717d7b9cca1f104fde39 && \ + ./autogen.sh && \ + ./configure && \ + make -j4 && \ + cd java && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ + cd ../javanano && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO + +################## +# PHP dependencies. +RUN wget http://am1.php.net/get/php-5.5.38.tar.bz2/from/this/mirror +RUN mv mirror php-5.5.38.tar.bz2 +RUN tar -xvf php-5.5.38.tar.bz2 +RUN cd php-5.5.38 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.5-zts && \ + make && make install && cd .. +RUN cd php-5.5.38 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.5 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-5.6.30.tar.bz2/from/this/mirror +RUN mv mirror php-5.6.30.tar.bz2 +RUN tar -xvf php-5.6.30.tar.bz2 +RUN cd php-5.6.30 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-5.6-zts && \ + make && make install && cd .. +RUN cd php-5.6.30 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-5.6 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.0.18.tar.bz2/from/this/mirror +RUN mv mirror php-7.0.18.tar.bz2 +RUN tar -xvf php-7.0.18.tar.bz2 +RUN cd php-7.0.18 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.0-zts && \ + make && make install && cd .. +RUN cd php-7.0.18 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.0 && \ + make && make install && cd .. + +RUN wget http://am1.php.net/get/php-7.1.4.tar.bz2/from/this/mirror +RUN mv mirror php-7.1.4.tar.bz2 +RUN tar -xvf php-7.1.4.tar.bz2 +RUN cd php-7.1.4 && ./configure --enable-maintainer-zts --prefix=/usr/local/php-7.1-zts && \ + make && make install && cd .. +RUN cd php-7.1.4 && make clean && ./configure --enable-bcmath --prefix=/usr/local/php-7.1 && \ + make && make install && cd .. + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" +RUN php composer-setup.php +RUN mv composer.phar /usr/bin/composer +RUN php -r "unlink('composer-setup.php');" +RUN composer config -g -- disable-tls true +RUN composer config -g -- secure-http false +RUN cd /tmp && \ + rm -rf protobuf && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + git reset --hard 49b44bff2b6257a119f9c6a342d6151c736586b8 && \ + cd php && \ + ln -sfn /usr/local/php-5.5/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.5/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.5/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.5 && \ + ln -sfn /usr/local/php-5.6/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-5.6/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-5.6/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-5.6 && \ + ln -sfn /usr/local/php-7.0/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.0/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.0/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.0 && \ + ln -sfn /usr/local/php-7.1/bin/php /usr/bin/php && \ + ln -sfn /usr/local/php-7.1/bin/php-config /usr/bin/php-config && \ + ln -sfn /usr/local/php-7.1/bin/phpize /usr/bin/phpize && \ + composer install && \ + mv vendor /usr/local/vendor-7.1 + +################## +# Go dependencies. +RUN apt-get install -y \ + # -- For go -- \ + golang + +################## +# Javascript dependencies. +RUN apt-get install -y \ + # -- For javascript -- \ + npm + +################## +# Python 3.5 3.6 dependencies. +RUN apt-get clean && apt-get update && apt-get install -y --force-yes \ + python3.5-dev \ + python3.6-dev \ + && apt-get clean + +# On Debian/Ubuntu, nodejs binary is named 'nodejs' because the name 'node' +# is taken by another legacy binary. We don't have that legacy binary and +# npm expects the binary to be named 'node', so we just create a symbol +# link here. +RUN ln -s `which nodejs` /usr/bin/node + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/kokoro/linux/64-bit/build.sh b/kokoro/linux/64-bit/build.sh new file mode 100755 index 00000000..48ddbce5 --- /dev/null +++ b/kokoro/linux/64-bit/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="csharp java_jdk7 javanano_jdk7 java_oracle7 javanano_oracle7 python python_cpp ruby_all javascript golang php_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/64-bit/continuous.cfg b/kokoro/linux/64-bit/continuous.cfg new file mode 100644 index 00000000..3a4faac1 --- /dev/null +++ b/kokoro/linux/64-bit/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/64-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/64-bit/presubmit.cfg b/kokoro/linux/64-bit/presubmit.cfg new file mode 100644 index 00000000..3a4faac1 --- /dev/null +++ b/kokoro/linux/64-bit/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/64-bit/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh new file mode 100755 index 00000000..d8aea724 --- /dev/null +++ b/kokoro/linux/bazel/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +git submodule update --init --recursive +bazel test :protobuf_test diff --git a/kokoro/macos/jruby/continuous.cfg b/kokoro/linux/bazel/continuous.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/continuous.cfg +++ b/kokoro/linux/bazel/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/macos/jruby/presubmit.cfg b/kokoro/linux/bazel/presubmit.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/presubmit.cfg +++ b/kokoro/linux/bazel/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh new file mode 100755 index 00000000..af5b299e --- /dev/null +++ b/kokoro/linux/benchmark/build.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Change to repo root +cd $(dirname $0)/../../.. + +export OUTPUT_DIR=testoutput +oldpwd=`pwd` + +# tcmalloc +if [ ! -f gperftools/.libs/libtcmalloc.so ]; then + git clone https://github.com/gperftools/gperftools.git + cd gperftools + ./autogen.sh + ./configure + make -j8 + cd .. +fi + +# download datasets for benchmark +cd benchmarks +./download_data.sh +datasets=`find . -type f -name "dataset.*.pb"` +cd $oldpwd + +# build Python protobuf +./autogen.sh +./configure CXXFLAGS="-fPIC -O2" +make -j8 +cd python +python setup.py build --cpp_implementation +pip install . --user + + +# build and run Python benchmark +cd ../benchmarks +make python-pure-python-benchmark +make python-cpp-reflection-benchmark +make -j8 python-cpp-generated-code-benchmark +echo "[" > tmp/python_result.json +echo "benchmarking pure python..." +./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp reflection..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp generated code..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json +echo "]" >> "tmp/python_result.json" +cd $oldpwd + +# build CPP protobuf +./configure +make clean && make -j8 + +# build Java protobuf +cd java +mvn package +cd .. + +# build CPP benchmark +cd benchmarks +mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp +echo "benchmarking cpp..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets +cd $oldpwd + +# build go protobuf +export PATH="`pwd`/src:$PATH" +export GOPATH="$HOME/gocode" +mkdir -p "$GOPATH/src/github.com/google" +rm -f "$GOPATH/src/github.com/google/protobuf" +ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" +export PATH="$GOPATH/bin:$PATH" +go get github.com/golang/protobuf/protoc-gen-go + +# build go benchmark +cd benchmarks +make go-benchmark +echo "benchmarking go..." +./go-benchmark $datasets > tmp/go_result.txt + +# build java benchmark +make java-benchmark +echo "benchmarking java..." +./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets + +# upload result to bq +make python_add_init +env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ + -python="../tmp/python_result.json" -go="../tmp/go_result.txt" + +cd $oldpwd diff --git a/kokoro/linux/benchmark/continuous.cfg b/kokoro/linux/benchmark/continuous.cfg new file mode 100755 index 00000000..a3558c65 --- /dev/null +++ b/kokoro/linux/benchmark/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/benchmark/build.sh" +timeout_mins: 240 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/build_and_run_docker.sh b/kokoro/linux/build_and_run_docker.sh new file mode 100755 index 00000000..6a1f327a --- /dev/null +++ b/kokoro/linux/build_and_run_docker.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Builds docker image and runs a command under it. +# This is a generic script that is configured with the following variables: +# +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) +# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# $@ - Extra args to pass to docker run + + +set -ex + +cd $(dirname $0)/../.. +git_root=$(pwd) +cd - + +# Use image name based on Dockerfile sha1 +DOCKERHUB_ORGANIZATION=grpctesting/protobuf +DOCKER_IMAGE_NAME=${DOCKERHUB_ORGANIZATION}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Pull dockerimage from Dockerhub +docker pull $DOCKER_IMAGE_NAME + +# Ensure existence of ccache directory +CCACHE_DIR=/tmp/protobuf-ccache +mkdir -p $CCACHE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +echo $git_root + +# Run command inside docker +docker run \ + "$@" \ + -e CCACHE_DIR=$CCACHE_DIR \ + -e EXTERNAL_GIT_ROOT="/var/local/kokoro/protobuf" \ + -e TEST_SET="$TEST_SET" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/kokoro/protobuf:ro" \ + -v $CCACHE_DIR:$CCACHE_DIR \ + -w /var/local/git/protobuf \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/kokoro/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "${git_root}/kokoro" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/kokoro/linux/golang/build.sh b/kokoro/linux/golang/build.sh new file mode 100755 index 00000000..79cddff1 --- /dev/null +++ b/kokoro/linux/golang/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="golang" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/golang/continuous.cfg b/kokoro/linux/golang/continuous.cfg new file mode 100644 index 00000000..2282f78f --- /dev/null +++ b/kokoro/linux/golang/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/golang/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/golang/presubmit.cfg b/kokoro/linux/golang/presubmit.cfg new file mode 100644 index 00000000..2282f78f --- /dev/null +++ b/kokoro/linux/golang/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/golang/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_jdk7/build.sh b/kokoro/linux/java_jdk7/build.sh new file mode 100755 index 00000000..0a5604b4 --- /dev/null +++ b/kokoro/linux/java_jdk7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="java_jdk7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/java_jdk7/continuous.cfg b/kokoro/linux/java_jdk7/continuous.cfg new file mode 100644 index 00000000..07a9df5f --- /dev/null +++ b/kokoro/linux/java_jdk7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_jdk7/presubmit.cfg b/kokoro/linux/java_jdk7/presubmit.cfg new file mode 100644 index 00000000..07a9df5f --- /dev/null +++ b/kokoro/linux/java_jdk7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_jdk7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_oracle7/build.sh b/kokoro/linux/java_oracle7/build.sh new file mode 100755 index 00000000..9ab90dad --- /dev/null +++ b/kokoro/linux/java_oracle7/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="java_oracle7" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/java_oracle7/continuous.cfg b/kokoro/linux/java_oracle7/continuous.cfg new file mode 100644 index 00000000..610469b4 --- /dev/null +++ b/kokoro/linux/java_oracle7/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/java_oracle7/presubmit.cfg b/kokoro/linux/java_oracle7/presubmit.cfg new file mode 100644 index 00000000..610469b4 --- /dev/null +++ b/kokoro/linux/java_oracle7/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/java_oracle7/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javascript/build.sh b/kokoro/linux/javascript/build.sh new file mode 100755 index 00000000..c2585fb7 --- /dev/null +++ b/kokoro/linux/javascript/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="javascript" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javascript/continuous.cfg b/kokoro/linux/javascript/continuous.cfg new file mode 100644 index 00000000..b42f522f --- /dev/null +++ b/kokoro/linux/javascript/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/javascript/presubmit.cfg b/kokoro/linux/javascript/presubmit.cfg new file mode 100644 index 00000000..b42f522f --- /dev/null +++ b/kokoro/linux/javascript/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/javascript/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/make_test_output.py b/kokoro/linux/make_test_output.py new file mode 100644 index 00000000..98536853 --- /dev/null +++ b/kokoro/linux/make_test_output.py @@ -0,0 +1,94 @@ +"""Gathers output from test runs and create an XML file in JUnit format. + +The output files from the individual tests have been written in a directory +structure like: + + $DIR/joblog (output from "parallel --joblog joblog") + $DIR/logs/1/cpp/stdout + $DIR/logs/1/cpp/stderr + $DIR/logs/1/csharp/stdout + $DIR/logs/1/csharp/stderr + $DIR/logs/1/java_jdk7/stdout + $DIR/logs/1/java_jdk7/stderr + etc. + +This script bundles them into a single output XML file so Jenkins can show +detailed test results. It runs as the last step before the Jenkins build +finishes. +""" + +import os +import sys +from yattag import Doc +from collections import defaultdict + + +def readtests(basedir): + tests = defaultdict(dict) + + # Sample input (note: separators are tabs). + # + # Seq Host Starttime Runtime Send Receive Exitval Signal Command + # 1 : 1456263838.313 0.005 0 0 0 0 echo A + with open(basedir + "/joblog") as jobs: + firstline = next(jobs) + for line in jobs: + values = line.split("\t") + + name = values[8].split()[-1] + test = tests[name] + test["name"] = name + test["time"] = values[3] + + exitval = values[6] + if int(exitval): + # We don't have a more specific message. User should look at stderr. + test["failure"] = "TEST FAILURE" + else: + test["failure"] = False + + for testname in os.listdir(basedir + "/logs/1"): + test = tests[testname] + + with open(basedir + "/logs/1/" + testname + "/stdout") as f: + test["stdout"] = f.read() + + with open(basedir + "/logs/1/" + testname + "/stderr") as f: + test["stderr"] = f.read() + + # The cpp test is special since it doesn't run under parallel so doesn't show + # up in the job log. + tests["cpp"]["name"] = "cpp" + + with open(basedir + '/logs/1/cpp/build_time', 'r') as f: + tests["cpp"]["time"] = f.read().strip() + tests["cpp"]["failure"] = False + + ret = tests.values() + ret.sort(key=lambda x: x["name"]) + + return ret + + +def genxml(tests): + doc, tag, text = Doc().tagtext() + + with tag("testsuites"): + with tag("testsuite", name="Protobuf Tests"): + for test in tests: + with tag("testcase", name=test["name"], classname=test["name"], + time=test["time"]): + with tag("system-out"): + text(test["stdout"]) + with tag("system-err"): + text(test["stderr"]) + if test["failure"]: + with tag("failure"): + text(test["failure"]) + + return doc.getvalue() + + +sys.stderr.write("make_test_output.py: writing XML from directory: " + + sys.argv[1] + "\n") +print(genxml(readtests(sys.argv[1]))) diff --git a/kokoro/linux/php_all/build.sh b/kokoro/linux/php_all/build.sh new file mode 100755 index 00000000..e9ab84cb --- /dev/null +++ b/kokoro/linux/php_all/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="php_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/php_all/continuous.cfg b/kokoro/linux/php_all/continuous.cfg new file mode 100644 index 00000000..dfa8037e --- /dev/null +++ b/kokoro/linux/php_all/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/php_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/php_all/presubmit.cfg b/kokoro/linux/php_all/presubmit.cfg new file mode 100644 index 00000000..dfa8037e --- /dev/null +++ b/kokoro/linux/php_all/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/php_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/prepare_build_linux_rc b/kokoro/linux/prepare_build_linux_rc index 4c3f255d..f64ea952 100644 --- a/kokoro/linux/prepare_build_linux_rc +++ b/kokoro/linux/prepare_build_linux_rc @@ -3,7 +3,11 @@ # Source this rc script to prepare the environment for Linux builds # Set up dotnet -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893 +sudo sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-trusty-prod trusty main" > /etc/apt/sources.list.d/dotnetdev.list' +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EB3E94ADBE1229CF sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.4 +# We use the .NET Core SDK 2.x to build... +sudo apt-get install -y dotnet-sdk-2.0.3 +# But we also need the 1.x framework to test against, as we +# target netstandard1.x +sudo apt-get install -y dotnet-sharedframework-microsoft.netcore.app-1.0.5 diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh new file mode 100755 index 00000000..df3636cc --- /dev/null +++ b/kokoro/linux/pull_request_in_docker.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# This is the script that runs inside Docker, once the image has been built, +# to execute all tests for the "pull request" project. + +WORKSPACE_BASE=`pwd` +MY_DIR="$(dirname "$0")" +TEST_SCRIPT=$MY_DIR/../../tests.sh +BUILD_DIR=/tmp/protobuf + +set -e # exit immediately on error +set -x # display all commands + +# The protobuf repository is mounted into our Docker image, but read-only. +# We clone into a directory inside Docker (this is faster than cp). +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone /var/local/kokoro/protobuf +cd protobuf + +# Initialize any submodules: +git submodule update --init --recursive + +# Set up the directory where our test output is going to go. +OUTPUT_DIR=`mktemp -d` +LOG_OUTPUT_DIR=$OUTPUT_DIR/logs +mkdir -p $LOG_OUTPUT_DIR/1/cpp + +################################################################################ +# cpp build needs to run first, non-parallelized, so that protoc is available +# for other builds. + +# Output filenames to follow the overall scheme used by parallel, ie: +# $DIR/logs/1/cpp/stdout +# $DIR/logs/1/cpp/stderr +# $DIR/logs/1/csharp/stdout +# $DIR/logs/1/csharp/stderr +# $DIR/logs/1/java_jdk7/stdout +# $DIR/logs/1/java_jdk7/stderr +CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout +CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr + +# Time the C++ build, so we can put this info in the test output. +# It's important that we get /usr/bin/time (which supports -f and -o) and not +# the bash builtin "time" which doesn't. +TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" + +$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) + +# Other tests are run in parallel. TEST_SET is defined in +# buildcmds/pull_request{_32}.sh + +parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ + $TEST_SET \ + || true # Process test results even if tests fail. + +cat $OUTPUT_DIR/joblog + +# The directory that is copied from Docker back into the Kokoro workspace. +COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput +mkdir -p $COPY_FROM_DOCKER +TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/sponge_log.xml + +# Process all the output files from "parallel" and package them into a single +# .xml file with detailed, broken-down test output. +python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE + +ls -l $TESTOUTPUT_XML_FILE diff --git a/kokoro/linux/python/build.sh b/kokoro/linux/python/build.sh new file mode 100755 index 00000000..86964f55 --- /dev/null +++ b/kokoro/linux/python/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python/continuous.cfg b/kokoro/linux/python/continuous.cfg new file mode 100644 index 00000000..e2fc4136 --- /dev/null +++ b/kokoro/linux/python/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python/presubmit.cfg b/kokoro/linux/python/presubmit.cfg new file mode 100644 index 00000000..e2fc4136 --- /dev/null +++ b/kokoro/linux/python/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python_cpp/build.sh b/kokoro/linux/python_cpp/build.sh new file mode 100755 index 00000000..add64651 --- /dev/null +++ b/kokoro/linux/python_cpp/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="python_cpp" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/python_cpp/continuous.cfg b/kokoro/linux/python_cpp/continuous.cfg new file mode 100644 index 00000000..b1b0e550 --- /dev/null +++ b/kokoro/linux/python_cpp/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python_cpp/presubmit.cfg b/kokoro/linux/python_cpp/presubmit.cfg new file mode 100644 index 00000000..b1b0e550 --- /dev/null +++ b/kokoro/linux/python_cpp/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_cpp/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/ruby_all/build.sh b/kokoro/linux/ruby_all/build.sh new file mode 100755 index 00000000..ea758958 --- /dev/null +++ b/kokoro/linux/ruby_all/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERFILE_DIR=kokoro/linux/64-bit +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="ruby_all" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/ruby_all/continuous.cfg b/kokoro/linux/ruby_all/continuous.cfg new file mode 100644 index 00000000..02d76437 --- /dev/null +++ b/kokoro/linux/ruby_all/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/ruby_all/presubmit.cfg b/kokoro/linux/ruby_all/presubmit.cfg new file mode 100644 index 00000000..02d76437 --- /dev/null +++ b/kokoro/linux/ruby_all/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby_all/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/macos/jruby/build.sh b/kokoro/macos/jruby/build.sh deleted file mode 100755 index c82eaebf..00000000 --- a/kokoro/macos/jruby/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -./tests.sh jruby diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index dfe66ff8..fb975523 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -5,7 +5,7 @@ ## # Select Xcode version -export DEVELOPER_DIR=/Applications/Xcode_8.1.app/Contents/Developer +export DEVELOPER_DIR=/Applications/Xcode_9.1.app/Contents/Developer ## # Select C/C++ compilers diff --git a/kokoro/release/linux/build_artifacts.sh b/kokoro/release/linux/build_artifacts.sh new file mode 100755 index 00000000..ae7dc813 --- /dev/null +++ b/kokoro/release/linux/build_artifacts.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../.. + +source kokoro/release/linux/prepare_build.sh + +# ruby environment +source kokoro/release/linux/ruby/ruby_build_environment.sh + +# build artifacts +bash kokoro/release/linux/ruby/ruby_build.sh diff --git a/kokoro/release/linux/prepare_build.sh b/kokoro/release/linux/prepare_build.sh new file mode 100755 index 00000000..d3ff8764 --- /dev/null +++ b/kokoro/release/linux/prepare_build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Move docker's storage location to scratch disk so we don't run out of space. +echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker +# Use container registry mirror for pulling docker images (should make downloads faster) +# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring +echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker +sudo service docker restart + +# Download Docker images from DockerHub +export DOCKERHUB_ORGANIZATION=grpctesting + +# All artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + diff --git a/kokoro/release/linux/release.cfg b/kokoro/release/linux/release.cfg new file mode 100644 index 00000000..5dae7019 --- /dev/null +++ b/kokoro/release/linux/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Linux release builds +build_file: "protobuf/kokoro/release/linux/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/linux/ruby/ruby_build.sh b/kokoro/release/linux/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/linux/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/linux/ruby/ruby_build_environment.sh b/kokoro/release/linux/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..ea04f905 --- /dev/null +++ b/kokoro/release/linux/ruby/ruby_build_environment.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +gem install bundler --update +set -ex diff --git a/kokoro/release/macos/build_artifacts.sh b/kokoro/release/macos/build_artifacts.sh new file mode 100755 index 00000000..23135ca3 --- /dev/null +++ b/kokoro/release/macos/build_artifacts.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../.. + +# all artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +# ruby environment +bash kokoro/release/macos/ruby/ruby_build_environment.sh + +gem install rubygems-update +update_rubygems + +# build artifacts +bash kokoro/release/macos/ruby/ruby_build.sh diff --git a/kokoro/release/macos/release.cfg b/kokoro/release/macos/release.cfg new file mode 100644 index 00000000..35d36dee --- /dev/null +++ b/kokoro/release/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX release builds +build_file: "protobuf/kokoro/release/macos/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/macos/ruby/ruby_build.sh b/kokoro/release/macos/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/macos/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/macos/ruby/ruby_build_environment.sh b/kokoro/release/macos/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..e9c619cd --- /dev/null +++ b/kokoro/release/macos/ruby/ruby_build_environment.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -ex + +set +ex # rvm script is very verbose and exits with errorcode +source $HOME/.rvm/scripts/rvm +set -e # rvm commands are very verbose +time rvm install 2.5.0 +rvm use 2.5.0 --default +gem install rake-compiler --no-ri --no-doc +gem install bundler --no-ri --no-doc +rvm osx-ssl-certs status all +rvm osx-ssl-certs update all +set -ex + +rm -rf ~/.rake-compiler + +CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) + +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" + +# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details +patch "$CROSS_RUBY" << EOF +--- cross-ruby.rake 2018-04-10 11:32:16.000000000 -0700 ++++ patched 2018-04-10 11:40:25.000000000 -0700 +@@ -133,8 +133,10 @@ + "--host=#{MINGW_HOST}", + "--target=#{MINGW_TARGET}", + "--build=#{RUBY_BUILD}", +- '--enable-shared', ++ '--enable-static', ++ '--disable-shared', + '--disable-install-doc', ++ '--without-gmp', + '--with-ext=' + ] + +@@ -151,6 +153,7 @@ + # make + file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + chdir File.dirname(t.prerequisites.first) do ++ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make + sh MAKE + end + end +EOF + +MAKE="make -j8" + +for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done + +sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" +mv "$CROSS_RUBY" ~/.rake-compiler/config.yml + diff --git a/kokoro/release/protoc/macos/build.sh b/kokoro/release/protoc/macos/build.sh new file mode 100644 index 00000000..db9d3140 --- /dev/null +++ b/kokoro/release/protoc/macos/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -x +CXXFLAGS_COMMON="-DNDEBUG -mmacosx-version-min=10.9" + +cd github/protobuf +./autogen.sh + +mkdir build32 && cd build32 +export CXXFLAGS="$CXXFLAGS_COMMON -m32" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. + +mkdir build64 && cd build64 +export CXXFLAGS="$CXXFLAGS_COMMON -m64" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. diff --git a/kokoro/release/protoc/macos/release.cfg b/kokoro/release/protoc/macos/release.cfg new file mode 100644 index 00000000..da903d03 --- /dev/null +++ b/kokoro/release/protoc/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX protoc release builds +build_file: "protobuf/kokoro/release/protoc/macos/build.sh" + +action { + define_artifacts { + regex: "**/protoc" + } +} diff --git a/kokoro/release/protoc/windows/build.bat b/kokoro/release/protoc/windows/build.bat new file mode 100644 index 00000000..ae0fa94a --- /dev/null +++ b/kokoro/release/protoc/windows/build.bat @@ -0,0 +1,27 @@ +set PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% +set generator32=Visual Studio 14 +set generator64=Visual Studio 14 Win64 +set vcplatform32=win32 +set vcplatform64=x64 +set configuration=Release + +echo Building protoc +cd github\protobuf + +mkdir build32 +cd build32 +cmake -G "%generator32%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform32% || goto error +cd .. + +mkdir build64 +cd build64 +cmake -G "%generator64%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform64% || goto error +cd .. + +goto :EOF + +:error +echo Failed! +exit /b %ERRORLEVEL% diff --git a/kokoro/release/protoc/windows/release.cfg b/kokoro/release/protoc/windows/release.cfg new file mode 100644 index 00000000..0d0da8f1 --- /dev/null +++ b/kokoro/release/protoc/windows/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Windows protoc release builds +build_file: "protobuf/kokoro/release/protoc/windows/build.bat" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} |