diff options
Diffstat (limited to 'kokoro')
25 files changed, 336 insertions, 84 deletions
diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh new file mode 100755 index 00000000..d8aea724 --- /dev/null +++ b/kokoro/linux/bazel/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +git submodule update --init --recursive +bazel test :protobuf_test diff --git a/kokoro/macos/jruby/continuous.cfg b/kokoro/linux/bazel/continuous.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/continuous.cfg +++ b/kokoro/linux/bazel/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/macos/jruby/presubmit.cfg b/kokoro/linux/bazel/presubmit.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/presubmit.cfg +++ b/kokoro/linux/bazel/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh new file mode 100755 index 00000000..af5b299e --- /dev/null +++ b/kokoro/linux/benchmark/build.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# +# Change to repo root +cd $(dirname $0)/../../.. + +export OUTPUT_DIR=testoutput +oldpwd=`pwd` + +# tcmalloc +if [ ! -f gperftools/.libs/libtcmalloc.so ]; then + git clone https://github.com/gperftools/gperftools.git + cd gperftools + ./autogen.sh + ./configure + make -j8 + cd .. +fi + +# download datasets for benchmark +cd benchmarks +./download_data.sh +datasets=`find . -type f -name "dataset.*.pb"` +cd $oldpwd + +# build Python protobuf +./autogen.sh +./configure CXXFLAGS="-fPIC -O2" +make -j8 +cd python +python setup.py build --cpp_implementation +pip install . --user + + +# build and run Python benchmark +cd ../benchmarks +make python-pure-python-benchmark +make python-cpp-reflection-benchmark +make -j8 python-cpp-generated-code-benchmark +echo "[" > tmp/python_result.json +echo "benchmarking pure python..." +./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp reflection..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json +echo "," >> "tmp/python_result.json" +echo "benchmarking python cpp generated code..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json +echo "]" >> "tmp/python_result.json" +cd $oldpwd + +# build CPP protobuf +./configure +make clean && make -j8 + +# build Java protobuf +cd java +mvn package +cd .. + +# build CPP benchmark +cd benchmarks +mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp +echo "benchmarking cpp..." +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./cpp-benchmark --benchmark_min_time=5.0 --benchmark_out_format=json --benchmark_out="tmp/cpp_result.json" $datasets +cd $oldpwd + +# build go protobuf +export PATH="`pwd`/src:$PATH" +export GOPATH="$HOME/gocode" +mkdir -p "$GOPATH/src/github.com/google" +rm -f "$GOPATH/src/github.com/google/protobuf" +ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" +export PATH="$GOPATH/bin:$PATH" +go get github.com/golang/protobuf/protoc-gen-go + +# build go benchmark +cd benchmarks +make go-benchmark +echo "benchmarking go..." +./go-benchmark $datasets > tmp/go_result.txt + +# build java benchmark +make java-benchmark +echo "benchmarking java..." +./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets + +# upload result to bq +make python_add_init +env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ + -python="../tmp/python_result.json" -go="../tmp/go_result.txt" + +cd $oldpwd diff --git a/kokoro/linux/javanano_jdk7/continuous.cfg b/kokoro/linux/benchmark/continuous.cfg index 72936484..a3558c65 100644..100755 --- a/kokoro/linux/javanano_jdk7/continuous.cfg +++ b/kokoro/linux/benchmark/continuous.cfg @@ -1,8 +1,8 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh" -timeout_mins: 120 +build_file: "protobuf/kokoro/linux/benchmark/build.sh" +timeout_mins: 240 action { define_artifacts { diff --git a/kokoro/linux/javanano_jdk7/build.sh b/kokoro/linux/javanano_jdk7/build.sh deleted file mode 100755 index 2fc06475..00000000 --- a/kokoro/linux/javanano_jdk7/build.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERFILE_DIR=kokoro/linux/64-bit -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="javanano_jdk7" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javanano_jdk7/presubmit.cfg b/kokoro/linux/javanano_jdk7/presubmit.cfg deleted file mode 100644 index 72936484..00000000 --- a/kokoro/linux/javanano_jdk7/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/javanano_jdk7/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/javanano_oracle7/build.sh b/kokoro/linux/javanano_oracle7/build.sh deleted file mode 100755 index 651d17e8..00000000 --- a/kokoro/linux/javanano_oracle7/build.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# This is the top-level script we give to Kokoro as the entry point for -# running the "pull request" project: -# -# This script selects a specific Dockerfile (for building a Docker image) and -# a script to run inside that image. Then we delegate to the general -# build_and_run_docker.sh script. - -# Change to repo root -cd $(dirname $0)/../../.. - -export DOCKERFILE_DIR=kokoro/linux/64-bit -export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh -export OUTPUT_DIR=testoutput -export TEST_SET="javanano_oracle7" -./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/javanano_oracle7/continuous.cfg b/kokoro/linux/javanano_oracle7/continuous.cfg deleted file mode 100644 index 9cac8972..00000000 --- a/kokoro/linux/javanano_oracle7/continuous.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/javanano_oracle7/presubmit.cfg b/kokoro/linux/javanano_oracle7/presubmit.cfg deleted file mode 100644 index 9cac8972..00000000 --- a/kokoro/linux/javanano_oracle7/presubmit.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Config file for running tests in Kokoro - -# Location of the build script in repository -build_file: "protobuf/kokoro/linux/javanano_oracle7/build.sh" -timeout_mins: 120 - -action { - define_artifacts { - regex: "**/sponge_log.xml" - } -} diff --git a/kokoro/linux/pull_request_in_docker.sh b/kokoro/linux/pull_request_in_docker.sh index 23e63961..df3636cc 100755 --- a/kokoro/linux/pull_request_in_docker.sh +++ b/kokoro/linux/pull_request_in_docker.sh @@ -19,6 +19,9 @@ cd $BUILD_DIR git clone /var/local/kokoro/protobuf cd protobuf +# Initialize any submodules: +git submodule update --init --recursive + # Set up the directory where our test output is going to go. OUTPUT_DIR=`mktemp -d` LOG_OUTPUT_DIR=$OUTPUT_DIR/logs diff --git a/kokoro/macos/jruby/build.sh b/kokoro/macos/jruby/build.sh deleted file mode 100755 index c82eaebf..00000000 --- a/kokoro/macos/jruby/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -./tests.sh jruby diff --git a/kokoro/release/linux/build_artifacts.sh b/kokoro/release/linux/build_artifacts.sh new file mode 100755 index 00000000..ae7dc813 --- /dev/null +++ b/kokoro/release/linux/build_artifacts.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../.. + +source kokoro/release/linux/prepare_build.sh + +# ruby environment +source kokoro/release/linux/ruby/ruby_build_environment.sh + +# build artifacts +bash kokoro/release/linux/ruby/ruby_build.sh diff --git a/kokoro/release/linux/prepare_build.sh b/kokoro/release/linux/prepare_build.sh new file mode 100755 index 00000000..d3ff8764 --- /dev/null +++ b/kokoro/release/linux/prepare_build.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Move docker's storage location to scratch disk so we don't run out of space. +echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker +# Use container registry mirror for pulling docker images (should make downloads faster) +# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring +echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker +sudo service docker restart + +# Download Docker images from DockerHub +export DOCKERHUB_ORGANIZATION=grpctesting + +# All artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + diff --git a/kokoro/release/linux/release.cfg b/kokoro/release/linux/release.cfg new file mode 100644 index 00000000..5dae7019 --- /dev/null +++ b/kokoro/release/linux/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Linux release builds +build_file: "protobuf/kokoro/release/linux/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/linux/ruby/ruby_build.sh b/kokoro/release/linux/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/linux/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/linux/ruby/ruby_build_environment.sh b/kokoro/release/linux/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..ea04f905 --- /dev/null +++ b/kokoro/release/linux/ruby/ruby_build_environment.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set +ex +[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh +set -e # rvm commands are very verbose +rvm --default use ruby-2.4.1 +gem install bundler --update +set -ex diff --git a/kokoro/release/macos/build_artifacts.sh b/kokoro/release/macos/build_artifacts.sh new file mode 100755 index 00000000..23135ca3 --- /dev/null +++ b/kokoro/release/macos/build_artifacts.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +# change to repo root +cd $(dirname $0)/../../.. + +# all artifacts come here +mkdir artifacts +export ARTIFACT_DIR=$(pwd)/artifacts + +# ruby environment +bash kokoro/release/macos/ruby/ruby_build_environment.sh + +gem install rubygems-update +update_rubygems + +# build artifacts +bash kokoro/release/macos/ruby/ruby_build.sh diff --git a/kokoro/release/macos/release.cfg b/kokoro/release/macos/release.cfg new file mode 100644 index 00000000..35d36dee --- /dev/null +++ b/kokoro/release/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX release builds +build_file: "protobuf/kokoro/release/macos/build_artifacts.sh" + +action { + define_artifacts { + regex: "github/protobuf/artifacts/**" + } +} diff --git a/kokoro/release/macos/ruby/ruby_build.sh b/kokoro/release/macos/ruby/ruby_build.sh new file mode 100755 index 00000000..cf6b433d --- /dev/null +++ b/kokoro/release/macos/ruby/ruby_build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Build protoc +if test ! -e src/protoc; then + ./autogen.sh + ./configure + make -j4 +fi + +umask 0022 +pushd ruby +bundle install && bundle exec rake gem:native +ls pkg +mv pkg/* $ARTIFACT_DIR +popd diff --git a/kokoro/release/macos/ruby/ruby_build_environment.sh b/kokoro/release/macos/ruby/ruby_build_environment.sh new file mode 100755 index 00000000..e9c619cd --- /dev/null +++ b/kokoro/release/macos/ruby/ruby_build_environment.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -ex + +set +ex # rvm script is very verbose and exits with errorcode +source $HOME/.rvm/scripts/rvm +set -e # rvm commands are very verbose +time rvm install 2.5.0 +rvm use 2.5.0 --default +gem install rake-compiler --no-ri --no-doc +gem install bundler --no-ri --no-doc +rvm osx-ssl-certs status all +rvm osx-ssl-certs update all +set -ex + +rm -rf ~/.rake-compiler + +CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) + +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" + +# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details +patch "$CROSS_RUBY" << EOF +--- cross-ruby.rake 2018-04-10 11:32:16.000000000 -0700 ++++ patched 2018-04-10 11:40:25.000000000 -0700 +@@ -133,8 +133,10 @@ + "--host=#{MINGW_HOST}", + "--target=#{MINGW_TARGET}", + "--build=#{RUBY_BUILD}", +- '--enable-shared', ++ '--enable-static', ++ '--disable-shared', + '--disable-install-doc', ++ '--without-gmp', + '--with-ext=' + ] + +@@ -151,6 +153,7 @@ + # make + file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + chdir File.dirname(t.prerequisites.first) do ++ sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make + sh MAKE + end + end +EOF + +MAKE="make -j8" + +for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done + +sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" +mv "$CROSS_RUBY" ~/.rake-compiler/config.yml + diff --git a/kokoro/release/protoc/macos/build.sh b/kokoro/release/protoc/macos/build.sh new file mode 100644 index 00000000..db9d3140 --- /dev/null +++ b/kokoro/release/protoc/macos/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -x +CXXFLAGS_COMMON="-DNDEBUG -mmacosx-version-min=10.9" + +cd github/protobuf +./autogen.sh + +mkdir build32 && cd build32 +export CXXFLAGS="$CXXFLAGS_COMMON -m32" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. + +mkdir build64 && cd build64 +export CXXFLAGS="$CXXFLAGS_COMMON -m64" +../configure --disable-shared +make -j4 +file src/protoc +otool -L src/protoc | grep dylib +cd .. diff --git a/kokoro/release/protoc/macos/release.cfg b/kokoro/release/protoc/macos/release.cfg new file mode 100644 index 00000000..da903d03 --- /dev/null +++ b/kokoro/release/protoc/macos/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Mac OSX protoc release builds +build_file: "protobuf/kokoro/release/protoc/macos/build.sh" + +action { + define_artifacts { + regex: "**/protoc" + } +} diff --git a/kokoro/release/protoc/windows/build.bat b/kokoro/release/protoc/windows/build.bat new file mode 100644 index 00000000..ae0fa94a --- /dev/null +++ b/kokoro/release/protoc/windows/build.bat @@ -0,0 +1,27 @@ +set PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% +set generator32=Visual Studio 14 +set generator64=Visual Studio 14 Win64 +set vcplatform32=win32 +set vcplatform64=x64 +set configuration=Release + +echo Building protoc +cd github\protobuf + +mkdir build32 +cd build32 +cmake -G "%generator32%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform32% || goto error +cd .. + +mkdir build64 +cd build64 +cmake -G "%generator64%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake +msbuild protobuf.sln /p:Platform=%vcplatform64% || goto error +cd .. + +goto :EOF + +:error +echo Failed! +exit /b %ERRORLEVEL% diff --git a/kokoro/release/protoc/windows/release.cfg b/kokoro/release/protoc/windows/release.cfg new file mode 100644 index 00000000..0d0da8f1 --- /dev/null +++ b/kokoro/release/protoc/windows/release.cfg @@ -0,0 +1,8 @@ +# Configuration for Windows protoc release builds +build_file: "protobuf/kokoro/release/protoc/windows/build.bat" + +action { + define_artifacts { + regex: "**/protoc.exe" + } +} |