18 files changed, 249 insertions, 20 deletions
diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh
new file mode 100755
index 00000000..d8aea724
--- /dev/null
+++ b/kokoro/linux/bazel/build.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Build file to set up and run tests
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+git submodule update --init --recursive
+bazel test :protobuf_test
diff --git a/kokoro/macos/jruby/continuous.cfg b/kokoro/linux/bazel/continuous.cfg
index f1310fd3..13cfef15 100644
--- a/kokoro/macos/jruby/continuous.cfg
+++ b/kokoro/linux/bazel/continuous.cfg
@@ -1,5 +1,5 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/macos/jruby/build.sh"
-timeout_mins: 1440
+build_file: "protobuf/kokoro/linux/bazel/build.sh"
+timeout_mins: 15
diff --git a/kokoro/macos/jruby/presubmit.cfg b/kokoro/linux/bazel/presubmit.cfg
index f1310fd3..13cfef15 100644
--- a/kokoro/macos/jruby/presubmit.cfg
+++ b/kokoro/linux/bazel/presubmit.cfg
@@ -1,5 +1,5 @@
 # Config file for running tests in Kokoro
 
 # Location of the build script in repository
-build_file: "protobuf/kokoro/macos/jruby/build.sh"
-timeout_mins: 1440
+build_file: "protobuf/kokoro/linux/bazel/build.sh"
+timeout_mins: 15
diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh
index 750c3e5c..af5b299e 100755
--- a/kokoro/linux/benchmark/build.sh
+++ b/kokoro/linux/benchmark/build.sh
@@ -27,8 +27,8 @@ cd $oldpwd
 ./configure CXXFLAGS="-fPIC -O2"
 make -j8
 cd python
-python setup.py -q build --cpp_implementation
-pip install .
+python setup.py build --cpp_implementation
+pip install . --user
 
 
 # build and run Python benchmark
@@ -41,10 +41,10 @@ echo "benchmarking pure python..."
 ./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp reflection..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp generated code..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
 echo "]" >> "tmp/python_result.json"
 cd $oldpwd
 
@@ -52,6 +52,11 @@ cd $oldpwd
 ./configure
 make clean && make -j8
 
+# build Java protobuf
+cd java
+mvn package
+cd ..
+
 # build CPP benchmark
 cd benchmarks
 mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
@@ -81,7 +86,7 @@ echo "benchmarking java..."
 
 # upload result to bq
 make python_add_init
-python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
+env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
     -python="../tmp/python_result.json" -go="../tmp/go_result.txt"
 
 cd $oldpwd
diff --git a/kokoro/macos/jruby/build.sh b/kokoro/macos/jruby/build.sh
deleted file mode 100755
index c82eaebf..00000000
--- a/kokoro/macos/jruby/build.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#
-# Build file to set up and run tests
-
-# Change to repo root
-cd $(dirname $0)/../../..
-
-# Prepare worker environment to run tests
-source kokoro/macos/prepare_build_macos_rc
-
-./tests.sh jruby
diff --git a/kokoro/release/linux/build_artifacts.sh b/kokoro/release/linux/build_artifacts.sh
new file mode 100755
index 00000000..ae7dc813
--- /dev/null
+++ b/kokoro/release/linux/build_artifacts.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+# change to repo root
+cd $(dirname $0)/../../..
+
+source kokoro/release/linux/prepare_build.sh
+
+# ruby environment
+source kokoro/release/linux/ruby/ruby_build_environment.sh
+
+# build artifacts
+bash kokoro/release/linux/ruby/ruby_build.sh
diff --git a/kokoro/release/linux/prepare_build.sh b/kokoro/release/linux/prepare_build.sh
new file mode 100755
index 00000000..d3ff8764
--- /dev/null
+++ b/kokoro/release/linux/prepare_build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Move docker's storage location to scratch disk so we don't run out of space.
+echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /etc/default/docker
+# Use container registry mirror for pulling docker images (should make downloads faster)
+# See https://cloud.google.com/container-registry/docs/using-dockerhub-mirroring
+echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker
+sudo service docker restart
+
+# Download Docker images from DockerHub
+export DOCKERHUB_ORGANIZATION=grpctesting
+
+# All artifacts come here
+mkdir artifacts
+export ARTIFACT_DIR=$(pwd)/artifacts
+
diff --git a/kokoro/release/linux/release.cfg b/kokoro/release/linux/release.cfg
new file mode 100644
index 00000000..5dae7019
--- /dev/null
+++ b/kokoro/release/linux/release.cfg
@@ -0,0 +1,8 @@
+# Configuration for Linux release builds
+build_file: "protobuf/kokoro/release/linux/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}
diff --git a/kokoro/release/linux/ruby/ruby_build.sh b/kokoro/release/linux/ruby/ruby_build.sh
new file mode 100755
index 00000000..cf6b433d
--- /dev/null
+++ b/kokoro/release/linux/ruby/ruby_build.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Build protoc
+if test ! -e src/protoc; then
+  ./autogen.sh
+  ./configure
+  make -j4
+fi
+
+umask 0022
+pushd ruby
+bundle install && bundle exec rake gem:native
+ls pkg
+mv pkg/* $ARTIFACT_DIR
+popd
diff --git a/kokoro/release/linux/ruby/ruby_build_environment.sh b/kokoro/release/linux/ruby/ruby_build_environment.sh
new file mode 100755
index 00000000..ea04f905
--- /dev/null
+++ b/kokoro/release/linux/ruby/ruby_build_environment.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+gem install bundler --update
+set -ex
diff --git a/kokoro/release/macos/build_artifacts.sh b/kokoro/release/macos/build_artifacts.sh
new file mode 100755
index 00000000..23135ca3
--- /dev/null
+++ b/kokoro/release/macos/build_artifacts.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# change to repo root
+cd $(dirname $0)/../../..
+
+# all artifacts come here
+mkdir artifacts
+export ARTIFACT_DIR=$(pwd)/artifacts
+
+# ruby environment
+bash kokoro/release/macos/ruby/ruby_build_environment.sh
+
+gem install rubygems-update
+update_rubygems
+
+# build artifacts
+bash kokoro/release/macos/ruby/ruby_build.sh
diff --git a/kokoro/release/macos/release.cfg b/kokoro/release/macos/release.cfg
new file mode 100644
index 00000000..35d36dee
--- /dev/null
+++ b/kokoro/release/macos/release.cfg
@@ -0,0 +1,8 @@
+# Configuration for Mac OSX release builds
+build_file: "protobuf/kokoro/release/macos/build_artifacts.sh"
+
+action {
+  define_artifacts {
+    regex: "github/protobuf/artifacts/**"
+  }
+}
diff --git a/kokoro/release/macos/ruby/ruby_build.sh b/kokoro/release/macos/ruby/ruby_build.sh
new file mode 100755
index 00000000..cf6b433d
--- /dev/null
+++ b/kokoro/release/macos/ruby/ruby_build.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Build protoc
+if test ! -e src/protoc; then
+  ./autogen.sh
+  ./configure
+  make -j4
+fi
+
+umask 0022
+pushd ruby
+bundle install && bundle exec rake gem:native
+ls pkg
+mv pkg/* $ARTIFACT_DIR
+popd
diff --git a/kokoro/release/macos/ruby/ruby_build_environment.sh b/kokoro/release/macos/ruby/ruby_build_environment.sh
new file mode 100755
index 00000000..e9c619cd
--- /dev/null
+++ b/kokoro/release/macos/ruby/ruby_build_environment.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -ex
+
+set +ex  # rvm script is very verbose and exits with errorcode
+source $HOME/.rvm/scripts/rvm
+set -e  # rvm commands are very verbose
+time rvm install 2.5.0
+rvm use 2.5.0 --default
+gem install rake-compiler --no-ri --no-doc
+gem install bundler --no-ri --no-doc
+rvm osx-ssl-certs status all
+rvm osx-ssl-certs update all
+set -ex
+
+rm -rf ~/.rake-compiler
+
+CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX)
+
+curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY"
+
+# See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details
+patch "$CROSS_RUBY" << EOF
+--- cross-ruby.rake	2018-04-10 11:32:16.000000000 -0700
++++ patched	2018-04-10 11:40:25.000000000 -0700
+@@ -133,8 +133,10 @@
+     "--host=#{MINGW_HOST}",
+     "--target=#{MINGW_TARGET}",
+     "--build=#{RUBY_BUILD}",
+-    '--enable-shared',
++    '--enable-static',
++    '--disable-shared',
+     '--disable-install-doc',
++    '--without-gmp',
+     '--with-ext='
+   ]
+ 
+@@ -151,6 +153,7 @@
+ # make
+ file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t|
+   chdir File.dirname(t.prerequisites.first) do
++    sh "test -s verconf.h || rm -f verconf.h"  # if verconf.h has size 0, make sure it gets re-built by make
+     sh MAKE
+   end
+ end
+EOF
+
+MAKE="make -j8"
+
+for v in 2.5.0 2.4.0 2.3.0 2.2.2 2.1.6 2.0.0-p645 ; do
+  ccache -c
+  rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE"
+done
+
+sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY"
+mv "$CROSS_RUBY" ~/.rake-compiler/config.yml
+
diff --git a/kokoro/release/protoc/macos/build.sh b/kokoro/release/protoc/macos/build.sh
new file mode 100644
index 00000000..db9d3140
--- /dev/null
+++ b/kokoro/release/protoc/macos/build.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -x
+CXXFLAGS_COMMON="-DNDEBUG -mmacosx-version-min=10.9"
+
+cd github/protobuf
+./autogen.sh
+
+mkdir build32 && cd build32
+export CXXFLAGS="$CXXFLAGS_COMMON -m32"
+../configure --disable-shared
+make -j4
+file src/protoc
+otool -L src/protoc | grep dylib
+cd ..
+
+mkdir build64 && cd build64
+export CXXFLAGS="$CXXFLAGS_COMMON -m64"
+../configure --disable-shared
+make -j4
+file src/protoc
+otool -L src/protoc | grep dylib
+cd ..
diff --git a/kokoro/release/protoc/macos/release.cfg b/kokoro/release/protoc/macos/release.cfg
new file mode 100644
index 00000000..da903d03
--- /dev/null
+++ b/kokoro/release/protoc/macos/release.cfg
@@ -0,0 +1,8 @@
+# Configuration for Mac OSX protoc release builds
+build_file: "protobuf/kokoro/release/protoc/macos/build.sh"
+
+action {
+  define_artifacts {
+    regex: "**/protoc"
+  }
+}
diff --git a/kokoro/release/protoc/windows/build.bat b/kokoro/release/protoc/windows/build.bat
new file mode 100644
index 00000000..ae0fa94a
--- /dev/null
+++ b/kokoro/release/protoc/windows/build.bat
@@ -0,0 +1,27 @@
+set PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH%
+set generator32=Visual Studio 14
+set generator64=Visual Studio 14 Win64
+set vcplatform32=win32
+set vcplatform64=x64
+set configuration=Release
+
+echo Building protoc
+cd github\protobuf
+
+mkdir build32
+cd build32
+cmake -G "%generator32%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake
+msbuild protobuf.sln /p:Platform=%vcplatform32% || goto error
+cd ..
+
+mkdir build64
+cd build64
+cmake -G "%generator64%" -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_UNICODE=ON ../cmake
+msbuild protobuf.sln /p:Platform=%vcplatform64% || goto error
+cd ..
+
+goto :EOF
+
+:error
+echo Failed!
+exit /b %ERRORLEVEL%
diff --git a/kokoro/release/protoc/windows/release.cfg b/kokoro/release/protoc/windows/release.cfg
new file mode 100644
index 00000000..0d0da8f1
--- /dev/null
+++ b/kokoro/release/protoc/windows/release.cfg
@@ -0,0 +1,8 @@
+# Configuration for Windows protoc release builds
+build_file: "protobuf/kokoro/release/protoc/windows/build.bat"
+
+action {
+  define_artifacts {
+    regex: "**/protoc.exe"
+  }
+}