From 67c727cd411a339eb4233c84d9a1afadd8c20566 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Fri, 4 Mar 2016 14:21:18 -0800 Subject: Rearranged and commented files for running under Jenkins. --- .travis.yml | 2 +- jenkins/README.md | 6 + jenkins/build_and_run_docker.sh | 56 ++++++ jenkins/buildcmds/README.md | 6 + jenkins/buildcmds/pull_request.sh | 15 ++ jenkins/docker/Dockerfile | 130 +++++++++++++ jenkins/make_test_output.py | 91 +++++++++ jenkins/pull_request_in_docker.sh | 72 +++++++ tests.sh | 350 +++++++++++++++++++++++++++++++++ tools/docker/Dockerfile | 151 --------------- tools/jenkins/build_and_run_docker.sh | 84 -------- tools/jenkins/make_test_output.py | 90 --------- tools/jenkins/pull_request.sh | 7 - tools/run_tests/jenkins.sh | 68 ------- tools/run_tests/tests.sh | 351 ---------------------------------- 15 files changed, 727 insertions(+), 752 deletions(-) create mode 100644 jenkins/README.md create mode 100755 jenkins/build_and_run_docker.sh create mode 100644 jenkins/buildcmds/README.md create mode 100755 jenkins/buildcmds/pull_request.sh create mode 100644 jenkins/docker/Dockerfile create mode 100644 jenkins/make_test_output.py create mode 100755 jenkins/pull_request_in_docker.sh create mode 100755 tests.sh delete mode 100644 tools/docker/Dockerfile delete mode 100755 tools/jenkins/build_and_run_docker.sh delete mode 100644 tools/jenkins/make_test_output.py delete mode 100755 tools/jenkins/pull_request.sh delete mode 100755 tools/run_tests/jenkins.sh delete mode 100755 tools/run_tests/tests.sh diff --git a/.travis.yml b/.travis.yml index 26b9dcaf..f8d2347c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ os: # The Objective C build needs Xcode 7.0 or later. osx_image: xcode7.2 script: - - ./tools/run_tests/tests.sh $CONFIG + - ./tests.sh $CONFIG env: - CONFIG=cpp - CONFIG=cpp_distcheck diff --git a/jenkins/README.md b/jenkins/README.md new file mode 100644 index 00000000..29f664f2 --- /dev/null +++ b/jenkins/README.md @@ -0,0 +1,6 @@ + +Jenkins Infrastructure +---------------------- + +The scripts in this directory serve as plumbing for running the protobuf +tests under Jenkins. diff --git a/jenkins/build_and_run_docker.sh b/jenkins/build_and_run_docker.sh new file mode 100755 index 00000000..abc6f055 --- /dev/null +++ b/jenkins/build_and_run_docker.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Builds docker image and runs a command under it. +# This is a generic script that is configured with the following variables: +# +# DOCKERFILE_DIR - Directory in which Dockerfile file is located. +# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) +# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# $@ - Extra args to pass to docker run + + +set -ex + +cd $(dirname $0)/.. +git_root=$(pwd) +cd - + +# Use image name based on Dockerfile location checksum +DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + +# Make sure docker image has been built. Should be instantaneous if so. +docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR + +# Ensure existence of ccache directory +CCACHE_DIR=/tmp/protobuf-ccache +mkdir -p $CCACHE_DIR + +# Choose random name for docker container +CONTAINER_NAME="build_and_run_docker_$(uuidgen)" + +# Run command inside docker +docker run \ + "$@" \ + -e CCACHE_DIR=$CCACHE_DIR \ + -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \ + -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ + -v "$git_root:/var/local/jenkins/protobuf:ro" \ + -v $CCACHE_DIR:$CCACHE_DIR \ + -w /var/local/git/protobuf \ + --name=$CONTAINER_NAME \ + $DOCKER_IMAGE_NAME \ + bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" + +# Copy output artifacts +if [ "$OUTPUT_DIR" != "" ] +then + docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true" +fi + +# remove the container, possibly killing it first +docker rm -f $CONTAINER_NAME || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/jenkins/buildcmds/README.md b/jenkins/buildcmds/README.md new file mode 100644 index 00000000..7a48f2d5 --- /dev/null +++ b/jenkins/buildcmds/README.md @@ -0,0 +1,6 @@ + +Jenkins Build Commands +---------------------- + +The scripts in this directory are designed to be top-level entry points for +Jenkins projects. diff --git a/jenkins/buildcmds/pull_request.sh b/jenkins/buildcmds/pull_request.sh new file mode 100755 index 00000000..01fda798 --- /dev/null +++ b/jenkins/buildcmds/pull_request.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This is the top-level script we give to Jenkins as the entry point for +# running the "pull request" project: +# +# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/ +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +export DOCKERFILE_DIR=jenkins/docker +export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +./jenkins/build_and_run_docker.sh diff --git a/jenkins/docker/Dockerfile b/jenkins/docker/Dockerfile new file mode 100644 index 00000000..8467aeff --- /dev/null +++ b/jenkins/docker/Dockerfile @@ -0,0 +1,130 @@ +# This Dockerfile specifies the recipe for creating an image for the tests +# to run in. +# +# We install as many test dependencies here as we can, because these setup +# steps can be cached. They do *not* run every time we run the build. +# The Docker image is only rebuilt when the Dockerfile (ie. this file) +# changes. + +# Base Dockerfile for gRPC dev images +FROM debian:latest + +# Apt source for old Python versions. +RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C + +# Apt source for Oracle Java. +run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections + +# Apt source for Mono +run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ + echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + +# Install dependencies. We start with the basic ones require to build protoc +# and the C++ build +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + git \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + parallel \ + time \ + wget \ + # -- For csharp -- + mono-devel \ + referenceassemblies-pcl \ + nunit \ + # -- For all Java builds -- \ + maven \ + # -- For java_jdk6 -- \ + # oops! not in jessie. too old? openjdk-6-jdk \ + # -- For java_jdk7 -- \ + openjdk-7-jdk \ + # -- For java_oracle7 -- \ + oracle-java7-installer \ + # -- For python / python_cpp -- \ + python-setuptools \ + python-pip \ + python-dev \ + python2.6-dev \ + python3.3-dev \ + python3.4-dev \ + # -- For Ruby -- + ruby \ + && apt-get clean + +################## +# C# dependencies + +RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe + +################## +# Python dependencies + +# These packages exist in apt-get, but their versions are too old, so we have +# to get updates from pip. + +RUN pip install pip --upgrade +RUN pip install virtualenv tox yattag + + +################## +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +################## +# Java dependencies + +# This step requires compiling protoc. :( + +ENV MAVEN_REPO /var/maven_local_repository +ENV MVN mvn --batch-mode + +RUN cd /tmp && \ + git clone https://github.com/google/protobuf.git && \ + cd protobuf && \ + ./autogen.sh && \ + ./configure && \ + make -j6 && \ + cd java && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ + cd ../javanano && \ + $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO + +################## +# Prepare ccache + +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +# Define the default command. +CMD ["bash"] diff --git a/jenkins/make_test_output.py b/jenkins/make_test_output.py new file mode 100644 index 00000000..b1f2e2c0 --- /dev/null +++ b/jenkins/make_test_output.py @@ -0,0 +1,91 @@ +"""Gathers output from test runs and create an XML file in JUnit format. + +The output files from the individual tests have been written in a directory +structure like: + + $DIR/joblog (output from "parallel --joblog joblog") + $DIR/logs/1/cpp/stdout + $DIR/logs/1/cpp/stderr + $DIR/logs/1/csharp/stdout + $DIR/logs/1/csharp/stderr + $DIR/logs/1/java_jdk7/stdout + $DIR/logs/1/java_jdk7/stderr + etc. + +This script bundles them into a single output XML file so Jenkins can show +detailed test results. It runs as the last step before the Jenkins build +finishes. +""" + +import os; +import sys; +from yattag import Doc +from collections import defaultdict + +def readtests(basedir): + tests = defaultdict(dict) + + # Sample input (note: separators are tabs). + # + # Seq Host Starttime Runtime Send Receive Exitval Signal Command + # 1 : 1456263838.313 0.005 0 0 0 0 echo A + with open(basedir + "/joblog") as jobs: + firstline = next(jobs) + for line in jobs: + values = line.split("\t") + + name = values[8].split()[-1] + test = tests[name] + test["name"] = name + test["time"] = values[3] + + exitval = values[6] + if int(exitval): + # We don't have a more specific message. User should look at stderr. + test["failure"] = "TEST FAILURE" + else: + test["failure"] = False + + for testname in os.listdir(basedir + "/logs/1"): + test = tests[testname] + + with open(basedir + "/logs/1/" + testname + "/stdout") as f: + test["stdout"] = f.read() + + with open(basedir + "/logs/1/" + testname + "/stderr") as f: + test["stderr"] = f.read() + + # The cpp test is special since it doesn't run under parallel so doesn't show + # up in the job log. + tests["cpp"]["name"] = "cpp" + + with open(basedir + '/logs/1/cpp/build_time', 'r') as f: + tests["cpp"]["time"] = f.read().strip() + tests["cpp"]["failure"] = False + + ret = tests.values() + ret.sort(key=lambda x: x["name"]) + + return ret + +def genxml(tests): + doc, tag, text = Doc().tagtext() + + with tag("testsuites"): + with tag("testsuite", name="Protobuf Tests"): + for test in tests: + with tag("testcase", name=test["name"], classname=test["name"], + time=test["time"]): + with tag("system-out"): + text(test["stdout"]) + with tag("system-err"): + text(test["stderr"]) + if test["failure"]: + with tag("failure"): + text(test["failure"]) + + return doc.getvalue() + +sys.stderr.write("make_test_output.py: writing XML from directory: " + + sys.argv[1] + "\n"); +print genxml(readtests(sys.argv[1])) diff --git a/jenkins/pull_request_in_docker.sh b/jenkins/pull_request_in_docker.sh new file mode 100755 index 00000000..887f97c5 --- /dev/null +++ b/jenkins/pull_request_in_docker.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# This is the script that runs inside Docker, once the image has been built, +# to execute all tests for the "pull request" project. + +WORKSPACE_BASE=`pwd` +MY_DIR="$(dirname "$0")" +TEST_SCRIPT=$MY_DIR/../tests.sh +BUILD_DIR=/tmp/protobuf + +set -e # exit immediately on error +set -x # display all commands + +# The protobuf repository is mounted into our Docker image, but read-only. +# We clone into a directory inside Docker (this is faster than cp). +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR +cd $BUILD_DIR +git clone /var/local/jenkins/protobuf +cd protobuf + +# Set up the directory where our test output is going to go. +OUTPUT_DIR=`mktemp -d` +LOG_OUTPUT_DIR=$OUTPUT_DIR/logs +mkdir -p $LOG_OUTPUT_DIR/1/cpp + +################################################################################ +# cpp build needs to run first, non-parallelized, so that protoc is available +# for other builds. + +# Output filenames to follow the overall scheme used by parallel, ie: +# $DIR/logs/1/cpp/stdout +# $DIR/logs/1/cpp/stderr +# $DIR/logs/1/csharp/stdout +# $DIR/logs/1/csharp/stderr +# $DIR/logs/1/java_jdk7/stdout +# $DIR/logs/1/java_jdk7/stderr +CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout +CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr + +# Time the C++ build, so we can put this info in the test output. +# It's important that we get /usr/bin/time (which supports -f and -o) and not +# the bash builtin "time" which doesn't. +TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" + +$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) + +# Other tests are run in parallel. + +parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ + csharp \ + java_jdk7 \ + javanano_jdk7 \ + java_oracle7 \ + javanano_oracle7 \ + python \ + python_cpp \ + ruby21 \ + || true # Process test results even if tests fail. + +cat $OUTPUT_DIR/joblog + +# The directory that is copied from Docker back into the Jenkins workspace. +COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput +mkdir -p $COPY_FROM_DOCKER +TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml + +# Process all the output files from "parallel" and package them into a single +# .xml file with detailed, broken-down test output. +python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE + +ls -l $TESTOUTPUT_XML_FILE diff --git a/tests.sh b/tests.sh new file mode 100755 index 00000000..a4624c0c --- /dev/null +++ b/tests.sh @@ -0,0 +1,350 @@ +#!/bin/bash +# +# Build and runs tests for the protobuf project. The tests as written here are +# used by both Jenkins and Travis, though some specialized logic is required to +# handle the differences between them. + +on_travis() { + if [ "$TRAVIS" == "true" ]; then + "$@" + fi +} + +# For when some other test needs the C++ main build, including protoc and +# libprotobuf. +internal_build_cpp() { + if [ -f src/protoc ]; then + # Already built. + return + fi + + if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then + # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more + # decent C++ 11 support in order to compile conformance tests. + sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y + sudo apt-get update -qq + sudo apt-get install -qq g++-4.8 + export CXX="g++-4.8" CC="gcc-4.8" + fi + + ./autogen.sh + ./configure + make -j2 +} + +build_cpp() { + internal_build_cpp + make check -j2 + cd conformance && make test_cpp && cd .. +} + +build_cpp_distcheck() { + ./autogen.sh + ./configure + make distcheck -j2 +} + +build_csharp() { + # Just for the conformance tests. We don't currently + # need to really build protoc, but it's simplest to keep with the + # conventions of the other builds. + internal_build_cpp + NUGET=/usr/local/bin/nuget.exe + + if [ "$TRAVIS" == "true" ]; then + # Install latest version of Mono + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list + echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list + sudo apt-get update -qq + sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit + wget www.nuget.org/NuGet.exe -O nuget.exe + NUGET=../../nuget.exe + fi + + (cd csharp/src; mono $NUGET restore) + csharp/buildall.sh + cd conformance && make test_csharp && cd .. +} + +build_golang() { + # Go build needs `protoc`. + internal_build_cpp + # Add protoc to the path so that the examples build finds it. + export PATH="`pwd`/src:$PATH" + + # Install Go and the Go protobuf compiler plugin. + sudo apt-get update -qq + sudo apt-get install -qq golang + export GOPATH="$HOME/gocode" + mkdir -p "$GOPATH/src/github.com/google" + ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" + export PATH="$GOPATH/bin:$PATH" + go get github.com/golang/protobuf/protoc-gen-go + + cd examples && make gotest && cd .. +} + +use_java() { + version=$1 + case "$version" in + jdk6) + on_travis sudo apt-get install openjdk-6-jdk + export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH + ;; + jdk7) + on_travis sudo apt-get install openjdk-7-jdk + export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH + ;; + oracle7) + if [ "$TRAVIS" == "true" ]; then + sudo apt-get install python-software-properties # for apt-add-repository + echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ + sudo debconf-set-selections + yes | sudo apt-add-repository ppa:webupd8team/java + yes | sudo apt-get install oracle-java7-installer + fi; + export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH + ;; + esac + + if [ "$TRAVIS" != "true" ]; then + MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository + MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" + fi; + + which java + java -version +} + +# --batch-mode supresses download progress output that spams the logs. +MVN="mvn --batch-mode" + +build_java() { + version=$1 + dir=java_$version + # Java build needs `protoc`. + internal_build_cpp + cp -r java $dir + cd $dir && $MVN clean && $MVN test + cd ../.. +} + +# The conformance tests are hard-coded to work with the $ROOT/java directory. +# So this can't run in parallel with two different sets of tests. +build_java_with_conformance_tests() { + # Java build needs `protoc`. + internal_build_cpp + cd java && $MVN test && $MVN install + cd util && $MVN package assembly:single + cd ../.. + cd conformance && make test_java && cd .. +} + +build_javanano() { + # Java build needs `protoc`. + internal_build_cpp + cd javanano && $MVN test && cd .. +} + +build_java_jdk6() { + use_java jdk6 + build_java jdk6 +} +build_java_jdk7() { + use_java jdk7 + build_java_with_conformance_tests +} +build_java_oracle7() { + use_java oracle7 + build_java oracle7 +} + +build_javanano_jdk6() { + use_java jdk6 + build_javanano +} +build_javanano_jdk7() { + use_java jdk7 + build_javanano +} +build_javanano_oracle7() { + use_java oracle7 + build_javanano +} + +internal_install_python_deps() { + if [ "$TRAVIS" != "true" ]; then + return; + fi + # Install tox (OS X doesn't have pip). + if [ $(uname -s) == "Darwin" ]; then + sudo easy_install tox + else + sudo pip install tox + fi + # Only install Python2.6/3.x on Linux. + if [ $(uname -s) == "Linux" ]; then + sudo apt-get install -y python-software-properties # for apt-add-repository + sudo apt-add-repository -y ppa:fkrull/deadsnakes + sudo apt-get update -qq + sudo apt-get install -y python2.6 python2.6-dev + sudo apt-get install -y python3.3 python3.3-dev + sudo apt-get install -y python3.4 python3.4-dev + fi +} + +internal_objectivec_common () { + # Make sure xctool is up to date. Adapted from + # http://docs.travis-ci.com/user/osx-ci-environment/ + # We don't use a before_install because we test multiple OSes. + brew update + # xctool 0.2.8 seems to have a bug where it randomly kills tests saying + # they failed. Disabling the updates, but letting it report about being + # updates as a hint that this needs to eventually get re-enabled. + # https://github.com/facebook/xctool/issues/619 + # https://github.com/google/protobuf/issues/1232 + brew outdated xctool || true + #brew outdated xctool || brew upgrade xctool + # Reused the build script that takes care of configuring and ensuring things + # are up to date. Xcode and conformance tests will be directly invoked. + objectivec/DevTools/full_mac_build.sh \ + --core-only --skip-xcode --skip-objc-conformance +} + +internal_xctool_debug_and_release() { + # Always use -reporter plain to avoid escape codes in output (makes travis + # logs easier to read). + xctool -reporter plain -configuration Debug "$@" + xctool -reporter plain -configuration Release "$@" +} + +build_objectivec_ios() { + internal_objectivec_common + # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool + # doesn't support >1 destination, so we have to build first and then run the + # tests one destination at a time. + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_iOS.xcodeproj \ + -scheme ProtocolBuffers \ + -sdk iphonesimulator \ + build-tests + IOS_DESTINATIONS=( + "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit + "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit + "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit + "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit + ) + for i in "${IOS_DESTINATIONS[@]}" ; do + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_iOS.xcodeproj \ + -scheme ProtocolBuffers \ + -sdk iphonesimulator \ + -destination "${i}" \ + run-tests + done +} + +build_objectivec_osx() { + internal_objectivec_common + internal_xctool_debug_and_release \ + -project objectivec/ProtocolBuffers_OSX.xcodeproj \ + -scheme ProtocolBuffers \ + -destination "platform=OS X,arch=x86_64" \ + test + cd conformance && make test_objc && cd .. +} + +build_python() { + internal_build_cpp + internal_install_python_deps + cd python + # Only test Python 2.6/3.x on Linux + if [ $(uname -s) == "Linux" ]; then + envlist=py\{26,27,33,34\}-python + else + envlist=py27-python + fi + tox -e $envlist + cd .. +} + +build_python_cpp() { + internal_build_cpp + internal_install_python_deps + export LD_LIBRARY_PATH=../src/.libs # for Linux + export DYLD_LIBRARY_PATH=../src/.libs # for OS X + cd python + # Only test Python 2.6/3.x on Linux + if [ $(uname -s) == "Linux" ]; then + # py26 is currently disabled due to json_format + envlist=py\{27,33,34\}-cpp + else + envlist=py27-cpp + fi + tox -e $envlist + cd .. +} + +build_ruby19() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-1.9 && cd .. +} +build_ruby20() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.0 && cd .. +} +build_ruby21() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.1 && cd .. +} +build_ruby22() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-2.2 && cd .. +} +build_jruby() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh jruby && cd .. +} + +build_javascript() { + internal_build_cpp + cd js && npm install && npm test && cd .. +} + +# Note: travis currently does not support testing more than one language so the +# .travis.yml cheats and claims to only be cpp. If they add multiple language +# support, this should probably get updated to install steps and/or +# rvm/gemfile/jdk/etc. entries rather than manually doing the work. + +# .travis.yml uses matrix.exclude to block the cases where app-get can't be +# use to install things. + +# -------- main -------- + +if [ "$#" -ne 1 ]; then + echo " +Usage: $0 { cpp | + csharp | + java_jdk6 | + java_jdk7 | + java_oracle7 | + javanano_jdk6 | + javanano_jdk7 | + javanano_oracle7 | + objectivec_ios | + objectivec_osx | + python | + python_cpp | + ruby19 | + ruby20 | + ruby21 | + ruby22 | + jruby } +" + exit 1 +fi + +set -e # exit immediately on error +set -x # display all commands +eval "build_$1" diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile deleted file mode 100644 index 5136ee6b..00000000 --- a/tools/docker/Dockerfile +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Base Dockerfile for gRPC dev images -FROM debian:latest - -# Apt source for old Python versions. -RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C - -# Apt source for Oracle Java. -run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections - -# Apt source for Mono -run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \ - echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - -# Install dependencies. We start with the basic ones require to build protoc -# and the C++ build -RUN apt-get update && apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - git \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libgtest-dev \ - libtool \ - make \ - parallel \ - time \ - wget \ - # -- For csharp -- - mono-devel \ - referenceassemblies-pcl \ - nunit \ - # -- For all Java builds -- \ - maven \ - # -- For java_jdk6 -- \ - # oops! not in jessie. too old? openjdk-6-jdk \ - # -- For java_jdk7 -- \ - openjdk-7-jdk \ - # -- For java_oracle7 -- \ - oracle-java7-installer \ - # -- For python / python_cpp -- \ - python-setuptools \ - python-pip \ - python-dev \ - python2.6-dev \ - python3.3-dev \ - python3.4-dev \ - # -- For Ruby -- - ruby \ - && apt-get clean - -################## -# C# dependencies - -RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe - -################## -# Python dependencies - -# These packages exist in apt-get, but their versions are too old, so we have -# to get updates from pip. - -RUN pip install pip --upgrade -RUN pip install virtualenv tox yattag - - -################## -# Ruby dependencies - -# Install rvm -RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -RUN \curl -sSL https://get.rvm.io | bash -s stable - -# Install Ruby 2.1 -RUN /bin/bash -l -c "rvm install ruby-2.1" -RUN /bin/bash -l -c "rvm use --default ruby-2.1" -RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" -RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" -RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" -RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" - -################## -# Java dependencies - -# This step requires compiling protoc. :( - -ENV MAVEN_REPO /var/maven_local_repository -ENV MVN mvn --batch-mode - -RUN cd /tmp && \ - git clone https://github.com/google/protobuf.git && \ - cd protobuf && \ - ./autogen.sh && \ - ./configure && \ - make -j6 && \ - cd java && \ - $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \ - $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \ - cd ../javanano && \ - $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO - -################## -# Prepare ccache - -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - -# Define the default command. -CMD ["bash"] diff --git a/tools/jenkins/build_and_run_docker.sh b/tools/jenkins/build_and_run_docker.sh deleted file mode 100755 index ad1075fa..00000000 --- a/tools/jenkins/build_and_run_docker.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Builds docker image and runs a command under it. -# You should never need to call this script on your own. - -set -ex - -cd $(dirname $0)/../.. -git_root=$(pwd) -cd - - -# Inputs -# DOCKERFILE_DIR - Directory in which Dockerfile file is located. -# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root) -# OUTPUT_DIR - Directory that will be copied from inside docker after finishing. -# $@ - Extra args to pass to docker run - -# Use image name based on Dockerfile location checksum -DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) - -# Make sure docker image has been built. Should be instantaneous if so. -docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR - -# Ensure existence of ccache directory -CCACHE_DIR=/tmp/protobuf-ccache -mkdir -p $CCACHE_DIR - -# Choose random name for docker container -CONTAINER_NAME="build_and_run_docker_$(uuidgen)" - -# Run command inside docker -docker run \ - "$@" \ - -e CCACHE_DIR=$CCACHE_DIR \ - -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \ - -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ - -v "$git_root:/var/local/jenkins/protobuf:ro" \ - -v $CCACHE_DIR:$CCACHE_DIR \ - -w /var/local/git/protobuf \ - --name=$CONTAINER_NAME \ - $DOCKER_IMAGE_NAME \ - bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true" - -# Copy output artifacts -if [ "$OUTPUT_DIR" != "" ] -then - docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true" -fi - -# remove the container, possibly killing it first -docker rm -f $CONTAINER_NAME || true - -if [ "$FAILED" != "" ] -then - exit 1 -fi diff --git a/tools/jenkins/make_test_output.py b/tools/jenkins/make_test_output.py deleted file mode 100644 index 986d1979..00000000 --- a/tools/jenkins/make_test_output.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Gather output from test runs and create an XML file in JUnit format. - -The output files from the individual tests have been written in a directory -structure like: - - $DIR/joblog (--joblog from "parallel") - $DIR/logs/1/cpp/stdout - $DIR/logs/1/cpp/stderr - $DIR/logs/1/csharp/stdout - $DIR/logs/1/csharp/stderr - $DIR/logs/1/java_jdk7/stdout - $DIR/logs/1/java_jdk7/stderr - etc. - -This script bundles them into a single output XML file so Jenkins can show -detailed test results. -""" - -import os; -import sys; -from yattag import Doc -from collections import defaultdict - -def readtests(basedir): - tests = defaultdict(dict) - - # Sample input (note: separators are tabs). - # - # Seq Host Starttime Runtime Send Receive Exitval Signal Command - # 1 : 1456263838.313 0.005 0 0 0 0 echo A - with open(basedir + "/joblog") as jobs: - firstline = next(jobs) - for line in jobs: - values = line.split("\t") - - name = values[8].split()[-1] - test = tests[name] - test["name"] = name - test["time"] = values[3] - - exitval = values[6] - if int(exitval): - # We don't have a more specific message. User should look at stderr. - test["failure"] = "TEST FAILURE" - else: - test["failure"] = False - - for testname in os.listdir(basedir + "/logs/1"): - test = tests[testname] - - with open(basedir + "/logs/1/" + testname + "/stdout") as f: - test["stdout"] = f.read() - - with open(basedir + "/logs/1/" + testname + "/stderr") as f: - test["stderr"] = f.read() - - # The cpp test is special since it doesn't run under parallel so doesn't show - # up in the job log. - tests["cpp"]["name"] = "cpp" - - with open(basedir + '/logs/1/cpp/build_time', 'r') as f: - tests["cpp"]["time"] = f.read().strip() - tests["cpp"]["failure"] = False - - ret = tests.values() - ret.sort(key=lambda x: x["name"]) - - return ret - -def genxml(tests): - doc, tag, text = Doc().tagtext() - - with tag("testsuites"): - with tag("testsuite", name="Protobuf Tests"): - for test in tests: - with tag("testcase", name=test["name"], classname=test["name"], - time=test["time"]): - with tag("system-out"): - text(test["stdout"]) - with tag("system-err"): - text(test["stderr"]) - if test["failure"]: - with tag("failure"): - text(test["failure"]) - - return doc.getvalue() - -sys.stderr.write("make_test_output.py: writing XML from directory: " + - sys.argv[1] + "\n"); -print genxml(readtests(sys.argv[1])) diff --git a/tools/jenkins/pull_request.sh b/tools/jenkins/pull_request.sh deleted file mode 100755 index 00538b9c..00000000 --- a/tools/jenkins/pull_request.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - - -export DOCKERFILE_DIR=tools/docker -export DOCKER_RUN_SCRIPT=tools/run_tests/jenkins.sh -export OUTPUT_DIR=testoutput -./tools/jenkins/build_and_run_docker.sh diff --git a/tools/run_tests/jenkins.sh b/tools/run_tests/jenkins.sh deleted file mode 100755 index ea67345b..00000000 --- a/tools/run_tests/jenkins.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -WORKSPACE_BASE=`pwd` -MY_DIR="$(dirname "$0")" -TEST_SCRIPT=$MY_DIR/tests.sh -BUILD_DIR=/tmp/protobuf - -# Set value used in tests.sh. -PARALLELISM=-j8 - -set -e # exit immediately on error -set -x # display all commands - -rm -rf $BUILD_DIR -mkdir -p $BUILD_DIR -cd $BUILD_DIR -git clone /var/local/jenkins/protobuf -cd protobuf - -OUTPUT_DIR=`mktemp -d` -LOG_OUTPUT_DIR=$OUTPUT_DIR/logs -mkdir -p $LOG_OUTPUT_DIR/1/cpp - -################################################################################ -# cpp build needs to run first, non-parallelized, so that protoc is available -# for other builds. - -# Output filenames to follow the overall scheme used by parallel, ie: -# $DIR/logs/1/cpp/stdout -# $DIR/logs/1/cpp/stderr -# $DIR/logs/1/csharp/stdout -# $DIR/logs/1/csharp/stderr -# $DIR/logs/1/java_jdk7/stdout -# $DIR/logs/1/java_jdk7/stderr -CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout -CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr - -# It's important that we get /usr/bin/time (which supports -f and -o) and not -# the bash builtin "time" which doesn't. -TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time" - -$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2) - -# Other tests are run in parallel. The overall run fails if any one of them -# fails. - -parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \ - csharp \ - java_jdk7 \ - javanano_jdk7 \ - java_oracle7 \ - javanano_oracle7 \ - python \ - python_cpp \ - ruby21 \ - || true # Process test results even if tests fail. - -# The directory that is copied from Docker back into the Jenkins workspace. -COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput -mkdir -p $COPY_FROM_DOCKER -TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml - -python $MY_DIR/../jenkins/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE - -ls -l $TESTOUTPUT_XML_FILE - -### disabled tests -# java_jdk6 \ diff --git a/tools/run_tests/tests.sh b/tools/run_tests/tests.sh deleted file mode 100755 index c28a5daa..00000000 --- a/tools/run_tests/tests.sh +++ /dev/null @@ -1,351 +0,0 @@ -#!/bin/bash - -on_travis() { - if [ "$TRAVIS" == "true" ]; then - "$@" - fi -} - -# For when some other test needs the C++ main build, including protoc and -# libprotobuf. -internal_build_cpp() { - if [ -f src/protoc ]; then - # Already built. - return - fi - - if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then - # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more - # decent C++ 11 support in order to compile conformance tests. - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -qq - sudo apt-get install -qq g++-4.8 - export CXX="g++-4.8" CC="gcc-4.8" - fi - - ./autogen.sh - ./configure - make $PARALLELISM -} - -build_cpp() { - internal_build_cpp - make check $PARALLELISM - cd conformance && make test_cpp && cd .. -} - -build_cpp_distcheck() { - ./autogen.sh - ./configure - make distcheck $PARALLELISM -} - -build_csharp() { - # Just for the conformance tests. We don't currently - # need to really build protoc, but it's simplest to keep with the - # conventions of the other builds. - internal_build_cpp - NUGET=/usr/local/bin/nuget.exe - - if [ "$TRAVIS" == "true" ]; then - # Install latest version of Mono - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list - echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | sudo tee -a /etc/apt/sources.list.d/mono-xamarin.list - sudo apt-get update -qq - sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit - wget www.nuget.org/NuGet.exe -O nuget.exe - NUGET=../../nuget.exe - fi - - (cd csharp/src; mono $NUGET restore) - csharp/buildall.sh - cd conformance && make test_csharp && cd .. -} - -build_golang() { - # Go build needs `protoc`. - internal_build_cpp - # Add protoc to the path so that the examples build finds it. - export PATH="`pwd`/src:$PATH" - - # Install Go and the Go protobuf compiler plugin. - sudo apt-get update -qq - sudo apt-get install -qq golang - export GOPATH="$HOME/gocode" - mkdir -p "$GOPATH/src/github.com/google" - ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" - export PATH="$GOPATH/bin:$PATH" - go get github.com/golang/protobuf/protoc-gen-go - - cd examples && make gotest && cd .. -} - -use_java() { - version=$1 - case "$version" in - jdk6) - on_travis sudo apt-get install openjdk-6-jdk - export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH - ;; - jdk7) - on_travis sudo apt-get install openjdk-7-jdk - export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH - ;; - oracle7) - if [ "$TRAVIS" == "true" ]; then - sudo apt-get install python-software-properties # for apt-add-repository - echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ - sudo debconf-set-selections - yes | sudo apt-add-repository ppa:webupd8team/java - yes | sudo apt-get install oracle-java7-installer - fi; - export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH - ;; - esac - - if [ "$TRAVIS" != "true" ]; then - MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository - MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" - fi; - - which java - java -version -} - -# --batch-mode supresses download progress output that spams the logs. -MVN="mvn --batch-mode" - -build_java() { - version=$1 - dir=java_$version - # Java build needs `protoc`. - internal_build_cpp - cp -r java $dir - cd $dir && $MVN clean && $MVN test - cd ../.. -} - -# The conformance tests are hard-coded to work with the $ROOT/java directory. -# So this can't run in parallel with two different sets of tests. -build_java_with_conformance_tests() { - # Java build needs `protoc`. - internal_build_cpp - cd java && $MVN test && $MVN install - cd util && $MVN package assembly:single - cd ../.. - cd conformance && make test_java && cd .. -} - -build_javanano() { - # Java build needs `protoc`. - internal_build_cpp - cd javanano && $MVN test && cd .. -} - -build_java_jdk6() { - use_java jdk6 - build_java jdk6 -} -build_java_jdk7() { - use_java jdk7 - build_java_with_conformance_tests -} -build_java_oracle7() { - use_java oracle7 - build_java oracle7 -} - -build_javanano_jdk6() { - use_java jdk6 - build_javanano -} -build_javanano_jdk7() { - use_java jdk7 - build_javanano -} -build_javanano_oracle7() { - use_java oracle7 - build_javanano -} - -internal_install_python_deps() { - if [ "$TRAVIS" != "true" ]; then - return; - fi - # Install tox (OS X doesn't have pip). - if [ $(uname -s) == "Darwin" ]; then - sudo easy_install tox - else - sudo pip install tox - fi - # Only install Python2.6/3.x on Linux. - if [ $(uname -s) == "Linux" ]; then - sudo apt-get install -y python-software-properties # for apt-add-repository - sudo apt-add-repository -y ppa:fkrull/deadsnakes - sudo apt-get update -qq - sudo apt-get install -y python2.6 python2.6-dev - sudo apt-get install -y python3.3 python3.3-dev - sudo apt-get install -y python3.4 python3.4-dev - fi -} - -internal_objectivec_common () { - # Make sure xctool is up to date. Adapted from - # http://docs.travis-ci.com/user/osx-ci-environment/ - # We don't use a before_install because we test multiple OSes. - brew update - # xctool 0.2.8 seems to have a bug where it randomly kills tests saying - # they failed. Disabling the updates, but letting it report about being - # updates as a hint that this needs to eventually get re-enabled. - # https://github.com/facebook/xctool/issues/619 - # https://github.com/google/protobuf/issues/1232 - brew outdated xctool || true - #brew outdated xctool || brew upgrade xctool - # Reused the build script that takes care of configuring and ensuring things - # are up to date. Xcode and conformance tests will be directly invoked. - objectivec/DevTools/full_mac_build.sh \ - --core-only --skip-xcode --skip-objc-conformance -} - -internal_xctool_debug_and_release() { - # Always use -reporter plain to avoid escape codes in output (makes travis - # logs easier to read). - xctool -reporter plain -configuration Debug "$@" - xctool -reporter plain -configuration Release "$@" -} - -build_objectivec_ios() { - internal_objectivec_common - # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool - # doesn't support >1 destination, so we have to build first and then run the - # tests one destination at a time. - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_iOS.xcodeproj \ - -scheme ProtocolBuffers \ - -sdk iphonesimulator \ - build-tests - IOS_DESTINATIONS=( - "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit - "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit - "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit - "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit - ) - for i in "${IOS_DESTINATIONS[@]}" ; do - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_iOS.xcodeproj \ - -scheme ProtocolBuffers \ - -sdk iphonesimulator \ - -destination "${i}" \ - run-tests - done -} - -build_objectivec_osx() { - internal_objectivec_common - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_OSX.xcodeproj \ - -scheme ProtocolBuffers \ - -destination "platform=OS X,arch=x86_64" \ - test - cd conformance && make test_objc && cd .. -} - -build_python() { - internal_build_cpp - internal_install_python_deps - cd python - # Only test Python 2.6/3.x on Linux - if [ $(uname -s) == "Linux" ]; then - envlist=py\{26,27,33,34\}-python - else - envlist=py27-python - fi - tox -e $envlist - cd .. -} - -build_python_cpp() { - internal_build_cpp - internal_install_python_deps - export LD_LIBRARY_PATH=../src/.libs # for Linux - export DYLD_LIBRARY_PATH=../src/.libs # for OS X - cd python - # Only test Python 2.6/3.x on Linux - if [ $(uname -s) == "Linux" ]; then - # py26 is currently disabled due to json_format - envlist=py\{27,33,34\}-cpp - else - envlist=py27-cpp - fi - tox -e $envlist - cd .. -} - -build_ruby19() { - internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh ruby-1.9 && cd .. -} -build_ruby20() { - internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh ruby-2.0 && cd .. -} -build_ruby21() { - internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh ruby-2.1 && cd .. -} -build_ruby22() { - internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh ruby-2.2 && cd .. -} -build_jruby() { - internal_build_cpp # For conformance tests. - cd ruby && bash travis-test.sh jruby && cd .. -} - -build_javascript() { - internal_build_cpp - cd js && npm install && npm test && cd .. -} - -[ -n "${PARALLELISM}" ] && PARALLELISM=-j8 - -# Note: travis currently does not support testing more than one language so the -# .travis.yml cheats and claims to only be cpp. If they add multiple language -# support, this should probably get updated to install steps and/or -# rvm/gemfile/jdk/etc. entries rather than manually doing the work. - -# .travis.yml uses matrix.exclude to block the cases where app-get can't be -# use to install things. - -# -------- main -------- - -# Set value used in tests.sh. -PARALLELISM=-j2 - -if [ "$#" -ne 1 ]; then - echo " -Usage: $0 { cpp | - csharp | - java_jdk6 | - java_jdk7 | - java_oracle7 | - javanano_jdk6 | - javanano_jdk7 | - javanano_oracle7 | - objectivec_ios | - objectivec_osx | - python | - python_cpp | - ruby19 | - ruby20 | - ruby21 | - ruby22 | - jruby } -" - exit 1 -fi - -set -e # exit immediately on error -set -x # display all commands -eval "build_$1" -- cgit v1.2.3