From 7a55684a1305c73b0d7924d5cf04a52c093bd1bc Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 5 Feb 2016 11:33:30 -0800 Subject: Clang format and fix copyrights --- test/distrib/python/distribtest.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'test/distrib') diff --git a/test/distrib/python/distribtest.py b/test/distrib/python/distribtest.py index 428ffe2b34..66c1f88796 100644 --- a/test/distrib/python/distribtest.py +++ b/test/distrib/python/distribtest.py @@ -1,3 +1,32 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + from grpc.beta import implementations # This code doesn't do much but makes sure the native extension is loaded -- cgit v1.2.3 From 5c04760d46f2628a83106d048e732c997fed8801 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 Feb 2016 14:25:56 -0800 Subject: Change default on Python install option --- src/python/grpcio/commands.py | 2 +- test/distrib/python/run_distrib_test.sh | 3 +-- tools/run_tests/build_artifact_python.sh | 10 ++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'test/distrib') diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 490d0f07b3..31bd00e94e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -135,7 +135,7 @@ class Install(install.install, EggNameMixin): def initialize_options(self): install.install.initialize_options(self) - self.use_grpc_custom_bdist = False + self.use_grpc_custom_bdist = bool(int(os.environ.get('GRPC_PYTHON_USE_CUSTOM_BDIST', '1'))) def finalize_options(self): install.install.finalize_options(self) diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index a185bcbc26..2148be121a 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -52,7 +52,6 @@ pip install --upgrade futures GRPC_PYTHON_BINARIES_REPOSITORY="${BDIST_DIR}" \ pip install \ - "${SDIST_ARCHIVE}" \ - --install-option="--use-grpc-custom-bdist" + "${SDIST_ARCHIVE}" python distribtest.py diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 48cf390f69..835fad83e1 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -39,11 +39,13 @@ then pip install -rrequirements.txt fi -GRPC_PYTHON_BUILD_WITH_CYTHON=1 ${SETARCH_CMD} python setup.py \ - bdist_wheel \ - sdist \ +GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ +GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ +${SETARCH_CMD} python setup.py \ + bdist_wheel \ + sdist \ bdist_egg_grpc_custom mkdir -p artifacts -cp -r dist/* artifacts \ No newline at end of file +cp -r dist/* artifacts -- cgit v1.2.3 From 6b5555ead1a0a7141d28bec505f5421a4101763d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 5 Feb 2016 16:07:18 -0800 Subject: Fix docker files, distrib test script --- test/distrib/python/run_distrib_test.sh | 17 +++++++++-------- tools/dockerfile/distribtest/python_arch_x64/Dockerfile | 5 +++-- .../distribtest/python_centos6_x64/Dockerfile | 2 ++ 3 files changed, 14 insertions(+), 10 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index 2148be121a..08856ad388 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -42,16 +42,17 @@ then exit 1 fi -# TODO(jtattermusch): this shouldn't be required -pip install --upgrade six +PIP=pip2 +which $PIP || PIP=pip +PYTHON=python2 +which $PYTHON || PYTHON=python -# TODO(jtattermusch): if these don't get preinstalled, pip tries to install them -# with --use-grpc-custom-bdist option, which obviously fails. -pip install --upgrade enum34 -pip install --upgrade futures +# TODO(jtattermusch): this shouldn't be required +$PIP install --upgrade six GRPC_PYTHON_BINARIES_REPOSITORY="${BDIST_DIR}" \ - pip install \ + $PIP install \ "${SDIST_ARCHIVE}" -python distribtest.py +$PYTHON distribtest.py + diff --git a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile index 4d10e240c6..536ad86d53 100644 --- a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile @@ -31,6 +31,7 @@ FROM base/archlinux RUN pacman --noconfirm -Syy RUN pacman --noconfirm -S git -RUN pacman --noconfirm -S python -RUN pacman --noconfirm -S python-pip +RUN pacman --noconfirm -S openssl +RUN pacman --noconfirm -S python2 +RUN pacman --noconfirm -S python2-pip diff --git a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile index 683686c6db..55677fc10c 100644 --- a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile @@ -35,3 +35,5 @@ RUN yum install -y python RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm RUN yum install -y python-pip +RUN pip install --upgrade pip + -- cgit v1.2.3 From 0bd1da55936bae4bdd82ef40526b5de6105c8624 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 5 Feb 2016 20:21:17 -0800 Subject: fix some node distribtests --- test/distrib/node/run_distrib_test.sh | 7 ++++++- tools/dockerfile/distribtest/node_jessie_x86/Dockerfile | 2 +- tools/run_tests/distribtest_targets.py | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/node/run_distrib_test.sh b/test/distrib/node/run_distrib_test.sh index 90637cc01d..ac9e7d7090 100755 --- a/test/distrib/node/run_distrib_test.sh +++ b/test/distrib/node/run_distrib_test.sh @@ -32,7 +32,12 @@ set -ex cd $(dirname $0) -nvm install $1 +NODE_VERSION="$1" + +# make sure nvm is available +source ~/.nvm/nvm.sh || true + +nvm install $NODE_VERSION npm install -g node-static diff --git a/tools/dockerfile/distribtest/node_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/node_jessie_x86/Dockerfile index bd40ab2b4c..0408cbeeb0 100644 --- a/tools/dockerfile/distribtest/node_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/node_jessie_x86/Dockerfile @@ -29,7 +29,7 @@ FROM 32bit/debian:jessie -RUN apt-get update && apt-get install -y git +RUN apt-get update && apt-get install -y curl # Install nvm RUN touch .profile diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index a1e45ece9d..f3ebc00b4e 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -105,8 +105,7 @@ class NodeDistribTest(object): 'tools/dockerfile/distribtest/node_%s_%s' % ( self.docker_suffix, self.arch), - # bash -l needed to make nvm available - 'bash -l test/distrib/node/run_distrib_test.sh %s' % ( + 'test/distrib/node/run_distrib_test.sh %s' % ( self.node_version)) def __str__(self): return self.name -- cgit v1.2.3 From c83a9c01870c21ee0632342fe58b69d33cea7fb3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 6 Feb 2016 14:53:28 -0800 Subject: enable C# distribtest on macos --- test/distrib/csharp/run_distrib_test.sh | 2 +- test/distrib/csharp/update_version.sh | 2 +- tools/run_tests/distribtest_targets.py | 36 ++++++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 9 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/csharp/run_distrib_test.sh b/test/distrib/csharp/run_distrib_test.sh index 5a01c91875..8ddb4c4bff 100755 --- a/test/distrib/csharp/run_distrib_test.sh +++ b/test/distrib/csharp/run_distrib_test.sh @@ -32,7 +32,7 @@ set -ex cd $(dirname $0) -unzip "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets.zip" -d TestNugetFeed +unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets.zip" -d TestNugetFeed # TODO(jtattermusch): replace the version number ./update_version.sh 0.13.0 diff --git a/test/distrib/csharp/update_version.sh b/test/distrib/csharp/update_version.sh index 569f7ff4bd..f2554e8998 100755 --- a/test/distrib/csharp/update_version.sh +++ b/test/distrib/csharp/update_version.sh @@ -33,4 +33,4 @@ set -e cd $(dirname $0) # Replaces version placeholder with value provided as first argument. -sed -i "s/__GRPC_NUGET_VERSION__/$1/g" DistribTest/packages.config DistribTest/DistribTest.csproj +sed -ibak "s/__GRPC_NUGET_VERSION__/$1/g" DistribTest/packages.config DistribTest/DistribTest.csproj diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index a1e45ece9d..965c7f8316 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -55,28 +55,49 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, return jobspec +def create_jobspec(name, cmdline, environ=None, shell=False, + flake_retries=0, timeout_retries=0): + """Creates jobspec.""" + jobspec = jobset.JobSpec( + cmdline=cmdline, + environ=environ, + shortname='distribtest.%s' % (name), + timeout_seconds=10*60, + flake_retries=flake_retries, + timeout_retries=timeout_retries, + shell=shell) + return jobspec + + class CSharpDistribTest(object): """Tests C# NuGet package""" - def __init__(self, platform, arch, docker_suffix): - self.name = 'csharp_nuget_%s_%s_%s' % (platform, arch, docker_suffix) + def __init__(self, platform, arch, docker_suffix=None): + self.name = 'csharp_nuget_%s_%s' % (platform, arch) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'csharp', platform, arch, docker_suffix] + self.labels = ['distribtest', 'csharp', platform, arch] + if docker_suffix: + self.name += '_%s' % docker_suffix + self.labels.append(docker_suffix) def pre_build_jobspecs(self): return [] def build_jobspec(self): - if not self.platform == 'linux': - raise Exception("Not supported yet.") - - return create_docker_jobspec(self.name, + if self.platform == 'linux': + return create_docker_jobspec(self.name, 'tools/dockerfile/distribtest/csharp_%s_%s' % ( self.docker_suffix, self.arch), 'test/distrib/csharp/run_distrib_test.sh') + elif self.platform == 'macos': + return create_jobspec(self.name, + ['test/distrib/csharp/run_distrib_test.sh'], + environ={'EXTERNAL_GIT_ROOT': '../../..'}) + else: + raise Exception("Not supported yet.") def __str__(self): return self.name @@ -176,6 +197,7 @@ def targets(): CSharpDistribTest('linux', 'x64', 'ubuntu1504'), CSharpDistribTest('linux', 'x64', 'ubuntu1510'), CSharpDistribTest('linux', 'x64', 'ubuntu1604'), + CSharpDistribTest('macos', 'x86'), PythonDistribTest('linux', 'x64', 'wheezy'), PythonDistribTest('linux', 'x64', 'jessie'), PythonDistribTest('linux', 'x86', 'jessie'), -- cgit v1.2.3 From 2713c5169fc8e5e540b69bda95d8b17aaac6adeb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Feb 2016 07:49:44 -0800 Subject: do not hardcode version to test --- test/distrib/csharp/run_distrib_test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/csharp/run_distrib_test.sh b/test/distrib/csharp/run_distrib_test.sh index 8ddb4c4bff..1de62041b3 100755 --- a/test/distrib/csharp/run_distrib_test.sh +++ b/test/distrib/csharp/run_distrib_test.sh @@ -34,8 +34,9 @@ cd $(dirname $0) unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets.zip" -d TestNugetFeed -# TODO(jtattermusch): replace the version number -./update_version.sh 0.13.0 +# Extract the version number from Grpc nuget package name. +CSHARP_VERSION=$(ls TestNugetFeed | grep '^Grpc\.[0-9].*\.nupkg$' | sed s/^Grpc\.// | sed s/\.nupkg$//) +./update_version.sh $CSHARP_VERSION nuget restore -- cgit v1.2.3 From 6677a86283989ea70803b2292b5836fbd8f0fe96 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Feb 2016 08:00:41 -0800 Subject: do not hardcode the python package version --- test/distrib/python/run_distrib_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index 08856ad388..d941869733 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -32,8 +32,8 @@ set -ex cd $(dirname $0) -# TODO(jtattermusch): replace the version number -SDIST_ARCHIVE="$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-0.12.0b8.tar.gz" +# Pick up the source dist archive whatever its version is +SDIST_ARCHIVE=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-*.tar.gz BDIST_DIR="file://$EXTERNAL_GIT_ROOT/input_artifacts" if [ ! -f "${SDIST_ARCHIVE}" ] -- cgit v1.2.3 From 0474a1c393811034b281b8fc014604055c30a862 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Feb 2016 08:34:02 -0800 Subject: python fixes --- test/distrib/python/run_distrib_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/python/run_distrib_test.sh b/test/distrib/python/run_distrib_test.sh index d941869733..8fd7ffb54c 100755 --- a/test/distrib/python/run_distrib_test.sh +++ b/test/distrib/python/run_distrib_test.sh @@ -36,7 +36,7 @@ cd $(dirname $0) SDIST_ARCHIVE=$EXTERNAL_GIT_ROOT/input_artifacts/grpcio-*.tar.gz BDIST_DIR="file://$EXTERNAL_GIT_ROOT/input_artifacts" -if [ ! -f "${SDIST_ARCHIVE}" ] +if [ ! -f ${SDIST_ARCHIVE} ] then echo "Archive ${SDIST_ARCHIVE} does not exist." exit 1 @@ -52,7 +52,7 @@ $PIP install --upgrade six GRPC_PYTHON_BINARIES_REPOSITORY="${BDIST_DIR}" \ $PIP install \ - "${SDIST_ARCHIVE}" + ${SDIST_ARCHIVE} $PYTHON distribtest.py -- cgit v1.2.3 From a24026f554dabbbd8a30517b497f7d7bc98aa8b8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 9 Feb 2016 08:49:52 -0800 Subject: fix node artifact building on mac --- test/distrib/node/run_distrib_test.sh | 7 ++----- tools/jenkins/docker_run.sh | 2 -- tools/run_tests/build_artifact_node.sh | 6 +++++- tools/run_tests/build_package_node.sh | 3 +++ 4 files changed, 10 insertions(+), 8 deletions(-) (limited to 'test/distrib') diff --git a/test/distrib/node/run_distrib_test.sh b/test/distrib/node/run_distrib_test.sh index ac9e7d7090..99a51f01f7 100755 --- a/test/distrib/node/run_distrib_test.sh +++ b/test/distrib/node/run_distrib_test.sh @@ -28,15 +28,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +NODE_VERSION=$1 +source ~/.nvm/nvm.sh set -ex cd $(dirname $0) -NODE_VERSION="$1" - -# make sure nvm is available -source ~/.nvm/nvm.sh || true - nvm install $NODE_VERSION npm install -g node-static diff --git a/tools/jenkins/docker_run.sh b/tools/jenkins/docker_run.sh index 519dc5cade..df7b6571d7 100755 --- a/tools/jenkins/docker_run.sh +++ b/tools/jenkins/docker_run.sh @@ -47,8 +47,6 @@ then rvm use ruby-2.1 fi -nvm use 4 || true - cd /var/local/git/grpc $RUN_COMMAND diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh index ec3fb6462f..8aeec79d08 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/build_artifact_node.sh @@ -28,8 +28,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +NODE_TARGET_ARCH=$1 +source ~/.nvm/nvm.sh set -ex +nvm use 4 + cd $(dirname $0)/../.. rm -rf build @@ -42,6 +46,6 @@ node_versions=( 0.10.41 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 ) for version in ${node_versions[@]} do - node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$1 + node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH cp -r build/stage/* artifacts/ done diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index 1358740264..a8b9448973 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -28,8 +28,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +source ~/.nvm/nvm.sh set -ex +nvm use 4 + cd $(dirname $0)/../.. mkdir -p artifacts/ -- cgit v1.2.3