diff options
42 files changed, 394 insertions, 358 deletions
diff --git a/.yardopts b/.yardopts new file mode 100644 index 0000000000..3bc779d421 --- /dev/null +++ b/.yardopts @@ -0,0 +1 @@ +src/ruby/**/*.rb
\ No newline at end of file diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 3c46d61170..072089ac51 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,6 +7,7 @@ graft third_party/zlib include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py include src/python/grpcio/grpc_core_dependencies.py +include src/python/grpcio/precompiled.py include src/python/grpcio/support.py include src/python/grpcio/README.rst include requirements.txt diff --git a/binding.gyp b/binding.gyp index 3659bfcf13..82e36c421d 100644 --- a/binding.gyp +++ b/binding.gyp @@ -55,7 +55,8 @@ 'UNICODE', '_UNICODE', 'NOMINMAX', - 'OPENSSL_NO_ASM' + 'OPENSSL_NO_ASM', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], "msvs_settings": { 'VCCLCompilerTool': { @@ -78,7 +79,8 @@ # supports ALPN. The target is "[major].[minor].[patch]". We split by # periods and take the first field to get the major version. 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)' + 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], 'include_dirs': [ '<(node_root_dir)/deps/openssl/openssl/include', diff --git a/examples/node/README.md b/examples/node/README.md index 09c56f7fa6..7a2bc9794f 100644 --- a/examples/node/README.md +++ b/examples/node/README.md @@ -4,14 +4,10 @@ gRPC in 3 minutes (Node.js) PREREQUISITES ------------- -- `node`: This requires Node 0.10.x or greater. -- [homebrew][] on Mac OS X. This simplifies the installation of the gRPC C core. +- `node`: This requires Node 0.12.x or greater. INSTALL ------- - - [Install gRPC Node][] - - - Install this package's dependencies ```sh $ cd examples/node @@ -35,15 +31,9 @@ TRY IT! $ node ./greeter_client.js ``` -NOTE ----- -This directory has a copy of `helloworld.proto` because it currently depends on -some Protocol Buffer 2.0 syntax that is deprecated in Protocol Buffer 3.0. - TUTORIAL -------- You can find a more detailed tutorial in [gRPC Basics: Node.js][] -[homebrew]:http://brew.sh [Install gRPC Node]:../../src/node [gRPC Basics: Node.js]:http://www.grpc.io/docs/tutorials/basic/node.html diff --git a/examples/node/greeter_client.js b/examples/node/greeter_client.js index 9b4b0a7782..ca5781514d 100644 --- a/examples/node/greeter_client.js +++ b/examples/node/greeter_client.js @@ -31,7 +31,7 @@ * */ -var PROTO_PATH = __dirname + '/helloworld.proto'; +var PROTO_PATH = __dirname + '/../protos/helloworld.proto'; var grpc = require('grpc'); var hello_proto = grpc.load(PROTO_PATH).helloworld; diff --git a/examples/node/greeter_server.js b/examples/node/greeter_server.js index 2712b3dd3a..47d9892816 100644 --- a/examples/node/greeter_server.js +++ b/examples/node/greeter_server.js @@ -31,7 +31,7 @@ * */ -var PROTO_PATH = __dirname + '/helloworld.proto'; +var PROTO_PATH = __dirname + '/../protos/helloworld.proto'; var grpc = require('grpc'); var hello_proto = grpc.load(PROTO_PATH).helloworld; diff --git a/examples/node/helloworld.proto b/examples/node/helloworld.proto deleted file mode 100644 index a52c947f89..0000000000 --- a/examples/node/helloworld.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option java_package = "ex.grpc"; - -package helloworld; - -// The greeting service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - optional string name = 1; -} - -// The response message containing the greetings -message HelloReply { - optional string message = 1; -} diff --git a/examples/node/package.json b/examples/node/package.json index 65c5789ed7..00ba428d96 100644 --- a/examples/node/package.json +++ b/examples/node/package.json @@ -2,6 +2,6 @@ "name": "grpc-examples", "version": "0.1.0", "dependencies": { - "grpc": "0.12.0" + "grpc": "0.13.0" } } diff --git a/examples/node/route_guide/route_guide.proto b/examples/node/route_guide/route_guide.proto deleted file mode 100644 index 38daa933cd..0000000000 --- a/examples/node/route_guide/route_guide.proto +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2015, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -option java_package = "io.grpc.routeguide"; - -package routeguide; - -// Interface exported by the server. -service RouteGuide { - // A simple RPC. - // - // Obtains the feature at a given position. - rpc GetFeature(Point) returns (Feature) {} - - // A server-to-client streaming RPC. - // - // Obtains the Features available within the given Rectangle. Results are - // streamed rather than returned at once (e.g. in a response message with a - // repeated field), as the rectangle may cover a large area and contain a - // huge number of features. - rpc ListFeatures(Rectangle) returns (stream Feature) {} - - // A client-to-server streaming RPC. - // - // Accepts a stream of Points on a route being traversed, returning a - // RouteSummary when traversal is completed. - rpc RecordRoute(stream Point) returns (RouteSummary) {} - - // A Bidirectional streaming RPC. - // - // Accepts a stream of RouteNotes sent while a route is being traversed, - // while receiving other RouteNotes (e.g. from other users). - rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} -} - -// Points are represented as latitude-longitude pairs in the E7 representation -// (degrees multiplied by 10**7 and rounded to the nearest integer). -// Latitudes should be in the range +/- 90 degrees and longitude should be in -// the range +/- 180 degrees (inclusive). -message Point { - optional int32 latitude = 1; - optional int32 longitude = 2; -} - -// A latitude-longitude rectangle, represented as two diagonally opposite -// points "lo" and "hi". -message Rectangle { - // One corner of the rectangle. - optional Point lo = 1; - - // The other corner of the rectangle. - optional Point hi = 2; -} - -// A feature names something at a given point. -// -// If a feature could not be named, the name is empty. -message Feature { - // The name of the feature. - optional string name = 1; - - // The point where the feature is detected. - optional Point location = 2; -} - -// A RouteNote is a message sent while at a given point. -message RouteNote { - // The location from which the message is sent. - optional Point location = 1; - - // The message to be sent. - optional string message = 2; -} - -// A RouteSummary is received in response to a RecordRoute rpc. -// -// It contains the number of individual points received, the number of -// detected features, and the total distance covered as the cumulative sum of -// the distance between each point. -message RouteSummary { - // The number of points received. - optional int32 point_count = 1; - - // The number of known features passed while traversing the route. - optional int32 feature_count = 2; - - // The distance covered in metres. - optional int32 distance = 3; - - // The duration of the traversal in seconds. - optional int32 elapsed_time = 4; -} diff --git a/examples/node/route_guide/route_guide_client.js b/examples/node/route_guide/route_guide_client.js index e38a21f422..6ff0279184 100644 --- a/examples/node/route_guide/route_guide_client.js +++ b/examples/node/route_guide/route_guide_client.js @@ -31,15 +31,17 @@ * */ +var PROTO_PATH = __dirname + '/../../protos/route_guide.proto'; + var async = require('async'); var fs = require('fs'); var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(__dirname + '/route_guide.proto').routeguide; +var routeguide = grpc.load(PROTO_PATH).routeguide; var client = new routeguide.RouteGuide('localhost:50051', - grpc.Credentials.createInsecure()); + grpc.credentials.createInsecure()); var COORD_FACTOR = 1e7; diff --git a/examples/node/route_guide/route_guide_server.js b/examples/node/route_guide/route_guide_server.js index 06cf2925e1..9fa9827991 100644 --- a/examples/node/route_guide/route_guide_server.js +++ b/examples/node/route_guide/route_guide_server.js @@ -31,12 +31,14 @@ * */ +var PROTO_PATH = __dirname + '/../../protos/route_guide.proto'; + var fs = require('fs'); var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(__dirname + '/route_guide.proto').routeguide; +var routeguide = grpc.load(PROTO_PATH).routeguide; var COORD_FACTOR = 1e7; diff --git a/grpc.gemspec b/grpc.gemspec index 9e4683f41c..32fe4932a9 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 2.0.0' - s.files = %w( Makefile ) + s.files = %w( Makefile .yardopts ) s.files += %w( etc/roots.pem ) s.files += Dir.glob('src/ruby/bin/**/*') s.files += Dir.glob('src/ruby/ext/**/*') @@ -31,7 +31,7 @@ Gem::Specification.new do |s| s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.2' + s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' diff --git a/include/grpc++/create_channel.h b/include/grpc++/create_channel.h index aff9c291c2..80eed067b7 100644 --- a/include/grpc++/create_channel.h +++ b/include/grpc++/create_channel.h @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,6 +36,7 @@ #include <memory> +#include <grpc++/channel.h> #include <grpc++/security/credentials.h> #include <grpc++/support/channel_arguments.h> #include <grpc++/support/config.h> diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 258063fd54..92569043fc 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -34,6 +34,14 @@ #ifndef GRPC_IMPL_CODEGEN_PORT_PLATFORM_H #define GRPC_IMPL_CODEGEN_PORT_PLATFORM_H +/* + * Define GPR_BACKWARDS_COMPATIBILITY_MODE to try harder to be ABI + * compatible with older platforms (currently only on Linux) + * Causes: + * - some libc calls to be gotten via dlsym + * - some syscalls to be made directly + */ + /* Get windows.h included everywhere (we need it) */ #if defined(_WIN64) || defined(WIN64) || defined(_WIN32) || defined(WIN32) #ifndef WIN32_LEAN_AND_MEAN diff --git a/package.json b/package.json index 1b79ecf92d..8cbfb29055 100644 --- a/package.json +++ b/package.json @@ -23,13 +23,12 @@ "test": "./node_modules/.bin/mocha src/node/test && npm run-script lint", "gen_docs": "./node_modules/.bin/jsdoc -c src/node/jsdoc_conf.json", "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", - "preinstall": "npm install node-pre-gyp", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, + "bundledDependencies": ["node-pre-gyp"], "dependencies": { "lodash": "^3.9.3", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.19", "protobufjs": "^4.0.0" }, "devDependencies": { diff --git a/requirements.txt b/requirements.txt index a1cc88cfe7..e3208e6355 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ futures>=2.2.0 cython>=0.23 coverage>=4.0 six>=1.10 +wheel>=0.29 @@ -53,6 +53,7 @@ sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. import commands +import precompiled import grpc_core_dependencies import grpc_version @@ -94,10 +95,10 @@ if "linux" in sys.platform: if not "win32" in sys.platform: EXTENSION_LIBRARIES += ('m',) -DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600)) +DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) -CFLAGS = () LDFLAGS = () +CFLAGS = () if "linux" in sys.platform: LDFLAGS += ('-Wl,-wrap,memcpy',) if "linux" in sys.platform or "darwin" in sys.platform: @@ -156,15 +157,14 @@ SETUP_REQUIRES = ( ) + INSTALL_REQUIRES COMMAND_CLASS = { - 'install': commands.Install, 'doc': commands.SphinxDocumentation, 'build_proto_modules': commands.BuildProtoModules, 'build_project_metadata': commands.BuildProjectMetadata, 'build_py': commands.BuildPy, 'build_ext': commands.BuildExt, + 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, - 'bdist_egg_grpc_custom': commands.BdistEggCustomName, } # Ensure that package data is copied over before any commands have been run: @@ -202,10 +202,13 @@ TEST_LOADER = 'tests:Loader' TEST_RUNNER = 'tests:Runner' PACKAGE_DATA = { + # Binaries that may or may not be present in the final installation, but are + # mentioned here for completeness. 'grpc._cython': [ '_credentials/roots.pem', '_windows/grpc_c.32.python', '_windows/grpc_c.64.python', + 'cygrpc.so', ], } if INSTALL_TESTS: @@ -215,19 +218,23 @@ else: PACKAGES = setuptools.find_packages( PYTHON_STEM, exclude=['tests', 'tests.*']) -setuptools.setup( - name='grpcio', - version=grpc_version.VERSION, - license=LICENSE, - ext_modules=CYTHON_EXTENSION_MODULES, - packages=list(PACKAGES), - package_dir=PACKAGE_DIRECTORIES, - package_data=PACKAGE_DATA, - install_requires=INSTALL_REQUIRES, - setup_requires=SETUP_REQUIRES, - cmdclass=COMMAND_CLASS, - tests_require=TESTS_REQUIRE, - test_suite=TEST_SUITE, - test_loader=TEST_LOADER, - test_runner=TEST_RUNNER, -) +setup_arguments = { + 'name': 'grpcio', + 'version': grpc_version.VERSION, + 'license': LICENSE, + 'ext_modules': CYTHON_EXTENSION_MODULES, + 'packages': list(PACKAGES), + 'package_dir': PACKAGE_DIRECTORIES, + 'package_data': PACKAGE_DATA, + 'install_requires': INSTALL_REQUIRES, + 'setup_requires': SETUP_REQUIRES, + 'cmdclass': COMMAND_CLASS, + 'tests_require': TESTS_REQUIRE, + 'test_suite': TEST_SUITE, + 'test_loader': TEST_LOADER, + 'test_runner': TEST_RUNNER, +} + +precompiled.update_setup_arguments(setup_arguments) + +setuptools.setup(**setup_arguments) diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c index 442cd8298e..1ca6fa1aff 100644 --- a/src/core/support/env_linux.c +++ b/src/core/support/env_linux.c @@ -52,6 +52,7 @@ #include "src/core/support/string.h" char *gpr_getenv(const char *name) { +#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE) typedef char *(*getenv_type)(const char *); static getenv_type getenv_func = NULL; /* Check to see which getenv variant is supported (go from most @@ -62,6 +63,10 @@ char *gpr_getenv(const char *name) { } char *result = getenv_func(name); return result == NULL ? result : gpr_strdup(result); +#else + char *result = secure_getenv(name); + return result == NULL ? result : gpr_strdup(result); +#endif } void gpr_setenv(const char *name, const char *value) { diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c index 1f92d7f090..36d75e7da2 100644 --- a/src/core/support/time_posix.c +++ b/src/core/support/time_posix.c @@ -86,7 +86,7 @@ gpr_timespec gpr_now(gpr_clock_type clock_type) { gpr_precise_clock_now(&ret); return ret; } else { -#if defined(__linux__) && !defined(GPR_NO_DIRECT_SYSCALLS) +#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE) && defined(__linux__) /* avoid ABI problems by invoking syscalls directly */ syscall(SYS_clock_gettime, clockid_for_gpr_clock[clock_type], &now); #else diff --git a/src/core/surface/validate_metadata.c b/src/core/surface/validate_metadata.c index df2e80b4b7..bf4126867f 100644 --- a/src/core/surface/validate_metadata.c +++ b/src/core/surface/validate_metadata.c @@ -50,7 +50,7 @@ static int conforms_to(const char *s, size_t len, const uint8_t *legal_bits) { int grpc_header_key_is_legal(const char *key, size_t length) { static const uint8_t legal_header_bits[256 / 8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xff, 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0x00, 0x00, 0x00, 0x80, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; if (length == 0) { diff --git a/src/node/performance/worker_server.js b/src/node/performance/worker.js index 7c8ab00026..7c8ab00026 100644 --- a/src/node/performance/worker_server.js +++ b/src/node/performance/worker.js diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index c7b5a3bde4..698c760ebe 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -1,22 +1,40 @@ gRPC Python =========== -Package for GRPC Python. +Package for gRPC Python. -Dependencies +Installation ------------ -Ensure you have installed the gRPC core. On Mac OS X, install homebrew_. -Run the following command to install gRPC Python. +gRPC Python is available for Linux and Mac OS X running Python 2.7. + +From PyPI +~~~~~~~~~ + +If you are installing locally... :: - $ curl -fsSL https://goo.gl/getgrpc | bash -s python + $ pip install grpcio + +Else system wide (on Ubuntu)... + +:: -This will download and run the [gRPC install script][] to install grpc core. The script then uses pip to install this package. It also installs the Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin for python. + $ sudo pip install grpcio + +From Source +~~~~~~~~~~~ + +Building from source requires that you have the Python headers (usually a +package named `python-dev`). + +:: -Otherwise, `install from source`_ + $ export REPO_ROOT=grpc + $ git clone https://github.com/grpc/grpc.git $REPO_ROOT + $ cd $REPO_ROOT + $ pip install . -.. _`install from source`: https://github.com/grpc/grpc/blob/master/src/python/README.md#building-from-source -.. _homebrew: http://brew.sh -.. _`gRPC install script`: https://raw.githubusercontent.com/grpc/homebrew-grpc/master/scripts/install +Note that `$REPO_ROOT` can be assigned to whatever directory name floats your +fancy. diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 774e7ad6a1..aa29c728f2 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -41,7 +41,6 @@ import sys import traceback import setuptools -from setuptools.command import bdist_egg from setuptools.command import build_ext from setuptools.command import build_py from setuptools.command import easy_install @@ -52,13 +51,6 @@ import support PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) -BINARIES_REPOSITORY = os.environ.get( - 'GRPC_PYTHON_BINARIES_REPOSITORY', - 'https://storage.googleapis.com/grpc-precompiled-binaries/python') - -USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get( - 'GRPC_PYTHON_USE_CUSTOM_BDIST', '1'))) - CONF_PY_ADDENDUM = """ extensions.append('sphinx.ext.napoleon') napoleon_google_docstring = True @@ -74,126 +66,39 @@ class CommandError(Exception): # TODO(atash): Remove this once PyPI has better Linux bdist support. See # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported -def _get_grpc_custom_bdist_egg(decorated_basename, target_egg_basename): - """Returns a string path to a .egg file for Linux to install. +def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): + """Returns a string path to a bdist file for Linux to install. - If we can retrieve a pre-compiled egg from online, uses it. Else, emits a + If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a warning and builds from source. """ + # TODO(atash): somehow the name that's returned from `wheel` is different + # between different versions of 'wheel' (but from a compatibility standpoint, + # the names are compatible); we should have some way of determining name + # compatibility in the same way `wheel` does to avoid having to rename all of + # the custom wheels that we build/upload to GCS. + # Break import style to ensure that setup.py has had a chance to install the - # relevant package eggs. + # relevant package. from six.moves.urllib import request - decorated_path = decorated_basename + '.egg' + decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT try: url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path) - egg_data = request.urlopen(url).read() + bdist_data = request.urlopen(url).read() except IOError as error: raise CommandError( - '{}\n\nCould not find the bdist egg {}: {}' + '{}\n\nCould not find the bdist {}: {}' .format(traceback.format_exc(), decorated_path, error.message)) - # Our chosen local egg path. - egg_path = target_egg_basename + '.egg' + # Our chosen local bdist path. + bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT try: - with open(egg_path, 'w') as egg_file: - egg_file.write(egg_data) + with open(bdist_path, 'w') as bdist_file: + bdist_file.write(bdist_data) except IOError as error: raise CommandError( - '{}\n\nCould not write grpcio egg: {}' + '{}\n\nCould not write grpcio bdist: {}' .format(traceback.format_exc(), error.message)) - return egg_path - - -class EggNameMixin(object): - """Mixin for setuptools.Command classes to enable acquiring the egg name.""" - - def egg_name(self, with_custom): - """ - Args: - with_custom: Boolean describing whether or not to decorate the egg name - with custom gRPC-specific target information. - """ - egg_command = self.get_finalized_command('bdist_egg') - base = os.path.splitext(os.path.basename(egg_command.egg_output))[0] - if with_custom: - flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' - return '{base}-{flavor}'.format(base=base, flavor=flavor) - else: - return base - - -class Install(install.install, EggNameMixin): - """Custom Install command for gRPC Python. - - This is for bdist shims and whatever else we might need a custom install - command for. - """ - - user_options = install.install.user_options + [ - # TODO(atash): remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - ('use-grpc-custom-bdist', None, - 'Whether to retrieve a binary from the gRPC binary repository instead ' - 'of building from source.'), - ] - - def initialize_options(self): - install.install.initialize_options(self) - self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST - - def finalize_options(self): - install.install.finalize_options(self) - - def run(self): - if self.use_grpc_custom_bdist: - try: - try: - egg_path = _get_grpc_custom_bdist_egg(self.egg_name(True), - self.egg_name(False)) - except CommandError as error: - sys.stderr.write( - '\nWARNING: Failed to acquire grpcio prebuilt binary:\n' - '{}.\n\n'.format(error.message)) - raise - try: - self._run_bdist_retrieval_install(egg_path) - except Exception as error: - # if anything else happens (and given how there's no way to really know - # what's happening in setuptools here, I mean *anything*), warn the user - # and fall back to building from source. - sys.stderr.write( - '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n' - .format(traceback.format_exc())) - raise - except Exception: - install.install.run(self) - else: - install.install.run(self) - - # TODO(atash): Remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - def _run_bdist_retrieval_install(self, bdist_egg): - easy_install = self.distribution.get_command_class('easy_install') - easy_install_command = easy_install( - self.distribution, args='x', root=self.root, record=self.record, - ) - easy_install_command.ensure_finalized() - easy_install_command.always_copy_from = '.' - easy_install_command.package_index.scan(glob.glob('*.egg')) - arguments = [bdist_egg] - if setuptools.bootstrap_install_from: - args.insert(0, setuptools.bootstrap_install_from) - easy_install_command.args = arguments - easy_install_command.run() - setuptools.bootstrap_install_from = None - - -class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin): - """Thin wrapper around the bdist_egg command to build with our custom name.""" - - def run(self): - bdist_egg.bdist_egg.run(self) - target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True))) - shutil.move(self.get_outputs()[0], target) + return bdist_path class SphinxDocumentation(setuptools.Command): diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py new file mode 100644 index 0000000000..05c651b506 --- /dev/null +++ b/src/python/grpcio/precompiled.py @@ -0,0 +1,102 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import platform +import shutil +import sys + +import setuptools + +import commands +import grpc_version + +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen + +PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) +BINARIES_REPOSITORY = os.environ.get( + 'GRPC_PYTHON_BINARIES_REPOSITORY', + 'https://storage.googleapis.com/grpc-precompiled-binaries/python') +USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( + 'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1'))) + +def _tagged_ext_name(base): + uname = platform.uname() + tags = '-'.join((grpc_version.VERSION, uname[0], uname[4])) + flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' + return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor) + + +class BuildTaggedExt(setuptools.Command): + + description = 'build the gRPC tagged extensions' + user_options = [] + + def initialize_options(self): + # distutils requires this override. + pass + + def finalize_options(self): + # distutils requires this override. + pass + + def run(self): + if 'linux' in sys.platform: + self.run_command('build_ext') + try: + os.makedirs('dist/') + except OSError: + pass + shutil.copyfile( + os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'), + 'dist/{}.so'.format(_tagged_ext_name('cygrpc'))) + else: + sys.stderr.write('nothing to do for build_tagged_ext\n') + + +def update_setup_arguments(setup_arguments): + url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc')) + target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so') + try: + extension = urlopen(url).read() + except: + sys.stderr.write( + 'could not download precompiled extension: {}\n'.format(url)) + return + try: + with open(target_path, 'w') as target: + target.write(extension) + setup_arguments['ext_modules'] = [] + except: + sys.stderr.write( + 'could not write precompiled extension to directory: {} -> {}\n' + .format(url, target_path)) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index b7c6cb3d7e..6b7001a489 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -79,6 +79,7 @@ unless File.exist?(File.join(grpc_lib_dir, 'libgrpc.a')) or windows ENV['EMBED_ZLIB'] = 'true' ENV['ARCH_FLAGS'] = RbConfig::CONFIG['ARCH_FLAG'] ENV['ARCH_FLAGS'] = '-arch i386 -arch x86_64' if RUBY_PLATFORM =~ /darwin/ + ENV['CFLAGS'] = '-DGPR_BACKWARDS_COMPATIBILITY_MODE' output_dir = File.expand_path(RbConfig::CONFIG['topdir']) grpc_lib_dir = File.join(output_dir, 'libs', grpc_config) diff --git a/templates/binding.gyp.template b/templates/binding.gyp.template index 7cee562caa..3ffbc06643 100644 --- a/templates/binding.gyp.template +++ b/templates/binding.gyp.template @@ -57,7 +57,8 @@ 'UNICODE', '_UNICODE', 'NOMINMAX', - 'OPENSSL_NO_ASM' + 'OPENSSL_NO_ASM', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], "msvs_settings": { 'VCCLCompilerTool': { @@ -80,7 +81,8 @@ # supports ALPN. The target is "[major].[minor].[patch]". We split by # periods and take the first field to get the major version. 'defines': [ - 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)' + 'TSI_OPENSSL_ALPN_SUPPORT=<!(echo <(target) | cut -d. -f1)', + 'GPR_BACKWARDS_COMPATIBILITY_MODE' ], 'include_dirs': [ '<(node_root_dir)/deps/openssl/openssl/include', diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index 6cc3e964a3..701e1c7485 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -17,7 +17,7 @@ s.required_ruby_version = '>= 2.0.0' - s.files = %w( Makefile ) + s.files = %w( Makefile .yardopts ) s.files += %w( etc/roots.pem ) s.files += Dir.glob('src/ruby/bin/**/*') s.files += Dir.glob('src/ruby/ext/**/*') @@ -33,7 +33,7 @@ s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) s.platform = Gem::Platform::RUBY - s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.2' + s.add_dependency 'google-protobuf', '~> 3.0.0.alpha.5.0.3' s.add_dependency 'googleauth', '~> 0.5.1' s.add_development_dependency 'bundler', '~> 1.9' diff --git a/templates/package.json.template b/templates/package.json.template index d6279b996e..99e8287b35 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -25,13 +25,12 @@ "test": "./node_modules/.bin/mocha src/node/test && npm run-script lint", "gen_docs": "./node_modules/.bin/jsdoc -c src/node/jsdoc_conf.json", "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test", - "preinstall": "npm install node-pre-gyp", "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build" }, + "bundledDependencies": ["node-pre-gyp"], "dependencies": { "lodash": "^3.9.3", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.19", "protobufjs": "^4.0.0" }, "devDependencies": { diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 252bda3798..a194c615cd 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -479,8 +479,10 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { send_request.set_message("Hello"); std::pair<grpc::string, grpc::string> meta1("key1", "val1"); std::pair<grpc::string, grpc::string> meta2("key2", "val2"); + std::pair<grpc::string, grpc::string> meta3("g.r.d-bin", "xyz"); cli_ctx.AddMetadata(meta1.first, meta1.second); cli_ctx.AddMetadata(meta2.first, meta2.second); + cli_ctx.AddMetadata(meta3.first, meta3.second); std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader( stub_->AsyncEcho(&cli_ctx, send_request, cq_.get())); @@ -494,6 +496,8 @@ TEST_P(AsyncEnd2endTest, ClientInitialMetadataRpc) { ToString(client_initial_metadata.find(meta1.first)->second)); EXPECT_EQ(meta2.second, ToString(client_initial_metadata.find(meta2.first)->second)); + EXPECT_EQ(meta3.second, + ToString(client_initial_metadata.find(meta3.first)->second)); EXPECT_GE(client_initial_metadata.size(), static_cast<size_t>(2)); send_response.set_message(recv_request.message()); diff --git a/test/distrib/node/run_distrib_test.sh b/test/distrib/node/run_distrib_test.sh index 99a51f01f7..9b8f15771b 100755 --- a/test/distrib/node/run_distrib_test.sh +++ b/test/distrib/node/run_distrib_test.sh @@ -38,6 +38,9 @@ nvm install $NODE_VERSION npm install -g node-static +# Kill off existing static servers +kill -9 $(ps aux | grep '[n]ode .*static' | awk '{print $2}') || true + STATIC_SERVER=127.0.0.1 STATIC_PORT=8080 diff --git a/tools/README.md b/tools/README.md index eb6633a866..a0c41eb79f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,6 +8,8 @@ dockerfile: Docker files to test gRPC. doxygen: gRPC C/C++ documentation generation via Doxygen. +gce: scripts to help setup testing infrastructure on GCE. + jenkins: support for running tests on Jenkins. profile_analyzer: pretty printer for gRPC profiling data. diff --git a/tools/codegen/core/gen_legal_metadata_characters.c b/tools/codegen/core/gen_legal_metadata_characters.c index 3c9e1c7619..10605d52bf 100644 --- a/tools/codegen/core/gen_legal_metadata_characters.c +++ b/tools/codegen/core/gen_legal_metadata_characters.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -52,7 +52,7 @@ static void legal(int x) { static void dump(void) { int i; - printf("static const gpr_uint8 legal_header_bits[256/8] = "); + printf("static const uint8_t legal_header_bits[256/8] = "); for (i = 0; i < 256 / 8; i++) printf("%c 0x%02x", i ? ',' : '{', legal_bits[i]); printf(" };\n"); diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile index b44fcff7e3..1eb4c1e775 100644 --- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile @@ -29,4 +29,9 @@ FROM fedora:21 +# Make yum work properly under docker when using overlay storage driver. +# https://bugzilla.redhat.com/show_bug.cgi?id=1213602#c9 +# https://github.com/docker/docker/issues/10180 +RUN yum install -y yum-plugin-ovl + RUN yum clean all && yum update -y && yum install -y python python-pip diff --git a/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile index 598dac5a11..b567c5b109 100644 --- a/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/ruby_fedora21_x64/Dockerfile @@ -29,6 +29,11 @@ FROM fedora:21 +# Make yum work properly under docker when using overlay storage driver. +# https://bugzilla.redhat.com/show_bug.cgi?id=1213602#c9 +# https://github.com/docker/docker/issues/10180 +RUN yum install -y yum-plugin-ovl + RUN yum clean all && yum update -y && yum install -y ruby RUN gem install bundler diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh new file mode 100755 index 0000000000..2a9e77ab17 --- /dev/null +++ b/tools/gce/create_linux_worker.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Creates a standard jenkins worker on GCE. + +set -ex + +cd $(dirname $0) + +CLOUD_PROJECT=grpc-testing +ZONE=us-central1-a + +INSTANCE_NAME=grpc-jenkins-worker1 + +gcloud compute instances create $INSTANCE_NAME \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + --machine-type n1-standard-8 \ + --image ubuntu-14-04 \ + --boot-disk-size 1000 + +echo 'Created GCE instance, waiting 60 seconds for it to come online.' +sleep 60 + +gcloud compute copy-files \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + jenkins_master.pub linux_worker_init.sh ${INSTANCE_NAME}:~ + +gcloud compute ssh \ + --project="$CLOUD_PROJECT" \ + --zone "$ZONE" \ + $INSTANCE_NAME --command "./linux_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub new file mode 100644 index 0000000000..e9853224e1 --- /dev/null +++ b/tools/gce/jenkins_master.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh new file mode 100755 index 0000000000..f56cac0ce2 --- /dev/null +++ b/tools/gce/linux_worker_init.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Initializes a fresh GCE VM to become a jenkins linux worker. +# You shouldn't run this script on your own, use create_linux_worker.sh +# instead. + +set -ex + +sudo apt-get update + +# Install JRE +sudo apt-get install -y openjdk-7-jre +sudo apt-get install -y unzip lsof + +# Install Docker +curl -sSL https://get.docker.com/ | sh + +# Setup jenkins user (or the user will already exist bcuz magic) +sudo adduser jenkins --disabled-password || true + +# Enable jenkins to use docker without sudo: +sudo usermod -aG docker jenkins + +# Use "overlay" storage driver for docker +# see https://github.com/grpc/grpc/issues/4988 +echo 'DOCKER_OPTS="${DOCKER_OPTS} --storage-driver=overlay"' | sudo tee --append /etc/default/docker + +# Install RVM +# TODO(jtattermusch): why is RVM needed? +gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +curl -sSL https://get.rvm.io | bash -s stable --ruby + +# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ +# This needs to happen as the last step to prevent Jenkins master from connecting +# to a machine that hasn't been properly setup yet. +cat jenkins_master.pub | sudo tee --append ~jenkins/.ssh/authorized_keys + +# Restart for docker to pickup the config changes. +echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' +sleep 10 + +sudo reboot diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 74d2a4859c..9cd02c5e43 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -175,7 +175,8 @@ class CSharpExtArtifact: else: environ = {'CONFIG': 'opt', 'EMBED_OPENSSL': 'true', - 'EMBED_ZLIB': 'true'} + 'EMBED_ZLIB': 'true', + 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'} if self.platform == 'linux': return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, diff --git a/tools/run_tests/build_artifact_node.sh b/tools/run_tests/build_artifact_node.sh index 9a3b9bd1ba..6aa4824538 100755 --- a/tools/run_tests/build_artifact_node.sh +++ b/tools/run_tests/build_artifact_node.sh @@ -36,7 +36,7 @@ nvm use 4 cd $(dirname $0)/../.. -rm -rf build +rm -rf build || true mkdir -p artifacts @@ -46,6 +46,6 @@ node_versions=( 0.12.0 1.0.0 1.1.0 2.0.0 3.0.0 4.0.0 5.0.0 ) for version in ${node_versions[@]} do - node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH + ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=$NODE_TARGET_ARCH cp -r build/stage/* artifacts/ done diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 835fad83e1..6e7ab911d5 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -39,12 +39,20 @@ then pip install -rrequirements.txt fi +# The bdist_wheel_grpc_custom command is finicky about command output ordering +# and thus ought to be run in a shell command separate of others. Further, it +# trashes the actual bdist_wheel output, so it should be run first so that +# bdist_wheel may be run unmolested. +GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ +GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ +${SETARCH_CMD} python setup.py \ + build_tagged_ext + GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ bdist_wheel \ - sdist \ - bdist_egg_grpc_custom + sdist mkdir -p artifacts diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index a8b9448973..aca90a3750 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -38,6 +38,7 @@ cd $(dirname $0)/../.. mkdir -p artifacts/ cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* artifacts/ || true +npm update npm pack cp grpc-*.tgz artifacts/grpc.tgz diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py index b26a870778..261f44bc6d 100644 --- a/tools/run_tests/distribtest_targets.py +++ b/tools/run_tests/distribtest_targets.py @@ -122,11 +122,15 @@ class NodeDistribTest(object): def build_jobspec(self): if self.platform == 'linux': + linux32 = '' + if self.arch == 'x86': + linux32 = 'linux32' return create_docker_jobspec(self.name, 'tools/dockerfile/distribtest/node_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/node/run_distrib_test.sh %s' % ( + '%s test/distrib/node/run_distrib_test.sh %s' % ( + linux32, self.node_version)) elif self.platform == 'macos': return create_jobspec(self.name, @@ -236,11 +240,7 @@ def targets(): RubyDistribTest('linux', 'x64', 'ubuntu1504'), RubyDistribTest('linux', 'x64', 'ubuntu1510'), RubyDistribTest('linux', 'x64', 'ubuntu1604'), - NodeDistribTest('macos', 'x64', None, '0.10'), - NodeDistribTest('macos', 'x64', None, '0.12'), - NodeDistribTest('macos', 'x64', None, '3'), NodeDistribTest('macos', 'x64', None, '4'), - NodeDistribTest('macos', 'x64', None, '5'), NodeDistribTest('linux', 'x86', 'jessie', '4') ] + [ NodeDistribTest('linux', 'x64', os, version) |