diff options
Diffstat (limited to 'tools/distrib/python')
-rwxr-xr-x | tools/distrib/python/check_grpcio_tools.py | 11 | ||||
-rwxr-xr-x | tools/distrib/python/docgen.py | 135 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/grpc_tools/__init__.py | 1 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/grpc_tools/command.py | 63 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/grpc_tools/protoc.py | 12 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/grpc_version.py | 2 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/setup.py | 251 | ||||
-rwxr-xr-x | tools/distrib/python/make_grpcio_tools.py | 149 | ||||
-rwxr-xr-x | tools/distrib/python/submit.py | 67 |
9 files changed, 370 insertions, 321 deletions
diff --git a/tools/distrib/python/check_grpcio_tools.py b/tools/distrib/python/check_grpcio_tools.py index b56ccaea7a..2363017113 100755 --- a/tools/distrib/python/check_grpcio_tools.py +++ b/tools/distrib/python/check_grpcio_tools.py @@ -23,12 +23,11 @@ Have you called tools/distrib/python/make_grpcio_tools.py since upgrading protob submodule_commit_hash = _make.protobuf_submodule_commit_hash() with open(_make.GRPC_PYTHON_PROTOC_LIB_DEPS, 'r') as _protoc_lib_deps_file: - content = _protoc_lib_deps_file.read().splitlines() + content = _protoc_lib_deps_file.read().splitlines() -testString = (_make.COMMIT_HASH_PREFIX + - submodule_commit_hash + - _make.COMMIT_HASH_SUFFIX) +testString = ( + _make.COMMIT_HASH_PREFIX + submodule_commit_hash + _make.COMMIT_HASH_SUFFIX) if testString not in content: - print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS)) - raise SystemExit(1) + print(OUT_OF_DATE_MESSAGE.format(_make.GRPC_PYTHON_PROTOC_LIB_DEPS)) + raise SystemExit(1) diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index 1822e51d09..4d6fcb5d65 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -24,14 +24,20 @@ import sys import tempfile parser = argparse.ArgumentParser() -parser.add_argument('--config', metavar='c', type=str, nargs=1, - help='GRPC/GPR libraries build configuration', - default='opt') +parser.add_argument( + '--config', + metavar='c', + type=str, + nargs=1, + help='GRPC/GPR libraries build configuration', + default='opt') parser.add_argument('--submit', action='store_true') parser.add_argument('--gh-user', type=str, help='GitHub user to push as.') -parser.add_argument('--gh-repo-owner', type=str, - help=('Owner of the GitHub repository to be pushed; ' - 'defaults to --gh-user.')) +parser.add_argument( + '--gh-repo-owner', + type=str, + help=('Owner of the GitHub repository to be pushed; ' + 'defaults to --gh-user.')) parser.add_argument('--doc-branch', type=str) args = parser.parse_args() @@ -59,60 +65,75 @@ environment.update({ }) subprocess_arguments_list = [ - {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, - {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==9.0.1'], - 'env': environment}, - {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], - 'env': environment}, - {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment}, - {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], 'env': environment}, + { + 'args': ['virtualenv', VIRTUALENV_DIR], + 'env': environment + }, + { + 'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip==9.0.1'], + 'env': environment + }, + { + 'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], + 'env': environment + }, + { + 'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], + 'env': environment + }, + { + 'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], + 'env': environment + }, ] for subprocess_arguments in subprocess_arguments_list: - print('Running command: {}'.format(subprocess_arguments['args'])) - subprocess.check_call(**subprocess_arguments) + print('Running command: {}'.format(subprocess_arguments['args'])) + subprocess.check_call(**subprocess_arguments) if args.submit: - assert args.gh_user - assert args.doc_branch - github_user = args.gh_user - github_repository_owner = ( - args.gh_repo_owner if args.gh_repo_owner else args.gh_user) - # Create a temporary directory out of tree, checkout gh-pages from the - # specified repository, edit it, and push it. It's up to the user to then go - # onto GitHub and make a PR against grpc/grpc:gh-pages. - repo_parent_dir = tempfile.mkdtemp() - print('Documentation parent directory: {}'.format(repo_parent_dir)) - repo_dir = os.path.join(repo_parent_dir, 'grpc') - python_doc_dir = os.path.join(repo_dir, 'python') - doc_branch = args.doc_branch + assert args.gh_user + assert args.doc_branch + github_user = args.gh_user + github_repository_owner = (args.gh_repo_owner + if args.gh_repo_owner else args.gh_user) + # Create a temporary directory out of tree, checkout gh-pages from the + # specified repository, edit it, and push it. It's up to the user to then go + # onto GitHub and make a PR against grpc/grpc:gh-pages. + repo_parent_dir = tempfile.mkdtemp() + print('Documentation parent directory: {}'.format(repo_parent_dir)) + repo_dir = os.path.join(repo_parent_dir, 'grpc') + python_doc_dir = os.path.join(repo_dir, 'python') + doc_branch = args.doc_branch - print('Cloning your repository...') - subprocess.check_call([ - 'git', 'clone', 'https://{}@github.com/{}/grpc'.format( - github_user, github_repository_owner) - ], cwd=repo_parent_dir) - subprocess.check_call([ - 'git', 'remote', 'add', 'upstream', 'https://github.com/grpc/grpc' - ], cwd=repo_dir) - subprocess.check_call(['git', 'fetch', 'upstream'], cwd=repo_dir) - subprocess.check_call([ - 'git', 'checkout', 'upstream/gh-pages', '-b', doc_branch - ], cwd=repo_dir) - print('Updating documentation...') - shutil.rmtree(python_doc_dir, ignore_errors=True) - shutil.copytree(DOC_PATH, python_doc_dir) - print('Attempting to push documentation...') - try: - subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir) - subprocess.check_call([ - 'git', 'commit', '-m', 'Auto-update Python documentation' - ], cwd=repo_dir) - subprocess.check_call([ - 'git', 'push', '--set-upstream', 'origin', doc_branch - ], cwd=repo_dir) - except subprocess.CalledProcessError: - print('Failed to push documentation. Examine this directory and push ' - 'manually: {}'.format(repo_parent_dir)) - sys.exit(1) - shutil.rmtree(repo_parent_dir) + print('Cloning your repository...') + subprocess.check_call( + [ + 'git', 'clone', 'https://{}@github.com/{}/grpc'.format( + github_user, github_repository_owner) + ], + cwd=repo_parent_dir) + subprocess.check_call( + ['git', 'remote', 'add', 'upstream', 'https://github.com/grpc/grpc'], + cwd=repo_dir) + subprocess.check_call(['git', 'fetch', 'upstream'], cwd=repo_dir) + subprocess.check_call( + ['git', 'checkout', 'upstream/gh-pages', '-b', doc_branch], + cwd=repo_dir) + print('Updating documentation...') + shutil.rmtree(python_doc_dir, ignore_errors=True) + shutil.copytree(DOC_PATH, python_doc_dir) + print('Attempting to push documentation...') + try: + subprocess.check_call(['git', 'add', '--all'], cwd=repo_dir) + subprocess.check_call( + ['git', 'commit', '-m', 'Auto-update Python documentation'], + cwd=repo_dir) + subprocess.check_call( + ['git', 'push', '--set-upstream', 'origin', doc_branch], + cwd=repo_dir) + except subprocess.CalledProcessError: + print('Failed to push documentation. Examine this directory and push ' + 'manually: {}'.format(repo_parent_dir)) + sys.exit(1) + shutil.rmtree(repo_parent_dir) diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py index 1454b678e1..5772620b60 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/command.py b/tools/distrib/python/grpcio_tools/grpc_tools/command.py index 28be137045..c0f9d93b0a 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/command.py @@ -22,43 +22,44 @@ from grpc_tools import protoc def build_package_protos(package_root): - proto_files = [] - inclusion_root = os.path.abspath(package_root) - for root, _, files in os.walk(inclusion_root): - for filename in files: - if filename.endswith('.proto'): - proto_files.append(os.path.abspath(os.path.join(root, filename))) + proto_files = [] + inclusion_root = os.path.abspath(package_root) + for root, _, files in os.walk(inclusion_root): + for filename in files: + if filename.endswith('.proto'): + proto_files.append( + os.path.abspath(os.path.join(root, filename))) - well_known_protos_include = pkg_resources.resource_filename( - 'grpc_tools', '_proto') + well_known_protos_include = pkg_resources.resource_filename('grpc_tools', + '_proto') - for proto_file in proto_files: - command = [ - 'grpc_tools.protoc', - '--proto_path={}'.format(inclusion_root), - '--proto_path={}'.format(well_known_protos_include), - '--python_out={}'.format(inclusion_root), - '--grpc_python_out={}'.format(inclusion_root), - ] + [proto_file] - if protoc.main(command) != 0: - sys.stderr.write('warning: {} failed'.format(command)) + for proto_file in proto_files: + command = [ + 'grpc_tools.protoc', + '--proto_path={}'.format(inclusion_root), + '--proto_path={}'.format(well_known_protos_include), + '--python_out={}'.format(inclusion_root), + '--grpc_python_out={}'.format(inclusion_root), + ] + [proto_file] + if protoc.main(command) != 0: + sys.stderr.write('warning: {} failed'.format(command)) class BuildPackageProtos(setuptools.Command): - """Command to generate project *_pb2.py modules from proto files.""" + """Command to generate project *_pb2.py modules from proto files.""" - description = 'build grpc protobuf modules' - user_options = [] + description = 'build grpc protobuf modules' + user_options = [] - def initialize_options(self): - pass + def initialize_options(self): + pass - def finalize_options(self): - pass + def finalize_options(self): + pass - def run(self): - # due to limitations of the proto generator, we require that only *one* - # directory is provided as an 'include' directory. We assume it's the '' key - # to `self.distribution.package_dir` (and get a key error if it's not - # there). - build_package_protos(self.distribution.package_dir['']) + def run(self): + # due to limitations of the proto generator, we require that only *one* + # directory is provided as an 'include' directory. We assume it's the '' key + # to `self.distribution.package_dir` (and get a key error if it's not + # there). + build_package_protos(self.distribution.package_dir['']) diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py index efad51e07b..582cba0e39 100644 --- a/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py +++ b/tools/distrib/python/grpcio_tools/grpc_tools/protoc.py @@ -19,16 +19,18 @@ import sys from grpc_tools import _protoc_compiler + def main(command_arguments): - """Run the protocol buffer compiler with the given command-line arguments. + """Run the protocol buffer compiler with the given command-line arguments. Args: command_arguments: a list of strings representing command line arguments to `protoc`. """ - command_arguments = [argument.encode() for argument in command_arguments] - return _protoc_compiler.run_main(command_arguments) + command_arguments = [argument.encode() for argument in command_arguments] + return _protoc_compiler.run_main(command_arguments) + if __name__ == '__main__': - proto_include = pkg_resources.resource_filename('grpc_tools', '_proto') - sys.exit(main(sys.argv + ['-I{}'.format(proto_include)])) + proto_include = pkg_resources.resource_filename('grpc_tools', '_proto') + sys.exit(main(sys.argv + ['-I{}'.format(proto_include)])) diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f613025be3..c4ed066122 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.9.0.dev0' +VERSION = '1.9.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 8d95cb5a28..342a220d5e 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -66,42 +66,42 @@ BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) if EXTRA_ENV_COMPILE_ARGS is None: - EXTRA_ENV_COMPILE_ARGS = '-std=c++11' - if 'win32' in sys.platform: - if sys.version_info < (3, 5): - # We use define flags here and don't directly add to DEFINE_MACROS below to - # ensure that the expert user/builder has a way of turning it off (via the - # envvars) without adding yet more GRPC-specific envvars. - # See https://sourceforge.net/p/mingw-w64/bugs/363/ - if '32' in platform.architecture()[0]: - EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot' - else: - EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot' - else: - # We need to statically link the C++ Runtime, only the C runtime is - # available dynamically - EXTRA_ENV_COMPILE_ARGS += ' /MT' - elif "linux" in sys.platform or "darwin" in sys.platform: - EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti' + EXTRA_ENV_COMPILE_ARGS = '-std=c++11' + if 'win32' in sys.platform: + if sys.version_info < (3, 5): + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot' + else: + # We need to statically link the C++ Runtime, only the C runtime is + # available dynamically + EXTRA_ENV_COMPILE_ARGS += ' /MT' + elif "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti' if EXTRA_ENV_LINK_ARGS is None: - EXTRA_ENV_LINK_ARGS = '' - if "linux" in sys.platform or "darwin" in sys.platform: - EXTRA_ENV_LINK_ARGS += ' -lpthread' - elif "win32" in sys.platform and sys.version_info < (3, 5): - msvcr = cygwinccompiler.get_msvcr()[0] - # TODO(atash) sift through the GCC specs to see if libstdc++ can have any - # influence on the linkage outcome on MinGW for non-C++ programs. - EXTRA_ENV_LINK_ARGS += ( - ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' - '-static'.format(msvcr=msvcr)) + EXTRA_ENV_LINK_ARGS = '' + if "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_LINK_ARGS += ' -lpthread' + elif "win32" in sys.platform and sys.version_info < (3, 5): + msvcr = cygwinccompiler.get_msvcr()[0] + # TODO(atash) sift through the GCC specs to see if libstdc++ can have any + # influence on the linkage outcome on MinGW for non-C++ programs. + EXTRA_ENV_LINK_ARGS += ( + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' + '-static'.format(msvcr=msvcr)) EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) -CC_FILES = [ - os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] PROTO_FILES = [ - os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES] + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES +] CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) @@ -110,107 +110,114 @@ GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' DEFINE_MACROS = () if "win32" in sys.platform: - DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) - if '64bit' in platform.architecture()[0]: - DEFINE_MACROS += (('MS_WIN64', 1),) + DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) + if '64bit' in platform.architecture()[0]: + DEFINE_MACROS += (('MS_WIN64', 1),) elif "linux" in sys.platform or "darwin" in sys.platform: - DEFINE_MACROS += (('HAVE_PTHREAD', 1),) + DEFINE_MACROS += (('HAVE_PTHREAD', 1),) # By default, Python3 distutils enforces compatibility of # c plugins (.so files) with the OSX version Python3 was built with. # For Python3.4, this is OSX 10.6, but we need Thread Local Support (__thread) if 'darwin' in sys.platform and PY3: - mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') - if mac_target and (pkg_resources.parse_version(mac_target) < - pkg_resources.parse_version('10.9.0')): - os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' - os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( - r'macosx-[0-9]+\.[0-9]+-(.+)', - r'macosx-10.9-\1', - util.get_platform()) + mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + if mac_target and (pkg_resources.parse_version(mac_target) < + pkg_resources.parse_version('10.9.0')): + os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.9-\1', + util.get_platform()) + def package_data(): - tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) - proto_resources_path = os.path.join(tools_path, - GRPC_PYTHON_PROTO_RESOURCES_NAME) - proto_files = [] - for proto_file in PROTO_FILES: - source = os.path.join(PROTO_INCLUDE, proto_file) - target = os.path.join(proto_resources_path, proto_file) - relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file) - try: - os.makedirs(os.path.dirname(target)) - except OSError as error: - if error.errno == errno.EEXIST: - pass - else: - raise - shutil.copy(source, target) - proto_files.append(relative_target) - return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} + tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) + proto_resources_path = os.path.join(tools_path, + GRPC_PYTHON_PROTO_RESOURCES_NAME) + proto_files = [] + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) + target = os.path.join(proto_resources_path, proto_file) + relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, + proto_file) + try: + os.makedirs(os.path.dirname(target)) + except OSError as error: + if error.errno == errno.EEXIST: + pass + else: + raise + shutil.copy(source, target) + proto_files.append(relative_target) + return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} + def extension_modules(): - if BUILD_WITH_CYTHON: - plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')] - else: - plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')] - - plugin_sources += [ - os.path.join('grpc_tools', 'main.cc'), - os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] - - #HACK: Substitute the embed.cc, which is a JS to C++ - # preprocessor with the generated code. - # The generated code should not be material - # to the parts of protoc we use (it affects - # the JavaScript code generator, supposedly), - # but we need to be cautious about it. - cc_files_clone = list(CC_FILES) - embed_cc_file = os.path.normpath('google/protobuf/compiler/js/embed.cc') - well_known_types_file = os.path.normpath( - 'google/protobuf/compiler/js/well_known_types_embed.cc') - if embed_cc_file in cc_files_clone: - cc_files_clone.remove(embed_cc_file) - if well_known_types_file in cc_files_clone: - cc_files_clone.remove(well_known_types_file) - plugin_sources += [os.path.join('grpc_tools', 'protobuf_generated_well_known_types_embed.cc')] - plugin_sources += [os.path.join(CC_INCLUDE, cc_file) for cc_file in cc_files_clone] - - plugin_ext = extension.Extension( - name='grpc_tools._protoc_compiler', - sources=plugin_sources, - include_dirs=[ - '.', - 'grpc_root', - os.path.join('grpc_root', 'include'), - CC_INCLUDE, - ], - language='c++', - define_macros=list(DEFINE_MACROS), - extra_compile_args=list(EXTRA_COMPILE_ARGS), - extra_link_args=list(EXTRA_LINK_ARGS), - ) - extensions = [plugin_ext] - if BUILD_WITH_CYTHON: - from Cython import Build - return Build.cythonize(extensions) - else: - return extensions + if BUILD_WITH_CYTHON: + plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')] + else: + plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')] + + plugin_sources += [ + os.path.join('grpc_tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc') + ] + + #HACK: Substitute the embed.cc, which is a JS to C++ + # preprocessor with the generated code. + # The generated code should not be material + # to the parts of protoc we use (it affects + # the JavaScript code generator, supposedly), + # but we need to be cautious about it. + cc_files_clone = list(CC_FILES) + embed_cc_file = os.path.normpath('google/protobuf/compiler/js/embed.cc') + well_known_types_file = os.path.normpath( + 'google/protobuf/compiler/js/well_known_types_embed.cc') + if embed_cc_file in cc_files_clone: + cc_files_clone.remove(embed_cc_file) + if well_known_types_file in cc_files_clone: + cc_files_clone.remove(well_known_types_file) + plugin_sources += [ + os.path.join('grpc_tools', + 'protobuf_generated_well_known_types_embed.cc') + ] + plugin_sources += [ + os.path.join(CC_INCLUDE, cc_file) for cc_file in cc_files_clone + ] + + plugin_ext = extension.Extension( + name='grpc_tools._protoc_compiler', + sources=plugin_sources, + include_dirs=[ + '.', + 'grpc_root', + os.path.join('grpc_root', 'include'), + CC_INCLUDE, + ], + language='c++', + define_macros=list(DEFINE_MACROS), + extra_compile_args=list(EXTRA_COMPILE_ARGS), + extra_link_args=list(EXTRA_LINK_ARGS),) + extensions = [plugin_ext] + if BUILD_WITH_CYTHON: + from Cython import Build + return Build.cythonize(extensions) + else: + return extensions + setuptools.setup( - name='grpcio-tools', - version=grpc_version.VERSION, - description='Protobuf code generator for gRPC', - author='The gRPC Authors', - author_email='grpc-io@googlegroups.com', - url='https://grpc.io', - license='Apache License 2.0', - classifiers=CLASSIFIERS, - ext_modules=extension_modules(), - packages=setuptools.find_packages('.'), - install_requires=[ - 'protobuf>=3.5.0.post1', - 'grpcio>={version}'.format(version=grpc_version.VERSION), - ], - package_data=package_data(), -) + name='grpcio-tools', + version=grpc_version.VERSION, + description='Protobuf code generator for gRPC', + author='The gRPC Authors', + author_email='grpc-io@googlegroups.com', + url='https://grpc.io', + license='Apache License 2.0', + classifiers=CLASSIFIERS, + ext_modules=extension_modules(), + packages=setuptools.find_packages('.'), + install_requires=[ + 'protobuf>=3.5.0.post1', + 'grpcio>={version}'.format(version=grpc_version.VERSION), + ], + package_data=package_data(),) diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index c865f0bcc0..216492a2f3 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -27,7 +27,7 @@ import sys import traceback import uuid -DEPS_FILE_CONTENT=""" +DEPS_FILE_CONTENT = """ # Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -60,15 +60,16 @@ PROTOBUF_CC_PREFIX = '//:src/' PROTOBUF_PROTO_PREFIX = '//:src/' GRPC_ROOT = os.path.abspath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), - '..', '..', '..')) + os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) -GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib', - 'python', 'grpcio_tools') +GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', + 'grpcio_tools') -GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src') +GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', + 'src') GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT) -GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', 'protobuf') +GRPC_PROTOBUF_SUBMODULE_ROOT = os.path.join(GRPC_ROOT, 'third_party', + 'protobuf') GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler') GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf', 'src') @@ -80,81 +81,91 @@ GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include') GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'include') -BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.sh') +BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', + 'bazel_deps.sh') BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos' + def protobuf_submodule_commit_hash(): - """Gets the commit hash for the HEAD of the protobuf submodule currently + """Gets the commit hash for the HEAD of the protobuf submodule currently checked out.""" - cwd = os.getcwd() - os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT) - output = subprocess.check_output(['git', 'rev-parse', 'HEAD']) - os.chdir(cwd) - return output.splitlines()[0].strip() + cwd = os.getcwd() + os.chdir(GRPC_PROTOBUF_SUBMODULE_ROOT) + output = subprocess.check_output(['git', 'rev-parse', 'HEAD']) + os.chdir(cwd) + return output.splitlines()[0].strip() + def bazel_query(query): - output = subprocess.check_output([BAZEL_DEPS, query]) - return output.splitlines() + output = subprocess.check_output([BAZEL_DEPS, query]) + return output.splitlines() + def get_deps(): - """Write the result of the bazel query `query` against protobuf to + """Write the result of the bazel query `query` against protobuf to `out_file`.""" - cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) - cc_files = [ - name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output - if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX)] - proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY) - proto_files = [ - name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output - if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX)] - commit_hash = protobuf_submodule_commit_hash() - deps_file_content = DEPS_FILE_CONTENT.format( - cc_files=cc_files, - proto_files=proto_files, - cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), - proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), - commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX) - return deps_file_content + cc_files_output = bazel_query(BAZEL_DEPS_PROTOC_LIB_QUERY) + cc_files = [ + name[len(PROTOBUF_CC_PREFIX):] for name in cc_files_output + if name.endswith('.cc') and name.startswith(PROTOBUF_CC_PREFIX) + ] + proto_files_output = bazel_query(BAZEL_DEPS_COMMON_PROTOS_QUERY) + proto_files = [ + name[len(PROTOBUF_PROTO_PREFIX):] for name in proto_files_output + if name.endswith('.proto') and name.startswith(PROTOBUF_PROTO_PREFIX) + ] + commit_hash = protobuf_submodule_commit_hash() + deps_file_content = DEPS_FILE_CONTENT.format( + cc_files=cc_files, + proto_files=proto_files, + cc_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), + proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT), + commit_hash=COMMIT_HASH_PREFIX + commit_hash + COMMIT_HASH_SUFFIX) + return deps_file_content + def long_path(path): - if os.name == 'nt': - return '\\\\?\\' + path - else: - return path + if os.name == 'nt': + return '\\\\?\\' + path + else: + return path + def main(): - os.chdir(GRPC_ROOT) - - for source, target in [ - (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), - (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), - (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: - for source_dir, _, files in os.walk(source): - target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source))) - try: - os.makedirs(target_dir) - except OSError as error: - if error.errno != errno.EEXIST: - raise - for relative_file in files: - source_file = os.path.abspath(os.path.join(source_dir, relative_file)) - target_file = os.path.abspath(os.path.join(target_dir, relative_file)) - shutil.copyfile(source_file, target_file) - - try: - protoc_lib_deps_content = get_deps() - except Exception as error: - # We allow this script to succeed even if we couldn't get the dependencies, - # as then we can assume that even without a successful bazel run the - # dependencies currently in source control are 'good enough'. - sys.stderr.write("Got non-fatal error:\n") - traceback.print_exc(file=sys.stderr) - return - # If we successfully got the dependencies, truncate and rewrite the deps file. - with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: - deps_file.write(protoc_lib_deps_content) + os.chdir(GRPC_ROOT) + + for source, target in [(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), + (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), + (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: + for source_dir, _, files in os.walk(source): + target_dir = os.path.abspath( + os.path.join(target, os.path.relpath(source_dir, source))) + try: + os.makedirs(target_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + for relative_file in files: + source_file = os.path.abspath( + os.path.join(source_dir, relative_file)) + target_file = os.path.abspath( + os.path.join(target_dir, relative_file)) + shutil.copyfile(source_file, target_file) + + try: + protoc_lib_deps_content = get_deps() + except Exception as error: + # We allow this script to succeed even if we couldn't get the dependencies, + # as then we can assume that even without a successful bazel run the + # dependencies currently in source control are 'good enough'. + sys.stderr.write("Got non-fatal error:\n") + traceback.print_exc(file=sys.stderr) + return + # If we successfully got the dependencies, truncate and rewrite the deps file. + with open(GRPC_PYTHON_PROTOC_LIB_DEPS, 'w') as deps_file: + deps_file.write(protoc_lib_deps_content) -if __name__ == '__main__': - main() +if __name__ == '__main__': + main() diff --git a/tools/distrib/python/submit.py b/tools/distrib/python/submit.py index 92eab5ad65..aff71b5eb1 100755 --- a/tools/distrib/python/submit.py +++ b/tools/distrib/python/submit.py @@ -21,43 +21,52 @@ import subprocess parser = argparse.ArgumentParser( description='Submit the package to a PyPI repository.') parser.add_argument( - '--repository', '-r', metavar='r', type=str, default='pypi', + '--repository', + '-r', + metavar='r', + type=str, + default='pypi', help='The repository to push the package to. ' - 'Ensure the value appears in your .pypirc file. ' - 'Defaults to "pypi".' -) + 'Ensure the value appears in your .pypirc file. ' + 'Defaults to "pypi".') parser.add_argument( - '--identity', '-i', metavar='i', type=str, - help='GPG identity to sign the files with.' -) + '--identity', + '-i', + metavar='i', + type=str, + help='GPG identity to sign the files with.') parser.add_argument( - '--username', '-u', metavar='u', type=str, + '--username', + '-u', + metavar='u', + type=str, help='Username to authenticate with the repository. Not needed if you have ' - 'configured your .pypirc to include your username.' -) + 'configured your .pypirc to include your username.') parser.add_argument( - '--password', '-p', metavar='p', type=str, + '--password', + '-p', + metavar='p', + type=str, help='Password to authenticate with the repository. Not needed if you have ' - 'configured your .pypirc to include your password.' -) + 'configured your .pypirc to include your password.') parser.add_argument( - '--bdist', '-b', action='store_true', - help='Generate a binary distribution (wheel) for the current OS.' -) + '--bdist', + '-b', + action='store_true', + help='Generate a binary distribution (wheel) for the current OS.') parser.add_argument( - '--dist-args', type=str, - help='Additional arguments to pass to the *dist setup.py command.' -) + '--dist-args', + type=str, + help='Additional arguments to pass to the *dist setup.py command.') args = parser.parse_args() # Move to the root directory of Python GRPC. -pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../../../') +pkgdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../') # Remove previous distributions; they somehow confuse twine. try: - shutil.rmtree(os.path.join(pkgdir, 'dist/')) + shutil.rmtree(os.path.join(pkgdir, 'dist/')) except: - pass + pass # Build the Cython C files build_env = os.environ.copy() @@ -67,20 +76,20 @@ subprocess.call(cmd, cwd=pkgdir, env=build_env) # Make the push. if args.bdist: - cmd = ['python', 'setup.py', 'bdist_wheel'] + cmd = ['python', 'setup.py', 'bdist_wheel'] else: - cmd = ['python', 'setup.py', 'sdist'] + cmd = ['python', 'setup.py', 'sdist'] if args.dist_args: - cmd += args.dist_args.split() + cmd += args.dist_args.split() subprocess.call(cmd, cwd=pkgdir) cmd = ['twine', 'upload', '-r', args.repository] if args.identity is not None: - cmd.extend(['-i', args.identity]) + cmd.extend(['-i', args.identity]) if args.username is not None: - cmd.extend(['-u', args.username]) + cmd.extend(['-u', args.username]) if args.password is not None: - cmd.extend(['-p', args.password]) + cmd.extend(['-p', args.password]) cmd.append('dist/*') subprocess.call(cmd, cwd=pkgdir) |