aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/distrib/python
diff options
context:
space:
mode:
Diffstat (limited to 'tools/distrib/python')
-rwxr-xr-xtools/distrib/python/docgen.py3
-rw-r--r--tools/distrib/python/grpcio_tools/README.rst40
-rw-r--r--tools/distrib/python/grpcio_tools/grpc/tools/command.py38
-rw-r--r--tools/distrib/python/grpcio_tools/grpc_version.py2
-rw-r--r--tools/distrib/python/grpcio_tools/setup.py109
-rwxr-xr-xtools/distrib/python/make_grpcio_tools.py94
6 files changed, 227 insertions, 59 deletions
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py
index 72c65ad14a..15bd8d855f 100755
--- a/tools/distrib/python/docgen.py
+++ b/tools/distrib/python/docgen.py
@@ -70,8 +70,9 @@ environment.update({
})
subprocess_arguments_list = [
- {'args': ['make'], 'cwd': PROJECT_ROOT},
{'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment},
+ {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'],
+ 'env': environment},
{'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH],
'env': environment},
{'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment},
diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst
index 8946a1d5b3..55521d17bb 100644
--- a/tools/distrib/python/grpcio_tools/README.rst
+++ b/tools/distrib/python/grpcio_tools/README.rst
@@ -122,6 +122,7 @@ Help, I ...
third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX'
static const Type kPosMax = SIGNED_INT_MAX(Type); \\
^
+
And your toolchain is GCC (at the time of this writing, up through at least
GCC 6.0), this is probably a bug where GCC chokes on constant expressions
when the :code:`-fwrapv` flag is specified. You should consider setting your
@@ -136,3 +137,42 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory
::
$ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES
+
+To use as a build step in distutils-based projects, you may use the provided
+command class in your :code:`setup.py`:
+
+::
+
+ setuptools.setup(
+ # ...
+ cmdclass={
+ 'build_proto_modules': grpc.tools.command.BuildPackageProtos,
+ }
+ # ...
+ )
+
+Invocation of the command will walk the project tree and transpile every
+:code:`.proto` file into a :code:`_pb2.py` file in the same directory.
+
+Note that this particular approach requires :code:`grpcio-tools` to be
+installed on the machine before the setup script is invoked (i.e. no
+combination of :code:`setup_requires` or :code:`install_requires` will provide
+access to :code:`grpc.tools.command.BuildPackageProtos` if it isn't already
+installed). One way to work around this can be found in our
+:code:`grpcio-health-checking`
+`package <https://pypi.python.org/pypi/grpcio-health-checking>`_:
+
+::
+
+ class BuildPackageProtos(setuptools.Command):
+ """Command to generate project *_pb2.py modules from proto files."""
+ # ...
+ def run(self):
+ from grpc.tools import command
+ command.build_package_protos(self.distribution.package_dir[''])
+
+Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the
+command on-setup as desired.
+
+For more information on command classes, consult :code:`distutils` and
+:code:`setuptools` documentation.
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/command.py b/tools/distrib/python/grpcio_tools/grpc/tools/command.py
index ccf38b7d56..2520099835 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/command.py
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/command.py
@@ -35,7 +35,26 @@ import setuptools
from grpc.tools import protoc
-class BuildProtoModules(setuptools.Command):
+def build_package_protos(package_root):
+ proto_files = []
+ inclusion_root = os.path.abspath(package_root)
+ for root, _, files in os.walk(inclusion_root):
+ for filename in files:
+ if filename.endswith('.proto'):
+ proto_files.append(os.path.abspath(os.path.join(root, filename)))
+
+ for proto_file in proto_files:
+ command = [
+ 'grpc.tools.protoc',
+ '--proto_path={}'.format(inclusion_root),
+ '--python_out={}'.format(inclusion_root),
+ '--grpc_python_out={}'.format(inclusion_root),
+ ] + [proto_file]
+ if protoc.main(command) != 0:
+ sys.stderr.write('warning: {} failed'.format(command))
+
+
+class BuildPackageProtos(setuptools.Command):
"""Command to generate project *_pb2.py modules from proto files."""
description = 'build grpc protobuf modules'
@@ -52,19 +71,4 @@ class BuildProtoModules(setuptools.Command):
# directory is provided as an 'include' directory. We assume it's the '' key
# to `self.distribution.package_dir` (and get a key error if it's not
# there).
- proto_files = []
- inclusion_root = os.path.abspath(self.distribution.package_dir[''])
- for root, _, files in os.walk(inclusion_root):
- for filename in files:
- if filename.endswith('.proto'):
- proto_files.append(os.path.abspath(os.path.join(root, filename)))
-
- for proto_file in proto_files:
- command = [
- 'grpc.tools.protoc',
- '--proto_path={}'.format(inclusion_root),
- '--python_out={}'.format(inclusion_root),
- '--grpc_python_out={}'.format(inclusion_root),
- ] + [proto_file]
- if protoc.main(command) != 0:
- sys.stderr.write('warning: {} failed'.format(command))
+ build_package_protos(self.distribution.package_dir[''])
diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py
index 4b1e7fcd58..79c40717dd 100644
--- a/tools/distrib/python/grpcio_tools/grpc_version.py
+++ b/tools/distrib/python/grpcio_tools/grpc_version.py
@@ -29,4 +29,4 @@
# AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!!
-VERSION='0.16.0.dev0'
+VERSION='1.1.0.dev0'
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index afb6063906..bb1f1cf085 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -28,10 +28,13 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from distutils import extension
+from distutils import util
import errno
import os
import os.path
import pkg_resources
+import platform
+import re
import shlex
import shutil
import sys
@@ -45,23 +48,62 @@ from setuptools.command import build_ext
os.chdir(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.abspath('.'))
+import protoc_lib_deps
+import grpc_version
+
PY3 = sys.version_info.major == 3
+# Environment variable to determine whether or not the Cython extension should
+# *use* Cython or use the generated C files. Note that this requires the C files
+# to have been generated by building first *with* Cython support.
+BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False)
+
# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
# ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
- '-fno-wrapv -frtti -std=c++11'))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
- '-lpthread'))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+ EXTRA_ENV_COMPILE_ARGS = '-fno-wrapv -frtti -std=c++11'
+ if 'win32' in sys.platform:
+ # We use define flags here and don't directly add to DEFINE_MACROS below to
+ # ensure that the expert user/builder has a way of turning it off (via the
+ # envvars) without adding yet more GRPC-specific envvars.
+ # See https://sourceforge.net/p/mingw-w64/bugs/363/
+ if '32' in platform.architecture()[0]:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+ else:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+if EXTRA_ENV_LINK_ARGS is None:
+ EXTRA_ENV_LINK_ARGS = '-lpthread'
+ if 'win32' in sys.platform:
+ # TODO(atash) check if this is actually safe to just import and call on
+ # non-Windows (to avoid breaking import style)
+ from distutils.cygwinccompiler import get_msvcr
+ msvcr = get_msvcr()[0]
+ EXTRA_ENV_LINK_ARGS += (
+ ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+ '-static'.format(msvcr=msvcr))
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
+
+CC_FILES = [
+ os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
+PROTO_FILES = [
+ os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES]
+CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
+PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
-import protoc_lib_deps
-import grpc_version
+DEFINE_MACROS = (('HAVE_PTHREAD', 1),)
+if "win32" in sys.platform and '64bit' in platform.architecture()[0]:
+ DEFINE_MACROS += (('MS_WIN64', 1),)
# By default, Python3 distutils enforces compatibility of
# c plugins (.so files) with the OSX version Python3 was built with.
@@ -71,14 +113,18 @@ if 'darwin' in sys.platform and PY3:
if mac_target and (pkg_resources.parse_version(mac_target) <
pkg_resources.parse_version('10.9.0')):
os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
+ os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
+ r'macosx-[0-9]+\.[0-9]+-(.+)',
+ r'macosx-10.9-\1',
+ util.get_platform())
def package_data():
tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
proto_resources_path = os.path.join(tools_path,
GRPC_PYTHON_PROTO_RESOURCES_NAME)
proto_files = []
- for proto_file in protoc_lib_deps.PROTO_FILES:
- source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
+ for proto_file in PROTO_FILES:
+ source = os.path.join(PROTO_INCLUDE, proto_file)
target = os.path.join(proto_resources_path, proto_file)
relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
try:
@@ -92,44 +138,47 @@ def package_data():
proto_files.append(relative_target)
return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
-def protoc_ext_module():
- plugin_sources = [
- 'grpc/tools/main.cc',
- 'grpc_root/src/compiler/python_generator.cc'] + [
- os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
- for cc_file in protoc_lib_deps.CC_FILES]
+def extension_modules():
+ if BUILD_WITH_CYTHON:
+ plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')]
+ else:
+ plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.cpp')]
+ plugin_sources += [
+ os.path.join('grpc', 'tools', 'main.cc'),
+ os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [
+ os.path.join(CC_INCLUDE, cc_file)
+ for cc_file in CC_FILES]
plugin_ext = extension.Extension(
name='grpc.tools._protoc_compiler',
- sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources,
+ sources=plugin_sources,
include_dirs=[
'.',
'grpc_root',
- 'grpc_root/include',
- protoc_lib_deps.CC_INCLUDE,
+ os.path.join('grpc_root', 'include'),
+ CC_INCLUDE,
],
language='c++',
- define_macros=[('HAVE_PTHREAD', 1)],
- extra_compile_args=EXTRA_COMPILE_ARGS,
- extra_link_args=EXTRA_LINK_ARGS,
+ define_macros=list(DEFINE_MACROS),
+ extra_compile_args=list(EXTRA_COMPILE_ARGS),
+ extra_link_args=list(EXTRA_LINK_ARGS),
)
- return plugin_ext
-
-def maybe_cythonize(exts):
- from Cython import Build
- return Build.cythonize(exts)
+ extensions = [plugin_ext]
+ if BUILD_WITH_CYTHON:
+ from Cython import Build
+ return Build.cythonize(extensions)
+ else:
+ return extensions
setuptools.setup(
name='grpcio_tools',
version=grpc_version.VERSION,
license='3-clause BSD',
- ext_modules=maybe_cythonize([
- protoc_ext_module(),
- ]),
+ ext_modules=extension_modules(),
packages=setuptools.find_packages('.'),
namespace_packages=['grpc'],
install_requires=[
'protobuf>=3.0.0a3',
- 'grpcio>=0.14.0',
+ 'grpcio>=0.15.0',
],
package_data=package_data(),
)
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index fd9b38b084..adf58445af 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -29,12 +29,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from __future__ import print_function
+
+import errno
+import filecmp
+import glob
import os
import os.path
import shutil
import subprocess
import sys
import traceback
+import uuid
DEPS_FILE_CONTENT="""
# Copyright 2016, Google Inc.
@@ -124,20 +130,88 @@ def get_deps():
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
return deps_file_content
+def long_path(path):
+ if os.name == 'nt':
+ return '\\\\?\\' + path
+ else:
+ return path
+
+def atomic_file_copy(src, dst):
+ """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
+ renaming being atomic. Described at http://stackoverflow.com/a/28090883.
+ """
+ try:
+ if filecmp.cmp(src, dst):
+ return
+ except:
+ pass
+ dst_dir = os.path.abspath(os.path.dirname(dst))
+ dst_base = os.path.basename(dst)
+ this_id = str(uuid.uuid4()).replace('.', '-')
+ temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
+ mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
+ mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
+ src = long_path(src)
+ dst = long_path(dst)
+ temporary_file = long_path(temporary_file)
+ mole_file = long_path(mole_file)
+ mole_pattern = long_path(mole_pattern)
+ shutil.copy2(src, temporary_file)
+ try:
+ os.rename(temporary_file, mole_file)
+ except:
+ print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
+ print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
+ raise
+ for other_file in glob.glob(mole_pattern):
+ other_id = other_file.split('.')[-3]
+ if this_id == other_id:
+ pass
+ elif this_id < other_id:
+ try:
+ os.remove(other_file)
+ except:
+ pass
+ else:
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ this_id = other_id
+ mole_file = other_file
+ try:
+ if filecmp.cmp(src, dst):
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ return
+ except:
+ pass
+ try:
+ os.rename(mole_file, dst)
+ except:
+ pass
+
def main():
os.chdir(GRPC_ROOT)
- for tree in [GRPC_PYTHON_PROTOBUF,
- GRPC_PYTHON_PROTOC_PLUGINS,
- GRPC_PYTHON_INCLUDE]:
- try:
- shutil.rmtree(tree)
- except Exception as _:
- pass
- shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
- shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
- shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
+ for source, target in [
+ (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF),
+ (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS),
+ (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]:
+ for source_dir, _, files in os.walk(source):
+ target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source)))
+ try:
+ os.makedirs(target_dir)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+ for relative_file in files:
+ source_file = os.path.abspath(os.path.join(source_dir, relative_file))
+ target_file = os.path.abspath(os.path.join(target_dir, relative_file))
+ atomic_file_copy(source_file, target_file)
try:
protoc_lib_deps_content = get_deps()
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-29 03:42:52 +0000