Automatically generate Python proto and grpc files.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/bb2b7064 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/bb2b7064 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/bb2b7064 Branch: refs/heads/master Commit: bb2b7064960c458a63c627d96e948f38ef11a898 Parents: 539e229 Author: Robert Bradshaw <[email protected]> Authored: Tue May 23 12:28:59 2017 -0700 Committer: Robert Bradshaw <[email protected]> Committed: Wed May 24 15:01:39 2017 -0700 ---------------------------------------------------------------------- .gitignore | 1 + pom.xml | 2 +- sdks/python/MANIFEST.in | 1 + sdks/python/apache_beam/runners/api/__init__.py | 13 +-- sdks/python/gen_protos.py | 116 +++++++++++++++++++ sdks/python/run_pylint.sh | 1 + sdks/python/setup.py | 38 +++++- sdks/python/tox.ini | 3 + 8 files changed, 157 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 1ecb993..6e077bd 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ sdks/python/**/*.egg sdks/python/LICENSE sdks/python/NOTICE sdks/python/README.md +sdks/python/runners/api/*pb2*.* # Ignore IntelliJ files. .idea/ http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index c3a6b73..4f833cd 100644 --- a/pom.xml +++ b/pom.xml @@ -1301,7 +1301,7 @@ <exclude>**/.settings/**/*</exclude> <!-- Proto/grpc generated wrappers --> - <exclude>**/sdks/python/apache_beam/runners/api/*.py</exclude> + <exclude>**/apache_beam/runners/api/*_pb2*.py</exclude> </excludes> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/MANIFEST.in ---------------------------------------------------------------------- diff --git a/sdks/python/MANIFEST.in b/sdks/python/MANIFEST.in index 41d80ef..c97e57a 100644 --- a/sdks/python/MANIFEST.in +++ b/sdks/python/MANIFEST.in @@ -15,6 +15,7 @@ # limitations under the License. # +include gen_protos.py include README.md include NOTICE include LICENSE http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/apache_beam/runners/api/__init__.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/api/__init__.py b/sdks/python/apache_beam/runners/api/__init__.py index bf95208..2750859 100644 --- a/sdks/python/apache_beam/runners/api/__init__.py +++ b/sdks/python/apache_beam/runners/api/__init__.py @@ -17,16 +17,5 @@ """For internal use only; no backwards-compatibility guarantees. -Checked in to avoid protoc dependency for Python development. - -Regenerate files with:: - - protoc -I../common/runner-api/src/main/proto/ \ - --python_out=apache_beam/runners/api/ \ - ../common/runner-api/src/main/proto/*.proto - - protoc -I../common/{fn,runner}-api/src/main/proto/ \ - --python_out=apache_beam/runners/api/ \ - --grpc_python_out=apache_beam/runners/api/ \ - ../common/fn-api/src/main/proto/*.proto +Automatically generated when running setup.py sdist or build[_py]. """ http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/gen_protos.py ---------------------------------------------------------------------- diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py new file mode 100644 index 0000000..edaaff4 --- /dev/null +++ b/sdks/python/gen_protos.py @@ -0,0 +1,116 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Generates Python proto modules and grpc stubs for Beam protos.""" + +import glob +import logging +import multiprocessing +import os +import pkg_resources +import subprocess +import sys +import warnings + + +GRPC_TOOLS = 'grpcio-tools>=1.3.5' + +BEAM_PROTO_PATHS = [ + os.path.join('..', 'common', 'runner-api', 'src', 'main', 'proto'), + os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto') +] + +PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'runners', 'api') + + +def generate_proto_files(): + + try: + import grpc_tools + except ImportError: + warnings.warn('Installing grpcio-tools is recommended for development.') + + py_sdk_root = os.path.dirname(os.path.abspath(__file__)) + common = os.path.join(py_sdk_root, '..', 'common') + proto_dirs = [os.path.join(py_sdk_root, path) for path in BEAM_PROTO_PATHS] + proto_files = sum( + [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) + out_dir = os.path.join(py_sdk_root, PYTHON_OUTPUT_PATH) + out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] + + if out_files and not proto_files: + # We have out_files but no protos; assume they're up to date. + # This is actually the common case (e.g. installation from an sdist). + return + + elif not out_files and not proto_files: + if not common: + raise RuntimeError( + 'Not in apache git tree; unable to find proto definitions.') + else: + raise RuntimeError( + 'No proto files found in %s.' % proto_dirs) + + # Regenerate iff the proto files are newer. + elif not out_files or len(out_files) < len(proto_files) or ( + min(os.path.getmtime(path) for path in out_files) + <= max(os.path.getmtime(path) for path in proto_files)): + try: + from grpc_tools import protoc + except ImportError: + # Use a subprocess to avoid messing with this process' path and imports. + # Note that this requires a separate module from setup.py for Windows: + # https://docs.python.org/2/library/multiprocessing.html#windows + p = multiprocessing.Process( + target=_install_grpcio_tools_and_generate_proto_files) + p.start() + p.join() + else: + logging.info('Regenerating out-of-date Python proto definitions.') + builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') + args = ( + [sys.executable] + # expecting to be called from command line + ['--proto_path=%s' % builtin_protos] + + ['--proto_path=%s' % d for d in proto_dirs] + + ['--python_out=%s' % out_dir] + + ['--grpc_python_out=%s' % out_dir] + + proto_files) + ret_code = protoc.main(args) + if ret_code: + raise RuntimeError( + 'Protoc returned non-zero status (see logs for details): ' + '%s' % ret_code) + + +# Though wheels are available for grpcio-tools, setup_requires uses +# easy_install which doesn't understand them. This means that it is +# compiled from scratch (which is expensive as it compiles the full +# protoc compiler). Instead, we attempt to install a wheel in a temporary +# directory and add it to the path as needed. +# See https://github.com/pypa/setuptools/issues/377 +def _install_grpcio_tools_and_generate_proto_files(): + install_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), '.eggs', 'grpcio-wheels') + logging.warning('Downloading a grpcio-tools to %s' % install_path) + subprocess.check_call( + ['pip', 'install', '-t', install_path, '--upgrade', GRPC_TOOLS]) + sys.path.append(install_path) + generate_proto_files() + + +if __name__ == '__main__': + generate_proto_files() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/run_pylint.sh ---------------------------------------------------------------------- diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index a5e3fa1..400c577 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -49,6 +49,7 @@ EXCLUDED_GENERATED_FILES=( "apache_beam/runners/api/beam_fn_api_pb2.py" "apache_beam/runners/api/beam_fn_api_pb2_grpc.py" "apache_beam/runners/api/beam_runner_api_pb2.py" +"apache_beam/runners/api/beam_runner_api_pb2_grpc.py" ) FILES_TO_IGNORE="" http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/setup.py ---------------------------------------------------------------------- diff --git a/sdks/python/setup.py b/sdks/python/setup.py index befc024..8a8ce48 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -19,15 +19,25 @@ from distutils.version import StrictVersion +import glob import os +import pkg_resources import platform import shutil +import subprocess +import sys import warnings import setuptools +from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist +from setuptools.command.test import test + from pkg_resources import get_distribution, DistributionNotFound +import gen_protos + def get_version(): global_names = {} @@ -98,8 +108,12 @@ REQUIRED_PACKAGES = [ 'pyyaml>=3.12,<4.0.0', ] +REQUIRED_SETUP_PACKAGES = [ + 'nose>=1.0', + ] + REQUIRED_TEST_PACKAGES = [ - 'pyhamcrest>=1.9,<2.0' + 'pyhamcrest>=1.9,<2.0', ] GCP_REQUIREMENTS = [ @@ -111,6 +125,15 @@ GCP_REQUIREMENTS = [ ] +# We must generate protos after setup_requires are installed. +def generate_protos_first(original_cmd): + class cmd(original_cmd, object): + def run(self): + gen_protos.generate_proto_files() + super(cmd, self).run() + return cmd + + setuptools.setup( name=PACKAGE_NAME, version=PACKAGE_VERSION, @@ -135,7 +158,7 @@ setuptools.setup( 'apache_beam/utils/counters.py', 'apache_beam/utils/windowed_value.py', ]), - setup_requires=['nose>=1.0'], + setup_requires=REQUIRED_SETUP_PACKAGES, install_requires=REQUIRED_PACKAGES, test_suite='nose.collector', tests_require=REQUIRED_TEST_PACKAGES, @@ -153,11 +176,16 @@ setuptools.setup( 'Programming Language :: Python :: 2.7', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', - ], + ], license='Apache License, Version 2.0', keywords=PACKAGE_KEYWORDS, entry_points={ 'nose.plugins.0.10': [ 'beam_test_plugin = test_config:BeamTestPlugin' - ]} - ) + ]}, + cmdclass={ + 'build_py': generate_protos_first(build_py), + 'sdist': generate_protos_first(sdist), + 'test': generate_protos_first(test), + }, +) http://git-wip-us.apache.org/repos/asf/beam/blob/bb2b7064/sdks/python/tox.ini ---------------------------------------------------------------------- diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 2592b17..917e907 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -28,6 +28,7 @@ select = E3 # autocomplete_test depends on nose when invoked directly. deps = nose==1.3.7 + grpcio-tools==1.3.5 commands = python --version pip install -e .[test] @@ -44,6 +45,7 @@ platform = linux2 # autocomplete_test depends on nose when invoked directly. deps = nose==1.3.7 + grpcio-tools==1.3.5 cython==0.25.2 whitelist_externals=find commands = @@ -87,6 +89,7 @@ passenv = TRAVIS* [testenv:docs] deps= nose==1.3.7 + grpcio-tools==1.3.5 Sphinx==1.5.5 commands = pip install -e .[test,gcp,docs]
