Repository: beam Updated Branches: refs/heads/master 94d7f747b -> d5aff5dad
Add zip to the list of accepted extra package file types. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/41417f77 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/41417f77 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/41417f77 Branch: refs/heads/master Commit: 41417f778fc3f62f3a27ddbb5c1fb717cc25f7b6 Parents: 94d7f74 Author: Ahmet Altay <[email protected]> Authored: Fri Oct 20 14:02:48 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Wed Oct 25 16:33:19 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/options/pipeline_options.py | 11 ++++++----- .../apache_beam/runners/dataflow/internal/dependency.py | 5 +++-- .../runners/dataflow/internal/dependency_test.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/options/pipeline_options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index a09c7c3..5278b8a 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -602,11 +602,12 @@ class SetupOptions(PipelineOptions): default=None, help= ('Local path to a Python package file. The file is expected to be (1) ' - 'a package tarball (".tar") or (2) a compressed package tarball ' - '(".tar.gz") which can be installed using the "pip install" command ' - 'of the standard pip package. Multiple --extra_package options can ' - 'be specified if more than one package is needed. During job ' - 'submission, the files will be staged in the staging area ' + 'a package tarball (".tar"), (2) a compressed package tarball ' + '(".tar.gz"), (3) a Wheel file (".whl") or (4) a compressed package ' + 'zip file (".zip") which can be installed using the "pip install" ' + 'command of the standard pip package. Multiple --extra_package ' + 'options can be specified if more than one package is needed. During ' + 'job submission, the files will be staged in the staging area ' '(--staging_location option) and the workers will install them in ' 'same order they were specified on the command line.')) http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/runners/dataflow/internal/dependency.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index c1edf7d..fba2df2 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -179,10 +179,11 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir, for package in extra_packages: if not (os.path.basename(package).endswith('.tar') or os.path.basename(package).endswith('.tar.gz') or - os.path.basename(package).endswith('.whl')): + os.path.basename(package).endswith('.whl') or + os.path.basename(package).endswith('.zip')): raise RuntimeError( 'The --extra_package option expects a full path ending with ' - '".tar" or ".tar.gz" instead of %s' % package) + '".tar", ".tar.gz", ".whl" or ".zip" instead of %s' % package) if os.path.basename(package).endswith('.whl'): logging.warning( 'The .whl package "%s" is provided in --extra_package. ' http://git-wip-us.apache.org/repos/asf/beam/blob/41417f77/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py index 6d9b061..f0e59bc 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency_test.py @@ -427,8 +427,9 @@ class SetupTest(unittest.TestCase): dependency.stage_job_resources(options) self.assertEqual( cm.exception.message, - 'The --extra_package option expects a full path ending with ".tar" or ' - '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz')) + 'The --extra_package option expects a full path ending with ' + '".tar", ".tar.gz", ".whl" or ".zip" ' + 'instead of %s' % os.path.join(source_dir, 'abc.tgz')) if __name__ == '__main__':
