Repository: incubator-beam Updated Branches: refs/heads/python-sdk 8a333a661 -> 19e3eff91
Allow .whl files to be staged with --extra_package Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/71c474a5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/71c474a5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/71c474a5 Branch: refs/heads/python-sdk Commit: 71c474a5d7227be9391388fda41875305b952f93 Parents: 8a333a6 Author: Charles Chen <c...@google.com> Authored: Thu Sep 22 20:19:05 2016 -0700 Committer: Robert Bradshaw <rober...@google.com> Committed: Mon Sep 26 15:05:00 2016 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/utils/dependency.py | 14 +++++++++++--- sdks/python/apache_beam/utils/dependency_test.py | 12 +++++++----- sdks/python/apache_beam/utils/options.py | 16 ++++++++-------- 3 files changed, 26 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index 314bd1b..ab311a0 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -141,10 +141,18 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir, local_packages = [] for package in extra_packages: if not (os.path.basename(package).endswith('.tar') or - os.path.basename(package).endswith('.tar.gz')): + os.path.basename(package).endswith('.tar.gz') or + os.path.basename(package).endswith('.whl')): raise RuntimeError( - 'The --extra_packages option expects a full path ending with ' - '\'.tar\' or \'.tar.gz\' instead of %s' % package) + 'The --extra_package option expects a full path ending with ' + '".tar" or ".tar.gz" instead of %s' % package) + if os.path.basename(package).endswith('.whl'): + logging.warning( + 'The .whl package "%s" is provided in --extra_package. ' + 'This functionality is not officially supported. Since wheel ' + 'packages are binary distributions, this package must be ' + 'binary-compatible with the worker environment (e.g. Python 2.7 ' + 'running on an x64 Linux host).') if not os.path.isfile(package): if package.startswith('gs://'): http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/dependency_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index ca31806..3549a07 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -342,6 +342,8 @@ class SetupTest(unittest.TestCase): self.create_temp_file( os.path.join(source_dir, 'xyz2.tar'), 'nothing') self.create_temp_file( + os.path.join(source_dir, 'whl.whl'), 'nothing') + self.create_temp_file( os.path.join(source_dir, dependency.EXTRA_PACKAGES_FILE), 'nothing') options = PipelineOptions() @@ -351,6 +353,7 @@ class SetupTest(unittest.TestCase): os.path.join(source_dir, 'abc.tar.gz'), os.path.join(source_dir, 'xyz.tar.gz'), os.path.join(source_dir, 'xyz2.tar'), + os.path.join(source_dir, 'whl.whl'), 'gs://my-gcs-bucket/gcs.tar.gz'] gcs_copied_files = [] @@ -369,12 +372,12 @@ class SetupTest(unittest.TestCase): dependency._dependency_file_copy = file_copy self.assertEqual( - ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz', + ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'whl.whl', 'gcs.tar.gz', dependency.EXTRA_PACKAGES_FILE], dependency.stage_job_resources(options)) with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f: self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n', - 'gcs.tar.gz\n'], f.readlines()) + 'whl.whl\n', 'gcs.tar.gz\n'], f.readlines()) self.assertEqual(['gs://my-gcs-bucket/gcs.tar.gz'], gcs_copied_files) def test_with_extra_packages_missing_files(self): @@ -406,9 +409,8 @@ class SetupTest(unittest.TestCase): dependency.stage_job_resources(options) self.assertEqual( cm.exception.message, - 'The --extra_packages option expects a full path ending with ' - '\'.tar\' or \'.tar.gz\' instead of %s' % os.path.join(source_dir, - 'abc.tgz')) + 'The --extra_package option expects a full path ending with ".tar" or ' + '".tar.gz" instead of %s' % os.path.join(source_dir, 'abc.tgz')) if __name__ == '__main__': http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/71c474a5/sdks/python/apache_beam/utils/options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/options.py b/sdks/python/apache_beam/utils/options.py index 700c080..a964036 100644 --- a/sdks/python/apache_beam/utils/options.py +++ b/sdks/python/apache_beam/utils/options.py @@ -442,14 +442,14 @@ class SetupOptions(PipelineOptions): action='append', default=None, help= - ('Local path to a Python package file. The file is expected to be a ' - 'compressed tarball with the suffix \'.tar.gz\' which can be ' - 'installed using the easy_install command of the standard setuptools ' - 'package. Multiple --extra_package options can be specified if more ' - 'than one package is needed. During job submission the files will be ' - 'staged in the staging area (--staging_location option) and the ' - 'workers will install them in same order they were specified on the ' - 'command line.')) + ('Local path to a Python package file. The file is expected to be (1) ' + 'a package tarball (".tar") or (2) a compressed package tarball ' + '(".tar.gz") which can be installed using the "pip install" command ' + 'of the standard pip package. Multiple --extra_package options can ' + 'be specified if more than one package is needed. During job ' + 'submission, the files will be staged in the staging area ' + '(--staging_location option) and the workers will install them in ' + 'same order they were specified on the command line.')) # TODO(silviuc): Add --files_to_stage option. # This could potentially replace the --requirements_file and --setup_file.