Repository: incubator-beam Updated Branches: refs/heads/python-sdk 1b1c8d538 -> 39dda320a
Allow ".tar" files in extra_packages Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/516f15dd Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/516f15dd Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/516f15dd Branch: refs/heads/python-sdk Commit: 516f15dd656fd9a1df36c91a25e555ca1f7e1e42 Parents: 1b1c8d5 Author: Charles Chen <[email protected]> Authored: Sun Jul 10 20:59:08 2016 -0700 Committer: Dan Halperin <[email protected]> Committed: Mon Jul 11 13:14:12 2016 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/utils/dependency.py | 8 ++++---- sdks/python/apache_beam/utils/dependency_test.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/516f15dd/sdks/python/apache_beam/utils/dependency.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/dependency.py b/sdks/python/apache_beam/utils/dependency.py index ddb640a..b809cf2 100644 --- a/sdks/python/apache_beam/utils/dependency.py +++ b/sdks/python/apache_beam/utils/dependency.py @@ -29,7 +29,7 @@ and a --setup_file option. If --setup_file is present then it is assumed that the folder containing the file specified by the option has the typical layout required by setuptools and it will run 'python setup.py sdist' to produce a source distribution. The -resulting tarball (a file ending in .tar.gz) will be staged at the GCS staging +resulting tarball (a .tar or .tar.gz file) will be staged at the GCS staging location specified as job option. When a worker starts it will check for the presence of this file and will run 'easy_install tarball' to install the package in the worker. @@ -137,10 +137,11 @@ def _stage_extra_packages(extra_packages, staging_location, temp_dir, staging_temp_dir = None local_packages = [] for package in extra_packages: - if not os.path.basename(package).endswith('.tar.gz'): + if not (os.path.basename(package).endswith('.tar') or + os.path.basename(package).endswith('.tar.gz')): raise RuntimeError( 'The --extra_packages option expects a full path ending with ' - '\'.tar.gz\' instead of %s' % package) + '\'.tar\' or \'.tar.gz\' instead of %s' % package) if not os.path.isfile(package): if package.startswith('gs://'): @@ -460,4 +461,3 @@ def _download_pypi_sdk_package(temp_dir): 'Failed to download a source distribution for the running SDK. Expected ' 'either %s or %s to be found in the download folder.' % ( zip_expected, tgz_expected)) - http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/516f15dd/sdks/python/apache_beam/utils/dependency_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/dependency_test.py b/sdks/python/apache_beam/utils/dependency_test.py index ab6446d..b7a9296 100644 --- a/sdks/python/apache_beam/utils/dependency_test.py +++ b/sdks/python/apache_beam/utils/dependency_test.py @@ -333,6 +333,8 @@ class SetupTest(unittest.TestCase): self.create_temp_file( os.path.join(source_dir, 'xyz.tar.gz'), 'nothing') self.create_temp_file( + os.path.join(source_dir, 'xyz2.tar'), 'nothing') + self.create_temp_file( os.path.join(source_dir, dependency.EXTRA_PACKAGES_FILE), 'nothing') options = PipelineOptions() @@ -341,6 +343,7 @@ class SetupTest(unittest.TestCase): options.view_as(SetupOptions).extra_packages = [ os.path.join(source_dir, 'abc.tar.gz'), os.path.join(source_dir, 'xyz.tar.gz'), + os.path.join(source_dir, 'xyz2.tar'), 'gs://my-gcs-bucket/gcs.tar.gz'] gcs_copied_files = [] @@ -359,13 +362,13 @@ class SetupTest(unittest.TestCase): dependency._dependency_file_copy = file_copy self.assertEqual( - ['abc.tar.gz', 'xyz.tar.gz', 'gcs.tar.gz', + ['abc.tar.gz', 'xyz.tar.gz', 'xyz2.tar', 'gcs.tar.gz', dependency.EXTRA_PACKAGES_FILE, names.PICKLED_MAIN_SESSION_FILE], dependency.stage_job_resources(options)) with open(os.path.join(staging_dir, dependency.EXTRA_PACKAGES_FILE)) as f: - self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'gcs.tar.gz\n'], - f.readlines()) + self.assertEqual(['abc.tar.gz\n', 'xyz.tar.gz\n', 'xyz2.tar\n', + 'gcs.tar.gz\n'], f.readlines()) self.assertEqual(['gs://my-gcs-bucket/gcs.tar.gz'], gcs_copied_files) def test_with_extra_packages_missing_files(self): @@ -398,7 +401,8 @@ class SetupTest(unittest.TestCase): self.assertEqual( cm.exception.message, 'The --extra_packages option expects a full path ending with ' - '\'.tar.gz\' instead of %s' % os.path.join(source_dir, 'abc.tgz')) + '\'.tar\' or \'.tar.gz\' instead of %s' % os.path.join(source_dir, + 'abc.tgz')) if __name__ == '__main__':
