This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 6fe2f55 Use Python 3.6 and 3.7 containers for Dataflow runner
new e4a98b0 Merge pull request #7951 from
charlesccychen/python3637-container
6fe2f55 is described below
commit 6fe2f5557db46a1679b48db043a6f154bbb7ff18
Author: Charles Chen <[email protected]>
AuthorDate: Tue Feb 26 17:32:10 2019 -0800
Use Python 3.6 and 3.7 containers for Dataflow runner
---
.../runners/dataflow/internal/apiclient.py | 10 ++++++---
.../runners/dataflow/internal/apiclient_test.py | 24 ++++++++++++++++++++--
.../apache_beam/runners/dataflow/internal/names.py | 2 +-
3 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 4f0da29..9533dd4 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -878,11 +878,15 @@ def get_default_container_image_for_current_sdk(job_type):
"""
if sys.version_info[0] == 2:
version_suffix = ''
- elif sys.version_info[0] == 3:
+ elif sys.version_info[0:2] == (3, 5):
version_suffix = '3'
+ elif sys.version_info[0:2] == (3, 6):
+ version_suffix = '36'
+ elif sys.version_info[0:2] == (3, 7):
+ version_suffix = '37'
else:
- raise Exception('Dataflow only supports Python versions 2 and 3, got: %s'
- % sys.version_info[0])
+ raise Exception('Dataflow only supports Python versions 2 and 3.5+, got:
%s'
+ % str(sys.version_info[0:2]))
# TODO(tvalentyn): Use enumerated type instead of strings for job types.
if job_type == 'FNAPI_BATCH' or job_type == 'FNAPI_STREAMING':
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 054b62f..b3fe941 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -248,11 +248,21 @@ class UtilTest(unittest.TestCase):
pipeline_options,
'2.0.0', #any environment version
FAKE_PIPELINE_URL)
- if sys.version_info[0] == 3:
+ if sys.version_info[0:2] == (3, 5):
self.assertEqual(
env.proto.workerPools[0].workerHarnessContainerImage,
(names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
'/python3-fnapi:' + names.BEAM_FNAPI_CONTAINER_VERSION))
+ elif sys.version_info[0:2] == (3, 6):
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/python36-fnapi:' + names.BEAM_FNAPI_CONTAINER_VERSION))
+ elif sys.version_info[0:2] == (3, 7):
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/python37-fnapi:' + names.BEAM_FNAPI_CONTAINER_VERSION))
else:
self.assertEqual(
env.proto.workerPools[0].workerHarnessContainerImage,
@@ -266,11 +276,21 @@ class UtilTest(unittest.TestCase):
pipeline_options,
'2.0.0', #any environment version
FAKE_PIPELINE_URL)
- if sys.version_info[0] == 3:
+ if sys.version_info[0:2] == (3, 5):
self.assertEqual(
env.proto.workerPools[0].workerHarnessContainerImage,
(names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
'/python3:' + names.BEAM_CONTAINER_VERSION))
+ elif sys.version_info[0:2] == (3, 6):
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/python36:' + names.BEAM_CONTAINER_VERSION))
+ elif sys.version_info[0:2] == (3, 7):
+ self.assertEqual(
+ env.proto.workerPools[0].workerHarnessContainerImage,
+ (names.DATAFLOW_CONTAINER_IMAGE_REPOSITORY +
+ '/python37:' + names.BEAM_CONTAINER_VERSION))
else:
self.assertEqual(
env.proto.workerPools[0].workerHarnessContainerImage,
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py
b/sdks/python/apache_beam/runners/dataflow/internal/names.py
index aa60087..560bd3d 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/names.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py
@@ -38,7 +38,7 @@ SERIALIZED_SOURCE_KEY = 'serialized_source'
# Update this version to the next version whenever there is a change that will
# require changes to legacy Dataflow worker execution environment.
-BEAM_CONTAINER_VERSION = 'beam-master-20190213'
+BEAM_CONTAINER_VERSION = 'beam-master-20190226'
# Update this version to the next version whenever there is a change that
# requires changes to SDK harness container or SDK harness launcher.
BEAM_FNAPI_CONTAINER_VERSION = 'beam-master-20190213'
