This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new dbe86f8 [BEAM-6583] Update minimal Python 3 requirements. (#7839)
dbe86f8 is described below
commit dbe86f8590ae494314c8e96308aaf46d1f8123b3
Author: tvalentyn <tvalen...@users.noreply.github.com>
AuthorDate: Wed Feb 13 21:57:17 2019 -0800
[BEAM-6583] Update minimal Python 3 requirements. (#7839)
* Refine python_requires stanza.
* Update Dataflow Runner requirements on Python 3
---
.../runners/dataflow/internal/apiclient.py | 26 +++++++++++++
.../runners/dataflow/internal/apiclient_test.py | 45 ++++++++++++++++++++--
sdks/python/setup.py | 5 +--
3 files changed, 70 insertions(+), 6 deletions(-)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 6d5e140..4f0da29 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -31,6 +31,7 @@ import re
import sys
import tempfile
import time
+import warnings
from datetime import datetime
from builtins import object
@@ -159,6 +160,7 @@ class Environment(object):
key='version', value=to_json_value(beam_version.__version__))])
# Version information.
self.proto.version = dataflow.Environment.VersionValue()
+ _verify_interpreter_version_is_supported(options)
if self.standard_options.streaming:
job_type = 'FNAPI_STREAMING'
else:
@@ -938,6 +940,30 @@ def get_response_encoding():
return None if sys.version_info[0] < 3 else 'utf8'
+def _verify_interpreter_version_is_supported(pipeline_options):
+ if sys.version_info[0] == 2:
+ return
+
+ if sys.version_info[0] == 3 and sys.version_info[1] == 5:
+ if sys.version_info[2] < 3:
+ warnings.warn(
+ 'You are using an early release of Python 3.5. It is recommended '
+ 'to use Python 3.5.3 or higher with Dataflow '
+ 'runner.')
+ return
+
+ debug_options = pipeline_options.view_as(DebugOptions)
+ if (debug_options.experiments and 'ignore_py3_minor_version' in
+ debug_options.experiments):
+ return
+
+ raise Exception(
+ 'Dataflow runner currently supports Python versions '
+ '2.7 and 3.5. To ignore this requirement and start a job using a '
+ 'different version of Python 3 interpreter, pass '
+ '--experiment ignore_py3_minor_version pipeline option.')
+
+
# To enable a counter on the service, add it to this dictionary.
structured_counter_translations = {
cy_combiners.CountCombineFn: (
diff --git
a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 0e37a0a..054b62f 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -396,19 +396,58 @@ class UtilTest(unittest.TestCase):
self.assertNotIn(
"use_multiple_sdk_containers", environment.proto.experiments)
- @mock.patch('apache_beam.runners.dataflow.internal.apiclient.sys')
- def test_get_python_sdk_name(self, mock_sys):
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [2, 333])
+ def test_get_python_sdk_name(self):
pipeline_options = PipelineOptions(
['--project', 'test_project', '--job_name', 'test_job_name',
'--temp_location', 'gs://test-location/temp',
'--experiments', 'beam_fn_api',
'--experiments', 'use_multiple_sdk_containers'])
- mock_sys.version_info = [2, 333]
environment = apiclient.Environment(
[], pipeline_options, 1, FAKE_PIPELINE_URL)
self.assertEqual('Apache Beam Python 2.333 SDK',
environment._get_python_sdk_name())
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [2, 7])
+ def test_interpreter_version_check_passes_py27(self):
+ pipeline_options = PipelineOptions([])
+ apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [3, 5, 2])
+ def test_interpreter_version_check_passes_py352(self):
+ pipeline_options = PipelineOptions([])
+ apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [3, 5, 6])
+ def test_interpreter_version_check_passes_py356(self):
+ pipeline_options = PipelineOptions([])
+ apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [3, 9, 0])
+ def test_interpreter_version_check_passes_with_experiment(self):
+ pipeline_options = PipelineOptions(
+ ["--experiment=ignore_py3_minor_version"])
+ apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+ @mock.patch(
+ 'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+ [3, 9, 0])
+ def test_interpreter_version_check_fails_py39(self):
+ pipeline_options = PipelineOptions([])
+ self.assertRaises(
+ Exception,
+ apiclient._verify_interpreter_version_is_supported, pipeline_options)
+
if __name__ == '__main__':
unittest.main()
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 8b23605..020ab43 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -167,9 +167,7 @@ def generate_protos_first(original_cmd):
return original_cmd
-# TODO(BEAM-6583): audit Python 3.x version compatibility and refine this
-# requirement range if necessary.
-python_requires = '>=2.7<=3.7'
+python_requires = '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*'
if sys.version_info[0] == 3:
warnings.warn(
@@ -216,6 +214,7 @@ setuptools.setup(
'License :: OSI Approved :: Apache Software License',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.5',
'Topic :: Software Development :: Libraries',
'Topic :: Software Development :: Libraries :: Python Modules',
],
