[
https://issues.apache.org/jira/browse/BEAM-1251?focusedWorklogId=147747&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-147747
]
ASF GitHub Bot logged work on BEAM-1251:
----------------------------------------
Author: ASF GitHub Bot
Created on: 25/Sep/18 19:55
Start Date: 25/Sep/18 19:55
Worklog Time Spent: 10m
Work Description: aaltay closed pull request #6489: [BEAM-1251] Upgrade
pylint version for py27-lint3
URL: https://github.com/apache/beam/pull/6489
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/examples/complete/distribopt.py
b/sdks/python/apache_beam/examples/complete/distribopt.py
index f79489516e9..a16a40b6137 100644
--- a/sdks/python/apache_beam/examples/complete/distribopt.py
+++ b/sdks/python/apache_beam/examples/complete/distribopt.py
@@ -211,12 +211,12 @@ def process(self, element, quantities):
# Create (crop, quantity) lists for each greenhouse
greenhouses = defaultdict(list)
- for crop, greenhouse in mapping.iteritems():
+ for crop, greenhouse in mapping.items():
quantity = quantities[crop]
greenhouses[greenhouse].append((crop, quantity))
# Create input for OptimizeProductParameters
- for greenhouse, crops in greenhouses.iteritems():
+ for greenhouse, crops in greenhouses.items():
key = (mapping_identifier, greenhouse)
yield (key, crops)
diff --git a/sdks/python/apache_beam/io/vcfio.py
b/sdks/python/apache_beam/io/vcfio.py
index d96c8f79d65..abe33fc0dc4 100644
--- a/sdks/python/apache_beam/io/vcfio.py
+++ b/sdks/python/apache_beam/io/vcfio.py
@@ -321,6 +321,7 @@ def _create_generator(self):
def __iter__(self):
return self
+ # pylint: disable=next-method-defined
def next(self):
return self.__next__()
diff --git a/sdks/python/apache_beam/options/pipeline_options.py
b/sdks/python/apache_beam/options/pipeline_options.py
index c1bb2ed89e9..7da91a668da 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -240,7 +240,7 @@ def _visible_option_list(self):
for option in dir(self._visible_options) if option[0] != '_')
def __dir__(self):
- return sorted(dir(type(self)) + list(self.__dict__.keys()) +
+ return sorted(dir(type(self)) + list(self.__dict__) +
self._visible_option_list())
def __getattr__(self, name):
diff --git a/sdks/python/apache_beam/runners/direct/evaluation_context.py
b/sdks/python/apache_beam/runners/direct/evaluation_context.py
index 01d0631c5ee..24b05b61527 100644
--- a/sdks/python/apache_beam/runners/direct/evaluation_context.py
+++ b/sdks/python/apache_beam/runners/direct/evaluation_context.py
@@ -82,7 +82,7 @@ def __init__(self, side_inputs):
def __repr__(self):
views_string = (', '.join(str(elm) for elm in self._views.values())
- if self._views.values() else '[]')
+ if self._views else '[]')
return '_SideInputsContainer(_views=%s)' % views_string
def get_value_or_block_until_ready(self, side_input, task, block_until):
diff --git a/sdks/python/apache_beam/runners/interactive/cache_manager.py
b/sdks/python/apache_beam/runners/interactive/cache_manager.py
index a9dd03d6455..d0c0b75716c 100644
--- a/sdks/python/apache_beam/runners/interactive/cache_manager.py
+++ b/sdks/python/apache_beam/runners/interactive/cache_manager.py
@@ -191,6 +191,7 @@ def expand(self, pcoll):
class SafeFastPrimitivesCoder(coders.Coder):
"""This class add an quote/unquote step to escape special characters."""
+ # pylint: disable=deprecated-urllib-function
def encode(self, value):
return urllib.quote(coders.coders.FastPrimitivesCoder().encode(value))
diff --git a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer.py
b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer.py
index 1f597810270..b0cf1342c46 100644
--- a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer.py
+++ b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer.py
@@ -116,7 +116,7 @@ def _analyze_pipeline(self):
sample=True)
required_transforms['_root'] = beam_runner_api_pb2.PTransform(
- subtransforms=top_level_required_transforms.keys())
+ subtransforms=list(top_level_required_transforms.keys()))
referenced_pcollection_ids = self._referenced_pcollection_ids(
required_transforms)
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
index 2fc92e887bf..1ea7b42be6a 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py
@@ -837,8 +837,8 @@ def fuse(producer, consumer):
# Everything that was originally a stage or a replacement, but wasn't
# replaced, should be in the final graph.
- final_stages = frozenset(stages).union(replacements.values()).difference(
- list(replacements.keys()))
+ final_stages = frozenset(stages).union(list(replacements.values()))\
+ .difference(list(replacements))
for stage in final_stages:
# Update all references to their final values before throwing
diff --git a/sdks/python/apache_beam/transforms/trigger.py
b/sdks/python/apache_beam/transforms/trigger.py
index c185a522218..6e581db56d9 100644
--- a/sdks/python/apache_beam/transforms/trigger.py
+++ b/sdks/python/apache_beam/transforms/trigger.py
@@ -1064,7 +1064,7 @@ def process_elements(self, state, windowed_values,
output_watermark):
# First handle merging.
if self.is_merging:
old_windows = set(state.known_windows())
- all_windows = old_windows.union(windows_to_elements.keys())
+ all_windows = old_windows.union(list(windows_to_elements))
if all_windows != old_windows:
merged_away = {}
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py
b/sdks/python/apache_beam/transforms/trigger_test.py
index 034abae65c8..1a0d66c921c 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -425,12 +425,14 @@ def format_result(k_v):
| beam.GroupByKey()
| beam.Map(format_result))
assert_that(result, equal_to(
- {
- 'A-5': {1, 2, 3, 4, 5},
- # A-10, A-11 never emitted due to AfterCount(3) never firing.
- 'B-4': {6, 7, 8, 9},
- 'B-3': {10, 15, 16},
- }.items()))
+ list(
+ {
+ 'A-5': {1, 2, 3, 4, 5},
+ # A-10, A-11 never emitted due to AfterCount(3) never firing.
+ 'B-4': {6, 7, 8, 9},
+ 'B-3': {10, 15, 16},
+ }.items()
+ )))
class TranscriptTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py
b/sdks/python/apache_beam/typehints/trivial_inference.py
index d8181ad7c78..c17d4ff03fd 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -55,6 +55,7 @@ def instance_to_type(o):
if o is None:
return type(None)
elif t not in typehints.DISALLOWED_PRIMITIVE_TYPES:
+ # pylint: disable=deprecated-types-field
if sys.version_info[0] == 2 and t == types.InstanceType:
return o.__class__
if t == BoundMethod:
diff --git a/sdks/python/apache_beam/utils/counters.py
b/sdks/python/apache_beam/utils/counters.py
index 5696bc43f80..1df81949c77 100644
--- a/sdks/python/apache_beam/utils/counters.py
+++ b/sdks/python/apache_beam/utils/counters.py
@@ -225,4 +225,4 @@ def get_counters(self):
this method returns hence the returned iterable may be stale.
"""
with self._lock:
- return self.counters.values()
+ return self.counters.values() # pylint:
disable=dict-values-not-iterating
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 9cdfcc62c02..cecf71e2ce6 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -106,7 +106,7 @@ commands =
# Checks for py2/3 compatibility issues
deps =
pycodestyle==2.3.1
- pylint==1.7.6
+ pylint==1.9.3
future==0.16.0
isort==4.2.15
flake8==3.5.0
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 147747)
Time Spent: 21h 20m (was: 21h 10m)
> Python 3 Support
> ----------------
>
> Key: BEAM-1251
> URL: https://issues.apache.org/jira/browse/BEAM-1251
> Project: Beam
> Issue Type: Improvement
> Components: sdk-py-core
> Reporter: Eyad Sibai
> Assignee: Robbe
> Priority: Major
> Time Spent: 21h 20m
> Remaining Estimate: 0h
>
> I have been trying to use google datalab with python3. As I see there are
> several packages that does not support python3 yet which google datalab
> depends on. This is one of them.
> https://github.com/GoogleCloudPlatform/DataflowPythonSDK/issues/6
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
