Repository: incubator-beam Updated Branches: refs/heads/python-sdk df7ecd55f -> 5ab5567ea
Enables unused-import and used-before-assignment rules Also cleans up some left over lint warnings from previous iterations. Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/2094e009 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/2094e009 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/2094e009 Branch: refs/heads/python-sdk Commit: 2094e0090bf98933c08c835c15958fc01512730c Parents: bff9801 Author: Ahmet Altay <[email protected]> Authored: Thu Jun 30 14:47:20 2016 -0700 Committer: Ahmet Altay <[email protected]> Committed: Thu Jun 30 14:47:20 2016 -0700 ---------------------------------------------------------------------- sdks/python/.pylintrc | 2 -- sdks/python/apache_beam/coders/coder_impl.py | 1 - .../apache_beam/examples/complete/autocomplete.py | 2 +- .../examples/complete/autocomplete_test.py | 6 +----- .../apache_beam/examples/complete/estimate_pi.py | 4 ++-- sdks/python/apache_beam/examples/complete/tfidf.py | 6 +++--- .../examples/complete/top_wikipedia_sessions.py | 9 +++++---- .../examples/cookbook/bigquery_side_input.py | 6 +++--- .../examples/cookbook/bigquery_side_input_test.py | 10 +++++----- .../examples/cookbook/bigquery_tornadoes_test.py | 2 +- sdks/python/apache_beam/examples/cookbook/coders.py | 11 +++++------ .../python/apache_beam/examples/cookbook/filters.py | 4 ++-- .../apache_beam/examples/cookbook/filters_test.py | 2 +- .../apache_beam/examples/cookbook/mergecontacts.py | 16 ++++++++-------- .../examples/cookbook/multiple_output_pardo.py | 8 +++++--- sdks/python/apache_beam/internal/json_value_test.py | 1 - sdks/python/apache_beam/transforms/core.py | 3 --- sdks/python/apache_beam/transforms/ptransform.py | 1 - sdks/python/apache_beam/transforms/util.py | 3 --- sdks/python/apache_beam/transforms/window.py | 2 +- 20 files changed, 43 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/.pylintrc ---------------------------------------------------------------------- diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc index b283f04..c138f1a 100644 --- a/sdks/python/.pylintrc +++ b/sdks/python/.pylintrc @@ -124,9 +124,7 @@ disable = unnecessary-lambda, unneeded-not, unused-argument, - unused-import, unused-wildcard-import, - used-before-assignment, wildcard-import, http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/coders/coder_impl.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py index a623f2b..8ff73bf 100644 --- a/sdks/python/apache_beam/coders/coder_impl.py +++ b/sdks/python/apache_beam/coders/coder_impl.py @@ -26,7 +26,6 @@ coder_impl.pxd file for type hints. """ import collections -from cPickle import loads, dumps # pylint: disable=wrong-import-order, wrong-import-position http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/complete/autocomplete.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/autocomplete.py b/sdks/python/apache_beam/examples/complete/autocomplete.py index 616a2c9..2fb2d92 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete.py @@ -44,7 +44,7 @@ def run(argv=None): | beam.FlatMap('split', lambda x: re.findall(r'[A-Za-z\']+', x)) | TopPerPrefix('TopPerPrefix', 5) | beam.Map('format', - lambda (prefix, candidates): '%s: %s' % (prefix, candidates)) + lambda (prefix, candidates): '%s: %s' % (prefix, candidates)) | beam.io.Write('write', beam.io.TextFileSink(known_args.output))) p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/complete/autocomplete_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/autocomplete_test.py b/sdks/python/apache_beam/examples/complete/autocomplete_test.py index 706c4c8..bd0a6cb 100644 --- a/sdks/python/apache_beam/examples/complete/autocomplete_test.py +++ b/sdks/python/apache_beam/examples/complete/autocomplete_test.py @@ -18,16 +18,12 @@ """Test for the wordcount example.""" import collections -import logging -import re -import tempfile import unittest import apache_beam as beam from apache_beam.examples.complete import autocomplete from apache_beam.pvalue import AsIter -from apache_beam.utils import options # TODO(robertwb): Move to testing utilities. @@ -41,7 +37,7 @@ def assert_that(pcoll, matcher): def check_matcher(_, side_value): assert matcher(side_value) return [] - singleton | beam.FlatMap(check_matcher, AsIter(pcoll)) + singleton | beam.FlatMap(check_matcher, AsIter(pcoll)) # pylint: disable=expression-not-assigned def contains_in_any_order(expected): http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/complete/estimate_pi.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/estimate_pi.py b/sdks/python/apache_beam/examples/complete/estimate_pi.py index d6ae619..8b0f202 100644 --- a/sdks/python/apache_beam/examples/complete/estimate_pi.py +++ b/sdks/python/apache_beam/examples/complete/estimate_pi.py @@ -100,8 +100,8 @@ def run(argv=None): | beam.Map('Run trials', run_trials) | beam.CombineGlobally('Sum', combine_results).without_defaults() | beam.io.Write('Write', - beam.io.TextFileSink(known_args.output, - coder=JsonCoder()))) + beam.io.TextFileSink(known_args.output, + coder=JsonCoder()))) # Actually run the pipeline (all operations above are deferred). p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/complete/tfidf.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/tfidf.py b/sdks/python/apache_beam/examples/complete/tfidf.py index 1447f5d..cd85651 100644 --- a/sdks/python/apache_beam/examples/complete/tfidf.py +++ b/sdks/python/apache_beam/examples/complete/tfidf.py @@ -98,7 +98,7 @@ class TfIdf(beam.PTransform): uri_to_word_and_count = ( uri_and_word_to_count | beam.Map('shift keys', - lambda ((uri, word), count): (uri, (word, count)))) + lambda ((uri, word), count): (uri, (word, count)))) # Perform a CoGroupByKey (a sort of pre-join) on the prepared # uri_to_word_total and uri_to_word_and_count tagged by 'word totals' and @@ -146,8 +146,8 @@ class TfIdf(beam.PTransform): word_to_df = ( word_to_doc_count | beam.Map('compute doc frequencies', - lambda (word, count), total: (word, float(count) / total), - AsSingleton(total_documents))) + lambda (word, count), total: (word, float(count) / total), + AsSingleton(total_documents))) # Join the term frequency and document frequency collections, # each keyed on the word. http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py index 2209f80..55b7857 100644 --- a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py +++ b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py @@ -76,7 +76,7 @@ class ComputeSessions(beam.PTransform): def apply(self, pcoll): return (pcoll | beam.WindowInto('ComputeSessionsWindow', - window.Sessions(gap_size=ONE_HOUR_IN_SECONDS)) + window.Sessions(gap_size=ONE_HOUR_IN_SECONDS)) | combiners.Count.PerElement()) @@ -89,8 +89,8 @@ class TopPerMonth(beam.PTransform): def apply(self, pcoll): return (pcoll | beam.WindowInto('TopPerMonthWindow', - window.FixedWindows( - size=THIRTY_DAYS_IN_SECONDS)) + window.FixedWindows( + size=THIRTY_DAYS_IN_SECONDS)) | combiners.core.CombineGlobally( 'Top', combiners.TopCombineFn( @@ -126,7 +126,8 @@ class ComputeTopSessions(beam.PTransform): def apply(self, pcoll): return (pcoll - | beam.ParDo('ExtractUserAndTimestamp', ExtractUserAndTimestampDoFn()) + | beam.ParDo('ExtractUserAndTimestamp', + ExtractUserAndTimestampDoFn()) | beam.Filter( lambda x: abs(hash(x)) <= sys.maxint * self.sampling_threshold) | ComputeSessions() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py index 65fe815..c26f6cd 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input.py @@ -96,9 +96,9 @@ def run(argv=None): ignore_word = known_args.ignore_word pcoll_corpus = p | beam.Read('read corpus', - beam.io.BigQuerySource(query=query_corpus)) + beam.io.BigQuerySource(query=query_corpus)) pcoll_word = p | beam.Read('read words', - beam.io.BigQuerySource(query=query_word)) + beam.io.BigQuerySource(query=query_word)) pcoll_ignore_corpus = p | beam.Create('create_ignore_corpus', [ignore_corpus]) pcoll_ignore_word = p | beam.Create('create_ignore_word', [ignore_word]) pcoll_group_ids = p | beam.Create('create groups', group_ids) @@ -108,7 +108,7 @@ def run(argv=None): # pylint:disable=expression-not-assigned pcoll_groups | beam.io.Write('WriteToText', - beam.io.TextFileSink(known_args.output)) + beam.io.TextFileSink(known_args.output)) p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py index 813a93a..c16518f 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_side_input_test.py @@ -31,12 +31,12 @@ class BigQuerySideInputTest(unittest.TestCase): group_ids_pcoll = p | beam.Create('create_group_ids', ['A', 'B', 'C']) corpus_pcoll = p | beam.Create('create_corpus', - [{'f': 'corpus1'}, - {'f': 'corpus2'}, - {'f': 'corpus3'}]) + [{'f': 'corpus1'}, + {'f': 'corpus2'}, + {'f': 'corpus3'}]) words_pcoll = p | beam.Create('create_words', [{'f': 'word1'}, - {'f': 'word2'}, - {'f': 'word3'}]) + {'f': 'word2'}, + {'f': 'word3'}]) ignore_corpus_pcoll = p | beam.Create('create_ignore_corpus', ['corpus1']) ignore_word_pcoll = p | beam.Create('create_ignore_word', ['word1']) http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py index 211b779..2547849 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_test.py @@ -35,7 +35,7 @@ class BigQueryTornadoesTest(unittest.TestCase): {'month': 2, 'day': 1, 'tornado': True}])) results = bigquery_tornadoes.count_tornadoes(rows) beam.assert_that(results, beam.equal_to([{'month': 1, 'tornado_count': 2}, - {'month': 2, 'tornado_count': 1}])) + {'month': 2, 'tornado_count': 1}])) p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/coders.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/coders.py b/sdks/python/apache_beam/examples/cookbook/coders.py index 0a27356..f5158c2 100644 --- a/sdks/python/apache_beam/examples/cookbook/coders.py +++ b/sdks/python/apache_beam/examples/cookbook/coders.py @@ -81,12 +81,11 @@ def run(argv=None): p = beam.Pipeline(argv=pipeline_args) (p # pylint: disable=expression-not-assigned | beam.io.Read('read', - beam.io.TextFileSource(known_args.input, - coder=JsonCoder())) - | beam.FlatMap('points', compute_points) | beam.CombinePerKey(sum) | beam.io.Write( - 'write', - beam.io.TextFileSink(known_args.output, - coder=JsonCoder()))) + beam.io.TextFileSource(known_args.input, coder=JsonCoder())) + | beam.FlatMap('points', compute_points) + | beam.CombinePerKey(sum) + | beam.io.Write('write', + beam.io.TextFileSink(known_args.output, coder=JsonCoder()))) p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/filters.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/filters.py b/sdks/python/apache_beam/examples/cookbook/filters.py index 8741d63..c309941 100644 --- a/sdks/python/apache_beam/examples/cookbook/filters.py +++ b/sdks/python/apache_beam/examples/cookbook/filters.py @@ -54,7 +54,7 @@ def filter_cold_days(input_data, month_filter): fields_of_interest = ( input_data | beam.Map('projected', - lambda row: {f: row[f] for f in projection_fields})) + lambda row: {f: row[f] for f in projection_fields})) # Compute the global mean temperature. global_mean = AsSingleton( @@ -68,7 +68,7 @@ def filter_cold_days(input_data, month_filter): fields_of_interest | beam.Filter('desired month', lambda row: row['month'] == month_filter) | beam.Filter('below mean', - lambda row, mean: row['mean_temp'] < mean, global_mean)) + lambda row, mean: row['mean_temp'] < mean, global_mean)) def run(argv=None): http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/filters_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/filters_test.py b/sdks/python/apache_beam/examples/cookbook/filters_test.py index bac5f73..cf1ca7e 100644 --- a/sdks/python/apache_beam/examples/cookbook/filters_test.py +++ b/sdks/python/apache_beam/examples/cookbook/filters_test.py @@ -47,7 +47,7 @@ class FiltersTest(unittest.TestCase): beam.assert_that( results, beam.equal_to([{'year': 2010, 'month': 1, 'day': 1, 'mean_temp': 3}, - {'year': 2012, 'month': 1, 'day': 2, 'mean_temp': 3}])) + {'year': 2012, 'month': 1, 'day': 2, 'mean_temp': 3}])) results.pipeline.run() def test_basic_empty(self): http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/mergecontacts.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py index 886171f..bf8f4d0 100644 --- a/sdks/python/apache_beam/examples/cookbook/mergecontacts.py +++ b/sdks/python/apache_beam/examples/cookbook/mergecontacts.py @@ -69,9 +69,9 @@ def run(argv=None, assert_results=None): return (p | beam.io.Read('read_%s' % label, textfile) | beam.Map('backslash_%s' % label, - lambda x: re.sub(r'\\', r'\\\\', x)) + lambda x: re.sub(r'\\', r'\\\\', x)) | beam.Map('escape_quotes_%s' % label, - lambda x: re.sub(r'"', r'\"', x)) + lambda x: re.sub(r'"', r'\"', x)) | beam.Map('split_%s' % label, lambda x: re.split(r'\t+', x, 1))) # Read input databases. @@ -79,8 +79,8 @@ def run(argv=None, assert_results=None): beam.io.TextFileSource(known_args.input_email)) phone = read_kv_textfile('phone', beam.io.TextFileSource(known_args.input_phone)) - snailmail = read_kv_textfile('snailmail', - beam.io.TextFileSource(known_args.input_snailmail)) + snailmail = read_kv_textfile('snailmail', beam.io.TextFileSource( + known_args.input_snailmail)) # Group together all entries under the same name. grouped = (email, phone, snailmail) | beam.CoGroupByKey('group_by_name') @@ -109,17 +109,17 @@ def run(argv=None, assert_results=None): # Write tab-delimited output. # pylint: disable=expression-not-assigned tsv_lines | beam.io.Write('write_tsv', - beam.io.TextFileSink(known_args.output_tsv)) + beam.io.TextFileSink(known_args.output_tsv)) # TODO(silviuc): Move the assert_results logic to the unit test. if assert_results is not None: expected_luddites, expected_writers, expected_nomads = assert_results beam.assert_that(num_luddites, beam.equal_to([expected_luddites]), - label='assert:luddites') + label='assert:luddites') beam.assert_that(num_writers, beam.equal_to([expected_writers]), - label='assert:writers') + label='assert:writers') beam.assert_that(num_nomads, beam.equal_to([expected_nomads]), - label='assert:nomads') + label='assert:nomads') # Execute pipeline. p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py index 861f95c..48b8b1e 100644 --- a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py +++ b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py @@ -153,18 +153,20 @@ def run(argv=None): | beam.Map('pair_with_key', lambda x: ('chars_temp_key', x)) | beam.GroupByKey() | beam.Map('count chars', lambda (_, counts): sum(counts)) - | beam.Write('write chars', beam.io.TextFileSink(known_args.output + '-chars'))) + | beam.Write('write chars', + beam.io.TextFileSink(known_args.output + '-chars'))) # pylint: disable=expression-not-assigned (short_words | CountWords('count short words') | beam.Write('write short words', - beam.io.TextFileSink(known_args.output + '-short-words'))) + beam.io.TextFileSink(known_args.output + '-short-words'))) # pylint: disable=expression-not-assigned (words | CountWords('count words') - | beam.Write('write words', beam.io.TextFileSink(known_args.output + '-words'))) + | beam.Write('write words', + beam.io.TextFileSink(known_args.output + '-words'))) p.run() http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/internal/json_value_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/internal/json_value_test.py b/sdks/python/apache_beam/internal/json_value_test.py index f2ae0c1..ad2a394 100644 --- a/sdks/python/apache_beam/internal/json_value_test.py +++ b/sdks/python/apache_beam/internal/json_value_test.py @@ -20,7 +20,6 @@ import unittest from apitools.base.py.extra_types import JsonValue -from apitools.base.py.extra_types import JsonObject from apache_beam.internal.json_value import from_json_value from apache_beam.internal.json_value import to_json_value http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/transforms/core.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index 8f4c246..12c0a97 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -20,14 +20,11 @@ from __future__ import absolute_import import copy -import uuid from apache_beam import pvalue from apache_beam import typehints from apache_beam.coders import typecoders from apache_beam.internal import util -from apache_beam.pvalue import AsIter -from apache_beam.pvalue import AsSingleton from apache_beam.transforms import ptransform from apache_beam.transforms import window from apache_beam.transforms.ptransform import PTransform http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/transforms/ptransform.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index 106c44f..2b3ad32 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -42,7 +42,6 @@ import operator import os import sys -from apache_beam import coders from apache_beam import error from apache_beam import pvalue from apache_beam import typehints http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/transforms/util.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index 87990a3..6b5b16c 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -20,9 +20,6 @@ from __future__ import absolute_import -import collections -import operator - from apache_beam.pvalue import AsIter as AllOf from apache_beam.transforms.core import CombinePerKey, Create, Flatten, GroupByKey, Map from apache_beam.transforms.ptransform import PTransform http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/2094e009/sdks/python/apache_beam/transforms/window.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index e58cf3c..cc17bcf 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -369,7 +369,7 @@ class Sessions(WindowFn): to_merge = [] for w in sorted(merge_context.windows, key=lambda w: w.start): if to_merge: - if end > w.start: + if end > w.start: # pylint: disable=used-before-assignment to_merge.append(w) if w.end > end: end = w.end
