This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 7bd73a5 [BEAM-4751] fix missing pylint3 check for io subpackage
(#5916)
7bd73a5 is described below
commit 7bd73a51b670755bbb19e1291003722d5d16bdc5
Author: Matthias Feys <[email protected]>
AuthorDate: Mon Jul 30 19:59:06 2018 +0200
[BEAM-4751] fix missing pylint3 check for io subpackage (#5916)
* fix missing pylint3 check for io subpackage
* resolved hash if dict issues
* remove and simplify __hash__
---
sdks/python/apache_beam/io/concat_source_test.py | 6 +++++-
sdks/python/apache_beam/io/filebasedsink.py | 1 +
sdks/python/apache_beam/io/filebasedsource_test.py | 1 +
sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py | 1 +
sdks/python/apache_beam/io/gcp/datastore/v1/helper.py | 2 ++
sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter.py | 1 +
sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py | 2 ++
sdks/python/apache_beam/io/gcp/gcsio_test.py | 1 +
sdks/python/apache_beam/io/gcp/pubsub_integration_test.py | 1 +
sdks/python/apache_beam/io/gcp/pubsub_it_pipeline.py | 2 ++
sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py | 2 ++
sdks/python/apache_beam/io/hadoopfilesystem_test.py | 1 +
sdks/python/apache_beam/io/range_trackers.py | 6 ++++--
sdks/python/apache_beam/io/restriction_trackers.py | 3 +++
sdks/python/apache_beam/io/tfrecordio.py | 9 +++++----
sdks/python/apache_beam/io/vcfio.py | 6 +++++-
sdks/python/tox.ini | 1 +
18 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/sdks/python/apache_beam/io/concat_source_test.py
b/sdks/python/apache_beam/io/concat_source_test.py
index 31e4392..1e8603a 100644
--- a/sdks/python/apache_beam/io/concat_source_test.py
+++ b/sdks/python/apache_beam/io/concat_source_test.py
@@ -35,6 +35,8 @@ from apache_beam.testing.util import equal_to
class RangeSource(iobase.BoundedSource):
+ __hash__ = None
+
def __init__(self, start, end, split_freq=1):
assert start <= end
self._start = start
@@ -76,7 +78,9 @@ class RangeSource(iobase.BoundedSource):
# For testing
def __eq__(self, other):
return (type(self) == type(other)
- and self._start == other._start and self._end == other._end)
+ and self._start == other._start
+ and self._end == other._end
+ and self._split_freq == other._split_freq)
def __ne__(self, other):
return not self == other
diff --git a/sdks/python/apache_beam/io/filebasedsink.py
b/sdks/python/apache_beam/io/filebasedsink.py
index dc6726e..5a09582 100644
--- a/sdks/python/apache_beam/io/filebasedsink.py
+++ b/sdks/python/apache_beam/io/filebasedsink.py
@@ -60,6 +60,7 @@ class FileBasedSink(iobase.Sink):
# Max number of threads to be used for renaming.
_MAX_RENAME_THREADS = 64
+ __hash__ = None
def __init__(self,
file_path_prefix,
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py
b/sdks/python/apache_beam/io/filebasedsource_test.py
index e931223..71498d0 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -15,6 +15,7 @@
#
from __future__ import absolute_import
+from __future__ import division
import bz2
import gzip
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
index 437f388..66cca24 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
@@ -22,6 +22,7 @@ from __future__ import division
import logging
import time
from builtins import object
+from builtins import round
from apache_beam.io.gcp.datastore.v1 import helper
from apache_beam.io.gcp.datastore.v1 import query_splitter
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
index a27df09..5fdc3a7 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/helper.py
@@ -20,6 +20,8 @@
For internal use only; no backwards-compatibility guarantees.
"""
+from __future__ import absolute_import
+
import errno
import logging
import sys
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter.py
index 7723fb7..ef3c1e4 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/query_splitter.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
from __future__ import division
from builtins import range
+from builtins import round
from apache_beam.io.gcp.datastore.v1 import helper
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
index 8f17c21..8b05ebe 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
@@ -16,6 +16,8 @@
#
"""Tests for util.py."""
+from __future__ import absolute_import
+
import unittest
from apache_beam.io.gcp.datastore.v1 import util
diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
index 88f7ce9..757475d 100644
--- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_test.py
@@ -18,6 +18,8 @@
"""Unit tests for GCS File System."""
+from __future__ import absolute_import
+
import logging
import unittest
from builtins import zip
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_test.py
b/sdks/python/apache_beam/io/gcp/gcsio_test.py
index b10926c..0a0b16d 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_test.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
"""Tests for Google Cloud Storage client."""
+from __future__ import absolute_import
from __future__ import division
import errno
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_integration_test.py
b/sdks/python/apache_beam/io/gcp/pubsub_integration_test.py
index d287ef2..c3921bb 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_integration_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_integration_test.py
@@ -17,6 +17,7 @@
"""
Integration test for Google Cloud Pub/Sub.
"""
+from __future__ import absolute_import
import logging
import unittest
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_it_pipeline.py
b/sdks/python/apache_beam/io/gcp/pubsub_it_pipeline.py
index 407ed88..0c4535f 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_it_pipeline.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_it_pipeline.py
@@ -18,6 +18,8 @@
Test pipeline for use by pubsub_integration_test.
"""
+from __future__ import absolute_import
+
import argparse
import apache_beam as beam
diff --git a/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
b/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
index 83f21a2..0e59481 100644
--- a/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
+++ b/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
@@ -17,6 +17,8 @@
"""Unit test for PubSub verifier."""
+from __future__ import absolute_import
+
import logging
import unittest
diff --git a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
index cce0ac7..a943a12 100644
--- a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
+++ b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
@@ -35,6 +35,7 @@ from apache_beam.options.pipeline_options import
PipelineOptions
class FakeFile(io.BytesIO):
"""File object for FakeHdfs"""
+ __hash__ = None
def __init__(self, path, mode='', type='FILE'):
io.BytesIO.__init__(self)
diff --git a/sdks/python/apache_beam/io/range_trackers.py
b/sdks/python/apache_beam/io/range_trackers.py
index 50439e1..0514b74 100644
--- a/sdks/python/apache_beam/io/range_trackers.py
+++ b/sdks/python/apache_beam/io/range_trackers.py
@@ -20,9 +20,11 @@
from __future__ import absolute_import
from __future__ import division
+import codecs
import logging
import math
import threading
+from builtins import zip
from past.builtins import long
@@ -402,7 +404,7 @@ class
LexicographicKeyRangeTracker(OrderedPositionRangeTracker):
s += '\0' * (prec - len(s))
else:
s = s[:prec]
- return int(s.encode('hex'), 16)
+ return int(codecs.encode(s, 'hex'), 16)
@staticmethod
def _string_from_int(i, prec):
@@ -410,4 +412,4 @@ class
LexicographicKeyRangeTracker(OrderedPositionRangeTracker):
Inverse of _string_to_int.
"""
h = '%x' % i
- return ('0' * (2 * prec - len(h)) + h).decode('hex')
+ return codecs.decode('0' * (2 * prec - len(h)) + h, 'hex')
diff --git a/sdks/python/apache_beam/io/restriction_trackers.py
b/sdks/python/apache_beam/io/restriction_trackers.py
index 014125f..e722420 100644
--- a/sdks/python/apache_beam/io/restriction_trackers.py
+++ b/sdks/python/apache_beam/io/restriction_trackers.py
@@ -42,6 +42,9 @@ class OffsetRange(object):
return self.start == other.start and self.stop == other.stop
+ def __hash__(self):
+ return hash((type(self), self.start, self.stop))
+
def split(self, desired_num_offsets_per_split, min_num_offsets_per_split=1):
current_split_start = self.start
max_split_size = max(desired_num_offsets_per_split,
diff --git a/sdks/python/apache_beam/io/tfrecordio.py
b/sdks/python/apache_beam/io/tfrecordio.py
index 2ef7c5b..5561d2c 100644
--- a/sdks/python/apache_beam/io/tfrecordio.py
+++ b/sdks/python/apache_beam/io/tfrecordio.py
@@ -18,6 +18,7 @@
from __future__ import absolute_import
+import codecs
import logging
import struct
from builtins import object
@@ -120,24 +121,24 @@ class _TFRecordUtil(object):
# Validate all length related payloads.
if len(buf) != buf_length_expected:
raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
- (buf_length_expected, buf.encode('hex')))
+ (buf_length_expected, codecs.encode(buf, 'hex')))
length, length_mask_expected = struct.unpack('<QI', buf)
length_mask_actual = cls._masked_crc32c(buf[:8])
if length_mask_actual != length_mask_expected:
raise ValueError('Not a valid TFRecord. Mismatch of length mask: %s' %
- buf.encode('hex'))
+ codecs.encode(buf, 'hex'))
# Validate all data related payloads.
buf_length_expected = length + 4
buf = file_handle.read(buf_length_expected)
if len(buf) != buf_length_expected:
raise ValueError('Not a valid TFRecord. Fewer than %d bytes: %s' %
- (buf_length_expected, buf.encode('hex')))
+ (buf_length_expected, codecs.encode(buf, 'hex')))
data, data_mask_expected = struct.unpack('<%dsI' % length, buf)
data_mask_actual = cls._masked_crc32c(data)
if data_mask_actual != data_mask_expected:
raise ValueError('Not a valid TFRecord. Mismatch of data mask: %s' %
- buf.encode('hex'))
+ codecs.encode(buf, 'hex'))
# All validation checks passed.
return data
diff --git a/sdks/python/apache_beam/io/vcfio.py
b/sdks/python/apache_beam/io/vcfio.py
index a5e606b..d96c8f7 100644
--- a/sdks/python/apache_beam/io/vcfio.py
+++ b/sdks/python/apache_beam/io/vcfio.py
@@ -29,6 +29,7 @@ from builtins import object
from collections import namedtuple
from future.utils import iteritems
+from past.builtins import long
from past.builtins import unicode
import vcf
@@ -72,6 +73,7 @@ class Variant(object):
Each object corresponds to a single record in a VCF file.
"""
+ __hash__ = None
def __init__(self,
reference_name=None,
@@ -187,6 +189,8 @@ class VariantCall(object):
variant. It may include associated information such as quality and phasing.
"""
+ __hash__ = None
+
def __init__(self, name=None, genotype=None, phaseset=None, info=None):
"""Initialize the :class:`VariantCall` object.
@@ -407,7 +411,7 @@ class _VcfSource(filebasedsource.FileBasedSource):
# Note: this is already done for INFO fields in PyVCF.
if (field in formats and
formats[field].num is None and
- isinstance(data, (int, float, int, str, unicode, bool))):
+ isinstance(data, (int, float, long, str, unicode, bool))):
data = [data]
call.info[field] = data
variant.calls.append(call)
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 51d91b2..de2352a 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -103,6 +103,7 @@ modules =
apache_beam/runners
apache_beam/examples
apache_beam/portability
+ apache_beam/io
apache_beam/internal
apache_beam/metrics
apache_beam/options
