[
https://issues.apache.org/jira/browse/BEAM-5620?focusedWorklogId=152430&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-152430
]
ASF GitHub Bot logged work on BEAM-5620:
----------------------------------------
Author: ASF GitHub Bot
Created on: 08/Oct/18 21:51
Start Date: 08/Oct/18 21:51
Worklog Time Spent: 10m
Work Description: aaltay closed pull request #6599: [BEAM-5620] rename
assertItemsEqual to assertCountEqual for PY3 compatibility
URL: https://github.com/apache/beam/pull/6599
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py
b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index 4588ef4b99f..7a3e5169f49 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -24,6 +24,7 @@
import gzip
import logging
import os
+import sys
import tempfile
import unittest
import uuid
@@ -474,6 +475,12 @@ def expand(self, pcoll):
return pcoll | 'DummyWriteForTesting' >> beam.ParDo(
SnippetsTest.DummyWriteTransform.WriteDoFn(self.file_to_write))
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
self.old_read_from_text = beam.io.ReadFromText
self.old_write_to_text = beam.io.WriteToText
@@ -591,7 +598,7 @@ def rename_table(self, access_token, old_name, new_name):
for line in f:
received_output.append(line.rstrip(os.linesep))
- self.assertItemsEqual(expected_output, received_output)
+ self.assertCountEqual(expected_output, received_output)
glob_pattern = tempdir_name + os.sep + 'final_table_with_ptransform*'
output_files = glob.glob(glob_pattern)
@@ -603,7 +610,7 @@ def rename_table(self, access_token, old_name, new_name):
for line in f:
received_output.append(line.rstrip(os.linesep))
- self.assertItemsEqual(expected_output, received_output)
+ self.assertCountEqual(expected_output, received_output)
def test_model_textio(self):
temp_path = self.create_temp_file('aa bb cc\n bb cc\n cc')
diff --git a/sdks/python/apache_beam/io/avroio_test.py
b/sdks/python/apache_beam/io/avroio_test.py
index 93f2ba9ebfd..daec98c38dd 100644
--- a/sdks/python/apache_beam/io/avroio_test.py
+++ b/sdks/python/apache_beam/io/avroio_test.py
@@ -19,6 +19,7 @@
import json
import logging
import os
+import sys
import tempfile
import unittest
from builtins import range
@@ -59,6 +60,12 @@ def __init__(self, methodName='runTest'):
super(TestAvro, self).__init__(methodName)
self.use_fastavro = False
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
# Reducing the size of thread pools. Without this test execution may fail
in
# environments with limited amount of resources.
@@ -150,7 +157,7 @@ def _run_avro_test(self, pattern, desired_bundle_size,
perform_splitting,
(source, None, None), sources_info)
else:
read_records = source_test_utils.read_from_source(source, None, None)
- self.assertItemsEqual(expected_result, read_records)
+ self.assertCountEqual(expected_result, read_records)
def test_read_without_splitting(self):
file_name = self._write_data()
diff --git a/sdks/python/apache_beam/io/filebasedsink_test.py
b/sdks/python/apache_beam/io/filebasedsink_test.py
index b79370eff4a..55e5a16dff8 100644
--- a/sdks/python/apache_beam/io/filebasedsink_test.py
+++ b/sdks/python/apache_beam/io/filebasedsink_test.py
@@ -24,6 +24,7 @@
import logging
import os
import shutil
+import sys
import tempfile
import unittest
from builtins import range
@@ -96,6 +97,12 @@ def close(self, file_handle):
class TestFileBasedSink(_TestCaseWithTempDirCleanUp):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def _common_init(self, sink):
# Manually invoke the generic Sink API.
init_token = sink.initialize_write()
@@ -136,7 +143,7 @@ def test_file_sink_writing(self):
self.assertEqual(open(shard2).read(), '[start][x][y][z][end]')
# Check that any temp files are deleted.
- self.assertItemsEqual([shard1, shard2], glob.glob(temp_path + '*'))
+ self.assertCountEqual([shard1, shard2], glob.glob(temp_path + '*'))
def test_file_sink_display_data(self):
temp_path = os.path.join(self._new_tempdir(), 'display')
@@ -277,7 +284,7 @@ def test_file_sink_multi_shards(self):
open(shard_name).read(), ('[start][a][b][%s][end]' % uuid))
# Check that any temp files are deleted.
- self.assertItemsEqual(res, glob.glob(temp_path + '*'))
+ self.assertCountEqual(res, glob.glob(temp_path + '*'))
@mock.patch.object(filebasedsink.FileSystems, 'rename')
def test_file_sink_rename_error(self, rename_mock):
diff --git a/sdks/python/apache_beam/io/filebasedsource_test.py
b/sdks/python/apache_beam/io/filebasedsource_test.py
index 3ed27022300..0e57a04356d 100644
--- a/sdks/python/apache_beam/io/filebasedsource_test.py
+++ b/sdks/python/apache_beam/io/filebasedsource_test.py
@@ -24,6 +24,7 @@
import math
import os
import random
+import sys
import tempfile
import unittest
from builtins import object
@@ -579,6 +580,12 @@ def default_output_coder(self):
class TestSingleFileSource(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
# Reducing the size of thread pools. Without this test execution may fail
in
# environments with limited amount of resources.
@@ -650,7 +657,7 @@ def test_read_range_at_beginning(self):
source = SingleFileSource(fbs, file_name, 0, 10 * 6)
range_tracker = source.get_range_tracker(0, 20)
read_data = [value for value in source.read(range_tracker)]
- self.assertItemsEqual(expected_data[:4], read_data)
+ self.assertCountEqual(expected_data[:4], read_data)
def test_read_range_at_end(self):
fbs = LineSource('dummy_pattern', validate=False)
@@ -661,7 +668,7 @@ def test_read_range_at_end(self):
source = SingleFileSource(fbs, file_name, 0, 10 * 6)
range_tracker = source.get_range_tracker(40, 60)
read_data = [value for value in source.read(range_tracker)]
- self.assertItemsEqual(expected_data[-3:], read_data)
+ self.assertCountEqual(expected_data[-3:], read_data)
def test_read_range_at_middle(self):
fbs = LineSource('dummy_pattern', validate=False)
@@ -672,7 +679,7 @@ def test_read_range_at_middle(self):
source = SingleFileSource(fbs, file_name, 0, 10 * 6)
range_tracker = source.get_range_tracker(20, 40)
read_data = [value for value in source.read(range_tracker)]
- self.assertItemsEqual(expected_data[4:7], read_data)
+ self.assertCountEqual(expected_data[4:7], read_data)
def test_produces_splits_desiredsize_large_than_size(self):
fbs = LineSource('dummy_pattern', validate=False)
@@ -688,7 +695,7 @@ def test_produces_splits_desiredsize_large_than_size(self):
range_tracker = splits[0].source.get_range_tracker(None, None)
read_data = [value for value in splits[0].source.read(range_tracker)]
- self.assertItemsEqual(expected_data, read_data)
+ self.assertCountEqual(expected_data, read_data)
def test_produces_splits_desiredsize_smaller_than_size(self):
fbs = LineSource('dummy_pattern', validate=False)
@@ -706,7 +713,7 @@ def
test_produces_splits_desiredsize_smaller_than_size(self):
split.stop_position)
data_from_split = [data for data in source.read(range_tracker)]
read_data.extend(data_from_split)
- self.assertItemsEqual(expected_data, read_data)
+ self.assertCountEqual(expected_data, read_data)
def test_produce_split_with_start_and_end_positions(self):
fbs = LineSource('dummy_pattern', validate=False)
@@ -726,7 +733,7 @@ def test_produce_split_with_start_and_end_positions(self):
split.stop_position)
data_from_split = [data for data in source.read(range_tracker)]
read_data.extend(data_from_split)
- self.assertItemsEqual(expected_data[2:9], read_data)
+ self.assertCountEqual(expected_data[2:9], read_data)
if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
b/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
index 0f9351d346d..1261aa13159 100644
--- a/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
+++ b/sdks/python/apache_beam/io/gcp/tests/pubsub_matcher_test.py
@@ -20,6 +20,7 @@
from __future__ import absolute_import
import logging
+import sys
import unittest
import mock
@@ -42,6 +43,12 @@
@mock.patch('google.cloud.pubsub.SubscriberClient')
class PubSubMatcherTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
self.mock_presult = mock.MagicMock()
@@ -123,7 +130,7 @@ def test_message_matcher_mismatch(self, mock_get_sub,
unused_mock):
with self.assertRaises(AssertionError) as error:
hc_assert_that(self.mock_presult, self.pubsub_matcher)
self.assertEqual(mock_sub.pull.call_count, 1)
- self.assertItemsEqual(['c', 'd'], self.pubsub_matcher.messages)
+ self.assertCountEqual(['c', 'd'], self.pubsub_matcher.messages)
self.assertTrue(
'\nExpected: Expected 1 messages.\n but: Got 2 messages.'
in str(error.exception.args[0]))
diff --git a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
index a943a12bb4d..b3b44b55508 100644
--- a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
+++ b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
@@ -22,6 +22,7 @@
import io
import logging
import posixpath
+import sys
import unittest
from builtins import object
@@ -197,6 +198,12 @@ def checksum(self, path):
class HadoopFileSystemTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
self._fake_hdfs = FakeHdfs()
hdfs.hdfs.InsecureClient = (
@@ -258,7 +265,7 @@ def test_match_file(self):
returned_files = [f.path
for match_result in result
for f in match_result.metadata_list]
- self.assertItemsEqual(expected_files, returned_files)
+ self.assertCountEqual(expected_files, returned_files)
def test_match_file_with_limits(self):
expected_files = [self.fs.join(self.tmpdir, filename)
@@ -296,7 +303,7 @@ def test_match_directory(self):
# structure, so listing without a '/' will return no results.
result = self.fs.match([self.tmpdir + '/'])[0]
files = [f.path for f in result.metadata_list]
- self.assertItemsEqual(files, expected_files)
+ self.assertCountEqual(files, expected_files)
def test_match_directory_trailing_slash(self):
expected_files = [self.fs.join(self.tmpdir, filename)
@@ -304,7 +311,7 @@ def test_match_directory_trailing_slash(self):
result = self.fs.match([self.tmpdir + '/'])[0]
files = [f.path for f in result.metadata_list]
- self.assertItemsEqual(files, expected_files)
+ self.assertCountEqual(files, expected_files)
def test_create_success(self):
url = self.fs.join(self.tmpdir, 'new_file')
diff --git a/sdks/python/apache_beam/io/localfilesystem_test.py
b/sdks/python/apache_beam/io/localfilesystem_test.py
index 1f813b4cc48..46ed6a29f41 100644
--- a/sdks/python/apache_beam/io/localfilesystem_test.py
+++ b/sdks/python/apache_beam/io/localfilesystem_test.py
@@ -24,6 +24,7 @@
import logging
import os
import shutil
+import sys
import tempfile
import unittest
@@ -60,6 +61,12 @@ def _split(path):
class LocalFileSystemTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
pipeline_options = PipelineOptions()
@@ -185,7 +192,7 @@ def test_match_glob(self, pattern, files, expected):
full_pattern = os.path.join(self.tmpdir, pattern)
result = self.fs.match([full_pattern])[0]
files = [os.path.relpath(f.path, self.tmpdir) for f in
result.metadata_list]
- self.assertItemsEqual(files, expected)
+ self.assertCountEqual(files, expected)
def test_match_directory(self):
result = self.fs.match([self.tmpdir])[0]
@@ -200,7 +207,7 @@ def test_match_directory_contents(self):
result = self.fs.match([self.tmpdir + '/'])[0]
files = [f.path for f in result.metadata_list]
- self.assertItemsEqual(files, [path1, path2])
+ self.assertCountEqual(files, [path1, path2])
def test_copy(self):
path1 = os.path.join(self.tmpdir, 'f1')
diff --git a/sdks/python/apache_beam/io/source_test_utils_test.py
b/sdks/python/apache_beam/io/source_test_utils_test.py
index 94eb4401f6b..38a2e8eff74 100644
--- a/sdks/python/apache_beam/io/source_test_utils_test.py
+++ b/sdks/python/apache_beam/io/source_test_utils_test.py
@@ -18,6 +18,7 @@
from __future__ import absolute_import
import logging
+import sys
import tempfile
import unittest
from builtins import range
@@ -28,6 +29,12 @@
class SourceTestUtilsTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def _create_file_with_data(self, lines):
assert isinstance(lines, list)
with tempfile.NamedTemporaryFile(delete=False) as f:
@@ -50,7 +57,7 @@ def _create_source(self, data):
def test_read_from_source(self):
data = self._create_data(100)
source = self._create_source(data)
- self.assertItemsEqual(
+ self.assertCountEqual(
data, source_test_utils.read_from_source(source, None, None))
def test_source_equals_reference_source(self):
diff --git a/sdks/python/apache_beam/io/sources_test.py
b/sdks/python/apache_beam/io/sources_test.py
index 40629ae8395..cba8922976e 100644
--- a/sdks/python/apache_beam/io/sources_test.py
+++ b/sdks/python/apache_beam/io/sources_test.py
@@ -20,6 +20,7 @@
import logging
import os
+import sys
import tempfile
import unittest
@@ -83,6 +84,12 @@ def default_output_coder(self):
class SourcesTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def _create_temp_file(self, contents):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(contents)
@@ -95,7 +102,7 @@ def test_read_from_source(self):
range_tracker = source.get_range_tracker(None, None)
result = [line for line in source.read(range_tracker)]
- self.assertItemsEqual(['aaaa', 'bbbb', 'cccc', 'dddd'], result)
+ self.assertCountEqual(['aaaa', 'bbbb', 'cccc', 'dddd'], result)
def test_run_direct(self):
file_name = self._create_temp_file('aaaa\nbbbb\ncccc\ndddd')
diff --git a/sdks/python/apache_beam/io/textio_test.py
b/sdks/python/apache_beam/io/textio_test.py
index 3606897049d..adb96d1b8d1 100644
--- a/sdks/python/apache_beam/io/textio_test.py
+++ b/sdks/python/apache_beam/io/textio_test.py
@@ -25,6 +25,7 @@
import logging
import os
import shutil
+import sys
import tempfile
import unittest
from builtins import range
@@ -55,6 +56,12 @@ class TextSourceTest(unittest.TestCase):
# Number of records that will be written by most tests.
DEFAULT_NUM_RECORDS = 100
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def _run_read_test(self, file_or_pattern, expected_data,
buffer_size=DEFAULT_NUM_RECORDS,
compression=CompressionTypes.UNCOMPRESSED):
@@ -65,7 +72,7 @@ def _run_read_test(self, file_or_pattern, expected_data,
True, coders.StrUtf8Coder(), buffer_size)
range_tracker = source.get_range_tracker(None, None)
read_data = list(source.read(range_tracker))
- self.assertItemsEqual(expected_data, read_data)
+ self.assertCountEqual(expected_data, read_data)
def test_read_single_file(self):
file_name, expected_data = write_data(TextSourceTest.DEFAULT_NUM_RECORDS)
@@ -187,7 +194,7 @@ def test_read_single_file_without_striping_eol_lf(self):
range_tracker = source.get_range_tracker(None, None)
read_data = list(source.read(range_tracker))
- self.assertItemsEqual([line + '\n' for line in written_data], read_data)
+ self.assertCountEqual([line + '\n' for line in written_data], read_data)
def test_read_single_file_without_striping_eol_crlf(self):
file_name, written_data = write_data(TextSourceTest.DEFAULT_NUM_RECORDS,
@@ -198,7 +205,7 @@ def test_read_single_file_without_striping_eol_crlf(self):
range_tracker = source.get_range_tracker(None, None)
read_data = list(source.read(range_tracker))
- self.assertItemsEqual([line + '\r\n' for line in written_data], read_data)
+ self.assertCountEqual([line + '\r\n' for line in written_data], read_data)
def test_read_file_pattern_with_empty_files(self):
pattern, expected_data = write_pattern(
@@ -249,8 +256,8 @@ def store_header(lines):
splits[0].start_position, splits[0].stop_position)
read_data = list(source.read_records(file_name, range_tracker))
- self.assertItemsEqual(expected_data[:5], header_lines)
- self.assertItemsEqual(expected_data[5:], read_data)
+ self.assertCountEqual(expected_data[:5], header_lines)
+ self.assertCountEqual(expected_data[5:], read_data)
def test_progress(self):
file_name, expected_data = write_data(10)
@@ -713,7 +720,7 @@ def test_read_skip_header_single(self):
skip_header_lines)
read_data = self._read_skip_header_lines(file_name, skip_header_lines)
self.assertEqual(len(expected_data), len(read_data))
- self.assertItemsEqual(expected_data, read_data)
+ self.assertCountEqual(expected_data, read_data)
def test_read_skip_header_pattern(self):
line_counts = [
@@ -730,7 +737,7 @@ def test_read_skip_header_pattern(self):
expected_data = self._remove_lines(data, line_counts, skip_header_lines)
read_data = self._read_skip_header_lines(pattern, skip_header_lines)
self.assertEqual(len(expected_data), len(read_data))
- self.assertItemsEqual(expected_data, read_data)
+ self.assertCountEqual(expected_data, read_data)
def test_read_skip_header_pattern_insufficient_lines(self):
line_counts = [
@@ -743,7 +750,7 @@ def test_read_skip_header_pattern_insufficient_lines(self):
data = self._remove_lines(data, line_counts, skip_header_lines)
read_data = self._read_skip_header_lines(pattern, skip_header_lines)
self.assertEqual(len(data), len(read_data))
- self.assertItemsEqual(data, read_data)
+ self.assertCountEqual(data, read_data)
def test_read_gzip_with_skip_lines(self):
_, lines = write_data(15)
@@ -782,6 +789,12 @@ def test_read_after_splitting_skip_header(self):
class TextSinkTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def setUp(self):
super(TextSinkTest, self).setUp()
self.lines = ['Line %d' % d for d in range(100)]
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py
b/sdks/python/apache_beam/transforms/ptransform_test.py
index c594e6ab28b..95ea03f2ba2 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -24,6 +24,7 @@
import collections
import operator
import re
+import sys
import unittest
from builtins import map
from builtins import range
@@ -62,6 +63,12 @@ class PTransformTest(unittest.TestCase):
# Enable nose tests running in parallel
_multiprocess_can_split_ = True
+ @classmethod
+ def setUpClass(cls):
+ # Method has been renamed in Python 3
+ if sys.version_info[0] < 3:
+ cls.assertCountEqual = cls.assertItemsEqual
+
def assertStartswith(self, msg, prefix):
self.assertTrue(msg.startswith(prefix),
'"%s" does not start with "%s"' % (msg, prefix))
@@ -659,15 +666,15 @@ def test_chained_ptransforms(self):
pipeline.run()
def test_apply_to_list(self):
- self.assertItemsEqual(
+ self.assertCountEqual(
[1, 2, 3], [0, 1, 2] | 'AddOne' >> beam.Map(lambda x: x + 1))
self.assertItemsEqual([1],
[0, 1, 2] | 'Odd' >> beam.Filter(lambda x: x % 2))
- self.assertItemsEqual([1, 2, 100, 3],
+ self.assertCountEqual([1, 2, 100, 3],
([1, 2, 3], [100]) | beam.Flatten())
join_input = ([('k', 'a')],
[('k', 'b'), ('k', 'c')])
- self.assertItemsEqual([('k', (['a'], ['b', 'c']))],
+ self.assertCountEqual([('k', (['a'], ['b', 'c']))],
join_input | beam.CoGroupByKey())
def test_multi_input_ptransform(self):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 152430)
Time Spent: 0.5h (was: 20m)
> Some tests use assertItemsEqual method, not available in Python 3
> -----------------------------------------------------------------
>
> Key: BEAM-5620
> URL: https://issues.apache.org/jira/browse/BEAM-5620
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-py-core
> Reporter: Valentyn Tymofieiev
> Assignee: Matthias Feys
> Priority: Major
> Time Spent: 0.5h
> Remaining Estimate: 0h
>
> See:
> https://github.com/apache/beam/search?q=assertItemsEqual&unscoped_q=assertItemsEqual
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
