[
https://issues.apache.org/jira/browse/BEAM-5626?focusedWorklogId=153376&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153376
]
ASF GitHub Bot logged work on BEAM-5626:
----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Oct/18 03:23
Start Date: 11/Oct/18 03:23
Worklog Time Spent: 10m
Work Description: manuzhang closed pull request #6628: [BEAM-5626] Unskip
hadoopfilesystem_test which is already passing in Python 3.
URL: https://github.com/apache/beam/pull/6628
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
index 3dd94b7ed8c..ae2b810f5fc 100644
--- a/sdks/python/apache_beam/io/hadoopfilesystem_test.py
+++ b/sdks/python/apache_beam/io/hadoopfilesystem_test.py
@@ -322,10 +322,6 @@ def test_create_success(self):
expected_file = FakeFile(url, 'wb')
self.assertEqual(self._fake_hdfs.files[url], expected_file)
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_create_write_read_compressed(self):
url = self.fs.join(self.tmpdir, 'new_file.gz')
@@ -363,10 +359,6 @@ def _cmpfiles(self, url1, url2):
data2 = f2.read()
return data1 == data2
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_copy_file(self):
url1 = self.fs.join(self.tmpdir, 'new_file1')
url2 = self.fs.join(self.tmpdir, 'new_file2')
@@ -377,10 +369,6 @@ def test_copy_file(self):
self.assertTrue(self._cmpfiles(url1, url2))
self.assertTrue(self._cmpfiles(url1, url3))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_copy_file_overwrite_error(self):
url1 = self.fs.join(self.tmpdir, 'new_file1')
url2 = self.fs.join(self.tmpdir, 'new_file2')
@@ -392,10 +380,6 @@ def test_copy_file_overwrite_error(self):
BeamIOError, r'already exists.*%s' % posixpath.basename(url2)):
self.fs.copy([url1], [url2])
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_copy_file_error(self):
url1 = self.fs.join(self.tmpdir, 'new_file1')
url2 = self.fs.join(self.tmpdir, 'new_file2')
@@ -409,10 +393,6 @@ def test_copy_file_error(self):
self.fs.copy([url1, url3], [url2, url4])
self.assertTrue(self._cmpfiles(url3, url4))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_copy_directory(self):
url_t1 = self.fs.join(self.tmpdir, 't1')
url_t1_inner = self.fs.join(self.tmpdir, 't1/inner')
@@ -430,10 +410,6 @@ def test_copy_directory(self):
self.fs.copy([url_t1], [url_t2])
self.assertTrue(self._cmpfiles(url1, url2))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_copy_directory_overwrite_error(self):
url_t1 = self.fs.join(self.tmpdir, 't1')
url_t1_inner = self.fs.join(self.tmpdir, 't1/inner')
@@ -458,10 +434,6 @@ def test_copy_directory_overwrite_error(self):
with self.assertRaisesRegexp(BeamIOError, r'already exists'):
self.fs.copy([url_t1], [url_t2])
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_rename_file(self):
url1 = self.fs.join(self.tmpdir, 'f1')
url2 = self.fs.join(self.tmpdir, 'f2')
@@ -490,10 +462,6 @@ def test_rename_file_error(self):
self.assertFalse(self.fs.exists(url3))
self.assertTrue(self.fs.exists(url4))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_rename_directory(self):
url_t1 = self.fs.join(self.tmpdir, 't1')
url_t2 = self.fs.join(self.tmpdir, 't2')
@@ -515,20 +483,12 @@ def test_exists(self):
self.assertTrue(self.fs.exists(url1))
self.assertFalse(self.fs.exists(url2))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_size(self):
url = self.fs.join(self.tmpdir, 'f1')
with self.fs.create(url) as f:
f.write(b'Hello')
self.assertEqual(5, self.fs.size(url))
- @unittest.skipIf(sys.version_info[0] == 3 and
- os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
- 'This test still needs to be fixed on Python 3'
- 'TODO: BEAM-5627')
def test_checksum(self):
url = self.fs.join(self.tmpdir, 'f1')
with self.fs.create(url) as f:
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 5ed55e9635e..09c794b16db 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -58,7 +58,7 @@ setenv =
BEAM_EXPERIMENTAL_PY3=1
RUN_SKIPPED_PY3_TESTS=0
modules =
-
apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.gcsfilesystem_test,apache_beam.io.gcp.gcsio_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.internal,apache_beam.io.filesystem_test,apache_beam.io.filesystems,apache_beam.io.range_trackers_test,apache_beam.io.sources_test
+
apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.gcsfilesystem_test,apache_beam.io.gcp.gcsio_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.io.hdfs_integration_test,apache_beam.io.gcp.internal,apache_beam.io.filesystem_test,apache_beam.io.filesystems,apache_beam.io.range_trackers_test,apache_beam.io.sources_test
commands =
python --version
pip --version
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 153376)
Time Spent: 5h 10m (was: 5h)
> Several IO tests fail in Python 3 with RuntimeError('dictionary changed size
> during iteration',)}
> -------------------------------------------------------------------------------------------------
>
> Key: BEAM-5626
> URL: https://issues.apache.org/jira/browse/BEAM-5626
> Project: Beam
> Issue Type: Sub-task
> Components: sdk-py-core
> Reporter: Valentyn Tymofieiev
> Assignee: Ruoyun Huang
> Priority: Major
> Fix For: 2.8.0
>
> Time Spent: 5h 10m
> Remaining Estimate: 0h
>
> ERROR: test_delete_dir
> (apache_beam.io.hadoopfilesystem_test.HadoopFileSystemTest)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/apache_beam/io/hadoopfilesystem_test.py",
> line 506, in test_delete_dir
> self.fs.delete([url_t1])
> File
> "/usr/local/google/home/valentyn/projects/beam/clean_head/beam/sdks/python/apache_beam/io/hadoopfilesystem.py",
> line 370, in delete
> raise BeamIOError("Delete operation failed", exceptions)
> apache_beam.io.filesystem.BeamIOError: Delete operation failed with
> exceptions {'hdfs://test_dir/new_dir1': RuntimeError('dictionary changed size
> during iteration', )}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
