This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c6c12cc [BEAM-5315] python 3 datastore fail gracefully
new 5406b98 Merge pull request #7804 from Juta/io-gcp
c6c12cc is described below
commit c6c12ccbcea10cde99dfbc39c1d58585977a4adc
Author: Juta <juta_st...@hotmail.com>
AuthorDate: Mon Feb 11 15:13:49 2019 +0100
[BEAM-5315] python 3 datastore fail gracefully
---
.../examples/cookbook/datastore_wordcount_it_test.py | 6 ++++++
sdks/python/apache_beam/examples/snippets/snippets_test.py | 4 ++++
.../apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py | 6 ++++++
sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py | 7 ++++++-
.../python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py | 10 ++++++++++
sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py | 6 ++++++
sdks/python/apache_beam/io/gcp/datastore_write_it_test.py | 6 ++++++
sdks/python/tox.ini | 2 +-
8 files changed, 45 insertions(+), 2 deletions(-)
diff --git
a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
index d3d63cc..5a4fea9 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
@@ -20,6 +20,8 @@
from __future__ import absolute_import
import logging
+import os
+import sys
import time
import unittest
@@ -32,6 +34,10 @@ from apache_beam.testing.pipeline_verifiers import
PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
class DatastoreWordCountIT(unittest.TestCase):
DATASTORE_WORDCOUNT_KIND = "DatastoreWordCount"
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py
b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index d6ac1f5..64ed9fe 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -631,6 +631,10 @@ class SnippetsTest(unittest.TestCase):
snippets.model_textio_compressed(
{'read': gzip_file_name}, ['aa', 'bb', 'cc'])
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
@unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
def test_model_datastoreio(self):
# We cannot test DatastoreIO functionality in unit tests, therefore we
limit
diff --git
a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
index e3ccb92..1d9c46e 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
@@ -18,6 +18,8 @@
from __future__ import absolute_import
from __future__ import division
+import os
+import sys
import unittest
from builtins import range
@@ -26,6 +28,10 @@ from mock import patch
from apache_beam.io.gcp.datastore.v1.adaptive_throttler import
AdaptiveThrottler
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
class AdaptiveThrottlerTest(unittest.TestCase):
START_TIME = 1500000000000
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
index 123d063..d12be78 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
@@ -20,7 +20,9 @@ from __future__ import absolute_import
from __future__ import division
import logging
+import sys
import time
+import warnings
from builtins import object
from builtins import round
@@ -44,7 +46,10 @@ try:
from google.cloud.proto.datastore.v1 import datastore_pb2
from googledatastore import helper as datastore_helper
except ImportError:
- pass
+ if sys.version_info[0] == 3:
+ warnings.warn('Datastore IO will support Python 3 after replacing '
+ 'googledatastore by google-cloud-datastore, '
+ 'see: BEAM-4543.')
# pylint: enable=wrong-import-order, wrong-import-position
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
index b7bc22a..d345fef 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
@@ -19,6 +19,8 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+import os
+import sys
import unittest
from builtins import map
from builtins import range
@@ -48,6 +50,10 @@ except ImportError:
# pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
@unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
class DatastoreioTest(unittest.TestCase):
_PROJECT = 'project'
@@ -272,6 +278,10 @@ class DatastoreioTest(unittest.TestCase):
return split_queries
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
@unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
class DynamicWriteBatcherTest(unittest.TestCase):
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
index 8b05ebe..0f04727 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
@@ -18,11 +18,17 @@
"""Tests for util.py."""
from __future__ import absolute_import
+import os
+import sys
import unittest
from apache_beam.io.gcp.datastore.v1 import util
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
class MovingSumTest(unittest.TestCase):
TIMESTAMP = 1500000000
diff --git a/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
b/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
index a7947a2..44af815 100644
--- a/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
@@ -28,7 +28,9 @@ results in the pipeline.
from __future__ import absolute_import
import logging
+import os
import random
+import sys
import unittest
from datetime import datetime
@@ -40,6 +42,10 @@ from apache_beam.testing.pipeline_verifiers import
PipelineStateMatcher
from apache_beam.testing.test_pipeline import TestPipeline
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3'
+ 'TODO: BEAM-4543')
class DatastoreWriteIT(unittest.TestCase):
NUM_ENTITIES = 1001
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 743e0bd..3661c9d 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -93,7 +93,7 @@ setenv =
RUN_SKIPPED_PY3_TESTS=0
extras = test,gcp
modules =
-
apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.i
[...]
+
apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.i
[...]
commands =
python --version
pip --version
