[
https://issues.apache.org/jira/browse/BEAM-4543?focusedWorklogId=231842&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-231842
]
ASF GitHub Bot logged work on BEAM-4543:
----------------------------------------
Author: ASF GitHub Bot
Created on: 24/Apr/19 02:40
Start Date: 24/Apr/19 02:40
Worklog Time Spent: 10m
Work Description: udim commented on pull request #8262: [BEAM-4543]
Python Datastore IO using google-cloud-datastore
URL: https://github.com/apache/beam/pull/8262#discussion_r277905066
##########
File path: sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py
##########
@@ -0,0 +1,350 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for datastoreio."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import datetime
+import math
+import unittest
+
+from mock import MagicMock
+from mock import call
+from mock import patch
+
+# Protect against environments where datastore library is not available.
+try:
+ from apache_beam.io.gcp.datastore.v1new import helper
+ from apache_beam.io.gcp.datastore.v1new import query_splitter
+ from apache_beam.io.gcp.datastore.v1new.datastoreio import
DeleteFromDatastore
+ from apache_beam.io.gcp.datastore.v1new.datastoreio import QueryDatastore
+ from apache_beam.io.gcp.datastore.v1new.datastoreio import WriteToDatastore
+ from apache_beam.io.gcp.datastore.v1new.datastoreio import _Mutate
+ from google.cloud.datastore import client
+ from google.cloud.datastore import entity
+ from google.cloud.datastore import helpers
+ from google.cloud.datastore import key
+# TODO(BEAM-4543): Remove TypeError once googledatastore dependency is removed.
+except (ImportError, TypeError):
+ client = None
+
+
+class FakeMutation(object):
+ def __init__(self, entity=None, key=None):
+ """Fake mutation request object.
+
+ Requires exactly one of entity or key to be set.
+
+ Args:
+ entity: (``google.cloud.datastore.entity.Entity``) entity representing
+ this upsert mutation
+ key: (``google.cloud.datastore.key.Key``) key representing
+ this delete mutation
+ """
+ self.entity = entity
+ self.key = key
+
+ def ByteSize(self):
+ if self.entity is not None:
+ return helpers.entity_to_protobuf(self.entity).ByteSize()
+ else:
+ return self.key.to_protobuf().ByteSize()
+
+
+class FakeBatch(object):
+ def __init__(self, all_batch_items=None, commit_count=None):
+ """Fake ``google.cloud.datastore.batch.Batch`` object.
+
+ Args:
+ all_batch_items: (list) If set, will append all entities/keys added to
+ this batch.
+ commit_count: (list of int) If set, will increment commit_count[0] on
+ each ``commit``.
+ """
+ self._all_batch_items = all_batch_items
+ self._commit_count = commit_count
+ self.mutations = []
+
+ def put(self, _entity):
+ assert isinstance(_entity, entity.Entity)
+ self.mutations.append(FakeMutation(entity=_entity))
+ if self._all_batch_items is not None:
+ self._all_batch_items.append(_entity)
+
+ def delete(self, _key):
+ assert isinstance(_key, key.Key)
+ self.mutations.append(FakeMutation(key=_key))
+ if self._all_batch_items is not None:
+ self._all_batch_items.append(_key)
+
+ def begin(self):
+ pass
+
+ def commit(self):
+ if self._commit_count:
+ self._commit_count[0] += 1
+
+
[email protected](client is None, 'Datastore dependencies are not installed')
+class DatastoreioTest(unittest.TestCase):
Review comment:
I did move `DynamicWriteBatcher` and `DynamicWriteBatcherTest` to the shared
v1/util.py, v1/util_test.py files.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 231842)
Time Spent: 3h (was: 2h 50m)
> Remove dependency on googledatastore in favor of google-cloud-datastore.
> ------------------------------------------------------------------------
>
> Key: BEAM-4543
> URL: https://issues.apache.org/jira/browse/BEAM-4543
> Project: Beam
> Issue Type: Improvement
> Components: sdk-py-core
> Reporter: Valentyn Tymofieiev
> Assignee: Udi Meiri
> Priority: Minor
> Labels: triaged
> Time Spent: 3h
> Remaining Estimate: 0h
>
> apache-beam[gcp] package depends [1] on googledatastore package [2]. We
> should replace this dependency with google-cloud-datastore [3] which is
> officially supported, has better release cadence and also has Python 3
> support.
> [1]
> https://github.com/apache/beam/blob/fad655462f8fadfdfaab0b7a09cab538f076f94e/sdks/python/setup.py#L126
> [2] [https://pypi.org/project/googledatastore/]
> [3] [https://pypi.org/project/google-cloud-datastore/]
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
