benWize commented on a change in pull request #15328:
URL: https://github.com/apache/beam/pull/15328#discussion_r691436271
##########
File path: sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio_test.py
##########
@@ -177,6 +180,50 @@ def test_write_mutations_non_retryable_error(self):
mock_throttler, rpc_stats_callback, throttle_delay=0)
rpc_stats_callback.assert_called_once_with(errors=1)
+ def test_write_mutations_metric_on_failure(self):
Review comment:
Done!
##########
File path: sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py
##########
@@ -26,20 +26,27 @@
import logging
import time
+from google.api_core.exceptions import ClientError
Review comment:
Done, but I'm not sure what to do in the except
https://github.com/apache/beam/blob/bd4f5c773b03d5bb8b5837335cf590366f2b1b36/sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py#L52
##########
File path: sdks/python/apache_beam/io/gcp/datastore/v1new/datastoreio.py
##########
@@ -265,10 +272,33 @@ def get_estimated_num_splits(client, query):
class _QueryFn(DoFn):
"""A DoFn that fetches entities from Cloud Datastore, for a given query."""
def process(self, query, *unused_args, **unused_kwargs):
+ if query.namespace is None:
+ query.namespace = ''
_client = helper.get_client(query.project, query.namespace)
client_query = query._to_client_query(_client)
- for client_entity in client_query.fetch(query.limit):
- yield types.Entity.from_client_entity(client_entity)
+ # Create request count metric
+ resource = resource_identifiers.DatastoreNamespace(
+ query.project, query.namespace)
+ labels = {
+ monitoring_infos.SERVICE_LABEL: 'Datastore',
+ monitoring_infos.METHOD_LABEL: 'BatchDatastoreRead',
+ monitoring_infos.RESOURCE_LABEL: resource,
+ monitoring_infos.DATASTORE_NAMESPACE_LABEL: query.namespace,
+ monitoring_infos.DATASTORE_PROJECT_ID_LABEL: query.project,
+ monitoring_infos.STATUS_LABEL: 'ok'
+ }
+ service_call_metric = ServiceCallMetric(
+ request_count_urn=monitoring_infos.API_REQUEST_COUNT_URN,
+ base_labels=labels)
+ try:
+ for client_entity in client_query.fetch(query.limit):
+ yield types.Entity.from_client_entity(client_entity)
+ service_call_metric.call('ok')
+ except (ClientError, GoogleAPICallError) as e:
Review comment:
Yes, I've forced an error sending a negative limit in the integration
test, and it throws an InvalidArgument exception which is an instance of
ClientError as shown here in the debug console.
<img width="1283" alt="Screen Shot 2021-08-18 at 11 46 36"
src="https://user-images.githubusercontent.com/74670721/129940700-77d1622d-b533-4816-ad36-04fa5e356df3.png";>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
![https://user-images.githubusercontent.com/74670721/129940700-77d1622d-b533-4816-ad36-04fa5e356df3.png"](https://user-images.githubusercontent.com/74670721/129940700-77d1622d-b533-4816-ad36-04fa5e356df3.png"){kind=link}