jrmccluskey commented on code in PR #37639:
URL: https://github.com/apache/beam/pull/37639#discussion_r3319554540
##########
sdks/python/apache_beam/runners/dataflow/internal/apiclient.py:
##########
@@ -1130,38 +1114,32 @@ class
DataflowJobAlreadyExistsError(retry.PermanentException):
pass
-def to_split_int(n):
- res = dataflow.SplitInt64()
- res.lowBits = n & 0xffffffff
- res.highBits = n >> 32
- return res
-
-
# TODO: Used in legacy batch worker. Move under MetricUpdateTranslators
# after Runner V2 transition.
-def translate_distribution(distribution_update, metric_update_proto):
+def translate_distribution(
+ distribution_update, metric_update_proto: dataflow.MetricUpdate):
"""Translate metrics DistributionUpdate to dataflow distribution update.
Args:
distribution_update: Instance of DistributionData,
DistributionInt64Accumulator or DataflowDistributionCounter.
metric_update_proto: Used for report metrics.
"""
- dist_update_proto = dataflow.DistributionUpdate()
- dist_update_proto.min = to_split_int(distribution_update.min)
- dist_update_proto.max = to_split_int(distribution_update.max)
- dist_update_proto.count = to_split_int(distribution_update.count)
- dist_update_proto.sum = to_split_int(distribution_update.sum)
+ dist_update_proto = Struct()
+ dist_update_proto.update({"min": distribution_update.min})
+ dist_update_proto.update({"max": distribution_update.max})
+ dist_update_proto.update({"count": distribution_update.count})
+ dist_update_proto.update({"sum": distribution_update.sum})
# DataflowDistributionCounter needs to translate histogram
- if isinstance(distribution_update, DataflowDistributionCounter):
- dist_update_proto.histogram = dataflow.Histogram()
- distribution_update.translate_to_histogram(dist_update_proto.histogram)
+ # if isinstance(distribution_update, DataflowDistributionCounter):
+ # dist_update_proto.histogram = dataflow.Histogram()
+ # distribution_update.translate_to_histogram(dist_update_proto.histogram)
Review Comment:
Re: to_split_int() there's a SplitInt64 proto
(https://docs.cloud.google.com/workflows/docs/reference/googleapis/dataflow/v1b3/Overview#splitint64)
that the GAPIC client does not have an equivalent for. Based on testing, using
ints outside of this format doesn't cause any issues anyway.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
