chamikaramj commented on code in PR #26100:
URL: https://github.com/apache/beam/pull/26100#discussion_r1163109261
##########
sdks/python/apache_beam/transforms/external.py:
##########
@@ -327,17 +327,61 @@ class SchemaAwareExternalTransform(ptransform.PTransform):
:param expansion_service: an expansion service to use. This should already be
available and the Schema-aware transforms to be used must already be
deployed.
+ :param rearrange_based_on_discovery: if this flag is set, the input kwargs
+ will be rearranged to match the order of fields in the external
+ SchemaTransform configuration. A discovery call will be made to fetch
+ the configuration.
:param classpath: (Optional) A list paths to additional jars to place on the
expansion service classpath.
:kwargs: field name to value mapping for configuring the schema transform.
keys map to the field names of the schema of the SchemaTransform
(in-order).
"""
- def __init__(self, identifier, expansion_service, classpath=None, **kwargs):
+ def __init__(
+ self,
+ identifier,
+ expansion_service,
+ rearrange_based_on_discovery=False,
+ classpath=None,
+ **kwargs):
self._expansion_service = expansion_service
- self._payload_builder = SchemaTransformPayloadBuilder(identifier, **kwargs)
+ self._kwargs = kwargs
self._classpath = classpath
+ _kwargs = kwargs
+ if rearrange_based_on_discovery:
+ _kwargs = self._rearrange_kwargs(identifier)
+
+ self._payload_builder = SchemaTransformPayloadBuilder(identifier,
**_kwargs)
+
+ def _rearrange_kwargs(self, identifier):
+ # discover and fetch the external SchemaTransform configuration then
+ # use it to build an appropriate payload
+ schematransform_config = SchemaAwareExternalTransform.discover_config(
+ self._expansion_service, identifier)
+
+ external_config_fields =
schematransform_config.configuration_schema._fields
+ ordered_kwargs = {}
Review Comment:
I think this has to be an OrderedDict. Otherwise the order is not guaranteed
by Python.
https://docs.python.org/3/library/collections.html#collections.OrderedDict
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
