robertwb commented on a change in pull request #11185: [BEAM-8019] Some
generalizations to support cross-language transforms.
URL: https://github.com/apache/beam/pull/11185#discussion_r397542649
##########
File path: sdks/python/apache_beam/pvalue.py
##########
@@ -181,11 +185,32 @@ def from_(pcoll):
"""
return PCollection(pcoll.pipeline, is_bounded=pcoll.is_bounded)
+ def _recursively_add_external_coders(
+ self, new_context, old_context, coder_proto):
+ for component_coder_id in coder_proto.component_coder_ids:
+ component_coder_proto = (
+ old_context.coders.get_id_to_proto_map()[component_coder_id])
+ new_context.coders.get_id_to_proto_map()[component_coder_id] = (
+ component_coder_proto)
+ self._recursively_add_external_coders(
+ new_context, old_context, component_coder_proto)
+
def to_runner_api(self, context):
# type: (PipelineContext) -> beam_runner_api_pb2.PCollection
+
+ if isinstance(self.element_type, ElementTypeHolder):
+ # This is potentially an coder in an external SDK that cannot be directly
+ # parsed in Python SDK.
+ coder_proto = self.element_type.coder
+ coder_id = context.coders.get_by_proto(coder_proto, deduplicate=True)
+ self._recursively_add_external_coders(
+ context, self.element_type.context, coder_proto)
+ else:
+ coder_id = context.coder_id_from_element_type(self.element_type)
Review comment:
Maybe this'd require populating the map above with actual Coders as well as
their protos.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
