This is an automated email from the ASF dual-hosted git repository. echauchot pushed a commit to branch spark-runner_structured-streaming in repository https://gitbox.apache.org/repos/asf/beam.git
commit b37da3e80a8f678463fb5e3cf2c991ff93de5b23 Author: Etienne Chauchot <[email protected]> AuthorDate: Mon Nov 26 11:31:36 2018 +0100 Add precise TODO for multiple TransformTranslator per transform URN --- .../translation/batch/PipelineTranslatorBatch.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java index e883131..318d74c 100644 --- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java +++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java @@ -42,6 +42,12 @@ public class PipelineTranslatorBatch extends PipelineTranslator { @SuppressWarnings("rawtypes") private static final Map<String, TransformTranslator> TRANSFORM_TRANSLATORS = new HashMap<>(); + //TODO the ability to have more than one TransformTranslator per URN + // that could be dynamically chosen by a predicated that evaluates based on PCollection + // obtainable though node.getInputs.getValue() + // See https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L83 + // And https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L106 + static { TRANSFORM_TRANSLATORS.put( PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch());
