johnyangk commented on a change in pull request #122: [NEMO-213] Use Beam's
DoFnRunners to execute DoFn
URL: https://github.com/apache/incubator-nemo/pull/122#discussion_r223655975
##########
File path:
compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
##########
@@ -117,34 +130,92 @@ private static void boundedReadTranslator(final
TranslationContext ctx,
private static void parDoSingleOutputTranslator(final TranslationContext ctx,
final
PrimitiveTransformVertex transformVertex,
final ParDo.SingleOutput<?,
?> transform) {
- final DoTransform doTransform = new DoTransform(transform.getFn(),
ctx.pipelineOptions);
- final IRVertex vertex = new OperatorVertex(doTransform);
- ctx.addVertex(vertex);
- transformVertex.getNode().getInputs().values().stream()
+ try {
+ final AppliedPTransform pTransform =
transformVertex.getNode().toAppliedPTransform(PIPELINE.get());
+ final DoFn doFn = (DoFn) ParDoTranslation.getDoFn(pTransform);
+ final TupleTag mainOutputTag = (TupleTag)
ParDoTranslation.getMainOutputTag(pTransform);
+ final List<PCollectionView<?>> sideInputs =
ParDoTranslation.getSideInputs(pTransform);
+ final TupleTagList additionalOutputTags =
ParDoTranslation.getAdditionalOutputTags(pTransform);
+
+ final PCollection<?> mainInput = (PCollection<?>)
+
Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform));
+
+ final SimpleDoFnTransform doFnTransform =
+ new SimpleDoFnTransform(
+ doFn,
+ mainInput.getCoder(),
+ getOutputCoders(pTransform),
+ mainOutputTag,
+ additionalOutputTags.getAll(),
+ mainInput.getWindowingStrategy(),
+ sideInputs,
+ ctx.pipelineOptions);
+
+ //final DoTransform doTransform = new DoTransform(transform.getFn(),
ctx.pipelineOptions);
Review comment:
Remove unused code
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
