[GitHub] sanha commented on a change in pull request #2: [NEMO-7] Intra-TaskGroup pipelining

[GitBox]

sanha commented on a change in pull request #2: [NEMO-7] Intra-TaskGroup 
pipelining
URL: https://github.com/apache/incubator-nemo/pull/2#discussion_r173048627
 
 

Advertising

 ##########
 File path: 
compiler/frontend/spark/src/main/java/edu/snu/nemo/compiler/frontend/spark/transform/ReduceTransform.java
 ##########
 @@ -15,41 +15,62 @@
  */
 package edu.snu.nemo.compiler.frontend.spark.transform;
 
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Output;
 import edu.snu.nemo.common.ir.OutputCollector;
 import edu.snu.nemo.common.ir.vertex.transform.Transform;
+import edu.snu.nemo.compiler.frontend.spark.core.java.JavaRDD;
 import org.apache.spark.api.java.function.Function2;
 
 import javax.annotation.Nullable;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.util.Iterator;
 
+
 /**
  * Reduce Transform for Spark.
+ *
  * @param <T> element type.
  */
 public final class ReduceTransform<T> implements Transform<T, T> {
   private final Function2<T, T, T> func;
-  private OutputCollector<T> oc;
+  private OutputCollector<T> outputCollector;
+  private T result;
+  private String filename;
 
   /**
    * Constructor.
    * @param func function to run for the reduce transform.
    */
   public ReduceTransform(final Function2<T, T, T> func) {
     this.func = func;
+    this.result = null;
+    this.filename = filename + JavaRDD.getResultId();
   }
 
   @Override
-  public void prepare(final Context context, final OutputCollector<T> 
outputCollector) {
-    this.oc = outputCollector;
+  public void prepare(final Context context, final OutputCollector<T> p) {
+    this.outputCollector = p;
   }
 
   @Override
-  public void onData(final Iterator<T> elements, final String srcVertexId) {
-    final T res = reduceIterator(elements, func);
-    if (res == null) { // nothing to be done.
+  public void onData(final Object element) {
+    if (element == null) { // nothing to be done.
       return;
     }
-    oc.emit(res);
+
+    if (result == null) {
+      result = (T) element;
 
 Review comment:
   Is this a right function? In my guess, the first element will be reduced 
twice.
   I guess that you omitted `return;` in here. 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services