[SYSTEMML-1388] Fix load imbalance parfor spark w/ factoring Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/600e641b Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/600e641b Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/600e641b
Branch: refs/heads/master Commit: 600e641b2c5e39fca1e7cde197f1f1e26ce112b6 Parents: b945543 Author: Matthias Boehm <[email protected]> Authored: Thu Mar 9 18:51:21 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Thu Mar 9 18:51:21 2017 -0800 ---------------------------------------------------------------------- .../runtime/controlprogram/parfor/RemoteParForSpark.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/600e641b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java index 5e69b19..49ac9db 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSpark.java @@ -71,10 +71,10 @@ public class RemoteParForSpark //run remote_spark parfor job //(w/o lazy evaluation to fit existing parfor framework, e.g., result merge) RemoteParForSparkWorker func = new RemoteParForSparkWorker(program, clsMap, cpCaching, aTasks, aIters); - List<Tuple2<Long,String>> out = - sc.parallelize( tasks, numMappers ) //create rdd of parfor tasks - .flatMapToPair( func ) //execute parfor tasks - .collect(); //get output handles + List<Tuple2<Long,String>> out = sc + .parallelize(tasks, tasks.size()) //create rdd of parfor tasks + .flatMapToPair(func) //execute parfor tasks + .collect(); //get output handles //de-serialize results LocalVariableMap[] results = RemoteParForUtils.getResults(out, LOG);
