Repository: systemml Updated Branches: refs/heads/master 1c0efe31a -> c6a8715ee
[SYSTEMML-2204] Fix deregistration of parallelized RDDs on cleanup This patch fixes the meta data handling of parallelized RDDs to allow for proper deregistration during cleanup (e.g., on rmvar). With this patch, we avoid unnecessarily conservative guarded parallelize by correctly reflecting the currently consumed memory of live RDDs in the driver. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/81e17152 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/81e17152 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/81e17152 Branch: refs/heads/master Commit: 81e17152c66c1ab465c87e78d42b9a8c64734051 Parents: 1c0efe3 Author: Matthias Boehm <[email protected]> Authored: Thu Mar 22 14:47:19 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Thu Mar 22 14:47:19 2018 -0700 ---------------------------------------------------------------------- .../sysml/runtime/controlprogram/context/SparkExecutionContext.java | 1 + 1 file changed, 1 insertion(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/81e17152/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index 86c3adc..179c24a 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -380,6 +380,7 @@ public class SparkExecutionContext extends ExecutionContext //keep rdd handle for future operations on it RDDObject rddhandle = new RDDObject(rdd); rddhandle.setHDFSFile(fromFile); + rddhandle.setParallelizedRDD(!fromFile); mo.setRDDHandle(rddhandle); } //CASE 3: non-dirty (file exists on HDFS)
