Repository: systemml Updated Branches: refs/heads/master 1bcdfaac1 -> add561b38
[SYSTEMML-2361] Fix invalid cleanup of matrices/frames read in functions This patch fixes a severe bug that led to invalid cleanup of matrices or frames that are read from persistent storage (e.g., HDFS) in functions. Although there are never rmvar instructions generated for pread variables, the function call cleans up unexpected function outputs to safe guard against special case memory leaks. We now globally disable the cleanup of all persistently read matrices or frames. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/add561b3 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/add561b3 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/add561b3 Branch: refs/heads/master Commit: add561b38e55e16dca7fd0d8a520b5abca5671a3 Parents: 1bcdfaa Author: Matthias Boehm <[email protected]> Authored: Mon Jun 4 19:44:32 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Jun 4 19:44:32 2018 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/lops/Data.java | 6 +++--- .../sysml/runtime/instructions/cp/VariableCPInstruction.java | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/lops/Data.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/Data.java b/src/main/java/org/apache/sysml/lops/Data.java index 53f169b..95df961 100644 --- a/src/main/java/org/apache/sysml/lops/Data.java +++ b/src/main/java/org/apache/sysml/lops/Data.java @@ -37,11 +37,11 @@ import org.apache.sysml.parser.Expression.ValueType; * variables, literals. Can be for both input and output. */ -public class Data extends Lop +public class Data extends Lop { - public enum OperationTypes {READ,WRITE} - + public static final String PREAD_PREFIX = "p"+OperationTypes.READ.name(); + FileFormatTypes formatType = FileFormatTypes.BINARY; OperationTypes operation; boolean literal_var = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/add561b3/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index b46f4df..26fcb2e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -485,6 +485,8 @@ public class VariableCPInstruction extends CPInstruction { mobj.setMetaData((MetaData)metadata.clone()); mobj.setFileFormatProperties(_formatProperties); mobj.setUpdateType(_updateType); + mobj.enableCleanup(!getInput1().getName() + .startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX)); ec.setVariable(getInput1().getName(), mobj); if(DMLScript.STATISTICS && _updateType.isInPlace()) Statistics.incrementTotalUIPVar(); @@ -496,6 +498,8 @@ public class VariableCPInstruction extends CPInstruction { fobj.setFileFormatProperties(_formatProperties); if( _schema != null ) fobj.setSchema(_schema); //after metadata + fobj.enableCleanup(!getInput1().getName() + .startsWith(org.apache.sysml.lops.Data.PREAD_PREFIX)); ec.setVariable(getInput1().getName(), fobj); } else if ( getInput1().getDataType() == DataType.SCALAR ){
