Repository: incubator-systemml Updated Branches: refs/heads/master b2be71738 -> 3353bf9ec
[SYSTEMML-1240] Cleanup crc files when write scalar Cleanup crc files when write scalar to local file system. Closes #386. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/3353bf9e Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/3353bf9e Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/3353bf9e Branch: refs/heads/master Commit: 3353bf9ec5419f0b65154975e4e537e44a1fb56f Parents: b2be717 Author: Deron Eriksson <[email protected]> Authored: Tue Feb 14 14:36:05 2017 -0800 Committer: Deron Eriksson <[email protected]> Committed: Tue Feb 14 14:36:05 2017 -0800 ---------------------------------------------------------------------- .../instructions/cp/VariableCPInstruction.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3353bf9e/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index 78fe330..907c102 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -22,7 +22,12 @@ package org.apache.sysml.runtime.instructions.cp; import java.io.IOException; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.JobConf; import org.apache.sysml.api.DMLScript; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.UnaryCP; import org.apache.sysml.parser.Expression.DataType; @@ -37,6 +42,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysml.runtime.instructions.Instruction; import org.apache.sysml.runtime.instructions.InstructionUtils; +import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.io.WriterMatrixMarket; import org.apache.sysml.runtime.io.WriterTextCSV; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; @@ -892,6 +898,14 @@ public class VariableCPInstruction extends CPInstruction ScalarObject scalar = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); MapReduceTool.writeObjectToHDFS(scalar.getValue(), fname); MapReduceTool.writeScalarMetaDataFile(fname +".mtd", input1.getValueType()); + + JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); + FileSystem fs = FileSystem.get(job); + if (fs instanceof LocalFileSystem) { + Path path = new Path(fname); + IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path); + } + } catch ( IOException e ) { throw new DMLRuntimeException(e); }
