Repository: incubator-systemml Updated Branches: refs/heads/master 5baebe4f7 -> f4d99cf01
Provide error message when the transform and output path overlaps. We will throw error message for following invocation: hadoop jar SystemML.jar -f algorithms/transform.dml -nvargs TRANSFORM_PATH=foo/output OUTPUT_DATA_PATH=foo/output.mtx ... However, following invocation is valid hadoop jar SystemML.jar -f algorithms/transform.dml -nvargs TRANSFORM_PATH=foo/maps OUTPUT_DATA_PATH=foo/data/output.mtx ... Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f4d99cf0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f4d99cf0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f4d99cf0 Branch: refs/heads/master Commit: f4d99cf0108ab03a73d5cdfdc36951f2f762e53e Parents: 5baebe4 Author: Niketan Pansare <npan...@us.ibm.com> Authored: Wed Feb 17 18:00:12 2016 -0800 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Wed Feb 17 18:00:12 2016 -0800 ---------------------------------------------------------------------- .../sysml/runtime/transform/DataTransform.java | 25 +++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f4d99cf0/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java index 1b85a3f..ea2f264 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java +++ b/src/main/java/org/apache/sysml/runtime/transform/DataTransform.java @@ -795,7 +795,7 @@ public class DataTransform { // Parse transform instruction (the first instruction) to obtain relevant fields TransformOperands oprnds = new TransformOperands(insts[0], inputMatrices[0]); - JobConf job = new JobConf(DataTransform.class); // ConfigurationManager.getCachedJobConf()); + JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); FileSystem fs = FileSystem.get(job); // find the first file in alphabetical ordering of partfiles in directory inputPath @@ -826,6 +826,8 @@ public class DataTransform { boolean isBB = (bboutputs.size() > 0); String tmpPath = MRJobConfiguration.constructTempOutputFilename(); + checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs); + JobReturn retCSV = null, retBB = null; if (!oprnds.isApply) { @@ -1097,6 +1099,8 @@ public class DataTransform { boolean isCSV = (csvoutputs.size() > 0); boolean isBB = (bboutputs.size() > 0); + checkIfOutputOverlapsWithTxMtd(outputMatrices, oprnds, isCSV, isBB, csvoutputs, bboutputs); + JobReturn ret = null; if (!oprnds.isApply) { @@ -1397,6 +1401,23 @@ public class DataTransform { br.close(); } + private static void checkIfOutputOverlapsWithTxMtd(MatrixObject[] outputMatrices, TransformOperands oprnds, + boolean isCSV, boolean isBB, ArrayList<Integer> csvoutputs, ArrayList<Integer> bboutputs) throws DMLRuntimeException { + if(isCSV) { + checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[csvoutputs.get(0)].getFileName()); + } + else if(isBB) { + checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[bboutputs.get(0)].getFileName()); + } + } + + private static void checkIfOutputOverlapsWithTxMtd(String txMtdPath, String outputPath) throws DMLRuntimeException { + if(txMtdPath.startsWith(outputPath) || outputPath.startsWith(txMtdPath)) { + throw new DMLRuntimeException("The transform path \'" + txMtdPath + + "\' cannot overlap with the output path \'" + outputPath + "\'"); + } + } + public static void spDataTransform(ParameterizedBuiltinSPInstruction inst, MatrixObject[] inputMatrices, MatrixObject[] outputMatrices, ExecutionContext ec) throws Exception { SparkExecutionContext sec = (SparkExecutionContext)ec; @@ -1404,6 +1425,8 @@ public class DataTransform { // Parse transform instruction (the first instruction) to obtain relevant fields TransformOperands oprnds = new TransformOperands(inst, inputMatrices[0]); + checkIfOutputOverlapsWithTxMtd(oprnds.txMtdPath, outputMatrices[0].getFileName()); + JobConf job = new JobConf(); FileSystem fs = FileSystem.get(job);
