[HOTFIX] Fix csv frame readers (robust size computation), tests Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/10007504 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/10007504 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/10007504
Branch: refs/heads/master Commit: 1000750464d470a7b56db71d896c6ca25e11e0bd Parents: b3ba991 Author: Matthias Boehm <[email protected]> Authored: Sun Feb 12 20:15:00 2017 +0100 Committer: Matthias Boehm <[email protected]> Committed: Sun Feb 12 20:15:00 2017 +0100 ---------------------------------------------------------------------- .../sysml/runtime/io/IOUtilFunctions.java | 10 ++-- .../TransformCSVFrameEncodeReadTest.java | 48 ++++++++++++++++---- .../transform/TransformCSVFrameEncodeRead.dml | 3 +- 3 files changed, 48 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 3f0ea56..9d3b0ab 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -382,19 +382,21 @@ public class IOUtilFunctions { LongWritable key = new LongWritable(); Text value = new Text(); - int ncol = -1; + int ncol = -1; for( int i=0; i<splits.length && ncol<=0; i++ ) { RecordReader<LongWritable, Text> reader = informat.getRecordReader(splits[i], job, Reporter.NULL); try { if( reader.next(key, value) ) { + boolean hasValue = true; if( value.toString().startsWith(TfUtils.TXMTD_MVPREFIX) ) - reader.next(key, value); + hasValue = reader.next(key, value); if( value.toString().startsWith(TfUtils.TXMTD_NDPREFIX) ) - reader.next(key, value); + hasValue = reader.next(key, value); String row = value.toString().trim(); - if( !row.isEmpty() ) + if( hasValue && !row.isEmpty() ) { ncol = IOUtilFunctions.countTokensCSV(row, delim); + } } } finally { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java index b28c2df..b35f2ac 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java @@ -51,33 +51,64 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase @Test public void testFrameReadMetaSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false); + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, false); } @Test public void testFrameReadMetaSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false); + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, false); } @Test public void testFrameReadMetaHybridCSV() { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false); + runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, false); } @Test public void testFrameParReadMetaSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true); + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", false, true); } @Test public void testFrameParReadMetaSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true); + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", false, true); } @Test public void testFrameParReadMetaHybridCSV() { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true); + runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", false, true); } + + @Test + public void testFrameReadSubMetaSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, false); + } + + @Test + public void testFrameReadSubMetaSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, false); + } + + @Test + public void testFrameReadSubMetaHybridCSV() { + runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, false); + } + + @Test + public void testFrameParReadSubMetaSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", true, true); + } + + @Test + public void testFrameParReadSubMetaSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", true, true); + } + + @Test + public void testFrameParReadSubMetaHybridCSV() { + runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", true, true); + } + /** * @@ -85,7 +116,7 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase * @param ofmt * @param dataset */ - private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean parRead ) + private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, boolean subset, boolean parRead ) { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; @@ -104,9 +135,10 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase getAndLoadTestConfiguration(TEST_NAME1); String HOME = SCRIPT_DIR + TEST_DIR; + int nrows = subset ? 4 : 13; fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; programArgs = new String[]{"-explain", "-stats","-args", - HOME + "input/" + DATASET, output("R") }; + HOME + "input/" + DATASET, String.valueOf(nrows), output("R") }; OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; runTest(true, false, null, -1); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/10007504/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml index 9da935f..d75cf8a 100644 --- a/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml +++ b/src/test/scripts/functions/transform/TransformCSVFrameEncodeRead.dml @@ -24,6 +24,7 @@ jspec = "{\"ids\": true, \"recode\": [1,2,3]}"; [X, M] = transformencode(target=F1, spec=jspec); +M = M[1:$2,] print(toString(M)) -write(M, $2, format="csv"); +write(M, $3, format="csv");
