http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java index 911f35f..3ec542e 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java @@ -32,6 +32,9 @@ import org.apache.hadoop.fs.Path; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMField; +import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMFormat; +import org.apache.sysml.runtime.io.FileFormatPropertiesMM.MMSymmetry; import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -48,24 +51,6 @@ public class MatrixMarketFormatTest extends AutomatedTestBase private final static int dim = 1200; private final static double sparsity = 0.1; - private enum MMFormat { - COORDINATE, - ARRAY, - } - - private enum MMField { - REAL, - INTEGER, - COMPLEX, - PATTERN, - } - - private enum MMSymmetry { - GENERAL, - SYMMETRIC, - SKEW_SYMMETRIC, //- instead _ - } - @Override public void setUp() { TestUtils.clearAssertionInformation(); @@ -337,10 +322,8 @@ public class MatrixMarketFormatTest extends AutomatedTestBase MatrixBlock tmp = MatrixBlock.randOperations( rows, cols, sparsity, -10, 10, "uniform", 7); - String header = "%%MatrixMarket matrix " - + fmt.name().toLowerCase() + " " - + field.name().toLowerCase() + " " - + symmetry.name().toLowerCase().replace("_", "-") + "\n"; + String header = "%%MatrixMarket matrix " + fmt.toString() + " " + + field.toString() + " " + symmetry.toString() + "\n"; String meta = rows + " " + cols + ((fmt == MMFormat.COORDINATE) ? " " + tmp.getNonZeros() : "") + "\n";
http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java index e055b38..6dcb673 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java @@ -42,6 +42,7 @@ import org.apache.sysml.runtime.instructions.spark.functions.CopyFrameBlockPairF import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils; import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongFrameToLongWritableFrameFunction; import org.apache.sysml.runtime.instructions.spark.utils.FrameRDDConverterUtils.LongWritableFrameToLongFrameFunction; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.io.FrameWriter; @@ -51,7 +52,6 @@ import org.apache.sysml.runtime.io.MatrixReaderFactory; import org.apache.sysml.runtime.io.MatrixWriter; import org.apache.sysml.runtime.io.MatrixWriterFactory; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -441,7 +441,7 @@ public class FrameConverterTest extends AutomatedTestBase InputInfo iinfo = InputInfo.BinaryBlockInputInfo; JavaPairRDD<LongWritable, FrameBlock> rddIn = sc.hadoopFile(fnameIn, iinfo.inputFormatClass, LongWritable.class, FrameBlock.class); JavaPairRDD<Long, FrameBlock> rddIn2 = rddIn.mapToPair(new CopyFrameBlockPairFunction(false)); - CSVFileFormatProperties fprop = new CSVFileFormatProperties(); + FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV(); JavaRDD<String> rddOut = FrameRDDConverterUtils.binaryBlockToCsv(rddIn2, mc, fprop, true); rddOut.saveAsTextFile(fnameOut); break; http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java index 77ebf10..7f7e089 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java @@ -24,11 +24,11 @@ import java.io.IOException; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.io.FrameWriter; import org.apache.sysml.runtime.io.FrameWriterFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.util.MapReduceTool; @@ -179,7 +179,7 @@ public class FrameReadWriteTest extends AutomatedTestBase initFrameData(frame2, B, schema2); //Write frame data to disk - CSVFileFormatProperties fprop = new CSVFileFormatProperties(); + FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV(); fprop.setDelim(DELIMITER); fprop.setHeader(HEADER); @@ -218,7 +218,7 @@ public class FrameReadWriteTest extends AutomatedTestBase } } - void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, CSVFileFormatProperties fprop) + void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, FileFormatPropertiesCSV fprop) throws IOException { String fname1 = SCRIPT_DIR + TEST_DIR + "/frameData1"; http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java index 4fd825b..0f34622 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/CSVReadUnknownSizeTest.java @@ -24,8 +24,8 @@ import org.junit.Test; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; @@ -114,7 +114,7 @@ public class CSVReadUnknownSizeTest extends AutomatedTestBase { double[][] X = getRandomMatrix(rows, cols, -1, 1, 1.0d, 7); MatrixBlock mb = DataConverter.convertToMatrixBlock(X); MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, 1000, 1000); - CSVFileFormatProperties fprop = new CSVFileFormatProperties(); + FileFormatPropertiesCSV fprop = new FileFormatPropertiesCSV(); DataConverter.writeMatrixToHDFS(mb, input("X"), OutputInfo.CSVOutputInfo, mc, -1, fprop); mc.set(-1, -1, -1, -1); MapReduceTool.writeMetaDataFile(input("X.mtd"), ValueType.DOUBLE, mc, OutputInfo.CSVOutputInfo, fprop); http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java index 056e619..04684a5 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/FrameCSVReadWriteTest.java @@ -22,9 +22,9 @@ package org.apache.sysml.test.integration.functions.transform; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -96,7 +96,7 @@ public class FrameCSVReadWriteTest extends AutomatedTestBase //read input/output and compare FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(false, ",", false)); + new FileFormatPropertiesCSV(false, ",", false)); FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L); FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo); FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L); http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java index 059bdb8..86a0cfd 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java @@ -23,9 +23,9 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -96,7 +96,7 @@ public class TransformCSVFrameEncodeDecodeTest extends AutomatedTestBase //read input/output and compare FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(false, ",", false)); + new FileFormatPropertiesCSV(false, ",", false)); FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L); FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo); FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L); http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java index 535d88d..d63890f 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java @@ -22,10 +22,10 @@ package org.apache.sysml.test.integration.functions.transform; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderTextCSV; import org.apache.sysml.runtime.io.FrameReaderTextCSVParallel; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.test.integration.AutomatedTestBase; @@ -142,8 +142,8 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase //read input/output and compare FrameReader reader2 = parRead ? - new FrameReaderTextCSVParallel( new CSVFileFormatProperties() ) : - new FrameReaderTextCSV( new CSVFileFormatProperties() ); + new FrameReaderTextCSVParallel( new FileFormatPropertiesCSV() ) : + new FrameReaderTextCSV( new FileFormatPropertiesCSV() ); FrameBlock fb2 = reader2.readFrameFromHDFS(output("R"), -1L, -1L); System.out.println(DataConverter.toString(fb2)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java index 790f661..45605df 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeColmapTest.java @@ -21,9 +21,9 @@ package org.apache.sysml.test.integration.functions.transform; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -125,7 +125,7 @@ public class TransformFrameEncodeColmapTest extends AutomatedTestBase //read input/output and compare FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(true, ",", false)); + new FileFormatPropertiesCSV(true, ",", false)); FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L); FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo); FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L); http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java index 3af3149..302ac2e 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java @@ -21,9 +21,9 @@ package org.apache.sysml.test.integration.functions.transform; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -172,7 +172,7 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase //read input/output and compare FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(true, ",", false)); + new FileFormatPropertiesCSV(true, ",", false)); FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L); FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo); FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L); http://git-wip-us.apache.org/repos/asf/systemml/blob/11d11987/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java index f9dbe06..8d01d60 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java @@ -23,9 +23,9 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.io.FileFormatPropertiesCSV; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -105,10 +105,10 @@ public class TransformFrameEncodeDecodeTokenTest extends AutomatedTestBase //read input/output and compare FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(false, " ", false)); + new FileFormatPropertiesCSV(false, " ", false)); FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET1, -1L, -1L); FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, - new CSVFileFormatProperties(false, " ", false)); + new FileFormatPropertiesCSV(false, " ", false)); FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L); String[][] R1 = DataConverter.convertToStringFrame(fb1); String[][] R2 = DataConverter.convertToStringFrame(fb2);
