Repository: incubator-systemml Updated Branches: refs/heads/master 7d4acddc0 -> 293c81c6c
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java index 2ce82f3..3fad934 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameReadMetaTest.java @@ -108,10 +108,10 @@ public class FrameReadMetaTest extends AutomatedTestBase Connection conn = new Connection(); //read meta data frame - String spec = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR+"tfmtd_example/spec.json"); + String spec = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR + TEST_DIR+"tfmtd_example2/spec.json"); FrameBlock M = readFrame ? DataConverter.convertToFrameBlock(conn.readStringFrame(SCRIPT_DIR + TEST_DIR+"tfmtd_frame_example/tfmtd_frame")) : - conn.readTransformMetaDataFromFile(spec, SCRIPT_DIR + TEST_DIR+"tfmtd_example/"); + conn.readTransformMetaDataFromFile(spec, SCRIPT_DIR + TEST_DIR+"tfmtd_example2/"); try { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java new file mode 100644 index 0000000..6f1c74c --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.transform; + +import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.runtime.io.MatrixReader; +import org.apache.sysml.runtime.io.MatrixReaderFactory; +import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class TransformFrameApplyTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "Transform"; + private final static String TEST_NAME2 = "ApplyFrame"; + private final static String TEST_DIR = "functions/transform/"; + private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameApplyTest.class.getSimpleName() + "/"; + + //dataset and transform tasks without missing values + private final static String DATASET1 = "homes3/homes.csv"; + private final static String SPEC1 = "homes3/homes.tfspec_recode.json"; + private final static String SPEC2 = "homes3/homes.tfspec_dummy.json"; + private final static String SPEC3 = "homes3/homes.tfspec_bin.json"; //incl recode + + //dataset and transform tasks with missing values + private final static String DATASET2 = "homes/homes.csv"; + private final static String SPEC4 = "homes3/homes.tfspec_impute.json"; + private final static String SPEC5 = "homes3/homes.tfspec_omit.json"; + + public enum TransformType { + RECODE, + DUMMY, + BIN, + IMPUTE, + OMIT, + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); + } + + @Test + public void testHomesRecodeSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE); + } + + @Test + public void testHomesRecodeSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE); + } + + @Test + public void testHomesDummycodeSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY); + } + + @Test + public void testHomesDummycodeSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY); + } + + @Test + public void testHomesBinningSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN); + } + + @Test + public void testHomesBinningSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN); + } + + @Test + public void testHomesOmitSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT); + } + + @Test + public void testHomesOmitSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT); + } + + @Test + public void testHomesImputeSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE); + } + + @Test + public void testHomesImputeSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE); + } + + /** + * + * @param rt + * @param ofmt + * @param dataset + */ + private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type ) + { + //set runtime platform + RUNTIME_PLATFORM rtold = rtplatform; + boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; + rtplatform = rt; + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + //set transform specification + String SPEC = null; String DATASET = null; + switch( type ) { + case RECODE: SPEC = SPEC1; DATASET = DATASET1; break; + case DUMMY: SPEC = SPEC2; DATASET = DATASET1; break; + case BIN: SPEC = SPEC3; DATASET = DATASET1; break; + case IMPUTE: SPEC = SPEC4; DATASET = DATASET2; break; + case OMIT: SPEC = SPEC5; DATASET = DATASET2; break; + } + + if( !ofmt.equals("csv") ) + throw new RuntimeException("Unsupported test output format"); + + try + { + getAndLoadTestConfiguration(TEST_NAME1); + + /* This is for running the junit test the new way, i.e., construct the arguments directly */ + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; + programArgs = new String[]{"-nvargs", + "DATA=" + HOME + "input/" + DATASET, + "TFSPEC=" + HOME + "input/" + SPEC, + "TFMTD=" + output("tfmtd"), + "TFDATA=" + output("tfout"), + "OFMT=" + ofmt }; + + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; + runTest(true, false, null, -1); + + fullDMLScriptName = HOME + TEST_NAME2 + ".dml"; + programArgs = new String[]{"-explain","-nvargs", + "DATA=" + HOME + "input/" + DATASET, + "TFSPEC=" + HOME + "input/" + SPEC, + "APPLYMTD=" + output("tfmtd"), // generated above + "TFDATA=" + output("test_tfout"), + "OFMT=" + ofmt }; + + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; + runTest(true, false, null, -1); + + //read both outputs and compare + MatrixReader reader1 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); + MatrixBlock mb1 = reader1.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1); + MatrixReader reader2 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); + MatrixBlock mb2 = reader2.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1); + double[][] R1 = DataConverter.convertToDoubleMatrix(mb1); + double[][] R2 = DataConverter.convertToDoubleMatrix(mb2); + TestUtils.compareMatrices(R1, R2, R1.length, R1[0].length, 0); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = rtold; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java new file mode 100644 index 0000000..946bb3d --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.transform; + +import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.runtime.io.FrameReader; +import org.apache.sysml.runtime.io.FrameReaderFactory; +import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; +import org.apache.sysml.runtime.matrix.data.FrameBlock; +import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class TransformFrameEncodeDecodeTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "TransformFrameEncodeDecode"; + private final static String TEST_DIR = "functions/transform/"; + private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameEncodeDecodeTest.class.getSimpleName() + "/"; + + //dataset and transform tasks without missing values + private final static String DATASET1 = "homes3/homes.csv"; + private final static String SPEC1 = "homes3/homes.tfspec_recode.json"; + private final static String SPEC2 = "homes3/homes.tfspec_dummy.json"; + + public enum TransformType { + RECODE, + DUMMY, + BIN, + IMPUTE, + OMIT, + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); + } + + @Test + public void testHomesRecodeSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE); + } + + @Test + public void testHomesRecodeSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE); + } + + @Test + public void testHomesDummycodeSingleNodeCSV() { + runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY); + } + + @Test + public void testHomesDummycodeSparkCSV() { + runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY); + } + + /** + * + * @param rt + * @param ofmt + * @param dataset + */ + private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type ) + { + //set runtime platform + RUNTIME_PLATFORM rtold = rtplatform; + boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; + rtplatform = rt; + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + //set transform specification + String SPEC = null; String DATASET = null; + switch( type ) { + case RECODE: SPEC = SPEC1; DATASET = DATASET1; break; + case DUMMY: SPEC = SPEC2; DATASET = DATASET1; break; + default: throw new RuntimeException("Unsupported transform type for encode/decode test."); + } + + if( !ofmt.equals("csv") ) + throw new RuntimeException("Unsupported test output format"); + + try + { + getAndLoadTestConfiguration(TEST_NAME1); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; + programArgs = new String[]{"-explain","-nvargs", + "DATA=" + HOME + "input/" + DATASET, + "TFSPEC=" + HOME + "input/" + SPEC, + "TFDATA=" + output("tfout"), + "OFMT=" + ofmt }; + + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; + runTest(true, false, null, -1); + + //System.exit(1); + + //read input/output and compare + FrameReader reader1 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo, + new CSVFileFormatProperties(true, ",", false)); + FrameBlock fb1 = reader1.readFrameFromHDFS(HOME + "input/" + DATASET, -1L, -1L); + FrameReader reader2 = FrameReaderFactory.createFrameReader(InputInfo.CSVInputInfo); + FrameBlock fb2 = reader2.readFrameFromHDFS(output("tfout"), -1L, -1L); + String[][] R1 = DataConverter.convertToStringFrame(fb1); + String[][] R2 = DataConverter.convertToStringFrame(fb2); + TestUtils.compareFrames(R1, R2, R1.length, R1[0].length); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = rtold; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameTest.java deleted file mode 100644 index 6921b77..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameTest.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; -import org.apache.sysml.runtime.io.MatrixReader; -import org.apache.sysml.runtime.io.MatrixReaderFactory; -import org.apache.sysml.runtime.matrix.data.InputInfo; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.util.DataConverter; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -public class TransformFrameTest extends AutomatedTestBase -{ - private final static String TEST_NAME1 = "Transform"; - private final static String TEST_NAME2 = "ApplyFrame"; - private final static String TEST_DIR = "functions/transform/"; - private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameTest.class.getSimpleName() + "/"; - - //dataset and transform tasks without missing values - private final static String DATASET1 = "homes3/homes.csv"; - private final static String SPEC1 = "homes3/homes.tfspec_recode.json"; - private final static String SPEC2 = "homes3/homes.tfspec_dummy.json"; - private final static String SPEC3 = "homes3/homes.tfspec_bin.json"; //incl recode - - //dataset and transform tasks with missing values - private final static String DATASET2 = "homes/homes.csv"; - private final static String SPEC4 = "homes3/homes.tfspec_impute.json"; - private final static String SPEC5 = "homes3/homes.tfspec_omit.json"; - - public enum TransformType { - RECODE, - DUMMY, - BIN, - IMPUTE, - OMIT, - } - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); - } - - @Test - public void testHomesRecodeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE); - } - - @Test - public void testHomesRecodeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE); - } - - @Test - public void testHomesDummycodeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY); - } - - @Test - public void testHomesDummycodeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY); - } - - @Test - public void testHomesBinningSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN); - } - - @Test - public void testHomesBinningSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN); - } - - @Test - public void testHomesOmitSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT); - } - - @Test - public void testHomesOmitSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT); - } - - @Test - public void testHomesImputeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE); - } - - @Test - public void testHomesImputeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE); - } - - /** - * - * @param rt - * @param ofmt - * @param dataset - */ - private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type ) - { - //set runtime platform - RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - //set transform specification - String SPEC = null; String DATASET = null; - switch( type ) { - case RECODE: SPEC = SPEC1; DATASET = DATASET1; break; - case DUMMY: SPEC = SPEC2; DATASET = DATASET1; break; - case BIN: SPEC = SPEC3; DATASET = DATASET1; break; - case IMPUTE: SPEC = SPEC4; DATASET = DATASET2; break; - case OMIT: SPEC = SPEC5; DATASET = DATASET2; break; - } - - if( !ofmt.equals("csv") ) - throw new RuntimeException("Unsupported test output format"); - - try - { - getAndLoadTestConfiguration(TEST_NAME1); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "TFSPEC=" + HOME + "input/" + SPEC, - "TFMTD=" + output("tfmtd"), - "TFDATA=" + output("tfout"), - "OFMT=" + ofmt }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; - runTest(true, false, null, -1); - - fullDMLScriptName = HOME + TEST_NAME2 + ".dml"; - programArgs = new String[]{"-explain","-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "TFSPEC=" + HOME + "input/" + SPEC, - "APPLYMTD=" + output("tfmtd"), // generated above - "TFDATA=" + output("test_tfout"), - "OFMT=" + ofmt }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; - runTest(true, false, null, -1); - - //read both outputs and compare - MatrixReader reader1 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); - MatrixBlock mb1 = reader1.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1); - MatrixReader reader2 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); - MatrixBlock mb2 = reader2.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1); - double[][] R1 = DataConverter.convertToDoubleMatrix(mb1); - double[][] R2 = DataConverter.convertToDoubleMatrix(mb2); - TestUtils.compareMatrices(R1, R2, R1.length, R1[0].length, 0); - } - catch(Exception ex) { - throw new RuntimeException(ex); - } - finally { - rtplatform = rtold; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/java/org/apache/sysml/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java index 834064b..85557bf 100644 --- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java +++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java @@ -747,6 +747,27 @@ public class TestUtils assertTrue("" + countErrors + " values are not in equal", countErrors == 0); } + /** + * + * @param expectedMatrix + * @param actualMatrix + * @param rows + * @param cols + * @param epsilon + */ + public static void compareFrames(String[][] expectedMatrix, String[][] actualMatrix, int rows, int cols ) { + int countErrors = 0; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + if( !(expectedMatrix[i][j].equals(actualMatrix[i][j]) || (expectedMatrix[i][j]+".0").equals(actualMatrix[i][j])) ) { + System.out.println(expectedMatrix[i][j] +" vs actual: "+actualMatrix[i][j]+" at "+i+" "+j); + countErrors++; + } + } + } + assertTrue("" + countErrors + " values are not in equal", countErrors == 0); + } + public static void compareScalars(double d1, double d2, double tol) { if(!compareCellValue(d1, d2, tol, false)) { assertTrue("Given scalars do not match: " + d1 + " != " + d2 , false); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.map new file mode 100644 index 0000000..6f092f5 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.map @@ -0,0 +1,4 @@ +"east",1,39 +"north",2,46 +"south",3,63 +"west",4,52 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.mode new file mode 100644 index 0000000..a8f28f5 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.mode @@ -0,0 +1 @@ +"south",3,63 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.ndistinct new file mode 100644 index 0000000..bf0d87a --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/district.ndistinct @@ -0,0 +1 @@ +4 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.map new file mode 100644 index 0000000..4a2b4de --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.map @@ -0,0 +1,3 @@ +"1",1,78 +"2",2,66 +"3",3,56 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.mode new file mode 100644 index 0000000..c4c00f9 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.mode @@ -0,0 +1 @@ +"1",1,78 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.ndistinct new file mode 100644 index 0000000..e440e5c --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/floors.ndistinct @@ -0,0 +1 @@ +3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.map new file mode 100644 index 0000000..7e62337 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.map @@ -0,0 +1,5 @@ +"1",1,54 +"1.5",2,42 +"2",3,39 +"2.5",4,33 +"3",5,32 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.mode new file mode 100644 index 0000000..389c785 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.mode @@ -0,0 +1 @@ +"1",1,54 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.ndistinct new file mode 100644 index 0000000..7813681 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbathrooms.ndistinct @@ -0,0 +1 @@ +5 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.map new file mode 100644 index 0000000..7046c6e --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.map @@ -0,0 +1,7 @@ +"1",1,19 +"2",2,38 +"3",3,32 +"4",4,32 +"5",5,23 +"6",6,32 +"7",7,24 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.mode new file mode 100644 index 0000000..97c003b --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.mode @@ -0,0 +1 @@ +"2",2,38 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.ndistinct new file mode 100644 index 0000000..c793025 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/numbedrooms.ndistinct @@ -0,0 +1 @@ +7 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.map new file mode 100644 index 0000000..4d97f0e --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.map @@ -0,0 +1,2 @@ +"FALSE",1,94 +"TRUE",2,89 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.mode new file mode 100644 index 0000000..653d895 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.mode @@ -0,0 +1 @@ +"FALSE",1,94 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.ndistinct new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/view.ndistinct @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.map ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.map b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.map new file mode 100644 index 0000000..ac6e504 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.map @@ -0,0 +1,5 @@ +"91312",1,38 +"94555",2,39 +"95141",3,42 +"96334",4,37 +"98755",5,42 http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.mode ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.mode b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.mode new file mode 100644 index 0000000..096266b --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.mode @@ -0,0 +1 @@ +"95141",3,42 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.ndistinct ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.ndistinct b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.ndistinct new file mode 100644 index 0000000..7813681 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/Recode/zipcode.ndistinct @@ -0,0 +1 @@ +5 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/coltypes.csv ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/coltypes.csv b/src/test/scripts/functions/jmlc/tfmtd_example2/coltypes.csv new file mode 100644 index 0000000..0475aed --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/coltypes.csv @@ -0,0 +1 @@ +2,2,1,2,2,2,2,1,1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/column.names ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/column.names b/src/test/scripts/functions/jmlc/tfmtd_example2/column.names new file mode 100644 index 0000000..0600650 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/column.names @@ -0,0 +1 @@ +zipcode,district,sqft,numbedrooms,numbathrooms,floors,view,saleprice,askingprice http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/tfmtd_example2/spec.json ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/tfmtd_example2/spec.json b/src/test/scripts/functions/jmlc/tfmtd_example2/spec.json new file mode 100644 index 0000000..68187c4 --- /dev/null +++ b/src/test/scripts/functions/jmlc/tfmtd_example2/spec.json @@ -0,0 +1 @@ +{"recode":{"attributes":[1,2,4,5,6,7]}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/jmlc/transform5.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/transform5.dml b/src/test/scripts/functions/jmlc/transform5.dml index 569e715..a36f769 100644 --- a/src/test/scripts/functions/jmlc/transform5.dml +++ b/src/test/scripts/functions/jmlc/transform5.dml @@ -42,6 +42,7 @@ F2 = transformdecode(target=X1, meta=M1, spec=specJson1); F22 = transformdecode(target=X2, meta=M2, spec=specJson2); #frame leftindexing +F2 = append(F2, F2[,2]) F2[,3] = F22; write(F2, $F2); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test/scripts/functions/transform/TransformFrameEncodeDecode.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformFrameEncodeDecode.dml b/src/test/scripts/functions/transform/TransformFrameEncodeDecode.dml new file mode 100644 index 0000000..48c3929 --- /dev/null +++ b/src/test/scripts/functions/transform/TransformFrameEncodeDecode.dml @@ -0,0 +1,32 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +F1 = read($DATA, data_type="frame", format="csv"); +jspec = read($TFSPEC, data_type="scalar", value_type="string"); + +[X, M] = transformencode(target=F1, spec=jspec); + +if(1==1){} + +F2 = transformdecode(target=X, spec=jspec, meta=M); + +write(F2, $TFDATA, format=$OFMT); + http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/293c81c6/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java index dbf9302..5122a60 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java @@ -30,7 +30,8 @@ import org.junit.runners.Suite; ScalingTest.class, TransformAndApplyTest.class, TransformEncodeDecodeTest.class, - TransformFrameTest.class, + TransformFrameApplyTest.class, + TransformFrameEncodeDecodeTest.class, TransformReadMetaTest.class, TransformTest.class, })
