Repository: systemml Updated Branches: refs/heads/master 8a2757341 -> 0cd3905f5
http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java deleted file mode 100644 index a80e8b3..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/ScalingTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import static org.junit.Assert.assertTrue; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.OutputStreamWriter; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.wink.json4j.JSONArray; -import org.apache.wink.json4j.JSONObject; -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.io.IOUtilFunctions; -import org.apache.sysml.runtime.io.ReaderBinaryBlock; -import org.apache.sysml.runtime.io.ReaderTextCSV; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.transform.TfUtils; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - * - */ -public class ScalingTest extends AutomatedTestBase -{ - - private final static String TEST_NAME = "Scaling"; - - private final static String TEST_DIR = "functions/transform/"; - private final static String TEST_CLASS_DIR = TEST_DIR + ScalingTest.class.getSimpleName() + "/"; - - private final static int rows1 = 1500; - private final static int cols1 = 16; - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[]{"R"})); - } - - // ---- Scaling CSV ---- - - @Test - public void testTransformScalingHybridCSV() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HYBRID, "csv"); - } - - @Test - public void testTransformScalingSPHybridCSV() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HYBRID_SPARK, "csv"); - } - - @Test - public void testTransformScalingHadoopCSV() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HADOOP, "csv"); - } - - @Test - public void testTransformScalingSparkCSV() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.SPARK, "csv"); - } - - // ---- Scaling BinaryBlock ---- - - @Test - public void testTransformScalingHybridBinary() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HYBRID, "binary"); - } - - @Test - public void testTransformScalingSPHybridBinary() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HYBRID_SPARK, "binary"); - } - - @Test - public void testTransformScalingHadoopBinary() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.HADOOP, "binary"); - } - - @Test - public void testTransformScalingSparkBinary() throws IOException, DMLRuntimeException, Exception - { - runScalingTest(rows1, cols1, RUNTIME_PLATFORM.SPARK, "binary"); - } - - // ---------------------------- - - private void generateSpecFile(int cols, String specFile) throws IOException , Exception - { - final String NAME = "name"; - final String METHOD = "method"; - final String SCALE_METHOD_Z = "z-score"; - final String SCALE_METHOD_M = "mean-subtraction"; - - JSONObject outputSpec = new JSONObject(); - JSONArray scaleSpec = new JSONArray(); - - for(int colID=1; colID <= cols; colID++) - { - JSONObject obj = new JSONObject(); - obj.put(NAME, "V"+colID); - if(colID <= cols/2) - obj.put(METHOD, SCALE_METHOD_M); - else - obj.put(METHOD, SCALE_METHOD_Z); - scaleSpec.add(obj); - } - outputSpec.put(TfUtils.TXMETHOD_SCALE, scaleSpec); - - FileSystem fs = IOUtilFunctions.getFileSystem(specFile); - try( BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(specFile),true))) ) { - out.write(outputSpec.toString()); - } - - } - - private void generateFrameMTD(String datafile) throws IllegalArgumentException, IOException , Exception - { - JSONObject mtd = new JSONObject(); - - mtd.put("data_type", "frame"); - mtd.put("format", "csv"); - mtd.put("header", false); - - FileSystem fs = IOUtilFunctions.getFileSystem(datafile); - try( BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(new Path(datafile+".mtd"),true))) ) { - out.write(mtd.toString()); - } - } - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - * @throws IOException - * @throws DMLRuntimeException - */ - private void runScalingTest( int rows, int cols, RUNTIME_PLATFORM rt, String ofmt) throws IOException, DMLRuntimeException, Exception - { - RUNTIME_PLATFORM platformOld = rtplatform; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - TestConfiguration config = getTestConfiguration(TEST_NAME); - loadTestConfiguration(config); - - String HOME = SCRIPT_DIR + TEST_DIR; - String specFile = input("spec.json"); - String inputFile = input("X"); - String outputFile = output(config.getOutputFiles()[0]); - String outputFileR = expected(config.getOutputFiles()[0]); - - generateSpecFile(cols, specFile); - - // This is for running the junit test the new way, i.e., construct the arguments directly - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{"-nvargs", - "DATA=" + inputFile, - "TFSPEC=" + specFile, - "TFMTD=" + output("tfmtd"), - "TFDATA=" + outputFile, - "OFMT=" + ofmt }; - - fullRScriptName = HOME + TEST_NAME + ".R"; - rCmd = "Rscript" + " " + fullRScriptName + " " + inputFile + " " + outputFileR; - - //generate actual dataset - double[][] X = getRandomMatrix(rows, cols, -50, 50, 1.0, 7); - TestUtils.writeCSVTestMatrix(inputFile, X); - generateFrameMTD(inputFile); - - runTest(true, false, null, -1); - runRScript(true); - - ReaderTextCSV expReader= new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null)); - MatrixBlock exp = expReader.readMatrixFromHDFS(outputFileR, -1, -1, -1, -1, -1); - MatrixBlock out = null; - - if ( ofmt.equals("csv") ) - { - ReaderTextCSV outReader= new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null)); - out = outReader.readMatrixFromHDFS(outputFile, -1, -1, -1, -1, -1); - } - else - { - ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false); - out = bbReader.readMatrixFromHDFS( - outputFile, exp.getNumRows(), exp.getNumColumns(), - ConfigurationManager.getBlocksize(), - ConfigurationManager.getBlocksize(), - -1); - } - - assertTrue("Incorrect output from data transform.", TransformTest.equals(out,exp, 1e-10)); - } - finally - { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformAndApplyTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformAndApplyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformAndApplyTest.java deleted file mode 100644 index 90c1e2f..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformAndApplyTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import java.io.IOException; -import java.util.HashMap; - -import org.junit.Assert; -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.io.MatrixWriter; -import org.apache.sysml.runtime.io.MatrixWriterFactory; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; -import org.apache.sysml.runtime.matrix.data.OutputInfo; -import org.apache.sysml.runtime.util.DataConverter; -import org.apache.sysml.runtime.util.MapReduceTool; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - * - */ -public class TransformAndApplyTest extends AutomatedTestBase -{ - private static final String TEST_NAME1 = "TransformAndApply"; - private static final String TEST_DIR = "functions/transform/"; - private static final String TEST_CLASS_DIR = TEST_DIR + TransformAndApplyTest.class.getSimpleName() + "/"; - - private static final String SPEC_X = "TransformAndApplySpecX.json"; - private static final String SPEC_Y = "TransformAndApplySpecY.json"; - - private static final int rows = 1234; - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"R1","R2"})); - } - - @Test - public void runTestCP() throws DMLRuntimeException, IOException { - runTransformAndApplyTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv"); - } - - @Test - public void runTestHadoop() throws DMLRuntimeException, IOException { - runTransformAndApplyTest(RUNTIME_PLATFORM.HADOOP, "csv"); - } - - @Test - public void runTestSpark() throws DMLRuntimeException, IOException { - runTransformAndApplyTest(RUNTIME_PLATFORM.SPARK, "csv"); - } - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - * @throws IOException - * @throws DMLRuntimeException - */ - private void runTransformAndApplyTest( RUNTIME_PLATFORM rt, String ofmt) throws IOException, DMLRuntimeException - { - RUNTIME_PLATFORM platformOld = rtplatform; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - getAndLoadTestConfiguration(TEST_NAME1); - - //generate input data - double[][] X = DataConverter.convertToDoubleMatrix( - MatrixBlock.seqOperations(0.5, rows/2, 0.5).appendOperations( - MatrixBlock.seqOperations(0.5, rows/2, 0.5), new MatrixBlock())); - double[][] Y = DataConverter.convertToDoubleMatrix( - MatrixBlock.seqOperations(rows/2, 0.5, -0.5)); - - //write inputs - MatrixBlock mbX = DataConverter.convertToMatrixBlock(X); - MatrixBlock mbY = DataConverter.convertToMatrixBlock(Y); - MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo); - writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1); - writer.writeMatrixToHDFS(mbY, input("Y"), rows, 1, -1, -1, -1); - - //read specs transform X and Y - String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR+TEST_DIR+SPEC_X); - String specY = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR+TEST_DIR+SPEC_Y); - - - fullDMLScriptName = SCRIPT_DIR+TEST_DIR + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-args", input("X"), input("Y"), specX, specY, - output("M1"), output("M2"), output("R1"), output("R2") }; - - //run test - runTest(true, false, null, -1); - - //compare matrices (values recoded to identical codes) - HashMap<CellIndex, Double> dml1 = readDMLMatrixFromHDFS("R1"); - HashMap<CellIndex, Double> dml2 = readDMLMatrixFromHDFS("R2"); - double[][] R1 = TestUtils.convertHashMapToDoubleArray(dml1); - double[][] R2 = TestUtils.convertHashMapToDoubleArray(dml2); - for( int i=0; i<rows; i++ ) { - Assert.assertEquals("Output values don't match: "+R1[i][0]+" vs "+R2[i][0], - new Double(R1[i][0]), new Double(R2[rows-i-1][0])); - } - } - catch(Exception ex) { - throw new IOException(ex); - } - finally - { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java index 9ee3d5f..059bdb8 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeDecodeTest.java @@ -23,7 +23,6 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; @@ -75,7 +74,6 @@ public class TransformCSVFrameEncodeDecodeTest extends AutomatedTestBase { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; rtplatform = rt; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -93,8 +91,7 @@ public class TransformCSVFrameEncodeDecodeTest extends AutomatedTestBase fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; programArgs = new String[]{"-explain","-args", HOME + "input/" + DATASET, output("R") }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; + runTest(true, false, null, -1); //read input/output and compare @@ -118,7 +115,6 @@ public class TransformCSVFrameEncodeDecodeTest extends AutomatedTestBase finally { rtplatform = rtold; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java index b35f2ac..535d88d 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformCSVFrameEncodeReadTest.java @@ -22,7 +22,6 @@ package org.apache.sysml.test.integration.functions.transform; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderTextCSV; import org.apache.sysml.runtime.io.FrameReaderTextCSVParallel; @@ -120,7 +119,6 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; rtplatform = rt; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -139,8 +137,7 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; programArgs = new String[]{"-explain", "-stats","-args", HOME + "input/" + DATASET, String.valueOf(nrows), output("R") }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; + runTest(true, false, null, -1); //read input/output and compare @@ -156,7 +153,6 @@ public class TransformCSVFrameEncodeReadTest extends AutomatedTestBase finally { rtplatform = rtold; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java deleted file mode 100644 index 6f1c74c..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameApplyTest.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; -import org.apache.sysml.runtime.io.MatrixReader; -import org.apache.sysml.runtime.io.MatrixReaderFactory; -import org.apache.sysml.runtime.matrix.data.InputInfo; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.util.DataConverter; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -public class TransformFrameApplyTest extends AutomatedTestBase -{ - private final static String TEST_NAME1 = "Transform"; - private final static String TEST_NAME2 = "ApplyFrame"; - private final static String TEST_DIR = "functions/transform/"; - private final static String TEST_CLASS_DIR = TEST_DIR + TransformFrameApplyTest.class.getSimpleName() + "/"; - - //dataset and transform tasks without missing values - private final static String DATASET1 = "homes3/homes.csv"; - private final static String SPEC1 = "homes3/homes.tfspec_recode.json"; - private final static String SPEC2 = "homes3/homes.tfspec_dummy.json"; - private final static String SPEC3 = "homes3/homes.tfspec_bin.json"; //incl recode - - //dataset and transform tasks with missing values - private final static String DATASET2 = "homes/homes.csv"; - private final static String SPEC4 = "homes3/homes.tfspec_impute.json"; - private final static String SPEC5 = "homes3/homes.tfspec_omit.json"; - - public enum TransformType { - RECODE, - DUMMY, - BIN, - IMPUTE, - OMIT, - } - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); - } - - @Test - public void testHomesRecodeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.RECODE); - } - - @Test - public void testHomesRecodeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.RECODE); - } - - @Test - public void testHomesDummycodeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.DUMMY); - } - - @Test - public void testHomesDummycodeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.DUMMY); - } - - @Test - public void testHomesBinningSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.BIN); - } - - @Test - public void testHomesBinningSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.BIN); - } - - @Test - public void testHomesOmitSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.OMIT); - } - - @Test - public void testHomesOmitSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.OMIT); - } - - @Test - public void testHomesImputeSingleNodeCSV() { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", TransformType.IMPUTE); - } - - @Test - public void testHomesImputeSparkCSV() { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", TransformType.IMPUTE); - } - - /** - * - * @param rt - * @param ofmt - * @param dataset - */ - private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, TransformType type ) - { - //set runtime platform - RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - //set transform specification - String SPEC = null; String DATASET = null; - switch( type ) { - case RECODE: SPEC = SPEC1; DATASET = DATASET1; break; - case DUMMY: SPEC = SPEC2; DATASET = DATASET1; break; - case BIN: SPEC = SPEC3; DATASET = DATASET1; break; - case IMPUTE: SPEC = SPEC4; DATASET = DATASET2; break; - case OMIT: SPEC = SPEC5; DATASET = DATASET2; break; - } - - if( !ofmt.equals("csv") ) - throw new RuntimeException("Unsupported test output format"); - - try - { - getAndLoadTestConfiguration(TEST_NAME1); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "TFSPEC=" + HOME + "input/" + SPEC, - "TFMTD=" + output("tfmtd"), - "TFDATA=" + output("tfout"), - "OFMT=" + ofmt }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; - runTest(true, false, null, -1); - - fullDMLScriptName = HOME + TEST_NAME2 + ".dml"; - programArgs = new String[]{"-explain","-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "TFSPEC=" + HOME + "input/" + SPEC, - "APPLYMTD=" + output("tfmtd"), // generated above - "TFDATA=" + output("test_tfout"), - "OFMT=" + ofmt }; - - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; - runTest(true, false, null, -1); - - //read both outputs and compare - MatrixReader reader1 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); - MatrixBlock mb1 = reader1.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1); - MatrixReader reader2 = MatrixReaderFactory.createMatrixReader(InputInfo.CSVInputInfo); - MatrixBlock mb2 = reader2.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1); - double[][] R1 = DataConverter.convertToDoubleMatrix(mb1); - double[][] R2 = DataConverter.convertToDoubleMatrix(mb2); - TestUtils.compareMatrices(R1, R2, R1.length, R1[0].length, 0); - } - catch(Exception ex) { - throw new RuntimeException(ex); - } - finally { - rtplatform = rtold; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java index 2d17c17..405661b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeApplyTest.java @@ -23,7 +23,6 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.io.MatrixReaderFactory; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -228,7 +227,6 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; rtplatform = rt; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -261,7 +259,6 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase "TFDATA2=" + output("tfout2"), "OFMT=" + ofmt }; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; runTest(true, false, null, -1); //read input/output and compare @@ -284,7 +281,6 @@ public class TransformFrameEncodeApplyTest extends AutomatedTestBase finally { rtplatform = rtold; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java index a879356..3af3149 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java @@ -21,7 +21,6 @@ package org.apache.sysml.test.integration.functions.transform; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; @@ -133,7 +132,6 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; rtplatform = rt; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -170,7 +168,6 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase // This is just a feature/bug and is reported in CLI-262, // though even a fix is unlikely to be backported to 1.2 - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; runTest(true, false, null, -1); //read input/output and compare @@ -194,7 +191,6 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase finally { rtplatform = rtold; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java index fa89e28..f9dbe06 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTokenTest.java @@ -23,7 +23,6 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; @@ -81,7 +80,6 @@ public class TransformFrameEncodeDecodeTokenTest extends AutomatedTestBase { //set runtime platform RUNTIME_PLATFORM rtold = rtplatform; - boolean csvReblockOld = OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK; rtplatform = rt; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -103,7 +101,6 @@ public class TransformFrameEncodeDecodeTokenTest extends AutomatedTestBase "TFDATA=" + output("tfout"), "SEP= ", "OFMT=" + ofmt, "OSEP= " }; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; runTest(true, false, null, -1); //read input/output and compare @@ -128,7 +125,6 @@ public class TransformFrameEncodeDecodeTokenTest extends AutomatedTestBase finally { rtplatform = rtold; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = csvReblockOld; } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java deleted file mode 100644 index 4e0240b..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import java.io.IOException; - -import org.junit.Assert; -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.parser.Expression.ValueType; -import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.io.FrameReader; -import org.apache.sysml.runtime.io.FrameReaderFactory; -import org.apache.sysml.runtime.io.MatrixWriter; -import org.apache.sysml.runtime.io.MatrixWriterFactory; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; -import org.apache.sysml.runtime.matrix.data.FrameBlock; -import org.apache.sysml.runtime.matrix.data.InputInfo; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.matrix.data.OutputInfo; -import org.apache.sysml.runtime.transform.meta.TfMetaUtils; -import org.apache.sysml.runtime.util.DataConverter; -import org.apache.sysml.runtime.util.MapReduceTool; -import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - * - */ -public class TransformReadMetaTest extends AutomatedTestBase -{ - private static final String TEST_NAME1 = "TransformReadMeta"; - private static final String TEST_NAME2 = "TransformReadMeta2"; - private static final String TEST_DIR = "functions/transform/"; - private static final String TEST_CLASS_DIR = TEST_DIR + TransformReadMetaTest.class.getSimpleName() + "/"; - private static final String SPEC_X = "TransformReadMetaSpecX.json"; - - private static final int rows = 1432; - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"M1, M"})); - addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2,new String[]{"M1, M"})); - } - - @Test - public void runTestCsvCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", ","); - } - - @Test - public void runTestCsvHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", ","); - } - - @Test - public void runTestCsvSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", ","); - } - - @Test - public void runTestCsvTabCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "\t"); - } - - @Test - public void runTestCsvTabHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", "\t"); - } - - @Test - public void runTestCsvTabSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", "\t"); - } - - @Test - public void runTestCsvColonCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", ":"); - } - - @Test - public void runTestCsvColonHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", ":"); - } - - @Test - public void runTestCsvColonSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", ":"); - } - - - @Test - public void runTestTextCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "text", ","); - } - - @Test - public void runTestTextHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "text", ","); - } - - @Test - public void runTestTextSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "text", ","); - } - - @Test - public void runTestBinaryCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", ","); - } - - @Test - public void runTestBinaryHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "binary", ","); - } - - @Test - public void runTestBinarySpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "binary", ","); - } - - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - * @throws IOException - * @throws DMLRuntimeException - */ - private void runTransformReadMetaTest( RUNTIME_PLATFORM rt, String ofmt, String delim) throws IOException, DMLRuntimeException - { - RUNTIME_PLATFORM platformOld = rtplatform; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - String testname = delim.equals(",") ? TEST_NAME1 : TEST_NAME2; - - getAndLoadTestConfiguration(testname); - - //generate input data - double[][] X = DataConverter.convertToDoubleMatrix( - MatrixBlock.seqOperations(0.5, rows/2, 0.5).appendOperations( - MatrixBlock.seqOperations(0.5, rows/2, 0.5), new MatrixBlock())); - MatrixBlock mbX = DataConverter.convertToMatrixBlock(X); - CSVFileFormatProperties fprops = new CSVFileFormatProperties(false, delim, false); - MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo, 1, fprops); - writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1); - - //read specs transform X and Y - String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR+TEST_DIR+SPEC_X); - - fullDMLScriptName = SCRIPT_DIR+TEST_DIR + testname + ".dml"; - programArgs = new String[]{"-args", input("X"), specX, output("M1"), output("M"), ofmt, delim}; - - //run test - runTest(true, false, null, -1); - - //compare meta data frames - InputInfo iinfo = InputInfo.stringExternalToInputInfo(ofmt); - FrameReader reader = FrameReaderFactory.createFrameReader(iinfo); - FrameBlock mExpected = TfMetaUtils.readTransformMetaDataFromFile(specX, output("M1"), delim); - FrameBlock mRet = reader.readFrameFromHDFS(output("M"), rows, 2); - for( int i=0; i<rows; i++ ) - for( int j=0; j<2; j++ ) { - Assert.assertTrue("Wrong result: "+mRet.get(i, j)+".", - UtilFunctions.compareTo(ValueType.STRING, mExpected.get(i, j), mRet.get(i, j))==0); - } - } - catch(Exception ex) { - throw new IOException(ex); - } - finally { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java deleted file mode 100644 index 3d799f0..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformTest.java +++ /dev/null @@ -1,709 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.transform; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.runtime.io.ReaderBinaryBlock; -import org.apache.sysml.runtime.io.ReaderTextCSV; -import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; -import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -public class TransformTest extends AutomatedTestBase -{ - - private final static String TEST_NAME1 = "Transform"; - private final static String TEST_NAME2 = "Apply"; - private final static String TEST_DIR = "functions/transform/"; - private final static String TEST_CLASS_DIR = TEST_DIR + TransformTest.class.getSimpleName() + "/"; - - private final static String HOMES_DATASET = "homes/homes.csv"; - private final static String HOMES_SPEC = "homes/homes.tfspec.json"; - private final static String HOMES_IDSPEC = "homes/homes.tfidspec.json"; - private final static String HOMES_TFDATA = "homes/homes.transformed.csv"; - - private final static String HOMES_OMIT_DATASET = "homes/homes.csv"; - private final static String HOMES_OMIT_SPEC = "homes/homesOmit.tfspec.json"; - private final static String HOMES_OMIT_IDSPEC = "homes/homesOmit.tfidspec.json"; - private final static String HOMES_OMIT_TFDATA = "homes/homesOmit.transformed.csv"; - - // Homes data set in two parts - private final static String HOMES2_DATASET = "homes2/homes.csv"; - private final static String HOMES2_SPEC = "homes2/homes.tfspec.json"; - private final static String HOMES2_IDSPEC = "homes2/homes.tfidspec.json"; - private final static String HOMES2_TFDATA = "homes/homes.transformed.csv"; // same as HOMES_TFDATA - - private final static String IRIS_DATASET = "iris/iris.csv"; - private final static String IRIS_SPEC = "iris/iris.tfspec.json"; - private final static String IRIS_IDSPEC = "iris/iris.tfidspec.json"; - private final static String IRIS_TFDATA = "iris/iris.transformed.csv"; - - @Override - public void setUp() - { - addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "y" }) ); - } - - // ---- Iris CSV ---- - - @Test - public void testIrisHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "iris", false); - } - - @Test - public void testIrisSingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "iris", false); - } - - @Test - public void testIrisSPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "iris", false); - } - - @Test - public void testIrisHadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "iris", false); - } - - @Test - public void testIrisSparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "iris", false); - } - - // ---- Iris BinaryBlock ---- - - @Test - public void testIrisHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "iris", false); - } - - @Test - public void testIrisSingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "iris", false); - } - - @Test - public void testIrisSPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "iris", false); - } - - @Test - public void testIrisHadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "iris", false); - } - - @Test - public void testIrisSparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "iris", false); - } - - // ---- Homes CSV ---- - - @Test - public void testHomesHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homes", false); - } - - @Test - public void testHomesSingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes", false); - } - - @Test - public void testHomesHadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes", false); - } - - @Test - public void testHomesSPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homes", false); - } - - @Test - public void testHomesSparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homes", false); - } - - // ---- Homes BinaryBlock ---- - - @Test - public void testHomesHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homes", false); - } - - @Test - public void testHomesSingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes", false); - } - - @Test - public void testHomesHadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes", false); - } - - @Test - public void testHomesSPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homes", false); - } - - @Test - public void testHomesSparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homes", false); - } - - // ---- OmitHomes CSV ---- - - @Test - public void testOmitHomesHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homesomit", false); - } - - @Test - public void testOmitHomesSingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homesomit", false); - } - - @Test - public void testOmitHomesHadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homesomit", false); - } - - @Test - public void testOmitHomesSPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homesomit", false); - } - - @Test - public void testOmitHomesSparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homesomit", false); - } - - // ---- OmitHomes BinaryBlock ---- - - @Test - public void testOmitHomesHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homesomit", false); - } - - @Test - public void testOmitHomesSingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homesomit", false); - } - - @Test - public void testOmitHomesHadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homesomit", false); - } - - @Test - public void testOmitHomesSPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homesomit", false); - } - - @Test - public void testOmitHomesSparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homesomit", false); - } - - // ---- Homes2 CSV ---- - - @Test - public void testHomes2HybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homes2", false); - } - - @Test - public void testHomes2SingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes2", false); - } - - @Test - public void testHomes2HadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes2", false); - } - - @Test - public void testHomes2SPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homes2", false); - } - - @Test - public void testHomes2SparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homes2", false); - } - - // ---- Homes2 BinaryBlock ---- - - @Test - public void testHomes2HybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homes2", false); - } - - @Test - public void testHomes2SingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes2", false); - } - - @Test - public void testHomes2HadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes2", false); - } - - @Test - public void testHomes2SPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homes2", false); - } - - @Test - public void testHomes2SparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homes2", false); - } - - // ---- Iris ID CSV ---- - - @Test - public void testIrisHybridIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "iris", true); - } - - @Test - public void testIrisSingleNodeIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "iris", true); - } - - @Test - public void testIrisSPHybridIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "iris", true); - } - - @Test - public void testIrisHadoopIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "iris", true); - } - - @Test - public void testIrisSparkIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "iris", true); - } - - // ---- Iris ID BinaryBlock ---- - - @Test - public void testIrisHybridIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "iris", true); - } - - @Test - public void testIrisSingleNodeIDBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "iris", true); - } - - @Test - public void testIrisSPHybridIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "iris", true); - } - - @Test - public void testIrisHadoopIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "iris", true); - } - - @Test - public void testIrisSparkIDBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "iris", true); - } - - // ---- Homes ID CSV ---- - - @Test - public void testHomesHybridIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homes", true); - } - - @Test - public void testHomesSingleNodeIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes", true); - } - - @Test - public void testHomesSPHybridIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homes", true); - } - - @Test - public void testHomesHadoopIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes", true); - } - - @Test - public void testHomesSparkIDCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homes", true); - } - - // ---- Homes ID BinaryBlock ---- - - @Test - public void testHomesHybridIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homes", true); - } - - @Test - public void testHomesSingleNodeIDBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes", true); - } - - @Test - public void testHomesSPHybridIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homes", true); - } - - @Test - public void testHomesHadoopIDBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes", true); - } - - @Test - public void testHomesSparkIDBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homes", true); - } - - // ---- OmitHomes CSV ---- - - @Test - public void testOmitHomesIDHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homesomit", true); - } - - @Test - public void testOmitHomesIDSingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homesomit", true); - } - - @Test - public void testOmitHomesIDHadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homesomit", true); - } - - @Test - public void testOmitHomesIDSPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homesomit", true); - } - - @Test - public void testOmitHomesIDSparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homesomit", true); - } - - // ---- OmitHomes BinaryBlock ---- - - @Test - public void testOmitHomesIDHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homesomit", true); - } - - @Test - public void testOmitHomesIDSingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homesomit", true); - } - - @Test - public void testOmitHomesIDHadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homesomit", true); - } - - @Test - public void testOmitHomesIDSPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homesomit", true); - } - - @Test - public void testOmitHomesIDSparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homes2", true); - } - - // ---- Homes2 CSV ---- - - @Test - public void testHomes2IDHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "csv", "homes2", true); - } - - @Test - public void testHomes2IDSingleNodeCSV() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "homes2", true); - } - - @Test - public void testHomes2IDHadoopCSV() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "csv", "homes2", true); - } - - @Test - public void testHomes2IDSPHybridCSV() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "csv", "homes2", true); - } - - @Test - public void testHomes2IDSparkCSV() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "csv", "homes2", true); - } - - // ---- Homes2 BinaryBlock ---- - - @Test - public void testHomes2IDHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID, "binary", "homes2", true); - } - - @Test - public void testHomes2IDSingleNodeBB() - { - runTransformTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", "homes2", true); - } - - @Test - public void testHomes2IDHadoopBB() - { - runTransformTest(RUNTIME_PLATFORM.HADOOP, "binary", "homes2", true); - } - - @Test - public void testHomes2IDSPHybridBB() - { - runTransformTest(RUNTIME_PLATFORM.HYBRID_SPARK, "binary", "homes2", true); - } - - @Test - public void testHomes2IDSparkBB() - { - runTransformTest(RUNTIME_PLATFORM.SPARK, "binary", "homes2", true); - } - - // ------------------------------ - - private void runTransformTest( RUNTIME_PLATFORM rt, String ofmt, String dataset, boolean byid ) - { - String DATASET = null, SPEC=null, TFDATA=null; - - if(dataset.equals("homes")) - { - DATASET = HOMES_DATASET; - SPEC = (byid ? HOMES_IDSPEC : HOMES_SPEC); - TFDATA = HOMES_TFDATA; - } - else if(dataset.equals("homesomit")) - { - DATASET = HOMES_OMIT_DATASET; - SPEC = (byid ? HOMES_OMIT_IDSPEC : HOMES_OMIT_SPEC); - TFDATA = HOMES_OMIT_TFDATA; - } - - else if(dataset.equals("homes2")) - { - DATASET = HOMES2_DATASET; - SPEC = (byid ? HOMES2_IDSPEC : HOMES2_SPEC); - TFDATA = HOMES2_TFDATA; - } - else if (dataset.equals("iris")) - { - DATASET = IRIS_DATASET; - SPEC = (byid ? IRIS_IDSPEC : IRIS_SPEC); - TFDATA = IRIS_TFDATA; - } - - RUNTIME_PLATFORM rtold = rtplatform; - rtplatform = rt; - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - getAndLoadTestConfiguration(TEST_NAME1); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "TFSPEC=" + HOME + "input/" + SPEC, - "TFMTD=" + output("tfmtd"), - "TFDATA=" + output("tfout"), - "OFMT=" + ofmt }; - - boolean exceptionExpected = false; - runTest(true, exceptionExpected, null, -1); - - fullDMLScriptName = HOME + TEST_NAME2 + ".dml"; - programArgs = new String[]{"-nvargs", - "DATA=" + HOME + "input/" + DATASET, - "APPLYMTD=" + output("tfmtd"), // generated above - "TFMTD=" + output("test_tfmtd"), - "TFDATA=" + output("test_tfout"), - "OFMT=" + ofmt }; - - exceptionExpected = false; - runTest(true, exceptionExpected, null, -1); - - try { - ReaderTextCSV csvReader= new ReaderTextCSV(new CSVFileFormatProperties(true, ",", true, 0, null)); - MatrixBlock exp = csvReader.readMatrixFromHDFS(HOME+"input/"+ TFDATA, -1, -1, -1, -1, -1); - - MatrixBlock out = null, out2=null; - if(ofmt.equals("csv")) - { - ReaderTextCSV outReader= new ReaderTextCSV(new CSVFileFormatProperties(false, ",", true, 0, null)); - out = outReader.readMatrixFromHDFS(output("tfout"), -1, -1, -1, -1, -1); - out2 = outReader.readMatrixFromHDFS(output("test_tfout"), -1, -1, -1, -1, -1); - } - else - { - ReaderBinaryBlock bbReader = new ReaderBinaryBlock(false); - out = bbReader.readMatrixFromHDFS( - output("tfout"), exp.getNumRows(), exp.getNumColumns(), - ConfigurationManager.getBlocksize(), - ConfigurationManager.getBlocksize(), - -1); - out2 = bbReader.readMatrixFromHDFS( - output("test_tfout"), exp.getNumRows(), exp.getNumColumns(), - ConfigurationManager.getBlocksize(), - ConfigurationManager.getBlocksize(), - -1); - } - - assertTrue("Incorrect output from data transform.", equals(out,exp, 1e-10)); - assertTrue("Incorrect output from apply transform.", equals(out2,exp, 1e-10)); - - } catch (Exception e) { - throw new RuntimeException(e); - } - } - finally - { - rtplatform = rtold; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } - - public static boolean equals(MatrixBlock mb1, MatrixBlock mb2, double epsilon) - { - if(mb1.getNumRows() != mb2.getNumRows() || mb1.getNumColumns() != mb2.getNumColumns() || mb1.getNonZeros() != mb2.getNonZeros() ) - return false; - - // TODO: this implementation is to be optimized for different block representations - for(int i=0; i < mb1.getNumRows(); i++) - for(int j=0; j < mb1.getNumColumns(); j++ ) - if(!TestUtils.compareCellValue(mb1.getValue(i, j), mb2.getValue(i,j), epsilon, false)) - { - System.err.println("(i="+ (i+1) + ",j=" + (j+1) + ") " + mb1.getValue(i, j) + " != " + mb2.getValue(i, j)); - return false; - } - - return true; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/Apply.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/Apply.dml b/src/test/scripts/functions/transform/Apply.dml deleted file mode 100644 index 8cbec31..0000000 --- a/src/test/scripts/functions/transform/Apply.dml +++ /dev/null @@ -1,30 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -raw = read($DATA); - -A = transform(target = raw, - transformPath = $TFMTD, - applyTransformPath = $APPLYMTD); - -write(A, $TFDATA, format=$OFMT); - http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/ApplyFrame.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/ApplyFrame.dml b/src/test/scripts/functions/transform/ApplyFrame.dml deleted file mode 100644 index 4fbcc36..0000000 --- a/src/test/scripts/functions/transform/ApplyFrame.dml +++ /dev/null @@ -1,29 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -raw = read($DATA); -jspec = read($TFSPEC, data_type="scalar", value_type="string") - -M = transformmeta(spec=jspec, transformPath=$APPLYMTD); -A = transformapply(target=raw, spec=jspec, meta=M); - -write(A, $TFDATA, format=$OFMT); - http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/Scaling.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/Scaling.R b/src/test/scripts/functions/transform/Scaling.R deleted file mode 100644 index a3bfe59..0000000 --- a/src/test/scripts/functions/transform/Scaling.R +++ /dev/null @@ -1,36 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -args <- commandArgs(TRUE) -options(digits=22) -library("Matrix") - -A = read.table(args[1], sep=","); -B = matrix(0, nrow=nrow(A), ncol=ncol(A)); - -cols = ncol(A); -A1 = A[, 1:cols/2]; -A2 = A[,(cols/2+1):cols] -B[, 1:cols/2] = scale(A1, center=T, scale=F) -B[, (cols/2+1):cols] = scale(A2) - -write.table(B, args[2], sep=",", row.names = F, col.names=F) http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/Scaling.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/Scaling.dml b/src/test/scripts/functions/transform/Scaling.dml deleted file mode 100644 index 1542477..0000000 --- a/src/test/scripts/functions/transform/Scaling.dml +++ /dev/null @@ -1,31 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -raw = read($DATA); -specJson = read($TFSPEC, data_type="scalar", value_type="string") - -A = transform(target = raw, - transformPath = $TFMTD, - spec = specJson); - -write(A, $TFDATA, format=$OFMT); - http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/Transform.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/Transform.dml b/src/test/scripts/functions/transform/Transform.dml deleted file mode 100644 index d36957a..0000000 --- a/src/test/scripts/functions/transform/Transform.dml +++ /dev/null @@ -1,31 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -raw = read($DATA); -specJson = read($TFSPEC, data_type="scalar", value_type="string"); - -A = transform(target = raw, - transformPath = $TFMTD, - spec = specJson); - -write(A, $TFDATA, format=$OFMT); - http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformAndApply.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformAndApply.dml b/src/test/scripts/functions/transform/TransformAndApply.dml deleted file mode 100644 index fc97402..0000000 --- a/src/test/scripts/functions/transform/TransformAndApply.dml +++ /dev/null @@ -1,37 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -#transform -X = read($1, data_type="frame", format="csv"); -specX = $3; -R1 = transform(target = X, spec = specX, transformPath = $5); - -if( 1==1 ){} - -#transform apply -Y = read($2, data_type="frame", format="csv"); -specY = $4; -R2 = transform(target = Y, spec = specY, applyTransformPath = $5, transformPath = $6); - -if( 1==1 ){} - -write(R1, $7) -write(R2, $8); http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformAndApplySpecX.json ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformAndApplySpecX.json b/src/test/scripts/functions/transform/TransformAndApplySpecX.json deleted file mode 100644 index b5c9a84..0000000 --- a/src/test/scripts/functions/transform/TransformAndApplySpecX.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ids": true - ,"recode": [ 1, 2 ] - -} http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformAndApplySpecY.json ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformAndApplySpecY.json b/src/test/scripts/functions/transform/TransformAndApplySpecY.json deleted file mode 100644 index e10a5fa..0000000 --- a/src/test/scripts/functions/transform/TransformAndApplySpecY.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ids": true - ,"recode": [ 1 ] - -} http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformReadMeta.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformReadMeta.dml b/src/test/scripts/functions/transform/TransformReadMeta.dml deleted file mode 100644 index 7328aa9..0000000 --- a/src/test/scripts/functions/transform/TransformReadMeta.dml +++ /dev/null @@ -1,33 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -#transform -X = read($1, data_type="frame", format="csv"); -specX = $2; -R1 = transform(target = X, spec = specX, transformPath = $3); - -if( 1==1 ){} - -print(sum(R1)); - -#transform read meta data and write as frame -M = transformmeta(spec = specX, transformPath = $3); -write(M, $4, format=$5); http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformReadMeta2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformReadMeta2.dml b/src/test/scripts/functions/transform/TransformReadMeta2.dml deleted file mode 100644 index f86911e..0000000 --- a/src/test/scripts/functions/transform/TransformReadMeta2.dml +++ /dev/null @@ -1,33 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -#transform -X = read($1, data_type="frame", format="csv", sep=$6); -specX = $2; -R1 = transform(target = X, spec = specX, transformPath = $3); - -if( 1==1 ){} - -print(sum(R1)); - -#transform read meta data and write as frame -M = transformmeta(spec = specX, transformPath = $3, sep=$6); -write(M, $4, format=$5); http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/TransformReadMetaSpecX.json ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformReadMetaSpecX.json b/src/test/scripts/functions/transform/TransformReadMetaSpecX.json deleted file mode 100644 index b5c9a84..0000000 --- a/src/test/scripts/functions/transform/TransformReadMetaSpecX.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ids": true - ,"recode": [ 1, 2 ] - -} http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test/scripts/functions/transform/Transform_colnames.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/Transform_colnames.dml b/src/test/scripts/functions/transform/Transform_colnames.dml deleted file mode 100644 index 8f40dcc..0000000 --- a/src/test/scripts/functions/transform/Transform_colnames.dml +++ /dev/null @@ -1,32 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -raw = read($DATA); -specJson = read($TFSPEC, data_type="scalar", value_type="string"); - -A = transform(target = raw, - transformPath = $TFMTD, - spec = specJson, - outputNames = $COLNAMES); - -write(A, $TFDATA, format=$OFMT); - http://git-wip-us.apache.org/repos/asf/systemml/blob/0cd3905f/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java index e36d4a0..645a468 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/transform/ZPackageSuite.java @@ -27,18 +27,12 @@ import org.junit.runners.Suite; @RunWith(Suite.class) @Suite.SuiteClasses({ FrameCSVReadWriteTest.class, - RunTest.class, - ScalingTest.class, - TransformAndApplyTest.class, TransformCSVFrameEncodeDecodeTest.class, TransformCSVFrameEncodeReadTest.class, TransformEncodeDecodeTest.class, - TransformFrameApplyTest.class, TransformFrameEncodeApplyTest.class, TransformFrameEncodeDecodeTest.class, TransformFrameEncodeDecodeTokenTest.class, - TransformReadMetaTest.class, - TransformTest.class, })
