Repository: incubator-systemml Updated Branches: refs/heads/master e9030ad39 -> 3ef044092
[SYSTEMML-583] Extended transformmeta builtin (optional sep arg), tests Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/3ab79dc5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/3ab79dc5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/3ab79dc5 Branch: refs/heads/master Commit: 3ab79dc56968b0dedc2675567f27bdab9cf6421e Parents: e9030ad Author: Matthias Boehm <[email protected]> Authored: Mon Apr 25 16:07:54 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Apr 25 16:08:14 2016 -0700 ---------------------------------------------------------------------- .../cp/ParameterizedBuiltinCPInstruction.java | 3 +- .../transform/TransformReadMetaTest.java | 67 +++++++++++++++----- .../functions/transform/TransformReadMeta2.dml | 33 ++++++++++ 3 files changed, 87 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3ab79dc5/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index a96c2b6..750ecfd 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -274,11 +274,12 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction //get input spec and path String spec = getParameterMap().get("spec"); String path = getParameterMap().get(ParameterizedBuiltinFunctionExpression.TF_FN_PARAM_MTD); + String delim = getParameterMap().containsKey("sep") ? getParameterMap().get("sep") : TfUtils.TXMTD_SEP; //execute transform meta data read FrameBlock meta = null; try { - meta = TfMetaUtils.readTransformMetaDataFromFile(spec, path, TfUtils.TXMTD_SEP); + meta = TfMetaUtils.readTransformMetaDataFromFile(spec, path, delim); } catch(Exception ex) { throw new DMLRuntimeException(ex); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3ab79dc5/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java index c0e09ba..4e0240b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformReadMetaTest.java @@ -31,6 +31,7 @@ import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; import org.apache.sysml.runtime.io.MatrixWriter; import org.apache.sysml.runtime.io.MatrixWriterFactory; +import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -50,6 +51,7 @@ import org.apache.sysml.test.utils.TestUtils; public class TransformReadMetaTest extends AutomatedTestBase { private static final String TEST_NAME1 = "TransformReadMeta"; + private static final String TEST_NAME2 = "TransformReadMeta2"; private static final String TEST_DIR = "functions/transform/"; private static final String TEST_CLASS_DIR = TEST_DIR + TransformReadMetaTest.class.getSimpleName() + "/"; private static final String SPEC_X = "TransformReadMetaSpecX.json"; @@ -60,51 +62,83 @@ public class TransformReadMetaTest extends AutomatedTestBase public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"M1, M"})); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2,new String[]{"M1, M"})); } @Test public void runTestCsvCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", ","); } @Test public void runTestCsvHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv"); + runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", ","); } @Test public void runTestCsvSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", ","); } @Test + public void runTestCsvTabCP() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", "\t"); + } + + @Test + public void runTestCsvTabHadoop() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", "\t"); + } + + @Test + public void runTestCsvTabSpark() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", "\t"); + } + + @Test + public void runTestCsvColonCP() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "csv", ":"); + } + + @Test + public void runTestCsvColonHadoop() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "csv", ":"); + } + + @Test + public void runTestCsvColonSpark() throws DMLRuntimeException, IOException { + runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "csv", ":"); + } + + + @Test public void runTestTextCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "text"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "text", ","); } @Test public void runTestTextHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "text"); + runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "text", ","); } @Test public void runTestTextSpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "text"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "text", ","); } @Test public void runTestBinaryCP() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SINGLE_NODE, "binary", ","); } @Test public void runTestBinaryHadoop() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "binary"); + runTransformReadMetaTest(RUNTIME_PLATFORM.HADOOP, "binary", ","); } @Test public void runTestBinarySpark() throws DMLRuntimeException, IOException { - runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "binary"); + runTransformReadMetaTest(RUNTIME_PLATFORM.SPARK, "binary", ","); } @@ -116,7 +150,7 @@ public class TransformReadMetaTest extends AutomatedTestBase * @throws IOException * @throws DMLRuntimeException */ - private void runTransformReadMetaTest( RUNTIME_PLATFORM rt, String ofmt) throws IOException, DMLRuntimeException + private void runTransformReadMetaTest( RUNTIME_PLATFORM rt, String ofmt, String delim) throws IOException, DMLRuntimeException { RUNTIME_PLATFORM platformOld = rtplatform; rtplatform = rt; @@ -127,21 +161,24 @@ public class TransformReadMetaTest extends AutomatedTestBase try { - getAndLoadTestConfiguration(TEST_NAME1); + String testname = delim.equals(",") ? TEST_NAME1 : TEST_NAME2; + + getAndLoadTestConfiguration(testname); //generate input data double[][] X = DataConverter.convertToDoubleMatrix( MatrixBlock.seqOperations(0.5, rows/2, 0.5).appendOperations( MatrixBlock.seqOperations(0.5, rows/2, 0.5), new MatrixBlock())); MatrixBlock mbX = DataConverter.convertToMatrixBlock(X); - MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo); + CSVFileFormatProperties fprops = new CSVFileFormatProperties(false, delim, false); + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter(OutputInfo.CSVOutputInfo, 1, fprops); writer.writeMatrixToHDFS(mbX, input("X"), rows, 2, -1, -1, -1); //read specs transform X and Y String specX = MapReduceTool.readStringFromHDFSFile(SCRIPT_DIR+TEST_DIR+SPEC_X); - fullDMLScriptName = SCRIPT_DIR+TEST_DIR + TEST_NAME1 + ".dml"; - programArgs = new String[]{"-args", input("X"), specX, output("M1"), output("M"), ofmt}; + fullDMLScriptName = SCRIPT_DIR+TEST_DIR + testname + ".dml"; + programArgs = new String[]{"-args", input("X"), specX, output("M1"), output("M"), ofmt, delim}; //run test runTest(true, false, null, -1); @@ -149,7 +186,7 @@ public class TransformReadMetaTest extends AutomatedTestBase //compare meta data frames InputInfo iinfo = InputInfo.stringExternalToInputInfo(ofmt); FrameReader reader = FrameReaderFactory.createFrameReader(iinfo); - FrameBlock mExpected = TfMetaUtils.readTransformMetaDataFromFile(specX, output("M1"), ","); + FrameBlock mExpected = TfMetaUtils.readTransformMetaDataFromFile(specX, output("M1"), delim); FrameBlock mRet = reader.readFrameFromHDFS(output("M"), rows, 2); for( int i=0; i<rows; i++ ) for( int j=0; j<2; j++ ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3ab79dc5/src/test/scripts/functions/transform/TransformReadMeta2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/transform/TransformReadMeta2.dml b/src/test/scripts/functions/transform/TransformReadMeta2.dml new file mode 100644 index 0000000..f86911e --- /dev/null +++ b/src/test/scripts/functions/transform/TransformReadMeta2.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +#transform +X = read($1, data_type="frame", format="csv", sep=$6); +specX = $2; +R1 = transform(target = X, spec = specX, transformPath = $3); + +if( 1==1 ){} + +print(sum(R1)); + +#transform read meta data and write as frame +M = transformmeta(spec = specX, transformPath = $3, sep=$6); +write(M, $4, format=$5);
