Repository: incubator-systemml Updated Branches: refs/heads/master 5096defa7 -> 328b644cb
[SYSTEMML-561] New matrix-frame/frame-matrix casting operations, tests Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c8c8d81e Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c8c8d81e Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c8c8d81e Branch: refs/heads/master Commit: c8c8d81eaaf43d6649f3956434eab36e2b87bae4 Parents: 5096def Author: Matthias Boehm <[email protected]> Authored: Mon Apr 4 14:43:43 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Tue Apr 5 00:18:09 2016 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 3 +- .../java/org/apache/sysml/hops/UnaryOp.java | 4 +- .../java/org/apache/sysml/lops/UnaryCP.java | 8 +- .../sysml/parser/BuiltinFunctionExpression.java | 34 +++- .../org/apache/sysml/parser/DMLTranslator.java | 5 + .../org/apache/sysml/parser/Expression.java | 1 + .../java/org/apache/sysml/parser/dml/Dml.g4 | 2 +- .../instructions/CPInstructionParser.java | 1 + .../runtime/instructions/cp/CPOperand.java | 4 +- .../instructions/cp/VariableCPInstruction.java | 40 +++- .../functions/jmlc/FrameCastingTest.java | 184 +++++++++++++++++++ .../functions/jmlc/FrameTransformTest.java | 8 +- src/test/scripts/functions/jmlc/transform6.dml | 29 +++ 13 files changed, 306 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index adf1828..fa45ed3 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -1033,7 +1033,7 @@ public abstract class Hop public enum OpOp1 { NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SIGN, SQRT, LOG, EXP, - CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, + CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, EIGEN, NROW, NCOL, LENGTH, ROUND, IQM, STOP, CEIL, FLOOR, MEDIAN, INVERSE, CHOLESKY, //cumulative sums, products, extreme values CUMSUM, CUMPROD, CUMMIN, CUMMAX, @@ -1279,6 +1279,7 @@ public abstract class Hop HopsOpOp1LopsUS.put(OpOp1.LOG, org.apache.sysml.lops.UnaryCP.OperationTypes.LOG); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX); + HopsOpOp1LopsUS.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_FRAME); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_INT); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/hops/UnaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java index e200691..4ed0225 100644 --- a/src/main/java/org/apache/sysml/hops/UnaryOp.java +++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java @@ -121,7 +121,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop { Hop input = getInput().get(0); - if (getDataType() == DataType.SCALAR || _op == OpOp1.CAST_AS_MATRIX) + if (getDataType() == DataType.SCALAR + || _op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME ) //TODO generalize frames to distributed ops { if (_op == Hop.OpOp1.IQM) //special handling IQM { @@ -633,6 +634,7 @@ public class UnaryOp extends Hop implements MultiThreadedHop { return ( _op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_SCALAR + || _op == OpOp1.CAST_AS_FRAME || _op == OpOp1.CAST_AS_BOOLEAN || _op == OpOp1.CAST_AS_DOUBLE || _op == OpOp1.CAST_AS_INT ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/lops/UnaryCP.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/UnaryCP.java b/src/main/java/org/apache/sysml/lops/UnaryCP.java index 74b3953..4fa3cc7 100644 --- a/src/main/java/org/apache/sysml/lops/UnaryCP.java +++ b/src/main/java/org/apache/sysml/lops/UnaryCP.java @@ -34,11 +34,14 @@ public class UnaryCP extends Lop { public enum OperationTypes { - NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, NROW, NCOL, LENGTH, ROUND, STOP, CEIL, FLOOR, CUMSUM, NOTSUPPORTED + NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP, + CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, + PRINT, NROW, NCOL, LENGTH, ROUND, STOP, CEIL, FLOOR, CUMSUM, NOTSUPPORTED }; public static final String CAST_AS_SCALAR_OPCODE = "castdts"; public static final String CAST_AS_MATRIX_OPCODE = "castdtm"; + public static final String CAST_AS_FRAME_OPCODE = "castdtf"; public static final String CAST_AS_DOUBLE_OPCODE = "castvtd"; public static final String CAST_AS_INT_OPCODE = "castvti"; public static final String CAST_AS_BOOLEAN_OPCODE = "castvtb"; @@ -119,6 +122,9 @@ public class UnaryCP extends Lop case CAST_AS_MATRIX: return CAST_AS_MATRIX_OPCODE; + + case CAST_AS_FRAME: + return CAST_AS_FRAME_OPCODE; case STOP: return "stop"; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java index 886aa0a..56f5857 100644 --- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java @@ -362,9 +362,21 @@ public class BuiltinFunctionExpression extends DataIdentifier break; case CAST_AS_MATRIX: checkNumParameters(1); - checkScalarParam(getFirstExpr()); + checkScalarFrameParam(getFirstExpr()); output.setDataType(DataType.MATRIX); - output.setDimensions(1, 1); + output.setDimensions(id.getDim1(), id.getDim2()); + if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) + output.setDimensions(1, 1); //correction scalars + output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock()); + output.setValueType(id.getValueType()); + break; + case CAST_AS_FRAME: + checkNumParameters(1); + checkMatrixParam(getFirstExpr()); + output.setDataType(DataType.FRAME); + output.setDimensions(id.getDim1(), id.getDim2()); + if( getFirstExpr().getOutput().getDataType()==DataType.SCALAR ) + output.setDimensions(1, 1); //correction scalars output.setBlockDimensions(id.getRowsInBlock(), id.getColumnsInBlock()); output.setValueType(id.getValueType()); break; @@ -1197,8 +1209,20 @@ public class BuiltinFunctionExpression extends DataIdentifier private void checkScalarParam(Expression e) //always unconditional throws LanguageException { - if (e.getOutput().getDataType() != DataType.SCALAR) - { + if (e.getOutput().getDataType() != DataType.SCALAR) { + raiseValidateError("Expecting scalar parameter for function " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + } + } + + /** + * + * @param e + * @throws LanguageException + */ + private void checkScalarFrameParam(Expression e) //always unconditional + throws LanguageException + { + if (e.getOutput().getDataType() != DataType.SCALAR && e.getOutput().getDataType() != DataType.FRAME) { raiseValidateError("Expecting scalar parameter for function " + this.getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); } } @@ -1415,6 +1439,8 @@ public class BuiltinFunctionExpression extends DataIdentifier bifop = Expression.BuiltinFunctionOp.CAST_AS_SCALAR; else if (functionName.equals("as.matrix")) bifop = Expression.BuiltinFunctionOp.CAST_AS_MATRIX; + else if (functionName.equals("as.frame")) + bifop = Expression.BuiltinFunctionOp.CAST_AS_FRAME; else if (functionName.equals("as.double")) bifop = Expression.BuiltinFunctionOp.CAST_AS_DOUBLE; else if (functionName.equals("as.integer")) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index d4bd062..2adcd77 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -2446,13 +2446,18 @@ public class DMLTranslator } break; + //data type casts case CAST_AS_SCALAR: currBuiltinOp = new UnaryOp(target.getName(), DataType.SCALAR, target.getValueType(), Hop.OpOp1.CAST_AS_SCALAR, expr); break; case CAST_AS_MATRIX: currBuiltinOp = new UnaryOp(target.getName(), DataType.MATRIX, target.getValueType(), Hop.OpOp1.CAST_AS_MATRIX, expr); break; + case CAST_AS_FRAME: + currBuiltinOp = new UnaryOp(target.getName(), DataType.FRAME, target.getValueType(), Hop.OpOp1.CAST_AS_FRAME, expr); + break; + //value type casts case CAST_AS_DOUBLE: currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.DOUBLE, Hop.OpOp1.CAST_AS_DOUBLE, expr); break; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/parser/Expression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java index f6acf5a..8695cab 100644 --- a/src/main/java/org/apache/sysml/parser/Expression.java +++ b/src/main/java/org/apache/sysml/parser/Expression.java @@ -58,6 +58,7 @@ public abstract class Expression CAST_AS_INT, CAST_AS_MATRIX, CAST_AS_SCALAR, + CAST_AS_FRAME, CBIND, //previously APPEND CEIL, CHOLESKY, http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/parser/dml/Dml.g4 ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/dml/Dml.g4 b/src/main/java/org/apache/sysml/parser/dml/Dml.g4 index e37b15c..0ee12c6 100644 --- a/src/main/java/org/apache/sysml/parser/dml/Dml.g4 +++ b/src/main/java/org/apache/sysml/parser/dml/Dml.g4 @@ -181,7 +181,7 @@ strictParameterizedKeyValueString : paramName=ID '=' paramVal=STRING ; ID : (ALPHABET (ALPHABET|DIGIT|'_')* '::')? ALPHABET (ALPHABET|DIGIT|'_')* // Special ID cases: // | 'matrix' // --> This is a special case which causes lot of headache - | 'as.scalar' | 'as.matrix' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail' + | 'as.scalar' | 'as.matrix' | 'as.frame' | 'as.double' | 'as.integer' | 'as.logical' | 'index.return' | 'lower.tail' ; // Unfortunately, we have datatype name clashing with builtin function name: matrix :( // Therefore, ugly work around for checking datatype http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java index 8ffc4e7..8ecb2e1 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java @@ -192,6 +192,7 @@ public class CPInstructionParser extends InstructionParser String2CPInstructionType.put( "rmfilevar" , CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( UnaryCP.CAST_AS_SCALAR_OPCODE, CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( UnaryCP.CAST_AS_MATRIX_OPCODE, CPINSTRUCTION_TYPE.Variable); + String2CPInstructionType.put( UnaryCP.CAST_AS_FRAME_OPCODE, CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( UnaryCP.CAST_AS_DOUBLE_OPCODE, CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( UnaryCP.CAST_AS_INT_OPCODE, CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( UnaryCP.CAST_AS_BOOLEAN_OPCODE, CPINSTRUCTION_TYPE.Variable); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java index 195b9d0..2f7fdb6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java @@ -68,7 +68,7 @@ public class CPOperand return _isLiteral; } - public void set_name(String name) { + public void setName(String name) { _name = name; } @@ -80,7 +80,7 @@ public class CPOperand _dataType = dt; } - public void set_literal(boolean literal) { + public void setLiteral(boolean literal) { _isLiteral = literal; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index dac114b..ac5d456 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -42,9 +42,11 @@ import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; import org.apache.sysml.runtime.matrix.MetaData; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FileFormatProperties; +import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; +import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.MapReduceTool; import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.utils.Statistics; @@ -80,7 +82,8 @@ public class VariableCPInstruction extends CPInstruction RemoveVariable, RemoveVariableAndFile, CastAsScalarVariable, - CastAsMatrixVariable, + CastAsMatrixVariable, + CastAsFrameVariable, CastAsDoubleVariable, CastAsIntegerVariable, CastAsBooleanVariable, @@ -133,6 +136,9 @@ public class VariableCPInstruction extends CPInstruction else if ( str.equalsIgnoreCase(UnaryCP.CAST_AS_MATRIX_OPCODE) ) return VariableOperationCode.CastAsMatrixVariable; + else if ( str.equalsIgnoreCase(UnaryCP.CAST_AS_FRAME_OPCODE) ) + return VariableOperationCode.CastAsFrameVariable; + else if ( str.equalsIgnoreCase(UnaryCP.CAST_AS_DOUBLE_OPCODE) ) return VariableOperationCode.CastAsDoubleVariable; @@ -379,6 +385,7 @@ public class VariableCPInstruction extends CPInstruction case CastAsScalarVariable: case CastAsMatrixVariable: + case CastAsFrameVariable: case CastAsDoubleVariable: case CastAsIntegerVariable: case CastAsBooleanVariable: @@ -512,12 +519,35 @@ public class VariableCPInstruction extends CPInstruction ec.setScalarOutput(output.getName(), new DoubleObject(value)); break; case CastAsMatrixVariable:{ - ScalarObject scalarInput = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); - MatrixBlock out = new MatrixBlock(1,1,false); - out.quickSetValue(0, 0, scalarInput.getDoubleValue()); + MatrixBlock out = null; + if( input1.getDataType()==DataType.SCALAR ) { + ScalarObject scalarInput = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); + out = new MatrixBlock(1,1,false); + out.quickSetValue(0, 0, scalarInput.getDoubleValue()); + } + else { //DataType.FRAME + FrameBlock fin = ec.getFrameInput(input1.getName()); + out = DataConverter.convertToMatrixBlock(fin); + ec.releaseFrameInput(input1.getName()); + } ec.setMatrixOutput(output.getName(), out); break; } + case CastAsFrameVariable:{ + FrameBlock out = null; + if( input1.getDataType()==DataType.SCALAR ) { + ScalarObject scalarInput = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); + out = new FrameBlock(1, input1.getValueType()); + out.set(0, 0, scalarInput.getStringValue()); + } + else { //DataType.FRAME + MatrixBlock min = ec.getMatrixInput(input1.getName()); + out = DataConverter.convertToFrameBlock(min); + ec.releaseMatrixInput(input1.getName()); + } + ec.setFrameOutput(output.getName(), out); + break; + } case CastAsDoubleVariable:{ ScalarObject scalarInput = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); ec.setScalarOutput(output.getName(), new DoubleObject(scalarInput.getDoubleValue())); @@ -1000,7 +1030,7 @@ public class VariableCPInstruction extends CPInstruction || opcode == VariableOperationCode.SetFileName ) { //replace in-memory instruction - input2.set_name(input2.getName().replaceAll(pattern, replace)); + input2.setName(input2.getName().replaceAll(pattern, replace)); // Find a start position of file name string. int iPos = StringUtils.ordinalIndexOf(instString, Lop.OPERAND_DELIMITOR, CREATEVAR_FILE_NAME_VAR_POS); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameCastingTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameCastingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameCastingTest.java new file mode 100644 index 0000000..d4f1c33 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameCastingTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.jmlc; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; +import org.apache.sysml.api.DMLException; +import org.apache.sysml.api.jmlc.Connection; +import org.apache.sysml.api.jmlc.PreparedScript; +import org.apache.sysml.api.jmlc.ResultVariables; +import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +/** + * + * + */ +public class FrameCastingTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "transform6"; + private final static String TEST_DIR = "functions/jmlc/"; + private final static String TEST_CLASS_DIR = TEST_DIR + FrameCastingTest.class.getSimpleName() + "/"; + + private final static int rows = 700; + private final static int cols = 3; + + private final static int nRuns = 2; + + private final static double sparsity1 = 0.7; + private final static double sparsity2 = 0.1; + + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "F2" }) ); + } + + @Test + public void testJMLCTransformDense() throws IOException { + runJMLCReuseTest(TEST_NAME1, false, false); + } + + @Test + public void testJMLCTransformSparse() throws IOException { + runJMLCReuseTest(TEST_NAME1, true, false); + } + + @Test + public void testJMLCTransformDenseReuse() throws IOException { + runJMLCReuseTest(TEST_NAME1, false, true); + } + + @Test + public void testJMLCTransformSparseReuse() throws IOException { + runJMLCReuseTest(TEST_NAME1, true, true); + } + + /** + * + * @param sparseM1 + * @param sparseM2 + * @param instType + * @throws IOException + */ + private void runJMLCReuseTest( String testname, boolean sparse, boolean modelReuse ) + throws IOException + { + String TEST_NAME = testname; + + TestConfiguration config = getTestConfiguration(TEST_NAME); + loadTestConfiguration(config); + + //generate inputs + double[][] Fd = TestUtils.round(getRandomMatrix(rows, cols, 0.51, 7.49, sparse?sparsity2:sparsity1, 1234)); + String[][] F1s = FrameTransformTest.createFrameData(Fd, ""); + + //run DML via JMLC + ArrayList<String[][]> F2set = execDMLScriptviaJMLC( TEST_NAME, F1s, modelReuse ); + + //check correct result + double[][] cF1 = add(Fd, 7); + for( String[][] data : F2set ) + for( int i=0; i<F1s.length; i++ ) + for( int j=0; j<F1s[i].length; j++ ) + Assert.assertEquals("Wrong result: "+data[i][j]+".", new Double(data[i][j]), new Double(cF1[i][j])); + } + + /** + * + * @param X + * @return + * @throws DMLException + * @throws IOException + */ + private ArrayList<String[][]> execDMLScriptviaJMLC( String testname, String[][] F1, boolean modelReuse) + throws IOException + { + Timing time = new Timing(true); + + ArrayList<String[][]> ret = new ArrayList<String[][]>(); + + //establish connection to SystemML + Connection conn = new Connection(); + + try + { + //prepare input arguments + HashMap<String,String> args = new HashMap<String,String>(); + args.put("$TRANSFORM_SPEC", "{ \"ids\": true ,\"recode\": [ 1, 2, 3] }"); + + //read and precompile script + String script = conn.readScript(SCRIPT_DIR + TEST_DIR + testname + ".dml"); + PreparedScript pstmt = conn.prepareScript(script, args, new String[]{"F1","M"}, new String[]{"F2"}, false); + + if( modelReuse ) + pstmt.setFrame("F1", F1, true); + + //execute script multiple times + for( int i=0; i<nRuns; i++ ) + { + //bind input parameters + if( !modelReuse ) + pstmt.setFrame("F1", F1); + + //execute script + ResultVariables rs = pstmt.executeScript(); + + //get output parameter + String[][] Y = rs.getFrame("F2"); + ret.add(Y); //keep result for comparison + } + } + catch(Exception ex) + { + ex.printStackTrace(); + throw new IOException(ex); + } + finally + { + if( conn != null ) + conn.close(); + } + + System.out.println("JMLC scoring w/ "+nRuns+" runs in "+time.stop()+"ms."); + + return ret; + } + + /** + * + * @param data + * @param val + * @return + */ + private double[][] add(double[][] data, double val) { + for( int i=0; i<data.length; i++ ) + for( int j=0; j<data[i].length; j++ ) + data[i][j] += val; + return data; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameTransformTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameTransformTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameTransformTest.java index 2821664..912f324 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameTransformTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/FrameTransformTest.java @@ -170,17 +170,21 @@ public class FrameTransformTest extends AutomatedTestBase return ret; } + protected static String[][] createFrameData(double[][] data) { + return createFrameData(data, "V"); + } + /** * * @param data * @return */ - protected static String[][] createFrameData(double[][] data) { + protected static String[][] createFrameData(double[][] data, String prefix) { String[][] ret = new String[data.length][]; for( int i=0; i<data.length; i++ ) { String[] row = new String[data[i].length]; for( int j=0; j<data[i].length; j++ ) - row[j] = "V"+String.valueOf(data[i][j]); + row[j] = prefix+String.valueOf(data[i][j]); ret[i] = row; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c8c8d81e/src/test/scripts/functions/jmlc/transform6.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/jmlc/transform6.dml b/src/test/scripts/functions/jmlc/transform6.dml new file mode 100644 index 0000000..2082e21 --- /dev/null +++ b/src/test/scripts/functions/jmlc/transform6.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +F1 = read($F1, data_type="frame", format="csv"); #new data + +X = as.matrix(F1); +X = X+7; +F2 = as.frame(X); + +write(F2, $F2); +
