Repository: systemml Updated Branches: refs/heads/master 9da5eab00 -> 70e5f29e8
[SYSTEMML-1826] Fix scalar-frame casts and frame rbind operations This patch fixes the missing compiler support for scalar-frame casting as well as missing meta data handling on frame rbind operations. This also includes a suite of related testcases and some minor cleanups in AutomatedTestBase (base class of all tests). Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/70e5f29e Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/70e5f29e Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/70e5f29e Branch: refs/heads/master Commit: 70e5f29e8fff0e698d7abc647efe3f79be38abba Parents: 9da5eab Author: Matthias Boehm <[email protected]> Authored: Sun Aug 6 20:45:51 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Aug 6 20:46:16 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/UnaryOp.java | 6 +- .../sysml/runtime/matrix/data/FrameBlock.java | 9 +- .../test/integration/AutomatedTestBase.java | 4 - .../frame/FrameScalarCastingIntegratedTest.java | 171 +++++++++++++++++++ .../scripts/functions/frame/FrameScalarCast.dml | 30 ++++ .../functions/frame/ZPackageSuite.java | 1 + 6 files changed, 211 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/hops/UnaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java b/src/main/java/org/apache/sysml/hops/UnaryOp.java index 35902b7..2b31247 100644 --- a/src/main/java/org/apache/sysml/hops/UnaryOp.java +++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java @@ -618,8 +618,7 @@ public class UnaryOp extends Hop implements MultiThreadedHop || _op == OpOp1.CUMMAX ); } - public boolean isCastUnaryOperation() - { + public boolean isCastUnaryOperation() { return ( _op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_SCALAR || _op == OpOp1.CAST_AS_FRAME @@ -695,7 +694,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop { //do nothing always known } - else if( _op == OpOp1.CAST_AS_MATRIX && getInput().get(0).getDataType()==DataType.SCALAR ) + else if( (_op == OpOp1.CAST_AS_MATRIX || _op == OpOp1.CAST_AS_FRAME) + && getInput().get(0).getDataType()==DataType.SCALAR ) { //prevent propagating 0 from scalar (which would be interpreted as unknown) setDim1( 1 ); http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index 10b39fe..45ad26c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -967,7 +967,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable //concatenate column data (w/ deep copy to prevent side effects) ret._coldata = (Array[]) ArrayUtils.addAll(_coldata, that._coldata); - for( int i=0; i<ret._coldata.length; i++ ) + for( int i=0; i<ret.getNumColumns(); i++ ) ret._coldata[i] = ret._coldata[i].clone(); } else //ROW APPEND @@ -984,10 +984,13 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable ret._numRows = _numRows; ret._schema = _schema.clone(); ret._colnames = (_colnames!=null) ? _colnames.clone() : null; + ret._colmeta = new ColumnMetadata[getNumColumns()]; + for( int j=0; j<_schema.length; j++ ) + ret._colmeta[j] = new ColumnMetadata(0); //concatenate data (deep copy first, append second) - ret._coldata = new Array[_coldata.length]; - for( int j=0; j<_coldata.length; j++ ) + ret._coldata = new Array[getNumColumns()]; + for( int j=0; j<getNumColumns(); j++ ) ret._coldata[j] = _coldata[j].clone(); Iterator<Object[]> iter = that.getObjectRowIterator(); while( iter.hasNext() ) http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index 5a1b904..07802dd 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -195,8 +195,6 @@ public abstract class AutomatedTestBase protected static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID; protected static final boolean DEBUG = false; - protected static final boolean VISUALIZE = false; - protected static final boolean RUNNETEZZA = false; protected String fullDMLScriptName; // utilize for both DML and PyDML, should probably be renamed. // protected String fullPYDMLScriptName; @@ -1174,8 +1172,6 @@ public abstract class AutomatedTestBase } } // program-independent parameters - if(VISUALIZE) - args.add("-v"); args.add("-exec"); if(rtplatform == RUNTIME_PLATFORM.HADOOP) args.add("hadoop"); http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java new file mode 100644 index 0000000..7302126 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.frame; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.runtime.util.MapReduceTool; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; +import org.apache.sysml.utils.Statistics; + +public class FrameScalarCastingIntegratedTest extends AutomatedTestBase +{ + private final static String TEST_DIR = "functions/frame/"; + private static final String TEST_CLASS_DIR = TEST_DIR + FrameScalarCastingIntegratedTest.class.getSimpleName() + "/"; + private final static String TEST_NAME = "FrameScalarCast"; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"})); + } + + @Test + public void testFrameStringCP0() { + runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.SINGLE_NODE); + } + + @Test + public void testFrameLongCP0() { + runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.SINGLE_NODE); + } + + @Test + public void testFrameBooleanCP0() { + runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.SINGLE_NODE); + } + + @Test + public void testFrameDoubleCP0() { + runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.SINGLE_NODE); + } + + @Test + public void testFrameStringCP1() { + runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.HYBRID); + } + + @Test + public void testFrameLongCP1() { + runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.HYBRID); + } + + @Test + public void testFrameBooleanCP1() { + runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.HYBRID); + } + + @Test + public void testFrameDoubleCP1() { + runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.HYBRID); + } + + @Test + public void testFrameStringCP2() { + runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.HYBRID_SPARK); + } + + @Test + public void testFrameLongCP2() { + runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.HYBRID_SPARK); + } + + @Test + public void testFrameBooleanCP2() { + runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.HYBRID_SPARK); + } + + @Test + public void testFrameDoubleCP2() { + runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.HYBRID_SPARK); + } + + @Test + public void testFrameStringSP() { + runFrameScalarCastingTest(ValueType.STRING, RUNTIME_PLATFORM.SPARK); + } + + @Test + public void testFrameLongSP() { + runFrameScalarCastingTest(ValueType.INT, RUNTIME_PLATFORM.SPARK); + } + + @Test + public void testFrameBooleanSP() { + runFrameScalarCastingTest(ValueType.BOOLEAN, RUNTIME_PLATFORM.SPARK); + } + + @Test + public void testFrameDoubleSP() { + runFrameScalarCastingTest(ValueType.DOUBLE, RUNTIME_PLATFORM.SPARK); + } + + private void runFrameScalarCastingTest(ValueType vtIn, RUNTIME_PLATFORM et) + { + RUNTIME_PLATFORM platformOld = rtplatform; + rtplatform = et; + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try + { + getAndLoadTestConfiguration(TEST_NAME); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-stats", "-args", input("V"), output("R") }; + + //generate input data + switch( vtIn ) { + case STRING: MapReduceTool.writeStringToHDFS("foo", input("V")); break; + case INT: MapReduceTool.writeIntToHDFS(7, input("V")); break; + case BOOLEAN: MapReduceTool.writeBooleanToHDFS(true, input("V")); break; + case DOUBLE: MapReduceTool.writeDoubleToHDFS(7.3, input("V")); break; + default: throw new RuntimeException("Unsupported type: "+vtIn); + } + MapReduceTool.writeScalarMetaDataFile(input("V")+".mtd", vtIn); + + //run tests + runTest(true, false, null, -1); + + //compare output + Assert.assertEquals(readDMLMatrixFromHDFS("R").get(new CellIndex(1,1)), Double.valueOf(1)); + if( et != RUNTIME_PLATFORM.SPARK ) { + Assert.assertTrue(Statistics.getNoOfCompiledSPInst()==0); + Assert.assertTrue(Statistics.getNoOfExecutedSPInst()==0); + } + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/scripts/functions/frame/FrameScalarCast.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/frame/FrameScalarCast.dml b/src/test/scripts/functions/frame/FrameScalarCast.dml new file mode 100644 index 0000000..25d9bd2 --- /dev/null +++ b/src/test/scripts/functions/frame/FrameScalarCast.dml @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +s1 = read($1); + +F1 = as.frame(s1); +F2 = rbind(F1, F1); +s2 = as.scalar(F2[1,1]) +R = as.matrix(s1==s2); + +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java index 50d7a67..45617ba 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java @@ -41,6 +41,7 @@ import org.junit.runners.Suite; FrameMatrixWriteTest.class, FrameMetaReadWriteTest.class, FrameReadWriteTest.class, + FrameScalarCastingIntegratedTest.class, FrameScalarCastingTest.class, FrameSchemaReadTest.class, FrameSerializationTest.class,
