Repository: systemml
Updated Branches:
  refs/heads/master 9da5eab00 -> 70e5f29e8


[SYSTEMML-1826] Fix scalar-frame casts and frame rbind operations

This patch fixes the missing compiler support for scalar-frame casting
as well as missing meta data handling on frame rbind operations. This
also includes a suite of related testcases and some minor cleanups in
AutomatedTestBase (base class of all tests).


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/70e5f29e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/70e5f29e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/70e5f29e

Branch: refs/heads/master
Commit: 70e5f29e8fff0e698d7abc647efe3f79be38abba
Parents: 9da5eab
Author: Matthias Boehm <mboe...@gmail.com>
Authored: Sun Aug 6 20:45:51 2017 -0700
Committer: Matthias Boehm <mboe...@gmail.com>
Committed: Sun Aug 6 20:46:16 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/sysml/hops/UnaryOp.java     |   6 +-
 .../sysml/runtime/matrix/data/FrameBlock.java   |   9 +-
 .../test/integration/AutomatedTestBase.java     |   4 -
 .../frame/FrameScalarCastingIntegratedTest.java | 171 +++++++++++++++++++
 .../scripts/functions/frame/FrameScalarCast.dml |  30 ++++
 .../functions/frame/ZPackageSuite.java          |   1 +
 6 files changed, 211 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/hops/UnaryOp.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/UnaryOp.java 
b/src/main/java/org/apache/sysml/hops/UnaryOp.java
index 35902b7..2b31247 100644
--- a/src/main/java/org/apache/sysml/hops/UnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/UnaryOp.java
@@ -618,8 +618,7 @@ public class UnaryOp extends Hop implements MultiThreadedHop
                                || _op == OpOp1.CUMMAX  );
        }
 
-       public boolean isCastUnaryOperation() 
-       {
+       public boolean isCastUnaryOperation() {
                return (   _op == OpOp1.CAST_AS_MATRIX
                                || _op == OpOp1.CAST_AS_SCALAR
                                || _op == OpOp1.CAST_AS_FRAME
@@ -695,7 +694,8 @@ public class UnaryOp extends Hop implements MultiThreadedHop
                {
                        //do nothing always known
                }
-               else if( _op == OpOp1.CAST_AS_MATRIX && 
getInput().get(0).getDataType()==DataType.SCALAR )
+               else if( (_op == OpOp1.CAST_AS_MATRIX || _op == 
OpOp1.CAST_AS_FRAME)
+                       && getInput().get(0).getDataType()==DataType.SCALAR )
                {
                        //prevent propagating 0 from scalar (which would be 
interpreted as unknown)
                        setDim1( 1 );

http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java 
b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
index 10b39fe..45ad26c 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java
@@ -967,7 +967,7 @@ public class FrameBlock implements Writable, CacheBlock, 
Externalizable
                        
                        //concatenate column data (w/ deep copy to prevent side 
effects)
                        ret._coldata = (Array[]) ArrayUtils.addAll(_coldata, 
that._coldata);
-                       for( int i=0; i<ret._coldata.length; i++ )
+                       for( int i=0; i<ret.getNumColumns(); i++ )
                                ret._coldata[i] = ret._coldata[i].clone();
                }
                else //ROW APPEND
@@ -984,10 +984,13 @@ public class FrameBlock implements Writable, CacheBlock, 
Externalizable
                        ret._numRows = _numRows;
                        ret._schema = _schema.clone();
                        ret._colnames = (_colnames!=null) ? _colnames.clone() : 
null;
+                       ret._colmeta = new ColumnMetadata[getNumColumns()];
+                       for( int j=0; j<_schema.length; j++ )
+                               ret._colmeta[j] = new ColumnMetadata(0);
                        
                        //concatenate data (deep copy first, append second)
-                       ret._coldata = new Array[_coldata.length];
-                       for( int j=0; j<_coldata.length; j++ )
+                       ret._coldata = new Array[getNumColumns()];
+                       for( int j=0; j<getNumColumns(); j++ )
                                ret._coldata[j] = _coldata[j].clone();
                        Iterator<Object[]> iter = that.getObjectRowIterator();
                        while( iter.hasNext() )

http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
----------------------------------------------------------------------
diff --git 
a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java 
b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
index 5a1b904..07802dd 100644
--- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
+++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java
@@ -195,8 +195,6 @@ public abstract class AutomatedTestBase
        protected static RUNTIME_PLATFORM rtplatform = RUNTIME_PLATFORM.HYBRID;
 
        protected static final boolean DEBUG = false;
-       protected static final boolean VISUALIZE = false;
-       protected static final boolean RUNNETEZZA = false;
 
        protected String fullDMLScriptName; // utilize for both DML and PyDML, 
should probably be renamed.
        // protected String fullPYDMLScriptName;
@@ -1174,8 +1172,6 @@ public abstract class AutomatedTestBase
                        }
                }
                // program-independent parameters
-               if(VISUALIZE)
-                       args.add("-v");
                args.add("-exec");
                if(rtplatform == RUNTIME_PLATFORM.HADOOP)
                        args.add("hadoop");

http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
----------------------------------------------------------------------
diff --git 
a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
 
b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
new file mode 100644
index 0000000..7302126
--- /dev/null
+++ 
b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameScalarCastingIntegratedTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysml.test.integration.functions.frame;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.sysml.api.DMLScript;
+import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM;
+import org.apache.sysml.parser.Expression.ValueType;
+import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex;
+import org.apache.sysml.runtime.util.MapReduceTool;
+import org.apache.sysml.test.integration.AutomatedTestBase;
+import org.apache.sysml.test.integration.TestConfiguration;
+import org.apache.sysml.test.utils.TestUtils;
+import org.apache.sysml.utils.Statistics;
+
+public class FrameScalarCastingIntegratedTest extends AutomatedTestBase
+{
+       private final static String TEST_DIR = "functions/frame/";
+       private static final String TEST_CLASS_DIR = TEST_DIR + 
FrameScalarCastingIntegratedTest.class.getSimpleName() + "/";
+       private final static String TEST_NAME = "FrameScalarCast";
+               
+       @Override
+       public void setUp() {
+               TestUtils.clearAssertionInformation();
+               addTestConfiguration(TEST_NAME, new 
TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"R"}));
+       }
+       
+       @Test
+       public void testFrameStringCP0() { 
+               runFrameScalarCastingTest(ValueType.STRING, 
RUNTIME_PLATFORM.SINGLE_NODE); 
+       }
+       
+       @Test
+       public void testFrameLongCP0() { 
+               runFrameScalarCastingTest(ValueType.INT, 
RUNTIME_PLATFORM.SINGLE_NODE); 
+       }
+       
+       @Test
+       public void testFrameBooleanCP0() { 
+               runFrameScalarCastingTest(ValueType.BOOLEAN, 
RUNTIME_PLATFORM.SINGLE_NODE); 
+       }
+       
+       @Test
+       public void testFrameDoubleCP0() { 
+               runFrameScalarCastingTest(ValueType.DOUBLE, 
RUNTIME_PLATFORM.SINGLE_NODE); 
+       }
+       
+       @Test
+       public void testFrameStringCP1() { 
+               runFrameScalarCastingTest(ValueType.STRING, 
RUNTIME_PLATFORM.HYBRID); 
+       }
+       
+       @Test
+       public void testFrameLongCP1() { 
+               runFrameScalarCastingTest(ValueType.INT, 
RUNTIME_PLATFORM.HYBRID); 
+       }
+       
+       @Test
+       public void testFrameBooleanCP1() { 
+               runFrameScalarCastingTest(ValueType.BOOLEAN, 
RUNTIME_PLATFORM.HYBRID); 
+       }
+       
+       @Test
+       public void testFrameDoubleCP1() { 
+               runFrameScalarCastingTest(ValueType.DOUBLE, 
RUNTIME_PLATFORM.HYBRID); 
+       }
+       
+       @Test
+       public void testFrameStringCP2() { 
+               runFrameScalarCastingTest(ValueType.STRING, 
RUNTIME_PLATFORM.HYBRID_SPARK); 
+       }
+       
+       @Test
+       public void testFrameLongCP2() { 
+               runFrameScalarCastingTest(ValueType.INT, 
RUNTIME_PLATFORM.HYBRID_SPARK); 
+       }
+       
+       @Test
+       public void testFrameBooleanCP2() { 
+               runFrameScalarCastingTest(ValueType.BOOLEAN, 
RUNTIME_PLATFORM.HYBRID_SPARK); 
+       }
+       
+       @Test
+       public void testFrameDoubleCP2() { 
+               runFrameScalarCastingTest(ValueType.DOUBLE, 
RUNTIME_PLATFORM.HYBRID_SPARK); 
+       }
+       
+       @Test
+       public void testFrameStringSP() { 
+               runFrameScalarCastingTest(ValueType.STRING, 
RUNTIME_PLATFORM.SPARK); 
+       }
+       
+       @Test
+       public void testFrameLongSP() { 
+               runFrameScalarCastingTest(ValueType.INT, 
RUNTIME_PLATFORM.SPARK); 
+       }
+       
+       @Test
+       public void testFrameBooleanSP() { 
+               runFrameScalarCastingTest(ValueType.BOOLEAN, 
RUNTIME_PLATFORM.SPARK); 
+       }
+       
+       @Test
+       public void testFrameDoubleSP() { 
+               runFrameScalarCastingTest(ValueType.DOUBLE, 
RUNTIME_PLATFORM.SPARK); 
+       }
+       
+       private void runFrameScalarCastingTest(ValueType vtIn, RUNTIME_PLATFORM 
et) 
+       {
+               RUNTIME_PLATFORM platformOld = rtplatform;
+               rtplatform = et;
+               boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG;
+               if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == 
RUNTIME_PLATFORM.HYBRID_SPARK )
+                       DMLScript.USE_LOCAL_SPARK_CONFIG = true;
+               
+               try
+               {               
+                       getAndLoadTestConfiguration(TEST_NAME);
+                   
+                   String HOME = SCRIPT_DIR + TEST_DIR;
+                       fullDMLScriptName = HOME + TEST_NAME + ".dml";
+                       programArgs = new String[]{"-stats", "-args", 
input("V"), output("R") };
+                       
+                       //generate input data
+                       switch( vtIn ) {
+                               case STRING: 
MapReduceTool.writeStringToHDFS("foo", input("V")); break;
+                               case INT: MapReduceTool.writeIntToHDFS(7, 
input("V")); break;
+                               case BOOLEAN: 
MapReduceTool.writeBooleanToHDFS(true, input("V")); break;
+                               case DOUBLE: 
MapReduceTool.writeDoubleToHDFS(7.3, input("V")); break;
+                               default: throw new 
RuntimeException("Unsupported type: "+vtIn);
+                       }
+                       
MapReduceTool.writeScalarMetaDataFile(input("V")+".mtd", vtIn);
+                       
+                       //run tests
+                       runTest(true, false, null, -1);
+
+                       //compare output 
+                       Assert.assertEquals(readDMLMatrixFromHDFS("R").get(new 
CellIndex(1,1)), Double.valueOf(1));
+                       if( et != RUNTIME_PLATFORM.SPARK ) {
+                               
Assert.assertTrue(Statistics.getNoOfCompiledSPInst()==0);
+                               
Assert.assertTrue(Statistics.getNoOfExecutedSPInst()==0);
+                       }
+               }
+               catch(Exception ex) {
+                       throw new RuntimeException(ex);
+               }
+               finally {
+                       rtplatform = platformOld;
+                       DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld;
+               }
+       }
+}

http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test/scripts/functions/frame/FrameScalarCast.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/frame/FrameScalarCast.dml 
b/src/test/scripts/functions/frame/FrameScalarCast.dml
new file mode 100644
index 0000000..25d9bd2
--- /dev/null
+++ b/src/test/scripts/functions/frame/FrameScalarCast.dml
@@ -0,0 +1,30 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+
+s1 = read($1);
+
+F1 = as.frame(s1);
+F2 = rbind(F1, F1);
+s2 = as.scalar(F2[1,1])
+R = as.matrix(s1==s2);
+
+write(R, $2);

http://git-wip-us.apache.org/repos/asf/systemml/blob/70e5f29e/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
----------------------------------------------------------------------
diff --git 
a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
 
b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
index 50d7a67..45617ba 100644
--- 
a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
+++ 
b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java
@@ -41,6 +41,7 @@ import org.junit.runners.Suite;
        FrameMatrixWriteTest.class,
        FrameMetaReadWriteTest.class,
        FrameReadWriteTest.class,
+       FrameScalarCastingIntegratedTest.class,
        FrameScalarCastingTest.class,
        FrameSchemaReadTest.class,
        FrameSerializationTest.class,

Reply via email to