Repository: incubator-systemml Updated Branches: refs/heads/master 1371ab4cf -> c9516e768
[SYSTEMML-558] FrameBlock indexing and append operations, incl tests This changes adds the following operations to FrameBlock: (1) column append (cbind), (2) row append (rbind), (3) right indexing (slice), and (4) left indexing. Furthermore, this also includes minor extensions of internal array data structure operations, utilities, and tests. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c9516e76 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c9516e76 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c9516e76 Branch: refs/heads/master Commit: c9516e76860276701adc4b26a7a1bb58269d0244 Parents: 1371ab4 Author: Matthias Boehm <[email protected]> Authored: Wed Mar 9 22:42:04 2016 -0800 Committer: Matthias Boehm <[email protected]> Committed: Wed Mar 9 22:42:04 2016 -0800 ---------------------------------------------------------------------- .../sysml/runtime/matrix/data/FrameBlock.java | 209 ++++++++++++++++++- .../sysml/runtime/util/UtilFunctions.java | 33 +++ .../functions/frame/FrameAppendTest.java | 141 +++++++++++++ .../functions/frame/FrameGetSetTest.java | 51 +---- .../functions/frame/FrameIndexingTest.java | 160 ++++++++++++++ .../functions/frame/FrameSerializationTest.java | 37 +--- .../functions/frame/ZPackageSuite.java | 2 + 7 files changed, 548 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index c522c39..d08376e 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -32,6 +32,7 @@ import java.util.List; import org.apache.hadoop.io.Writable; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.DMLRuntimeException; /** * @@ -68,7 +69,7 @@ public class FrameBlock implements Writable, Externalizable } /** - * Get the number of rows of the data frame. + * Get the number of rows of the frame block. * * @return */ @@ -77,7 +78,7 @@ public class FrameBlock implements Writable, Externalizable } /** - * Get the number of columns of the data frame, that is + * Get the number of columns of the frame block, that is * the number of columns defined in the schema. * * @return @@ -87,6 +88,15 @@ public class FrameBlock implements Writable, Externalizable } /** + * Returns the schema of the frame block. + * + * @return + */ + public List<ValueType> getSchema() { + return _schema; + } + + /** * Allocate column data structures if necessary, i.e., if schema specified * but not all column data structures created yet. */ @@ -297,6 +307,154 @@ public class FrameBlock implements Writable, Externalizable } /////// + // indexing and append operations + + /** + * + * @param rhsFrame + * @param rl + * @param ru + * @param cl + * @param cu + * @param ret + * @return + */ + public FrameBlock leftIndexingOperations(FrameBlock rhsFrame, int rl, int ru, int cl, int cu, FrameBlock ret) + throws DMLRuntimeException + { + // check the validity of bounds + if ( rl < 0 || rl >= getNumRows() || ru < rl || ru >= getNumRows() + || cl < 0 || cu >= getNumColumns() || cu < cl || cu >= getNumColumns() ) { + throw new DMLRuntimeException("Invalid values for frame indexing: ["+(rl+1)+":"+(ru+1)+"," + (cl+1)+":"+(cu+1)+"] " + + "must be within frame dimensions ["+getNumRows()+","+getNumColumns()+"]."); + } + if ( (ru-rl+1) < rhsFrame.getNumRows() || (cu-cl+1) < rhsFrame.getNumColumns()) { + throw new DMLRuntimeException("Invalid values for frame indexing: " + + "dimensions of the source frame ["+rhsFrame.getNumRows()+"x" + rhsFrame.getNumColumns() + "] " + + "do not match the shape of the frame specified by indices [" + + (rl+1) +":" + (ru+1) + ", " + (cl+1) + ":" + (cu+1) + "]."); + } + + //allocate output frame (incl deep copy schema) + if( ret == null ) + ret = new FrameBlock(_schema); + else + ret._schema = new ArrayList<ValueType>(_schema); + ret._numRows = _numRows; + + //copy data to output and partial overwrite w/ rhs + for( int j=0; j<getNumColumns(); j++ ) { + Array tmp = _coldata.get(j).clone(); + if( j>=cl && j<=cu ) + tmp.set(rl, ru, rhsFrame._coldata.get(j-cl)); + ret._coldata.add(tmp); + } + + return ret; + } + + /** + * Right indexing operations to slice a subframe out of this frame block. + * Note that the existing column value types are preserved. + * + * @param rl row lower index, inclusive, 0-based + * @param ru row upper index, inclusive, 0-based + * @param cl column lower index, inclusive, 0-based + * @param cu column upper index, inclusive, 0-based + * @param ret + * @return + */ + public FrameBlock sliceOperations(int rl, int ru, int cl, int cu, FrameBlock ret) + throws DMLRuntimeException + { + // check the validity of bounds + if ( rl < 0 || rl >= getNumRows() || ru < rl || ru >= getNumRows() + || cl < 0 || cu >= getNumColumns() || cu < cl || cu >= getNumColumns() ) { + throw new DMLRuntimeException("Invalid values for frame indexing: ["+(rl+1)+":"+(ru+1)+"," + (cl+1)+":"+(cu+1)+"] " + + "must be within frame dimensions ["+getNumRows()+","+getNumColumns()+"]"); + } + + //allocate output frame + if( ret == null ) + ret = new FrameBlock(); + ret._numRows = ru-rl+1; + + //copy output schema + for( int j=cl; j<=cu; j++ ) + ret._schema.add(_schema.get(j)); + + //copy output data + for( int j=cl; j<=cu; j++ ) + ret._coldata.add(_coldata.get(j).slice(rl,ru)); + + return ret; + } + + /** + * Appends the given argument frameblock 'that' to this frameblock by + * creating a deep copy to prevent side effects. For cbind, the frames + * are appended column-wise (same number of rows), while for rbind the + * frames are appended row-wise (same number of columns). + * + * @param that + * @param ret + * @param cbind + * @return + */ + public FrameBlock appendOperations( FrameBlock that, FrameBlock ret, boolean cbind ) + throws DMLRuntimeException + { + if( cbind ) //COLUMN APPEND + { + //sanity check row dimension mismatch + if( getNumRows() != that.getNumRows() ) { + throw new DMLRuntimeException("Incompatible number of rows for cbind: "+ + that.getNumRows()+" (expected: "+getNumRows()+")"); + } + + //allocate output frame + if( ret == null ) + ret = new FrameBlock(); + ret._numRows = _numRows; + + //concatenate schemas (w/ deep copy to prevent side effects) + ret._schema = new ArrayList<ValueType>(_schema); + ret._schema.addAll(that._schema); + + //concatenate column data (w/ deep copy to prevent side effects) + for( Array tmp : _coldata ) + ret._coldata.add(tmp.clone()); + for( Array tmp : that._coldata ) + ret._coldata.add(tmp.clone()); + } + else //ROW APPEND + { + //sanity check column dimension mismatch + if( getNumColumns() != that.getNumColumns() ) { + throw new DMLRuntimeException("Incompatible number of columns for rbind: "+ + that.getNumColumns()+" (expected: "+getNumColumns()+")"); + } + + //allocate output frame (incl deep copy schema) + if( ret == null ) + ret = new FrameBlock(_schema); + else + ret._schema = new ArrayList<ValueType>(_schema); + ret._numRows = _numRows; + + //concatenate data (deep copy first, append second) + for( Array tmp : _coldata ) + ret._coldata.add(tmp.clone()); + Iterator<Object[]> iter = that.getObjectRowIterator(); + while( iter.hasNext() ) + ret.appendRow(iter.next()); + } + + return ret; + } + + + /////// // row iterators (over strings and boxed objects) /** @@ -375,8 +533,11 @@ public class FrameBlock implements Writable, Externalizable } public abstract T get(int index); public abstract void set(int index, T value); + public abstract void set(int rl, int ru, Array value); public abstract void append(String value); public abstract void append(T value); + public abstract Array clone(); + public abstract Array slice(int rl, int ru); } /** @@ -395,22 +556,29 @@ public class FrameBlock implements Writable, Externalizable public void set(int index, String value) { _data[index] = value; } + public void set(int rl, int ru, Array value) { + System.arraycopy(((StringArray)value)._data, 0, _data, rl, ru-rl+1); + } public void append(String value) { if( _data.length <= _size ) _data = Arrays.copyOf(_data, newSize()); _data[_size++] = value; } - @Override public void write(DataOutput out) throws IOException { for( int i=0; i<_size; i++ ) out.writeUTF(_data[i]); } - @Override public void readFields(DataInput in) throws IOException { _size = _data.length; for( int i=0; i<_size; i++ ) _data[i] = in.readUTF(); } + public Array clone() { + return new StringArray(Arrays.copyOf(_data, _size)); + } + public Array slice(int rl, int ru) { + return new StringArray(Arrays.copyOfRange(_data,rl,ru+1)); + } } /** @@ -429,6 +597,9 @@ public class FrameBlock implements Writable, Externalizable public void set(int index, Boolean value) { _data[index] = value; } + public void set(int rl, int ru, Array value) { + System.arraycopy(((BooleanArray)value)._data, 0, _data, rl, ru-rl+1); + } public void append(String value) { append(Boolean.parseBoolean(value)); } @@ -437,17 +608,21 @@ public class FrameBlock implements Writable, Externalizable _data = Arrays.copyOf(_data, newSize()); _data[_size++] = value; } - @Override public void write(DataOutput out) throws IOException { for( int i=0; i<_size; i++ ) out.writeBoolean(_data[i]); } - @Override public void readFields(DataInput in) throws IOException { _size = _data.length; for( int i=0; i<_size; i++ ) _data[i] = in.readBoolean(); } + public Array clone() { + return new BooleanArray(Arrays.copyOf(_data, _size)); + } + public Array slice(int rl, int ru) { + return new BooleanArray(Arrays.copyOfRange(_data,rl,ru+1)); + } } /** @@ -466,6 +641,9 @@ public class FrameBlock implements Writable, Externalizable public void set(int index, Long value) { _data[index] = value; } + public void set(int rl, int ru, Array value) { + System.arraycopy(((LongArray)value)._data, 0, _data, rl, ru-rl+1); + } public void append(String value) { append(Long.parseLong(value)); } @@ -474,17 +652,21 @@ public class FrameBlock implements Writable, Externalizable _data = Arrays.copyOf(_data, newSize()); _data[_size++] = value; } - @Override public void write(DataOutput out) throws IOException { for( int i=0; i<_size; i++ ) out.writeLong(_data[i]); } - @Override public void readFields(DataInput in) throws IOException { _size = _data.length; for( int i=0; i<_size; i++ ) _data[i] = in.readLong(); } + public Array clone() { + return new LongArray(Arrays.copyOf(_data, _size)); + } + public Array slice(int rl, int ru) { + return new LongArray(Arrays.copyOfRange(_data,rl,ru+1)); + } } /** @@ -503,6 +685,9 @@ public class FrameBlock implements Writable, Externalizable public void set(int index, Double value) { _data[index] = value; } + public void set(int rl, int ru, Array value) { + System.arraycopy(((DoubleArray)value)._data, 0, _data, rl, ru-rl+1); + } public void append(String value) { append(Double.parseDouble(value)); } @@ -511,16 +696,20 @@ public class FrameBlock implements Writable, Externalizable _data = Arrays.copyOf(_data, newSize()); _data[_size++] = value; } - @Override public void write(DataOutput out) throws IOException { for( int i=0; i<_size; i++ ) out.writeDouble(_data[i]); } - @Override public void readFields(DataInput in) throws IOException { _size = _data.length; for( int i=0; i<_size; i++ ) _data[i] = in.readDouble(); } + public Array clone() { + return new DoubleArray(Arrays.copyOf(_data, _size)); + } + public Array slice(int rl, int ru) { + return new DoubleArray(Arrays.copyOfRange(_data,rl,ru+1)); + } } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java index 3126c00..4442585 100644 --- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java @@ -19,6 +19,7 @@ package org.apache.sysml.runtime.util; +import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; @@ -282,6 +283,38 @@ public class UtilFunctions else return ((Integer)obj).intValue(); } + + /** + * + * @param vt + * @param in + * @return + */ + public static Object doubleToObject(ValueType vt, double in) { + switch( vt ) { + case STRING: return String.valueOf(in); + case BOOLEAN: return (in!=0); + case INT: return UtilFunctions.toLong(in); + case DOUBLE: return in; + default: throw new RuntimeException("Unsupported value type: "+vt); + } + } + + /** + * + * @param vt + * @param in + * @return + */ + public static double objectToDouble(ValueType vt, Object in) { + switch( vt ) { + case STRING: return Double.parseDouble((String)in); + case BOOLEAN: return ((Boolean)in)?1d:0d; + case INT: return (Long)in; + case DOUBLE: return (Double)in; + default: throw new RuntimeException("Unsupported value type: "+vt); + } + } public static boolean isIntegerNumber( String str ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java new file mode 100644 index 0000000..555cf55 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameAppendTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.frame; + +import java.util.Arrays; +import java.util.List; + +import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction.AppendType; +import org.apache.sysml.runtime.matrix.data.FrameBlock; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class FrameAppendTest extends AutomatedTestBase +{ + private final static int rows = 1593; + private final static ValueType[] schemaStrings = new ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING}; + private final static ValueType[] schemaMixed = new ValueType[]{ValueType.STRING, ValueType.DOUBLE, ValueType.INT, ValueType.BOOLEAN}; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + } + + @Test + public void testFrameStringsStringsCBind() { + runFrameAppendTest(schemaStrings, schemaStrings, AppendType.CBIND); + } + + @Test + public void testFrameStringsStringsRBind() { //note: ncol(A)=ncol(B) + runFrameAppendTest(schemaStrings, schemaStrings, AppendType.RBIND); + } + + @Test + public void testFrameMixedStringsCBind() { + runFrameAppendTest(schemaMixed, schemaStrings, AppendType.CBIND); + } + + @Test + public void testFrameStringsMixedCBind() { + runFrameAppendTest(schemaStrings, schemaMixed, AppendType.CBIND); + } + + @Test + public void testFrameMixedMixedCBind() { + runFrameAppendTest(schemaMixed, schemaMixed, AppendType.CBIND); + } + + @Test + public void testFrameMixedMixedRBind() { //note: ncol(A)=ncol(B) + runFrameAppendTest(schemaMixed, schemaMixed, AppendType.RBIND); + } + + + /** + * + * @param sparseM1 + * @param sparseM2 + * @param instType + */ + private void runFrameAppendTest( ValueType[] schema1, ValueType[] schema2, AppendType atype) + { + try + { + //data generation + double[][] A = getRandomMatrix(rows, schema1.length, -10, 10, 0.9, 2373); + double[][] B = getRandomMatrix(rows, schema2.length, -10, 10, 0.9, 129); + + //init data frame 1 + List<ValueType> lschema1 = Arrays.asList(schema1); + FrameBlock frame1 = new FrameBlock(lschema1); + Object[] row1 = new Object[lschema1.size()]; + for( int i=0; i<rows; i++ ) { + for( int j=0; j<lschema1.size(); j++ ) + A[i][j] = UtilFunctions.objectToDouble(lschema1.get(j), + row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j])); + frame1.appendRow(row1); + } + + //init data frame 2 + List<ValueType> lschema2 = Arrays.asList(schema2); + FrameBlock frame2 = new FrameBlock(lschema2); + Object[] row2 = new Object[lschema2.size()]; + for( int i=0; i<rows; i++ ) { + for( int j=0; j<lschema2.size(); j++ ) + B[i][j] = UtilFunctions.objectToDouble(lschema2.get(j), + row2[j] = UtilFunctions.doubleToObject(lschema2.get(j), B[i][j])); + frame2.appendRow(row2); + } + + + //core append operations matrix blocks + MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); + MatrixBlock mbB = DataConverter.convertToMatrixBlock(B); + MatrixBlock mbC = mbA.appendOperations(mbB, new MatrixBlock(), atype==AppendType.CBIND); + + //core append operations frame blocks + FrameBlock frame3 = frame1.appendOperations(frame2, new FrameBlock(), atype==AppendType.CBIND); + + //check basic meta data + if( frame3.getNumRows() != mbC.getNumRows() ) + Assert.fail("Wrong number of rows: "+frame3.getNumRows()+", expected: "+mbC.getNumRows()); + + //check correct values + List<ValueType> lschema = frame3.getSchema(); + for( int i=0; i<rows; i++ ) + for( int j=0; j<lschema.size(); j++ ) { + double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame3.get(i, j)); + if( tmp != mbC.quickGetValue(i, j) ) + Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+mbC.quickGetValue(i, j)); + } + } + catch(Exception ex) { + ex.printStackTrace(); + throw new RuntimeException(ex); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java index acdb89f..c2b3161 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameGetSetTest.java @@ -104,25 +104,25 @@ public class FrameGetSetTest extends AutomatedTestBase case STRING: String[] tmp1 = new String[rows]; for( int i=0; i<rows; i++ ) - tmp1[i] = (String)toObject(vt, A[i][j]); + tmp1[i] = (String)UtilFunctions.doubleToObject(vt, A[i][j]); frame.appendColumn(tmp1); break; case BOOLEAN: boolean[] tmp2 = new boolean[rows]; for( int i=0; i<rows; i++ ) - A[i][j] = (tmp2[i] = (Boolean)toObject(vt, A[i][j]))?1:0; + A[i][j] = (tmp2[i] = (Boolean)UtilFunctions.doubleToObject(vt, A[i][j]))?1:0; frame.appendColumn(tmp2); break; case INT: long[] tmp3 = new long[rows]; for( int i=0; i<rows; i++ ) - A[i][j] = tmp3[i] = (Long)toObject(vt, A[i][j]); + A[i][j] = tmp3[i] = (Long)UtilFunctions.doubleToObject(vt, A[i][j]); frame.appendColumn(tmp3); break; case DOUBLE: double[] tmp4 = new double[rows]; for( int i=0; i<rows; i++ ) - tmp4[i] = (Double)toObject(vt, A[i][j]); + tmp4[i] = (Double)UtilFunctions.doubleToObject(vt, A[i][j]); frame.appendColumn(tmp4); break; default: @@ -134,7 +134,8 @@ public class FrameGetSetTest extends AutomatedTestBase Object[] row = new Object[lschema.size()]; for( int i=0; i<rows; i++ ) { for( int j=0; j<lschema.size(); j++ ) - A[i][j] = fromObject(lschema.get(j), row[j] = toObject(lschema.get(j), A[i][j])); + A[i][j] = UtilFunctions.objectToDouble(lschema.get(j), + row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j])); frame.appendRow(row); } } @@ -142,8 +143,8 @@ public class FrameGetSetTest extends AutomatedTestBase String[] row = new String[lschema.size()]; for( int i=0; i<rows; i++ ) { for( int j=0; j<lschema.size(); j++ ) { - Object obj = toObject(lschema.get(j), A[i][j]); - A[i][j] = fromObject(lschema.get(j), obj); + Object obj = UtilFunctions.doubleToObject(lschema.get(j), A[i][j]); + A[i][j] = UtilFunctions.objectToDouble(lschema.get(j), obj); row[j] = obj.toString(); } frame.appendRow(row); @@ -153,7 +154,7 @@ public class FrameGetSetTest extends AutomatedTestBase //some updates via set for( int i=7; i<13; i++ ) for( int j=0; j<=2; j++ ) { - frame.set(i, j, toObject(lschema.get(j), (double)i*j)); + frame.set(i, j, UtilFunctions.doubleToObject(lschema.get(j), (double)i*j)); A[i][j] = (double)i*j; } @@ -164,7 +165,7 @@ public class FrameGetSetTest extends AutomatedTestBase //check correct values for( int i=0; i<rows; i++ ) for( int j=0; j<lschema.size(); j++ ) { - double tmp = fromObject(lschema.get(j), frame.get(i, j)); + double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j)); if( tmp != A[i][j] ) Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]); } @@ -174,36 +175,4 @@ public class FrameGetSetTest extends AutomatedTestBase throw new RuntimeException(ex); } } - - /** - * - * @param vt - * @param in - * @return - */ - private Object toObject(ValueType vt, double in) { - switch( vt ) { - case STRING: return String.valueOf(in); - case BOOLEAN: return (in!=0); - case INT: return UtilFunctions.toLong(in); - case DOUBLE: return in; - default: throw new RuntimeException("Unsupported value type: "+vt); - } - } - - /** - * - * @param vt - * @param in - * @return - */ - private double fromObject(ValueType vt, Object in) { - switch( vt ) { - case STRING: return Double.parseDouble((String)in); - case BOOLEAN: return ((Boolean)in)?1d:0d; - case INT: return (Long)in; - case DOUBLE: return (Double)in; - default: throw new RuntimeException("Unsupported value type: "+vt); - } - } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java new file mode 100644 index 0000000..6d0b6c7 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.frame; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.matrix.data.FrameBlock; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +public class FrameIndexingTest extends AutomatedTestBase +{ + private final static int rows = 3345; + private final static int rl = 234; + private final static int ru = 1432; + private final static int cl = 0; + private final static int cu = 2; + + private final static ValueType[] schemaStrings = new ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING}; + private final static ValueType[] schemaMixed = new ValueType[]{ValueType.STRING, ValueType.DOUBLE, ValueType.INT, ValueType.BOOLEAN}; + + private enum IXType { + RIX, + LIX, + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + } + + @Test + public void testFrameStringsRIX() { + runFrameIndexingTest(schemaStrings, IXType.RIX); + } + + @Test + public void testFrameMixedRIX() { + runFrameIndexingTest(schemaMixed, IXType.RIX); + } + + @Test + public void testFrameStringsLIX() { + runFrameIndexingTest(schemaStrings, IXType.LIX); + } + + @Test + public void testFrameMixedLIX() { + runFrameIndexingTest(schemaMixed, IXType.LIX); + } + + + + /** + * + * @param sparseM1 + * @param sparseM2 + * @param instType + */ + private void runFrameIndexingTest( ValueType[] schema, IXType itype) + { + try + { + //data generation + double[][] A = getRandomMatrix(rows, schema.length, -10, 10, 0.9, 2412); + + //init data frame 1 + List<ValueType> lschema1 = Arrays.asList(schema); + FrameBlock frame1 = new FrameBlock(lschema1); + Object[] row1 = new Object[lschema1.size()]; + for( int i=0; i<rows; i++ ) { + for( int j=0; j<lschema1.size(); j++ ) + A[i][j] = UtilFunctions.objectToDouble(lschema1.get(j), + row1[j] = UtilFunctions.doubleToObject(lschema1.get(j), A[i][j])); + frame1.appendRow(row1); + } + + //core indexing operation + MatrixBlock mbC = null; + FrameBlock frame3 = null; + if( itype == IXType.RIX ) + { + //matrix indexing + MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); + mbC = mbA.sliceOperations(rl, ru, cl, cu, new MatrixBlock()); + + //frame indexing + frame3 = frame1.sliceOperations(rl, ru, cl, cu, new FrameBlock()); + } + else if( itype == IXType.LIX ) + { + //data generation + double[][] B = getRandomMatrix(ru-rl+1, cu-cl+1, -10, 10, 0.9, 7); + + //init data frame 2 + List<ValueType> lschema2 = new ArrayList<ValueType>(); + for( int j=cl; j<=cu; j++ ) + lschema2.add(schema[j]); + FrameBlock frame2 = new FrameBlock(lschema2); + Object[] row2 = new Object[lschema2.size()]; + for( int i=0; i<ru-rl+1; i++ ) { + for( int j=0; j<lschema2.size(); j++ ) + B[i][j] = UtilFunctions.objectToDouble(lschema2.get(j), + row2[j] = UtilFunctions.doubleToObject(lschema2.get(j), B[i][j])); + frame2.appendRow(row2); + } + + //matrix indexing + MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); + MatrixBlock mbB = DataConverter.convertToMatrixBlock(B); + mbC = mbA.leftIndexingOperations(mbB, rl, ru, cl, cu, new MatrixBlock(), false); + + //frame indexing + frame3 = frame1.leftIndexingOperations(frame2, rl, ru, cl, cu, new FrameBlock()); + } + + //check basic meta data + if( frame3.getNumRows() != mbC.getNumRows() ) + Assert.fail("Wrong number of rows: "+frame3.getNumRows()+", expected: "+mbC.getNumRows()); + + //check correct values + List<ValueType> lschema = frame3.getSchema(); + for( int i=0; i<ru-rl+1; i++ ) + for( int j=0; j<lschema.size(); j++ ) { + double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame3.get(i, j)); + if( tmp != mbC.quickGetValue(i, j) ) + Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+mbC.quickGetValue(i, j)); + } + } + catch(Exception ex) { + ex.printStackTrace(); + throw new RuntimeException(ex); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java index 2481599..f36d076 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameSerializationTest.java @@ -94,7 +94,8 @@ public class FrameSerializationTest extends AutomatedTestBase Object[] row = new Object[lschema.size()]; for( int i=0; i<rows; i++ ) { for( int j=0; j<lschema.size(); j++ ) - A[i][j] = fromObject(lschema.get(j), row[j] = toObject(lschema.get(j), A[i][j])); + A[i][j] = UtilFunctions.objectToDouble(lschema.get(j), + row[j] = UtilFunctions.doubleToObject(lschema.get(j), A[i][j])); frame.appendRow(row); } @@ -130,7 +131,7 @@ public class FrameSerializationTest extends AutomatedTestBase //check correct values for( int i=0; i<rows; i++ ) for( int j=0; j<lschema.size(); j++ ) { - double tmp = fromObject(lschema.get(j), frame.get(i, j)); + double tmp = UtilFunctions.objectToDouble(lschema.get(j), frame.get(i, j)); if( tmp != A[i][j] ) Assert.fail("Wrong get value for cell ("+i+","+j+"): "+tmp+", expected: "+A[i][j]); } @@ -140,36 +141,4 @@ public class FrameSerializationTest extends AutomatedTestBase throw new RuntimeException(ex); } } - - /** - * - * @param vt - * @param in - * @return - */ - private Object toObject(ValueType vt, double in) { - switch( vt ) { - case STRING: return String.valueOf(in); - case BOOLEAN: return (in!=0); - case INT: return UtilFunctions.toLong(in); - case DOUBLE: return in; - default: throw new RuntimeException("Unsupported value type: "+vt); - } - } - - /** - * - * @param vt - * @param in - * @return - */ - private double fromObject(ValueType vt, Object in) { - switch( vt ) { - case STRING: return Double.parseDouble((String)in); - case BOOLEAN: return ((Boolean)in)?1d:0d; - case INT: return (Long)in; - case DOUBLE: return (Double)in; - default: throw new RuntimeException("Unsupported value type: "+vt); - } - } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c9516e76/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java index 33b026b..f2afef9 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/frame/ZPackageSuite.java @@ -26,7 +26,9 @@ import org.junit.runners.Suite; * won't run two of them at once. */ @RunWith(Suite.class) @Suite.SuiteClasses({ + FrameAppendTest.class, FrameGetSetTest.class, + FrameIndexingTest.class, FrameSerializationTest.class, })
