This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 3f0a6f8b03 [SYSTEMDS-3484] Frame Append Optimizations
3f0a6f8b03 is described below
commit 3f0a6f8b03d1b164bf0338a10cd4741fea8bec45
Author: baunsgaard <[email protected]>
AuthorDate: Thu Jan 5 00:57:53 2023 +0100
[SYSTEMDS-3484] Frame Append Optimizations
This commit optimize the frame append to allow for quick append operations
of Frames, the observed speedup is 236x faster append of 64kx2k
Frames append to itself, going going ~10 sec operations to ~0.05 sec
operations.
Also included is minor additional tests, to improve coverage to 97% in
Arrays.
Closes #1759
---
src/main/java/org/apache/sysds/common/Types.java | 72 ++++++
.../sysds/runtime/frame/data/FrameBlock.java | 52 -----
.../sysds/runtime/frame/data/columns/Array.java | 13 +-
.../runtime/frame/data/columns/ArrayFactory.java | 27 ++-
.../runtime/frame/data/columns/BitSetArray.java | 13 +-
.../runtime/frame/data/columns/BooleanArray.java | 33 ++-
.../runtime/frame/data/columns/DoubleArray.java | 14 +-
.../runtime/frame/data/columns/FloatArray.java | 15 +-
.../runtime/frame/data/columns/IntegerArray.java | 14 +-
.../runtime/frame/data/columns/LongArray.java | 14 +-
.../runtime/frame/data/columns/StringArray.java | 17 +-
.../runtime/frame/data/lib/FrameLibAppend.java | 50 +++--
src/test/java/org/apache/sysds/test/TestUtils.java | 18 +-
.../test/component/frame/FrameAppendTest.java | 244 +++++++++++++--------
...ameAppendTest.java => FrameAppendTest_Old.java} | 119 +++++-----
.../component/frame/array/FrameArrayTests.java | 59 ++++-
.../primitives/FederatedCastToMatrixTest.java | 2 +-
17 files changed, 514 insertions(+), 262 deletions(-)
diff --git a/src/main/java/org/apache/sysds/common/Types.java
b/src/main/java/org/apache/sysds/common/Types.java
index a136929f72..68cc801a1c 100644
--- a/src/main/java/org/apache/sysds/common/Types.java
+++ b/src/main/java/org/apache/sysds/common/Types.java
@@ -117,9 +117,81 @@ public class Types
throw new DMLRuntimeException("Unknown
value type: "+value);
}
}
+
public static boolean isSameTypeString(ValueType vt1, ValueType
vt2) {
return
vt1.toExternalString().equals(vt2.toExternalString());
}
+
+ /**
+ * Get the highest common type that both ValueTypes can be
contained in
+ *
+ * @param a First ValueType
+ * @param b Second ValueType
+ * @return The common higest type to represent both
+ */
+ public static ValueType getHighestCommonType(ValueType a,
ValueType b){
+ if(a == b)
+ return a;
+ if(b == UNKNOWN)
+ throw new DMLRuntimeException(
+ "Invalid or not implemented support for
comparing valueType of: " + a + " and " + b);
+
+ switch(a){
+ case STRING:
+ return a;
+ case FP64:
+ switch(b){
+ case STRING:
+ return b;
+ default:
+ return a;
+ }
+ case FP32:
+ switch(b){
+ case STRING:
+ case FP64:
+ return b;
+ default:
+ return a;
+ }
+ case INT64:
+ switch(b){
+ case STRING:
+ case FP64:
+ case FP32:
+ return b;
+ default:
+ return a;
+ }
+ case INT32:
+ switch(b){
+ case STRING:
+ case FP64:
+ case FP32:
+ case INT64:
+ return b;
+ default:
+ return a;
+ }
+ case UINT8:
+ switch(b){
+ case STRING:
+ case FP64:
+ case FP32:
+ case INT64:
+ case INT32:
+ return b;
+ default:
+ return a;
+ }
+ case BOOLEAN:
+ return b; // always higher type in b;
+ case UNKNOWN:
+ default:
+ throw new DMLRuntimeException(
+ "Invalid or not implemented
support for comparing valueType of: " + a + " and " + b);
+ }
+ }
}
/**
diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
index b037b3b5cc..4bf3495946 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
@@ -31,8 +31,6 @@ import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
@@ -1034,56 +1032,6 @@ public class FrameBlock implements
CacheBlock<FrameBlock>, Externalizable {
*/
public FrameBlock append(FrameBlock that, boolean cbind) {
return FrameLibAppend.append(this, that, cbind);
-
- // FrameBlock ret = new FrameBlock();
- // if(cbind) // COLUMN APPEND
- // {
- // // sanity check row dimension mismatch
- // if(getNumRows() != that.getNumRows()) {
- // throw new DMLRuntimeException(
- // "Incompatible number of rows for cbind:
" + that.getNumRows() + " (expected: " + getNumRows() + ")");
- // }
-
- // // allocate output frame
- // ret._numRows = _numRows;
-
- // // concatenate schemas (w/ deep copy to prevent side
effects)
- // ret._schema = (ValueType[]) ArrayUtils.addAll(_schema,
that._schema);
- // ret._colnames = (String[])
ArrayUtils.addAll(getColumnNames(), that.getColumnNames());
- // ret._colmeta = (ColumnMetadata[])
ArrayUtils.addAll(_colmeta, that._colmeta);
-
- // // check and enforce unique columns names
- // if(!Arrays.stream(ret._colnames).allMatch(new
HashSet<>()::add))
- // ret._colnames =
createColNames(ret.getNumColumns());
-
- // // concatenate column data (w/ shallow copy which is
safe due to copy on write semantics)
- // ret._coldata = (Array[]) ArrayUtils.addAll(_coldata,
that._coldata);
- // }
- // else // ROW APPEND
- // {
- // // sanity check column dimension mismatch
- // if(getNumColumns() != that.getNumColumns()) {
- // throw new DMLRuntimeException("Incompatible
number of columns for rbind: " + that.getNumColumns()
- // + " (expected: " + getNumColumns() +
")");
- // }
- // ret._numRows = _numRows; // note set to previous since
each row is appended on.
- // ret._schema = _schema.clone();
- // ret._colnames = (_colnames != null) ? _colnames.clone()
: null;
- // ret._colmeta = new ColumnMetadata[getNumColumns()];
- // for(int j = 0; j < _schema.length; j++)
- // ret._colmeta[j] = new ColumnMetadata();
-
- // // concatenate data (deep copy first, append second)
- // ret._coldata = new Array[getNumColumns()];
- // for(int j = 0; j < getNumColumns(); j++)
- // ret._coldata[j] = _coldata[j].clone();
- // Iterator<Object[]> iter =
IteratorFactory.getObjectRowIterator(that, _schema);
- // while(iter.hasNext())
- // ret.appendRow(iter.next());
- // }
-
- // ret._msize = -1;
- // return ret;
}
public FrameBlock copy() {
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java
index c2f892f35f..0bdd8453fc 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java
@@ -173,7 +173,7 @@ public abstract class Array<T> implements Writable {
* @param value array of other type
*/
public final void setFromOtherTypeNz(Array<?> value) {
- setFromOtherTypeNz(0, value.size(), value);
+ setFromOtherTypeNz(0, value.size()-1, value);
}
/**
@@ -201,6 +201,17 @@ public abstract class Array<T> implements Writable {
*/
public abstract void append(T value);
+ /**
+ * append other array, if the other array is fitting in current
allocated size use that allocated size, otherwise
+ * allocate new array to combine the other with this.
+ *
+ * This method should use the set range function, and should be
preferred over the append single values.
+ *
+ * @param other The other array of same type to append to this.
+ * @return The combined arrays.
+ */
+ public abstract Array<T> append(Array<T> other);
+
/**
* Slice out the sub range and return new array with the specified type.
*
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java
index c8414fb3cd..c98c9eb11f 100644
---
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java
+++
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java
@@ -88,10 +88,10 @@ public interface ArrayFactory {
}
}
- public static Array<?> allocate(ValueType v, int nRow, String val){
+ public static Array<?> allocate(ValueType v, int nRow, String val) {
Array<?> a = allocate(v, nRow);
a.fill(val);
- return a;
+ return a;
}
public static Array<?> allocate(ValueType v, int nRow) {
@@ -146,4 +146,27 @@ public interface ArrayFactory {
arr.readFields(in);
return arr;
}
+
+ /**
+ * append arrays to each other, and cast to highest common type if
different types.
+ *
+ * @param <C> The type to return, java automatically make this Object,
and this is fine.
+ * @param a The first array to append to (potentially modifying this
a if applicable)
+ * @param b THe array to append to a, (not getting modified).
+ * @return A array containing the concatenation of the two.
+ */
+ @SuppressWarnings("unchecked")
+ public static <C> Array<C> append(Array<?> a, Array<?> b) {
+
+ // get common highest datatype.
+ final ValueType ta = a.getValueType();
+ final ValueType tb = b.getValueType();
+ final ValueType tc = ValueType.getHighestCommonType(ta, tb);
+
+ Array<C> ac = (Array<C>) (ta != tc ? a.changeType(tc) : a);
+ Array<C> bc = (Array<C>) (tb != tc ? b.changeType(tc) : b);
+
+ return ac.append(bc);
+ }
+
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java
index 785c39ce8b..69fa75a6e0 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java
@@ -236,6 +236,15 @@ public class BitSetArray extends Array<Boolean> {
append(BooleanArray.parseBoolean(value));
}
+ @Override
+ public BitSetArray append(Array<Boolean> other) {
+ final int endSize = this._size + other.size();
+ final BitSetArray retBS = new BitSetArray(endSize);
+ retBS.set(0, this._size - 1, this, 0);
+ retBS.set(this._size, endSize - 1, other,0);
+ return retBS;
+ }
+
@Override
public void append(Boolean value) {
if(_data.length * 64 < _size + 1)
@@ -447,8 +456,8 @@ public class BitSetArray extends Array<Boolean> {
}
@Override
- public double getAsDouble(int i){
- return get(i) ? 1.0: 0.0;
+ public double getAsDouble(int i) {
+ return get(i) ? 1.0 : 0.0;
}
public static String longToBits(long l) {
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java
index 72a67421a4..f8a5f61ebc 100644
---
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java
+++
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java
@@ -34,6 +34,11 @@ import org.apache.sysds.utils.MemoryEstimates;
public class BooleanArray extends Array<Boolean> {
protected boolean[] _data;
+ public BooleanArray(int size) {
+ super(size);
+ _data = new boolean[size];
+ }
+
public BooleanArray(boolean[] data) {
super(data.length);
_data = data;
@@ -118,6 +123,23 @@ public class BooleanArray extends Array<Boolean> {
_data[_size++] = (value != null) ? value : false;
}
+ @Override
+ public Array<Boolean> append(Array<Boolean> other) {
+ final int endSize = this._size + other.size();
+ if(other instanceof BooleanArray && endSize <
ArrayFactory.bitSetSwitchPoint) {
+ final boolean[] ret = new boolean[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+ System.arraycopy((boolean[]) other.get(), 0, ret,
this._size, other.size());
+ return new BooleanArray(ret);
+ }
+ else {
+ final BooleanArray retBS = new BooleanArray(endSize);
+ retBS.set(0, this._size - 1, this, 0);
+ retBS.set(this._size, endSize - 1, other, 0);
+ return retBS;
+ }
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.BOOLEAN.ordinal());
@@ -144,8 +166,11 @@ public class BooleanArray extends Array<Boolean> {
@Override
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new boolean[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = false;
_size = size;
}
@@ -249,7 +274,7 @@ public class BooleanArray extends Array<Boolean> {
public void fill(Boolean value) {
for(int i = 0; i < _size; i++)
_data[i] = value;
-
+
}
@Override
@@ -264,8 +289,8 @@ public class BooleanArray extends Array<Boolean> {
}
@Override
- public double getAsDouble(int i){
- return _data[i] ? 1.0: 0.0;
+ public double getAsDouble(int i) {
+ return _data[i] ? 1.0 : 0.0;
}
protected static boolean parseBoolean(String value) {
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java
index d382e93501..a7026bb747 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java
@@ -108,6 +108,15 @@ public class DoubleArray extends Array<Double> {
_data[_size++] = (value != null) ? value : 0d;
}
+ @Override
+ public DoubleArray append(Array<Double> other) {
+ final int endSize = this._size + other.size();
+ final double[] ret = new double[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+ System.arraycopy((double[])other.get(), 0, ret, this._size,
other.size());
+ return new DoubleArray(ret);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.FP64.ordinal());
@@ -134,8 +143,11 @@ public class DoubleArray extends Array<Double> {
@Override
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new double[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = 0;
_size = size;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java
index 839e64d41f..90072a86a9 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java
@@ -107,6 +107,16 @@ public class FloatArray extends Array<Float> {
_data[_size++] = (value != null) ? value : 0f;
}
+ @Override
+ public FloatArray append(Array<Float> other) {
+ final int endSize = this._size + other.size();
+ final float[] ret = new float[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+
+ System.arraycopy((float[])other.get(), 0, ret, this._size,
other.size());
+ return new FloatArray(ret);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.FP32.ordinal());
@@ -133,8 +143,11 @@ public class FloatArray extends Array<Float> {
@Override
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new float[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = 0;
_size = size;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java
index 37f6f2b910..71c6a17051 100644
---
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java
+++
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java
@@ -108,6 +108,15 @@ public class IntegerArray extends Array<Integer> {
_data[_size++] = (value != null) ? value : 0;
}
+ @Override
+ public IntegerArray append(Array<Integer> other) {
+ final int endSize = this._size + other.size();
+ final int[] ret = new int[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+ System.arraycopy((int[])other.get(), 0, ret, this._size,
other.size());
+ return new IntegerArray(ret);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.INT32.ordinal());
@@ -133,8 +142,11 @@ public class IntegerArray extends Array<Integer> {
}
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new int[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = 0;
_size = size;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java
index 36ceda15d8..a705b3a095 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java
@@ -107,6 +107,15 @@ public class LongArray extends Array<Long> {
_data[_size++] = (value != null) ? value : 0L;
}
+ @Override
+ public LongArray append(Array<Long> other) {
+ final int endSize = this._size + other.size();
+ final long[] ret = new long[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+ System.arraycopy((long[])other.get(), 0, ret, this._size,
other.size());
+ return new LongArray(ret);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.INT64.ordinal());
@@ -133,8 +142,11 @@ public class LongArray extends Array<Long> {
@Override
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new long[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = 0;
_size = size;
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java
b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java
index 87201d1758..908509b9f7 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java
@@ -106,6 +106,16 @@ public class StringArray extends Array<String> {
_data[_size++] = value;
}
+ @Override
+ public StringArray append(Array<String> other) {
+ final int endSize = this._size + other.size();
+ final String[] ret = new String[endSize];
+ System.arraycopy(_data, 0, ret, 0, this._size);
+ System.arraycopy((String[]) other.get(), 0, ret, this._size,
other.size());
+ ;
+ return new StringArray(ret);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeByte(FrameArrayType.STRING.ordinal());
@@ -134,8 +144,11 @@ public class StringArray extends Array<String> {
@Override
public void reset(int size) {
- if(_data.length < size)
+ if(_data.length < size || _data.length > 2 * size)
_data = new String[size];
+ else
+ for(int i = 0; i < size; i++)
+ _data[i] = null;
_size = size;
}
@@ -380,7 +393,7 @@ public class StringArray extends Array<String> {
}
@Override
- public double getAsDouble(int i){
+ public double getAsDouble(int i) {
return Double.parseDouble(_data[i]);
}
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
index 9a3995f610..54475b2640 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
@@ -21,15 +21,14 @@ package org.apache.sysds.runtime.frame.data.lib;
import java.util.Arrays;
import java.util.HashSet;
-import java.util.Iterator;
import org.apache.commons.lang.ArrayUtils;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.frame.data.FrameBlock;
import org.apache.sysds.runtime.frame.data.columns.Array;
+import org.apache.sysds.runtime.frame.data.columns.ArrayFactory;
import org.apache.sysds.runtime.frame.data.columns.ColumnMetadata;
-import org.apache.sysds.runtime.frame.data.iterators.IteratorFactory;
public class FrameLibAppend {
@@ -52,10 +51,14 @@ public class FrameLibAppend {
public static FrameBlock appendCbind(FrameBlock a, FrameBlock b) {
final int nRow = a.getNumRows();
- // sanity check row dimension mismatch
- if(nRow != b.getNumRows())
- throw new DMLRuntimeException(
- "Incompatible number of rows for cbind: " +
b.getNumRows() + " (expected: " + nRow + ")");
+ final int nRowB = b.getNumRows();
+
+ if(nRow != nRowB)
+ throw new DMLRuntimeException("Incompatible number of
rows for cbind: " + nRowB + " expected: " + nRow);
+ else if(a.getNumColumns() == 0)
+ return b;
+ else if(b.getNumColumns() == 0)
+ return a;
final ValueType[] _schema = addAll(a.getSchema(),
b.getSchema());
final ColumnMetadata[] _colmeta = addAll(a.getColumnMetadata(),
b.getColumnMetadata());
@@ -71,27 +74,30 @@ public class FrameLibAppend {
public static FrameBlock appendRbind(FrameBlock a, FrameBlock b) {
final int nCol = a.getNumColumns();
- // sanity check column dimension mismatch
- if(nCol != b.getNumColumns()) {
- throw new DMLRuntimeException(
- "Incompatible number of columns for rbind: " +
b.getNumColumns() + " (expected: " + nCol + ")");
- }
+ final int nColB = b.getNumColumns();
+
+ if(nCol != nColB)
+ throw new DMLRuntimeException("Incompatible number of
columns for rbind: " + nColB + " expected: " + nCol);
+ else if(a.getNumRows() == 0)
+ return b;
+ else if(b.getNumRows() == 0)
+ return a;
// ret._schema = a.getSchema().clone();
- String[] _colnames = (a.getColumnNames(false) != null) ?
a.getColumnNames().clone() : null;
- ColumnMetadata[] _colmeta = new
ColumnMetadata[a.getNumColumns()];
+ String[] retColNames = (a.getColumnNames(false) != null) ?
a.getColumnNames().clone() : null;
+ ColumnMetadata[] retColMeta = new
ColumnMetadata[a.getNumColumns()];
for(int j = 0; j < nCol; j++)
- _colmeta[j] = new ColumnMetadata();
+ retColMeta[j] = new ColumnMetadata();
// concatenate data (deep copy first, append second)
- Array<?>[] _coldata = new Array[a.getNumColumns()];
- for(int j = 0; j < a.getNumColumns(); j++)
- _coldata[j] = a.getColumn(j).clone();
- Iterator<Object[]> iter =
IteratorFactory.getObjectRowIterator(b, a.getSchema());
- FrameBlock ret = new FrameBlock(a.getSchema().clone(),
_colnames, _colmeta, _coldata);
- while(iter.hasNext())
- ret.appendRow(iter.next());
- return ret;
+ Array<?>[] retCols = new Array[a.getNumColumns()];
+ ValueType[] retSchema = new ValueType[a.getNumColumns()];
+ for(int j = 0; j < a.getNumColumns(); j++) {
+ retCols[j] = ArrayFactory.append(a.getColumn(j),
b.getColumn(j));
+ retSchema[j] = retCols[j].getValueType();
+ }
+
+ return new FrameBlock(retSchema, retColNames, retColMeta,
retCols);
}
@SuppressWarnings("unchecked")
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java
b/src/test/java/org/apache/sysds/test/TestUtils.java
index 88211197d1..b2b06a231e 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.sysds.common.Types.FileFormat;
import org.apache.sysds.common.Types.ValueType;
+import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.data.DenseBlockFP64;
import org.apache.sysds.runtime.data.SparseBlock;
@@ -600,7 +601,7 @@ public class TestUtils
public static boolean readDMLBoolean(String filePath) {
try {
- Boolean b = null;
+ boolean b = false;
Path outDirectory = new Path(filePath);
FileSystem fs =
IOUtilFunctions.getFileSystem(outDirectory, conf);
String line;
@@ -613,7 +614,7 @@ public class TestUtils
}
}
}
- return b.booleanValue();
+ return b;
} catch (IOException e) {
assertTrue("could not read from file " + filePath,
false);
}
@@ -1584,20 +1585,15 @@ public class TestUtils
}
}
- /**
- *
- * @param vt
- * @param in1
- * @param inR
- * @return
- */
public static int compareToR(ValueType vt, Object in1, Object inR,
double tolerance) {
if(in1 == null && (inR == null ||
(inR.toString().compareTo("NA")==0))) return 0;
else if(in1 == null && vt == ValueType.STRING) return -1;
else if(inR == null) return 1;
-
switch( vt ) {
- case STRING: return
((String)in1).compareTo((String)inR);
+ case STRING:
+ if (in1 == null)
+ throw new DMLRuntimeException("Fail");
+ return ((String)in1).compareTo((String)inR);
case BOOLEAN:
if(in1 == null)
return
Boolean.FALSE.compareTo(((Boolean)inR).booleanValue());
diff --git
a/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
b/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
index e010436bdf..b3348b0f02 100644
--- a/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
+++ b/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,121 +19,177 @@
package org.apache.sysds.test.component.frame;
-import org.junit.Assert;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
import org.apache.sysds.common.Types.ValueType;
+import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.frame.data.FrameBlock;
-import org.apache.sysds.runtime.instructions.cp.AppendCPInstruction.AppendType;
-import org.apache.sysds.runtime.matrix.data.MatrixBlock;
-import org.apache.sysds.runtime.util.DataConverter;
import org.apache.sysds.runtime.util.UtilFunctions;
-import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(value = Parameterized.class)
+public class FrameAppendTest {
+ public FrameBlock f;
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ ArrayList<Object[]> tests = new ArrayList<>();
+
+ try {
+ for(int i = 0; i < 10; i++) {
+ tests.add(new Object[]
{TestUtils.generateRandomFrameBlock(300, 300, i)});
+ tests.add(new Object[]
{TestUtils.generateRandomFrameBlock(100, 10, i)});
+ tests.add(new Object[]
{TestUtils.generateRandomFrameBlock(10, 10, i)});
+ tests.add(new Object[]
{TestUtils.generateRandomFrameBlock(1, 1, i)});
+ tests.add(new Object[]
{TestUtils.generateRandomFrameBlock(1, 10, i)});
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("failed constructing tests");
+ }
+
+ return tests;
+ }
+
+ public FrameAppendTest(FrameBlock f) {
+ this.f = f;
+ }
+
+ @Test
+ public void appendSelfRBind() {
+ FrameBlock ff = append(f, f, false);
+ final int nRow = f.getNumRows();
+ for(int r = 0; r < ff.getNumRows(); r++)
+ for(int c = 0; c < ff.getNumColumns(); c++)
+ assertEquals(ff.get(r, c).toString(), f.get(r %
nRow, c).toString());
+ }
-public class FrameAppendTest extends AutomatedTestBase
-{
- private final static int rows = 1593;
- private final static ValueType[] schemaStrings = new
ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING};
- private final static ValueType[] schemaMixed = new
ValueType[]{ValueType.STRING, ValueType.FP64, ValueType.INT64,
ValueType.BOOLEAN};
-
- @Override
- public void setUp() {
- TestUtils.clearAssertionInformation();
+ @Test
+ public void appendSelfCBind() {
+ FrameBlock ff = append(f, f, true);
+ final int nCol = f.getNumColumns();
+ for(int r = 0; r < ff.getNumRows(); r++)
+ for(int c = 0; c < ff.getNumColumns(); c++)
+ assertEquals(ff.get(r, c).toString(), f.get(r,
c % nCol).toString());
}
@Test
- public void testFrameStringsStringsCBind() {
- runFrameAppendTest(schemaStrings, schemaStrings,
AppendType.CBIND);
+ public void rBindEmptySameScheme() {
+ ValueType[] s = f.getSchema();
+ FrameBlock b = new FrameBlock(s);
+ FrameBlock ff = append(b, f, false);
+ TestUtils.compareFrames(f, ff, true);
}
-
+
@Test
- public void testFrameStringsStringsRBind() {
- runFrameAppendTest(schemaStrings, schemaStrings,
AppendType.RBIND);
+ public void rBindEmpty() {
+ FrameBlock b = new FrameBlock(f.getNumColumns(),
ValueType.STRING);
+ FrameBlock ff = append(b, f, false);
+ TestUtils.compareFrames(f, ff, true);
}
-
+
@Test
- public void testFrameMixedStringsCBind() {
- runFrameAppendTest(schemaMixed, schemaStrings,
AppendType.CBIND);
+ public void rBindEmptyAfter() {
+ FrameBlock b = new FrameBlock(f.getNumColumns(),
ValueType.STRING);
+ FrameBlock ff = append(f, b, false);
+ TestUtils.compareFrames(f, ff, true);
}
-
+
@Test
- public void testFrameStringsMixedCBind() {
- runFrameAppendTest(schemaStrings, schemaMixed,
AppendType.CBIND);
+ public void rBindEmptyAfterSameScheme() {
+ ValueType[] s = f.getSchema();
+ FrameBlock b = new FrameBlock(s);
+ FrameBlock ff = append(f, b, false);
+ TestUtils.compareFrames(f, ff, true);
}
-
+
+ @Test(expected = DMLRuntimeException.class)
+ public void cBindEmpty() {
+ // must have same number of rows.
+ FrameBlock b = new FrameBlock();
+ b.append(f, true);
+ }
+
+ @Test(expected = DMLRuntimeException.class)
+ public void cBindEmptyAfter() {
+ // must have same number of rows.
+ FrameBlock b = new FrameBlock();
+ f.append(b, true);
+ }
+
@Test
- public void testFrameMixedMixedCBind() {
- runFrameAppendTest(schemaMixed, schemaMixed, AppendType.CBIND);
+ public void cBindStringColAfter() {
+ // must have same number of rows.
+ FrameBlock b = new FrameBlock(new ValueType[]
{ValueType.STRING}, "Hi", f.getNumRows());
+ FrameBlock ff = append(f, b, true);
+ for(int r = 0; r < f.getNumRows(); r++) {
+ for(int c = 0; c < f.getNumColumns(); c++)
+ assertEquals(ff.get(r, c).toString(), f.get(r,
c).toString());
+ assertEquals(ff.get(r, f.getNumColumns()).toString(),
"Hi");
+ }
}
-
+
@Test
- public void testFrameMixedMixedRBind() {
- runFrameAppendTest(schemaMixed, schemaMixed, AppendType.RBIND);
+ public void cBindStringCol() {
+ // must have same number of rows.
+ FrameBlock b = new FrameBlock(new ValueType[]
{ValueType.STRING}, "Hi", f.getNumRows());
+ FrameBlock ff = append(b, f, true);
+ for(int r = 0; r < f.getNumRows(); r++) {
+ assertEquals(ff.get(r, 0), "Hi");
+ for(int c = 0; c < f.getNumColumns(); c++)
+ assertEquals(ff.get(r, c + 1).toString(),
f.get(r, c).toString());
+ }
}
-
- /**
- *
- * @param sparseM1
- * @param sparseM2
- * @param instType
- */
- private void runFrameAppendTest( ValueType[] schema1, ValueType[]
schema2, AppendType atype)
- {
- try
- {
- //data generation
- double[][] A = getRandomMatrix(rows, schema1.length,
-10, 10, 0.9, 2373);
- double[][] B = getRandomMatrix(rows, schema2.length,
-10, 10, 0.9, 129);
-
- //init data frame 1
- FrameBlock frame1 = new FrameBlock(schema1);
- Object[] row1 = new Object[schema1.length];
- for( int i=0; i<rows; i++ ) {
- for( int j=0; j<schema1.length; j++ )
- A[i][j] =
UtilFunctions.objectToDouble(schema1[j],
- row1[j] =
UtilFunctions.doubleToObject(schema1[j], A[i][j]));
- frame1.appendRow(row1);
+ @Test
+ public void rBindZeros() {
+ ValueType[] bools = UtilFunctions.nCopies(f.getNumColumns(),
ValueType.BOOLEAN);
+ FrameBlock b = new FrameBlock(bools, "0", 10);
+ FrameBlock ff = append(b, f, false);
+ for(int r = 0; r < 10; r++)
+ for(int c = 0; c < f.getNumColumns(); c++) {
+ String v = ff.get(r, c).toString();
+ assertTrue(v, v.equals("0") || v.equals("0.0")
|| v.equals("false"));
}
-
- //init data frame 2
- FrameBlock frame2 = new FrameBlock(schema2);
- Object[] row2 = new Object[schema2.length];
- for( int i=0; i<rows; i++ ) {
- for( int j=0; j<schema2.length; j++ )
- B[i][j] =
UtilFunctions.objectToDouble(schema2[j],
- row2[j] =
UtilFunctions.doubleToObject(schema2[j], B[i][j]));
- frame2.appendRow(row2);
+ for(int r = 0; r < f.getNumRows(); r++)
+ for(int c = 0; c < f.getNumColumns(); c++)
+ assertEquals(ff.get(r + 10, c).toString(),
f.get(r, c).toString());
+ }
+
+ @Test
+ public void rBindZerosMany() {
+ ValueType[] bools = UtilFunctions.nCopies(f.getNumColumns(),
ValueType.BOOLEAN);
+ FrameBlock b = new FrameBlock(bools, "0", 240);
+ FrameBlock ff = append(b, f, false);
+ for(int r = 0; r < 240; r++)
+ for(int c = 0; c < f.getNumColumns(); c++) {
+ String v = ff.get(r, c).toString();
+ assertTrue(v, v.equals("0") || v.equals("0.0")
|| v.equals("false"));
}
-
-
- //core append operations matrix blocks
- MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
- MatrixBlock mbB = DataConverter.convertToMatrixBlock(B);
- MatrixBlock mbC = mbA.append(mbB, new MatrixBlock(),
atype==AppendType.CBIND);
-
- //core append operations frame blocks
- FrameBlock frame3 = frame1.append(frame2,
atype==AppendType.CBIND);
-
- //check basic meta data
- if( frame3.getNumRows() != mbC.getNumRows() )
- Assert.fail("Wrong number of rows:
"+frame3.getNumRows()+", expected: "+mbC.getNumRows());
-
- //check correct values
- ValueType[] lschema = frame3.getSchema();
- for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.length; j++ ) {
- double tmp =
UtilFunctions.objectToDouble(lschema[j], frame3.get(i, j));
- double tmpm =
Double.isNaN(mbC.quickGetValue(i, j))? 0.0: mbC.quickGetValue(i, j);
- tmp = (Double.isNaN(tmp)? 0.0: tmp);
-
- if( tmp != tmpm )
- Assert.fail("Wrong get value
for cell ("+i+","+j+"): "+tmp+", expected: "+tmpm);
- }
+ for(int r = 0; r < f.getNumRows(); r++)
+ for(int c = 0; c < f.getNumColumns(); c++)
+ assertEquals(ff.get(r + 240, c).toString(),
f.get(r, c).toString());
+ }
+
+ private static FrameBlock append(FrameBlock a, FrameBlock b, boolean
cBind) {
+ try {
+ return a.append(b, cBind);
}
- catch(Exception ex) {
- ex.printStackTrace();
- throw new RuntimeException(ex);
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
}
+ return null;
}
}
diff --git
a/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
b/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest_Old.java
similarity index 57%
copy from
src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
copy to
src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest_Old.java
index e010436bdf..935b034372 100644
--- a/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest.java
+++
b/src/test/java/org/apache/sysds/test/component/frame/FrameAppendTest_Old.java
@@ -19,8 +19,6 @@
package org.apache.sysds.test.component.frame;
-import org.junit.Assert;
-import org.junit.Test;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.runtime.frame.data.FrameBlock;
import org.apache.sysds.runtime.instructions.cp.AppendCPInstruction.AppendType;
@@ -29,107 +27,100 @@ import org.apache.sysds.runtime.util.DataConverter;
import org.apache.sysds.runtime.util.UtilFunctions;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
-public class FrameAppendTest extends AutomatedTestBase
-{
+public class FrameAppendTest_Old extends AutomatedTestBase {
private final static int rows = 1593;
- private final static ValueType[] schemaStrings = new
ValueType[]{ValueType.STRING, ValueType.STRING, ValueType.STRING};
- private final static ValueType[] schemaMixed = new
ValueType[]{ValueType.STRING, ValueType.FP64, ValueType.INT64,
ValueType.BOOLEAN};
-
+ private final static ValueType[] schemaStrings = new ValueType[]
{ValueType.STRING, ValueType.STRING,
+ ValueType.STRING};
+ private final static ValueType[] schemaMixed = new ValueType[]
{ValueType.STRING, ValueType.FP64, ValueType.INT64,
+ ValueType.BOOLEAN};
+
@Override
public void setUp() {
TestUtils.clearAssertionInformation();
}
@Test
- public void testFrameStringsStringsCBind() {
+ public void testFrameStringsStringsCBind() {
runFrameAppendTest(schemaStrings, schemaStrings,
AppendType.CBIND);
}
-
+
@Test
- public void testFrameStringsStringsRBind() {
+ public void testFrameStringsStringsRBind() {
runFrameAppendTest(schemaStrings, schemaStrings,
AppendType.RBIND);
}
-
+
@Test
- public void testFrameMixedStringsCBind() {
+ public void testFrameMixedStringsCBind() {
runFrameAppendTest(schemaMixed, schemaStrings,
AppendType.CBIND);
}
-
+
@Test
- public void testFrameStringsMixedCBind() {
+ public void testFrameStringsMixedCBind() {
runFrameAppendTest(schemaStrings, schemaMixed,
AppendType.CBIND);
}
-
+
@Test
- public void testFrameMixedMixedCBind() {
+ public void testFrameMixedMixedCBind() {
runFrameAppendTest(schemaMixed, schemaMixed, AppendType.CBIND);
}
-
+
@Test
- public void testFrameMixedMixedRBind() {
+ public void testFrameMixedMixedRBind() {
runFrameAppendTest(schemaMixed, schemaMixed, AppendType.RBIND);
}
-
- /**
- *
- * @param sparseM1
- * @param sparseM2
- * @param instType
- */
- private void runFrameAppendTest( ValueType[] schema1, ValueType[]
schema2, AppendType atype)
- {
- try
- {
- //data generation
- double[][] A = getRandomMatrix(rows, schema1.length,
-10, 10, 0.9, 2373);
- double[][] B = getRandomMatrix(rows, schema2.length,
-10, 10, 0.9, 129);
-
- //init data frame 1
+ private void runFrameAppendTest(ValueType[] schema1, ValueType[]
schema2, AppendType atype) {
+ try {
+ // data generation
+ double[][] A = getRandomMatrix(rows, schema1.length,
-10, 10, 0.9, 2373);
+ double[][] B = getRandomMatrix(rows, schema2.length,
-10, 10, 0.9, 129);
+
+ // init data frame 1
FrameBlock frame1 = new FrameBlock(schema1);
Object[] row1 = new Object[schema1.length];
- for( int i=0; i<rows; i++ ) {
- for( int j=0; j<schema1.length; j++ )
- A[i][j] =
UtilFunctions.objectToDouble(schema1[j],
- row1[j] =
UtilFunctions.doubleToObject(schema1[j], A[i][j]));
+ for(int i = 0; i < rows; i++) {
+ for(int j = 0; j < schema1.length; j++)
+ A[i][j] =
UtilFunctions.objectToDouble(schema1[j],
+ row1[j] =
UtilFunctions.doubleToObject(schema1[j], A[i][j]));
frame1.appendRow(row1);
}
-
- //init data frame 2
+
+ // init data frame 2
FrameBlock frame2 = new FrameBlock(schema2);
Object[] row2 = new Object[schema2.length];
- for( int i=0; i<rows; i++ ) {
- for( int j=0; j<schema2.length; j++ )
- B[i][j] =
UtilFunctions.objectToDouble(schema2[j],
- row2[j] =
UtilFunctions.doubleToObject(schema2[j], B[i][j]));
+ for(int i = 0; i < rows; i++) {
+ for(int j = 0; j < schema2.length; j++)
+ B[i][j] =
UtilFunctions.objectToDouble(schema2[j],
+ row2[j] =
UtilFunctions.doubleToObject(schema2[j], B[i][j]));
frame2.appendRow(row2);
}
-
-
- //core append operations matrix blocks
+
+ // core append operations matrix blocks
MatrixBlock mbA = DataConverter.convertToMatrixBlock(A);
MatrixBlock mbB = DataConverter.convertToMatrixBlock(B);
- MatrixBlock mbC = mbA.append(mbB, new MatrixBlock(),
atype==AppendType.CBIND);
-
- //core append operations frame blocks
- FrameBlock frame3 = frame1.append(frame2,
atype==AppendType.CBIND);
-
- //check basic meta data
- if( frame3.getNumRows() != mbC.getNumRows() )
- Assert.fail("Wrong number of rows:
"+frame3.getNumRows()+", expected: "+mbC.getNumRows());
-
- //check correct values
+ MatrixBlock mbC = mbA.append(mbB, new MatrixBlock(),
atype == AppendType.CBIND);
+
+ // core append operations frame blocks
+ FrameBlock frame3 = frame1.append(frame2, atype ==
AppendType.CBIND);
+
+ // check basic meta data
+ if(frame3.getNumRows() != mbC.getNumRows())
+ Assert.fail("Wrong number of rows: " +
frame3.getNumRows() + ", expected: " + mbC.getNumRows());
+
+ // check correct values
ValueType[] lschema = frame3.getSchema();
- for( int i=0; i<rows; i++ )
- for( int j=0; j<lschema.length; j++ ) {
+ for(int i = 0; i < rows; i++)
+ for(int j = 0; j < lschema.length; j++) {
double tmp =
UtilFunctions.objectToDouble(lschema[j], frame3.get(i, j));
- double tmpm =
Double.isNaN(mbC.quickGetValue(i, j))? 0.0: mbC.quickGetValue(i, j);
- tmp = (Double.isNaN(tmp)? 0.0: tmp);
+ double tmpm =
Double.isNaN(mbC.quickGetValue(i, j)) ? 0.0 : mbC.quickGetValue(i, j);
+ tmp = (Double.isNaN(tmp) ? 0.0 : tmp);
- if( tmp != tmpm )
- Assert.fail("Wrong get value
for cell ("+i+","+j+"): "+tmp+", expected: "+tmpm);
- }
+ if(tmp != tmpm)
+ Assert.fail("Wrong get value
for cell (" + i + "," + j + "): " + tmp + ", expected: " + tmpm);
+ }
}
catch(Exception ex) {
ex.printStackTrace();
diff --git
a/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java
b/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java
index 303c55d244..22b8066355 100644
---
a/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java
+++
b/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java
@@ -713,11 +713,11 @@ public class FrameArrayTests {
public void append60Null() {
Array<?> aa = a.clone();
- try{
+ try {
for(int i = 0; i < 60; i++)
aa.append((String) null);
-
+
switch(a.getValueType()) {
case BOOLEAN:
assertEquals((Boolean) aa.get(aa.size()
- 1), false);
@@ -745,7 +745,7 @@ public class FrameArrayTests {
throw new DMLRuntimeException("Invalid
type");
}
}
- catch(Exception e){
+ catch(Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
@@ -790,6 +790,59 @@ public class FrameArrayTests {
compare(aa, a);
}
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSetNzString() {
+ Array<?> aa = a.clone();
+ Array<String> af =
(Array<String>)aa.changeType(ValueType.STRING);
+ try{
+
+ aa.setFromOtherTypeNz(af);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+ compare(aa, a);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSetFromString() {
+ Array<?> aa = a.clone();
+ Array<String> af =
(Array<String>)aa.changeType(ValueType.STRING);
+ try{
+
+ aa.setFromOtherType(0, af.size()-1, af);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+ compare(aa, a);
+ }
+
+ @Test
+ public void resetTestCase() {
+ Array<?> aa = a.clone();
+ aa.reset(10);
+ if(aa.getValueType() == ValueType.STRING){
+ for(int i = 0; i < 10; i++) {
+ assertEquals(null, aa.get(i));
+ }
+ }
+ else{
+
+ String v = aa.get(0).toString();
+ for(int i = 1; i < 10; i++) {
+ assertEquals(v, aa.get(i).toString());
+ }
+ }
+ }
+
protected static void compare(Array<?> a, Array<?> b) {
int size = a.size();
assertTrue(a.size() == b.size());
diff --git
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedCastToMatrixTest.java
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedCastToMatrixTest.java
index 07c16d7d5b..ebe7cab9a0 100644
---
a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedCastToMatrixTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedCastToMatrixTest.java
@@ -147,7 +147,7 @@ public class FederatedCastToMatrixTest extends
AutomatedTestBase {
LOG.debug(fedOut);
fedOut = fedOut.split("SystemDS Statistics:")[0];
- Assert.assertTrue("Equal Printed Output",
out.equals(fedOut));
+ Assert.assertTrue("Equal Printed Output: \n" + out +
"\n" + fedOut, out.equals(fedOut));
Assert.assertTrue("Contains federated Cast to frame",
heavyHittersContainsString("fed_castdtm"));
Assert.assertTrue(heavyHittersContainsString("fed_uak+")); // verify output is
federated
TestUtils.shutdownThreads(t1, t2);
