Repository: incubator-systemml Updated Branches: refs/heads/master 3031d340d -> 30dcd64b4
[SYSTEMML-1558] Efficient quantile/centralMoment on compressed matrices This patch adds support for selected order statistics over compressed matrices, specifically quantiles and centralMoment. The core idea is to count the frequency per value without accessing all data (e.g., only OLE segment length fields), and perform the actual statistics computation over the reduced input leveraging the distinct values and their counts as weights. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/30dcd64b Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/30dcd64b Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/30dcd64b Branch: refs/heads/master Commit: 30dcd64b4058033a81b775836b6a915e15d47814 Parents: 3031d34 Author: Matthias Boehm <[email protected]> Authored: Mon Apr 24 15:31:34 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Apr 24 15:31:50 2017 -0700 ---------------------------------------------------------------------- .../sysml/runtime/compress/ColGroupDDC1.java | 19 +- .../sysml/runtime/compress/ColGroupDDC2.java | 19 +- .../sysml/runtime/compress/ColGroupOLE.java | 19 ++ .../sysml/runtime/compress/ColGroupRLE.java | 21 ++ .../sysml/runtime/compress/ColGroupValue.java | 37 +++- .../runtime/compress/CompressedMatrixBlock.java | 39 +++- .../sysml/runtime/matrix/data/MatrixBlock.java | 1 - .../compress/BasicMatrixCentralMomentTest.java | 191 +++++++++++++++++++ .../compress/BasicMatrixQuantileTest.java | 187 ++++++++++++++++++ .../functions/compress/ZPackageSuite.java | 2 + 10 files changed, 517 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java index c9e5c9d..4cd103e 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java +++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC1.java @@ -177,6 +177,19 @@ public class ColGroupDDC1 extends ColGroupDDC c[i] = _values[(_data[i]&0xFF)*ncol+colpos]; target.recomputeNonZeros(); } + + @Override + public int[] getCounts() { + final int nrow = getNumRows(); + final int numVals = getNumValues(); + + int[] counts = new int[numVals]; + for( int i=0; i<nrow; i++ ) { + counts[_data[i]&0xFF] ++; + } + + return counts; + } @Override protected void countNonZerosPerRow(int[] rnnz, int rl, int ru) { @@ -272,15 +285,11 @@ public class ColGroupDDC1 extends ColGroupDDC @Override protected void computeSum(MatrixBlock result, KahanFunction kplus) { - final int nrow = getNumRows(); final int ncol = getNumCols(); final int numVals = getNumValues(); //iterative over codes and count per code (guaranteed <=255) - int[] counts = new int[numVals]; - for( int i=0; i<nrow; i++ ) { - counts[_data[i]&0xFF] ++; - } + int[] counts = getCounts(); //post-scaling of pre-aggregate with distinct values KahanObject kbuff = new KahanObject(result.quickGetValue(0, 0), result.quickGetValue(0, 1)); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java index 2584a47..f447aaf 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java +++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC2.java @@ -180,6 +180,19 @@ public class ColGroupDDC2 extends ColGroupDDC target.recomputeNonZeros(); } + @Override + public int[] getCounts() { + final int nrow = getNumRows(); + final int numVals = getNumValues(); + + int[] counts = new int[numVals]; + for( int i=0; i<nrow; i++ ) { + counts[_data[i]] ++; + } + + return counts; + } + @Override protected void countNonZerosPerRow(int[] rnnz, int rl, int ru) { final int ncol = getNumCols(); @@ -264,17 +277,13 @@ public class ColGroupDDC2 extends ColGroupDDC @Override protected void computeSum(MatrixBlock result, KahanFunction kplus) { - final int nrow = getNumRows(); final int ncol = getNumCols(); final int numVals = getNumValues(); if( numVals < MAX_TMP_VALS ) { //iterative over codes and count per code - int[] counts = new int[numVals]; - for( int i=0; i<nrow; i++ ) { - counts[_data[i]] ++; - } + int[] counts = getCounts(); //post-scaling of pre-aggregate with distinct values KahanObject kbuff = new KahanObject(result.quickGetValue(0, 0), result.quickGetValue(0, 1)); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java index b4b9169..fab8917 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java +++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupOLE.java @@ -229,6 +229,25 @@ public class ColGroupOLE extends ColGroupOffset target.recomputeNonZeros(); } + @Override + public int[] getCounts() { + final int numVals = getNumValues(); + + int[] counts = new int[numVals]; + for (int k = 0; k < numVals; k++) { + int boff = _ptr[k]; + int blen = len(k); + + //iterate over bitmap blocks and count partial lengths + int count = 0; + for (int bix=0; bix < blen; bix+=_data[boff+bix]+1) + count += _data[boff+bix]; + counts[k] = count; + } + + return counts; + } + @Override public ColGroup scalarOperation(ScalarOperator op) throws DMLRuntimeException http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java index b9fe5d3..cd658b0 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java +++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupRLE.java @@ -226,6 +226,27 @@ public class ColGroupRLE extends ColGroupOffset target.recomputeNonZeros(); } + @Override + public int[] getCounts() { + final int numVals = getNumValues(); + + int[] counts = new int[numVals]; + for (int k = 0; k < numVals; k++) { + int boff = _ptr[k]; + int blen = len(k); + int curRunEnd = 0; + int count = 0; + for (int bix = 0; bix < blen; bix+=2) { + int curRunStartOff = curRunEnd + _data[boff+bix]; + curRunEnd = curRunStartOff + _data[boff+bix+1]; + count += curRunEnd-curRunStartOff; + } + counts[k] = count; + } + + return counts; + } + @Override public void rightMultByVector(MatrixBlock vector, MatrixBlock result, int rl, int ru) throws DMLRuntimeException http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java index a374137..8bfc019 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java +++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupValue.java @@ -115,7 +115,7 @@ public abstract class ColGroupValue extends ColGroup } /** - * Obtain number of distrinct sets of values associated with the bitmaps in this column group. + * Obtain number of distinct sets of values associated with the bitmaps in this column group. * * @return the number of distinct sets of values associated with the bitmaps * in this column group @@ -128,6 +128,41 @@ public abstract class ColGroupValue extends ColGroup return _values; } + public MatrixBlock getValuesAsBlock() { + boolean containsZeros = (this instanceof ColGroupOffset) ? + ((ColGroupOffset)this)._zeros : false; + int rlen = containsZeros ? _values.length+1 : _values.length; + MatrixBlock ret = new MatrixBlock(rlen, 1, false); + for( int i=0; i<_values.length; i++ ) + ret.quickSetValue(i, 0, _values[i]); + return ret; + } + + public abstract int[] getCounts(); + + public int[] getCounts(boolean inclZeros) { + int[] counts = getCounts(); + if( inclZeros && this instanceof ColGroupOffset ) { + counts = Arrays.copyOf(counts, counts.length+1); + int sum = 0; + for( int i=0; i<counts.length; i++ ) + sum += counts[i]; + counts[counts.length-1] = getNumRows()-sum; + } + return counts; + } + + public MatrixBlock getCountsAsBlock() { + return getCountsAsBlock(getCounts()); + } + + public static MatrixBlock getCountsAsBlock(int[] counts) { + MatrixBlock ret = new MatrixBlock(counts.length, 1, false); + for( int i=0; i<counts.length; i++ ) + ret.quickSetValue(i, 0, counts[i]); + return ret; + } + protected int containsAllZeroValue() { int numVals = getNumValues(); int numCols = getNumCols(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java index fc59065..003f31a 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java @@ -89,6 +89,7 @@ import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.matrix.operators.ScalarOperator; import org.apache.sysml.runtime.matrix.operators.UnaryOperator; import org.apache.sysml.runtime.util.IndexRange; +import org.apache.sysml.runtime.util.SortUtils; /** * Experimental version of MatrixBlock that allows a compressed internal @@ -1901,17 +1902,29 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable @Override public CM_COV_Object cmOperations(CMOperator op) throws DMLRuntimeException { printDecompressWarning("cmOperations"); - MatrixBlock tmp = isCompressed() ? decompress() : this; - return tmp.cmOperations(op); + if( !isCompressed() || isEmptyBlock() ) + return super.cmOperations(op); + ColGroup grp = _colGroups.get(0); + if( grp instanceof ColGroupUncompressed ) + return ((ColGroupUncompressed)grp).getData().cmOperations(op); + + ColGroupValue grpVal = (ColGroupValue)grp; + MatrixBlock vals = grpVal.getValuesAsBlock(); + MatrixBlock counts = ColGroupValue.getCountsAsBlock(grpVal.getCounts(true)); + return vals.cmOperations(op, counts); } @Override public CM_COV_Object cmOperations(CMOperator op, MatrixBlock weights) throws DMLRuntimeException { printDecompressWarning("cmOperations"); - MatrixBlock left = isCompressed() ? decompress() : this; MatrixBlock right = getUncompressed(weights); - return left.cmOperations(op, right); + if( !isCompressed() || isEmptyBlock() ) + return super.cmOperations(op, right); + ColGroup grp = _colGroups.get(0); + if( grp instanceof ColGroupUncompressed ) + return ((ColGroupUncompressed)grp).getData().cmOperations(op); + return decompress().cmOperations(op, right); } @Override @@ -1937,9 +1950,23 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable public MatrixValue sortOperations(MatrixValue weights, MatrixValue result) throws DMLRuntimeException { printDecompressWarning("sortOperations"); - MatrixBlock left = isCompressed() ? decompress() : this; MatrixBlock right = getUncompressed(weights); - return left.sortOperations(right, result); + if( !isCompressed() ) + return super.sortOperations(right, result); + ColGroup grp = _colGroups.get(0); + if( grp instanceof ColGroupUncompressed ) + return ((ColGroupUncompressed)grp).getData().sortOperations(right, result); + + if( right == null ) { + ColGroupValue grpVal = (ColGroupValue)grp; + MatrixBlock vals = grpVal.getValuesAsBlock(); + int[] counts = grpVal.getCounts(true); + SortUtils.sortByValue(0, vals.getNumRows(), vals.getDenseBlock(), counts); + MatrixBlock counts2 = ColGroupValue.getCountsAsBlock(counts); + return vals.sortOperations(counts2, result); + } + else + return decompress().sortOperations(right, result); } @Override http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index f5d0a45..c458b07 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -4821,7 +4821,6 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab t += quickGetValue(i,1); } while(t<pos && i < getNumRows()); - //System.out.println("values: " + quickGetValue(i,0) + "," + quickGetValue(i,1) + " -- " + quickGetValue(i+1,0) + "," + quickGetValue(i+1,1)); if ( quickGetValue(i,1) != 0 ) { // i^th value is present in the data set, simply return it if ( average ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java new file mode 100644 index 0000000..b3dd43b --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixCentralMomentTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.compress; + +import org.apache.sysml.runtime.compress.CompressedMatrixBlock; +import org.apache.sysml.runtime.functionobjects.CM; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.matrix.operators.CMOperator; +import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; +import org.junit.Test; + +/** + * + */ +public class BasicMatrixCentralMomentTest extends AutomatedTestBase +{ + private static final int rows = 2071; + private static final int cols = 1; + private static final double sparsity1 = 0.9; + private static final double sparsity2 = 0.1; + private static final double sparsity3 = 0.0; + + public enum SparsityType { + DENSE, + SPARSE, + EMPTY, + } + + public enum ValueType { + RAND, //UC + CONST, //RLE + RAND_ROUND_OLE, //OLE + RAND_ROUND_DDC, //RLE + } + + @Override + public void setUp() { + + } + + @Test + public void testDenseRandDataCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, true); + } + + @Test + public void testSparseRandDataCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, true); + } + + @Test + public void testEmptyCompression() { + runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, true); + } + + @Test + public void testDenseRoundRandDataOLECompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, true); + } + + @Test + public void testSparseRoundRandDataOLECompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, true); + } + + @Test + public void testDenseRoundRandDataDDCCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_DDC, true); + } + + @Test + public void testSparseRoundRandDataDDCCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, true); + } + + @Test + public void testDenseConstantDataCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, true); + } + + @Test + public void testSparseConstDataCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, true); + } + + @Test + public void testDenseRandDataNoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, false); + } + + @Test + public void testSparseRandDataNoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, false); + } + + @Test + public void testEmptyNoCompression() { + runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, false); + } + + @Test + public void testDenseRoundRandDataOLENoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, false); + } + + @Test + public void testSparseRoundRandDataOLENoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, false); + } + + @Test + public void testDenseConstDataNoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, false); + } + + @Test + public void testSparseConstDataNoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, false); + } + + + /** + * + * @param mb + */ + private void runMatrixAppendTest(SparsityType sptype, ValueType vtype, boolean compress) + { + try + { + //prepare sparsity for input data + double sparsity = -1; + switch( sptype ){ + case DENSE: sparsity = sparsity1; break; + case SPARSE: sparsity = sparsity2; break; + case EMPTY: sparsity = sparsity3; break; + } + + //generate input data + double min = (vtype==ValueType.CONST)? 10 : -10; + double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7); + if( vtype==ValueType.RAND_ROUND_OLE || vtype==ValueType.RAND_ROUND_DDC ) { + CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype==ValueType.RAND_ROUND_DDC); + input = TestUtils.round(input); + } + MatrixBlock mb = DataConverter.convertToMatrixBlock(input); + + //compress given matrix block + CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb); + if( compress ) + cmb.compress(); + + //quantile uncompressed + AggregateOperationTypes opType = CMOperator.getCMAggOpType(2); + CMOperator cm = new CMOperator(CM.getCMFnObject(opType), opType); + + double ret1 = mb.cmOperations(cm).getRequiredResult(opType); + + //quantile compressed + double ret2 = cmb.cmOperations(cm).getRequiredResult(opType); + + //compare result with input + TestUtils.compareScalars(ret1, ret2, 0.0000001); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + CompressedMatrixBlock.ALLOW_DDC_ENCODING = true; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java new file mode 100644 index 0000000..77fc471 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/compress/BasicMatrixQuantileTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.compress; + +import org.apache.sysml.runtime.compress.CompressedMatrixBlock; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.util.DataConverter; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; +import org.junit.Test; + +/** + * + */ +public class BasicMatrixQuantileTest extends AutomatedTestBase +{ + private static final int rows = 2071; + private static final int cols = 1; + private static final double sparsity1 = 0.9; + private static final double sparsity2 = 0.1; + private static final double sparsity3 = 0.0; + + public enum SparsityType { + DENSE, + SPARSE, + EMPTY, + } + + public enum ValueType { + RAND, //UC + CONST, //RLE + RAND_ROUND_OLE, //OLE + RAND_ROUND_DDC, //RLE + } + + @Override + public void setUp() { + + } + + @Test + public void testDenseRandDataCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, true); + } + + @Test + public void testSparseRandDataCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, true); + } + + @Test + public void testEmptyCompression() { + runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, true); + } + + @Test + public void testDenseRoundRandDataOLECompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, true); + } + + @Test + public void testSparseRoundRandDataOLECompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, true); + } + + @Test + public void testDenseRoundRandDataDDCCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_DDC, true); + } + + @Test + public void testSparseRoundRandDataDDCCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, true); + } + + @Test + public void testDenseConstantDataCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, true); + } + + @Test + public void testSparseConstDataCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, true); + } + + @Test + public void testDenseRandDataNoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND, false); + } + + @Test + public void testSparseRandDataNoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND, false); + } + + @Test + public void testEmptyNoCompression() { + runMatrixAppendTest(SparsityType.EMPTY, ValueType.RAND, false); + } + + @Test + public void testDenseRoundRandDataOLENoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.RAND_ROUND_OLE, false); + } + + @Test + public void testSparseRoundRandDataOLENoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, false); + } + + @Test + public void testDenseConstDataNoCompression() { + runMatrixAppendTest(SparsityType.DENSE, ValueType.CONST, false); + } + + @Test + public void testSparseConstDataNoCompression() { + runMatrixAppendTest(SparsityType.SPARSE, ValueType.CONST, false); + } + + + /** + * + * @param mb + */ + private void runMatrixAppendTest(SparsityType sptype, ValueType vtype, boolean compress) + { + try + { + //prepare sparsity for input data + double sparsity = -1; + switch( sptype ){ + case DENSE: sparsity = sparsity1; break; + case SPARSE: sparsity = sparsity2; break; + case EMPTY: sparsity = sparsity3; break; + } + + //generate input data + double min = (vtype==ValueType.CONST)? 10 : -10; + double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7); + if( vtype==ValueType.RAND_ROUND_OLE || vtype==ValueType.RAND_ROUND_DDC ) { + CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype==ValueType.RAND_ROUND_DDC); + input = TestUtils.round(input); + } + MatrixBlock mb = DataConverter.convertToMatrixBlock(input); + + //compress given matrix block + CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb); + if( compress ) + cmb.compress(); + + //quantile uncompressed + MatrixBlock tmp1 = (MatrixBlock) mb.sortOperations(null, new MatrixBlock()); + double ret1 = tmp1.pickValue(0.95); + + //quantile compressed + MatrixBlock tmp2 = (MatrixBlock) cmb.sortOperations(null, new MatrixBlock()); + double ret2 = tmp2.pickValue(0.95); + + //compare result with input + TestUtils.compareScalars(ret1, ret2, 0.0000001); + } + catch(Exception ex) { + throw new RuntimeException(ex); + } + finally { + CompressedMatrixBlock.ALLOW_DDC_ENCODING = true; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30dcd64b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java index 52270e9..4563465 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/compress/ZPackageSuite.java @@ -29,7 +29,9 @@ import org.junit.runners.Suite; BasicCompressionTest.class, BasicGetValueTest.class, BasicMatrixAppendTest.class, + BasicMatrixCentralMomentTest.class, BasicMatrixMultChainTest.class, + BasicMatrixQuantileTest.class, BasicMatrixTransposeSelfMultTest.class, BasicMatrixVectorMultTest.class, BasicScalarOperationsSparseUnsafeTest.class,
