Repository: systemml Updated Branches: refs/heads/master ff4dbb3ee -> 1e851ef62
[SYSTEMML-2479] Extended bitset estimator (and, or, flip, trans, cbind) Closes #827. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1e851ef6 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1e851ef6 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1e851ef6 Branch: refs/heads/master Commit: 1e851ef62dba0565aab4b72cbadf6a039cb738c0 Parents: ff4dbb3 Author: Johanna Sommer <joha...@mail-sommer.com> Authored: Fri Aug 10 09:59:10 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Sat Aug 11 00:24:48 2018 -0700 ---------------------------------------------------------------------- .../sysml/hops/estim/EstimatorBitsetMM.java | 178 ++++++++++++- .../functions/estim/OpBindChainTest.java | 162 ++++++++++++ .../integration/functions/estim/OpBindTest.java | 27 +- .../functions/estim/OpElemWChainTest.java | 153 +++++++++++ .../functions/estim/OpElemWTest.java | 18 +- .../integration/functions/estim/OpSingle.java | 252 ------------------ .../functions/estim/OpSingleTest.java | 260 +++++++++++++++++++ .../functions/estim/ZPackageSuite.java | 2 +- 8 files changed, 771 insertions(+), 281 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java b/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java index 0bb4e5d..16a8a3a 100644 --- a/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java +++ b/src/main/java/org/apache/sysml/hops/estim/EstimatorBitsetMM.java @@ -81,20 +81,26 @@ public class EstimatorBitsetMM extends SparsityEstimator @Override public double estim(MatrixBlock m, OpCode op) { - return estim(m, null, op); + if( isExactMetadataOp(op) ) + return estimExactMetaData(m.getMatrixCharacteristics(), null, op).getSparsity(); + BitsetMatrix m1Map = new BitsetMatrix1(m); + BitsetMatrix outMap = estimInternal(m1Map, null, op); + return OptimizerUtils.getSparsity(outMap.getNumRows(), + outMap.getNumColumns(), outMap.getNonZeros()); } private BitsetMatrix estimInternal(BitsetMatrix m1Map, BitsetMatrix m2Map, OpCode op) { switch(op) { case MM: return m1Map.matMult(m2Map); + case MULT: return m1Map.and(m2Map); + case PLUS: return m1Map.or(m2Map); case RBIND: return m1Map.rbind(m2Map); + case CBIND: return m1Map.cbind(m2Map); + case NEQZERO: return m1Map; + case EQZERO: return m1Map.flip(); + case TRANS: return m1Map.transpose(); + //TODO implement all as bitset operations in both BitsetMatrix1 and BitsetMatrix2 - case MULT: - case PLUS: - case CBIND: - case TRANS: - case NEQZERO: - case EQZERO: case DIAG: case RESHAPE: default: @@ -125,6 +131,10 @@ public class EstimatorBitsetMM extends SparsityEstimator return _nonZeros; } + public abstract boolean get(int r, int c); + + public abstract void set(int r, int c); + protected void init(MatrixBlock in) { if (in.isEmptyBlock(false)) return; @@ -165,7 +175,28 @@ public class EstimatorBitsetMM extends SparsityEstimator protected abstract long matMultIntern(BitsetMatrix bsb, BitsetMatrix bsc, int rl, int ru); + protected abstract BitsetMatrix and(BitsetMatrix bsb); + + protected abstract BitsetMatrix or(BitsetMatrix bsb); + protected abstract BitsetMatrix rbind(BitsetMatrix bsb); + + protected abstract BitsetMatrix cbind(BitsetMatrix bsb); + + protected abstract BitsetMatrix flip(); + + public BitsetMatrix transpose() { + BitsetMatrix1 ret = new BitsetMatrix1(getNumRows(), getNumColumns()); + for(int i=0; i<getNumColumns(); i++) + for(int k=0; k<getNumRows(); k++) + if(get(i,k)) + ret.set(k, i); + return ret; + } + + //protected abstract BitsetMatrix diag(); + + //protected abstract BitsetMatrix reshape(int rows, int cols, boolean byrow); } /** @@ -264,6 +295,32 @@ public class EstimatorBitsetMM extends SparsityEstimator } @Override + public BitsetMatrix and(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix1) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix1 b = (BitsetMatrix1) bsb; + BitsetMatrix1 ret = new BitsetMatrix1(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) + ret._data[i] = _data[i] & b._data[i]; + ret._nonZeros = card(ret._data, 0, ret._data.length); + return ret; + } + + @Override + public BitsetMatrix or(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix1) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix1 b = (BitsetMatrix1) bsb; + BitsetMatrix1 ret = new BitsetMatrix1(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) + ret._data[i] = _data[i] | b._data[i]; + ret._nonZeros = card(ret._data, 0, ret._data.length); + return ret; + } + + @Override public BitsetMatrix rbind(BitsetMatrix bsb) { if( !(bsb instanceof BitsetMatrix1) ) throw new HopsException("Incompatible bitset types: " @@ -272,17 +329,47 @@ public class EstimatorBitsetMM extends SparsityEstimator BitsetMatrix1 ret = new BitsetMatrix1(getNumRows()+bsb.getNumRows(), getNumColumns()); System.arraycopy(_data, 0, ret._data, 0, _rlen*_rowLen); System.arraycopy(b._data, 0, ret._data, _rlen*_rowLen, b._rlen*_rowLen); + ret._nonZeros = card(ret._data, 0, ret._data.length); return ret; } - private void set(int r, int c) { + @Override + public BitsetMatrix cbind(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix1) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix1 b = (BitsetMatrix1) bsb; + BitsetMatrix1 ret = new BitsetMatrix1(getNumRows(), getNumColumns() + bsb.getNumColumns()); + //copy first matrix + for(int i=0; i<getNumRows(); i++) + System.arraycopy(_data, i*_rowLen, ret._data, i*ret._rowLen, _rowLen); + //copy second matrix + for(int i=0; i<getNumRows(); i++) + for(int j=0; j<b.getNumColumns(); j++) + if( b.get(i, j) ) + ret.set(i, getNumColumns()+j); + ret._nonZeros = card(ret._data, 0, ret._data.length); + return ret; + } + + @Override + public BitsetMatrix flip() { + BitsetMatrix1 ret = new BitsetMatrix1(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) + ret._data[i] = ~_data[i]; + ret._nonZeros = (long)getNumRows() * getNumColumns() - getNonZeros(); + return ret; + } + + @Override + public void set(int r, int c) { int off = r * _rowLen; int wordIndex = wordIndex(c); //see BitSet.java _data[off + wordIndex] |= (1L << c); //see BitSet.java } - @SuppressWarnings("unused") - private boolean get(int r, int c) { + @Override + public boolean get(int r, int c) { int off = r * _rowLen; int wordIndex = wordIndex(c); //see Bitset.java return (_data[off + wordIndex] & (1L << c)) != 0; //see BitSet.java @@ -391,6 +478,36 @@ public class EstimatorBitsetMM extends SparsityEstimator } @Override + public BitsetMatrix and(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix2) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix2 b = (BitsetMatrix2) bsb; + BitsetMatrix2 ret = new BitsetMatrix2(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) { + ret._data[i] = (BitSet)_data[i].clone(); + ret._data[i].and(b._data[i]); + ret._nonZeros += ret._data[i].cardinality(); + } + return ret; + } + + @Override + public BitsetMatrix or(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix2) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix2 b = (BitsetMatrix2) bsb; + BitsetMatrix2 ret = new BitsetMatrix2(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) { + ret._data[i] = (BitSet)_data[i].clone(); + ret._data[i].or(b._data[i]); + ret._nonZeros += ret._data[i].cardinality(); + } + return ret; + } + + @Override public BitsetMatrix rbind(BitsetMatrix bsb) { if( !(bsb instanceof BitsetMatrix2) ) throw new HopsException("Incompatible bitset types: " @@ -401,5 +518,46 @@ public class EstimatorBitsetMM extends SparsityEstimator System.arraycopy(b._data, 0, ret._data, _rlen, b._rlen); //shallow copy return ret; } + + @Override + protected BitsetMatrix cbind(BitsetMatrix bsb) { + if( !(bsb instanceof BitsetMatrix2) ) + throw new HopsException("Incompatible bitset types: " + + getClass().getSimpleName()+" and "+bsb.getClass().getSimpleName()); + BitsetMatrix2 b = (BitsetMatrix2) bsb; + BitsetMatrix2 ret = new BitsetMatrix2(getNumRows(), getNumColumns() + bsb.getNumColumns()); + //copy first matrix + for(int i=0; i<getNumRows(); i++) + ret._data[i] = (BitSet)_data[i].clone(); + //copy second matrix (via append) + for(int i=0; i<getNumRows(); i++) { + for(int j=0; j<b.getNumColumns(); j++) + if( b.get(i, j) ) + ret.set(i, getNumColumns()+j); + ret._nonZeros += ret._data[i].cardinality(); + } + return ret; + } + + @Override + public BitsetMatrix flip() { + BitsetMatrix2 ret = new BitsetMatrix2(getNumRows(), getNumColumns()); + for(int i=0; i<_data.length; i++) { + ret._data[i] = (BitSet)_data[i].clone(); + ret._data[i].flip(0, _data[i].size()); + ret._nonZeros += ret._data[i].cardinality(); + } + return ret; + } + + @Override + public boolean get(int r, int c) { + return _data[r].get(c); + } + + @Override + public void set(int r, int c) { + _data[r].set(c); + } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindChainTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindChainTest.java new file mode 100644 index 0000000..2d87e21 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindChainTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.estim; + +import org.junit.Test; +import org.apache.commons.lang.NotImplementedException; +import org.apache.sysml.hops.estim.EstimatorBasicAvg; +import org.apache.sysml.hops.estim.EstimatorBasicWorst; +import org.apache.sysml.hops.estim.EstimatorBitsetMM; +import org.apache.sysml.hops.estim.EstimatorMatrixHistogram; +import org.apache.sysml.hops.estim.MMNode; +import org.apache.sysml.hops.estim.SparsityEstimator; +import org.apache.sysml.hops.estim.SparsityEstimator.OpCode; +import org.apache.sysml.runtime.instructions.InstructionUtils; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; + +/** + * this is the basic operation check for all estimators with single operations + */ +public class OpBindChainTest extends AutomatedTestBase +{ + private final static int m = 600; + private final static int k = 300; + private final static int n = 100; + private final static double[] sparsity = new double[]{0.2, 0.4}; +// private final static OpCode mult = OpCode.MULT; +// private final static OpCode plus = OpCode.PLUS; + private final static OpCode rbind = OpCode.RBIND; + private final static OpCode cbind = OpCode.CBIND; +// private final static OpCode eqzero = OpCode.EQZERO; +// private final static OpCode diag = OpCode.DIAG; +// private final static OpCode neqzero = OpCode.NEQZERO; +// private final static OpCode trans = OpCode.TRANS; +// private final static OpCode reshape = OpCode.RESHAPE; + + @Override + public void setUp() { + //do nothing + } + + //Average Case + @Test + public void testAvgRbind() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, n, sparsity, rbind); + } + + @Test + public void testAvgCbind() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, n, sparsity, cbind); + } + + //Worst Case + @Test + public void testWorstRbind() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, n, sparsity, rbind); + } + + @Test + public void testWorstCbind() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, n, sparsity, cbind); + } + + //DensityMap + /*@Test + public void testDMCaserbind() { + runSparsityEstimateTest(new EstimatorDensityMap(), m, k, n, sparsity, rbind); + } + + @Test + public void testDMCasecbind() { + runSparsityEstimateTest(new EstimatorDensityMap(), m, k, n, sparsity, cbind); + }*/ + + //MNC + @Test + public void testMNCRbind() { + runSparsityEstimateTest(new EstimatorMatrixHistogram(), m, k, n, sparsity, rbind); + } + + @Test + public void testMNCCbind() { + runSparsityEstimateTest(new EstimatorMatrixHistogram(), m, k, n, sparsity, cbind); + } + + //Bitset + public void testBitsetCaserbind() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, rbind); + } + + @Test + public void testBitsetCasecbind() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, cbind); + } + + //Layered Graph + /*@Test + public void testLGCaserbind() { + runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, rbind); + } + + @Test + public void testLGCasecbind() { + runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, cbind); + }*/ + + + private void runSparsityEstimateTest(SparsityEstimator estim, int m, int k, int n, double[] sp, OpCode op) { + MatrixBlock m1; + MatrixBlock m2; + MatrixBlock m3 = new MatrixBlock(); + MatrixBlock m4; + MatrixBlock m5 = new MatrixBlock(); + double est = 0; + switch(op) { + case RBIND: + m1 = MatrixBlock.randOperations(m, k, sp[0], 1, 1, "uniform", 3); + m2 = MatrixBlock.randOperations(n, k, sp[1], 1, 1, "uniform", 7); + m1.append(m2, m3, false); + m4 = MatrixBlock.randOperations(k, m, sp[1], 1, 1, "uniform", 5); + m5 = m1.aggregateBinaryOperations(m3, m4, + new MatrixBlock(), InstructionUtils.getMatMultOperator(1)); + est = estim.estim(new MMNode(new MMNode(new MMNode(m1), new MMNode(m2), op), new MMNode(m4), OpCode.MM)).getSparsity(); + //System.out.println(est); + //System.out.println(m5.getSparsity()); + break; + case CBIND: + m1 = MatrixBlock.randOperations(m, k, sp[0], 1, 1, "uniform", 3); + m2 = MatrixBlock.randOperations(m, n, sp[1], 1, 1, "uniform", 7); + m1.append(m2, m3, true); + m4 = MatrixBlock.randOperations(k+n, m, sp[1], 1, 1, "uniform", 5); + m5 = m1.aggregateBinaryOperations(m3, m4, + new MatrixBlock(), InstructionUtils.getMatMultOperator(1)); + est = estim.estim(new MMNode(new MMNode(new MMNode(m1), new MMNode(m2), op), new MMNode(m4), OpCode.MM)).getSparsity(); + //System.out.println(est); + //System.out.println(m5.getSparsity()); + break; + default: + throw new NotImplementedException(); + } + //compare estimated and real sparsity + TestUtils.compareScalars(est, m5.getSparsity(), (estim instanceof EstimatorBasicWorst) ? 5e-1 : 1e-2); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindTest.java index a8f9e49..c04458b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpBindTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.apache.commons.lang.NotImplementedException; import org.apache.sysml.hops.estim.EstimatorBasicAvg; import org.apache.sysml.hops.estim.EstimatorBasicWorst; +import org.apache.sysml.hops.estim.EstimatorBitsetMM; import org.apache.sysml.hops.estim.EstimatorMatrixHistogram; import org.apache.sysml.hops.estim.SparsityEstimator; import org.apache.sysml.hops.estim.SparsityEstimator.OpCode; @@ -99,22 +100,22 @@ public class OpBindTest extends AutomatedTestBase } //Bitset - /*@Test - public void testBitsetCaserbind() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, rbind); - } - @Test public void testBitsetCasecbind() { runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, cbind); } - - //Layered Graph + @Test + public void testBitsetCaserbind() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, rbind); + } + + //Layered Graph + /*@Test public void testLGCaserbind() { runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, rbind); } - + @Test public void testLGCasecbind() { runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, cbind); @@ -125,7 +126,7 @@ public class OpBindTest extends AutomatedTestBase public void testSampleCaserbind() { runSparsityEstimateTest(new EstimatorSample(), m, k, n, sparsity, rbind); } - + @Test public void testSampleCasecbind() { runSparsityEstimateTest(new EstimatorSample(), m, k, n, sparsity, cbind); @@ -143,12 +144,16 @@ public class OpBindTest extends AutomatedTestBase m2 = MatrixBlock.randOperations(n, k, sp[1], 1, 1, "uniform", 3); m1.append(m2, m3, false); est = estim.estim(m1, m2, op); + System.out.println(est); + System.out.println(m3.getSparsity()); break; case CBIND: - m1 = MatrixBlock.randOperations(m, k, sp[0], 1, 1, "uniform", 3); - m2 = MatrixBlock.randOperations(m, n, sp[1], 1, 1, "uniform", 3); + m1 = MatrixBlock.randOperations(10, 130, sp[0], 1, 1, "uniform", 3); + m2 = MatrixBlock.randOperations(10, 70, sp[1], 1, 1, "uniform", 3); m1.append(m2, m3); est = estim.estim(m1, m2, op); + System.out.println(est); + System.out.println(m3.getSparsity()); break; default: throw new NotImplementedException(); http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWChainTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWChainTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWChainTest.java new file mode 100644 index 0000000..6d69b69 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWChainTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.estim; + +import org.junit.Test; +import org.apache.sysml.runtime.matrix.operators.BinaryOperator; +import org.apache.commons.lang.NotImplementedException; +import org.apache.sysml.hops.estim.EstimatorBasicAvg; +import org.apache.sysml.hops.estim.EstimatorBasicWorst; +import org.apache.sysml.hops.estim.EstimatorBitsetMM; +import org.apache.sysml.hops.estim.MMNode; +import org.apache.sysml.hops.estim.SparsityEstimator; +import org.apache.sysml.hops.estim.SparsityEstimator.OpCode; +import org.apache.sysml.runtime.functionobjects.Multiply; +import org.apache.sysml.runtime.functionobjects.Plus; +import org.apache.sysml.runtime.instructions.InstructionUtils; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; + +/** + * this is the basic operation check for all estimators with single operations + */ +public class OpElemWChainTest extends AutomatedTestBase +{ + private final static int m = 1600; + private final static int n = 700; + private final static double[] sparsity = new double[]{0.1, 0.04}; + private final static OpCode mult = OpCode.MULT; + private final static OpCode plus = OpCode.PLUS; + + @Override + public void setUp() { + //do nothing + } + //Average Case + @Test + public void testAvgMult() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, n, sparsity, mult); + } + + @Test + public void testAvgPlus() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, n, sparsity, plus); + } + + //Worst Case + @Test + public void testWorstMult() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, n, sparsity, mult); + } + + @Test + public void testWorstPlus() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, n, sparsity, plus); + } + + //DensityMap + /*@Test + public void testDMMult() { + runSparsityEstimateTest(new EstimatorDensityMap(), m, n, sparsity, mult); + } + + @Test + public void testDMPlus() { + runSparsityEstimateTest(new EstimatorDensityMap(), m, n, sparsity, plus); + } + + //MNC + @Test + public void testMNCMult() { + runSparsityEstimateTest(new EstimatorMatrixHistogram(), m, n, sparsity, mult); + } + + @Test + public void testMNCPlus() { + runSparsityEstimateTest(new EstimatorMatrixHistogram(), m, n, sparsity, plus); + }*/ + + //Bitset + @Test + public void testBitsetMult() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, n, sparsity, mult); + } + + @Test + public void testBitsetPlus() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, n, sparsity, plus); + } + + //Layered Graph + /*@Test + public void testLGCasemult() { + runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, mult); + } + + @Test + public void testLGCaseplus() { + runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, n, sparsity, plus); + }*/ + + + private void runSparsityEstimateTest(SparsityEstimator estim, int m, int n, double[] sp, OpCode op) { + MatrixBlock m1 = MatrixBlock.randOperations(m, n, sp[0], 1, 1, "uniform", 3); + MatrixBlock m2 = MatrixBlock.randOperations(m, n, sp[1], 1, 1, "uniform", 5); + MatrixBlock m3 = MatrixBlock.randOperations(n, m, sp[1], 1, 1, "uniform", 7); + MatrixBlock m4 = new MatrixBlock(); + MatrixBlock m5 = new MatrixBlock(); + BinaryOperator bOp; + double est = 0; + switch(op) { + case MULT: + bOp = new BinaryOperator(Multiply.getMultiplyFnObject()); + m1.binaryOperations(bOp, m2, m4); + m5 = m1.aggregateBinaryOperations(m4, m3, + new MatrixBlock(), InstructionUtils.getMatMultOperator(1)); + est = estim.estim(new MMNode(new MMNode(new MMNode(m1), new MMNode(m2), op), new MMNode(m3), OpCode.MM)).getSparsity(); + System.out.println(m5.getSparsity()); + System.out.println(est); + break; + case PLUS: + bOp = new BinaryOperator(Plus.getPlusFnObject()); + m1.binaryOperations(bOp, m2, m4); + m5 = m1.aggregateBinaryOperations(m4, m3, + new MatrixBlock(), InstructionUtils.getMatMultOperator(1)); + est = estim.estim(new MMNode(new MMNode(new MMNode(m1), new MMNode(m2), op), new MMNode(m3), OpCode.MM)).getSparsity(); + System.out.println(m5.getSparsity()); + System.out.println(est); + break; + default: + throw new NotImplementedException(); + } + //compare estimated and real sparsity + TestUtils.compareScalars(est, m5.getSparsity(), (estim instanceof EstimatorBasicWorst) ? 9e-1 : 1e-2); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWTest.java index 29cd607..7867f26 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpElemWTest.java @@ -24,6 +24,7 @@ import org.apache.sysml.runtime.matrix.operators.BinaryOperator; import org.apache.commons.lang.NotImplementedException; import org.apache.sysml.hops.estim.EstimatorBasicAvg; import org.apache.sysml.hops.estim.EstimatorBasicWorst; +import org.apache.sysml.hops.estim.EstimatorBitsetMM; import org.apache.sysml.hops.estim.EstimatorDensityMap; import org.apache.sysml.hops.estim.EstimatorMatrixHistogram; import org.apache.sysml.hops.estim.SparsityEstimator; @@ -39,8 +40,7 @@ import org.apache.sysml.test.utils.TestUtils; */ public class OpElemWTest extends AutomatedTestBase { - //TODO experiment with m>2n for MNC (currently suboptimal accuracy) - private final static int m = 600; + private final static int m = 1600; private final static int n = 700; private final static double[] sparsity = new double[]{0.1, 0.04}; private final static OpCode mult = OpCode.MULT; @@ -102,16 +102,16 @@ public class OpElemWTest extends AutomatedTestBase } //Bitset - /*@Test + @Test public void testBitsetCasemult() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, mult); + runSparsityEstimateTest(new EstimatorBitsetMM(), m, n, sparsity, mult); } @Test public void testBitsetCaseplus() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, n, sparsity, plus); + runSparsityEstimateTest(new EstimatorBitsetMM(), m, n, sparsity, plus); } - + /* //Layered Graph @Test public void testLGCasemult() { @@ -137,7 +137,7 @@ public class OpElemWTest extends AutomatedTestBase private void runSparsityEstimateTest(SparsityEstimator estim, int m, int n, double[] sp, OpCode op) { MatrixBlock m1 = MatrixBlock.randOperations(m, n, sp[0], 1, 1, "uniform", 3); - MatrixBlock m2 = MatrixBlock.randOperations(m, n, sp[1], 1, 1, "uniform", 3); + MatrixBlock m2 = MatrixBlock.randOperations(m, n, sp[1], 1, 1, "uniform", 7); MatrixBlock m3 = new MatrixBlock(); BinaryOperator bOp; double est = 0; @@ -146,11 +146,15 @@ public class OpElemWTest extends AutomatedTestBase bOp = new BinaryOperator(Multiply.getMultiplyFnObject()); m1.binaryOperations(bOp, m2, m3); est = estim.estim(m1, m2, op); + System.out.println(m3.getSparsity()); + System.out.println(est); break; case PLUS: bOp = new BinaryOperator(Plus.getPlusFnObject()); m1.binaryOperations(bOp, m2, m3); est = estim.estim(m1, m2, op); + System.out.println(m3.getSparsity()); + System.out.println(est); break; default: throw new NotImplementedException(); http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingle.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingle.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingle.java deleted file mode 100644 index 1756a8d..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingle.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.estim; - -import org.junit.Test; -import org.apache.sysml.hops.estim.EstimatorBasicAvg; -import org.apache.sysml.hops.estim.EstimatorBasicWorst; -import org.apache.sysml.hops.estim.EstimatorBitsetMM; -import org.apache.sysml.hops.estim.EstimatorDensityMap; -import org.apache.sysml.hops.estim.EstimatorLayeredGraph; -import org.apache.sysml.hops.estim.EstimatorSample; -import org.apache.sysml.hops.estim.SparsityEstimator; -import org.apache.sysml.hops.estim.SparsityEstimator.OpCode; -import org.apache.sysml.test.integration.AutomatedTestBase; - -/** - * this is the basic operation check for all estimators with single operations - */ -public class OpSingle extends AutomatedTestBase -{ - private final static int m = 600; - private final static int k = 300; - private final static double sparsity = 0.2; -// private final static OpCode mult = OpCode.MULT; -// private final static OpCode plus = OpCode.PLUS; -// private final static OpCode rbind = OpCode.RBIND; -// private final static OpCode cbind = OpCode.CBIND; - private final static OpCode eqzero = OpCode.EQZERO; - private final static OpCode diag = OpCode.DIAG; - private final static OpCode neqzero = OpCode.NEQZERO; - private final static OpCode trans = OpCode.TRANS; - private final static OpCode reshape = OpCode.RESHAPE; - - @Override - public void setUp() { - //do nothing - } - - //Average Case - @Test - public void testAvgCaseeqzero() { - runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, eqzero); - } - - @Test - public void testAvgCasediag() { - runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, diag); - } - - @Test - public void testAvgCaseneqzero() { - runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, neqzero); - } - - @Test - public void testAvgCasetrans() { - runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, trans); - } - - @Test - public void testAvgCasereshape() { - runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, reshape); - } - - //Worst Case - @Test - public void testWCaseeqzero() { - runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, eqzero); - } - - @Test - public void testWCasediag() { - runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, diag); - } - - @Test - public void testWCaseneqzero() { - runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, neqzero); - } - - @Test - public void testWCasetrans() { - runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, trans); - } - - @Test - public void testWCasereshape() { - runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, reshape); - } - - //DensityMap - @Test - public void testDMCaseeqzero() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero); - } - - @Test - public void testDMCasediag() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag); - } - - @Test - public void testDMCaseneqzero() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero); - } - - @Test - public void testDMCasetrans() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans); - } - - @Test - public void testDMCasereshape() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape); - } - - //MNC - @Test - public void testMNCCaseeqzero() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero); - } - - @Test - public void testMNCCasediag() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag); - } - - @Test - public void testMNCCaseneqzero() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero); - } - - @Test - public void testMNCCasetrans() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans); - } - - @Test - public void testMNCCasereshape() { - runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape); - } - - //Bitset - @Test - public void testBitsetCaseeqzero() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, eqzero); - } - - @Test - public void testBitsetCasediag() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, diag); - } - - @Test - public void testBitsetCaseneqzero() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, neqzero); - } - - @Test - public void testBitsetCasetrans() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, trans); - } - - @Test - public void testBitsetCasereshape() { - runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, reshape); - } - - //Layered Graph - @Test - public void testLGCaseeqzero() { - runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, eqzero); - } - - @Test - public void testLGCasediag() { - runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, diag); - } - - @Test - public void testLGCaseneqzero() { - runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, neqzero); - } - - @Test - public void testLGCasetans() { - runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, trans); - } - - @Test - public void testLGCasereshape() { - runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, reshape); - } - - //Sample - @Test - public void testSampleCaseeqzero() { - runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, eqzero); - } - - @Test - public void testSampleCasediag() { - runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, diag); - } - - @Test - public void testSampleCaseneqzero() { - runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, neqzero); - } - - @Test - public void testSampleCasetrans() { - runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, trans); - } - - @Test - public void testSampleCasereshape() { - runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, reshape); - } - - private void runSparsityEstimateTest(SparsityEstimator estim, int m, int k, double sp, OpCode op) { -// MatrixBlock m1 = MatrixBlock.randOperations(m, k, sp, 1, 1, "uniform", 3); -// MatrixBlock m2 = null; -// double est = 0; -// switch(op) { -// case EQZERO: -// case DIAG: -// case NEQZERO: -// case TRANS: -// case RESHAPE: -// } -// //compare estimated and real sparsity -// TestUtils.compareScalars(est, m2.getSparsity(), (estim instanceof EstimatorBasicWorst) ? 5e-1 : 1e-2); - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java new file mode 100644 index 0000000..fa567ed --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/estim/OpSingleTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.estim; + +import org.junit.Test; +import org.apache.directory.api.util.exception.NotImplementedException; +import org.apache.sysml.hops.estim.EstimatorBasicAvg; +import org.apache.sysml.hops.estim.EstimatorBasicWorst; +import org.apache.sysml.hops.estim.EstimatorBitsetMM; +import org.apache.sysml.hops.estim.SparsityEstimator; +import org.apache.sysml.hops.estim.SparsityEstimator.OpCode; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.utils.TestUtils; + +/** + * this is the basic operation check for all estimators with single operations + */ +public class OpSingleTest extends AutomatedTestBase +{ + private final static int m = 600; + private final static int k = 300; + private final static double sparsity = 0.2; +// private final static OpCode eqzero = OpCode.EQZERO; +// private final static OpCode diag = OpCode.DIAG; + private final static OpCode neqzero = OpCode.NEQZERO; + private final static OpCode trans = OpCode.TRANS; + private final static OpCode reshape = OpCode.RESHAPE; + + @Override + public void setUp() { + //do nothing + } + + //Average Case +// @Test +// public void testAvgEqzero() { +// runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, eqzero); +// } + +// @Test +// public void testAvgDiag() { +// runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, diag); +// } + + @Test + public void testAvgNeqzero() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, neqzero); + } + + @Test + public void testAvgTrans() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, trans); + } + + @Test + public void testAvgReshape() { + runSparsityEstimateTest(new EstimatorBasicAvg(), m, k, sparsity, reshape); + } + + //Worst Case +// @Test +// public void testWorstEqzero() { +// runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, eqzero); +// } + +// @Test +// public void testWCasediag() { +// runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, diag); +// } + + @Test + public void testWorstNeqzero() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, neqzero); + } + + @Test + public void testWoestTrans() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, trans); + } + + @Test + public void testWorstReshape() { + runSparsityEstimateTest(new EstimatorBasicWorst(), m, k, sparsity, reshape); + } + +// //DensityMap +// @Test +// public void testDMCaseeqzero() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero); +// } +// +// @Test +// public void testDMCasediag() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag); +// } +// +// @Test +// public void testDMCaseneqzero() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero); +// } +// +// @Test +// public void testDMCasetrans() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans); +// } +// +// @Test +// public void testDMCasereshape() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape); +// } +// +// //MNC +// @Test +// public void testMNCCaseeqzero() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, eqzero); +// } +// +// @Test +// public void testMNCCasediag() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, diag); +// } +// +// @Test +// public void testMNCCaseneqzero() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, neqzero); +// } +// +// @Test +// public void testMNCCasetrans() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, trans); +// } +// +// @Test +// public void testMNCCasereshape() { +// runSparsityEstimateTest(new EstimatorDensityMap(), m, k, sparsity, reshape); +// } +// + //Bitset +// @Test +// public void testBitsetCaseeqzero() { +// runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, eqzero); +// } + +// @Test +// public void testBitsetCasediag() { +// runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, diag); +// } + + @Test + public void testBitsetNeqzero() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, neqzero); + } + + @Test + public void testBitsetTrans() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, trans); + } + + @Test + public void testBitsetReshape() { + runSparsityEstimateTest(new EstimatorBitsetMM(), m, k, sparsity, reshape); + } + +// //Layered Graph +// @Test +// public void testLGCaseeqzero() { +// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, eqzero); +// } +// +// @Test +// public void testLGCasediag() { +// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, diag); +// } +// +// @Test +// public void testLGCaseneqzero() { +// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, neqzero); +// } +// +// @Test +// public void testLGCasetans() { +// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, trans); +// } +// +// @Test +// public void testLGCasereshape() { +// runSparsityEstimateTest(new EstimatorLayeredGraph(), m, k, sparsity, reshape); +// } +// +// //Sample +// @Test +// public void testSampleCaseeqzero() { +// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, eqzero); +// } +// +// @Test +// public void testSampleCasediag() { +// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, diag); +// } +// +// @Test +// public void testSampleCaseneqzero() { +// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, neqzero); +// } +// +// @Test +// public void testSampleCasetrans() { +// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, trans); +// } +// +// @Test +// public void testSampleCasereshape() { +// runSparsityEstimateTest(new EstimatorSample(), m, k, sparsity, reshape); +// } + + private void runSparsityEstimateTest(SparsityEstimator estim, int m, int k, double sp, OpCode op) { + MatrixBlock m1 = MatrixBlock.randOperations(m, k, sp, 1, 1, "uniform", 3); + MatrixBlock m2 = new MatrixBlock(); + double est = 0; + switch(op) { + case EQZERO: + //TODO find out how to do eqzero + case DIAG: + case NEQZERO: + m2 = m1; + est = estim.estim(m1, op); + break; + case TRANS: + m2 = m1; + est = estim.estim(m1, op); + break; + case RESHAPE: + m2 = m1; + est = estim.estim(m1, op); + break; + default: + throw new NotImplementedException(); + } + //compare estimated and real sparsity + TestUtils.compareScalars(est, m2.getSparsity(), (estim instanceof EstimatorBasicWorst) ? 5e-1 : 1e-2); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/1e851ef6/src/test_suites/java/org/apache/sysml/test/integration/functions/estim/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/estim/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/estim/ZPackageSuite.java index 2e9a6d6..2760063 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/estim/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/estim/ZPackageSuite.java @@ -28,11 +28,11 @@ import org.junit.runners.Suite; @Suite.SuiteClasses({ OpBindTest.class, OpElemWTest.class, + OpSingleTest.class, OuterProductTest.class, SelfProductTest.class, SquaredProductChainTest.class, SquaredProductTest.class, - //OpSingle.class })
