This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 3ec800ab7549dabb973c5d1d79fca88944cdf3ff Author: baunsgaard <[email protected]> AuthorDate: Wed Mar 24 20:29:31 2021 +0100 [SYSTEMDS-2912] AggregateFinalResults avoid Generic correction Aggregate of final results would use a generic aggregation if either inputs were different format from each other. This commit change the behavior to force uniform format. Furthermore allocating the initial result in dense improve performance slightly more with 1 second The change improve performance of: uack+ 59.179sec -> uack+ 9.0sec on CNN implementation on MNIST using the parameterserver Closes #1211 --- .../apache/sysds/runtime/matrix/data/LibMatrixAgg.java | 12 ++++++++++-- .../org/apache/sysds/runtime/matrix/data/MatrixValue.java | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java index 743dce9..86bd412 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java @@ -274,7 +274,7 @@ public class LibMatrixAgg pool.shutdown(); //aggregate partial results if( !(uaop.indexFn instanceof ReduceCol) ) { - out.copy(((PartialAggTask)tasks.get(0)).getResult()); //for init + out.copy(((PartialAggTask)tasks.get(0)).getResult(), false); //for init for( int i=1; i<tasks.size(); i++ ) aggregateFinalResult(uaop.aggOp, out, ((PartialAggTask)tasks.get(i)).getResult()); } @@ -468,7 +468,7 @@ public class LibMatrixAgg List<Future<MatrixBlock>> rtasks = pool.invokeAll(tasks); pool.shutdown(); //aggregate partial results and error handling - ret.copy(rtasks.get(0).get()); //for init + ret.copy(rtasks.get(0).get(), false); //for init for( int i=1; i<rtasks.size(); i++ ) aggregateFinalResult(op.aggOp, ret, rtasks.get(i).get()); } @@ -674,6 +674,14 @@ public class LibMatrixAgg laop = new AggregateOperator(0, KahanPlus.getKahanPlusFnObject(), aop.correction); } + if( out.isInSparseFormat() != partout.isInSparseFormat() ){ + if(partout.isInSparseFormat()) + partout.sparseToDense(); + + if(out.isInSparseFormat()) + out.sparseToDense(); + } + //incremental aggregation of final results if( laop.existsCorrection() ) out.incrementalAggregate(laop, partout); diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixValue.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixValue.java index d90e844..c9838d5 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixValue.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixValue.java @@ -96,7 +96,22 @@ public abstract class MatrixValue implements WritableComparable public abstract void reset(int rl, int cl, boolean sp, long nnzs); public abstract void reset(int rl, int cl, double v); + /** + * Copy this MatrixValue into that MatrixValue. + * + * If the MatrixValue is a MatrixBlock evaluate the sparsity of the original matrix, + * and copy into either a sparse or a dense matrix. + * + * @param that object to copy the values into. + */ public abstract void copy(MatrixValue that); + + /** + * Copy this MatrixValue into that MatrixValue. But select sparse destination block depending on boolean parameter. + * + * @param that object to copy the values into. + * @param sp boolean specifying if output should be forced sparse or dense. (only applicable if the 'that' is a MatrixBlock) + */ public abstract void copy(MatrixValue that, boolean sp); public abstract MatrixValue scalarOperations(ScalarOperator op, MatrixValue result);
