Repository: systemml Updated Branches: refs/heads/master 5e3fed25b -> f4954b78f
[MINOR] Fine-tuning codegen row ops (load balance, nnz recompute) This patch addresses minor issues of unnecessary many tasks (which is unnecessary for small inputs) and redundant nnz recomputation in spark codegen row ops. Furthermore, this also includes a minor fix of the maintenance of codegen optimization stats, which did not properly account for the plans evaluated by the opening heuristic. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f4954b78 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f4954b78 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f4954b78 Branch: refs/heads/master Commit: f4954b78f74cf494e61b8798709c26d411913d03 Parents: 5e3fed2 Author: Matthias Boehm <[email protected]> Authored: Sat Feb 17 18:21:28 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sat Feb 17 21:16:31 2018 -0800 ---------------------------------------------------------------------- .../sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java | 2 +- .../java/org/apache/sysml/runtime/codegen/SpoofRowwise.java | 8 +++++--- .../sysml/runtime/instructions/spark/SpoofSPInstruction.java | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java index 20cb955..84e4b4c 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java +++ b/src/main/java/org/apache/sysml/hops/codegen/opt/PlanSelectionFuseCostBasedV2.java @@ -227,7 +227,7 @@ public class PlanSelectionFuseCostBasedV2 extends PlanSelection //bestC is monotonically non-increasing and serves as the upper bound final int Mlen = matPoints.length-off; final long len = UtilFunctions.pow(2, Mlen); - long numEvalPlans = 0, numEvalPartPlans = 0; + long numEvalPlans = 2, numEvalPartPlans = 0; //evaluate heuristics fuse-all and fuse-no-redundancy to quickly obtain a good lower bound final boolean[] plan0 = createAssignment(Mlen, off, 0); // fuse-all http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java index b026b46..f10f857 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java @@ -163,14 +163,15 @@ public abstract class SpoofRowwise extends SpoofOperator //post-processing if( allocTmp &&_reqVectMem > 0 ) LibSpoofPrimitives.cleanupThreadLocalMemory(); - out.recomputeNonZeros(); if( flipOut ) { fixTransposeDimensions(out); out = LibMatrixReorg.transpose(out, new MatrixBlock( out.getNumColumns(), out.getNumRows(), false)); } - if( !aggIncr ) + if( !aggIncr ) { + out.recomputeNonZeros(); out.examSparsity(); + } return out; } @@ -207,7 +208,7 @@ public abstract class SpoofRowwise extends SpoofOperator ExecutorService pool = Executors.newFixedThreadPool( k ); ArrayList<Integer> blklens = (a instanceof CompressedMatrixBlock) ? LibMatrixMult.getAlignedBlockSizes(m, k, BitmapEncoder.BITMAP_BLOCK_SZ) : - LibMatrixMult.getBalancedBlockSizesDefault(m, k, false); + LibMatrixMult.getBalancedBlockSizesDefault(m, k, (long)m*n<16*PAR_NUMCELL_THRESHOLD); try { @@ -287,6 +288,7 @@ public abstract class SpoofRowwise extends SpoofOperator int rlen = out.getNumRows(); out.setNumRows(out.getNumColumns()); out.setNumColumns(rlen); + out.setNonZeros(out.getNumRows()*out.getNumColumns()); } private void executeDense(DenseBlock a, SideInput[] b, double[] scalars, DenseBlock c, int n, int rl, int ru) { http://git-wip-us.apache.org/repos/asf/systemml/blob/f4954b78/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java index 82b34df..edd7418 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java @@ -478,6 +478,7 @@ public class SpoofSPInstruction extends SPInstruction { //cleanup and final result preparations LibSpoofPrimitives.cleanupThreadLocalMemory(); if( aggIncr ) { + blkOut.recomputeNonZeros(); blkOut.examSparsity(); //deferred format change ret.add(new Tuple2<>(new MatrixIndexes(1,1), blkOut)); }
