Repository: incubator-systemml Updated Branches: refs/heads/master 0dfa71d35 -> 2a0e41029
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java index ec026fb..21c40e6 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java @@ -29,6 +29,7 @@ import java.util.concurrent.Future; import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.runtime.functionobjects.Builtin.BuiltinFunctionCode; import org.apache.sysml.runtime.functionobjects.CM; @@ -385,7 +386,7 @@ public class LibMatrixAgg MatrixBlock row = tasks.get(i).getResult(); if( uaop.aggOp.correctionExists ) row.dropLastRowsOrColums(uaop.aggOp.correctionLocation); - tmp.leftIndexingOperations(row, i, i, 0, n2-1, tmp, true); + tmp.leftIndexingOperations(row, i, i, 0, n2-1, tmp, UpdateType.INPLACE_PINNED); } MatrixBlock tmp2 = cumaggregateUnaryMatrix(tmp, new MatrixBlock(tasks.size(), n2, false), uop); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java index 67800fc..c1b859f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java @@ -34,6 +34,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.functionobjects.DiagIndex; import org.apache.sysml.runtime.functionobjects.RevIndex; import org.apache.sysml.runtime.functionobjects.SortIndex; @@ -296,7 +297,7 @@ public class LibMatrixReorg MatrixIndexes outix1 = new MatrixIndexes(blkix1, inix.getColumnIndex()); MatrixBlock outblk1 = new MatrixBlock(blklen1, inblk.getNumColumns(), inblk.isInSparseFormat()); MatrixBlock tmp1 = tmpblk.sliceOperations(0, iposCut, 0, tmpblk.getNumColumns()-1, new MatrixBlock()); - outblk1.leftIndexingOperations(tmp1, ipos1, outblk1.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk1, true); + outblk1.leftIndexingOperations(tmp1, ipos1, outblk1.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk1, UpdateType.INPLACE_PINNED); out.add(new IndexedMatrixValue(outix1, outblk1)); //slice second block (if necessary) @@ -304,7 +305,7 @@ public class LibMatrixReorg MatrixIndexes outix2 = new MatrixIndexes(blkix2, inix.getColumnIndex()); MatrixBlock outblk2 = new MatrixBlock(blklen2, inblk.getNumColumns(), inblk.isInSparseFormat()); MatrixBlock tmp2 = tmpblk.sliceOperations(iposCut+1, tmpblk.getNumRows()-1, 0, tmpblk.getNumColumns()-1, new MatrixBlock()); - outblk2.leftIndexingOperations(tmp2, 0, tmp2.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk2, true); + outblk2.leftIndexingOperations(tmp2, 0, tmp2.getNumRows()-1, 0, tmpblk.getNumColumns()-1, outblk2, UpdateType.INPLACE_PINNED); out.add(new IndexedMatrixValue(outix2, outblk2)); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index a7f7c38..e92cdf9 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -42,6 +42,7 @@ import org.apache.sysml.lops.MapMultChain.ChainType; import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.caching.CacheBlock; +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.runtime.functionobjects.CM; import org.apache.sysml.runtime.functionobjects.CTable; @@ -95,6 +96,8 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab public static final double ULTRA_SPARSITY_TURN_POINT = 0.00004; //default sparse block type: modified compressed sparse rows public static final SparseBlock.Type DEFAULT_SPARSEBLOCK = SparseBlock.Type.MCSR; + //default sparse block type for update in place: compressed sparse rows to prevent serialization + public static final SparseBlock.Type DEFAULT_INPLACE_SPARSEBLOCK = SparseBlock.Type.CSR; //basic header (int rlen, int clen, byte type) public static final int HEADER_SIZE = 9; @@ -3884,12 +3887,12 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab * @return * @throws DMLRuntimeException */ - public MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, IndexRange ixrange, MatrixBlock ret, boolean inplace) + public MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, IndexRange ixrange, MatrixBlock ret, UpdateType update) throws DMLRuntimeException { return leftIndexingOperations( rhsMatrix, (int)ixrange.rowStart, (int)ixrange.rowEnd, - (int)ixrange.colStart, (int)ixrange.colEnd, ret, inplace); + (int)ixrange.colStart, (int)ixrange.colEnd, ret, update); } /** @@ -3903,7 +3906,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab * @throws DMLRuntimeException */ public MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, int rl, int ru, - int cl, int cu, MatrixBlock ret, boolean inplace) + int cl, int cu, MatrixBlock ret, UpdateType update) throws DMLRuntimeException { // Check the validity of bounds @@ -3920,12 +3923,11 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab (rl+1) +":" + (ru+1) + ", " + (cl+1) + ":" + (cu+1) + "]."); } - MatrixBlock result = ret; - + MatrixBlock result = ret; boolean sp = estimateSparsityOnLeftIndexing(rlen, clen, nonZeros, rhsMatrix.getNumRows(), rhsMatrix.getNumColumns(), rhsMatrix.getNonZeros()); - if( !inplace ) //general case + if( !update.isInPlace() ) //general case { if(result==null) result=new MatrixBlock(rlen, clen, sp); @@ -3944,6 +3946,12 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab result.sparseToDense(); else if( !result.sparse && sp ) result.denseToSparse(); + + //ensure right sparse block representation to prevent serialization + if( result.sparse && update != UpdateType.INPLACE_PINNED ) { + result.sparseBlock = SparseBlockFactory.copySparseBlock( + DEFAULT_INPLACE_SPARSEBLOCK, result.sparseBlock, false); + } } //NOTE conceptually we could directly use a zeroout and copy(..., false) but @@ -3983,13 +3991,13 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab * @return * @throws DMLRuntimeException */ - public MatrixBlock leftIndexingOperations(ScalarObject scalar, int rl, int cl, MatrixBlock ret, boolean inplace) + public MatrixBlock leftIndexingOperations(ScalarObject scalar, int rl, int cl, MatrixBlock ret, UpdateType update) throws DMLRuntimeException { double inVal = scalar.getDoubleValue(); boolean sp = estimateSparsityOnLeftIndexing(rlen, clen, nonZeros, 1, 1, (inVal!=0)?1:0); - if( !inplace ) //general case + if( !update.isInPlace() ) //general case { if(ret==null) ret=new MatrixBlock(rlen, clen, sp); @@ -3999,7 +4007,16 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } else //update in-place + { + //use current block as in-place result ret = this; + + //ensure right sparse block representation to prevent serialization + if( ret.sparse && update != UpdateType.INPLACE_PINNED ) { + ret.sparseBlock = SparseBlockFactory.copySparseBlock( + DEFAULT_INPLACE_SPARSEBLOCK, ret.sparseBlock, false); + } + } ret.quickSetValue(rl, cl, inVal); return ret; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java index 4fe2834..03a36fe 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java @@ -23,6 +23,7 @@ package org.apache.sysml.runtime.matrix.data; import java.util.ArrayList; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.lops.PartialAggregate.CorrectionLocationType; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; @@ -417,7 +418,7 @@ public class OperationsOnMatrixValues int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen); int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen); MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false); - resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, false); + resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY); outlist.add(new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock)); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/main/java/org/apache/sysml/utils/Explain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java index 6b266ea..6e8b4f6 100644 --- a/src/main/java/org/apache/sysml/utils/Explain.java +++ b/src/main/java/org/apache/sysml/utils/Explain.java @@ -558,7 +558,10 @@ public class Explain if (sb instanceof WhileStatementBlock) { WhileStatementBlock wsb = (WhileStatementBlock) sb; builder.append(offset); - builder.append("WHILE (lines "+wsb.getBeginLine()+"-"+wsb.getEndLine()+")\n"); + if( !wsb.getUpdateInPlaceVars().isEmpty() ) + builder.append("WHILE (lines "+wsb.getBeginLine()+"-"+wsb.getEndLine()+") [in-place="+wsb.getUpdateInPlaceVars().toString()+"]\n"); + else + builder.append("WHILE (lines "+wsb.getBeginLine()+"-"+wsb.getEndLine()+")\n"); builder.append(explainHop(wsb.getPredicateHops(), level+1)); WhileStatement ws = (WhileStatement)sb.getStatement(0); @@ -588,9 +591,12 @@ public class Explain builder.append(offset); if (sb instanceof ParForStatementBlock) builder.append("PARFOR (lines "+fsb.getBeginLine()+"-"+fsb.getEndLine()+")\n"); - else - builder.append("FOR (lines "+fsb.getBeginLine()+"-"+fsb.getEndLine()+")\n"); - + else { + if( !fsb.getUpdateInPlaceVars().isEmpty() ) + builder.append("FOR (lines "+fsb.getBeginLine()+"-"+fsb.getEndLine()+") [in-place="+fsb.getUpdateInPlaceVars().toString()+"]\n"); + else + builder.append("FOR (lines "+fsb.getBeginLine()+"-"+fsb.getEndLine()+")\n"); + } if (fsb.getFromHops() != null) builder.append(explainHop(fsb.getFromHops(), level+1)); if (fsb.getToHops() != null) @@ -681,8 +687,8 @@ public class Explain + hop.getColsInBlock() + "," + hop.getNnz()); - if (hop.getUpdateInPlace()) - sb.append("," + hop.getUpdateInPlace()); + if (hop.getUpdateType().isInPlace()) + sb.append("," + hop.getUpdateType().toString().toLowerCase()); sb.append("]"); @@ -832,8 +838,12 @@ public class Explain else if (pb instanceof WhileProgramBlock) { WhileProgramBlock wpb = (WhileProgramBlock) pb; + StatementBlock wsb = pb.getStatementBlock(); sb.append(offset); - sb.append("WHILE (lines "+wpb.getBeginLine()+"-"+wpb.getEndLine()+")\n"); + if( wsb != null && !wsb.getUpdateInPlaceVars().isEmpty() ) + sb.append("WHILE (lines "+wpb.getBeginLine()+"-"+wpb.getEndLine()+") [in-place="+wsb.getUpdateInPlaceVars().toString()+"]\n"); + else + sb.append("WHILE (lines "+wpb.getBeginLine()+"-"+wpb.getEndLine()+")\n"); sb.append(explainInstructions(wpb.getPredicate(), level+1)); for( ProgramBlock pbc : wpb.getChildBlocks() ) sb.append( explainProgramBlock( pbc, level+1) ); @@ -857,11 +867,16 @@ public class Explain else if (pb instanceof ForProgramBlock) //incl parfor { ForProgramBlock fpb = (ForProgramBlock) pb; + StatementBlock fsb = pb.getStatementBlock(); sb.append(offset); if( pb instanceof ParForProgramBlock ) sb.append("PARFOR (lines "+fpb.getBeginLine()+"-"+fpb.getEndLine()+")\n"); - else - sb.append("FOR (lines "+fpb.getBeginLine()+"-"+fpb.getEndLine()+")\n"); + else { + if( fsb != null && !fsb.getUpdateInPlaceVars().isEmpty() ) + sb.append("FOR (lines "+fpb.getBeginLine()+"-"+fpb.getEndLine()+") [in-place="+fsb.getUpdateInPlaceVars().toString()+"]\n"); + else + sb.append("FOR (lines "+fpb.getBeginLine()+"-"+fpb.getEndLine()+")\n"); + } sb.append(explainInstructions(fpb.getFromInstructions(), level+1)); sb.append(explainInstructions(fpb.getToInstructions(), level+1)); sb.append(explainInstructions(fpb.getIncrementInstructions(), level+1)); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java index 6d0b6c7..36bafea 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameIndexingTest.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.List; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.DataConverter; @@ -133,7 +134,7 @@ public class FrameIndexingTest extends AutomatedTestBase //matrix indexing MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); MatrixBlock mbB = DataConverter.convertToMatrixBlock(B); - mbC = mbA.leftIndexingOperations(mbB, rl, ru, cl, cu, new MatrixBlock(), false); + mbC = mbA.leftIndexingOperations(mbB, rl, ru, cl, cu, new MatrixBlock(), UpdateType.COPY); //frame indexing frame3 = frame1.leftIndexingOperations(frame2, rl, ru, cl, cu, new FrameBlock()); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/78e161c0/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java index cb19d04..cc0a294 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java @@ -24,6 +24,7 @@ import java.util.List; import org.junit.Assert; import org.junit.Test; +import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.controlprogram.parfor.opt.OptimizerRuleBased; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; @@ -227,12 +228,15 @@ public class UpdateInPlaceTest extends AutomatedTestBase */ private void runUpdateInPlaceTest( String TEST_NAME, int iTestNumber, List<String> listUIPExp, long lTotalUIPVar, long lTotalLixUIP, long lTotalLix) { + boolean oldinplace = OptimizerUtils.ALLOW_LOOP_UPDATE_IN_PLACE; + try { TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); - + OptimizerUtils.ALLOW_LOOP_UPDATE_IN_PLACE = false; + // This is for running the junit test the new way, i.e., construct the arguments directly String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + iTestNumber + ".dml"; @@ -283,7 +287,8 @@ public class UpdateInPlaceTest extends AutomatedTestBase } } - finally{ + finally { + OptimizerUtils.ALLOW_LOOP_UPDATE_IN_PLACE = oldinplace; } }
