[SYSTEMML-2135] Compiler/runtime support for matrices w/ zero rows/cols This patch makes a deep-cutting change to the compiler and runtime in order to support matrices with zero rows and columns in order to simplify common scenarios such as init-empty & append. In detail, this includes tests for these common scenarios, modifies append and datagen operations accordingly, and modifies many compiler/runtime conditions to allow the modified matrix definition. In subsequent patches, we will further address the I/O path and operations such as unary aggregates that require special treatment.
Furthermore, this also fixes a hidden issue of MR map-append rbind operations, which read under special conditions wrong blocks from the distributed cache. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/92ee2cbf Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/92ee2cbf Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/92ee2cbf Branch: refs/heads/master Commit: 92ee2cbf8c52346a9f14f04d9a2a1c57e963b059 Parents: b7fd340 Author: Matthias Boehm <[email protected]> Authored: Wed Feb 7 22:33:24 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Thu Feb 8 11:56:24 2018 -0800 ---------------------------------------------------------------------- .../api/mlcontext/MLContextConversionUtil.java | 5 +- .../java/org/apache/sysml/conf/DMLConfig.java | 7 +- .../java/org/apache/sysml/hops/AggBinaryOp.java | 4 +- .../java/org/apache/sysml/hops/BinaryOp.java | 44 ++-- .../org/apache/sysml/hops/ConvolutionOp.java | 4 +- .../java/org/apache/sysml/hops/DataGenOp.java | 6 +- .../java/org/apache/sysml/hops/FunctionOp.java | 12 +- src/main/java/org/apache/sysml/hops/Hop.java | 31 +-- .../org/apache/sysml/hops/OptimizerUtils.java | 23 +- .../sysml/hops/ParameterizedBuiltinOp.java | 4 +- .../java/org/apache/sysml/hops/ReorgOp.java | 8 +- .../java/org/apache/sysml/hops/TernaryOp.java | 4 +- .../hops/codegen/template/TemplateCell.java | 4 +- .../hops/codegen/template/TemplateRow.java | 2 +- .../apache/sysml/hops/cost/CostEstimator.java | 6 - .../sysml/hops/ipa/InterProceduralAnalysis.java | 5 +- .../apache/sysml/hops/recompile/Recompiler.java | 7 +- .../sysml/hops/rewrite/HopRewriteUtils.java | 56 +++-- .../RewriteAlgebraicSimplificationDynamic.java | 6 +- .../org/apache/sysml/lops/compile/JobType.java | 1 - .../sysml/parser/BuiltinFunctionExpression.java | 12 +- .../org/apache/sysml/parser/DataExpression.java | 90 +++----- .../org/apache/sysml/parser/Identifier.java | 2 +- .../apache/sysml/parser/IndexedIdentifier.java | 20 +- .../sysml/parser/ParForStatementBlock.java | 2 +- .../sysml/runtime/codegen/SpoofRowwise.java | 2 +- .../controlprogram/caching/FrameObject.java | 2 +- .../controlprogram/parfor/DataPartitioner.java | 2 +- .../controlprogram/parfor/RemoteParForMR.java | 19 +- .../ParameterizedBuiltinCPFileInstruction.java | 9 +- .../instructions/mr/AppendMInstruction.java | 24 +- .../instructions/mr/AppendRInstruction.java | 3 +- .../instructions/mr/CtableInstruction.java | 2 +- .../instructions/mr/ZeroOutInstruction.java | 1 - .../spark/MatrixAppendMSPInstruction.java | 31 ++- .../spark/MatrixAppendRSPInstruction.java | 14 +- .../instructions/spark/RandSPInstruction.java | 10 +- .../spark/utils/FrameRDDConverterUtils.java | 8 +- .../apache/sysml/runtime/io/MatrixReader.java | 2 +- .../apache/sysml/runtime/io/ReaderTextCSV.java | 2 +- .../sysml/runtime/io/ReaderTextCSVParallel.java | 2 +- .../sysml/runtime/io/WriterBinaryBlock.java | 8 +- .../sysml/runtime/matrix/CSVReblockMR.java | 5 +- .../apache/sysml/runtime/matrix/CombineMR.java | 16 +- .../apache/sysml/runtime/matrix/DataGenMR.java | 4 +- .../runtime/matrix/MatrixCharacteristics.java | 30 ++- .../apache/sysml/runtime/matrix/ReblockMR.java | 2 - .../runtime/matrix/data/LibMatrixDatagen.java | 26 +-- .../sysml/runtime/matrix/data/MatrixBlock.java | 21 +- .../runtime/matrix/data/MatrixPackedCell.java | 4 +- .../runtime/matrix/mapred/CMCOVMRMapper.java | 21 +- .../runtime/matrix/mapred/CMCOVMRReducer.java | 4 +- .../matrix/mapred/DistributedCacheInput.java | 6 +- .../sysml/runtime/matrix/mapred/GMRMapper.java | 9 - .../runtime/matrix/mapred/MMCJMRCache.java | 6 +- .../runtime/matrix/mapred/MMCJMRMapper.java | 10 +- .../runtime/matrix/mapred/MMRJMRMapper.java | 13 +- .../runtime/matrix/mapred/MMRJMRReducer.java | 16 +- .../sysml/runtime/matrix/mapred/MapperBase.java | 33 +-- .../sysml/runtime/util/MapReduceTool.java | 5 - .../apache/sysml/runtime/util/SortUtils.java | 28 --- .../functions/misc/ZeroRowsColsMatrixTest.java | 223 +++++++++++++++++++ .../functions/misc/ZeroMatrix_Aggregates.R | 34 +++ .../functions/misc/ZeroMatrix_Aggregates.dml | 31 +++ .../scripts/functions/misc/ZeroMatrix_Cbind.R | 32 +++ .../scripts/functions/misc/ZeroMatrix_Cbind.dml | 29 +++ .../scripts/functions/misc/ZeroMatrix_Rbind.R | 33 +++ .../scripts/functions/misc/ZeroMatrix_Rbind.dml | 31 +++ .../functions/misc/ZeroMatrix_RemoveEmpty.R | 29 +++ .../functions/misc/ZeroMatrix_RemoveEmpty.dml | 28 +++ .../functions/misc/ZPackageSuite.java | 1 + 71 files changed, 748 insertions(+), 488 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java index 1536f8f..02b07c0 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextConversionUtil.java @@ -961,7 +961,7 @@ public class MLContextConversionUtil { int cols = mb.getNumColumns(); List<String> list = new ArrayList<>(); - if (mb.getNonZeros() > 0) { + if ( !mb.isEmptyBlock(false) ) { if (mb.isInSparseFormat()) { Iterator<IJV> iter = mb.getSparseBlockIterator(); int prevCellRow = -1; @@ -1004,8 +1004,7 @@ public class MLContextConversionUtil { matrixObject.release(); return list; } catch (CacheException e) { - throw new MLContextException("Cache exception while converting matrix object to List<String> CSV format", - e); + throw new MLContextException("Cache exception while converting matrix object to List<String> CSV format", e); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 3e70e36..f40c543 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -359,17 +359,14 @@ public class DMLConfig throws DMLRuntimeException { DMLConfig ret = null; - try - { - //System.out.println(content); + try { DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document domTree = null; domTree = builder.parse( new ByteArrayInputStream(content.getBytes("utf-8")) ); Element root = domTree.getDocumentElement(); ret = new DMLConfig( root ); } - catch(Exception ex) - { + catch(Exception ex) { throw new DMLRuntimeException("Unable to parse DML config.", ex); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/AggBinaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java index 6e87cfb..59cbf95 100644 --- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java +++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java @@ -1418,14 +1418,14 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop boolean ret = true; //right side cached (no agg if left has just one column block) - if( method == MMultMethod.MAPMM_R && getInput().get(0).getDim2() > 0 //known num columns + if( method == MMultMethod.MAPMM_R && getInput().get(0).getDim2() >= 0 //known num columns && getInput().get(0).getDim2() <= getInput().get(0).getColsInBlock() ) { ret = false; } //left side cached (no agg if right has just one row block) - if( method == MMultMethod.MAPMM_L && getInput().get(1).getDim1() > 0 //known num rows + if( method == MMultMethod.MAPMM_L && getInput().get(1).getDim1() >= 0 //known num rows && getInput().get(1).getDim1() <= getInput().get(1).getRowsInBlock() ) { ret = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/BinaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/BinaryOp.java b/src/main/java/org/apache/sysml/hops/BinaryOp.java index d3e36a3..d76cea9 100644 --- a/src/main/java/org/apache/sysml/hops/BinaryOp.java +++ b/src/main/java/org/apache/sysml/hops/BinaryOp.java @@ -910,7 +910,7 @@ public class BinaryOp extends Hop if( mc[0].nnzKnown() && mc[1].nnzKnown() ) lnnz = mc[0].getNonZeros() + mc[1].getNonZeros(); - if( ldim1 > 0 || ldim2 > 0 || lnnz >= 0 ) + if( ldim1 >= 0 || ldim2 >= 0 || lnnz >= 0 ) return new long[]{ldim1, ldim2, lnnz}; } else if( op == OpOp2.RBIND ) { @@ -923,12 +923,12 @@ public class BinaryOp extends Hop if( mc[0].nnzKnown() && mc[1].nnzKnown() ) lnnz = mc[0].getNonZeros() + mc[1].getNonZeros(); - if( ldim1 > 0 || ldim2 > 0 || lnnz >= 0 ) + if( ldim1 >= 0 || ldim2 >= 0 || lnnz >= 0 ) return new long[]{ldim1, ldim2, lnnz}; } else if ( op == OpOp2.SOLVE ) { // Output is a (likely to be dense) vector of size number of columns in the first input - if ( mc[0].getCols() > 0 ) { + if ( mc[0].getCols() >= 0 ) { ret = new long[]{ mc[0].getCols(), 1, mc[0].getCols()}; } } @@ -960,20 +960,20 @@ public class BinaryOp extends Hop } else //GENERAL CASE { - ldim1 = (mc[0].getRows()>0) ? mc[0].getRows() : + ldim1 = (mc[0].rowsKnown()) ? mc[0].getRows() : (mc[1].getRows()>1) ? mc[1].getRows() : -1; - ldim2 = (mc[0].getCols()>0) ? mc[0].getCols() : + ldim2 = (mc[0].colsKnown()) ? mc[0].getCols() : (mc[1].getCols()>1) ? mc[1].getCols() : -1; } sp1 = (mc[0].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[0].getNonZeros()):1.0; sp2 = (mc[1].getNonZeros()>0)?OptimizerUtils.getSparsity(ldim1, ldim2, mc[1].getNonZeros()):1.0; } - if( ldim1>0 && ldim2>0 ) + if( ldim1>=0 && ldim2>=0 ) { if( OptimizerUtils.isBinaryOpConditionalSparseSafe(op) && input2 instanceof LiteralOp ) { long lnnz = (long) (ldim1*ldim2*OptimizerUtils.getBinaryOpSparsityConditionalSparseSafe(sp1, op,(LiteralOp)input2)); - ret = new long[]{ldim1, ldim2, lnnz}; + ret = new long[]{ldim1, ldim2, lnnz}; } else { @@ -1097,11 +1097,11 @@ public class BinaryOp extends Hop Lop ret = null; long m1_dim1 = left.getDim1(); - long m1_dim2 = left.getDim2(); + long m1_dim2 = left.getDim2(); long m2_dim1 = right.getDim1(); long m2_dim2 = right.getDim2(); - long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1>0 && m2_dim1>0) ? (m1_dim1 + m2_dim1) : -1); //output rows - long m3_dim2 = cbind ? ((m1_dim2>0 && m2_dim2>0) ? (m1_dim2 + m2_dim2) : -1): m1_dim2; //output cols + long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1>=0 && m2_dim1>=0) ? (m1_dim1 + m2_dim1) : -1); //output rows + long m3_dim2 = cbind ? ((m1_dim2>=0 && m2_dim2>=0) ? (m1_dim2 + m2_dim2) : -1): m1_dim2; //output cols long m3_nnz = (left.getNnz()>0 && right.getNnz()>0) ? (left.getNnz() + right.getNnz()) : -1; //output nnz long brlen = left.getRowsInBlock(); long bclen = left.getColsInBlock(); @@ -1241,15 +1241,15 @@ public class BinaryOp extends Hop throws HopsException, LopsException { long m1_dim1 = left.getDim1(); - long m1_dim2 = left.getDim2(); + long m1_dim2 = left.getDim2(); long m2_dim1 = right1.getDim1(); long m2_dim2 = right1.getDim2(); long m3_dim1 = right2.getDim1(); - long m3_dim2 = right2.getDim2(); - long m41_dim2 = (m1_dim2>0 && m2_dim2>0) ? (m1_dim2 + m2_dim2) : -1; //output cols + long m3_dim2 = right2.getDim2(); + long m41_dim2 = (m1_dim2>=0 && m2_dim2>=0) ? (m1_dim2 + m2_dim2) : -1; //output cols long m41_nnz = (left.getNnz()>0 && right1.getNnz()>0) ? (left.getNnz() + right1.getNnz()) : -1; //output nnz - long m42_dim2 = (m1_dim2>0 && m2_dim2>0 && m3_dim2>0) ? (m1_dim2 + m2_dim2 + m3_dim2) : -1; //output cols + long m42_dim2 = (m1_dim2>=0 && m2_dim2>=0 && m3_dim2>=0) ? (m1_dim2 + m2_dim2 + m3_dim2) : -1; //output cols long m42_nnz = (left.getNnz()>0 && right1.getNnz()>0 && right2.getNnz()>0) ? (left.getNnz() + right1.getNnz()+ right2.getNnz()) : -1; //output nnz long brlen = left.getRowsInBlock(); @@ -1485,10 +1485,10 @@ public class BinaryOp extends Hop //TODO quantile if( op == OpOp2.CBIND ) { - setDim1( (input1.getDim1()>0) ? input1.getDim1() : input2.getDim1() ); + setDim1( input1.rowsKnown() ? input1.getDim1() : input2.getDim1() ); //ensure both columns are known, otherwise dangerous underestimation due to +(-1) - if( input1.getDim2()>0 && input2.getDim2()>0 ) + if( input1.colsKnown() && input2.colsKnown() ) setDim2( input1.getDim2() + input2.getDim2() ); else setDim2(-1); @@ -1500,10 +1500,10 @@ public class BinaryOp extends Hop } else if( op == OpOp2.RBIND ) { - setDim2( (input1.getDim2()>0) ? input1.getDim2() : input2.getDim2() ); + setDim2( colsKnown() ? input1.getDim2() : input2.getDim2() ); //ensure both rows are known, otherwise dangerous underestimation due to +(-1) - if( input1.getDim1()>0 && input2.getDim1()>0 ) + if( input1.rowsKnown() && input2.rowsKnown() ) setDim1( input1.getDim1() + input2.getDim1() ); else setDim1(-1); @@ -1546,10 +1546,10 @@ public class BinaryOp extends Hop } else //GENERAL CASE { - ldim1 = (input1.getDim1()>0) ? input1.getDim1() - : ((input2.getDim1()>1)?input2.getDim1():-1); - ldim2 = (input1.getDim2()>0) ? input1.getDim2() - : ((input2.getDim2()>1)?input2.getDim2():-1); + ldim1 = (input1.rowsKnown()) ? input1.getDim1() + : ((input2.getDim1()>1)?input2.getDim1():-1); + ldim2 = (input1.colsKnown()) ? input1.getDim2() + : ((input2.getDim2()>1)?input2.getDim2():-1); lnnz1 = input1.getNnz(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/ConvolutionOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java index 99e69b9..fce4958 100644 --- a/src/main/java/org/apache/sysml/hops/ConvolutionOp.java +++ b/src/main/java/org/apache/sysml/hops/ConvolutionOp.java @@ -271,7 +271,7 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop // Compute intermediate memory budget that can be passed to GPU operators // for better CuDNN operator selection at runtime double intermediateMemEstimate = computeIntermediateMemEstimate(-1, -1, -1 ); - if(et == ExecType.GPU && _dim1 > 0 && _dim2 > 0) { + if(et == ExecType.GPU && _dim1 >= 0 && _dim2 >= 0) { // This enables us to compile more efficient matrix-matrix CuDNN operation instead of // row-by-row invocation of multiple vector-matrix CuDNN operations. // This is possible as the operations on GPU are single-threaded @@ -512,7 +512,7 @@ public class ConvolutionOp extends Hop implements MultiThreadedHop ret[0] = mc[0].rowsKnown() ? mc[0].getRows() : -1; ret[1] = mc[0].colsKnown() ? mc[0].getCols() : -1; ret[2] = -1; - return (ret[0]>0 && ret[1]>0) ? ret : null; + return (ret[0]>=0 && ret[1]>=0) ? ret : null; } refreshSizeInformation(); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/DataGenOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java index 500bbd9..6980cd3 100644 --- a/src/main/java/org/apache/sysml/hops/DataGenOp.java +++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java @@ -161,9 +161,9 @@ public class DataGenOp extends Hop implements MultiThreadedHop HashMap<String, Lop> inputLops = new HashMap<>(); for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) { - if( cur.getKey().equals(DataExpression.RAND_ROWS) && _dim1>0 ) + if( cur.getKey().equals(DataExpression.RAND_ROWS) && rowsKnown() ) inputLops.put(cur.getKey(), new LiteralOp(_dim1).constructLops()); - else if( cur.getKey().equals(DataExpression.RAND_COLS) && _dim2>0 ) + else if( cur.getKey().equals(DataExpression.RAND_COLS) && colsKnown() ) inputLops.put(cur.getKey(), new LiteralOp(_dim2).constructLops()); else inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops()); @@ -243,7 +243,7 @@ public class DataGenOp extends Hop implements MultiThreadedHop long dim1 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_ROWS)), memo); long dim2 = computeDimParameterInformation(getInput().get(_paramIndexMap.get(DataExpression.RAND_COLS)), memo); long nnz = _sparsity >= 0 ? (long)(_sparsity * dim1 * dim2) : -1; - if( dim1>0 && dim2>0 ) + if( dim1>=0 && dim2>=0 ) return new long[]{ dim1, dim2, nnz }; } else if ( _op == DataGenMethod.SEQ ) http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/FunctionOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/FunctionOp.java b/src/main/java/org/apache/sysml/hops/FunctionOp.java index 570030f..49000e2 100644 --- a/src/main/java/org/apache/sysml/hops/FunctionOp.java +++ b/src/main/java/org/apache/sysml/hops/FunctionOp.java @@ -188,22 +188,16 @@ public class FunctionOp extends Hop else { if ( getFunctionName().equalsIgnoreCase("qr") ) { // matrix of size same as the input - double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0); - //System.out.println("QRInter " + interOutput/1024/1024); - return interOutput; + return OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0); } else if ( getFunctionName().equalsIgnoreCase("lu")) { // 1D vector - double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0); - //System.out.println("LUInter " + interOutput/1024/1024); - return interOutput; + return OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0); } else if ( getFunctionName().equalsIgnoreCase("eigen")) { // One matrix of size original input and three 1D vectors (used to represent tridiagonal matrix) - double interOutput = OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0) + return OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), getInput().get(0).getDim2(), 1.0) + 3*OptimizerUtils.estimateSizeExactSparsity(getInput().get(0).getDim1(), 1, 1.0); - //System.out.println("EigenInter " + interOutput/1024/1024); - return interOutput; } else if ( getFunctionName().equalsIgnoreCase("svd")) { double interOutput = OptimizerUtils.estimateSizeExactSparsity(1, getInput().get(0).getDim2(), 1.0); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 905d25d..8110838 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -884,19 +884,24 @@ public abstract class Hop implements ParseInfo public boolean dimsKnown() { return ( _dataType == DataType.SCALAR || ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME) - && _dim1 > 0 && _dim2 > 0) ); + && _dim1 >= 0 && _dim2 >= 0) ); } public boolean dimsKnown(boolean includeNnz) { - return ( _dataType == DataType.SCALAR - || ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME) - && _dim1 > 0 && _dim2 > 0 && ((includeNnz)? _nnz>=0 : true))); + return rowsKnown() && colsKnown() + && (_dataType.isScalar() || ((includeNnz) ? _nnz>=0 : true)); } public boolean dimsKnownAny() { - return ( _dataType == DataType.SCALAR - || ((_dataType==DataType.MATRIX || _dataType==DataType.FRAME) - && (_dim1 > 0 || _dim2 > 0)) ); + return rowsKnown() || colsKnown(); + } + + public boolean rowsKnown() { + return _dataType.isScalar() || _dim1 >= 0; + } + + public boolean colsKnown() { + return _dataType.isScalar() || _dim2 >= 0; } public static void resetVisitStatus( ArrayList<Hop> hops ) { @@ -920,7 +925,7 @@ public abstract class Hop implements ParseInfo if( !isVisited() ) return; for( Hop h : getInput() ) - h.resetVisitStatus(); + h.resetVisitStatus(); setVisited(false); } @@ -1742,16 +1747,14 @@ public abstract class Hop implements ParseInfo if( input instanceof UnaryOp ) { - if( ((UnaryOp)input).getOp() == Hop.OpOp1.NROW ) - { + if( ((UnaryOp)input).getOp() == Hop.OpOp1.NROW ) { MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0)); - if( mc.getRows()>0 ) + if( mc.rowsKnown() ) ret = mc.getRows(); } - else if ( ((UnaryOp)input).getOp() == Hop.OpOp1.NCOL ) - { + else if ( ((UnaryOp)input).getOp() == Hop.OpOp1.NCOL ) { MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0)); - if( mc.getCols()>0 ) + if( mc.colsKnown() ) ret = mc.getCols(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/OptimizerUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java index eba4dd7..224752d 100644 --- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java @@ -734,16 +734,16 @@ public class OptimizerUtils //estimate size of bottom boundary blocks long lrlen = rlen % brlen; - if( ncblks > 0 && lrlen > 0 ) + if( ncblks > 0 && lrlen >= 0 ) ret += ncblks * estimateSizeExactSparsity(lrlen, bclen, sp); //estimate size of right boundary blocks long lclen = clen % bclen; - if( nrblks > 0 && lclen > 0 ) + if( nrblks > 0 && lclen >= 0 ) ret += nrblks * estimateSizeExactSparsity(brlen, lclen, sp); //estimate size of bottom right boundary block - if( lrlen > 0 && lclen > 0 ) + if( lrlen >= 0 && lclen >= 0 ) ret += estimateSizeExactSparsity(lrlen, lclen, sp); return ret; @@ -1129,12 +1129,9 @@ public class OptimizerUtils return getSparsity(mc.getRows(), mc.getCols(), mc.getNonZeros()); } - public static double getSparsity( long dim1, long dim2, long nnz ) - { - if( dim1<=0 || dim2<=0 || nnz<0 ) - return 1.0; - else - return Math.min(((double)nnz)/dim1/dim2, 1.0); + public static double getSparsity( long dim1, long dim2, long nnz ) { + return ( dim1<=0 || dim2<=0 || nnz<0 ) ? 1.0 : + Math.min(((double)nnz)/dim1/dim2, 1.0); } public static String toMB(double inB) { @@ -1284,9 +1281,9 @@ public class OptimizerUtils Hop input = uroot.getInput().get(0); if(uroot.getOp() == Hop.OpOp1.NROW) - ret = (input.getDim1()>0) ? input.getDim1() : Double.MAX_VALUE; + ret = input.rowsKnown() ? input.getDim1() : Double.MAX_VALUE; else if( uroot.getOp() == Hop.OpOp1.NCOL ) - ret = (input.getDim2()>0) ? input.getDim2() : Double.MAX_VALUE; + ret = input.colsKnown() ? input.getDim2() : Double.MAX_VALUE; else { double lval = rEvalSimpleDoubleExpression(uroot.getInput().get(0), valMemo); @@ -1323,9 +1320,9 @@ public class OptimizerUtils Hop input = uroot.getInput().get(0); if(uroot.getOp() == Hop.OpOp1.NROW) - ret = (input.getDim1()>0) ? input.getDim1() : Double.MAX_VALUE; + ret = input.rowsKnown() ? input.getDim1() : Double.MAX_VALUE; else if( uroot.getOp() == Hop.OpOp1.NCOL ) - ret = (input.getDim2()>0) ? input.getDim2() : Double.MAX_VALUE; + ret = input.colsKnown() ? input.getDim2() : Double.MAX_VALUE; else { double lval = rEvalSimpleDoubleExpression(uroot.getInput().get(0), valMemo, vars); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java index 3fd6ec0..aa48061 100644 --- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java +++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java @@ -318,7 +318,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop long m2_dim1 = groups.getDim1(); long m2_dim2 = groups.getDim2(); long m3_dim1 = m1_dim1; - long m3_dim2 = ((m1_dim2>0 && m2_dim2>0) ? (m1_dim2 + m2_dim2) : -1); + long m3_dim2 = ((m1_dim2>=0 && m2_dim2>=0) ? (m1_dim2 + m2_dim2) : -1); long m3_nnz = (target.getNnz()>0 && groups.getNnz()>0) ? (target.getNnz() + groups.getNnz()) : -1; long brlen = target.getRowsInBlock(); long bclen = target.getColsInBlock(); @@ -602,7 +602,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop Lop rmEmpty = null; //a) broadcast-based PMM (permutation matrix mult) - if( rmRows && rlen > 0 && mestPM < OptimizerUtils.getRemoteMemBudgetMap() ) + if( rmRows && rlen >= 0 && mestPM < OptimizerUtils.getRemoteMemBudgetMap() ) { boolean needPart = !offsets.dimsKnown() || offsets.getDim1() > DistributedCacheInput.PARTITION_SIZE; if( needPart ){ //requires partitioning http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/ReorgOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ReorgOp.java b/src/main/java/org/apache/sysml/hops/ReorgOp.java index fc85b33..2ac6389 100644 --- a/src/main/java/org/apache/sysml/hops/ReorgOp.java +++ b/src/main/java/org/apache/sysml/hops/ReorgOp.java @@ -477,9 +477,9 @@ public class ReorgOp extends Hop implements MultiThreadedHop // input is a [k1,k2] matrix and output is a [k3,k4] matrix with k1*k2=k3*k4 // #nnz in output is exactly the same as in input if( mc.dimsKnown() ) { - if( _dim1 > 0 ) + if( _dim1 >= 0 ) ret = new long[]{ _dim1, mc.getRows()*mc.getCols()/_dim1, mc.getNonZeros()}; - else if( _dim2 > 0 ) + else if( _dim2 >= 0 ) ret = new long[]{ mc.getRows()*mc.getCols()/_dim2, _dim2, mc.getNonZeros()}; } break; @@ -608,9 +608,9 @@ public class ReorgOp extends Hop implements MultiThreadedHop refreshColsParameterInformation(input3); //refresh cols setNnz(input1.getNnz()); if( !dimsKnown() &&input1.dimsKnown() ) { //reshape allows to infer dims, if input and 1 dim known - if(_dim1 > 0) + if(_dim1 >= 0) _dim2 = (input1._dim1*input1._dim2)/_dim1; - else if(_dim2 > 0) + else if(_dim2 >= 0) _dim1 = (input1._dim1*input1._dim2)/_dim2; } break; http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/TernaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/TernaryOp.java b/src/main/java/org/apache/sysml/hops/TernaryOp.java index 49f67b1..4dd4c40 100644 --- a/src/main/java/org/apache/sysml/hops/TernaryOp.java +++ b/src/main/java/org/apache/sysml/hops/TernaryOp.java @@ -746,7 +746,7 @@ public class TernaryOp extends Hop { double ret = 0; if( _op == OpOp3.CTABLE ) { - if ( _dim1 > 0 && _dim2 > 0 ) { + if ( _dim1 >= 0 && _dim2 >= 0 ) { // output dimensions are known, and hence a MatrixBlock is allocated double sp = OptimizerUtils.getSparsity(_dim1, _dim2, Math.min(nnz, _dim1)); ret = OptimizerUtils.estimateSizeExactSparsity(_dim1, _dim2, sp ); @@ -812,7 +812,7 @@ public class TernaryOp extends Hop break; case IFELSE: for(MatrixCharacteristics lmc : mc) - if( lmc.dimsKnown() && lmc.getRows() > 0 ) //known matrix + if( lmc.dimsKnown() && lmc.getRows() >= 0 ) //known matrix return new long[]{lmc.getRows(), lmc.getCols(), -1}; break; case PLUS_MULT: http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java index 2b8db2a..cfac2d7 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java @@ -75,8 +75,8 @@ public class TemplateCell extends TemplateBase @Override public boolean open(Hop hop) { return hop.dimsKnown() && isValidOperation(hop) - && !(hop.getDim1()==1 && hop.getDim2()==1) - || (hop instanceof IndexingOp && hop.getInput().get(0).getDim2() > 0 + && !(hop.getDim1()==1 && hop.getDim2()==1) + || (hop instanceof IndexingOp && hop.getInput().get(0).getDim2() >= 0 && (((IndexingOp)hop).isColLowerEqualsUpper() || hop.getDim2()==1)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java index effe577..7cb67d0 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateRow.java @@ -95,7 +95,7 @@ public class TemplateRow extends TemplateBase || (hop instanceof AggUnaryOp && ((AggUnaryOp)hop).getDirection()!=Direction.RowCol && hop.getInput().get(0).getDim1()>1 && hop.getInput().get(0).getDim2()>1 && HopRewriteUtils.isAggUnaryOp(hop, SUPPORTED_ROW_AGG)) - || (hop instanceof IndexingOp && hop.getInput().get(0).getDim2() > 0 + || (hop instanceof IndexingOp && hop.getInput().get(0).getDim2() >= 0 && HopRewriteUtils.isColumnRangeIndexing((IndexingOp)hop)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java index 7b5f20e..341c320 100644 --- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java +++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java @@ -241,7 +241,6 @@ public abstract class CostEstimator } stats.put(varname, vs); - //System.out.println(varname+" "+vs); } } @@ -263,8 +262,6 @@ public abstract class CostEstimator long nnz = Long.parseLong(parts[10]); VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false); stats.put(varname, vs); - - //System.out.println(varname+" "+vs); } else if ( optype.equals("cpvar") ) { String varname = parts[1]; @@ -307,10 +304,7 @@ public abstract class CostEstimator FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst; ArrayList<String> outVars = finst.getBoundOutputParamNames(); for( String varname : outVars ) - { stats.put(varname, _unknownStats); - //System.out.println(varname+" "+vs); - } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java index 317f601..3ec5f6a 100644 --- a/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java +++ b/src/main/java/org/apache/sysml/hops/ipa/InterProceduralAnalysis.java @@ -638,8 +638,9 @@ public class InterProceduralAnalysis { MatrixObject moOut = (MatrixObject)dat; MatrixCharacteristics mc = moOut.getMatrixCharacteristics(); - if( OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros()>0)?((double)mc.getNonZeros())/mc.getRows()/mc.getCols():1.0) - < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns()) ) + if( OptimizerUtils.estimateSizeExactSparsity(mc.getRows(), mc.getCols(), (mc.getNonZeros()>0)? + OptimizerUtils.getSparsity(mc):1.0) + < OptimizerUtils.estimateSize(moIn.getNumRows(), moIn.getNumColumns()) ) { //update statistics if necessary mc.setDimension(moIn.getNumRows(), moIn.getNumColumns()); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java index 058e434..13bd81c 100644 --- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java +++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java @@ -1163,7 +1163,6 @@ public class Recompiler public static void extractDAGOutputStatistics(Hop hop, LocalVariableMap vars, boolean overwrite) { if( hop instanceof DataOp && ((DataOp)hop).getDataOpType()==DataOpTypes.TRANSIENTWRITE ) //for all writes to symbol table - //&& hop.getDim1()>0 && hop.getDim2()>0 ) //matrix with known dims { String varName = hop.getName(); if( !vars.keySet().contains(varName) || overwrite ) //not existing so far @@ -1172,10 +1171,8 @@ public class Recompiler if( hop.getDataType()==DataType.MATRIX ) { MatrixObject mo = new MatrixObject(ValueType.DOUBLE, null); - MatrixCharacteristics mc = new MatrixCharacteristics( - hop.getDim1(), hop.getDim2(), - ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), - hop.getNnz()); + MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), + ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), hop.getNnz()); MetaDataFormat meta = new MetaDataFormat(mc,null,null); mo.setMetaData(meta); vars.put(varName, mo); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index d9d9120..0484bb3 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -321,10 +321,10 @@ public class HopRewriteUtils public static Hop createDataGenOp( Hop input, double value ) throws HopsException { - Hop rows = (input.getDim1()>0) ? new LiteralOp(input.getDim1()) : - new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input); - Hop cols = (input.getDim2()>0) ? new LiteralOp(input.getDim2()) : - new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, input); + Hop rows = input.rowsKnown() ? new LiteralOp(input.getDim1()) : + new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input); + Hop cols = input.colsKnown() ? new LiteralOp(input.getDim2()) : + new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, input); Hop val = new LiteralOp(value); HashMap<String, Hop> params = new HashMap<>(); @@ -407,10 +407,10 @@ public class HopRewriteUtils public static Hop createDataGenOp( Hop rowInput, Hop colInput, double value ) throws HopsException { - Hop rows = (rowInput.getDim1()>0) ? new LiteralOp(rowInput.getDim1()) : - new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, rowInput); - Hop cols = (colInput.getDim2()>0) ? new LiteralOp(colInput.getDim2()) : - new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, colInput); + Hop rows = rowInput.rowsKnown() ? new LiteralOp(rowInput.getDim1()) : + new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, rowInput); + Hop cols = colInput.colsKnown() ? new LiteralOp(colInput.getDim2()) : + new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, colInput); Hop val = new LiteralOp(value); HashMap<String, Hop> params = new HashMap<>(); @@ -420,7 +420,7 @@ public class HopRewriteUtils params.put(DataExpression.RAND_MAX, val); params.put(DataExpression.RAND_PDF, new LiteralOp(DataExpression.RAND_PDF_UNIFORM)); params.put(DataExpression.RAND_LAMBDA, new LiteralOp(-1.0)); - params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0)); + params.put(DataExpression.RAND_SPARSITY, new LiteralOp(1.0)); params.put(DataExpression.RAND_SEED, new LiteralOp(DataGenOp.UNSPECIFIED_SEED) ); //note internal refresh size information @@ -440,10 +440,10 @@ public class HopRewriteUtils long nrow = tRowInput ? rowInput.getDim2() : rowInput.getDim1(); long ncol = tColInput ? colInput.getDim1() : rowInput.getDim2(); - Hop rows = (nrow>0) ? new LiteralOp(nrow) : - new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, tRowInput?OpOp1.NCOL:OpOp1.NROW, rowInput); - Hop cols = (ncol>0) ? new LiteralOp(ncol) : - new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, tColInput?OpOp1.NROW:OpOp1.NCOL, colInput); + Hop rows = (nrow>=0) ? new LiteralOp(nrow) : + new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, tRowInput?OpOp1.NCOL:OpOp1.NROW, rowInput); + Hop cols = (ncol>=0) ? new LiteralOp(ncol) : + new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, tColInput?OpOp1.NROW:OpOp1.NCOL, colInput); Hop val = new LiteralOp(value); HashMap<String, Hop> params = new HashMap<>(); @@ -677,12 +677,12 @@ public class HopRewriteUtils { Hop ret = null; if( row ){ - ret = (hop.getDim1()>0) ? new LiteralOp(hop.getDim1()) : - new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, hop); + ret = hop.rowsKnown() ? new LiteralOp(hop.getDim1()) : + new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, hop); } else{ - ret = (hop.getDim2()>0) ? new LiteralOp(hop.getDim2()) : - new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, hop); + ret = hop.colsKnown() ? new LiteralOp(hop.getDim2()) : + new UnaryOp("tmpcols", DataType.SCALAR, ValueType.INT, OpOp1.NCOL, hop); } return ret; @@ -698,8 +698,8 @@ public class HopRewriteUtils public static DataGenOp createSeqDataGenOp( Hop input, boolean asc ) throws HopsException { - Hop to = (input.getDim1()>0) ? new LiteralOp(input.getDim1()) : - new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input); + Hop to = input.rowsKnown() ? new LiteralOp(input.getDim1()) : + new UnaryOp("tmprows", DataType.SCALAR, ValueType.INT, OpOp1.NROW, input); HashMap<String, Hop> params = new HashMap<>(); if( asc ) { @@ -784,20 +784,18 @@ public class HopRewriteUtils /////////////////////////////////// // hop size information - public static boolean isDimsKnown( Hop hop ) - { - return ( hop.getDim1()>0 && hop.getDim2()>0 ); + public static boolean isDimsKnown( Hop hop ) { + return hop.dimsKnown(); } - public static boolean isEmpty( Hop hop ) - { + public static boolean isEmpty( Hop hop ) { return ( hop.getNnz()==0 ); } public static boolean isEqualSize( Hop hop1, Hop hop2 ) { return (hop1.dimsKnown() && hop2.dimsKnown() - && hop1.getDim1() == hop2.getDim1() - && hop1.getDim2() == hop2.getDim2()); + && hop1.getDim1() == hop2.getDim1() + && hop1.getDim2() == hop2.getDim2()); } public static boolean isEqualSize( Hop hop1, Hop... hops ) { @@ -830,8 +828,8 @@ public class HopRewriteUtils } //check row- or column-wise single block constraint - return cols ? (hop.getDim2()>0 && hop.getDim2()<=hop.getColsInBlock()) - : (hop.getDim1()>0 && hop.getDim1()<=hop.getRowsInBlock()); + return cols ? (hop.colsKnown() && hop.getDim2()<=hop.getColsInBlock()) : + (hop.rowsKnown() && hop.getDim1()<=hop.getRowsInBlock()); } public static boolean isOuterProductLikeMM( Hop hop ) { @@ -1365,6 +1363,6 @@ public class HopRewriteUtils public static boolean hasValidInputDims(Hop hop, boolean dim1) { return hop.getInput().stream().allMatch( - h -> (dim1?h.getDim1()>0:h.getDim2()>0)); + h -> dim1 ? h.rowsKnown() : h.colsKnown()); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java index 8453450..d64e4b8 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java @@ -998,8 +998,6 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule if( left instanceof ReorgOp && ((ReorgOp)left).getOp()==ReOrgOp.DIAG //left diag && HopRewriteUtils.isDimsKnown(left) && left.getDim2()>1 ) //diagV2M { - //System.out.println("diag mm rewrite: dim2(right)="+right.getDim2()); - if( right.getDim2()==1 ) //right column vector { //create binary operation over input and right @@ -2558,7 +2556,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule //even if the intermediate is otherwise not required, e.g., when part of a fused operator) if( hi instanceof UnaryOp ) { - if( ((UnaryOp)hi).getOp()==OpOp1.NROW && hi.getInput().get(0).getDim1()>0 ) { + if( ((UnaryOp)hi).getOp()==OpOp1.NROW && hi.getInput().get(0).rowsKnown() ) { Hop hnew = new LiteralOp(hi.getInput().get(0).getDim1()); HopRewriteUtils.replaceChildReference(parent, hi, hnew, pos, false); HopRewriteUtils.cleanupUnreferenced(hi); @@ -2566,7 +2564,7 @@ public class RewriteAlgebraicSimplificationDynamic extends HopRewriteRule + hnew.getName()+" (line "+hi.getBeginLine()+")."); hi = hnew; } - else if( ((UnaryOp)hi).getOp()==OpOp1.NCOL && hi.getInput().get(0).getDim2()>0 ) { + else if( ((UnaryOp)hi).getOp()==OpOp1.NCOL && hi.getInput().get(0).colsKnown() ) { Hop hnew = new LiteralOp(hi.getInput().get(0).getDim2()); HopRewriteUtils.replaceChildReference(parent, hi, hnew, pos, false); HopRewriteUtils.cleanupUnreferenced(hi); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/lops/compile/JobType.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/compile/JobType.java b/src/main/java/org/apache/sysml/lops/compile/JobType.java index 492be7e..2f55088 100644 --- a/src/main/java/org/apache/sysml/lops/compile/JobType.java +++ b/src/main/java/org/apache/sysml/lops/compile/JobType.java @@ -212,7 +212,6 @@ public enum JobType else if (id == 0) { // for ANY, return the bit vector with x number of 1's, // where x = number of actual job types (i.e., excluding INVALID,ANY) - //System.out.println("ANY --> " + JobType.values().length + ", " + (Math.pow(2, JobType.values().length-2)-1) + ", " + (Math.pow(2,13-2)-1)); return (int) Math.pow(2, maxJobID)-1; } else http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java index 3cb882c..7cc1ed3 100644 --- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java @@ -584,20 +584,20 @@ public class BuiltinFunctionExpression extends DataIdentifier long m2clen = getExpr(i).getOutput().getDim2(); if( getOpCode() == BuiltinFunctionOp.CBIND ) { - if (m1rlen > 0 && m2rlen > 0 && m1rlen!=m2rlen) { + if (m1rlen >= 0 && m2rlen >= 0 && m1rlen!=m2rlen) { raiseValidateError("inputs to cbind must have same number of rows: input 1 rows: " + m1rlen+", input 2 rows: "+m2rlen, conditional, LanguageErrorCodes.INVALID_PARAMETERS); } - appendDim1 = (m2rlen>0) ? m2rlen : appendDim1; - appendDim2 = (appendDim2>0 && m2clen>0) ? appendDim2 + m2clen : -1; + appendDim1 = (m2rlen>=0) ? m2rlen : appendDim1; + appendDim2 = (appendDim2>=0 && m2clen>=0) ? appendDim2 + m2clen : -1; } else if( getOpCode() == BuiltinFunctionOp.RBIND ) { - if (m1clen > 0 && m2clen > 0 && m1clen!=m2clen) { + if (m1clen >= 0 && m2clen >= 0 && m1clen!=m2clen) { raiseValidateError("inputs to rbind must have same number of columns: input 1 columns: " + m1clen+", input 2 columns: "+m2clen, conditional, LanguageErrorCodes.INVALID_PARAMETERS); } - appendDim1 = (appendDim1>0 && m2rlen>0)? appendDim1 + m2rlen : -1; - appendDim2 = (m2clen>0) ? m2clen : appendDim2; + appendDim1 = (appendDim1>=0 && m2rlen>=0)? appendDim1 + m2rlen : -1; + appendDim2 = (m2clen>=0) ? m2clen : appendDim2; } } output.setDimensions(appendDim1, appendDim2); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/parser/DataExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java index 4d14b66..d17443f 100644 --- a/src/main/java/org/apache/sysml/parser/DataExpression.java +++ b/src/main/java/org/apache/sysml/parser/DataExpression.java @@ -1123,22 +1123,18 @@ public class DataExpression extends DataIdentifier /////////////////////////////////////////////////////////////////// Expression rowsExpr = getVarParam(RAND_ROWS); if (rowsExpr instanceof IntIdentifier) { - if (((IntIdentifier)rowsExpr).getValue() >= 1 ) { - rowsLong = ((IntIdentifier)rowsExpr).getValue(); - } - else { + if( ((IntIdentifier)rowsExpr).getValue() < 0 ) { raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + ((IntIdentifier)rowsExpr).getValue(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + ((IntIdentifier)rowsExpr).getValue(), conditional); } + rowsLong = ((IntIdentifier)rowsExpr).getValue(); } else if (rowsExpr instanceof DoubleIdentifier) { - if (((DoubleIdentifier)rowsExpr).getValue() >= 1 ) { - rowsLong = UtilFunctions.toLong(Math.floor(((DoubleIdentifier)rowsExpr).getValue())); + if ( ((DoubleIdentifier)rowsExpr).getValue() < 0 ) { + raiseValidateError("In rand statement, can only assign rows a long " + + "(integer) value >= 0 -- attempted to assign value: " + rowsExpr.toString(), conditional); } - else { - raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + rowsExpr.toString(), conditional); - } + rowsLong = UtilFunctions.toLong(Math.floor(((DoubleIdentifier)rowsExpr).getValue())); } else if (rowsExpr instanceof DataIdentifier && !(rowsExpr instanceof IndexedIdentifier)) { @@ -1149,11 +1145,10 @@ public class DataExpression extends DataIdentifier // handle int constant ConstIdentifier constValue = currConstVars.get(identifierName); if (constValue instanceof IntIdentifier){ - // check rows is >= 1 --- throw exception - if (((IntIdentifier)constValue).getValue() < 1){ + if( ((IntIdentifier)constValue).getValue() < 0 ){ raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } // update row expr with new IntIdentifier long roundedValue = ((IntIdentifier)constValue).getValue(); @@ -1163,22 +1158,20 @@ public class DataExpression extends DataIdentifier } // handle double constant else if (constValue instanceof DoubleIdentifier){ - - if (((DoubleIdentifier)constValue).getValue() < 1.0){ + if (((DoubleIdentifier)constValue).getValue() < 0){ raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(double) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } // update row expr with new IntIdentifier (rounded down) long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue(); rowsExpr = new IntIdentifier(roundedValue, this); addVarParam(RAND_ROWS, rowsExpr); rowsLong = roundedValue; - } else { // exception -- rows must be integer or double constant raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } } else { @@ -1190,30 +1183,25 @@ public class DataExpression extends DataIdentifier // handle general expression rowsExpr.validateExpression(ids, currConstVars, conditional); } - - + /////////////////////////////////////////////////////////////////// // HANDLE COLUMNS /////////////////////////////////////////////////////////////////// Expression colsExpr = getVarParam(RAND_COLS); if (colsExpr instanceof IntIdentifier) { - if (((IntIdentifier)colsExpr).getValue() >= 1 ) { - colsLong = ((IntIdentifier)colsExpr).getValue(); - } - else { + if (((IntIdentifier)colsExpr).getValue() < 0 ) { raiseValidateError("In rand statement, can only assign cols a long " + - "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + colsExpr.toString(), conditional); } + colsLong = ((IntIdentifier)colsExpr).getValue(); } else if (colsExpr instanceof DoubleIdentifier) { - if (((DoubleIdentifier)colsExpr).getValue() >= 1 ) { - colsLong = Double.valueOf((Math.floor(((DoubleIdentifier)colsExpr).getValue()))).longValue(); + if (((DoubleIdentifier)colsExpr).getValue() < 0 ) { + raiseValidateError("In rand statement, can only assign cols a long " + + "(integer) value >= 0 -- attempted to assign value: " + colsExpr.toString(), conditional); } - else { - raiseValidateError("In rand statement, can only assign rows a long " + - "(integer) value >= 1 -- attempted to assign value: " + colsExpr.toString(), conditional); - } + colsLong = Double.valueOf((Math.floor(((DoubleIdentifier)colsExpr).getValue()))).longValue(); } else if (colsExpr instanceof DataIdentifier && !(colsExpr instanceof IndexedIdentifier)) { @@ -1224,44 +1212,39 @@ public class DataExpression extends DataIdentifier // handle int constant ConstIdentifier constValue = currConstVars.get(identifierName); if (constValue instanceof IntIdentifier){ - - // check cols is >= 1 --- throw exception - if (((IntIdentifier)constValue).getValue() < 1){ + if (((IntIdentifier)constValue).getValue() < 0 ){ raiseValidateError("In rand statement, can only assign cols a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } // update col expr with new IntIdentifier long roundedValue = ((IntIdentifier)constValue).getValue(); colsExpr = new IntIdentifier(roundedValue, this); addVarParam(RAND_COLS, colsExpr); - colsLong = roundedValue; + colsLong = roundedValue; } // handle double constant else if (constValue instanceof DoubleIdentifier){ - - if (((DoubleIdentifier)constValue).getValue() < 1){ + if (((DoubleIdentifier)constValue).getValue() < 0){ raiseValidateError("In rand statement, can only assign cols a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(double) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } // update col expr with new IntIdentifier (rounded down) long roundedValue = Double.valueOf(Math.floor(((DoubleIdentifier)constValue).getValue())).longValue(); colsExpr = new IntIdentifier(roundedValue, this); addVarParam(RAND_COLS, colsExpr); colsLong = roundedValue; - } else { // exception -- rows must be integer or double constant raiseValidateError("In rand statement, can only assign cols a long " + - "(integer) value >= 1 -- attempted to assign value: " + constValue.toString(), conditional); + "(integer) value >= 0 -- attempted to assign value: " + constValue.toString(), conditional); } } else { // handle general expression colsExpr.validateExpression(ids, currConstVars, conditional); } - - } + } else { // handle general expression colsExpr.validateExpression(ids, currConstVars, conditional); @@ -1307,8 +1290,7 @@ public class DataExpression extends DataIdentifier // handle general expression minExpr.validateExpression(ids, currConstVars, conditional); } - - } + } else { // handle general expression minExpr.validateExpression(ids, currConstVars, conditional); @@ -1325,25 +1307,21 @@ public class DataExpression extends DataIdentifier // check if the DataIdentifier variable is a ConstIdentifier String identifierName = ((DataIdentifier)maxExpr).getName(); - if (currConstVars.containsKey(identifierName)){ - + if (currConstVars.containsKey(identifierName)) { // handle int constant ConstIdentifier constValue = currConstVars.get(identifierName); - if (constValue instanceof IntIdentifier){ - + if (constValue instanceof IntIdentifier) { // update min expr with new IntIdentifier long roundedValue = ((IntIdentifier)constValue).getValue(); maxExpr = new DoubleIdentifier(roundedValue, this); addVarParam(RAND_MAX, maxExpr); } // handle double constant - else if (constValue instanceof DoubleIdentifier){ - + else if (constValue instanceof DoubleIdentifier) { // update col expr with new IntIdentifier (rounded down) double roundedValue = ((DoubleIdentifier)constValue).getValue(); maxExpr = new DoubleIdentifier(roundedValue, this); addVarParam(RAND_MAX, maxExpr); - } else { // exception -- rows must be integer or double constant @@ -1354,8 +1332,8 @@ public class DataExpression extends DataIdentifier else { // handle general expression maxExpr.validateExpression(ids, currConstVars, conditional); - } - } + } + } else { // handle general expression maxExpr.validateExpression(ids, currConstVars, conditional); @@ -1376,8 +1354,6 @@ public class DataExpression extends DataIdentifier ((IndexedIdentifier) getOutput()).setOriginalDimensions(targetAsSeen.getDim1(), targetAsSeen.getDim2()); //((IndexedIdentifier) getOutput()).setOriginalDimensions(getOutput().getDim1(), getOutput().getDim2()); } - //getOutput().computeDataType(); - if (getOutput() instanceof IndexedIdentifier){ LOG.warn(this.printWarningLocation() + "Output for Rand Statement may have incorrect size information"); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/parser/Identifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Identifier.java b/src/main/java/org/apache/sysml/parser/Identifier.java index b2a679d..47a8034 100644 --- a/src/main/java/org/apache/sysml/parser/Identifier.java +++ b/src/main/java/org/apache/sysml/parser/Identifier.java @@ -251,6 +251,6 @@ public abstract class Identifier extends Expression } public boolean dimsKnown(){ - return ( _dim1 > 0 && _dim2 > 0); + return ( _dim1 >= 0 && _dim2 >= 0); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java index 9c2b2d8..09dafc2 100644 --- a/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java +++ b/src/main/java/org/apache/sysml/parser/IndexedIdentifier.java @@ -102,7 +102,7 @@ public class IndexedIdentifier extends DataIdentifier if (rowLB_1_1 < 1){ raiseValidateError("lower-bound row index " + rowLB_1_1 + " initialized to out of bounds value. Value must be >= 1", conditional); } - if ((this.getOrigDim1() > 0) && (rowLB_1_1 > this.getOrigDim1())) { + if ((this.getOrigDim1() >= 0) && (rowLB_1_1 > this.getOrigDim1())) { raiseValidateError("lower-bound row index " + rowLB_1_1 + " initialized to out of bounds value. Rows in " + this.getName() + ": " + this.getOrigDim1(), conditional); } // valid lower row bound value @@ -138,7 +138,7 @@ public class IndexedIdentifier extends DataIdentifier LOG.info(this.printInfoLocation() + "lower-bound row index " + identifierName + " initialized to " + tempRowLB + " May cause runtime exception (runtime value must be >= 1)"); validRowLB = false; } - if (this.getOrigDim1() > 0 && tempRowLB > this.getOrigDim1()){ + if (this.getOrigDim1() >= 0 && tempRowLB > this.getOrigDim1()){ LOG.info(this.printInfoLocation() + "lower-bound row index " + identifierName + " initialized to " + tempRowLB + " May cause runtime exception (Rows in " + this.getName() + ": " + this.getOrigDim1() +")"); @@ -185,7 +185,7 @@ public class IndexedIdentifier extends DataIdentifier if (rowUB_1_1 < 1){ raiseValidateError("upper-bound row index " + rowUB_1_1 + " out of bounds value. Value must be >= 1", conditional); } - if ((this.getOrigDim1() > 0) && (rowUB_1_1 > this.getOrigDim1())) { + if ((this.getOrigDim1() >= 0) && (rowUB_1_1 > this.getOrigDim1())) { raiseValidateError("upper-bound row index " + rowUB_1_1 + " out of bounds value. Rows in " + this.getName() + ": " + this.getOrigDim1(), conditional); } if (isConst_rowLowerBound && rowUB_1_1 < rowLB_1){ @@ -220,7 +220,7 @@ public class IndexedIdentifier extends DataIdentifier LOG.info(this.printInfoLocation() + "upper-bound row index " + identifierName + " initialized to " + tempRowUB + " May cause runtime exception (runtime value must be >= 1)"); validRowUB = false; } - if (this.getOrigDim1() > 0 && tempRowUB > this.getOrigDim1()){ + if (this.getOrigDim1() >= 0 && tempRowUB > this.getOrigDim1()){ LOG.info(this.printInfoLocation() + "upper-bound row index " + identifierName + " initialized to " + tempRowUB + " May cause runtime exception (Rows in " + this.getName() + ": " + this.getOrigDim1() +")"); validRowUB = false; } @@ -262,7 +262,7 @@ public class IndexedIdentifier extends DataIdentifier if (colLB_1_1 < 1){ raiseValidateError("lower-bound column index " + colLB_1_1 + " initialized to out of bounds value. Value must be >= 1", conditional); } - if ((this.getOrigDim2() > 0) && (colLB_1_1 > this.getOrigDim2())){ + if ((this.getOrigDim2() >= 0) && (colLB_1_1 > this.getOrigDim2())){ raiseValidateError("lower-bound column index " + colLB_1_1 + " initialized to out of bounds value. Columns in " + this.getName() + ": " + this.getOrigDim2(), conditional); } // valid lower row bound value @@ -296,7 +296,7 @@ public class IndexedIdentifier extends DataIdentifier LOG.info(this.printInfoLocation() + "lower-bound column index " + identifierName + " initialized to " + tempColLB + " May cause runtime exception (runtime value must be >= 1)"); validColLB = false; } - if (this.getOrigDim2() > 0 && tempColLB > this.getOrigDim2()){ + if (this.getOrigDim2() >= 0 && tempColLB > this.getOrigDim2()){ LOG.info(this.printInfoLocation() + "lower-bound column index " + identifierName + " initialized to " + tempColLB + " May cause runtime exception (Columns in " + this.getName() @@ -344,7 +344,7 @@ public class IndexedIdentifier extends DataIdentifier if (colUB_1_1 < 1){ raiseValidateError("upper-bound column index " + colUB_1_1 + " out of bounds value. Value must be >= 1", conditional); } - if ((this.getOrigDim2() > 0) && (colUB_1_1 > this.getOrigDim2())) { + if ((this.getOrigDim2() >= 0) && (colUB_1_1 > this.getOrigDim2())) { raiseValidateError("upper-bound column index " + colUB_1_1 + " out of bounds value. Columns in " + this.getName() + ": " + this.getOrigDim2(), conditional); } if (isConst_rowLowerBound && colUB_1_1 < colLB_1) { @@ -386,7 +386,7 @@ public class IndexedIdentifier extends DataIdentifier + " May cause runtime exception (runtime value must be >= 1)"); validColUB = false; } - if (this.getOrigDim2() > 0 && tempColUB > this.getOrigDim2()){ + if (this.getOrigDim2() >= 0 && tempColUB > this.getOrigDim2()){ LOG.info(this.printInfoLocation() + "upper-bound column index " + identifierName + " initialized to " + tempColUB + " May cause runtime exception (Columns in " + this.getName() @@ -435,7 +435,7 @@ public class IndexedIdentifier extends DataIdentifier } // CASE: (lower == constant) && (upper == null) && (dimIndex > 0) --> rowCount - lower bound + 1 - else if (isConst_rowLowerBound && _rowUpperBound == null && this.getOrigDim1() > 0) { + else if (isConst_rowLowerBound && _rowUpperBound == null && this.getOrigDim1() >= 0) { long rowCount = this.getOrigDim1(); if (_rowLowerBound instanceof IntIdentifier) updatedRowDim = rowCount - ((IntIdentifier)_rowLowerBound).getValue() + 1; @@ -485,7 +485,7 @@ public class IndexedIdentifier extends DataIdentifier } // CASE: (lower == constant) && (upper == null) && (dimIndex > 0) --> colCount - lower bound + 1 - else if (isConst_colLowerBound && _colUpperBound == null && this.getOrigDim2() > 0) { + else if (isConst_colLowerBound && _colUpperBound == null && this.getOrigDim2() >= 0) { long colCount = this.getOrigDim2(); if (_colLowerBound instanceof IntIdentifier) updatedColDim = colCount - ((IntIdentifier)_colLowerBound).getValue() + 1; http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java index 1c1621a..cc33352 100644 --- a/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java +++ b/src/main/java/org/apache/sysml/parser/ParForStatementBlock.java @@ -1352,7 +1352,7 @@ public class ParForStatementBlock extends ForStatementBlock //scale row function 'out' with col dimensionality long colDim = _vsParent.getVariable(idat._name).getDim2(); - if( colDim > 0 ) { + if( colDim >= 0 ) { out.scale( colDim ); } else { http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java index 33d67c1..acc2e8d 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java @@ -74,7 +74,7 @@ public abstract class SpoofRowwise extends SpoofOperator } public boolean isConstDim2(long dim2) { return (this == NO_AGG_CONST || this == COL_AGG_CONST) - || (dim2>0 && isRowTypeB1()); + || (dim2>=0 && isRowTypeB1()); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java index 10e924f..28510cd 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java @@ -115,7 +115,7 @@ public class FrameObject extends CacheableData<FrameBlock> if( schema.equals("*") ) { //populate default schema int clen = (int) getNumColumns(); - if( clen > 0 ) //known number of cols + if( clen >= 0 ) //known number of cols _schema = UtilFunctions.nCopies(clen, ValueType.STRING); } else { http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java index afb2c71..9599993 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitioner.java @@ -104,7 +104,7 @@ public abstract class DataPartitioner int brlen = mc.getRowsPerBlock(); int bclen = mc.getColsPerBlock(); long nonZeros = mc.getNonZeros(); - double sparsity = (nonZeros>=0 && rows>0 && cols>0)? + double sparsity = mc.dimsKnown(true) ? ((double)nonZeros)/(rows*cols) : 1.0; if( !force ) //try to optimize, if format not forced http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java index 1994628..b998cba 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java @@ -266,27 +266,24 @@ public class RemoteParForMR for( Path lpath : IOUtilFunctions.getSequenceFilePaths(fs, path) ) { SequenceFile.Reader reader = new SequenceFile.Reader(fs,lpath,job); - try - { - while( reader.next(key, value) ) - { - //System.out.println("key="+key.get()+", value="+value.toString()); + try { + while( reader.next(key, value) ) { if( !tmp.containsKey( key.get() ) ) - tmp.put(key.get(), new LocalVariableMap ()); + tmp.put(key.get(), new LocalVariableMap ()); Object[] dat = ProgramConverter.parseDataObject( value.toString() ); - tmp.get( key.get() ).put((String)dat[0], (Data)dat[1]); - countAll++; + tmp.get( key.get() ).put((String)dat[0], (Data)dat[1]); + countAll++; } - } + } finally { IOUtilFunctions.closeSilently(reader); } - } + } LOG.debug("Num remote worker results (before deduplication): "+countAll); LOG.debug("Num remote worker results: "+tmp.size()); //create return array - return tmp.values().toArray(new LocalVariableMap[0]); + return tmp.values().toArray(new LocalVariableMap[0]); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java index 3cd19f5..14eadeb 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java @@ -178,16 +178,12 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC else if( ii == InputInfo.BinaryBlockInputInfo ) diagBlocks = createBinaryBlockStagingFile( fnameOld, stagingDir ); - //System.out.println("Executed phase 1 in "+time.stop()); - //Phase 2: scan empty rows/cols if( diagBlocks ) ret = createKeyMappingDiag(stagingDir, mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), ii); else ret = createKeyMapping(stagingDir, mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), ii); - //System.out.println("Executed phase 2 in "+time.stop()); - //Phase 3: create output files MapReduceTool.deleteFileIfExistOnHDFS(fnameNew); if( ii == InputInfo.TextCellInputInfo @@ -202,11 +198,8 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC else createBlockResultFile( fnameNew, stagingDir, mc.getRows(), mc.getCols(), ret, mc.getNonZeros(), mc.getRowsPerBlock(), mc.getColsPerBlock(), ii ); } - - //System.out.println("Executed phase 3 in "+time.stop()); } - catch( IOException ioe ) - { + catch( IOException ioe ) { throw new DMLRuntimeException( ioe ); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java index 6ac8cfc..0294121 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendMInstruction.java @@ -59,14 +59,12 @@ public class AppendMInstruction extends AppendInstruction implements IDistribute } @Override //IDistributedCacheConsumer - public boolean isDistCacheOnlyIndex( String inst, byte index ) - { + public boolean isDistCacheOnlyIndex( String inst, byte index ) { return (index==input2 && index!=input1); } @Override //IDistributedCacheConsumer - public void addDistCacheIndex( String inst, ArrayList<Byte> indexes ) - { + public void addDistCacheIndex( String inst, ArrayList<Byte> indexes ) { indexes.add(input2); } @@ -86,9 +84,9 @@ public class AppendMInstruction extends AppendInstruction implements IDistribute if(in1 == null) continue; - //check for boundary block + //check for boundary block w/ awareness of zero rows or columns int blen = _cbind ? blockColFactor : blockRowFactor; - long lastBlockColIndex = (long)Math.ceil((double)_offset/blen); + long lastBlockColIndex = Math.max((long)Math.ceil((double)_offset/blen),1); //case 1: pass through of non-boundary blocks MatrixIndexes ix = in1.getIndexes(); @@ -105,13 +103,13 @@ public class AppendMInstruction extends AppendInstruction implements IDistribute DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input2); if( _cbind ) { cachedValues.add(output, new IndexedMatrixValue( - new MatrixIndexes(ix.getRowIndex(), ix.getColumnIndex()+1), - dcInput.getDataBlock((int)ix.getRowIndex(), 1).getValue())); + new MatrixIndexes(ix.getRowIndex(), ix.getColumnIndex()+1), + dcInput.getDataBlock((int)ix.getRowIndex(), 1).getValue())); } else { cachedValues.add(output, new IndexedMatrixValue( - new MatrixIndexes(ix.getRowIndex()+1, ix.getColumnIndex()), - dcInput.getDataBlock(1, (int)ix.getColumnIndex()).getValue())); + new MatrixIndexes(ix.getRowIndex()+1, ix.getColumnIndex()), + dcInput.getDataBlock(1, (int)ix.getColumnIndex()).getValue())); } } //case 3: append operation on boundary block @@ -135,16 +133,16 @@ public class AppendMInstruction extends AppendInstruction implements IDistribute } } else { //rbind - value_in2 = dcInput.getDataBlock(1, (int)ix.getRowIndex()).getValue(); + value_in2 = dcInput.getDataBlock(1, (int)ix.getColumnIndex()).getValue(); if(in1.getValue().getNumRows()+value_in2.getNumRows()>blen) { IndexedMatrixValue second=cachedValues.holdPlace(output, valueClass); second.getIndexes().setIndexes(ix.getRowIndex()+1, ix.getColumnIndex()); outlist.add(second); } } - + OperationsOnMatrixValues.performAppend(in1.getValue(), value_in2, outlist, - blockRowFactor, blockColFactor, _cbind, true, 0); + blockRowFactor, blockColFactor, _cbind, true, 0); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java index fae8ddd..d3847a5 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/AppendRInstruction.java @@ -42,11 +42,10 @@ public class AppendRInstruction extends AppendInstruction { byte in2 = Byte.parseByte(parts[2]); byte out = Byte.parseByte(parts[3]); boolean cbind = Boolean.parseBoolean(parts[4]); - + return new AppendRInstruction(null, in1, in2, out, cbind, str); } - @Override public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int brlen, int bclen) http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/mr/CtableInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/CtableInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/CtableInstruction.java index 213f401..9bb9973 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/CtableInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/CtableInstruction.java @@ -99,7 +99,7 @@ public class CtableInstruction extends MRInstruction { } public boolean knownOutputDims() { - return (_outputDim1 >0 && _outputDim2>0); + return (_outputDim1 >=0 && _outputDim2>=0); } public static CtableInstruction parseInstruction ( String str ) http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java index 821193b..db78476 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/ZeroOutInstruction.java @@ -88,7 +88,6 @@ public class ZeroOutInstruction extends UnaryMRInstructionBase { if(tempRange.rowStart==-1 && !complementary)//if no overlap, directly write them out { cachedValues.add(output, in); - //System.out.println("just write down: "+in); return; } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java index affd145..eb04947 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java @@ -84,12 +84,12 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { private static boolean preservesPartitioning( MatrixCharacteristics mcIn1, MatrixCharacteristics mcIn2, boolean cbind ) { - long ncblksIn1 = cbind ? - (long)Math.ceil((double)mcIn1.getCols()/mcIn1.getColsPerBlock()) : - (long)Math.ceil((double)mcIn1.getRows()/mcIn1.getRowsPerBlock()); + //determine if append is partitioning-preserving based on number of input and output blocks + //with awareness of zero number of rows or columns + long ncblksIn1 = cbind ? mcIn1.getNumColBlocks() : mcIn1.getNumRowBlocks(); long ncblksOut = cbind ? - (long)Math.ceil(((double)mcIn1.getCols()+mcIn2.getCols())/mcIn1.getColsPerBlock()) : - (long)Math.ceil(((double)mcIn1.getRows()+mcIn2.getRows())/mcIn1.getRowsPerBlock()); + Math.max((long)Math.ceil(((double)mcIn1.getCols()+mcIn2.getCols())/mcIn1.getColsPerBlock()),1) : + Math.max((long)Math.ceil(((double)mcIn1.getRows()+mcIn2.getRows())/mcIn1.getRowsPerBlock()),1); //mappend is partitioning-preserving if in-block append (e.g., common case of colvector append) return (ncblksIn1 == ncblksOut); @@ -99,25 +99,22 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { { private static final long serialVersionUID = 2738541014432173450L; - private PartitionedBroadcast<MatrixBlock> _pm = null; - private boolean _cbind = true; - private long _offset; - private int _brlen; - private int _bclen; - private long _lastBlockColIndex; + private final PartitionedBroadcast<MatrixBlock> _pm; + private final boolean _cbind; + private final int _brlen; + private final int _bclen; + private final long _lastBlockColIndex; public MapSideAppendFunction(PartitionedBroadcast<MatrixBlock> binput, boolean cbind, long offset, int brlen, int bclen) { _pm = binput; _cbind = cbind; - - _offset = offset; _brlen = brlen; _bclen = bclen; //check for boundary block - int blen = cbind ? bclen : brlen; - _lastBlockColIndex = (long)Math.ceil((double)_offset/blen); + _lastBlockColIndex = Math.max((long)Math.ceil( + (double)offset/(cbind ? bclen : brlen)),1); } @Override @@ -199,8 +196,8 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { _cbind = cbind; //check for boundary block - int blen = cbind ? bclen : brlen; - _lastBlockColIndex = (long)Math.ceil((double)offset/blen); + _lastBlockColIndex = Math.max((long)Math.ceil( + (double)offset/(cbind ? bclen : brlen)),1); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendRSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendRSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendRSPInstruction.java index 40358bf..aba212e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendRSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendRSPInstruction.java @@ -52,14 +52,14 @@ public class MatrixAppendRSPInstruction extends AppendRSPInstruction { //execute reduce-append operations (partitioning preserving) JavaPairRDD<MatrixIndexes,MatrixBlock> out = in1 - .join(in2) - .mapValues(new ReduceSideAppendFunction(_cbind)); + .join(in2) + .mapValues(new ReduceSideAppendFunction(_cbind)); //put output RDD handle into symbol table updateBinaryAppendOutputMatrixCharacteristics(sec, _cbind); sec.setRDDHandleForVariable(output.getName(), out); sec.addLineageRDD(output.getName(), input1.getName()); - sec.addLineageRDD(output.getName(), input2.getName()); + sec.addLineageRDD(output.getName(), input2.getName()); } private static class ReduceSideAppendFunction implements Function<Tuple2<MatrixBlock, MatrixBlock>, MatrixBlock> @@ -67,7 +67,7 @@ public class MatrixAppendRSPInstruction extends AppendRSPInstruction { private static final long serialVersionUID = -6763904972560309095L; private boolean _cbind = true; - + public ReduceSideAppendFunction(boolean cbind) { _cbind = cbind; } @@ -76,11 +76,7 @@ public class MatrixAppendRSPInstruction extends AppendRSPInstruction { public MatrixBlock call(Tuple2<MatrixBlock, MatrixBlock> arg0) throws Exception { - MatrixBlock left = arg0._1(); - MatrixBlock right = arg0._2(); - - return left.appendOperations(right, new MatrixBlock(), _cbind); + return arg0._1().appendOperations(arg0._2(), new MatrixBlock(), _cbind); } } } - http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java index d6d177f..1a1633e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java @@ -278,8 +278,9 @@ public class RandSPInstruction extends UnarySPInstruction { double totalSize = OptimizerUtils.estimatePartitionedSizeExactSparsity( lrows, lcols, rowsInBlock, colsInBlock, sparsity); //overestimate for on disk, ensures hdfs block per partition double hdfsBlkSize = InfrastructureAnalyzer.getHDFSBlockSize(); - long numBlocks = new MatrixCharacteristics(lrows, lcols, rowsInBlock, colsInBlock).getNumBlocks(); - long numColBlocks = (long)Math.ceil((double)lcols/(double)colsInBlock); + MatrixCharacteristics tmp = new MatrixCharacteristics(lrows, lcols, rowsInBlock, colsInBlock); + long numBlocks = tmp.getNumBlocks(); + long numColBlocks = tmp.getNumColBlocks(); //a) in-memory seed rdd construction if( numBlocks < INMEMORY_NUMBLOCKS_THRESHOLD ) @@ -295,7 +296,7 @@ public class RandSPInstruction extends UnarySPInstruction { //for load balancing: degree of parallelism such that ~128MB per partition int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); - + //create seeds rdd seedsRDD = sec.getSparkContext().parallelizePairs(seeds, numPartitions); } @@ -754,6 +755,5 @@ public class RandSPInstruction extends UnarySPInstruction { return ( OptimizerUtils.isValidCPDimensions(lrows, lcols) && OptimizerUtils.isValidCPMatrixSize(lrows, lcols, sparsity) && size < OptimizerUtils.getLocalMemBudget() ); - } - + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java index 055b33b..e244c3c 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java @@ -648,8 +648,8 @@ public class FrameRDDConverterUtils private static void flushBlocksToList( Long ix, FrameBlock fb, ArrayList<Tuple2<Long,FrameBlock>> ret ) throws DMLRuntimeException - { - if( fb != null && fb.getNumRows()>0 ) + { + if( fb != null && fb.getNumRows()>=0 ) ret.add(new Tuple2<>(ix, fb)); } } @@ -788,8 +788,8 @@ public class FrameRDDConverterUtils private static void flushBlocksToList( Long ix, FrameBlock fb, ArrayList<Tuple2<Long,FrameBlock>> ret ) throws DMLRuntimeException - { - if( fb != null && fb.getNumRows()>0 ) + { + if( fb != null && fb.getNumRows()>=0 ) ret.add(new Tuple2<>(ix, fb)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java index ee1db4c..cfdacf9 100644 --- a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java +++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java @@ -91,7 +91,7 @@ public abstract class MatrixReader SparseBlock sblock = ret.getSparseBlock(); //create synchronization points for MCSR (start row per block row) if( sblock instanceof SparseBlockMCSR && clen > bclen //multiple col blocks - && clen > 0 && bclen > 0 && rlen > 0 && brlen > 0 ) { //all dims known + && clen >= 0 && bclen > 0 && rlen >= 0 && brlen > 0 ) { //all dims known //note: allocate w/ min 2 nnz to ensure allocated row object because //adaptive change from scalar to row could cause synchronization issues for( int i=0; i<rlen; i+=brlen ) http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java index c0f4d99..e8a8e1a 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java @@ -56,7 +56,7 @@ public class ReaderTextCSV extends MatrixReader { //allocate output matrix block MatrixBlock ret = null; - if( rlen>0 && clen>0 ) //otherwise allocated on read + if( rlen>=0 && clen>=0 ) //otherwise allocated on read ret = createOutputMatrixBlock(rlen, clen, (int)rlen, (int)clen, estnnz, true, false); //prepare file access
