[SYSTEMML-2190] Change parser/hops block sizes from long to int This patch refactors the parser and hop data structures to use int instead of long block sizes for consistency the runtime and avoid unnecessarily bloated compiler data structures.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/2b74a794 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/2b74a794 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/2b74a794 Branch: refs/heads/master Commit: 2b74a794480f6ee830443c393bbf164fc13676df Parents: b3419b0 Author: Matthias Boehm <[email protected]> Authored: Fri Mar 23 19:00:07 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Mar 23 22:18:26 2018 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/DataOp.java | 6 +-- src/main/java/org/apache/sysml/hops/Hop.java | 14 +++---- .../org/apache/sysml/hops/LeftIndexingOp.java | 9 ++-- .../sysml/hops/ParameterizedBuiltinOp.java | 14 +++---- .../apache/sysml/hops/cost/CostEstimator.java | 20 ++++----- .../hops/cost/CostEstimatorStaticRuntime.java | 12 +++--- .../org/apache/sysml/hops/cost/VarStats.java | 20 ++++----- .../sysml/hops/rewrite/HopRewriteUtils.java | 8 ++-- .../RewriteSplitDagDataDependentOperators.java | 4 +- .../rewrite/RewriteSplitDagUnknownCSVRead.java | 4 +- .../java/org/apache/sysml/lops/CSVReBlock.java | 31 ++++++-------- .../java/org/apache/sysml/lops/ReBlock.java | 9 ++-- .../sysml/parser/BuiltinFunctionExpression.java | 28 ++++++------- .../org/apache/sysml/parser/DataExpression.java | 7 ++-- .../org/apache/sysml/parser/DataIdentifier.java | 17 +++----- .../org/apache/sysml/parser/Expression.java | 43 +++++++------------- .../org/apache/sysml/parser/Identifier.java | 20 ++++----- .../sysml/parser/RelationalExpression.java | 8 ++-- .../context/ExecutionContext.java | 14 +++---- .../runtime/matrix/MatrixCharacteristics.java | 12 ++++-- .../functions/codegen/CPlanComparisonTest.java | 4 +- 21 files changed, 134 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/DataOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/DataOp.java b/src/main/java/org/apache/sysml/hops/DataOp.java index 01e4878..2495997 100644 --- a/src/main/java/org/apache/sysml/hops/DataOp.java +++ b/src/main/java/org/apache/sysml/hops/DataOp.java @@ -80,7 +80,7 @@ public class DataOp extends Hop * @param colsPerBlock cols per block */ public DataOp(String l, DataType dt, ValueType vt, DataOpTypes dop, - String fname, long dim1, long dim2, long nnz, long rowsPerBlock, long colsPerBlock) { + String fname, long dim1, long dim2, long nnz, int rowsPerBlock, int colsPerBlock) { super(l, dt, vt); _dataop = dop; @@ -96,7 +96,7 @@ public class DataOp extends Hop } public DataOp(String l, DataType dt, ValueType vt, DataOpTypes dop, - String fname, long dim1, long dim2, long nnz, UpdateType update, long rowsPerBlock, long colsPerBlock) { + String fname, long dim1, long dim2, long nnz, UpdateType update, int rowsPerBlock, int colsPerBlock) { this(l, dt, vt, dop, fname, dim1, dim2, nnz, rowsPerBlock, colsPerBlock); setUpdateType(update); } @@ -218,7 +218,7 @@ public class DataOp extends Hop _dataop = type; } - public void setOutputParams(long dim1, long dim2, long nnz, UpdateType update, long rowsPerBlock, long colsPerBlock) { + public void setOutputParams(long dim1, long dim2, long nnz, UpdateType update, int rowsPerBlock, int colsPerBlock) { setDim1(dim1); setDim2(dim2); setNnz(nnz); http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 31ccd33..0ced336 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -80,8 +80,8 @@ public abstract class Hop implements ParseInfo protected boolean _visited = false; protected long _dim1 = -1; protected long _dim2 = -1; - protected long _rows_in_block = -1; - protected long _cols_in_block = -1; + protected int _rows_in_block = -1; + protected int _cols_in_block = -1; protected long _nnz = -1; protected UpdateType _updateType = UpdateType.COPY; @@ -253,7 +253,7 @@ public abstract class Hop implements ParseInfo return false; } - public void setOutputBlocksizes( long brlen, long bclen ) { + public void setOutputBlocksizes(int brlen, int bclen) { setRowsInBlock( brlen ); setColsInBlock( bclen ); } @@ -787,19 +787,19 @@ public abstract class Hop implements ParseInfo h._parent.add(this); } - public long getRowsInBlock() { + public int getRowsInBlock() { return _rows_in_block; } - public void setRowsInBlock(long rowsInBlock) { + public void setRowsInBlock(int rowsInBlock) { _rows_in_block = rowsInBlock; } - public long getColsInBlock() { + public int getColsInBlock() { return _cols_in_block; } - public void setColsInBlock(long colsInBlock) { + public void setColsInBlock(int colsInBlock) { _cols_in_block = colsInBlock; } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java index a660d24..2ca90ab 100644 --- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java +++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java @@ -133,12 +133,9 @@ public class LeftIndexingOp extends Hop if (isRightHandSideScalar()) { //insert cast to matrix if necessary (for reuse MR runtime) rightInput = new UnaryCP(getInput().get(1).constructLops(), - OperationTypes.CAST_AS_MATRIX, - DataType.MATRIX, ValueType.DOUBLE); - rightInput.getOutputParameters().setDimensions( (long)1, (long)1, - (long)ConfigurationManager.getBlocksize(), - (long)ConfigurationManager.getBlocksize(), - (long)-1); + OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE); + rightInput.getOutputParameters().setDimensions(1L, 1L, + ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1L); } else rightInput = getInput().get(1).constructLops(); http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java index 302fb7a..a1cea17 100644 --- a/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java +++ b/src/main/java/org/apache/sysml/hops/ParameterizedBuiltinOp.java @@ -480,8 +480,8 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop { //get input vector (without materializing diag()) Hop input = targetHop.getInput().get(0); - long brlen = input.getRowsInBlock(); - long bclen = input.getColsInBlock(); + int brlen = input.getRowsInBlock(); + int bclen = input.getColsInBlock(); MemoTable memo = new MemoTable(); boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp)input).isPPredOperation()); @@ -545,8 +545,8 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop Hop input = targetHop; long rlen = input.getDim1(); long clen = input.getDim2(); - long brlen = input.getRowsInBlock(); - long bclen = input.getColsInBlock(); + int brlen = input.getRowsInBlock(); + int bclen = input.getColsInBlock(); long nnz = input.getNnz(); boolean rmRows = ((LiteralOp)marginHop).getStringValue().equals("rows"); @@ -576,7 +576,7 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop Hop cumsumInput = emptyInd; if( !rmRows ){ cumsumInput = HopRewriteUtils.createTranspose(emptyInd); - HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this); + HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this); } UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM); @@ -673,8 +673,8 @@ public class ParameterizedBuiltinOp extends Hop implements MultiThreadedHop Hop input = targetHop; long rlen = input.getDim1(); long clen = input.getDim2(); - long brlen = input.getRowsInBlock(); - long bclen = input.getColsInBlock(); + int brlen = input.getRowsInBlock(); + int bclen = input.getColsInBlock(); boolean rmRows = ((LiteralOp)marginHop).getStringValue().equals("rows"); //construct lops via new partial hop dag and subsequent lops construction http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java index 341c320..f09de20 100644 --- a/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java +++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimator.java @@ -229,8 +229,8 @@ public abstract class CostEstimator MatrixCharacteristics mc = mo.getMatrixCharacteristics(); long rlen = mc.getRows(); long clen = mc.getCols(); - long brlen = mc.getRowsPerBlock(); - long bclen = mc.getColsPerBlock(); + int brlen = mc.getRowsPerBlock(); + int bclen = mc.getColsPerBlock(); long nnz = mc.getNonZeros(); boolean inmem = mo.getStatusAsString().equals("CACHED"); vs = new VarStats(rlen, clen, brlen, bclen, nnz, inmem); @@ -257,8 +257,8 @@ public abstract class CostEstimator String varname = parts[1]; long rlen = Long.parseLong(parts[6]); long clen = Long.parseLong(parts[7]); - long brlen = Long.parseLong(parts[8]); - long bclen = Long.parseLong(parts[9]); + int brlen = Integer.parseInt(parts[8]); + int bclen = Integer.parseInt(parts[9]); long nnz = Long.parseLong(parts[10]); VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false); stats.put(varname, vs); @@ -285,8 +285,8 @@ public abstract class CostEstimator String varname = randInst.output.getName(); long rlen = randInst.getRows(); long clen = randInst.getCols(); - long brlen = randInst.getRowsInBlock(); - long bclen = randInst.getColsInBlock(); + int brlen = randInst.getRowsInBlock(); + int bclen = randInst.getColsInBlock(); long nnz = (long) (randInst.getSparsity() * rlen * clen); VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, true); stats.put(varname, vs); @@ -335,11 +335,11 @@ public abstract class CostEstimator byte outIndex = Byte.parseByte(parts[2]); long rlen = parts[3].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[3]); long clen = parts[4].contains(Lop.VARIABLE_NAME_PLACEHOLDER)?-1:UtilFunctions.parseToLong(parts[4]); - long brlen = Long.parseLong(parts[5]); - long bclen = Long.parseLong(parts[6]); + int brlen = Integer.parseInt(parts[5]); + int bclen = Integer.parseInt(parts[6]); long nnz = (long) (Double.parseDouble(parts[9]) * rlen * clen); VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false); - stats.put(String.valueOf(outIndex), vs); + stats.put(String.valueOf(outIndex), vs); } } @@ -354,7 +354,7 @@ public abstract class CostEstimator VarStats vs = e.getValue(); if( vs !=null ) { - MatrixCharacteristics mc = new MatrixCharacteristics(vs._rlen, vs._clen, (int)vs._brlen, (int)vs._bclen, (long)vs._nnz); + MatrixCharacteristics mc = new MatrixCharacteristics(vs._rlen, vs._clen, vs._brlen, vs._bclen, (long)vs._nnz); dims.put(ix, mc); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java index 1d4074d..8329493 100644 --- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java +++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java @@ -659,8 +659,8 @@ public class CostEstimatorStaticRuntime extends CostEstimator { boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn)); - double bytes = (double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn)); - double mbytes = bytes / (1024*1024); + double bytes = (double)MatrixBlock.estimateSizeOnDisk(dm, dn, (long)(ds*dm*dn)); + double mbytes = bytes / (1024*1024); double ret = -1; if( sparse ) @@ -679,8 +679,8 @@ public class CostEstimatorStaticRuntime extends CostEstimator { boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn)); - double bytes = (double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn)); - double mbytes = bytes / (1024*1024); + double bytes = (double)MatrixBlock.estimateSizeOnDisk(dm, dn, (long)(ds*dm*dn)); + double mbytes = bytes / (1024*1024); double ret = -1; @@ -719,7 +719,7 @@ public class CostEstimatorStaticRuntime extends CostEstimator { boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn)); - double ret = ((double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn))) / (1024*1024); + double ret = ((double)MatrixBlock.estimateSizeOnDisk(dm, dn, (long)(ds*dm*dn))) / (1024*1024); if( sparse ) ret /= DEFAULT_MBS_FSREAD_BINARYBLOCK_SPARSE; else //dense @@ -732,7 +732,7 @@ public class CostEstimatorStaticRuntime extends CostEstimator { boolean sparse = MatrixBlock.evalSparseFormatOnDisk(dm, dn, (long)(ds*dm*dn)); - double ret = ((double)MatrixBlock.estimateSizeOnDisk((long)dm, (long)dn, (long)(ds*dm*dn))) / (1024*1024); + double ret = ((double)MatrixBlock.estimateSizeOnDisk(dm, dn, (long)(ds*dm*dn))) / (1024*1024); if( sparse ) ret /= DEFAULT_MBS_FSWRITE_BINARYBLOCK_SPARSE; http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/cost/VarStats.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/cost/VarStats.java b/src/main/java/org/apache/sysml/hops/cost/VarStats.java index 8709a50..3fc2c03 100644 --- a/src/main/java/org/apache/sysml/hops/cost/VarStats.java +++ b/src/main/java/org/apache/sysml/hops/cost/VarStats.java @@ -19,19 +19,18 @@ package org.apache.sysml.hops.cost; -public class VarStats -{ +import org.apache.sysml.hops.OptimizerUtils; - +public class VarStats +{ long _rlen = -1; long _clen = -1; - long _brlen = -1; - long _bclen = -1; - double _nnz = -1; + int _brlen = -1; + int _bclen = -1; + long _nnz = -1; boolean _inmem = false; - public VarStats( long rlen, long clen, long brlen, long bclen, long nnz, boolean inmem ) - { + public VarStats( long rlen, long clen, int brlen, int bclen, long nnz, boolean inmem ) { _rlen = rlen; _clen = clen; _brlen = brlen; @@ -40,9 +39,8 @@ public class VarStats _inmem = inmem; } - public double getSparsity() - { - return (_nnz<0) ? 1.0 : (double)_nnz/_rlen/_clen; + public double getSparsity() { + return OptimizerUtils.getSparsity(_rlen, _clen, _nnz); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index 74711e3..abb1c7d 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -738,7 +738,7 @@ public class HopRewriteUtils return dop; } - public static void setOutputParameters( Hop hop, long rlen, long clen, long brlen, long bclen, long nnz ) { + public static void setOutputParameters( Hop hop, long rlen, long clen, int brlen, int bclen, long nnz ) { hop.setDim1( rlen ); hop.setDim2( clen ); hop.setOutputBlocksizes(brlen, bclen ); @@ -764,13 +764,11 @@ public class HopRewriteUtils dest.setParseInfo(src); } - public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, Hop src ) - { + public static void updateHopCharacteristics( Hop hop, int brlen, int bclen, Hop src ) { updateHopCharacteristics(hop, brlen, bclen, new MemoTable(), src); } - public static void updateHopCharacteristics( Hop hop, long brlen, long bclen, MemoTable memo, Hop src ) - { + public static void updateHopCharacteristics( Hop hop, int brlen, int bclen, MemoTable memo, Hop src ) { //update block sizes and dimensions hop.setOutputBlocksizes(brlen, bclen); hop.refreshSizeInformation(); http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java index 13f4a50..b7ab7d1 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagDataDependentOperators.java @@ -126,8 +126,8 @@ public class RewriteSplitDagDataDependentOperators extends StatementBlockRewrite long clen = c.getDim2(); long nnz = c.getNnz(); UpdateType update = c.getUpdateType(); - long brlen = c.getRowsInBlock(); - long bclen = c.getColsInBlock(); + int brlen = c.getRowsInBlock(); + int bclen = c.getColsInBlock(); if( hasTWrites && moveTWrite) //reuse existing transient_write { http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java index 5351c0d..3e423b8 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteSplitDagUnknownCSVRead.java @@ -86,8 +86,8 @@ public class RewriteSplitDagUnknownCSVRead extends StatementBlockRewriteRule long clen = reblock.getDim2(); long nnz = reblock.getNnz(); UpdateType update = reblock.getUpdateType(); - long brlen = reblock.getRowsInBlock(); - long bclen = reblock.getColsInBlock(); + int brlen = reblock.getRowsInBlock(); + int bclen = reblock.getColsInBlock(); //replace reblock inputs to avoid dangling references across dags //(otherwise, for instance, literal ops are shared across dags) http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/lops/CSVReBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/CSVReBlock.java b/src/main/java/org/apache/sysml/lops/CSVReBlock.java index 55571f7..1217cd1 100644 --- a/src/main/java/org/apache/sysml/lops/CSVReBlock.java +++ b/src/main/java/org/apache/sysml/lops/CSVReBlock.java @@ -32,34 +32,30 @@ import org.apache.sysml.parser.Expression.ValueType; */ public class CSVReBlock extends Lop { - public static final String OPCODE = "csvrblk"; - Long rows_per_block; - Long cols_per_block; + private int _rows_per_block; + private int _cols_per_block; - public CSVReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, ExecType et) throws LopsException + public CSVReBlock(Lop input, int rows_per_block, int cols_per_block, DataType dt, ValueType vt, ExecType et) throws LopsException { - super(Lop.Type.CSVReBlock, dt, vt); - this.addInput(input); + super(Lop.Type.CSVReBlock, dt, vt); + addInput(input); input.addOutput(this); - this.rows_per_block = rows_per_block; - this.cols_per_block = cols_per_block; + _rows_per_block = rows_per_block; + _cols_per_block = cols_per_block; - /* - * This lop can be executed only in CSVREBLOCK job. - */ boolean breaksAlignment = false; boolean aligner = false; boolean definesMRJob = true; lps.addCompatibility(JobType.CSV_REBLOCK); if(et == ExecType.MR) { - this.lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob ); + lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob ); } else if(et == ExecType.SPARK) { - this.lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob ); + lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob ); } else { throw new LopsException("Incorrect execution type for CSVReblock:" + et); @@ -67,9 +63,8 @@ public class CSVReBlock extends Lop } @Override - public String toString() { - - return "CSVReblock - rows per block = " + rows_per_block + " cols per block " + cols_per_block ; + public String toString() { + return "CSVReblock - rows per block = " + _rows_per_block + " cols per block " + _cols_per_block ; } private String prepCSVProperties() throws LopsException { @@ -130,9 +125,9 @@ public class CSVReBlock extends Lop sb.append( OPERAND_DELIMITOR ); sb.append( prepOutputOperand(output)); sb.append( OPERAND_DELIMITOR ); - sb.append( rows_per_block ); + sb.append( _rows_per_block ); sb.append( OPERAND_DELIMITOR ); - sb.append( cols_per_block ); + sb.append( _cols_per_block ); sb.append( OPERAND_DELIMITOR ); sb.append( prepCSVProperties() ); http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/lops/ReBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/ReBlock.java b/src/main/java/org/apache/sysml/lops/ReBlock.java index b8d4e14..15a2bf6 100644 --- a/src/main/java/org/apache/sysml/lops/ReBlock.java +++ b/src/main/java/org/apache/sysml/lops/ReBlock.java @@ -36,12 +36,12 @@ public class ReBlock extends Lop private boolean _outputEmptyBlocks = true; - private Long _rows_per_block; - private Long _cols_per_block; + private int _rows_per_block; + private int _cols_per_block; - public ReBlock(Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, boolean outputEmptyBlocks, ExecType et) throws LopsException + public ReBlock(Lop input, int rows_per_block, int cols_per_block, DataType dt, ValueType vt, boolean outputEmptyBlocks, ExecType et) throws LopsException { - super(Lop.Type.ReBlock, dt, vt); + super(Lop.Type.ReBlock, dt, vt); this.addInput(input); input.addOutput(this); @@ -66,7 +66,6 @@ public class ReBlock extends Lop @Override public String toString() { - return "Reblock - rows per block = " + _rows_per_block + " cols per block " + _cols_per_block ; } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java index 880b698..2da1f23 100644 --- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java @@ -26,6 +26,7 @@ import java.util.HashSet; import org.antlr.v4.runtime.ParserRuleContext; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.LanguageException.LanguageErrorCodes; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.util.ConvolutionUtils; import org.apache.sysml.runtime.util.UtilFunctions; @@ -1288,26 +1289,22 @@ public class BuiltinFunctionExpression extends DataIdentifier } } - private void setBinaryOutputProperties(DataIdentifier output) - throws LanguageException - { + private void setBinaryOutputProperties(DataIdentifier output) throws LanguageException { DataType dt1 = getFirstExpr().getOutput().getDataType(); DataType dt2 = getSecondExpr().getOutput().getDataType(); - DataType dtOut = (dt1==DataType.MATRIX || dt2==DataType.MATRIX) ? + DataType dtOut = (dt1==DataType.MATRIX || dt2==DataType.MATRIX) ? DataType.MATRIX : DataType.SCALAR; if( dt1==DataType.MATRIX && dt2==DataType.MATRIX ) checkMatchingDimensions(getFirstExpr(), getSecondExpr(), true); - long[] dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr()); + MatrixCharacteristics dims = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr()); output.setDataType(dtOut); - output.setValueType(dtOut==DataType.MATRIX ? ValueType.DOUBLE : + output.setValueType(dtOut==DataType.MATRIX ? ValueType.DOUBLE : computeValueType(getFirstExpr(), getSecondExpr(), true)); - output.setDimensions(dims[0], dims[1]); - output.setBlockDimensions (dims[2], dims[3]); + output.setDimensions(dims.getRows(), dims.getCols()); + output.setBlockDimensions (dims.getRowsPerBlock(), dims.getColsPerBlock()); } - private void setTernaryOutputProperties(DataIdentifier output, boolean conditional) - throws LanguageException - { + private void setTernaryOutputProperties(DataIdentifier output, boolean conditional) throws LanguageException { DataType dt1 = getFirstExpr().getOutput().getDataType(); DataType dt2 = getSecondExpr().getOutput().getDataType(); DataType dt3 = getThirdExpr().getOutput().getDataType(); @@ -1319,13 +1316,14 @@ public class BuiltinFunctionExpression extends DataIdentifier checkMatchingDimensions(getFirstExpr(), getThirdExpr(), false, conditional); if( dt2==DataType.MATRIX && dt3==DataType.MATRIX ) checkMatchingDimensions(getSecondExpr(), getThirdExpr(), false, conditional); - long[] dims1 = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr()); - long[] dims2 = getBinaryMatrixCharacteristics(getSecondExpr(), getThirdExpr()); + MatrixCharacteristics dims1 = getBinaryMatrixCharacteristics(getFirstExpr(), getSecondExpr()); + MatrixCharacteristics dims2 = getBinaryMatrixCharacteristics(getSecondExpr(), getThirdExpr()); output.setDataType(dtOut); output.setValueType(dtOut==DataType.MATRIX ? ValueType.DOUBLE : computeValueType(getSecondExpr(), getThirdExpr(), true)); - output.setDimensions(Math.max(dims1[0], dims2[0]), Math.max(dims1[1], dims2[1])); - output.setBlockDimensions (Math.max(dims1[2], dims2[2]), Math.max(dims1[3], dims2[3])); + output.setDimensions(Math.max(dims1.getRows(), dims2.getRows()), Math.max(dims1.getCols(), dims2.getCols())); + output.setBlockDimensions(Math.max(dims1.getRowsPerBlock(), dims2.getRowsPerBlock()), + Math.max(dims1.getColsPerBlock(), dims2.getColsPerBlock())); } private void expandArguments() { http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/DataExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java index 9764dfc..fa762f5 100644 --- a/src/main/java/org/apache/sysml/parser/DataExpression.java +++ b/src/main/java/org/apache/sysml/parser/DataExpression.java @@ -905,9 +905,10 @@ public class DataExpression extends DataIdentifier if (getVarParam(ROWBLOCKCOUNTPARAM) instanceof ConstIdentifier && getVarParam(COLUMNBLOCKCOUNTPARAM) instanceof ConstIdentifier) { - Long rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? null : Long.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); - Long columnBlockCount = (getVarParam(COLUMNBLOCKCOUNTPARAM) == null) ? null : Long.valueOf(getVarParam(COLUMNBLOCKCOUNTPARAM).toString()); - + Integer rowBlockCount = (getVarParam(ROWBLOCKCOUNTPARAM) == null) ? + null : Integer.valueOf(getVarParam(ROWBLOCKCOUNTPARAM).toString()); + Integer columnBlockCount = (getVarParam(COLUMNBLOCKCOUNTPARAM) == null) ? + null : Integer.valueOf(getVarParam(COLUMNBLOCKCOUNTPARAM).toString()); if ((rowBlockCount != null) && (columnBlockCount != null)) { getOutput().setBlockDimensions(rowBlockCount, columnBlockCount); } else if ((rowBlockCount != null) || (columnBlockCount != null)) { http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/DataIdentifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DataIdentifier.java b/src/main/java/org/apache/sysml/parser/DataIdentifier.java index 1873549..34caf4e 100644 --- a/src/main/java/org/apache/sysml/parser/DataIdentifier.java +++ b/src/main/java/org/apache/sysml/parser/DataIdentifier.java @@ -20,15 +20,15 @@ package org.apache.sysml.parser; -public class DataIdentifier extends Identifier +public class DataIdentifier extends Identifier { protected String _name; - protected String _valueTypeString; + protected String _valueTypeString; public DataIdentifier(DataIdentifier passed){ setProperties(passed); _name = passed.getName(); - _valueTypeString = passed.getValueType().toString(); + _valueTypeString = passed.getValueType().toString(); // set location information setParseInfo(passed); @@ -39,7 +39,6 @@ public class DataIdentifier extends Identifier DataIdentifier newId = new DataIdentifier(this); String newIdName = prefix + _name; newId.setName(newIdName); - return newId; } @@ -55,6 +54,7 @@ public class DataIdentifier extends Identifier public String getName(){ return _name; } + public void setName(String name){ _name = name; } @@ -88,11 +88,9 @@ public class DataIdentifier extends Identifier } @Override - public boolean equals(Object that) - { + public boolean equals(Object that) { if( !(that instanceof DataIdentifier) ) return false; - DataIdentifier target = (DataIdentifier)that; if(getName()!=null && !getName().equals(target.getName())) return false; @@ -106,14 +104,11 @@ public class DataIdentifier extends Identifier return false; if(!(this.getDim2() == target.getDim2())) return false; - return true; - } @Override - public int hashCode() - { + public int hashCode() { return super.hashCode(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/Expression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java index ac30657..2a6503d 100644 --- a/src/main/java/org/apache/sysml/parser/Expression.java +++ b/src/main/java/org/apache/sysml/parser/Expression.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.sysml.hops.Hop.FileFormatTypes; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; public abstract class Expression implements ParseInfo @@ -551,51 +552,35 @@ public abstract class Expression implements ParseInfo * Returns the matrix characteristics for scalar-scalar, scalar-matrix, matrix-scalar, matrix-matrix * operations. This method is aware of potentially unknowns and matrix-vector (col/row) operations. * - * * @param expression1 The first expression * @param expression2 The second expression - * @return long array of 4 values, where [0] is the number of rows (rlen), + * @return matrix characteristics * [1] is the number of columns (clen), [2] is the number of rows in a block (brlen), * and [3] is the number of columns in a block (bclen). Default (unknown) values are * -1. Scalar values are all 0. */ - public static long[] getBinaryMatrixCharacteristics(Expression expression1, Expression expression2) - { - long[] ret = new long[]{ -1, -1, -1, -1 }; - + public static MatrixCharacteristics getBinaryMatrixCharacteristics(Expression expression1, Expression expression2) { Identifier idleft = expression1.getOutput(); Identifier idright = expression2.getOutput(); - if( idleft.getDataType()==DataType.SCALAR && idright.getDataType()==DataType.SCALAR ) { - ret[0] = 0; - ret[1] = 0; - ret[2] = 0; - ret[3] = 0; + return new MatrixCharacteristics(0, 0, 0, 0); } else if( idleft.getDataType()==DataType.SCALAR && idright.getDataType()==DataType.MATRIX ) { - ret[0] = idright.getDim1(); - ret[1] = idright.getDim2(); - ret[2] = idright.getRowsInBlock(); - ret[3] = idright.getColumnsInBlock(); + return new MatrixCharacteristics(idright.getDim1(), idright.getDim2(), idright.getRowsInBlock(), idright.getColumnsInBlock()); } else if( idleft.getDataType()==DataType.MATRIX && idright.getDataType()==DataType.SCALAR ) { - ret[0] = idleft.getDim1(); - ret[1] = idleft.getDim2(); - ret[2] = idleft.getRowsInBlock(); - ret[3] = idleft.getColumnsInBlock(); + return new MatrixCharacteristics(idleft.getDim1(), idleft.getDim2(), idleft.getRowsInBlock(), idleft.getColumnsInBlock()); } else if( idleft.getDataType()==DataType.MATRIX && idright.getDataType()==DataType.MATRIX ) { - ret[0] = idleft.getDim1(); - ret[1] = idleft.getDim2(); - ret[2] = idleft.getRowsInBlock(); - ret[3] = idleft.getColumnsInBlock(); - if( ret[0] < 0 && idright.getDim1() > 1 ) //robustness for row vectors - ret[0] = idright.getDim1(); - if( ret[1] < 0 && idright.getDim2() > 1 ) //robustness for row vectors - ret[1] = idright.getDim2(); + MatrixCharacteristics mc = new MatrixCharacteristics( + idleft.getDim1(), idleft.getDim2(), idleft.getRowsInBlock(), idleft.getColumnsInBlock()); + if( mc.getRows() < 0 && idright.getDim1() > 1 ) //robustness for row vectors + mc.setRows(idright.getDim1()); + if( mc.getCols() < 0 && idright.getDim2() > 1 ) //robustness for row vectors + mc.setCols(idright.getDim2()); + return mc; } - - return ret; + return new MatrixCharacteristics(-1, -1, -1, -1); } /////////////////////////////////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/Identifier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Identifier.java b/src/main/java/org/apache/sysml/parser/Identifier.java index 47a8034..3c44452 100644 --- a/src/main/java/org/apache/sysml/parser/Identifier.java +++ b/src/main/java/org/apache/sysml/parser/Identifier.java @@ -29,13 +29,12 @@ public abstract class Identifier extends Expression protected ValueType _valueType; protected long _dim1; protected long _dim2; - protected long _rows_in_block; - protected long _columns_in_block; + protected int _rows_in_block; + protected int _columns_in_block; protected long _nnz; protected FormatType _formatType; - public Identifier() - { + public Identifier() { _dim1 = -1; _dim2 = -1; _dataType = DataType.UNKNOWN; @@ -47,11 +46,9 @@ public abstract class Identifier extends Expression _formatType = null; } - public void setProperties(Identifier i) - { + public void setProperties(Identifier i) { if (i == null) return; - _dataType = i.getDataType(); _valueType = i.getValueType(); if (i instanceof IndexedIdentifier) { @@ -68,8 +65,7 @@ public abstract class Identifier extends Expression _formatType = i.getFormatType(); } - public void setDimensionValueProperties(Identifier i) - { + public void setDimensionValueProperties(Identifier i) { if (i instanceof IndexedIdentifier) { IndexedIdentifier ixi = (IndexedIdentifier)i; _dim1 = ixi.getOrigDim1(); @@ -101,7 +97,7 @@ public abstract class Identifier extends Expression _dim2 = dim2; } - public void setBlockDimensions(long dim1, long dim2){ + public void setBlockDimensions(int dim1, int dim2){ _rows_in_block = dim1; _columns_in_block = dim2; } @@ -130,11 +126,11 @@ public abstract class Identifier extends Expression return _formatType; } - public long getRowsInBlock(){ + public int getRowsInBlock(){ return _rows_in_block; } - public long getColumnsInBlock(){ + public int getColumnsInBlock(){ return _columns_in_block; } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/parser/RelationalExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/RelationalExpression.java b/src/main/java/org/apache/sysml/parser/RelationalExpression.java index 91e9da3..af6424e 100644 --- a/src/main/java/org/apache/sysml/parser/RelationalExpression.java +++ b/src/main/java/org/apache/sysml/parser/RelationalExpression.java @@ -22,6 +22,7 @@ package org.apache.sysml.parser; import java.util.HashMap; import org.apache.sysml.parser.LanguageException.LanguageErrorCodes; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; public class RelationalExpression extends Expression { @@ -142,11 +143,10 @@ public class RelationalExpression extends Expression if(isLeftMatrix && isRightMatrix) { checkMatchingDimensions(_left, _right, true); } - - long[] dims = getBinaryMatrixCharacteristics(_left, _right); + MatrixCharacteristics dims = getBinaryMatrixCharacteristics(_left, _right); output.setDataType(DataType.MATRIX); - output.setDimensions(dims[0], dims[1]); - output.setBlockDimensions(dims[2], dims[3]); + output.setDimensions(dims.getRows(), dims.getCols()); + output.setBlockDimensions(dims.getRowsPerBlock(), dims.getColsPerBlock()); //since SystemML only supports double matrices, the value type is forced to //double; once we support boolean matrices this needs to change http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java index af8c8b9..5416d5d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java @@ -274,22 +274,18 @@ public class ExecutionContext { return mo.acquireRead(); } - public void setMetaData(String varName, long nrows, long ncols) - throws DMLRuntimeException - { + public void setMetaData(String varName, long nrows, long ncols) throws DMLRuntimeException { MatrixObject mo = getMatrixObject(varName); if(mo.getNumRows() == nrows && mo.getNumColumns() == ncols) return; - MetaData oldMetaData = mo.getMetaData(); if( oldMetaData == null || !(oldMetaData instanceof MetaDataFormat) ) throw new DMLRuntimeException("Metadata not available"); - - MatrixCharacteristics mc = new MatrixCharacteristics((long)nrows, (long)ncols, - (int) mo.getNumRowsPerBlock(), (int)mo.getNumColumnsPerBlock()); + MatrixCharacteristics mc = new MatrixCharacteristics(nrows, ncols, + (int) mo.getNumRowsPerBlock(), (int)mo.getNumColumnsPerBlock()); mo.setMetaData(new MetaDataFormat(mc, - ((MetaDataFormat)oldMetaData).getOutputInfo(), - ((MetaDataFormat)oldMetaData).getInputInfo())); + ((MetaDataFormat)oldMetaData).getOutputInfo(), + ((MetaDataFormat)oldMetaData).getInputInfo())); } /** http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java index e9554f9..0fd9f22 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java @@ -78,9 +78,7 @@ public class MatrixCharacteristics implements Serializable private long nonZero = -1; private boolean ubNnz = false; - public MatrixCharacteristics() { - - } + public MatrixCharacteristics() {} public MatrixCharacteristics(long nr, long nc, int bnr, int bnc) { set(nr, nc, bnr, bnc); @@ -122,11 +120,19 @@ public class MatrixCharacteristics implements Serializable public long getRows(){ return numRows; } + + public void setRows(long rlen) { + numRows = rlen; + } public long getCols(){ return numColumns; } + public void setCols(long clen) { + numColumns = clen; + } + public long getLength() { return numRows * numColumns; } http://git-wip-us.apache.org/repos/asf/systemml/blob/2b74a794/src/test/java/org/apache/sysml/test/integration/functions/codegen/CPlanComparisonTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CPlanComparisonTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CPlanComparisonTest.java index bc68e68..0b85fae 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CPlanComparisonTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CPlanComparisonTest.java @@ -240,11 +240,11 @@ public class CPlanComparisonTest extends AutomatedTestBase private Hop createDataOp(DataType dt) { return new DataOp("tmp"+_seq.getNextID(), dt, ValueType.DOUBLE, - DataOpTypes.TRANSIENTREAD, "tmp", 77L, 7L, -1L, 1000L, 1000L); + DataOpTypes.TRANSIENTREAD, "tmp", 77L, 7L, -1L, 1000, 1000); } private static Hop createDataOp(String name, DataType dt) { return new DataOp(name, dt, ValueType.DOUBLE, - DataOpTypes.TRANSIENTREAD, "tmp", 77L, 7L, -1L, 1000L, 1000L); + DataOpTypes.TRANSIENTREAD, "tmp", 77L, 7L, -1L, 1000, 1000); } }
