Repository: systemml Updated Branches: refs/heads/master d7f765331 -> d5f20b43b
[MINOR] Rename rangeReIndex to rightIndex and related cleanups This patch renames the rangeReIndex to rightIndex to make it consistent with the existing leftIndex. Furthermore, this also centralizes these left/right indexing opcodes and modifies their usage accordingly. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/e60a4c29 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/e60a4c29 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/e60a4c29 Branch: refs/heads/master Commit: e60a4c29dd1e6f3ccdcee8cae27628b75c2d4742 Parents: d7f7653 Author: Matthias Boehm <[email protected]> Authored: Tue Sep 5 12:29:29 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Tue Sep 5 21:18:09 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/IndexingOp.java | 8 +- .../org/apache/sysml/hops/LeftIndexingOp.java | 4 +- .../hops/cost/CostEstimatorStaticRuntime.java | 14 +- .../java/org/apache/sysml/lops/LeftIndex.java | 6 +- src/main/java/org/apache/sysml/lops/Lop.java | 20 +- .../apache/sysml/lops/RangeBasedReIndex.java | 191 ------------------- .../java/org/apache/sysml/lops/RightIndex.java | 188 ++++++++++++++++++ .../instructions/CPInstructionParser.java | 6 +- .../instructions/GPUInstructionParser.java | 4 +- .../instructions/MRInstructionParser.java | 11 +- .../instructions/SPInstructionParser.java | 8 +- .../cp/FrameIndexingCPInstruction.java | 6 +- .../instructions/cp/IndexingCPInstruction.java | 47 ++--- .../cp/MatrixIndexingCPInstruction.java | 6 +- .../cpfile/MatrixIndexingCPFileInstruction.java | 29 ++- .../gpu/MatrixIndexingGPUInstruction.java | 28 ++- .../runtime/instructions/mr/MRInstruction.java | 2 +- .../mr/RangeBasedReIndexInstruction.java | 7 +- .../spark/FrameIndexingSPInstruction.java | 17 +- .../spark/IndexingSPInstruction.java | 18 +- .../spark/MatrixIndexingSPInstruction.java | 18 +- .../runtime/matrix/data/LibMatrixCUDA.java | 2 +- .../sysml/runtime/matrix/data/MatrixBlock.java | 2 +- .../functions/codegen/RowAggTmplTest.java | 5 +- .../recompile/RemoveEmptyRecompileTest.java | 12 +- 25 files changed, 312 insertions(+), 347 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/IndexingOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java b/src/main/java/org/apache/sysml/hops/IndexingOp.java index 6a3ddf4..2463c7d 100644 --- a/src/main/java/org/apache/sysml/hops/IndexingOp.java +++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java @@ -27,7 +27,7 @@ import org.apache.sysml.lops.Data; import org.apache.sysml.lops.Group; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopsException; -import org.apache.sysml.lops.RangeBasedReIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; @@ -134,7 +134,7 @@ public class IndexingOp extends Hop input._dim1, input._dim2, _dim1, _dim2); Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1)); - RangeBasedReIndex reindex = new RangeBasedReIndex( + RightIndex reindex = new RightIndex( input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), et); @@ -170,7 +170,7 @@ public class IndexingOp extends Hop SparkAggType.NONE : SparkAggType.MULTI_BLOCK; Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1)); - RangeBasedReIndex reindex = new RangeBasedReIndex( + RightIndex reindex = new RightIndex( input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), aggtype, et); @@ -182,7 +182,7 @@ public class IndexingOp extends Hop else //CP or GPU { Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1)); - RangeBasedReIndex reindex = new RangeBasedReIndex( + RightIndex reindex = new RightIndex( input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), et); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java index 02e7753..f3c6614 100644 --- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java +++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java @@ -26,7 +26,7 @@ import org.apache.sysml.lops.LeftIndex; import org.apache.sysml.lops.LeftIndex.LixCacheType; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopsException; -import org.apache.sysml.lops.RangeBasedReIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.UnaryCP; import org.apache.sysml.lops.ZeroOut; import org.apache.sysml.lops.LopProperties.ExecType; @@ -144,7 +144,7 @@ public class LeftIndexingOp extends Hop rightInput = getInput().get(1).constructLops(); - RangeBasedReIndex reindex = new RangeBasedReIndex( + RightIndex reindex = new RightIndex( rightInput, top, bottom, left, right, nrow, ncol, getDataType(), getValueType(), et, true); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java index bb3ce3c..7cdde47 100644 --- a/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java +++ b/src/main/java/org/apache/sysml/hops/cost/CostEstimatorStaticRuntime.java @@ -24,8 +24,10 @@ import java.util.HashSet; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.DataGen; +import org.apache.sysml.lops.LeftIndex; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.MapMult; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.lops.MMTSJ.MMTSJType; import org.apache.sysml.lops.compile.JobType; @@ -1047,12 +1049,12 @@ public class CostEstimatorStaticRuntime extends CostEstimator } return 0; - case MatrixIndexing: //opcodes: rangeReIndex, leftIndex - if( optype.equals("leftIndex") ){ + case MatrixIndexing: //opcodes: rightIndex, leftIndex + if( optype.equals(LeftIndex.OPCODE) ){ return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n) + 2 * DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n ); } - else if( optype.equals("rangeReIndex") ){ + else if( optype.equals(RightIndex.OPCODE) ){ return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n ); } return 0; @@ -1200,11 +1202,11 @@ public class CostEstimatorStaticRuntime extends CostEstimator //String2MRInstructionType.put( "valuepick" , MRINSTRUCTION_TYPE.PickByCount); // for quantile() //String2MRInstructionType.put( "rangepick" , MRINSTRUCTION_TYPE.PickByCount); // for interQuantile() - case RangeReIndex: //opcodes: rangeReIndex, rangeReIndexForLeft + case RightIndex: //opcodes: rightIndex, rightIndexForLeft //TODO: requires category consolidation - if( optype.equals("rangeReIndex") ) + if( optype.equals(RightIndex.OPCODE) ) return DEFAULT_NFLOP_CP * ((leftSparse)? d2m*d2n*d2s : d2m*d2n ); - else //rangeReIndexForLeft + else //rightIndexForLeft return DEFAULT_NFLOP_CP * ((leftSparse)? d1m*d1n*d1s : d1m*d1n) + DEFAULT_NFLOP_CP * ((rightSparse)? d2m*d2n*d2s : d2m*d2n ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/LeftIndex.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/LeftIndex.java b/src/main/java/org/apache/sysml/lops/LeftIndex.java index 7c6117a..e841059 100644 --- a/src/main/java/org/apache/sysml/lops/LeftIndex.java +++ b/src/main/java/org/apache/sysml/lops/LeftIndex.java @@ -34,6 +34,8 @@ public class LeftIndex extends Lop NONE } + public static final String OPCODE = "leftIndex"; + private LixCacheType _type; public LeftIndex( @@ -104,7 +106,7 @@ public class LeftIndex extends Lop if( _type != LixCacheType.NONE ) return "mapLeftIndex"; else - return "leftIndex"; + return OPCODE; } @Override @@ -153,6 +155,6 @@ public class LeftIndex extends Lop @Override public String toString() { - return "leftIndex"; + return getOpcode(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/Lop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java index 33fbccb..238e329 100644 --- a/src/main/java/org/apache/sysml/lops/Lop.java +++ b/src/main/java/org/apache/sysml/lops/Lop.java @@ -38,27 +38,27 @@ public abstract class Lop { public enum Type { - Data, DataGen, //CP/MR read/write/datagen - ReBlock, CSVReBlock, //MR reblock operations + Data, DataGen, //CP/MR read/write/datagen + ReBlock, CSVReBlock, //MR reblock operations MMCJ, MMRJ, MMTSJ, PMMJ, MapMult, MapMultChain, //MR matrix multiplications UnaryCP, UNARY, BinaryCP, Binary, Ternary, //CP/MR unary/binary/ternary - RangeReIndex, LeftIndex, ZeroOut, //CP/MR indexing - Aggregate, PartialAggregate, //CP/MR aggregation - BinUaggChain, UaggOuterChain, //CP/MR aggregation + RightIndex, LeftIndex, ZeroOut, //CP/MR indexing + Aggregate, PartialAggregate, //CP/MR aggregation + BinUaggChain, UaggOuterChain, //CP/MR aggregation TernaryAggregate, //CP ternary-binary aggregates - Grouping, //MR grouping + Grouping, //MR grouping Append, //CP/MR append (column append) CombineUnary, CombineBinary, CombineTernary, //MR combine (stitch together) CentralMoment, CoVariance, GroupedAgg, GroupedAggM, Transform, DataPartition, RepMat, //CP/MR reorganization, partitioning, replication ParameterizedBuiltin, //CP/MR parameterized ops (name/value) - FunctionCallCP, FunctionCallCPSingle, //CP function calls + FunctionCallCP, FunctionCallCPSingle, //CP function calls CumulativePartialAggregate, CumulativeSplitAggregate, CumulativeOffsetBinary, //MR cumsum/cumprod/cummin/cummax WeightedSquaredLoss, WeightedSigmoid, WeightedDivMM, WeightedCeMM, WeightedUMM, SortKeys, PickValues, - Checkpoint, //Spark persist into storage level - PlusMult, MinusMult, //CP - SpoofFused, //CP/SP generated fused operator + Checkpoint, //Spark persist into storage level + PlusMult, MinusMult, //CP + SpoofFused, //CP/SP generated fused operator /** CP operation on a variable number of operands */ MULTIPLE_CP }; http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java b/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java deleted file mode 100644 index 401dfbd..0000000 --- a/src/main/java/org/apache/sysml/lops/RangeBasedReIndex.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.lops; - -import org.apache.sysml.hops.AggBinaryOp.SparkAggType; -import org.apache.sysml.lops.LopProperties.ExecLocation; -import org.apache.sysml.lops.LopProperties.ExecType; -import org.apache.sysml.lops.compile.JobType; -import org.apache.sysml.parser.Expression.DataType; -import org.apache.sysml.parser.Expression.ValueType; - - -public class RangeBasedReIndex extends Lop -{ - - private boolean forLeftIndexing = false; - - //optional attribute for spark exec type - private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK; - - public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, - DataType dt, ValueType vt, ExecType et, boolean forleft) - throws LopsException - { - super(Lop.Type.RangeReIndex, dt, vt); - init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft); - } - - public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, - DataType dt, ValueType vt, ExecType et) - throws LopsException - { - super(Lop.Type.RangeReIndex, dt, vt); - init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false); - } - - public RangeBasedReIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, - DataType dt, ValueType vt, SparkAggType aggtype, ExecType et) - throws LopsException - { - super(Lop.Type.RangeReIndex, dt, vt); - _aggtype = aggtype; - init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false); - } - - private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim, - Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft) - { - addInput(inputMatrix); - addInput(rowL); - addInput(rowU); - addInput(colL); - addInput(colU); - addInput(leftMatrixRowDim); - addInput(leftMatrixColDim); - - inputMatrix.addOutput(this); - rowL.addOutput(this); - rowU.addOutput(this); - colL.addOutput(this); - colU.addOutput(this); - leftMatrixRowDim.addOutput(this); - leftMatrixColDim.addOutput(this); - - boolean breaksAlignment = true; - boolean aligner = false; - boolean definesMRJob = false; - - if ( et == ExecType.MR ) { - - lps.addCompatibility(JobType.GMR); - lps.addCompatibility(JobType.DATAGEN); - lps.addCompatibility(JobType.MMCJ); - lps.addCompatibility(JobType.MMRJ); - lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob); - } - else { - lps.addCompatibility(JobType.INVALID); - lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob); - } - - forLeftIndexing=forleft; - } - - private String getOpcode() { - if(forLeftIndexing) - return "rangeReIndexForLeft"; - else - return "rangeReIndex"; - } - - @Override - public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output) - throws LopsException { - StringBuilder sb = new StringBuilder(); - sb.append( getExecType() ); - sb.append( OPERAND_DELIMITOR ); - sb.append( getOpcode() ); - sb.append( OPERAND_DELIMITOR ); - - sb.append( getInputs().get(0).prepInputOperand(input)); - sb.append( OPERAND_DELIMITOR ); - - // rowl, rowu - sb.append( getInputs().get(1).prepScalarInputOperand(rowl)); - sb.append( OPERAND_DELIMITOR ); - sb.append( getInputs().get(2).prepScalarInputOperand(rowu)); - sb.append( OPERAND_DELIMITOR ); - - // coll, colu - sb.append( getInputs().get(3).prepScalarInputOperand(coll)); - sb.append( OPERAND_DELIMITOR ); - sb.append( getInputs().get(4).prepScalarInputOperand(colu)); - sb.append( OPERAND_DELIMITOR ); - - sb.append( output ); - sb.append( DATATYPE_PREFIX ); - sb.append( getDataType() ); - sb.append( VALUETYPE_PREFIX ); - sb.append( getValueType() ); - - if(getExecType() == ExecType.MR) { - // following fields are added only when this lop is executed in MR (both for left & right indexing) - sb.append( OPERAND_DELIMITOR ); - - sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim)); - sb.append( OPERAND_DELIMITOR ); - sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim)); - } - - //in case of spark, we also compile the optional aggregate flag into the instruction. - if( getExecType() == ExecType.SPARK ) { - sb.append( OPERAND_DELIMITOR ); - sb.append( _aggtype ); - } - - return sb.toString(); - } - - @Override - public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index) - throws LopsException { - /* - * Example: B = A[row_l:row_u, col_l:col_u] - * A - input matrix (input_index1) - * row_l - lower bound in row dimension - * row_u - upper bound in row dimension - * col_l - lower bound in column dimension - * col_u - upper bound in column dimension - * - * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7) - * will be equal to -1. They should be ignored and the scalar value labels must - * be derived from input lops. - */ - String rowl = getInputs().get(1).prepScalarLabel(); - String rowu = getInputs().get(2).prepScalarLabel(); - String coll = getInputs().get(3).prepScalarLabel(); - String colu = getInputs().get(4).prepScalarLabel(); - - String left_nrow = getInputs().get(5).prepScalarLabel(); - String left_ncol = getInputs().get(6).prepScalarLabel(); - - return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index)); - } - - @Override - public String toString() { - if(forLeftIndexing) - return "rangeReIndexForLeft"; - else - return "rangeReIndex"; - } - -} http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/lops/RightIndex.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/RightIndex.java b/src/main/java/org/apache/sysml/lops/RightIndex.java new file mode 100644 index 0000000..812425a --- /dev/null +++ b/src/main/java/org/apache/sysml/lops/RightIndex.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.lops; + +import org.apache.sysml.hops.AggBinaryOp.SparkAggType; +import org.apache.sysml.lops.LopProperties.ExecLocation; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.compile.JobType; +import org.apache.sysml.parser.Expression.DataType; +import org.apache.sysml.parser.Expression.ValueType; + + +public class RightIndex extends Lop +{ + public static final String OPCODE = "rightIndex"; + + private boolean forLeftIndexing = false; + + //optional attribute for spark exec type + private SparkAggType _aggtype = SparkAggType.MULTI_BLOCK; + + public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, + DataType dt, ValueType vt, ExecType et, boolean forleft) + throws LopsException + { + super(Lop.Type.RightIndex, dt, vt); + init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, forleft); + } + + public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, + DataType dt, ValueType vt, ExecType et) + throws LopsException + { + super(Lop.Type.RightIndex, dt, vt); + init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false); + } + + public RightIndex(Lop input, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop rowDim, Lop colDim, + DataType dt, ValueType vt, SparkAggType aggtype, ExecType et) + throws LopsException + { + super(Lop.Type.RightIndex, dt, vt); + _aggtype = aggtype; + init(input, rowL, rowU, colL, colU, rowDim, colDim, dt, vt, et, false); + } + + private void init(Lop inputMatrix, Lop rowL, Lop rowU, Lop colL, Lop colU, Lop leftMatrixRowDim, + Lop leftMatrixColDim, DataType dt, ValueType vt, ExecType et, boolean forleft) + { + addInput(inputMatrix); + addInput(rowL); + addInput(rowU); + addInput(colL); + addInput(colU); + addInput(leftMatrixRowDim); + addInput(leftMatrixColDim); + + inputMatrix.addOutput(this); + rowL.addOutput(this); + rowU.addOutput(this); + colL.addOutput(this); + colU.addOutput(this); + leftMatrixRowDim.addOutput(this); + leftMatrixColDim.addOutput(this); + + boolean breaksAlignment = true; + boolean aligner = false; + boolean definesMRJob = false; + + if ( et == ExecType.MR ) { + lps.addCompatibility(JobType.GMR); + lps.addCompatibility(JobType.DATAGEN); + lps.addCompatibility(JobType.MMCJ); + lps.addCompatibility(JobType.MMRJ); + lps.setProperties(inputs, et, ExecLocation.Map, breaksAlignment, aligner, definesMRJob); + } + else { + lps.addCompatibility(JobType.INVALID); + lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob); + } + + forLeftIndexing=forleft; + } + + private String getOpcode() { + if(forLeftIndexing) + return OPCODE+"ForLeft"; + else + return OPCODE; + } + + @Override + public String getInstructions(String input, String rowl, String rowu, String coll, String colu, String leftRowDim, String leftColDim, String output) + throws LopsException { + StringBuilder sb = new StringBuilder(); + sb.append( getExecType() ); + sb.append( OPERAND_DELIMITOR ); + sb.append( getOpcode() ); + sb.append( OPERAND_DELIMITOR ); + + sb.append( getInputs().get(0).prepInputOperand(input)); + sb.append( OPERAND_DELIMITOR ); + + // rowl, rowu + sb.append( getInputs().get(1).prepScalarInputOperand(rowl)); + sb.append( OPERAND_DELIMITOR ); + sb.append( getInputs().get(2).prepScalarInputOperand(rowu)); + sb.append( OPERAND_DELIMITOR ); + + // coll, colu + sb.append( getInputs().get(3).prepScalarInputOperand(coll)); + sb.append( OPERAND_DELIMITOR ); + sb.append( getInputs().get(4).prepScalarInputOperand(colu)); + sb.append( OPERAND_DELIMITOR ); + + sb.append( output ); + sb.append( DATATYPE_PREFIX ); + sb.append( getDataType() ); + sb.append( VALUETYPE_PREFIX ); + sb.append( getValueType() ); + + if(getExecType() == ExecType.MR) { + // following fields are added only when this lop is executed in MR (both for left & right indexing) + sb.append( OPERAND_DELIMITOR ); + + sb.append( getInputs().get(5).prepScalarInputOperand(leftRowDim)); + sb.append( OPERAND_DELIMITOR ); + sb.append( getInputs().get(6).prepScalarInputOperand(leftColDim)); + } + + //in case of spark, we also compile the optional aggregate flag into the instruction. + if( getExecType() == ExecType.SPARK ) { + sb.append( OPERAND_DELIMITOR ); + sb.append( _aggtype ); + } + + return sb.toString(); + } + + @Override + public String getInstructions(int input_index1, int input_index2, int input_index3, int input_index4, int input_index5, int input_index6, int input_index7, int output_index) + throws LopsException { + /* + * Example: B = A[row_l:row_u, col_l:col_u] + * A - input matrix (input_index1) + * row_l - lower bound in row dimension + * row_u - upper bound in row dimension + * col_l - lower bound in column dimension + * col_u - upper bound in column dimension + * + * Since row_l,row_u,col_l,col_u are scalars, values for input_index(2,3,4,5,6,7) + * will be equal to -1. They should be ignored and the scalar value labels must + * be derived from input lops. + */ + String rowl = getInputs().get(1).prepScalarLabel(); + String rowu = getInputs().get(2).prepScalarLabel(); + String coll = getInputs().get(3).prepScalarLabel(); + String colu = getInputs().get(4).prepScalarLabel(); + + String left_nrow = getInputs().get(5).prepScalarLabel(); + String left_ncol = getInputs().get(6).prepScalarLabel(); + + return getInstructions(Integer.toString(input_index1), rowl, rowu, coll, colu, left_nrow, left_ncol, Integer.toString(output_index)); + } + + @Override + public String toString() { + return getOpcode(); + } + +} http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java index f1c9d3e..2f77710 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java @@ -24,7 +24,9 @@ import java.util.HashMap; import org.apache.sysml.lops.Append; import org.apache.sysml.lops.DataGen; +import org.apache.sysml.lops.LeftIndex; import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.UnaryCP; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.instructions.cp.AggregateBinaryCPInstruction; @@ -257,8 +259,8 @@ public class CPInstructionParser extends InstructionParser String2CPInstructionType.put( "qpick" , CPINSTRUCTION_TYPE.QPick); - String2CPInstructionType.put( "rangeReIndex", CPINSTRUCTION_TYPE.MatrixIndexing); - String2CPInstructionType.put( "leftIndex" , CPINSTRUCTION_TYPE.MatrixIndexing); + String2CPInstructionType.put( RightIndex.OPCODE, CPINSTRUCTION_TYPE.MatrixIndexing); + String2CPInstructionType.put( LeftIndex.OPCODE, CPINSTRUCTION_TYPE.MatrixIndexing); String2CPInstructionType.put( "tsmm" , CPINSTRUCTION_TYPE.MMTSJ); String2CPInstructionType.put( "pmm" , CPINSTRUCTION_TYPE.PMMJ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java index 8abfc69..6f1ed91 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/GPUInstructionParser.java @@ -20,6 +20,7 @@ package org.apache.sysml.runtime.instructions; import java.util.HashMap; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.instructions.gpu.AggregateBinaryGPUInstruction; import org.apache.sysml.runtime.instructions.gpu.ArithmeticBinaryGPUInstruction; @@ -131,8 +132,7 @@ public class GPUInstructionParser extends InstructionParser String2GPUInstructionType.put( ">=" , GPUINSTRUCTION_TYPE.RelationalBinary); // Indexing - // right indexing: output = X[1:3, 4:5] - String2GPUInstructionType.put( "rangeReIndex", GPUINSTRUCTION_TYPE.MatrixIndexing); + String2GPUInstructionType.put( RightIndex.OPCODE, GPUINSTRUCTION_TYPE.MatrixIndexing); } public static GPUInstruction parseSingleInstruction (String str ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java index 5ced2ab..afc9d35 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/MRInstructionParser.java @@ -25,6 +25,7 @@ import org.apache.sysml.lops.BinaryM; import org.apache.sysml.lops.DataGen; import org.apache.sysml.lops.MapMult; import org.apache.sysml.lops.MapMultChain; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.WeightedCrossEntropy; import org.apache.sysml.lops.WeightedCrossEntropyR; import org.apache.sysml.lops.WeightedDivMM; @@ -258,10 +259,10 @@ public class MRInstructionParser extends InstructionParser String2MRInstructionType.put( "groupedagg" , MRINSTRUCTION_TYPE.GroupedAggregate); String2MRInstructionType.put( "mapgroupedagg" , MRINSTRUCTION_TYPE.MapGroupedAggregate); - //rangereindexing - String2MRInstructionType.put( "rangeReIndex" , MRINSTRUCTION_TYPE.RangeReIndex); - String2MRInstructionType.put( "rangeReIndexForLeft" , MRINSTRUCTION_TYPE.RangeReIndex); - String2MRInstructionType.put( "zeroOut" , MRINSTRUCTION_TYPE.ZeroOut); + //right indexing + String2MRInstructionType.put( RightIndex.OPCODE , MRINSTRUCTION_TYPE.RightIndex); + String2MRInstructionType.put( RightIndex.OPCODE+"ForLeft" , MRINSTRUCTION_TYPE.RightIndex); + String2MRInstructionType.put( "zeroOut" , MRINSTRUCTION_TYPE.ZeroOut); //append String2MRInstructionType.put( "mappend" , MRINSTRUCTION_TYPE.Append); @@ -413,7 +414,7 @@ public class MRInstructionParser extends InstructionParser case MapGroupedAggregate: return GroupedAggregateMInstruction.parseInstruction(str); - case RangeReIndex: + case RightIndex: return RangeBasedReIndexInstruction.parseInstruction(str); case ZeroOut: http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java index a2b0ef1..2ebb2e1 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/SPInstructionParser.java @@ -24,6 +24,8 @@ import java.util.HashMap; import org.apache.sysml.lops.Checkpoint; import org.apache.sysml.lops.Compression; import org.apache.sysml.lops.DataGen; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.WeightedCrossEntropy; import org.apache.sysml.lops.WeightedCrossEntropyR; import org.apache.sysml.lops.WeightedDivMM; @@ -141,9 +143,9 @@ public class SPInstructionParser extends InstructionParser String2SPInstructionType.put( "maxpooling", SPINSTRUCTION_TYPE.Convolution); String2SPInstructionType.put( "relu_maxpooling", SPINSTRUCTION_TYPE.Convolution); - String2SPInstructionType.put( "rangeReIndex" , SPINSTRUCTION_TYPE.MatrixIndexing); - String2SPInstructionType.put( "leftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing); - String2SPInstructionType.put( "mapLeftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing); + String2SPInstructionType.put( RightIndex.OPCODE, SPINSTRUCTION_TYPE.MatrixIndexing); + String2SPInstructionType.put( LeftIndex.OPCODE, SPINSTRUCTION_TYPE.MatrixIndexing); + String2SPInstructionType.put( "mapLeftIndex" , SPINSTRUCTION_TYPE.MatrixIndexing); // Reorg Instruction Opcodes (repositioning of existing values) String2SPInstructionType.put( "r'" , SPINSTRUCTION_TYPE.Reorg); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java index fa45b9c..d585148 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/FrameIndexingCPInstruction.java @@ -19,6 +19,8 @@ package org.apache.sysml.runtime.instructions.cp; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; @@ -46,7 +48,7 @@ public final class FrameIndexingCPInstruction extends IndexingCPInstruction { IndexRange ixrange = getIndexRange(ec); //right indexing - if( opcode.equalsIgnoreCase("rangeReIndex") ) + if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { //execute right indexing operation FrameBlock in = ec.getFrameInput(input1.getName()); @@ -59,7 +61,7 @@ public final class FrameIndexingCPInstruction extends IndexingCPInstruction { ec.setFrameOutput(output.getName(), out); } //left indexing - else if ( opcode.equalsIgnoreCase("leftIndex")) + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) { FrameBlock lin = ec.getFrameInput(input1.getName()); FrameBlock out = null; http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java index 46f3f05..e3cb100 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/IndexingCPInstruction.java @@ -19,6 +19,8 @@ package org.apache.sysml.runtime.instructions.cp; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; @@ -62,22 +64,15 @@ public abstract class IndexingCPInstruction extends UnaryCPInstruction { String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); String opcode = parts[0]; - if ( opcode.equalsIgnoreCase("rangeReIndex") ) { + if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { if ( parts.length == 7 ) { - // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 CPOperand in, rl, ru, cl, cu, out; - in = new CPOperand(); - rl = new CPOperand(); - ru = new CPOperand(); - cl = new CPOperand(); - cu = new CPOperand(); - out = new CPOperand(); - in.split(parts[1]); - rl.split(parts[2]); - ru.split(parts[3]); - cl.split(parts[4]); - cu.split(parts[5]); - out.split(parts[6]); + in = new CPOperand(parts[1]); + rl = new CPOperand(parts[2]); + ru = new CPOperand(parts[3]); + cl = new CPOperand(parts[4]); + cu = new CPOperand(parts[5]); + out = new CPOperand(parts[6]); if( in.getDataType()==DataType.MATRIX ) return new MatrixIndexingCPInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str); else if (in.getDataType() == DataType.FRAME) @@ -89,24 +84,16 @@ public abstract class IndexingCPInstruction extends UnaryCPInstruction { throw new DMLRuntimeException("Invalid number of operands in instruction: " + str); } } - else if ( opcode.equalsIgnoreCase("leftIndex")) { + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) { if ( parts.length == 8 ) { - // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7 CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out; - lhsInput = new CPOperand(); - rhsInput = new CPOperand(); - rl = new CPOperand(); - ru = new CPOperand(); - cl = new CPOperand(); - cu = new CPOperand(); - out = new CPOperand(); - lhsInput.split(parts[1]); - rhsInput.split(parts[2]); - rl.split(parts[3]); - ru.split(parts[4]); - cl.split(parts[5]); - cu.split(parts[6]); - out.split(parts[7]); + lhsInput = new CPOperand(parts[1]); + rhsInput = new CPOperand(parts[2]); + rl = new CPOperand(parts[3]); + ru = new CPOperand(parts[4]); + cl = new CPOperand(parts[5]); + cu = new CPOperand(parts[6]); + out = new CPOperand(parts[7]); if( lhsInput.getDataType()==DataType.MATRIX ) return new MatrixIndexingCPInstruction(new SimpleOperator(null), lhsInput, rhsInput, rl, ru, cl, cu, out, opcode, str); else if (lhsInput.getDataType() == DataType.FRAME) http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java index 942f898..85c2a3d 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java @@ -20,6 +20,8 @@ package org.apache.sysml.runtime.instructions.cp; import org.apache.sysml.api.DMLScript; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -54,7 +56,7 @@ public final class MatrixIndexingCPInstruction extends IndexingCPInstruction { MatrixObject mo = ec.getMatrixObject(input1.getName()); //right indexing - if( opcode.equalsIgnoreCase("rangeReIndex") ) + if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { MatrixBlock resultBlock = null; @@ -78,7 +80,7 @@ public final class MatrixIndexingCPInstruction extends IndexingCPInstruction { ec.setMatrixOutput(output.getName(), resultBlock, getExtendedOpcode()); } //left indexing - else if ( opcode.equalsIgnoreCase("leftIndex")) + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) { UpdateType updateType = mo.getUpdateType(); if(DMLScript.STATISTICS) http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java index 33f3aed..acfd4d4 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/MatrixIndexingCPFileInstruction.java @@ -19,6 +19,8 @@ package org.apache.sysml.runtime.instructions.cpfile; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; @@ -55,29 +57,22 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); String opcode = parts[0]; - if ( opcode.equalsIgnoreCase("rangeReIndex") ) { + if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { if ( parts.length == 7 ) { - // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 CPOperand in, rl, ru, cl, cu, out; - in = new CPOperand(); - rl = new CPOperand(); - ru = new CPOperand(); - cl = new CPOperand(); - cu = new CPOperand(); - out = new CPOperand(); - in.split(parts[1]); - rl.split(parts[2]); - ru.split(parts[3]); - cl.split(parts[4]); - cu.split(parts[5]); - out.split(parts[6]); + in = new CPOperand(parts[1]); + rl = new CPOperand(parts[2]); + ru = new CPOperand(parts[3]); + cl = new CPOperand(parts[4]); + cu = new CPOperand(parts[5]); + out = new CPOperand(parts[6]); return new MatrixIndexingCPFileInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str); } else { throw new DMLRuntimeException("Invalid number of operands in instruction: " + str); } } - else if ( parts[0].equalsIgnoreCase("leftIndex")) + else if ( parts[0].equalsIgnoreCase(LeftIndex.OPCODE)) { throw new DMLRuntimeException("Invalid opcode while parsing a MatrixIndexingCPFileInstruction: " + str); } @@ -95,7 +90,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction IndexRange ixrange = getIndexRange(ec).add(1); MatrixObject mo = ec.getMatrixObject(input1.getName()); - if( mo.isPartitioned() && opcode.equalsIgnoreCase("rangeReIndex") ) + if( mo.isPartitioned() && opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { MatrixFormatMetaData meta = (MatrixFormatMetaData)mo.getMetaData(); MatrixCharacteristics mc = meta.getMatrixCharacteristics(); @@ -125,7 +120,7 @@ public final class MatrixIndexingCPFileInstruction extends IndexingCPInstruction mcNew = new MatrixCharacteristics( mc.getRows(), mo.getPartitionSize(), mc.getRowsPerBlock(), mc.getColsPerBlock() ); break; default: - throw new DMLRuntimeException("Unsupported partition format for CP_FILE rangeReIndex: "+ mo.getPartitionFormat()); + throw new DMLRuntimeException("Unsupported partition format for CP_FILE "+RightIndex.OPCODE+": "+ mo.getPartitionFormat()); } MatrixFormatMetaData metaNew = new MatrixFormatMetaData(mcNew,meta.getOutputInfo(),meta.getInputInfo()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java index ba3ccb5..9b29d4a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixIndexingGPUInstruction.java @@ -18,6 +18,8 @@ */ package org.apache.sysml.runtime.instructions.gpu; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; @@ -65,22 +67,15 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction { String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); String opcode = parts[0]; - if ( opcode.equalsIgnoreCase("rangeReIndex") ) { + if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { if ( parts.length == 7 ) { - // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 CPOperand in, rl, ru, cl, cu, out; - in = new CPOperand(); - rl = new CPOperand(); - ru = new CPOperand(); - cl = new CPOperand(); - cu = new CPOperand(); - out = new CPOperand(); - in.split(parts[1]); - rl.split(parts[2]); - ru.split(parts[3]); - cl.split(parts[4]); - cu.split(parts[5]); - out.split(parts[6]); + in = new CPOperand(parts[1]); + rl = new CPOperand(parts[2]); + ru = new CPOperand(parts[3]); + cl = new CPOperand(parts[4]); + cu = new CPOperand(parts[5]); + out = new CPOperand(parts[6]); if( in.getDataType()==DataType.MATRIX ) return new MatrixIndexingGPUInstruction(new SimpleOperator(null), in, rl, ru, cl, cu, out, opcode, str); else @@ -90,9 +85,8 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction { throw new DMLRuntimeException("Invalid number of operands in instruction: " + str); } } - else if ( opcode.equalsIgnoreCase("leftIndex")) { + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE)) { if ( parts.length == 8 ) { - // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7 CPOperand lhsInput, rhsInput, rl, ru, cl, cu, out; lhsInput = new CPOperand(); rhsInput = new CPOperand(); @@ -129,7 +123,7 @@ public class MatrixIndexingGPUInstruction extends GPUInstruction { String opcode = getOpcode(); IndexRange ixrange = getIndexRange(ec); - if ( opcode.equalsIgnoreCase("rangeReIndex") ) { + if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { MatrixObject mat1 = getMatrixInputForGPUInstruction(ec, input1.getName()); LibMatrixCUDA.sliceOperations(ec, ec.getGPUContext(0), getExtendedOpcode(), mat1, ixrange, output.getName()); ec.releaseMatrixInputForGPUInstruction(input1.getName()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java index 7d05010..0fd160a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/MRInstruction.java @@ -32,7 +32,7 @@ public abstract class MRInstruction extends Instruction { public enum MRINSTRUCTION_TYPE { INVALID, Append, Aggregate, ArithmeticBinary, ArithmeticBinary2, AggregateBinary, AggregateUnary, Rand, Seq, CSVReblock, CSVWrite, Reblock, Reorg, Replicate, Unary, CombineBinary, CombineUnary, CombineTernary, - PickByCount, Partition, Ternary, Quaternary, CM_N_COV, MapGroupedAggregate, GroupedAggregate, RangeReIndex, + PickByCount, Partition, Ternary, Quaternary, CM_N_COV, MapGroupedAggregate, GroupedAggregate, RightIndex, ZeroOut, MMTSJ, PMMJ, MatrixReshape, ParameterizedBuiltin, Sort, MapMultChain, CumsumAggregate, CumsumSplit, CumsumOffset, BinUaggChain, UaggOuterChain, RemoveEmpty }; http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java index 79d15ec..b7d73ea 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/mr/RangeBasedReIndexInstruction.java @@ -21,6 +21,7 @@ package org.apache.sysml.runtime.instructions.mr; import java.util.ArrayList; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.instructions.InstructionUtils; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; @@ -42,7 +43,7 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase { private RangeBasedReIndexInstruction(Operator op, byte in, byte out, IndexRange rng, boolean forleft, long rlen, long clen, String istr) { super(op, in, out); - mrtype = MRINSTRUCTION_TYPE.RangeReIndex; + mrtype = MRINSTRUCTION_TYPE.RightIndex; instString = istr; _ixrange = rng; _forLeft = forleft; @@ -66,9 +67,9 @@ public class RangeBasedReIndexInstruction extends UnaryMRInstructionBase { String opcode = parts[0]; boolean forLeft = false; - if(opcode.equalsIgnoreCase("rangeReIndexForLeft")) + if(opcode.equalsIgnoreCase(RightIndex.OPCODE+"ForLeft")) forLeft=true; - else if(!opcode.equalsIgnoreCase("rangeReIndex")) + else if(!opcode.equalsIgnoreCase(RightIndex.OPCODE)) throw new DMLRuntimeException("Unknown opcode while parsing a Select: " + str); byte in = Byte.parseByte(parts[1]); IndexRange rng=new IndexRange(UtilFunctions.parseToLong(parts[2]), http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java index 71178c3..71f17a0 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java @@ -29,6 +29,8 @@ import org.apache.spark.api.java.function.PairFunction; import scala.Tuple2; import org.apache.sysml.hops.AggBinaryOp.SparkAggType; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; @@ -48,16 +50,7 @@ import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; /** - * This class implements the frame indexing functionality inside Spark. - * Example instructions: - * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 - * input=mVar1, output=mVar6, - * bounds = (Var2,Var3,Var4,Var5) - * rowindex_lower: Var2, rowindex_upper: Var3 - * colindex_lower: Var4, colindex_upper: Var5 - * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7 - * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2" - * the result is stored in mVar7 + * This class implements the frame indexing functionality inside Spark. * */ public class FrameIndexingSPInstruction extends IndexingSPInstruction { @@ -87,7 +80,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction { IndexRange ixrange = new IndexRange(rl, ru, cl, cu); //right indexing - if( opcode.equalsIgnoreCase("rangeReIndex") ) + if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { //update and check output dimensions MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName()); @@ -116,7 +109,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction { sec.getFrameObject(input1.getName()).getSchema((int)cl, (int)cu)); } //left indexing - else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex")) + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) { JavaPairRDD<Long,FrameBlock> in1 = sec.getFrameBinaryBlockRDDHandleForVariable( input1.getName() ); PartitionedBroadcast<FrameBlock> broadcastIn2 = null; http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java index 0932cdd..38558d7 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/IndexingSPInstruction.java @@ -20,6 +20,8 @@ package org.apache.sysml.runtime.instructions.spark; import org.apache.sysml.hops.AggBinaryOp.SparkAggType; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.LeftIndex.LixCacheType; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -30,16 +32,6 @@ import org.apache.sysml.runtime.matrix.operators.SimpleOperator; /** * This class implements the matrix indexing functionality inside Spark. - * Example instructions: - * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 - * input=mVar1, output=mVar6, - * bounds = (Var2,Var3,Var4,Var5) - * rowindex_lower: Var2, rowindex_upper: Var3 - * colindex_lower: Var4, colindex_upper: Var5 - * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7 - * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2" - * the result is stored in mVar7 - * */ public abstract class IndexingSPInstruction extends UnarySPInstruction { protected CPOperand rowLower, rowUpper, colLower, colUpper; @@ -71,9 +63,8 @@ public abstract class IndexingSPInstruction extends UnarySPInstruction { String[] parts = InstructionUtils.getInstructionPartsWithValueType(str); String opcode = parts[0]; - if ( opcode.equalsIgnoreCase("rangeReIndex") ) { + if ( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { if ( parts.length == 8 ) { - // Example: rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 CPOperand in = new CPOperand(parts[1]); CPOperand rl = new CPOperand(parts[2]); CPOperand ru = new CPOperand(parts[3]); @@ -90,9 +81,8 @@ public abstract class IndexingSPInstruction extends UnarySPInstruction { throw new DMLRuntimeException("Invalid number of operands in instruction: " + str); } } - else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex")) { + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) { if ( parts.length == 9 ) { - // Example: leftIndex:mVar1:mvar2:Var3:Var4:Var5:Var6:mVar7 CPOperand lhsInput = new CPOperand(parts[1]); CPOperand rhsInput = new CPOperand(parts[2]); CPOperand rl = new CPOperand(parts[3]); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java index b4a0aeb..540ac74 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java @@ -37,6 +37,8 @@ import scala.reflect.ClassManifestFactory; import scala.runtime.AbstractFunction1; import org.apache.sysml.hops.AggBinaryOp.SparkAggType; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.lops.LeftIndex.LixCacheType; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.DMLRuntimeException; @@ -59,17 +61,7 @@ import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; /** - * This class implements the matrix indexing functionality inside CP. - * Example instructions: - * rangeReIndex:mVar1:Var2:Var3:Var4:Var5:mVar6 - * input=mVar1, output=mVar6, - * bounds = (Var2,Var3,Var4,Var5) - * rowindex_lower: Var2, rowindex_upper: Var3 - * colindex_lower: Var4, colindex_upper: Var5 - * leftIndex:mVar1:mVar2:Var3:Var4:Var5:Var6:mVar7 - * triggered by "mVar1[Var3:Var4, Var5:Var6] = mVar2" - * the result is stored in mVar7 - * + * This class implements the matrix indexing functionality inside CP. */ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { private final LixCacheType _type; @@ -101,7 +93,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { IndexRange ixrange = new IndexRange(rl, ru, cl, cu); //right indexing - if( opcode.equalsIgnoreCase("rangeReIndex") ) + if( opcode.equalsIgnoreCase(RightIndex.OPCODE) ) { //update and check output dimensions MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(input1.getName()); @@ -127,7 +119,7 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { } } //left indexing - else if ( opcode.equalsIgnoreCase("leftIndex") || opcode.equalsIgnoreCase("mapLeftIndex")) + else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) { String rddVar = (_type==LixCacheType.LEFT) ? input2.getName() : input1.getName(); String bcVar = (_type==LixCacheType.LEFT) ? input1.getName() : input2.getName(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java index 4be5c2d..09ffe9f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java @@ -3087,7 +3087,7 @@ public class LibMatrixCUDA { //********************************************************************/ /** - * Method to perform rangeReIndex operation for a given lower and upper bounds in row and column dimensions. + * Method to perform rightIndex operation for a given lower and upper bounds in row and column dimensions. * * @param ec current execution context * @param gCtx current gpu context http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 145eb97..d041fc4 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -3781,7 +3781,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } /** - * Method to perform rangeReIndex operation for a given lower and upper bounds in row and column dimensions. + * Method to perform rightIndex operation for a given lower and upper bounds in row and column dimensions. * Extracted submatrix is returned as "result". Note: This operation is now 0-based. * * @param rl row lower http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java index 6d25130..3ecfd6b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java @@ -28,6 +28,7 @@ import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; @@ -573,13 +574,13 @@ public class RowAggTmplTest extends AutomatedTestBase if( testname.equals(TEST_NAME15) ) Assert.assertTrue(!heavyHittersContainsSubString("uark+")); if( testname.equals(TEST_NAME17) ) - Assert.assertTrue(!heavyHittersContainsSubString("rangeReIndex")); + Assert.assertTrue(!heavyHittersContainsSubString(RightIndex.OPCODE)); if( testname.equals(TEST_NAME28) ) Assert.assertTrue(!heavyHittersContainsSubString("spoofRA", 2) && !heavyHittersContainsSubString("sp_spoofRA", 2)); if( testname.equals(TEST_NAME30) ) Assert.assertTrue(!heavyHittersContainsSubString("spoofRA", 2) - && !heavyHittersContainsSubString("rangeReIndex")); + && !heavyHittersContainsSubString(RightIndex.OPCODE)); } finally { rtplatform = platformOld; http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java index 630c774..c1bac4e 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/RemoveEmptyRecompileTest.java @@ -25,6 +25,8 @@ import org.junit.Assert; import org.junit.Test; import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.lops.LeftIndex; +import org.apache.sysml.lops.RightIndex; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; @@ -305,9 +307,9 @@ public class RemoveEmptyRecompileTest extends AutomatedTestBase { switch(type){ //for sum, literal replacement of unary aggregates applies - case SUM: return "rlit";//return "uak+"; + case SUM: return "rlit";//return "uak+"; - case ROUND: return "round"; + case ROUND: return "round"; case TRANSPOSE: return "r'"; case MULT_LEFT: case MULT_RIGHT: return "*"; @@ -316,9 +318,9 @@ public class RemoveEmptyRecompileTest extends AutomatedTestBase case MINUS_LEFT: case MINUS_RIGHT: return "-"; case MM_LEFT: - case MM_RIGHT: return "ba+*"; - case RIX: return "rangeReIndex"; - case LIX: return "leftIndex"; + case MM_RIGHT: return "ba+*"; + case RIX: return RightIndex.OPCODE; + case LIX: return LeftIndex.OPCODE; } return null;
