Repository: incubator-systemml Updated Branches: refs/heads/master 41d9d2c1d -> 8c3f97276
Cleanup unnecessary seq_incr operator and instruction (default handling) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/94641018 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/94641018 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/94641018 Branch: refs/heads/master Commit: 946410189658219d43a4a2588185a20b3671dd20 Parents: 41d9d2c Author: Matthias Boehm <[email protected]> Authored: Sun Dec 6 14:17:10 2015 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sun Dec 6 14:17:10 2015 -0800 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/DataGenOp.java | 14 +---------- src/main/java/org/apache/sysml/hops/Hop.java | 4 +-- .../apache/sysml/hops/recompile/Recompiler.java | 8 ++---- .../org/apache/sysml/lops/BinaryScalar.java | 4 --- .../org/apache/sysml/parser/DMLTranslator.java | 7 ++---- .../instructions/CPInstructionParser.java | 1 - .../instructions/cp/DataGenCPInstruction.java | 3 +++ .../instructions/cp/VariableCPInstruction.java | 26 +------------------- .../instructions/spark/RandSPInstruction.java | 3 +++ .../apache/sysml/runtime/matrix/DataGenMR.java | 3 +++ .../runtime/matrix/data/LibMatrixDatagen.java | 12 +++++++++ .../runtime/matrix/mapred/DataGenMapper.java | 3 +++ .../matrix/operators/BinaryOperator.java | 2 +- 13 files changed, 32 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/DataGenOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/DataGenOp.java b/src/main/java/org/apache/sysml/hops/DataGenOp.java index bfe42c3..7432fc2 100644 --- a/src/main/java/org/apache/sysml/hops/DataGenOp.java +++ b/src/main/java/org/apache/sysml/hops/DataGenOp.java @@ -258,14 +258,6 @@ public class DataGenOp extends Hop implements MultiThreadedHop if( toVal > 0 ) return new long[]{ toVal, 1, -1 }; } - //here, we check for the common case of seq(1,x,?), i.e., from=1, to=x, b(seqincr) operator - if( from instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)from)==1 - && incr instanceof BinaryOp && ((BinaryOp)incr).getOp() == Hop.OpOp2.SEQINCR ) //implicit 1 - { - long toVal = computeDimParameterInformation(to, memo); - if( toVal > 0 ) - return new long[]{ toVal, 1, -1 }; - } //here, we check for the common case of seq(x,1,-1), i.e. from=x, to=1 incr=-1 if( to instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)to)==1 && incr instanceof LiteralOp && HopRewriteUtils.getDoubleValueSafe((LiteralOp)incr)==-1 ) @@ -350,12 +342,8 @@ public class DataGenOp extends Hop implements MultiThreadedHop double incr = computeBoundsInformation(input3); boolean incrKnown = (incr != Double.MAX_VALUE); - - if( !incrKnown && input3 instanceof BinaryOp //special case for incr - && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR && fromKnown && toKnown) - { + if( fromKnown && toKnown && incr == 1) { incr = ( from >= to ) ? -1 : 1; - incrKnown = true; } if ( fromKnown && toKnown && incrKnown ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 9822fac..ca798ce 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -1040,7 +1040,7 @@ public abstract class Hop public enum OpOp2 { PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, LESS, LESSEQUAL, GREATER, GREATEREQUAL, EQUAL, NOTEQUAL, MIN, MAX, AND, OR, LOG, POW, PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM, - CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SEQINCR, SOLVE, MEDIAN, INVALID, + CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SOLVE, MEDIAN, INVALID, //fused ML-specific operators for performance MINUS_NZ, //sparse-safe minus: X-(mean*ppred(X,0,!=)) LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X,0.5) @@ -1192,7 +1192,6 @@ public abstract class Hop HopsOpOp2LopsBS.put(OpOp2.LOG, org.apache.sysml.lops.BinaryScalar.OperationTypes.LOG); HopsOpOp2LopsBS.put(OpOp2.POW, org.apache.sysml.lops.BinaryScalar.OperationTypes.POW); HopsOpOp2LopsBS.put(OpOp2.PRINT, org.apache.sysml.lops.BinaryScalar.OperationTypes.PRINT); - HopsOpOp2LopsBS.put(OpOp2.SEQINCR, org.apache.sysml.lops.BinaryScalar.OperationTypes.SEQINCR); } protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp2LopsU; @@ -1354,7 +1353,6 @@ public abstract class Hop HopsOpOp2String.put(OpOp2.CBIND, "cbind"); HopsOpOp2String.put(OpOp2.RBIND, "rbind"); HopsOpOp2String.put(OpOp2.SOLVE, "solve"); - HopsOpOp2String.put(OpOp2.SEQINCR, "seqincr"); } public static String getOpOp2String( OpOp2 op ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java index 600c5e2..48de372 100644 --- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java +++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java @@ -35,7 +35,6 @@ import org.apache.wink.json4j.JSONObject; import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; -import org.apache.sysml.hops.BinaryOp; import org.apache.sysml.hops.DataGenOp; import org.apache.sysml.hops.DataOp; import org.apache.sysml.hops.FunctionOp; @@ -1692,11 +1691,8 @@ public class Recompiler double incr = d.computeBoundsInformation(d.getInput().get(ix3), vars); //special case increment - Hop input3 = d.getInput().get(ix3); - if ( input3 instanceof BinaryOp && ((BinaryOp)input3).getOp() == Hop.OpOp2.SEQINCR - && from!=Double.MAX_VALUE && to!=Double.MAX_VALUE ) - { - incr =( from >= to )? -1.0 : 1.0; + if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE ) { + incr = ( from >= to && incr==1 ) ? -1.0 : 1.0; } if ( from!=Double.MAX_VALUE && to!=Double.MAX_VALUE && incr!=Double.MAX_VALUE ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/lops/BinaryScalar.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/BinaryScalar.java b/src/main/java/org/apache/sysml/lops/BinaryScalar.java index 4a2eb29..0f423e3 100644 --- a/src/main/java/org/apache/sysml/lops/BinaryScalar.java +++ b/src/main/java/org/apache/sysml/lops/BinaryScalar.java @@ -41,7 +41,6 @@ public class BinaryScalar extends Lop LOG,POW,MAX,MIN,PRINT, IQSIZE, Over, - SEQINCR } OperationTypes operation; @@ -190,9 +189,6 @@ public class BinaryScalar extends Lop case IQSIZE: return "iqsize"; - - case SEQINCR: - return "seqincr"; default: throw new UnsupportedOperationException("Instruction is not defined for BinaryScalar operator: " + op); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index e9e43b8..42ada29 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -2606,11 +2606,8 @@ public class DMLTranslator HashMap<String,Hop> randParams = new HashMap<String,Hop>(); randParams.put(Statement.SEQ_FROM, expr); randParams.put(Statement.SEQ_TO, expr2); - if ( expr3 == null ) { - expr3 = new BinaryOp(Statement.SEQ_INCR, DataType.SCALAR, ValueType.DOUBLE, Hop.OpOp2.SEQINCR, expr, expr2); - //throw new ParseException("Unable to search for increment field in seq()."); //randParams.put(Statement.SEQ_INCR, new LiteralOp("defaultIncr", Double.NaN)); - } - randParams.put(Statement.SEQ_INCR, expr3); + randParams.put(Statement.SEQ_INCR, (expr3!=null)?expr3 : new LiteralOp(1)); + //note incr: default -1 (for from>to) handled during runtime currBuiltinOp = new DataGenOp(DataGenMethod.SEQ, target, randParams); break; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java index 3d41154..d81e363 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java @@ -192,7 +192,6 @@ public class CPInstructionParser extends InstructionParser String2CPInstructionType.put( "read" , CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( "write" , CPINSTRUCTION_TYPE.Variable); String2CPInstructionType.put( "createvar" , CPINSTRUCTION_TYPE.Variable); - String2CPInstructionType.put( "seqincr" , CPINSTRUCTION_TYPE.Variable); // Reorg Instruction Opcodes (repositioning of existing values) String2CPInstructionType.put( "r'" , CPINSTRUCTION_TYPE.Reorg); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java index 6ae3313..9ed2714 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataGenCPInstruction.java @@ -302,6 +302,9 @@ public class DataGenCPInstruction extends UnaryCPInstruction } else if ( method == DataGenMethod.SEQ ) { + //handle default 1 to -1 for special case of from>to + seq_incr = LibMatrixDatagen.updateSeqIncr(seq_from, seq_to, seq_incr); + if( LOG.isTraceEnabled() ) LOG.trace("Process DataGenCPInstruction seq with seqFrom="+seq_from+", seqTo="+seq_to+", seqIncr"+seq_incr); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index 116eee1..9fa2ad0 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -86,7 +86,6 @@ public class VariableCPInstruction extends CPInstruction Write, Read, SetFileName, - SequenceIncrement, } private static IDSequence _uniqueVarID; @@ -150,9 +149,6 @@ public class VariableCPInstruction extends CPInstruction else if ( str.equalsIgnoreCase("setfilename") ) return VariableOperationCode.SetFileName; - else if ( str.equalsIgnoreCase("seqincr") ) - return VariableOperationCode.SequenceIncrement; - else throw new DMLUnsupportedOperationException("Invalid function: " + str); } @@ -234,7 +230,6 @@ public class VariableCPInstruction extends CPInstruction return 1; case Write: case SetFileName: - case SequenceIncrement: return 3; default: return 2; @@ -413,12 +408,6 @@ public class VariableCPInstruction extends CPInstruction //return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, str); break; - case SequenceIncrement: - in1 = new CPOperand(parts[1]); - in2 = new CPOperand(parts[2]); - out = new CPOperand(parts[3]); - break; - } return new VariableCPInstruction(getVariableOperationCode(opcode), in1, in2, in3, out, _arity, opcode, str); } @@ -583,20 +572,7 @@ public class VariableCPInstruction extends CPInstruction throw new DMLRuntimeException("Invalid data type (" + input1.getDataType() + ") in SetFileName instruction: " + instString); } break; - - case SequenceIncrement: - ScalarObject fromObj = ec.getScalarInput(input1.getName(), input1.getValueType(), input1.isLiteral()); - ScalarObject toObj = ec.getScalarInput(input2.getName(), input2.getValueType(), input2.isLiteral()); - double ret = Double.NaN; - if ( fromObj.getDoubleValue() > toObj.getDoubleValue() ) - ret = -1.0; - else - ret = 1.0; - ScalarObject incrObj = (ScalarObject) new DoubleObject(ret); - ec.setVariable(output.getName(), incrObj); - - break; - + default: throw new DMLRuntimeException("Unknown opcode: " + opcode ); } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java index 0e9da31..3b4f798 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java @@ -390,6 +390,9 @@ public class RandSPInstruction extends UnarySPInstruction throw new DMLRuntimeException("ERROR: While performing seq(" + seq_from + "," + seq_to + "," + seq_incr + ")"); } + //handle default 1 to -1 for special case of from>to + seq_incr = LibMatrixDatagen.updateSeqIncr(seq_from, seq_to, seq_incr); + if( LOG.isTraceEnabled() ) LOG.trace("Process RandSPInstruction seq with seqFrom="+seq_from+", seqTo="+seq_to+", seqIncr"+seq_incr); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java index 554b22c..ba933a4 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java @@ -195,6 +195,9 @@ public class DataGenMR double from = seqInst.fromValue; double to = seqInst.toValue; double incr = seqInst.incrValue; + + //handle default 1 to -1 for special case of from>to + incr = LibMatrixDatagen.updateSeqIncr(from, to, incr); // Correctness checks on (from, to, incr) boolean neg = (from > to); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java index 4e3e382..5f2664c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java @@ -70,6 +70,18 @@ public class LibMatrixDatagen } /** + * + * @param seq_from + * @param seq_to + * @param seq_incr + * @return + */ + public static double updateSeqIncr(double seq_from, double seq_to, double seq_incr) { + //handle default 1 to -1 for special case of from>to + return (seq_from>seq_to && seq_incr==1)? -1 : seq_incr; + } + + /** * A matrix of random numbers is generated by using multiple seeds, one for each * block. Such block-level seeds are produced via Well equidistributed long-period linear * generator (Well1024a). For a given seed, this function sets up the block-level seeds. http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java index 88fcb66..49b2804 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DataGenMapper.java @@ -98,6 +98,9 @@ implements Mapper<Writable, Writable, Writable, Writable> double to=Double.parseDouble(params[3]); double incr=Double.parseDouble(params[4]); + //handle default 1 to -1 for special case of from>to + incr = LibMatrixDatagen.updateSeqIncr(from, to, incr); + //sequence data generation try { indexes[i].setIndexes(blockRowNumber, blockColNumber); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/94641018/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java index bb09c76..a268f49 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/operators/BinaryOperator.java @@ -96,7 +96,7 @@ public class BinaryOperator extends Operator implements Serializable //non-supported ops (not required for sparsity estimates): //PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM, - //CENTRALMOMENT, COVARIANCE, APPEND, SEQINCR, SOLVE, MEDIAN, + //CENTRALMOMENT, COVARIANCE, APPEND, SOLVE, MEDIAN, return OpOp2.INVALID; }
