Repository: systemml Updated Branches: refs/heads/master ac0416883 -> aedceb611
[SYSTEMML-1444] HOP/LOP extensions for UDFs in expressions (part 1) Closes #603. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/aedceb61 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/aedceb61 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/aedceb61 Branch: refs/heads/master Commit: aedceb61152f15c392cbcca2fe6775c1ea4419ab Parents: ac04168 Author: Janardhan <[email protected]> Authored: Fri Aug 25 21:06:08 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Aug 25 21:06:08 2017 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/FunctionOp.java | 32 ++--- .../org/apache/sysml/lops/FunctionCallCP.java | 3 +- .../apache/sysml/lops/FunctionCallCPSingle.java | 128 +++++++++++++++++++ src/main/java/org/apache/sysml/lops/Lop.java | 2 +- .../org/apache/sysml/parser/DMLTranslator.java | 14 +- 5 files changed, 153 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/aedceb61/src/main/java/org/apache/sysml/hops/FunctionOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/FunctionOp.java b/src/main/java/org/apache/sysml/hops/FunctionOp.java index f40929b..d3daa15 100644 --- a/src/main/java/org/apache/sysml/hops/FunctionOp.java +++ b/src/main/java/org/apache/sysml/hops/FunctionOp.java @@ -22,6 +22,7 @@ package org.apache.sysml.hops; import java.util.ArrayList; import org.apache.sysml.lops.FunctionCallCP; +import org.apache.sysml.lops.FunctionCallCPSingle; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopsException; import org.apache.sysml.lops.LopProperties.ExecType; @@ -33,14 +34,11 @@ import org.apache.sysml.runtime.controlprogram.parfor.opt.CostEstimatorHops; /** * This FunctionOp represents the call to a DML-bodied or external function. * - * Note: Currently, we support expressions in function arguments but no function calls - * in expressions. + * Note: Currently, we support expressions in function arguments along with function calls + * in expressions with single outputs, leaving multiple outputs handling as it is. */ public class FunctionOp extends Hop { - - public static String OPSTRING = "extfunct"; - public enum FunctionType{ DML, EXTERNAL_MEM, @@ -49,22 +47,25 @@ public class FunctionOp extends Hop UNKNOWN } + public static final String OPSTRING = "extfunct"; + private FunctionType _type = null; private String _fnamespace = null; private String _fname = null; private String[] _outputs = null; private ArrayList<Hop> _outputHops = null; + private boolean _singleOutFun = false; private FunctionOp() { //default constructor for clone } public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, ArrayList<Hop> outputHops) { - this(type, fnamespace, fname, finputs, outputs); + this(type, fnamespace, fname, finputs, outputs, false); _outputHops = outputHops; } - public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs) + public FunctionOp(FunctionType type, String fnamespace, String fname, ArrayList<Hop> finputs, String[] outputs, boolean singleOut) { super(fnamespace + Program.KEY_DELIM + fname, DataType.UNKNOWN, ValueType.UNKNOWN ); @@ -72,9 +73,9 @@ public class FunctionOp extends Hop _fnamespace = fnamespace; _fname = fname; _outputs = outputs; + _singleOutFun = singleOut; - for( Hop in : finputs ) - { + for( Hop in : finputs ) { getInput().add(in); in.getParent().add(this); } @@ -230,19 +231,20 @@ public class FunctionOp extends Hop //return already created lops if( getLops() != null ) return getLops(); - + ExecType et = optFindExecType(); //construct input lops (recursive) ArrayList<Lop> tmp = new ArrayList<Lop>(); for( Hop in : getInput() ) tmp.add( in.constructLops() ); - + //construct function call - FunctionCallCP fcall = new FunctionCallCP( tmp, _fnamespace, _fname, _outputs, _outputHops, et ); - setLineNumbers( fcall ); - setLops( fcall ); - + Lop fcall = _singleOutFun ? new FunctionCallCPSingle( tmp, _fnamespace, _fname, et ) : + new FunctionCallCP(tmp, _fnamespace, _fname, _outputs, _outputHops, et); + setLineNumbers(fcall); + setLops(fcall); + //note: no reblock lop because outputs directly bound return getLops(); http://git-wip-us.apache.org/repos/asf/systemml/blob/aedceb61/src/main/java/org/apache/sysml/lops/FunctionCallCP.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/FunctionCallCP.java b/src/main/java/org/apache/sysml/lops/FunctionCallCP.java index 28d2f14..0fb997a 100644 --- a/src/main/java/org/apache/sysml/lops/FunctionCallCP.java +++ b/src/main/java/org/apache/sysml/lops/FunctionCallCP.java @@ -22,6 +22,7 @@ package org.apache.sysml.lops; import java.util.ArrayList; +import org.apache.sysml.hops.FunctionOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.HopsException; import org.apache.sysml.lops.LopProperties.ExecLocation; @@ -122,7 +123,7 @@ public class FunctionCallCP extends Lop inst.append(getExecType()); inst.append(Lop.OPERAND_DELIMITOR); - inst.append("extfunct"); + inst.append(FunctionOp.OPSTRING); inst.append(Lop.OPERAND_DELIMITOR); inst.append(_fnamespace); inst.append(Lop.OPERAND_DELIMITOR); http://git-wip-us.apache.org/repos/asf/systemml/blob/aedceb61/src/main/java/org/apache/sysml/lops/FunctionCallCPSingle.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/FunctionCallCPSingle.java b/src/main/java/org/apache/sysml/lops/FunctionCallCPSingle.java new file mode 100644 index 0000000..ef96ed5 --- /dev/null +++ b/src/main/java/org/apache/sysml/lops/FunctionCallCPSingle.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.lops; + + +import java.util.ArrayList; + +import org.apache.sysml.hops.FunctionOp; +import org.apache.sysml.lops.LopProperties.ExecLocation; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.compile.JobType; +import org.apache.sysml.parser.DMLProgram; +import org.apache.sysml.parser.Expression.DataType; +import org.apache.sysml.parser.Expression.ValueType; + +public class FunctionCallCPSingle extends Lop +{ + private String _fnamespace; + private String _fname; + + public FunctionCallCPSingle(ArrayList<Lop> inputs, String fnamespace, String fname, ExecType et) + { + super(Lop.Type.FunctionCallCPSingle, DataType.UNKNOWN, ValueType.UNKNOWN); + //note: data scalar in order to prevent generation of redundant createvar, rmvar + + _fnamespace = fnamespace; + _fname = fname; + + //wire inputs + for( Lop in : inputs ) { + addInput( in ); + in.addOutput( this ); + } + + //lop properties: always in CP + boolean breaksAlignment = false; + boolean aligner = false; + boolean definesMRJob = false; + lps.addCompatibility(JobType.INVALID); + lps.setProperties(inputs, et, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob ); + } + + + @Override + public String toString() { + return "function call: " + DMLProgram.constructFunctionKey(_fnamespace, _fname); + } + + @Override + public String getInstructions(String input1, String output) throws LopsException { + return getInstructions(new String[]{input1}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String output) throws LopsException { + return getInstructions(new String[]{input1, input2}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String input3, String output) throws LopsException { + return getInstructions(new String[]{input1, input2, input3}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String input3, String input4, String output) throws LopsException { + return getInstructions(new String[]{input1, input2, input3, input4}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String input3, String input4, String input5, String output) throws LopsException { + return getInstructions(new String[]{input1, input2, input3, input4, input5}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String input3, String input4, String input5, String input6, String output) throws LopsException { + return getInstructions(new String[]{input1, input2, input3, input4, input5, input6}, new String[]{output}); + } + + @Override + public String getInstructions(String input1, String input2, String input3, String input4, String input5, String input6, String input7, String output) throws LopsException { + return getInstructions(new String[]{input1, input2, input3, input4, input5, input6, input7}, new String[]{output}); + } + + @Override + public String getInstructions(String[] inputs, String output) throws LopsException + { + StringBuilder inst = new StringBuilder(); + inst.append(getExecType()); + + inst.append(Lop.OPERAND_DELIMITOR); + inst.append(FunctionOp.OPSTRING); + inst.append(Lop.OPERAND_DELIMITOR); + inst.append(_fnamespace); + inst.append(Lop.OPERAND_DELIMITOR); + inst.append(_fname); + inst.append(Lop.OPERAND_DELIMITOR); + inst.append(inputs.length); + inst.append(Lop.OPERAND_DELIMITOR); + inst.append("1"); //single output + + for(int i=0; i<inputs.length; i++) { + inst.append(Lop.OPERAND_DELIMITOR); + inst.append( getInputs().get(i).prepInputOperand(inputs[i]) ); + } + + inst.append(Lop.OPERAND_DELIMITOR); + inst.append(output); + + return inst.toString(); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/aedceb61/src/main/java/org/apache/sysml/lops/Lop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/Lop.java b/src/main/java/org/apache/sysml/lops/Lop.java index 1e88561..33fbccb 100644 --- a/src/main/java/org/apache/sysml/lops/Lop.java +++ b/src/main/java/org/apache/sysml/lops/Lop.java @@ -52,7 +52,7 @@ public abstract class Lop CentralMoment, CoVariance, GroupedAgg, GroupedAggM, Transform, DataPartition, RepMat, //CP/MR reorganization, partitioning, replication ParameterizedBuiltin, //CP/MR parameterized ops (name/value) - FunctionCallCP, //CP function calls + FunctionCallCP, FunctionCallCPSingle, //CP function calls CumulativePartialAggregate, CumulativeSplitAggregate, CumulativeOffsetBinary, //MR cumsum/cumprod/cummin/cummax WeightedSquaredLoss, WeightedSigmoid, WeightedDivMM, WeightedCeMM, WeightedUMM, SortKeys, PickValues, http://git-wip-us.apache.org/repos/asf/systemml/blob/aedceb61/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index eaae69a..e6b8590 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -1416,19 +1416,15 @@ public class DMLTranslator ArrayList<Hop> finputs = new ArrayList<Hop>(); for (ParameterExpression paramName : fci.getParamExprs()){ - Hop in = processExpression(paramName.getExpr(), null, ids); + Hop in = processExpression(paramName.getExpr(), null, ids); finputs.add(in); } //create function op FunctionType ftype = fsb.getFunctionOpType(); - FunctionOp fcall = null; - if (target == null) { - fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{}); - } else { - fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{target.getName()}); - } - + FunctionOp fcall = (target == null) ? + new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{}, false) : + new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new String[]{target.getName()}, false); output.add(fcall); //TODO function output dataops (phase 3) @@ -1465,7 +1461,7 @@ public class DMLTranslator } FunctionType ftype = fsb.getFunctionOpType(); - FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs); + FunctionOp fcall = new FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, foutputs, false); output.add(fcall); //TODO function output dataops (phase 3)
