[SYSTEMML-2207] New optional 'secondorder' arg for external functions This patch adds an optional argument to the specification of external functions in order to mark them as 'secondorder', i.e., indicate that they are calling other functions. Similar to the builtin eval functions, this flag is in turn used to disable the removal of unused functions, which might be called within this second order function.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a839b30b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a839b30b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a839b30b Branch: refs/heads/master Commit: a839b30b26e31f7c1702b24d9a900cb64ad05ec5 Parents: 1f32397 Author: Matthias Boehm <[email protected]> Authored: Sat Mar 24 19:34:28 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sat Mar 24 19:34:28 2018 -0700 ---------------------------------------------------------------------- .../sysml/hops/ipa/FunctionCallGraph.java | 8 ++ .../sysml/parser/ExternalFunctionStatement.java | 23 ++++-- .../org/apache/sysml/udf/lib/EvalFunction.java | 84 ++++++++++++++++++++ .../functions/external/EvalFunctionTest.java | 74 +++++++++++++++++ .../scripts/functions/external/SecondOrder.dml | 33 ++++++++ .../functions/external/ZPackageSuite.java | 3 +- 6 files changed, 216 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java b/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java index 78cba3d..4268784 100644 --- a/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java +++ b/src/main/java/org/apache/sysml/hops/ipa/FunctionCallGraph.java @@ -36,6 +36,7 @@ import org.apache.sysml.hops.Hop.DataOpTypes; import org.apache.sysml.hops.Hop.OpOpN; import org.apache.sysml.hops.rewrite.HopRewriteUtils; import org.apache.sysml.parser.DMLProgram; +import org.apache.sysml.parser.ExternalFunctionStatement; import org.apache.sysml.parser.ForStatement; import org.apache.sysml.parser.ForStatementBlock; import org.apache.sysml.parser.FunctionStatement; @@ -356,6 +357,13 @@ public class FunctionCallGraph //mark as visited for current function call context lfset.add( lfkey ); + + //handle second order external functions + FunctionStatementBlock fsb = sb.getDMLProg().getFunctionStatementBlock(lfkey); + if( fsb != null && fsb.getStatement(0) instanceof ExternalFunctionStatement + && ((ExternalFunctionStatement)fsb.getStatement(0)).isSecondOrder() ) { + ret = true; + } } else if( HopRewriteUtils.isData(h, DataOpTypes.TRANSIENTWRITE) && HopRewriteUtils.isNary(h.getInput().get(0), OpOpN.EVAL) ) { http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java index 1a87ebc..00ab76a 100644 --- a/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java +++ b/src/main/java/org/apache/sysml/parser/ExternalFunctionStatement.java @@ -28,7 +28,7 @@ public class ExternalFunctionStatement extends FunctionStatement public static final String EXEC_TYPE = "exectype"; public static final String CONFIG_FILE = "configfile"; public static final String SIDE_EFFECTS = "sideeffect"; - + public static final String SECOND_ORDER = "secondorder"; //valid attribute values for execlocation public static final String FILE_BASED = "file"; @@ -59,6 +59,11 @@ public class ExternalFunctionStatement extends FunctionStatement && Boolean.parseBoolean(_otherParams.get(SIDE_EFFECTS)); } + public boolean isSecondOrder() { + return _otherParams.containsKey(SECOND_ORDER) + && Boolean.parseBoolean(_otherParams.get(SECOND_ORDER)); + } + /** * Validates all attributes and attribute values. * @@ -69,7 +74,7 @@ public class ExternalFunctionStatement extends FunctionStatement //warnings for all not defined attributes for( String varName : _otherParams.keySet() ) if( !(varName.equals(CLASS_NAME) || varName.equals(EXEC_TYPE) - || varName.equals(CONFIG_FILE) || varName.equals(SIDE_EFFECTS) ) ) + || varName.equals(CONFIG_FILE) || varName.equals(SIDE_EFFECTS) || varName.equals(SECOND_ORDER) ) ) { LOG.warn( printWarningLocation() + "External function specifies undefined attribute type '"+varName+"'."); } @@ -95,12 +100,14 @@ public class ExternalFunctionStatement extends FunctionStatement _otherParams.put(EXEC_TYPE, DEFAULT_EXEC_TYPE); } - //side effects - if( _otherParams.containsKey( SIDE_EFFECTS ) ) { - String sideeffect = _otherParams.get(SIDE_EFFECTS); - if( !(sideeffect.equals("true") || sideeffect.equals("false")) ) //always unconditional (invalid parameter) - sb.raiseValidateError("External function specifies invalid value for (optional) attribute '" - + SIDE_EFFECTS+"' (valid values: true, false).", false); + //side effects/second order + for( String param : new String[]{ SIDE_EFFECTS, SECOND_ORDER } ) { + if( _otherParams.containsKey( param ) ) { + String sideeffect = _otherParams.get(param); + if( !(sideeffect.equals("true") || sideeffect.equals("false")) ) //always unconditional (invalid parameter) + sb.raiseValidateError("External function specifies invalid value for (optional) attribute '" + + param+"' (valid values: true, false).", false); + } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/main/java/org/apache/sysml/udf/lib/EvalFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/lib/EvalFunction.java b/src/main/java/org/apache/sysml/udf/lib/EvalFunction.java new file mode 100644 index 0000000..c9d1028 --- /dev/null +++ b/src/main/java/org/apache/sysml/udf/lib/EvalFunction.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.udf.lib; + +import java.util.ArrayList; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.sysml.parser.Expression.DataType; + +import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; +import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; +import org.apache.sysml.runtime.instructions.cp.CPOperand; +import org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction; +import org.apache.sysml.udf.FunctionParameter; +import org.apache.sysml.udf.Matrix; +import org.apache.sysml.udf.PackageFunction; +import org.apache.sysml.udf.Scalar; +import org.apache.sysml.udf.Matrix.ValueType; + +/** + * This function is for testing purposes only. + */ +public class EvalFunction extends PackageFunction +{ + private static final long serialVersionUID = 1L; + + private Matrix _ret; + + @Override + public int getNumFunctionOutputs() { + return 1; + } + + @Override + public FunctionParameter getFunctionOutput(int pos) { + switch(pos) { + case 0: return _ret; + default: + throw new RuntimeException("Invalid function output being requested"); + } + } + + @Override + public void execute(ExecutionContext ec) { + String fname = ((Scalar)getFunctionInput(0)).getValue(); + MatrixObject in = ((Matrix) getFunctionInput(1)).getMatrixObject(); + ArrayList<String> inputs = new ArrayList<>(); inputs.add("A"); + ArrayList<String> outputs = new ArrayList<>(); outputs.add("B"); + + ExecutionContext ec2 = ExecutionContextFactory.createContext(ec.getProgram()); + CPOperand inName = new CPOperand("TMP", org.apache.sysml.parser.Expression.ValueType.DOUBLE, DataType.MATRIX); + ec2.setVariable("TMP", in); + + FunctionCallCPInstruction fcpi = new FunctionCallCPInstruction( + null, fname, new CPOperand[]{inName}, inputs, outputs, "eval func"); + fcpi.processInstruction(ec2); + + MatrixObject out = (MatrixObject)ec2.getVariable("B"); + _ret = new Matrix(out, ValueType.Double); + } + + @Override + public void execute() { + throw new NotImplementedException(); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/test/java/org/apache/sysml/test/integration/functions/external/EvalFunctionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/external/EvalFunctionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/external/EvalFunctionTest.java new file mode 100644 index 0000000..3165089 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/external/EvalFunctionTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.external; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.util.MapReduceTool; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; + +public class EvalFunctionTest extends AutomatedTestBase +{ + private final static String TEST_NAME = "SecondOrder"; + private final static String TEST_DIR = "functions/external/"; + private final static String TEST_CLASS_DIR = TEST_DIR + EvalFunctionTest.class.getSimpleName() + "/"; + + private final static double eps = 1e-7; + private final static int rows = 120; + private final static int cols = 110; + private final static double sparsity = 0.7; + + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "Y" }) ); + } + + @Test + public void runEvalFunctionTest() { + + TestConfiguration config = getTestConfiguration(TEST_NAME); + config.addVariable("rows", rows); + config.addVariable("cols", cols); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-args", input("X"), output("Y") }; + + try { + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, -7); + writeInputMatrixWithMTD("X", X, false); + runTest(true, false, null, -1); + double[][] Y = MapReduceTool.readMatrixFromHDFS(output("Y"), + InputInfo.stringToInputInfo("textcell"), rows, cols, 1000, 1000); + for( int i=0; i<rows; i++ ) + for( int j=0; j<cols; j++ ) + if( Math.abs(X[i][j]-Y[i][j]+7) > eps ) + Assert.fail("Wrong results: "+X[i][j]+" vs "+Y[i][j]); + } + catch (Exception e) { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/test/scripts/functions/external/SecondOrder.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/external/SecondOrder.dml b/src/test/scripts/functions/external/SecondOrder.dml new file mode 100644 index 0000000..88cbf47 --- /dev/null +++ b/src/test/scripts/functions/external/SecondOrder.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +foo1 = externalFunction(String fname, Matrix[Double] A) +return (Matrix[Double] B) +implemented in (classname="org.apache.sysml.udf.lib.EvalFunction", exectype="mem", secondorder="true") + +foo2 = function(Matrix[Double] A) return(Matrix[Double] B) { + B = A + 7; +} + +X = read($1); +Y = foo1("foo2", X); +write(Y, $2) http://git-wip-us.apache.org/repos/asf/systemml/blob/a839b30b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java index 6da2a1c..cc9fd1b 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/external/ZPackageSuite.java @@ -28,8 +28,9 @@ import org.junit.runners.Suite; @Suite.SuiteClasses({ DynProjectTest.class, DynReadWriteTest.class, + EvalFunctionTest.class, FunctionExpressionsTest.class, - OrderTest.class + OrderTest.class, })
