Repository: incubator-systemml Updated Branches: refs/heads/master 18bb8c52c -> 66ea6f0c6
[SYSTEMML-670] Fix PyDML built-in functions random and cdf This patch fixes parsing of PyDML built-in functions random.normal, random.uniform, norm.cdf, expon.cdf, chi.cdf, f.cdf, and t.cdf. Also added test script to invoke each function to verify parsing issue no longer occurs for PyDML. Closes #156. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/66ea6f0c Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/66ea6f0c Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/66ea6f0c Branch: refs/heads/master Commit: 66ea6f0c6cf74d4d4413c80cc9b42a0b89f82b07 Parents: 18bb8c5 Author: Glenn Weidner <[email protected]> Authored: Mon May 16 11:45:11 2016 -0700 Committer: Mike Dusenberry <[email protected]> Committed: Mon May 16 11:45:11 2016 -0700 ---------------------------------------------------------------------- .../parser/pydml/PydmlSyntacticValidator.java | 45 +++++++++++--------- .../functions/misc/FunctionNamespaceTest.java | 14 ++++++ src/test/scripts/functions/misc/Functions14.dml | 43 +++++++++++++++++++ .../scripts/functions/misc/Functions14.pydml | 43 +++++++++++++++++++ 4 files changed, 126 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/66ea6f0c/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java index 7f73507..6bd9985 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java @@ -792,9 +792,10 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "ncol"; } } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("random.normal")) { + else if(namespace.equals("random") && functionName.equals("normal")) { if(paramExpression.size() != 3) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (number of rows, number of columns, sparsity)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 3 arguments (number of rows, number of columns, sparsity)", fnName); return null; } paramExpression.get(0).setName("rows"); @@ -804,9 +805,10 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "rand"; namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("random.uniform")) { + else if(namespace.equals("random") && functionName.equals("uniform")) { if(paramExpression.size() != 5) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 5 arguments (number of rows, number of columns, sparsity, min, max)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 5 arguments (number of rows, number of columns, sparsity, min, max)", fnName); return null; } paramExpression.get(0).setName("rows"); @@ -958,58 +960,63 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements functionName = "seq"; namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("norm.cdf")) { + else if(namespace.equals("norm") && functionName.equals("cdf")) { if(paramExpression.size() != 3) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, mean, sd)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 3 arguments (target, mean, sd)", fnName); return null; } - functionName = "cumulativeProbability"; + functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("mean"); paramExpression.get(2).setName("sd"); paramExpression.add(new ParameterExpression("dist", new StringIdentifier("normal", fileName, line, col, line, col))); namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("expon.cdf")) { + else if(namespace.equals("expon") && functionName.equals("cdf")) { if(paramExpression.size() != 2) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, mean)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 2 arguments (target, mean)", fnName); return null; } - functionName = "cumulativeProbability"; + functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("mean"); paramExpression.add(new ParameterExpression("dist", new StringIdentifier("exp", fileName, line, col, line, col))); namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("chi.cdf")) { + else if(namespace.equals("chi") && functionName.equals("cdf")) { if(paramExpression.size() != 2) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 2 arguments (target, df)", fnName); return null; } - functionName = "cumulativeProbability"; + functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df"); paramExpression.add(new ParameterExpression("dist", new StringIdentifier("chisq", fileName, line, col, line, col))); namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("f.cdf")) { + else if(namespace.equals("f") && functionName.equals("cdf")) { if(paramExpression.size() != 3) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 3 arguments (target, df1, df2)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 3 arguments (target, df1, df2)", fnName); return null; } - functionName = "cumulativeProbability"; + functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df1"); paramExpression.get(2).setName("df2"); paramExpression.add(new ParameterExpression("dist", new StringIdentifier("f", fileName, line, col, line, col))); namespace = DMLProgram.DEFAULT_NAMESPACE; } - else if(namespace.equals(DMLProgram.DEFAULT_NAMESPACE) && functionName.equals("t.cdf")) { + else if(namespace.equals("t") && functionName.equals("cdf")) { if(paramExpression.size() != 2) { - notifyErrorListeners("The builtin function \'" + functionName + "\' accepts exactly 2 arguments (target, df)", fnName); + String qualifiedName = namespace + namespaceResolutionOp() + functionName; + notifyErrorListeners("The builtin function \'" + qualifiedName + "\' accepts exactly 2 arguments (target, df)", fnName); return null; } - functionName = "cumulativeProbability"; + functionName = "cdf"; paramExpression.get(0).setName("target"); paramExpression.get(1).setName("df"); paramExpression.add(new ParameterExpression("dist", new StringIdentifier("t", fileName, line, col, line, col))); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/66ea6f0c/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionNamespaceTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionNamespaceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionNamespaceTest.java index 640f973..e2fc718 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionNamespaceTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/FunctionNamespaceTest.java @@ -47,6 +47,7 @@ public class FunctionNamespaceTest extends AutomatedTestBase private final static String TEST_NAME11 = "Functions11"; private final static String TEST_NAME12 = "Functions12"; private final static String TEST_NAME13 = "Functions13"; + private final static String TEST_NAME14 = "Functions14"; private final static String TEST_DIR = "functions/misc/"; private final static String TEST_CLASS_DIR = TEST_DIR + FunctionNamespaceTest.class.getSimpleName() + "/"; @@ -71,6 +72,7 @@ public class FunctionNamespaceTest extends AutomatedTestBase addTestConfiguration(TEST_NAME11, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME11)); addTestConfiguration(TEST_NAME12, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME12)); addTestConfiguration(TEST_NAME13, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME13)); + addTestConfiguration(TEST_NAME14, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME14)); } @Test @@ -169,6 +171,12 @@ public class FunctionNamespaceTest extends AutomatedTestBase } @Test + public void testFunctionRandomCDF() + { + runFunctionNamespaceTest(TEST_NAME14, ScriptType.DML); + } + + @Test public void testPyFunctionDefaultNS() { runFunctionNamespaceTest(TEST_NAME0, ScriptType.PYDML); @@ -263,6 +271,12 @@ public class FunctionNamespaceTest extends AutomatedTestBase { runFunctionNamespaceTest(TEST_NAME13, ScriptType.PYDML); } + + @Test + public void testPyFunctionRandomCDF() + { + runFunctionNamespaceTest(TEST_NAME14, ScriptType.PYDML); + } private void runFunctionNamespaceTest(String TEST_NAME, ScriptType scriptType) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/66ea6f0c/src/test/scripts/functions/misc/Functions14.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Functions14.dml b/src/test/scripts/functions/misc/Functions14.dml new file mode 100644 index 0000000..3ff0ae5 --- /dev/null +++ b/src/test/scripts/functions/misc/Functions14.dml @@ -0,0 +1,43 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +r = 4 +c = 3 +X = rand(rows=r, cols=c, sparsity=1, pdf="normal") +print("sum of rand normal matrix is " + sum(X)) + +Y = rand(rows=r, cols=c, sparsity=1, min=0, max=1, pdf="uniform") +print("avg of rand uniform matrix is " + avg(Y)) + +p = cdf(dist="normal", target=0.4, mean=0, sd=1) +print("normal cdf is " + p) + +p = cdf(dist="exp", target=0.7, rate=1.0) +print("exp cdf is " + p) + +p = cdf(dist="chisq", target=0.3, df=5) +print("chisq cdf is " + p) + +p = cdf(dist="f", target=2, df1=100, df2=200) +print("f cdf is " + p) + +p = cdf(dist="t", target=1, df=100) +print("t cdf is " + p) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/66ea6f0c/src/test/scripts/functions/misc/Functions14.pydml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Functions14.pydml b/src/test/scripts/functions/misc/Functions14.pydml new file mode 100644 index 0000000..6b7ed66 --- /dev/null +++ b/src/test/scripts/functions/misc/Functions14.pydml @@ -0,0 +1,43 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +r = 4 +c = 3 +X = random.normal(r, c, 1) +print("sum of random.normal matrix is " + sum(X)) + +Y = random.uniform(r, c, 1, 0, 1) +print("avg of random.uniform matrix is " + avg(Y)) + +p = norm.cdf(0.4, 0, 1) +print("norm.cdf is " + p) + +p = expon.cdf(0.7, 1.0) +print("expon.cdf is " + p) + +p = chi.cdf(0.3, 5) +print("chi.cdf is " + p) + +p = f.cdf(2, 100, 200) +print("f.cdf is " + p) + +p = t.cdf(1, 100) +print("t.cdf is " + p)
