Yingyi Bu has submitted this change and it was merged.

Change subject: ASTERIXDB-1652: fix dataverse.function(...) to check the 
existence of the dataverse.
......................................................................


ASTERIXDB-1652: fix dataverse.function(...) to check the existence of the 
dataverse.

Change-Id: I7779db56f540fdd645bb85c769baeaa37f620a0d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1192
Reviewed-by: Till Westmann <ti...@apache.org>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
M 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp
M 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp
M 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp
A 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp
A 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp
A 
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp
A 
asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm
M 
asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast
M 
asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M 
asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
M 
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
M 
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
M 
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
14 files changed, 226 insertions(+), 145 deletions(-)

Approvals:
  Till Westmann: Looks good to me, approved
  Jenkins: Verified; Verified

Objections:
  Jenkins: Violations found



diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp
index 6b7441e..0377995 100644
--- 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp
@@ -18,4 +18,4 @@
  */
 
 
-{'result1':(test.hex('AA') > null),'result2':(null >= 
test.hex('AA')),'result3':(test.hex('AA') < null),'result4':(null <= 
test.hex('AA')),'result5':(test.hex('AA') = null),'result6':(null != 
test.hex('AA'))};
+{'result1':(hex('AA') > null),'result2':(null >= 
hex('AA')),'result3':(hex('AA') < null),'result4':(null <= 
hex('AA')),'result5':(hex('AA') = null),'result6':(null != hex('AA'))};
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp
index 320ef99..74e25a7 100644
--- 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp
@@ -18,4 +18,4 @@
  */
 
 
-{'result1':(test.int64('3') > missing),'result2':(missing >= 
test.int64('3')),'result3':(test.int64('3') < missing),'result4':(missing <= 
test.int64('3')),'result5':(test.int64('3') = missing),'result6':(missing != 
test.int64('3'))};
+{'result1':(int64('3') > missing),'result2':(missing >= 
int64('3')),'result3':(int64('3') < missing),'result4':(missing <= 
int64('3')),'result5':(int64('3') = missing),'result6':(missing != int64('3'))};
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp
index 2af586e..a6a2163 100644
--- 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp
@@ -17,4 +17,4 @@
  * under the License.
  */
 
-{'result1':(test.int64('3') > null),'result2':(null >= 
test.int64('3')),'result3':(test.int64('3') < null),'result4':(null <= 
test.int64('3')),'result5':(test.int64('3') = null),'result6':(null != 
test.int64('3'))};
+{'result1':(int64('3') > null),'result2':(null >= 
int64('3')),'result3':(int64('3') < null),'result4':(null <= 
int64('3')),'result5':(int64('3') = null),'result6':(null != int64('3'))};
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp
new file mode 100644
index 0000000..cc938ae
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop  dataverse test if exists;
+create  dataverse test;
+
+use test;
+
+create function length(foo){
+   1
+}
+
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp
new file mode 100644
index 0000000..ba90535
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+// Tests if a builtin function can be overridden.
+test.length("test");
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp
 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp
new file mode 100644
index 0000000..650889c
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+test.length("test");
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm
 
b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm
@@ -0,0 +1 @@
+1
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast
 
b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast
index 3df4327..ee79f0c 100644
--- 
a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast
@@ -4,7 +4,7 @@
     LiteralExpr [STRING] [result1]
     :
     OperatorExpr [
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
       >
@@ -17,7 +17,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       >=
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
     ]
@@ -26,7 +26,7 @@
     LiteralExpr [STRING] [result3]
     :
     OperatorExpr [
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
       <
@@ -39,7 +39,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       <=
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
     ]
@@ -48,7 +48,7 @@
     LiteralExpr [STRING] [result5]
     :
     OperatorExpr [
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
       =
@@ -61,7 +61,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       !=
-      FunctionCall test.hex@1[
+      FunctionCall null.hex@1[
         LiteralExpr [STRING] [AA]
       ]
     ]
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast
 
b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast
index d1a8791..0866ec0 100644
--- 
a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast
+++ 
b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast
@@ -4,7 +4,7 @@
     LiteralExpr [STRING] [result1]
     :
     OperatorExpr [
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
       >
@@ -17,7 +17,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       >=
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
     ]
@@ -26,7 +26,7 @@
     LiteralExpr [STRING] [result3]
     :
     OperatorExpr [
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
       <
@@ -39,7 +39,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       <=
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
     ]
@@ -48,7 +48,7 @@
     LiteralExpr [STRING] [result5]
     :
     OperatorExpr [
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
       =
@@ -61,7 +61,7 @@
     OperatorExpr [
       LiteralExpr [NULL]
       !=
-      FunctionCall test.int64@1[
+      FunctionCall null.int64@1[
         LiteralExpr [STRING] [3]
       ]
     ]
diff --git 
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml 
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index a2bc4ce..2a57ac2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -7002,6 +7002,17 @@
       </compilation-unit>
     </test-case>
     <test-case FilePath="user-defined-functions">
+      <compilation-unit name="query-ASTERIXDB-1652">
+        <expected-error>In function call "test.length(...)", the dataverse 
"test" cannot be found!</expected-error>
+        <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="user-defined-functions">
+      <compilation-unit name="query-ASTERIXDB-1652-2">
+        <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="user-defined-functions">
       <compilation-unit name="query-issue218-2">
         <output-dir compare="Text">query-issue218-2</output-dir>
       </compilation-unit>
diff --git 
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
 
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index 5edc521..4568309 100644
--- 
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++ 
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -34,8 +34,8 @@
 import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
 import org.apache.asterix.lang.common.base.Clause;
 import org.apache.asterix.lang.common.base.Expression;
-import org.apache.asterix.lang.common.base.Expression.Kind;
 import org.apache.asterix.lang.common.base.IQueryRewriter;
+import org.apache.asterix.lang.common.base.Expression.Kind;
 import org.apache.asterix.lang.common.clause.GroupbyClause;
 import org.apache.asterix.lang.common.clause.LetClause;
 import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
@@ -46,11 +46,7 @@
 import org.apache.asterix.lang.common.struct.VarIdentifier;
 import org.apache.asterix.lang.common.util.FunctionUtil;
 import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
 
 class AqlQueryRewriter implements IQueryRewriter {
 
@@ -58,7 +54,6 @@
     private Query topExpr;
     private List<FunctionDecl> declaredFunctions;
     private LangRewritingContext context;
-    private MetadataTransactionContext mdTxnCtx;
     private AqlMetadataProvider metadataProvider;
 
     private void setup(List<FunctionDecl> declaredFunctions, Query topExpr, 
AqlMetadataProvider metadataProvider,
@@ -66,7 +61,6 @@
         this.topExpr = topExpr;
         this.context = context;
         this.declaredFunctions = declaredFunctions;
-        this.mdTxnCtx = metadataProvider.getMetadataTxnContext();
         this.metadataProvider = metadataProvider;
     }
 
@@ -108,9 +102,10 @@
             funIds.add(fdecl.getSignature());
         }
 
-        List<FunctionDecl> otherFDecls = new ArrayList<FunctionDecl>();
-        buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds);
-        declaredFunctions.addAll(otherFDecls);
+        List<FunctionDecl> storedFunctionDecls = 
FunctionUtil.retrieveUsedStoredFunctions(metadataProvider,
+                topExpr.getBody(), funIds, null,
+                expr -> getFunctionCalls(expr), func -> 
functionParser.getFunctionDecl(func), null);
+        declaredFunctions.addAll(storedFunctionDecls);
         if (!declaredFunctions.isEmpty()) {
             AQLInlineUdfsVisitor visitor =
                     new AQLInlineUdfsVisitor(context, new 
AQLRewriterFactory(), declaredFunctions, metadataProvider);
@@ -118,59 +113,7 @@
                 // loop until no more changes
             }
         }
-        declaredFunctions.removeAll(otherFDecls);
-    }
-
-    private void buildOtherUdfs(Expression expression, List<FunctionDecl> 
functionDecls,
-            List<FunctionSignature> declaredFunctions) throws AsterixException 
{
-        if (expression == null) {
-            return;
-        }
-        String value = 
metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS);
-        boolean includePrivateFunctions = (value != null) ? 
Boolean.valueOf(value.toLowerCase()) : false;
-        Set<FunctionSignature> functionCalls = getFunctionCalls(expression);
-        for (FunctionSignature signature : functionCalls) {
-
-            if (declaredFunctions != null && 
declaredFunctions.contains(signature)) {
-                continue;
-            }
-
-            Function function = lookupUserDefinedFunctionDecl(signature);
-            if (function == null) {
-                if 
(AsterixBuiltinFunctions.isBuiltinCompilerFunction(signature, 
includePrivateFunctions)) {
-                    continue;
-                }
-                StringBuilder messageBuilder = new StringBuilder();
-                if (functionDecls.size() > 0) {
-                    messageBuilder.append("function " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
-                            + " depends upon function " + signature + " which 
is undefined");
-                } else {
-                    messageBuilder.append("function " + signature + " is 
undefined ");
-                }
-                throw new AsterixException(messageBuilder.toString());
-            }
-
-            if 
(function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) {
-                FunctionDecl functionDecl = 
functionParser.getFunctionDecl(function);
-                if (functionDecl != null) {
-                    if (functionDecls.contains(functionDecl)) {
-                        throw new AsterixException(
-                                "Recursive invocation " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
-                                        + " <==> " + 
functionDecl.getSignature());
-                    }
-                    functionDecls.add(functionDecl);
-                    buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, 
declaredFunctions);
-                }
-            }
-        }
-
-    }
-
-    private Function lookupUserDefinedFunctionDecl(FunctionSignature 
signature) throws AsterixException {
-        if (signature.getNamespace() == null) {
-            return null;
-        }
-        return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature);
+        declaredFunctions.removeAll(storedFunctionDecls);
     }
 
     private Set<FunctionSignature> getFunctionCalls(Expression expression) 
throws AsterixException {
diff --git 
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
 
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index acd40d7..94866eb 100644
--- 
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ 
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -19,7 +19,19 @@
 
 package org.apache.asterix.lang.common.util;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
@@ -36,4 +48,102 @@
         return getFunctionInfo(new FunctionIdentifier(fs.getNamespace(), 
fs.getName(), fs.getArity()));
     }
 
+    @FunctionalInterface
+    public interface IFunctionCollector {
+        Set<FunctionSignature> getFunctionCalls(Expression expression) throws 
AsterixException;
+    }
+
+    @FunctionalInterface
+    public interface IFunctionParser {
+        FunctionDecl getFunctionDecl(Function function) throws 
AsterixException;
+    }
+
+    @FunctionalInterface
+    public interface IFunctionNormalizer {
+        FunctionSignature normalizeBuiltinFunctionSignature(FunctionSignature 
fs) throws AsterixException;
+    }
+
+    /**
+     * Retrieve stored functions (from CREATE FUNCTION statements) that have 
been used in an expression.
+     *
+     * @param metadataProvider,
+     *            the metadata provider
+     * @param expression,
+     *            the expression for analysis
+     * @param declaredFunctions,
+     *            a set of declared functions in the query, which can 
potentially override stored functions.
+     * @param functionCollector,
+     *            for collecting function calls in the <code>expression</code>
+     * @param functionParser,
+     *            for parsing stored functions in the string represetnation.
+     * @param functionNormalizer,
+     *            for normalizing function names.
+     * @throws AsterixException
+     */
+    public static List<FunctionDecl> 
retrieveUsedStoredFunctions(AqlMetadataProvider metadataProvider,
+            Expression expression, List<FunctionSignature> declaredFunctions, 
List<FunctionDecl> inputFunctionDecls,
+            IFunctionCollector functionCollector, IFunctionParser 
functionParser,
+            IFunctionNormalizer functionNormalizer) throws AsterixException {
+        List<FunctionDecl> functionDecls = inputFunctionDecls == null ? new 
ArrayList<>()
+                : new ArrayList<>(inputFunctionDecls);
+        if (expression == null) {
+            return functionDecls;
+        }
+        String value = 
metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS);
+        boolean includePrivateFunctions = (value != null) ? 
Boolean.valueOf(value.toLowerCase()) : false;
+        Set<FunctionSignature> functionCalls = 
functionCollector.getFunctionCalls(expression);
+        for (FunctionSignature signature : functionCalls) {
+            if (declaredFunctions != null && 
declaredFunctions.contains(signature)) {
+                continue;
+            }
+            String dataverseName = signature.getNamespace() == null ? 
metadataProvider.getDefaultDataverseName()
+                    : signature.getNamespace();
+            // Checks the existence of the referred dataverse.
+            if (metadataProvider.findDataverse(dataverseName) == null
+                    && !dataverseName.equals(FunctionConstants.ASTERIX_NS)) {
+                throw new AsterixException("In function call \"" + 
dataverseName + "." + signature.getName()
+                        + "(...)\", the dataverse \"" + dataverseName + "\" 
cannot be found!");
+            }
+            Function function = 
lookupUserDefinedFunctionDecl(metadataProvider.getMetadataTxnContext(), 
signature);
+            if (function == null) {
+                FunctionSignature normalizedSignature = functionNormalizer == 
null ? signature
+                        : 
functionNormalizer.normalizeBuiltinFunctionSignature(signature);
+                if 
(AsterixBuiltinFunctions.isBuiltinCompilerFunction(normalizedSignature, 
includePrivateFunctions)) {
+                    continue;
+                }
+                StringBuilder messageBuilder = new StringBuilder();
+                if (!functionDecls.isEmpty()) {
+                    messageBuilder.append("function " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
+                            + " depends upon function " + signature + " which 
is undefined");
+                } else {
+                    messageBuilder.append("function " + signature + " is 
undefined ");
+                }
+                throw new AsterixException(messageBuilder.toString());
+            }
+
+            if 
(function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) {
+                FunctionDecl functionDecl = 
functionParser.getFunctionDecl(function);
+                if (functionDecl != null) {
+                    if (functionDecls.contains(functionDecl)) {
+                        throw new AsterixException(
+                                "Recursive invocation " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
+                                        + " <==> " + 
functionDecl.getSignature());
+                    }
+                    functionDecls.add(functionDecl);
+                    functionDecls = 
retrieveUsedStoredFunctions(metadataProvider, functionDecl.getFuncBody(),
+                            declaredFunctions, functionDecls, 
functionCollector, functionParser, functionNormalizer);
+                }
+            }
+        }
+        return functionDecls;
+    }
+
+    private static Function 
lookupUserDefinedFunctionDecl(MetadataTransactionContext mdTxnCtx,
+            FunctionSignature signature) throws AsterixException {
+        if (signature.getNamespace() == null) {
+            return null;
+        }
+        return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature);
+    }
+
 }
diff --git 
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
 
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index dd79969..329f04b 100644
--- 
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ 
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -65,11 +65,7 @@
 import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
 import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
 import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
 
 class SqlppQueryRewriter implements IQueryRewriter {
     private static final String INLINE_WITH = "inline_with";
@@ -78,7 +74,6 @@
     private Query topExpr;
     private List<FunctionDecl> declaredFunctions;
     private LangRewritingContext context;
-    private MetadataTransactionContext mdTxnCtx;
     private AqlMetadataProvider metadataProvider;
 
     protected void setup(List<FunctionDecl> declaredFunctions, Query topExpr, 
AqlMetadataProvider metadataProvider,
@@ -86,7 +81,6 @@
         this.topExpr = topExpr;
         this.context = context;
         this.declaredFunctions = declaredFunctions;
-        this.mdTxnCtx = metadataProvider.getMetadataTxnContext();
         this.metadataProvider = metadataProvider;
     }
 
@@ -259,9 +253,11 @@
             funIds.add(fdecl.getSignature());
         }
 
-        List<FunctionDecl> otherFDecls = new ArrayList<FunctionDecl>();
-        buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds);
-        declaredFunctions.addAll(otherFDecls);
+        List<FunctionDecl> usedStoredFunctionDecls = 
FunctionUtil.retrieveUsedStoredFunctions(metadataProvider,
+                topExpr.getBody(), funIds, null,
+                expr -> getFunctionCalls(expr), func -> 
functionRepository.getFunctionDecl(func),
+                signature -> 
FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false));
+        declaredFunctions.addAll(usedStoredFunctionDecls);
         if (!declaredFunctions.isEmpty()) {
             SqlppInlineUdfsVisitor visitor = new 
SqlppInlineUdfsVisitor(context,
                     new SqlppFunctionBodyRewriterFactory() /* the rewriter for 
function bodies expressions*/,
@@ -270,61 +266,7 @@
                 // loop until no more changes
             }
         }
-        declaredFunctions.removeAll(otherFDecls);
-    }
-
-    protected void buildOtherUdfs(Expression expression, List<FunctionDecl> 
functionDecls,
-            List<FunctionSignature> declaredFunctions) throws AsterixException 
{
-        if (expression == null) {
-            return;
-        }
-        String value = 
metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS);
-        boolean includePrivateFunctions = (value != null) ? 
Boolean.valueOf(value.toLowerCase()) : false;
-        Set<FunctionSignature> functionCalls = getFunctionCalls(expression);
-        for (FunctionSignature signature : functionCalls) {
-
-            if (declaredFunctions != null && 
declaredFunctions.contains(signature)) {
-                continue;
-            }
-
-            Function function = lookupUserDefinedFunctionDecl(signature);
-            if (function == null) {
-                FunctionSignature normalizedSignature =
-                        
FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false);
-                if 
(AsterixBuiltinFunctions.isBuiltinCompilerFunction(normalizedSignature, 
includePrivateFunctions)) {
-                    continue;
-                }
-                StringBuilder messageBuilder = new StringBuilder();
-                if (functionDecls.size() > 0) {
-                    messageBuilder.append("function " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
-                            + " depends upon function " + signature + " which 
is undefined");
-                } else {
-                    messageBuilder.append("function " + signature + " is 
undefined ");
-                }
-                throw new AsterixException(messageBuilder.toString());
-            }
-
-            if 
(function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) {
-                FunctionDecl functionDecl = 
functionRepository.getFunctionDecl(function);
-                if (functionDecl != null) {
-                    if (functionDecls.contains(functionDecl)) {
-                        throw new AsterixException(
-                                "Recursive invocation " + 
functionDecls.get(functionDecls.size() - 1).getSignature()
-                                        + " <==> " + 
functionDecl.getSignature());
-                    }
-                    functionDecls.add(functionDecl);
-                    buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, 
declaredFunctions);
-                }
-            }
-        }
-
-    }
-
-    private Function lookupUserDefinedFunctionDecl(FunctionSignature 
signature) throws AsterixException {
-        if (signature.getNamespace() == null) {
-            return null;
-        }
-        return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature);
+        declaredFunctions.removeAll(usedStoredFunctionDecls);
     }
 
     private Set<FunctionSignature> getFunctionCalls(Expression expression) 
throws AsterixException {
diff --git 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 41a843f..29087ba 100644
--- 
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ 
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -26,10 +26,10 @@
 import java.util.Map;
 
 import org.apache.asterix.common.config.AsterixStorageProperties;
+import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import org.apache.asterix.common.context.ITransactionSubsystemProvider;
 import org.apache.asterix.common.context.TransactionSubsystemProvider;
@@ -42,8 +42,8 @@
 import 
org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
 import 
org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
 import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
 import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import 
org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
 import org.apache.asterix.external.adapter.factory.LookupAdapterFactory;
@@ -92,8 +92,8 @@
 import 
org.apache.asterix.runtime.operators.AsterixLSMInvertedIndexUpsertOperatorDescriptor;
 import 
org.apache.asterix.runtime.operators.AsterixLSMTreeUpsertOperatorDescriptor;
 import org.apache.asterix.runtime.util.AsterixAppContextInfo;
-import org.apache.asterix.runtime.util.ClusterStateManager;
 import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider;
+import org.apache.asterix.runtime.util.ClusterStateManager;
 import 
org.apache.asterix.transaction.management.opcallbacks.LockThenSearchOperationCallbackFactory;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory;
 import 
org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory;
@@ -417,6 +417,10 @@
         return format;
     }
 
+    public Dataverse findDataverse(String dataverseName) throws 
AsterixException {
+        return MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+    }
+
     public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, 
IAdapterFactory> buildFeedIntakeRuntime(
             JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor 
policyAccessor) throws Exception {
         Triple<IAdapterFactory, RecordDescriptor, 
IDataSourceAdapter.AdapterType> factoryOutput;

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1192
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I7779db56f540fdd645bb85c769baeaa37f620a0d
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <buyin...@gmail.com>
Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <ti...@apache.org>
Gerrit-Reviewer: Yingyi Bu <buyin...@gmail.com>

Reply via email to