Repository: asterixdb Updated Branches: refs/heads/master 2816885d1 -> 83108375a
ASTERIXDB-1652: fix dataverse.function(...) to check the existence of the dataverse. Change-Id: I7779db56f540fdd645bb85c769baeaa37f620a0d Reviewed-on: https://asterix-gerrit.ics.uci.edu/1192 Reviewed-by: Till Westmann <ti...@apache.org> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/83108375 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/83108375 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/83108375 Branch: refs/heads/master Commit: 83108375a45ffe9d81c9e3305595803066bc6139 Parents: 2816885 Author: Yingyi Bu <yin...@couchbase.com> Authored: Wed Sep 21 16:26:51 2016 -0700 Committer: Yingyi Bu <buyin...@gmail.com> Committed: Wed Sep 21 18:21:01 2016 -0700 ---------------------------------------------------------------------- .../binary_null/binary_null.1.query.sqlpp | 2 +- .../int64_missing/int64_missing.1.query.sqlpp | 2 +- .../int64_null/int64_null.1.query.sqlpp | 2 +- .../query-ASTERIXDB-1652-2.1.ddl.sqlpp | 28 +++++ .../query-ASTERIXDB-1652-2.2.query.sqlpp | 22 ++++ .../query-ASTERIXDB-1652.1.query.sqlpp | 20 ++++ .../query-ASTERIXDB-1652-2.1.adm | 1 + .../comparison/binary_null/binary_null.1.ast | 12 +- .../comparison/int64_null/int64_null.1.ast | 12 +- .../resources/runtimets/testsuite_sqlpp.xml | 11 ++ .../lang/aql/rewrites/AqlQueryRewriter.java | 69 +----------- .../asterix/lang/common/util/FunctionUtil.java | 110 +++++++++++++++++++ .../lang/sqlpp/rewrites/SqlppQueryRewriter.java | 70 +----------- .../metadata/declared/AqlMetadataProvider.java | 10 +- 14 files changed, 226 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp index 6b7441e..0377995 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/binary_null/binary_null.1.query.sqlpp @@ -18,4 +18,4 @@ */ -{'result1':(test.hex('AA') > null),'result2':(null >= test.hex('AA')),'result3':(test.hex('AA') < null),'result4':(null <= test.hex('AA')),'result5':(test.hex('AA') = null),'result6':(null != test.hex('AA'))}; +{'result1':(hex('AA') > null),'result2':(null >= hex('AA')),'result3':(hex('AA') < null),'result4':(null <= hex('AA')),'result5':(hex('AA') = null),'result6':(null != hex('AA'))}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp index 320ef99..74e25a7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_missing/int64_missing.1.query.sqlpp @@ -18,4 +18,4 @@ */ -{'result1':(test.int64('3') > missing),'result2':(missing >= test.int64('3')),'result3':(test.int64('3') < missing),'result4':(missing <= test.int64('3')),'result5':(test.int64('3') = missing),'result6':(missing != test.int64('3'))}; +{'result1':(int64('3') > missing),'result2':(missing >= int64('3')),'result3':(int64('3') < missing),'result4':(missing <= int64('3')),'result5':(int64('3') = missing),'result6':(missing != int64('3'))}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp index 2af586e..a6a2163 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/comparison/int64_null/int64_null.1.query.sqlpp @@ -17,4 +17,4 @@ * under the License. */ -{'result1':(test.int64('3') > null),'result2':(null >= test.int64('3')),'result3':(test.int64('3') < null),'result4':(null <= test.int64('3')),'result5':(test.int64('3') = null),'result6':(null != test.int64('3'))}; +{'result1':(int64('3') > null),'result2':(null >= int64('3')),'result3':(int64('3') < null),'result4':(null <= int64('3')),'result5':(int64('3') = null),'result6':(null != int64('3'))}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp new file mode 100644 index 0000000..cc938ae --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; + +use test; + +create function length(foo){ + 1 +} + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp new file mode 100644 index 0000000..ba90535 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.2.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +// Tests if a builtin function can be overridden. +test.length("test"); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp new file mode 100644 index 0000000..650889c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/query-ASTERIXDB-1652/query-ASTERIXDB-1652.1.query.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +test.length("test"); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/query-ASTERIXDB-1652-2/query-ASTERIXDB-1652-2.1.adm @@ -0,0 +1 @@ +1 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast index 3df4327..ee79f0c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/binary_null/binary_null.1.ast @@ -4,7 +4,7 @@ RecordConstructor [ LiteralExpr [STRING] [result1] : OperatorExpr [ - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] > @@ -17,7 +17,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] >= - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] ] @@ -26,7 +26,7 @@ RecordConstructor [ LiteralExpr [STRING] [result3] : OperatorExpr [ - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] < @@ -39,7 +39,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] <= - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] ] @@ -48,7 +48,7 @@ RecordConstructor [ LiteralExpr [STRING] [result5] : OperatorExpr [ - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] = @@ -61,7 +61,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] != - FunctionCall test.hex@1[ + FunctionCall null.hex@1[ LiteralExpr [STRING] [AA] ] ] http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast index d1a8791..0866ec0 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/comparison/int64_null/int64_null.1.ast @@ -4,7 +4,7 @@ RecordConstructor [ LiteralExpr [STRING] [result1] : OperatorExpr [ - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] > @@ -17,7 +17,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] >= - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] ] @@ -26,7 +26,7 @@ RecordConstructor [ LiteralExpr [STRING] [result3] : OperatorExpr [ - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] < @@ -39,7 +39,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] <= - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] ] @@ -48,7 +48,7 @@ RecordConstructor [ LiteralExpr [STRING] [result5] : OperatorExpr [ - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] = @@ -61,7 +61,7 @@ RecordConstructor [ OperatorExpr [ LiteralExpr [NULL] != - FunctionCall test.int64@1[ + FunctionCall null.int64@1[ LiteralExpr [STRING] [3] ] ] http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index a2bc4ce..2a57ac2 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -7002,6 +7002,17 @@ </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> + <compilation-unit name="query-ASTERIXDB-1652"> + <expected-error>In function call "test.length(...)", the dataverse "test" cannot be found!</expected-error> + <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="query-ASTERIXDB-1652-2"> + <output-dir compare="Text">query-ASTERIXDB-1652-2</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> <compilation-unit name="query-issue218-2"> <output-dir compare="Text">query-issue218-2</output-dir> </compilation-unit> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java index 5edc521..4568309 100644 --- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java +++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java @@ -34,8 +34,8 @@ import org.apache.asterix.lang.aql.visitor.AQLInlineUdfsVisitor; import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor; import org.apache.asterix.lang.common.base.Clause; import org.apache.asterix.lang.common.base.Expression; -import org.apache.asterix.lang.common.base.Expression.Kind; import org.apache.asterix.lang.common.base.IQueryRewriter; +import org.apache.asterix.lang.common.base.Expression.Kind; import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; @@ -46,11 +46,7 @@ import org.apache.asterix.lang.common.statement.Query; import org.apache.asterix.lang.common.struct.VarIdentifier; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor; -import org.apache.asterix.metadata.MetadataManager; -import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.AqlMetadataProvider; -import org.apache.asterix.metadata.entities.Function; -import org.apache.asterix.om.functions.AsterixBuiltinFunctions; class AqlQueryRewriter implements IQueryRewriter { @@ -58,7 +54,6 @@ class AqlQueryRewriter implements IQueryRewriter { private Query topExpr; private List<FunctionDecl> declaredFunctions; private LangRewritingContext context; - private MetadataTransactionContext mdTxnCtx; private AqlMetadataProvider metadataProvider; private void setup(List<FunctionDecl> declaredFunctions, Query topExpr, AqlMetadataProvider metadataProvider, @@ -66,7 +61,6 @@ class AqlQueryRewriter implements IQueryRewriter { this.topExpr = topExpr; this.context = context; this.declaredFunctions = declaredFunctions; - this.mdTxnCtx = metadataProvider.getMetadataTxnContext(); this.metadataProvider = metadataProvider; } @@ -108,9 +102,10 @@ class AqlQueryRewriter implements IQueryRewriter { funIds.add(fdecl.getSignature()); } - List<FunctionDecl> otherFDecls = new ArrayList<FunctionDecl>(); - buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds); - declaredFunctions.addAll(otherFDecls); + List<FunctionDecl> storedFunctionDecls = FunctionUtil.retrieveUsedStoredFunctions(metadataProvider, + topExpr.getBody(), funIds, null, + expr -> getFunctionCalls(expr), func -> functionParser.getFunctionDecl(func), null); + declaredFunctions.addAll(storedFunctionDecls); if (!declaredFunctions.isEmpty()) { AQLInlineUdfsVisitor visitor = new AQLInlineUdfsVisitor(context, new AQLRewriterFactory(), declaredFunctions, metadataProvider); @@ -118,59 +113,7 @@ class AqlQueryRewriter implements IQueryRewriter { // loop until no more changes } } - declaredFunctions.removeAll(otherFDecls); - } - - private void buildOtherUdfs(Expression expression, List<FunctionDecl> functionDecls, - List<FunctionSignature> declaredFunctions) throws AsterixException { - if (expression == null) { - return; - } - String value = metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS); - boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false; - Set<FunctionSignature> functionCalls = getFunctionCalls(expression); - for (FunctionSignature signature : functionCalls) { - - if (declaredFunctions != null && declaredFunctions.contains(signature)) { - continue; - } - - Function function = lookupUserDefinedFunctionDecl(signature); - if (function == null) { - if (AsterixBuiltinFunctions.isBuiltinCompilerFunction(signature, includePrivateFunctions)) { - continue; - } - StringBuilder messageBuilder = new StringBuilder(); - if (functionDecls.size() > 0) { - messageBuilder.append("function " + functionDecls.get(functionDecls.size() - 1).getSignature() - + " depends upon function " + signature + " which is undefined"); - } else { - messageBuilder.append("function " + signature + " is undefined "); - } - throw new AsterixException(messageBuilder.toString()); - } - - if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) { - FunctionDecl functionDecl = functionParser.getFunctionDecl(function); - if (functionDecl != null) { - if (functionDecls.contains(functionDecl)) { - throw new AsterixException( - "Recursive invocation " + functionDecls.get(functionDecls.size() - 1).getSignature() - + " <==> " + functionDecl.getSignature()); - } - functionDecls.add(functionDecl); - buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, declaredFunctions); - } - } - } - - } - - private Function lookupUserDefinedFunctionDecl(FunctionSignature signature) throws AsterixException { - if (signature.getNamespace() == null) { - return null; - } - return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature); + declaredFunctions.removeAll(storedFunctionDecls); } private Set<FunctionSignature> getFunctionCalls(Expression expression) throws AsterixException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java index acd40d7..94866eb 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java @@ -19,7 +19,19 @@ package org.apache.asterix.lang.common.util; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.functions.FunctionConstants; import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.lang.common.base.Expression; +import org.apache.asterix.lang.common.statement.FunctionDecl; +import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.metadata.MetadataTransactionContext; +import org.apache.asterix.metadata.declared.AqlMetadataProvider; +import org.apache.asterix.metadata.entities.Function; import org.apache.asterix.om.functions.AsterixBuiltinFunctions; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo; @@ -36,4 +48,102 @@ public class FunctionUtil { return getFunctionInfo(new FunctionIdentifier(fs.getNamespace(), fs.getName(), fs.getArity())); } + @FunctionalInterface + public interface IFunctionCollector { + Set<FunctionSignature> getFunctionCalls(Expression expression) throws AsterixException; + } + + @FunctionalInterface + public interface IFunctionParser { + FunctionDecl getFunctionDecl(Function function) throws AsterixException; + } + + @FunctionalInterface + public interface IFunctionNormalizer { + FunctionSignature normalizeBuiltinFunctionSignature(FunctionSignature fs) throws AsterixException; + } + + /** + * Retrieve stored functions (from CREATE FUNCTION statements) that have been used in an expression. + * + * @param metadataProvider, + * the metadata provider + * @param expression, + * the expression for analysis + * @param declaredFunctions, + * a set of declared functions in the query, which can potentially override stored functions. + * @param functionCollector, + * for collecting function calls in the <code>expression</code> + * @param functionParser, + * for parsing stored functions in the string represetnation. + * @param functionNormalizer, + * for normalizing function names. + * @throws AsterixException + */ + public static List<FunctionDecl> retrieveUsedStoredFunctions(AqlMetadataProvider metadataProvider, + Expression expression, List<FunctionSignature> declaredFunctions, List<FunctionDecl> inputFunctionDecls, + IFunctionCollector functionCollector, IFunctionParser functionParser, + IFunctionNormalizer functionNormalizer) throws AsterixException { + List<FunctionDecl> functionDecls = inputFunctionDecls == null ? new ArrayList<>() + : new ArrayList<>(inputFunctionDecls); + if (expression == null) { + return functionDecls; + } + String value = metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS); + boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false; + Set<FunctionSignature> functionCalls = functionCollector.getFunctionCalls(expression); + for (FunctionSignature signature : functionCalls) { + if (declaredFunctions != null && declaredFunctions.contains(signature)) { + continue; + } + String dataverseName = signature.getNamespace() == null ? metadataProvider.getDefaultDataverseName() + : signature.getNamespace(); + // Checks the existence of the referred dataverse. + if (metadataProvider.findDataverse(dataverseName) == null + && !dataverseName.equals(FunctionConstants.ASTERIX_NS)) { + throw new AsterixException("In function call \"" + dataverseName + "." + signature.getName() + + "(...)\", the dataverse \"" + dataverseName + "\" cannot be found!"); + } + Function function = lookupUserDefinedFunctionDecl(metadataProvider.getMetadataTxnContext(), signature); + if (function == null) { + FunctionSignature normalizedSignature = functionNormalizer == null ? signature + : functionNormalizer.normalizeBuiltinFunctionSignature(signature); + if (AsterixBuiltinFunctions.isBuiltinCompilerFunction(normalizedSignature, includePrivateFunctions)) { + continue; + } + StringBuilder messageBuilder = new StringBuilder(); + if (!functionDecls.isEmpty()) { + messageBuilder.append("function " + functionDecls.get(functionDecls.size() - 1).getSignature() + + " depends upon function " + signature + " which is undefined"); + } else { + messageBuilder.append("function " + signature + " is undefined "); + } + throw new AsterixException(messageBuilder.toString()); + } + + if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) { + FunctionDecl functionDecl = functionParser.getFunctionDecl(function); + if (functionDecl != null) { + if (functionDecls.contains(functionDecl)) { + throw new AsterixException( + "Recursive invocation " + functionDecls.get(functionDecls.size() - 1).getSignature() + + " <==> " + functionDecl.getSignature()); + } + functionDecls.add(functionDecl); + functionDecls = retrieveUsedStoredFunctions(metadataProvider, functionDecl.getFuncBody(), + declaredFunctions, functionDecls, functionCollector, functionParser, functionNormalizer); + } + } + } + return functionDecls; + } + + private static Function lookupUserDefinedFunctionDecl(MetadataTransactionContext mdTxnCtx, + FunctionSignature signature) throws AsterixException { + if (signature.getNamespace() == null) { + return null; + } + return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java index dd79969..329f04b 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java @@ -65,11 +65,7 @@ import org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVis import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil; import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor; -import org.apache.asterix.metadata.MetadataManager; -import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.AqlMetadataProvider; -import org.apache.asterix.metadata.entities.Function; -import org.apache.asterix.om.functions.AsterixBuiltinFunctions; class SqlppQueryRewriter implements IQueryRewriter { private static final String INLINE_WITH = "inline_with"; @@ -78,7 +74,6 @@ class SqlppQueryRewriter implements IQueryRewriter { private Query topExpr; private List<FunctionDecl> declaredFunctions; private LangRewritingContext context; - private MetadataTransactionContext mdTxnCtx; private AqlMetadataProvider metadataProvider; protected void setup(List<FunctionDecl> declaredFunctions, Query topExpr, AqlMetadataProvider metadataProvider, @@ -86,7 +81,6 @@ class SqlppQueryRewriter implements IQueryRewriter { this.topExpr = topExpr; this.context = context; this.declaredFunctions = declaredFunctions; - this.mdTxnCtx = metadataProvider.getMetadataTxnContext(); this.metadataProvider = metadataProvider; } @@ -259,9 +253,11 @@ class SqlppQueryRewriter implements IQueryRewriter { funIds.add(fdecl.getSignature()); } - List<FunctionDecl> otherFDecls = new ArrayList<FunctionDecl>(); - buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds); - declaredFunctions.addAll(otherFDecls); + List<FunctionDecl> usedStoredFunctionDecls = FunctionUtil.retrieveUsedStoredFunctions(metadataProvider, + topExpr.getBody(), funIds, null, + expr -> getFunctionCalls(expr), func -> functionRepository.getFunctionDecl(func), + signature -> FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false)); + declaredFunctions.addAll(usedStoredFunctionDecls); if (!declaredFunctions.isEmpty()) { SqlppInlineUdfsVisitor visitor = new SqlppInlineUdfsVisitor(context, new SqlppFunctionBodyRewriterFactory() /* the rewriter for function bodies expressions*/, @@ -270,61 +266,7 @@ class SqlppQueryRewriter implements IQueryRewriter { // loop until no more changes } } - declaredFunctions.removeAll(otherFDecls); - } - - protected void buildOtherUdfs(Expression expression, List<FunctionDecl> functionDecls, - List<FunctionSignature> declaredFunctions) throws AsterixException { - if (expression == null) { - return; - } - String value = metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS); - boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false; - Set<FunctionSignature> functionCalls = getFunctionCalls(expression); - for (FunctionSignature signature : functionCalls) { - - if (declaredFunctions != null && declaredFunctions.contains(signature)) { - continue; - } - - Function function = lookupUserDefinedFunctionDecl(signature); - if (function == null) { - FunctionSignature normalizedSignature = - FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false); - if (AsterixBuiltinFunctions.isBuiltinCompilerFunction(normalizedSignature, includePrivateFunctions)) { - continue; - } - StringBuilder messageBuilder = new StringBuilder(); - if (functionDecls.size() > 0) { - messageBuilder.append("function " + functionDecls.get(functionDecls.size() - 1).getSignature() - + " depends upon function " + signature + " which is undefined"); - } else { - messageBuilder.append("function " + signature + " is undefined "); - } - throw new AsterixException(messageBuilder.toString()); - } - - if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) { - FunctionDecl functionDecl = functionRepository.getFunctionDecl(function); - if (functionDecl != null) { - if (functionDecls.contains(functionDecl)) { - throw new AsterixException( - "Recursive invocation " + functionDecls.get(functionDecls.size() - 1).getSignature() - + " <==> " + functionDecl.getSignature()); - } - functionDecls.add(functionDecl); - buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, declaredFunctions); - } - } - } - - } - - private Function lookupUserDefinedFunctionDecl(FunctionSignature signature) throws AsterixException { - if (signature.getNamespace() == null) { - return null; - } - return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature); + declaredFunctions.removeAll(usedStoredFunctionDecls); } private Set<FunctionSignature> getFunctionCalls(Expression expression) throws AsterixException { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/83108375/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java index 41a843f..29087ba 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java @@ -26,10 +26,10 @@ import java.util.List; import java.util.Map; import org.apache.asterix.common.config.AsterixStorageProperties; +import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp; import org.apache.asterix.common.config.DatasetConfig.IndexType; -import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider; import org.apache.asterix.common.context.ITransactionSubsystemProvider; import org.apache.asterix.common.context.TransactionSubsystemProvider; @@ -42,8 +42,8 @@ import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallb import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory; import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory; import org.apache.asterix.common.library.ILibraryManager; -import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType; import org.apache.asterix.common.transactions.JobId; +import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory; import org.apache.asterix.external.adapter.factory.LookupAdapterFactory; @@ -92,8 +92,8 @@ import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.runtime.operators.AsterixLSMInvertedIndexUpsertOperatorDescriptor; import org.apache.asterix.runtime.operators.AsterixLSMTreeUpsertOperatorDescriptor; import org.apache.asterix.runtime.util.AsterixAppContextInfo; -import org.apache.asterix.runtime.util.ClusterStateManager; import org.apache.asterix.runtime.util.AsterixRuntimeComponentsProvider; +import org.apache.asterix.runtime.util.ClusterStateManager; import org.apache.asterix.transaction.management.opcallbacks.LockThenSearchOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory; @@ -417,6 +417,10 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin return format; } + public Dataverse findDataverse(String dataverseName) throws AsterixException { + return MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName); + } + public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime( JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception { Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> factoryOutput;
