This is an automated email from the ASF dual-hosted git repository. dlych pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6ce9bde265a5a9f5a6c308afb365d4807d16fdf2 Author: Dmitry Lychagin <[email protected]> AuthorDate: Fri Mar 5 18:34:53 2021 -0800 [NO ISSUE][COMP] Prohibit DROP SYNONYM if it is used by UDF - user model changes: no - storage format changes: no - interface changes: no Details: - Prohibit dropping synonym that is used by user-defined function - Prohibit dropping dataverse if it contains a synonym that used by user-defined function in another dataverse - Add testcases Change-Id: Ie05948b62fa3471b5989698073bbdf0e96364baf Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/10384 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- ...e.2.update.sqlpp => drop-dataverse.2.ddl.sqlpp} | 25 +++- .../check-dependencies-1.1.ddl.sqlpp | 8 ++ .../drop-dependency-4.3.ddl.sqlpp} | 46 ++----- .../drop-dependency-4.4.ddl.sqlpp} | 46 ++----- .../drop-dependency-6.3.ddl.sqlpp} | 50 ++----- .../drop-dependency-6.4.ddl.sqlpp} | 50 ++----- .../check-dependencies-1.1.adm | 1 + .../test/resources/runtimets/testsuite_sqlpp.xml | 13 +- .../asterix/lang/common/util/FunctionUtil.java | 55 ++++---- .../visitor/VariableCheckAndRewriteVisitor.java | 24 +++- .../sqlpp/visitor/SqlppSynonymRewriteVisitor.java | 10 +- .../org/apache/asterix/metadata/MetadataNode.java | 143 +++++++++++---------- .../metadata/declared/MetadataProvider.java | 9 +- .../apache/asterix/metadata/entities/Function.java | 39 ++++++ 14 files changed, 255 insertions(+), 264 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.ddl.sqlpp similarity index 69% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.update.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.ddl.sqlpp index 1f90ae9..8ecdeba 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.update.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/cross-dataverse/drop-dataverse/drop-dataverse.2.ddl.sqlpp @@ -18,8 +18,27 @@ */ /* * Description : Test cross dataverse functionality - * : create a dataset using a foreign datatype - * : Try dropping the foreign type's dataverse + * : create a function using a foreign synonym + * : Try dropping the foreign synonym's dataverse * Expected Res : Failure - * Date : 2015 Steven Jacobs */ + +drop dataverse b if exists; +drop dataverse a if exists; + +create dataverse a; +create dataverse b; + +create type a.a as { + id:int32 +}; + +create dataset a.b1(a.a) primary key id; + +create synonym a.s1 for a.b1; + +create function b.f1() { + select * from a.s1 +}; + +drop dataverse a; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp index c9639c4..0551845 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp @@ -41,6 +41,10 @@ create type TweetMessageType as closed { create dataset TweetMessages(TweetMessageType) primary key tweetid autogenerated; +create synonym TweetMessagesSyn1 for TweetMessages; + +create synonym TweetMessagesSyn2 for TweetMessages; + create function f1(message, text){ contains(message,text) }; @@ -49,6 +53,10 @@ create function f4(){ (select * from TweetMessages) }; +create function f5(){ +(select * from TweetMessagesSyn1, TweetMessagesSyn2) +}; + use B; create dataset TweetMessages2(C.TweetMessageType) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.3.ddl.sqlpp similarity index 57% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.3.ddl.sqlpp index c9639c4..7943d82 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.3.ddl.sqlpp @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + /* - * Description : Verify Function Dependency Metadata - * Expected Res : Success - * Date : Dec 15th 2017 + * Description : Try to drop a functional dependency + * Drop a synonym that is used by a non-varargs function + * Expected Res : Error */ drop dataverse B if exists; @@ -41,41 +42,14 @@ create type TweetMessageType as closed { create dataset TweetMessages(TweetMessageType) primary key tweetid autogenerated; -create function f1(message, text){ - contains(message,text) -}; - -create function f4(){ -(select * from TweetMessages) -}; +create synonym TweetMessagesSyn for TweetMessages; use B; -create dataset TweetMessages2(C.TweetMessageType) -primary key tweetid autogenerated; - -create function f0(message, text){ - contains(message,text) -}; - -create function C.f2(place, text) { - (select m.message_text - from TweetMessages m - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and f1(m.message_text,text) - and B.f0(m.message_text,text)) -}; - -create function C.f3(place, text) { - f2(place, text) +create function f2(place, text){ + (select m.message_text + from C.TweetMessagesSyn m) }; -create function f5(place, text){ - (select m.message_text - from TweetMessages2 m, C.TweetMessages m2 - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and C.f1(m.message_text,text) - and f0(m2.message_text,text)) -}; \ No newline at end of file +use C; +drop synonym TweetMessagesSyn; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.4.ddl.sqlpp similarity index 57% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.4.ddl.sqlpp index c9639c4..7f0ff98 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency-4.4.ddl.sqlpp @@ -16,10 +16,11 @@ * specific language governing permissions and limitations * under the License. */ + /* - * Description : Verify Function Dependency Metadata - * Expected Res : Success - * Date : Dec 15th 2017 + * Description : Try to drop a functional dependency + * Drop a synonym that is used by a varargs function + * Expected Res : Error */ drop dataverse B if exists; @@ -41,41 +42,14 @@ create type TweetMessageType as closed { create dataset TweetMessages(TweetMessageType) primary key tweetid autogenerated; -create function f1(message, text){ - contains(message,text) -}; - -create function f4(){ -(select * from TweetMessages) -}; +create synonym TweetMessagesSyn for TweetMessages; use B; -create dataset TweetMessages2(C.TweetMessageType) -primary key tweetid autogenerated; - -create function f0(message, text){ - contains(message,text) -}; - -create function C.f2(place, text) { - (select m.message_text - from TweetMessages m - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and f1(m.message_text,text) - and B.f0(m.message_text,text)) -}; - -create function C.f3(place, text) { - f2(place, text) +create function f2(...){ + (select m.message_text + from C.TweetMessagesSyn m) }; -create function f5(place, text){ - (select m.message_text - from TweetMessages2 m, C.TweetMessages m2 - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and C.f1(m.message_text,text) - and f0(m2.message_text,text)) -}; \ No newline at end of file +use C; +drop synonym TweetMessagesSyn; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.3.ddl.sqlpp similarity index 55% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.3.ddl.sqlpp index c9639c4..6cf428a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.3.ddl.sqlpp @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + /* - * Description : Verify Function Dependency Metadata - * Expected Res : Success - * Date : Dec 15th 2017 + * Description : Try to drop a functional dependency + * Drop a synonym that is used by a non-varargs function + * from the same dataverse + * Expected Res : Error */ -drop dataverse B if exists; drop dataverse C if exists; -create dataverse B; create dataverse C; use C; @@ -41,41 +41,11 @@ create type TweetMessageType as closed { create dataset TweetMessages(TweetMessageType) primary key tweetid autogenerated; -create function f1(message, text){ - contains(message,text) -}; - -create function f4(){ -(select * from TweetMessages) -}; - -use B; - -create dataset TweetMessages2(C.TweetMessageType) -primary key tweetid autogenerated; +create synonym TweetMessagesSyn for TweetMessages; -create function f0(message, text){ - contains(message,text) -}; - -create function C.f2(place, text) { - (select m.message_text - from TweetMessages m - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and f1(m.message_text,text) - and B.f0(m.message_text,text)) -}; - -create function C.f3(place, text) { - f2(place, text) +create function f2(place, text){ + (select m.message_text + from C.TweetMessagesSyn m) }; -create function f5(place, text){ - (select m.message_text - from TweetMessages2 m, C.TweetMessages m2 - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and C.f1(m.message_text,text) - and f0(m2.message_text,text)) -}; \ No newline at end of file +drop synonym TweetMessagesSyn; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.4.ddl.sqlpp similarity index 55% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.4.ddl.sqlpp index c9639c4..5198f66 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency-6.4.ddl.sqlpp @@ -16,15 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + /* - * Description : Verify Function Dependency Metadata - * Expected Res : Success - * Date : Dec 15th 2017 + * Description : Try to drop a functional dependency + * Drop a synonym that is used by a varargs function + * from the same dataverse + * Expected Res : Error */ -drop dataverse B if exists; drop dataverse C if exists; -create dataverse B; create dataverse C; use C; @@ -41,41 +41,11 @@ create type TweetMessageType as closed { create dataset TweetMessages(TweetMessageType) primary key tweetid autogenerated; -create function f1(message, text){ - contains(message,text) -}; - -create function f4(){ -(select * from TweetMessages) -}; - -use B; - -create dataset TweetMessages2(C.TweetMessageType) -primary key tweetid autogenerated; +create synonym TweetMessagesSyn for TweetMessages; -create function f0(message, text){ - contains(message,text) -}; - -create function C.f2(place, text) { - (select m.message_text - from TweetMessages m - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and f1(m.message_text,text) - and B.f0(m.message_text,text)) -}; - -create function C.f3(place, text) { - f2(place, text) +create function f2(...){ + (select m.message_text + from C.TweetMessagesSyn m) }; -create function f5(place, text){ - (select m.message_text - from TweetMessages2 m, C.TweetMessages m2 - where contains(m.message_text,text) - and spatial_intersect(m.sender_location, place) - and C.f1(m.message_text,text) - and f0(m2.message_text,text)) -}; \ No newline at end of file +drop synonym TweetMessagesSyn; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm index 9b32c3d..2279d69 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm @@ -4,3 +4,4 @@ { "DataverseName": "C", "Name": "f2", "Dependencies": [ [ [ "C", "TweetMessages" ] ], [ [ "C", "f1", "2" ], [ "B", "f0", "2" ] ], [ ] ] } { "DataverseName": "C", "Name": "f3", "Dependencies": [ [ ], [ [ "C", "f2", "2" ] ], [ ] ] } { "DataverseName": "C", "Name": "f4", "Dependencies": [ [ [ "C", "TweetMessages" ] ], [ ], [ ] ] } +{ "DataverseName": "C", "Name": "f5", "Dependencies": [ [ ], [ ], [ ], [ [ "C", "TweetMessagesSyn1" ], [ "C", "TweetMessagesSyn2" ] ] ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 8963f49..ce3fb7d 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -11913,6 +11913,7 @@ <compilation-unit name="drop-dataverse"> <output-dir compare="Text">drop-dataverse</output-dir> <expected-error>ASX1147: Cannot drop dataverse: type a.a being used by dataset b.b1</expected-error> + <expected-error>ASX1147: Cannot drop dataverse: synonym a.s1 being used by function b.f1()</expected-error> <source-location>false</source-location> </compilation-unit> </test-case> @@ -12056,8 +12057,10 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="drop-dependency-4"> <output-dir compare="Text">drop-dependency-4</output-dir> - <expected-error>Cannot drop dataset C.TweetMessages being used by function B.f2(2)</expected-error> - <expected-error>Cannot drop dataset C.TweetMessages being used by function B.f2(...)</expected-error> + <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function B.f2(2)</expected-error> + <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function B.f2(...)</expected-error> + <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function B.f2(2)</expected-error> + <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function B.f2(...)</expected-error> <source-location>false</source-location> </compilation-unit> </test-case> @@ -12074,8 +12077,10 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="drop-dependency-6"> <output-dir compare="Text">drop-dependency-6</output-dir> - <expected-error>Cannot drop dataset C.TweetMessages being used by function C.f2(2)</expected-error> - <expected-error>Cannot drop dataset C.TweetMessages being used by function C.f2(...)</expected-error> + <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function C.f2(2)</expected-error> + <expected-error>ASX1148: Cannot drop dataset C.TweetMessages being used by function C.f2(...)</expected-error> + <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function C.f2(2)</expected-error> + <expected-error>ASX1148: Cannot drop synonym C.TweetMessagesSyn being used by function C.f2(...)</expected-error> <source-location>false</source-location> </compilation-unit> </test-case> diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java index b491e82..7f4e078 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java @@ -276,16 +276,27 @@ public class FunctionUtil { Expression expression) throws CompilationException { Set<AbstractCallExpression> functionCalls = rewriter.getFunctionCalls(expression); //Get the List of used functions and used datasets - List<Triple<DataverseName, String, String>> datasourceDependencies = new ArrayList<>(); + List<Triple<DataverseName, String, String>> datasetDependencies = new ArrayList<>(); List<Triple<DataverseName, String, String>> functionDependencies = new ArrayList<>(); + List<Triple<DataverseName, String, String>> typeDependencies = Collections.emptyList(); + List<Triple<DataverseName, String, String>> synonymDependencies = new ArrayList<>(); for (AbstractCallExpression functionCall : functionCalls) { switch (functionCall.getKind()) { case CALL_EXPRESSION: FunctionSignature signature = functionCall.getFunctionSignature(); if (isBuiltinDatasetFunction(signature)) { - Pair<DataverseName, String> datasetReference = - parseDatasetFunctionArguments((CallExpr) functionCall); - datasourceDependencies.add(new Triple<>(datasetReference.first, datasetReference.second, null)); + CallExpr callExpr = (CallExpr) functionCall; + if (callExpr.getExprList().size() > 2) { + // resolved via synonym -> store synonym name as a dependency + Pair<DataverseName, String> synonymReference = parseDatasetFunctionArguments(callExpr, 2); + synonymDependencies + .add(new Triple<>(synonymReference.first, synonymReference.second, null)); + } else { + // resolved directly -> store dataset name as a dependency + Pair<DataverseName, String> datasetReference = parseDatasetFunctionArguments(callExpr, 0); + datasetDependencies + .add(new Triple<>(datasetReference.first, datasetReference.second, null)); + } } else if (BuiltinFunctions.getBuiltinFunctionInfo(signature.createFunctionIdentifier()) == null) { functionDependencies.add(new Triple<>(signature.getDataverseName(), signature.getName(), Integer.toString(signature.getArity()))); @@ -299,26 +310,21 @@ public class FunctionUtil { functionCall.getFunctionSignature().toString(false)); } } - List<List<Triple<DataverseName, String, String>>> dependencies = new ArrayList<>(3); - dependencies.add(datasourceDependencies); - dependencies.add(functionDependencies); - dependencies.add(Collections.emptyList()); - return dependencies; + return Function.createDependencies(datasetDependencies, functionDependencies, typeDependencies, + synonymDependencies); } public static List<List<Triple<DataverseName, String, String>>> getExternalFunctionDependencies( Collection<TypeSignature> dependentTypes) { - List<Triple<DataverseName, String, String>> datasourceDependencies = Collections.emptyList(); + List<Triple<DataverseName, String, String>> datasetDependencies = Collections.emptyList(); List<Triple<DataverseName, String, String>> functionDependencies = Collections.emptyList(); List<Triple<DataverseName, String, String>> typeDependencies = new ArrayList<>(dependentTypes.size()); + List<Triple<DataverseName, String, String>> synonymDependencies = Collections.emptyList(); for (TypeSignature t : dependentTypes) { typeDependencies.add(new Triple<>(t.getDataverseName(), t.getName(), null)); } - List<List<Triple<DataverseName, String, String>>> dependencies = new ArrayList<>(3); - dependencies.add(datasourceDependencies); - dependencies.add(functionDependencies); - dependencies.add(typeDependencies); - return dependencies; + return Function.createDependencies(datasetDependencies, functionDependencies, typeDependencies, + synonymDependencies); } public static boolean isBuiltinDatasetFunction(FunctionSignature fs) { @@ -328,30 +334,31 @@ public class FunctionUtil { public static Pair<DataverseName, String> parseDatasetFunctionArguments(CallExpr datasetFn) throws CompilationException { - return parseDatasetFunctionArguments(datasetFn.getExprList(), datasetFn.getSourceLocation(), + return parseDatasetFunctionArguments(datasetFn, 0); + } + + public static Pair<DataverseName, String> parseDatasetFunctionArguments(CallExpr datasetFn, int startPos) + throws CompilationException { + return parseDatasetFunctionArguments(datasetFn.getExprList(), startPos, datasetFn.getSourceLocation(), ExpressionUtils::getStringLiteral); } public static Pair<DataverseName, String> parseDatasetFunctionArguments(AbstractFunctionCallExpression datasetFn) throws CompilationException { - return parseDatasetFunctionArguments(datasetFn.getArguments(), datasetFn.getSourceLocation(), + return parseDatasetFunctionArguments(datasetFn.getArguments(), 0, datasetFn.getSourceLocation(), FunctionUtil::getStringConstant); } - private static <T> Pair<DataverseName, String> parseDatasetFunctionArguments(List<T> datasetFnArgs, + private static <T> Pair<DataverseName, String> parseDatasetFunctionArguments(List<T> datasetFnArgs, int startPos, SourceLocation sourceLoc, java.util.function.Function<T, String> argExtractFunction) throws CompilationException { - if (datasetFnArgs.size() != 2) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "Invalid number of arguments to dataset()"); - } - String dataverseNameArg = argExtractFunction.apply(datasetFnArgs.get(0)); + String dataverseNameArg = argExtractFunction.apply(datasetFnArgs.get(startPos)); if (dataverseNameArg == null) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Invalid argument to dataset()"); } DataverseName dataverseName = DataverseName.createFromCanonicalForm(dataverseNameArg); - String datasetName = argExtractFunction.apply(datasetFnArgs.get(1)); + String datasetName = argExtractFunction.apply(datasetFnArgs.get(startPos + 1)); if (datasetName == null) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Invalid argument to dataset()"); } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java index 291bcd0..8e8cfc7 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java @@ -51,6 +51,7 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.common.utils.Triple; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.api.exceptions.SourceLocation; @@ -155,13 +156,19 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopi return null; } SourceLocation sourceLoc = varExpr.getSourceLocation(); - Dataset dataset = findDataset(dataverseName, datasetName, sourceLoc); - if (dataset == null) { + Pair<Dataset, Boolean> datasetSynonymPair = findDataset(dataverseName, datasetName, sourceLoc); + if (datasetSynonymPair == null) { throw createUnresolvableError(dataverseName, datasetName, sourceLoc); } - List<Expression> argList = new ArrayList<>(2); + Dataset dataset = datasetSynonymPair.first; + boolean viaSynonym = datasetSynonymPair.second; + List<Expression> argList = new ArrayList<>(4); argList.add(new LiteralExpr(new StringLiteral(dataset.getDataverseName().getCanonicalForm()))); argList.add(new LiteralExpr(new StringLiteral(dataset.getDatasetName()))); + if (viaSynonym) { + argList.add(new LiteralExpr(new StringLiteral(dataverseName.getCanonicalForm()))); + argList.add(new LiteralExpr(new StringLiteral(datasetName))); + } CallExpr callExpr = new CallExpr(new FunctionSignature(BuiltinFunctions.DATASET), argList); callExpr.setSourceLocation(sourceLoc); return callExpr; @@ -221,16 +228,19 @@ public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopi dataverseName == null ? defaultDataverseName : dataverseName); } - private Dataset findDataset(DataverseName dataverseName, String datasetName, SourceLocation sourceLoc) - throws CompilationException { + private Pair<Dataset, Boolean> findDataset(DataverseName dataverseName, String datasetName, + SourceLocation sourceLoc) throws CompilationException { try { - Pair<DataverseName, String> dsName = + Boolean viaSynonym = false; + Triple<DataverseName, String, Boolean> dsName = metadataProvider.resolveDatasetNameUsingSynonyms(dataverseName, datasetName); if (dsName != null) { dataverseName = dsName.first; datasetName = dsName.second; + viaSynonym = dsName.third; } - return metadataProvider.findDataset(dataverseName, datasetName); + Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); + return dataset == null ? null : new Pair<>(dataset, viaSynonym); } catch (AlgebricksException e) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, e, sourceLoc, e.getMessage()); } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppSynonymRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppSynonymRewriteVisitor.java index ae999fd..a0ffdf2 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppSynonymRewriteVisitor.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/SqlppSynonymRewriteVisitor.java @@ -28,7 +28,7 @@ import org.apache.asterix.lang.common.statement.LoadStatement; import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppAstVisitor; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.common.utils.Triple; import org.apache.hyracks.api.exceptions.SourceLocation; /** @@ -43,7 +43,7 @@ public class SqlppSynonymRewriteVisitor extends AbstractSqlppAstVisitor<Void, Me @Override public Void visit(LoadStatement loadStmt, MetadataProvider metadataProvider) throws CompilationException { - Pair<DataverseName, String> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, + Triple<DataverseName, String, Boolean> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, loadStmt.getDataverseName(), loadStmt.getDatasetName(), loadStmt.getSourceLocation()); if (dsName != null) { loadStmt.setDataverseName(dsName.first); @@ -54,7 +54,7 @@ public class SqlppSynonymRewriteVisitor extends AbstractSqlppAstVisitor<Void, Me @Override public Void visit(InsertStatement insertStmt, MetadataProvider metadataProvider) throws CompilationException { - Pair<DataverseName, String> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, + Triple<DataverseName, String, Boolean> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, insertStmt.getDataverseName(), insertStmt.getDatasetName(), insertStmt.getSourceLocation()); if (dsName != null) { insertStmt.setDataverseName(dsName.first); @@ -65,7 +65,7 @@ public class SqlppSynonymRewriteVisitor extends AbstractSqlppAstVisitor<Void, Me @Override public Void visit(DeleteStatement deleteStmt, MetadataProvider metadataProvider) throws CompilationException { - Pair<DataverseName, String> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, + Triple<DataverseName, String, Boolean> dsName = resolveDatasetNameUsingSynonyms(metadataProvider, deleteStmt.getDataverseName(), deleteStmt.getDatasetName(), deleteStmt.getSourceLocation()); if (dsName != null) { deleteStmt.setDataverseName(dsName.first); @@ -74,7 +74,7 @@ public class SqlppSynonymRewriteVisitor extends AbstractSqlppAstVisitor<Void, Me return null; } - private Pair<DataverseName, String> resolveDatasetNameUsingSynonyms(MetadataProvider metadataProvider, + private Triple<DataverseName, String, Boolean> resolveDatasetNameUsingSynonyms(MetadataProvider metadataProvider, DataverseName dataverseName, String datasetName, SourceLocation sourceLoc) throws CompilationException { try { return metadataProvider.resolveDatasetNameUsingSynonyms(dataverseName, datasetName); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 088bbc6..85ab549 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -519,12 +519,6 @@ public class MetadataNode implements IMetadataNode { try { confirmDataverseCanBeDeleted(txnId, dataverseName); - // Drop all synonyms in this dataverse. - List<Synonym> dataverseSynonyms = getDataverseSynonyms(txnId, dataverseName); - for (Synonym synonym : dataverseSynonyms) { - dropSynonym(txnId, dataverseName, synonym.getSynonymName()); - } - // Drop all feeds and connections in this dataverse. // Feeds may depend on datatypes and adapters List<Feed> dataverseFeeds = getDataverseFeeds(txnId, dataverseName); @@ -543,7 +537,7 @@ public class MetadataNode implements IMetadataNode { } // Drop all functions in this dataverse. - // Functions may depend on datatypes and libraries + // Functions may depend on libraries, datasets, functions, datatypes, synonyms // As a side effect, acquires an S lock on the 'Function' dataset on behalf of txnId. List<Function> dataverseFunctions = getDataverseFunctions(txnId, dataverseName); for (Function function : dataverseFunctions) { @@ -564,6 +558,12 @@ public class MetadataNode implements IMetadataNode { dropLibrary(txnId, lib.getDataverseName(), lib.getName()); } + // Drop all synonyms in this dataverse. + List<Synonym> dataverseSynonyms = getDataverseSynonyms(txnId, dataverseName); + for (Synonym synonym : dataverseSynonyms) { + dropSynonym(txnId, dataverseName, synonym.getSynonymName(), true); + } + // Drop all datasets and indexes in this dataverse. // Datasets depend on datatypes List<Dataset> dataverseDatasets = getDataverseDatasets(txnId, dataverseName); @@ -949,37 +949,25 @@ public class MetadataNode implements IMetadataNode { } // If a function from a DIFFERENT dataverse - // uses datasets, functions or datatypes from this dataverse + // uses datasets, functions, datatypes, or synonyms from this dataverse // throw an error + Function.FunctionDependencyKind[] functionDependencyKinds = Function.FunctionDependencyKind.values(); List<Function> functions = getAllFunctions(txnId); for (Function function : functions) { if (function.getDataverseName().equals(dataverseName)) { continue; } - for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(0)) { - if (datasetDependency.first.equals(dataverseName)) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, - "dataset", - DatasetUtil.getFullyQualifiedDisplayName(datasetDependency.first, datasetDependency.second), - "function", function.getSignature()); - } - } - for (Triple<DataverseName, String, String> functionDependency : function.getDependencies().get(1)) { - if (functionDependency.first.equals(dataverseName)) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, - "function", new FunctionSignature(functionDependency.first, functionDependency.second, - Integer.parseInt(functionDependency.third)), - "function", function.getSignature()); - } - } - for (Triple<DataverseName, String, String> type : function.getDependencies().get(2)) { - if (type.first.equals(dataverseName)) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, - "type", TypeUtil.getFullyQualifiedDisplayName(type.first, type.second), "function", - function.getSignature()); + List<List<Triple<DataverseName, String, String>>> dependencies = function.getDependencies(); + for (int i = 0, n = dependencies.size(); i < n; i++) { + for (Triple<DataverseName, String, String> dependency : dependencies.get(i)) { + if (dependency.first.equals(dataverseName)) { + Function.FunctionDependencyKind functionDependencyKind = functionDependencyKinds[i]; + throw new AsterixException( + org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_DATAVERSE_DEPENDENT_EXISTS, + functionDependencyKind.toString().toLowerCase(), + functionDependencyKind.getDependencyDisplayName(dependency), "function", + function.getSignature()); + } } } } @@ -1003,19 +991,7 @@ public class MetadataNode implements IMetadataNode { } private void confirmFunctionCanBeDeleted(TxnId txnId, FunctionSignature signature) throws AlgebricksException { - // If any other function uses this function, throw an error - List<Function> functions = getAllFunctions(txnId); - for (Function function : functions) { - for (Triple<DataverseName, String, String> functionalDependency : function.getDependencies().get(1)) { - if (functionalDependency.first.equals(signature.getDataverseName()) - && functionalDependency.second.equals(signature.getName()) - && functionalDependency.third.equals(Integer.toString(signature.getArity()))) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, - "function", signature, "function", function.getSignature()); - } - } - } + confirmFunctionIsUnusedByFunctions(txnId, signature); // if any other feed connection uses this function, throw an error List<FeedConnection> feedConnections = getAllFeedConnections(txnId); @@ -1029,22 +1005,49 @@ public class MetadataNode implements IMetadataNode { } } - private void confirmDatasetCanBeDeleted(TxnId txnId, DataverseName dataverseName, String datasetName) + private void confirmFunctionIsUnusedByFunctions(TxnId txnId, FunctionSignature signature) throws AlgebricksException { - // If any function uses this type, throw an error + confirmObjectIsUnusedByFunctions(txnId, Function.FunctionDependencyKind.FUNCTION, signature.getDataverseName(), + signature.getName(), Integer.toString(signature.getArity())); + } + + private void confirmObjectIsUnusedByFunctions(TxnId txnId, Function.FunctionDependencyKind dependencyKind, + DataverseName dataverseName, String objectName, String objectArg) throws AlgebricksException { + // If any function uses this object, throw an error + int functionDependencyIdx = dependencyKind.ordinal(); List<Function> functions = getAllFunctions(txnId); for (Function function : functions) { - for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(0)) { - if (datasetDependency.first.equals(dataverseName) && datasetDependency.second.equals(datasetName)) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, - "dataset", DatasetUtil.getFullyQualifiedDisplayName(dataverseName, datasetName), "function", - function.getSignature()); + List<List<Triple<DataverseName, String, String>>> functionDependencies = function.getDependencies(); + if (functionDependencyIdx < functionDependencies.size()) { + List<Triple<DataverseName, String, String>> functionObjectDependencies = + functionDependencies.get(functionDependencyIdx); + if (functionObjectDependencies != null) { + for (Triple<DataverseName, String, String> dependency : functionObjectDependencies) { + if (dependency.first.equals(dataverseName) && dependency.second.equals(objectName) + && (objectArg == null || objectArg.equals(dependency.third))) { + throw new AsterixException( + org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, + dependencyKind.toString().toLowerCase(), + dependencyKind.getDependencyDisplayName(dependency), "function", + function.getSignature()); + } + } } } } } + private void confirmDatasetCanBeDeleted(TxnId txnId, DataverseName dataverseName, String datasetName) + throws AlgebricksException { + confirmDatasetIsUnusedByFunctions(txnId, dataverseName, datasetName); + } + + private void confirmDatasetIsUnusedByFunctions(TxnId txnId, DataverseName dataverseName, String datasetName) + throws AlgebricksException { + confirmObjectIsUnusedByFunctions(txnId, Function.FunctionDependencyKind.DATASET, dataverseName, datasetName, + null); + } + private void confirmLibraryCanBeDeleted(TxnId txnId, DataverseName dataverseName, String libraryName) throws AlgebricksException { confirmLibraryIsUnusedByFunctions(txnId, dataverseName, libraryName); @@ -1128,18 +1131,8 @@ public class MetadataNode implements IMetadataNode { private void confirmDatatypeIsUnusedByFunctions(TxnId txnId, DataverseName dataverseName, String dataTypeName) throws AlgebricksException { - // If any function uses this type, throw an error - List<Function> functions = getAllFunctions(txnId); - for (Function function : functions) { - for (Triple<DataverseName, String, String> datasetDependency : function.getDependencies().get(2)) { - if (datasetDependency.first.equals(dataverseName) && datasetDependency.second.equals(dataTypeName)) { - throw new AsterixException( - org.apache.asterix.common.exceptions.ErrorCode.CANNOT_DROP_OBJECT_DEPENDENT_EXISTS, "type", - TypeUtil.getFullyQualifiedDisplayName(dataverseName, dataTypeName), "function", - function.getSignature()); - } - } - } + confirmObjectIsUnusedByFunctions(txnId, Function.FunctionDependencyKind.TYPE, dataverseName, dataTypeName, + null); } private List<String> getNestedComplexDatatypeNamesForThisDatatype(TxnId txnId, DataverseName dataverseName, @@ -2034,6 +2027,15 @@ public class MetadataNode implements IMetadataNode { @Override public void dropSynonym(TxnId txnId, DataverseName dataverseName, String synonymName) throws AlgebricksException { + dropSynonym(txnId, dataverseName, synonymName, false); + } + + private void dropSynonym(TxnId txnId, DataverseName dataverseName, String synonymName, boolean force) + throws AlgebricksException { + if (!force) { + confirmSynonymCanBeDeleted(txnId, dataverseName, synonymName); + } + try { // Delete entry from the 'Synonym' dataset. ITupleReference searchKey = createTuple(dataverseName, synonymName); @@ -2052,6 +2054,17 @@ public class MetadataNode implements IMetadataNode { } } + private void confirmSynonymCanBeDeleted(TxnId txnId, DataverseName dataverseName, String synonymName) + throws AlgebricksException { + confirmSynonymIsUnusedByFunctions(txnId, dataverseName, synonymName); + } + + private void confirmSynonymIsUnusedByFunctions(TxnId txnId, DataverseName dataverseName, String synonymName) + throws AlgebricksException { + confirmObjectIsUnusedByFunctions(txnId, Function.FunctionDependencyKind.SYNONYM, dataverseName, synonymName, + null); + } + @Override public Synonym getSynonym(TxnId txnId, DataverseName dataverseName, String synonymName) throws AlgebricksException { try { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 5f7fdea..68def8d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -423,21 +423,22 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> return MetadataManagerUtil.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); } - public Pair<DataverseName, String> resolveDatasetNameUsingSynonyms(DataverseName dataverseName, String datasetName) - throws AlgebricksException { + public Triple<DataverseName, String, Boolean> resolveDatasetNameUsingSynonyms(DataverseName dataverseName, + String datasetName) throws AlgebricksException { DataverseName dvName = getActiveDataverseName(dataverseName); if (dvName == null) { return null; } + Synonym synonym = null; while (MetadataManagerUtil.findDataset(mdTxnCtx, dvName, datasetName) == null) { - Synonym synonym = findSynonym(dvName, datasetName); + synonym = findSynonym(dvName, datasetName); if (synonym == null) { return null; } dvName = synonym.getObjectDataverseName(); datasetName = synonym.getObjectName(); } - return new Pair<>(dvName, datasetName); + return new Triple<>(dvName, datasetName, synonym != null); } public Synonym findSynonym(DataverseName dataverseName, String synonymName) throws AlgebricksException { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java index 968cf14..3cd4346 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.metadata.entities; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -27,6 +28,9 @@ import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.metadata.MetadataCache; import org.apache.asterix.metadata.api.IMetadataEntity; +import org.apache.asterix.metadata.utils.DatasetUtil; +import org.apache.asterix.metadata.utils.MetadataUtil; +import org.apache.asterix.metadata.utils.TypeUtil; import org.apache.asterix.om.types.TypeSignature; import org.apache.hyracks.algebricks.common.utils.Triple; @@ -159,4 +163,39 @@ public class Function implements IMetadataEntity<Function> { public Function dropFromCache(MetadataCache cache) { return cache.dropFunction(this); } + + public static List<List<Triple<DataverseName, String, String>>> createDependencies( + List<Triple<DataverseName, String, String>> datasetDependencies, + List<Triple<DataverseName, String, String>> functionDependencies, + List<Triple<DataverseName, String, String>> typeDependencies, + List<Triple<DataverseName, String, String>> synonymDependencies) { + List<List<Triple<DataverseName, String, String>>> depList = new ArrayList<>(4); + depList.add(datasetDependencies); + depList.add(functionDependencies); + depList.add(typeDependencies); + if (!synonymDependencies.isEmpty()) { + depList.add(synonymDependencies); + } + return depList; + } + + public enum FunctionDependencyKind { + DATASET(dependency -> DatasetUtil.getFullyQualifiedDisplayName(dependency.first, dependency.second)), + FUNCTION( + dependency -> new FunctionSignature(dependency.first, dependency.second, + Integer.parseInt(dependency.third)).toString()), + TYPE(dependency -> TypeUtil.getFullyQualifiedDisplayName(dependency.first, dependency.second)), + SYNONYM(dependency -> MetadataUtil.getFullyQualifiedDisplayName(dependency.first, dependency.second)); + + private final java.util.function.Function<Triple<DataverseName, String, String>, String> dependencyDisplayNameAccessor; + + FunctionDependencyKind( + java.util.function.Function<Triple<DataverseName, String, String>, String> dependencyDisplayNameAccessor) { + this.dependencyDisplayNameAccessor = dependencyDisplayNameAccessor; + } + + public String getDependencyDisplayName(Triple<DataverseName, String, String> dependency) { + return dependencyDisplayNameAccessor.apply(dependency); + } + } }
