Steven Jacobs has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2253
Change subject: [ASTERIXDB-2180][FUN] Provent dropping of entities used by
functions
......................................................................
[ASTERIXDB-2180][FUN] Provent dropping of entities used by functions
Add dependencies to Functional Metadata
Check dependencies before dropping datasets or functions
Add tests
Change-Id: I2f08ff150dfd57432b88381c507814ddb57bd67b
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
M
asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
25 files changed, 690 insertions(+), 48 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/53/2253/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 1257a61..4428e05 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -180,7 +180,7 @@
}
public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl>
declaredFunctions,
- MetadataProvider metadataProvider, IReturningStatement q,
SessionOutput output)
+ MetadataProvider metadataProvider, IReturningStatement q,
SessionOutput output, boolean inlineUdfs)
throws CompilationException {
if (q == null) {
return null;
@@ -193,7 +193,7 @@
printPlanPostfix(output);
}
IQueryRewriter rw = rewriterFactory.createQueryRewriter();
- rw.rewrite(declaredFunctions, q, metadataProvider, new
LangRewritingContext(q.getVarCounter()));
+ rw.rewrite(declaredFunctions, q, metadataProvider, new
LangRewritingContext(q.getVarCounter()), inlineUdfs);
return new Pair<>(q, q.getVarCounter());
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index b013c60..9f05263 100755
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -248,7 +248,7 @@
}
Function f = new Function(dataverse, libraryName + "#" +
function.getName().trim(), args.size(),
args, function.getReturnType().trim(),
function.getDefinition().trim(),
- library.getLanguage().trim(),
function.getFunctionType().trim());
+ library.getLanguage().trim(),
function.getFunctionType().trim(), new ArrayList<>());
MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Installed function: " + libraryName + "#"
+ function.getName().trim());
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index de0c88f..5b68dc9 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -26,6 +26,7 @@
import java.io.InputStreamReader;
import java.rmi.RemoteException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@@ -83,6 +84,7 @@
import org.apache.asterix.lang.common.base.IRewriterFactory;
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.FieldBinding;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
import org.apache.asterix.lang.common.expression.LiteralExpr;
@@ -124,6 +126,7 @@
import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.util.CommonFunctionMapUtil;
import org.apache.asterix.lang.common.util.MergePolicyUtils;
import org.apache.asterix.lang.sqlpp.rewrites.SqlppRewriterFactory;
import org.apache.asterix.metadata.IDatasetDetails;
@@ -155,6 +158,7 @@
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.MetadataLockUtil;
import org.apache.asterix.metadata.utils.MetadataUtil;
+import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -185,6 +189,7 @@
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.data.IAWriterFactory;
import org.apache.hyracks.algebricks.data.IResultSerializerFactoryProvider;
import
org.apache.hyracks.algebricks.runtime.serializer.ResultSerializerFactoryProvider;
@@ -1701,12 +1706,40 @@
varIds.add(new VarIdentifier(v));
}
wrappedQuery.setExternalVars(varIds);
- apiFramework.reWriteQuery(declaredFunctions, metadataProvider,
wrappedQuery, sessionOutput);
+ apiFramework.reWriteQuery(declaredFunctions, metadataProvider,
wrappedQuery, sessionOutput, false);
+
+ Set<CallExpr> functionCalls =
+
rewriterFactory.createQueryRewriter().getFunctionCalls(cfs.getFunctionBodyExpression());
+
+ //Get the List of used functions and used datasets
+ List<List<List<String>>> dependencies = new ArrayList<>();
+ //dataset dependencies
+ dependencies.add(new ArrayList<>());
+ //functional dependencies
+ dependencies.add(new ArrayList<>());
+ for (CallExpr functionCall : functionCalls) {
+ FunctionSignature signature =
functionCall.getFunctionSignature();
+ FunctionIdentifier fid =
+ new FunctionIdentifier(signature.getNamespace(),
signature.getName(), signature.getArity());
+ if (fid.equals(BuiltinFunctions.DATASET)) {
+ Pair<String, String> path =
DatasetUtil.getDatasetInfo(metadataProvider,
+ ((LiteralExpr)
functionCall.getExprList().get(0)).getValue().getStringValue());
+ dependencies.get(0).add(new
ArrayList<>(Arrays.asList(path.first, path.second)));
+ }
+
+ else if (BuiltinFunctions.isBuiltinCompilerFunction(
+
CommonFunctionMapUtil.normalizeBuiltinFunctionSignature(signature), false)) {
+ continue;
+ } else {
+ dependencies.get(1).add(new
ArrayList<>(Arrays.asList(signature.getNamespace(), signature.getName(),
+ Integer.toString(signature.getArity()))));
+ }
+ }
Function function = new Function(dataverse, functionName,
cfs.getFunctionSignature().getArity(),
cfs.getParamList(), Function.RETURNTYPE_VOID,
cfs.getFunctionBody(),
rewriterFactory instanceof SqlppRewriterFactory ?
Function.LANGUAGE_SQLPP : Function.LANGUAGE_AQL,
- FunctionKind.SCALAR.toString());
+ FunctionKind.SCALAR.toString(), dependencies);
MetadataManager.INSTANCE.addFunction(mdTxnCtx, function);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1898,7 +1931,7 @@
// Query Rewriting (happens under the same ongoing metadata
transaction)
Pair<IReturningStatement, Integer> rewrittenResult =
apiFramework.reWriteQuery(declaredFunctions,
- metadataProvider, query, sessionOutput);
+ metadataProvider, query, sessionOutput, true);
// Query Compilation (happens under the same ongoing metadata
transaction)
return apiFramework.compileQuery(clusterInfoCollector,
metadataProvider, (Query) rewrittenResult.first,
@@ -1912,7 +1945,7 @@
// Insert/upsert statement rewriting (happens under the same ongoing
metadata
// transaction)
Pair<IReturningStatement, Integer> rewrittenResult =
apiFramework.reWriteQuery(declaredFunctions,
- metadataProvider, insertUpsert, sessionOutput);
+ metadataProvider, insertUpsert, sessionOutput, true);
InsertStatement rewrittenInsertUpsert = (InsertStatement)
rewrittenResult.first;
String dataverseName =
getActiveDataverse(rewrittenInsertUpsert.getDataverseName());
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp
new file mode 100644
index 0000000..c734c34
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Verify Function Dependency Metadata
+ * Expected Res : Success
+ * Date : Dec 15th 2017
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create type TweetMessageType as closed {
+ tweetid: uuid,
+ sender_location: point,
+ send_time: datetime,
+ referred_topics: {{ string }},
+ message_text: string,
+ countA: int32,
+ countB: int32
+};
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid autogenerated;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create function f4(){
+(select * from TweetMessages)
+};
+
+create dataverse two;
+use two;
+
+create dataset TweetMessages2(experiments.TweetMessageType)
+primary key tweetid autogenerated;
+
+create function f0(message, text){
+ contains(message,text)
+};
+
+create function experiments.f2(place, text) {
+ (select m.message_text
+ from TweetMessages m
+ where contains(m.message_text,text)
+ and spatial_intersect(m.sender_location, place)
+ and f1(m.message_text,text)
+ and two.f0(m.message_text,text))
+};
+
+create function experiments.f3(place, text) {
+ f2(place, text)
+};
+
+create function f5(place, text){
+ (select m.message_text
+ from TweetMessages2 m, experiments.TweetMessages m2
+ where contains(m.message_text,text)
+ and spatial_intersect(m.sender_location, place)
+ and experiments.f1(m.message_text,text)
+ and f0(m2.message_text,text))
+};
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp
new file mode 100644
index 0000000..8082b80
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Verify Function Dependency Metadata
+ * Expected Res : Success
+ * Date : Dec 15th 2017
+ */
+
+use experiments;
+
+
+select f.DataverseName,f.Name,f.Dependencies from Metadata.`Function` f
+order by f.DataverseName, f.Name;
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp
new file mode 100644
index 0000000..10f94ac
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create dataverse two;
+use two;
+
+create function f0(message, text){
+ experiments.f1(message,text)
+};
+
+drop dataverse experiments;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp
new file mode 100644
index 0000000..bbd1276
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create type TweetMessageType as closed {
+ tweetid: uuid,
+ sender_location: point,
+ send_time: datetime,
+ referred_topics: {{ string }},
+ message_text: string,
+ countA: int32,
+ countB: int32
+};
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid autogenerated;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create dataverse two;
+use two;
+
+create function f2(place, text){
+ (select m.message_text
+ from experiments.TweetMessages m)
+};
+
+drop dataverse experiments;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp
new file mode 100644
index 0000000..f42fd2a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create dataverse two;
+use two;
+
+create function f0(message, text){
+ experiments.f1(message,text)
+};
+
+use experiments;
+drop function f1@2;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp
new file mode 100644
index 0000000..8ac9cff
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create type TweetMessageType as closed {
+ tweetid: uuid,
+ sender_location: point,
+ send_time: datetime,
+ referred_topics: {{ string }},
+ message_text: string,
+ countA: int32,
+ countB: int32
+};
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid autogenerated;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create dataverse two;
+use two;
+
+create function f2(place, text){
+ (select m.message_text
+ from experiments.TweetMessages m)
+};
+
+use experiments;
+drop dataset TweetMessages;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp
new file mode 100644
index 0000000..235514c
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create function f0(message, text){
+ experiments.f1(message,text)
+};
+
+drop function f1@2;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp
new file mode 100644
index 0000000..d3ca1da
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Try to drop a functional dependency
+ * Expected Res : Error
+ */
+
+drop dataverse two if exists;
+drop dataverse experiments if exists;
+create dataverse experiments;
+use experiments;
+
+create type TweetMessageType as closed {
+ tweetid: uuid,
+ sender_location: point,
+ send_time: datetime,
+ referred_topics: {{ string }},
+ message_text: string,
+ countA: int32,
+ countB: int32
+};
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid autogenerated;
+
+create function f1(message, text){
+ contains(message,text)
+};
+
+create function f2(place, text){
+ (select m.message_text
+ from experiments.TweetMessages m)
+};
+
+drop dataset TweetMessages;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm
new file mode 100644
index 0000000..a5dcb7e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm
@@ -0,0 +1,6 @@
+{ "DataverseName": "experiments", "Name": "f1", "Dependencies": [ [ ], [ ] ]
}
+{ "DataverseName": "experiments", "Name": "f2", "Dependencies": [ [ [
"experiments", "TweetMessages" ] ], [ [ "experiments", "f1" ], [ "two", "f0" ]
] ] }
+{ "DataverseName": "experiments", "Name": "f3", "Dependencies": [ [ ], [ [
"experiments", "f2" ] ] ] }
+{ "DataverseName": "experiments", "Name": "f4", "Dependencies": [ [ [
"experiments", "TweetMessages" ] ], [ ] ] }
+{ "DataverseName": "two", "Name": "f0", "Dependencies": [ [ ], [ ] ] }
+{ "DataverseName": "two", "Name": "f5", "Dependencies": [ [ [ "experiments",
"TweetMessages" ], [ "two", "TweetMessages2" ] ], [ [ "two", "f0" ], [
"experiments", "f1" ] ] ] }
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 2c3d855..84f8473 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -7927,6 +7927,47 @@
</compilation-unit>
</test-case>
<test-case FilePath="user-defined-functions">
+ <compilation-unit name="check-dependencies-1">
+ <output-dir compare="Text">check-dependencies-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-1">
+ <output-dir compare="Text">drop-dependency-1</output-dir>
+ <expected-error>Cannot drop dataverse. Function two.f0@2 depends on
function experiments.f1@2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-2">
+ <output-dir compare="Text">drop-dependency-2</output-dir>
+ <expected-error>Cannot drop dataverse. Function two.f2@2 depends on
dataset experiments.TweetMessages</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-3">
+ <output-dir compare="Text">drop-dependency-3</output-dir>
+ <expected-error>Cannot drop function experiments.f1@2 being used by
function two.f0@2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-4">
+ <output-dir compare="Text">drop-dependency-4</output-dir>
+ <expected-error>Cannot drop dataset experiments.TweetMessages being
used by function two.f2@2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-5">
+ <output-dir compare="Text">drop-dependency-5</output-dir>
+ <expected-error>Cannot drop function experiments.f1@2 being used by
function experiments.f0@2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
+ <compilation-unit name="drop-dependency-6">
+ <output-dir compare="Text">drop-dependency-6</output-dir>
+ <expected-error>Cannot drop dataset experiments.TweetMessages being
used by function experiments.f2@2</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="user-defined-functions">
<compilation-unit name="single-line-definition">
<output-dir compare="Text">single-line-definition</output-dir>
</compilation-unit>
diff --git
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index 977f6bb..6ab69ad 100644
---
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -40,6 +40,7 @@
import org.apache.asterix.lang.common.base.IReturningStatement;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
@@ -68,7 +69,8 @@
@Override
public void rewrite(List<FunctionDecl> declaredFunctions,
IReturningStatement topStatement,
- MetadataProvider metadataProvider, LangRewritingContext context)
throws CompilationException {
+ MetadataProvider metadataProvider, LangRewritingContext context,
boolean inlineUdfs)
+ throws CompilationException {
setup(declaredFunctions, topStatement, metadataProvider, context);
if (topStatement.isTopLevel()) {
wrapInLets();
@@ -130,7 +132,8 @@
declaredFunctions.removeAll(storedFunctionDecls);
}
- private Set<FunctionSignature> getFunctionCalls(Expression expression)
throws CompilationException {
+ @Override
+ public Set<CallExpr> getFunctionCalls(Expression expression) throws
CompilationException {
GatherFunctionCalls gfc = new GatherFunctionCalls();
expression.accept(gfc, null);
return gfc.getCalls();
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
index 0c6c04c..f072917 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java
@@ -19,8 +19,10 @@
package org.apache.asterix.lang.common.base;
import java.util.List;
+import java.util.Set;
import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -39,6 +41,13 @@
* @param context,
* manages ids of variables and guarantees uniqueness of
variables.
*/
- public void rewrite(List<FunctionDecl> declaredFunctions,
IReturningStatement topExpr,
- MetadataProvider metadataProvider, LangRewritingContext context)
throws CompilationException;
+ void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement
topExpr,
+ MetadataProvider metadataProvider, LangRewritingContext context,
boolean inlineUdfs)
+ throws CompilationException;
+
+ /**
+ * Find the function calls used by a given expression
+ */
+ Set<CallExpr> getFunctionCalls(Expression expression) throws
CompilationException;
+
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index 1ca9316..cad84ee 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -27,6 +27,7 @@
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -52,7 +53,7 @@
@FunctionalInterface
public interface IFunctionCollector {
- Set<FunctionSignature> getFunctionCalls(Expression expression) throws
CompilationException;
+ Set<CallExpr> getFunctionCalls(Expression expression) throws
CompilationException;
}
@FunctionalInterface
@@ -95,8 +96,9 @@
}
String value =
metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS);
boolean includePrivateFunctions = (value != null) ?
Boolean.valueOf(value.toLowerCase()) : false;
- Set<FunctionSignature> functionCalls =
functionCollector.getFunctionCalls(expression);
- for (FunctionSignature signature : functionCalls) {
+ Set<CallExpr> functionCalls =
functionCollector.getFunctionCalls(expression);
+ for (CallExpr functionCall : functionCalls) {
+ FunctionSignature signature = functionCall.getFunctionSignature();
if (declaredFunctions != null &&
declaredFunctions.contains(signature)) {
continue;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
index 7d8c11e..8ae67d6 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
@@ -347,7 +347,7 @@
metadataProvider.setDefaultDataverse(fnDataverse);
try {
IQueryRewriter queryRewriter =
rewriterFactory.createQueryRewriter();
- queryRewriter.rewrite(declaredFunctions, wrappedQuery,
metadataProvider, context);
+ queryRewriter.rewrite(declaredFunctions, wrappedQuery,
metadataProvider, context, true);
return wrappedQuery.getBody();
} finally {
metadataProvider.setDefaultDataverse(defaultDataverse);
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
index 8842d86..3d149fb 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
@@ -23,7 +23,6 @@
import java.util.Set;
import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
@@ -53,11 +52,11 @@
public class GatherFunctionCallsVisitor extends
AbstractQueryExpressionVisitor<Void, Void> {
- protected final Set<FunctionSignature> calls = new
HashSet<FunctionSignature>();
+ protected final Set<CallExpr> calls = new HashSet<>();
@Override
public Void visit(CallExpr pf, Void arg) throws CompilationException {
- calls.add(pf.getFunctionSignature());
+ calls.add(pf);
for (Expression e : pf.getExprList()) {
e.accept(this, arg);
}
@@ -202,7 +201,7 @@
return null;
}
- public Set<FunctionSignature> getCalls() {
+ public Set<CallExpr> getCalls() {
return calls;
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
index a8b3dc6..cd57396 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java
@@ -31,7 +31,7 @@
@Override
public void rewrite(List<FunctionDecl> declaredFunctions,
IReturningStatement topStatement,
MetadataProvider metadataProvider,
- LangRewritingContext context) throws CompilationException {
+ LangRewritingContext context, boolean inlineUdfs) throws
CompilationException {
// Sets up parameters.
setup(declaredFunctions, topStatement, metadataProvider, context);
@@ -63,6 +63,6 @@
rewriteListInputFunctions();
// Inlines functions recursively.
- inlineDeclaredUdfs();
+ inlineDeclaredUdfs(inlineUdfs);
}
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index 3e9a873..befa5ab 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -28,6 +28,7 @@
import org.apache.asterix.lang.common.base.IQueryRewriter;
import org.apache.asterix.lang.common.base.IReturningStatement;
import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.util.FunctionUtil;
@@ -86,7 +87,8 @@
@Override
public void rewrite(List<FunctionDecl> declaredFunctions,
IReturningStatement topStatement,
- MetadataProvider metadataProvider, LangRewritingContext context)
throws CompilationException {
+ MetadataProvider metadataProvider, LangRewritingContext context,
boolean inlineUdfs)
+ throws CompilationException {
if (topStatement == null) {
return;
}
@@ -126,7 +128,7 @@
rewriteListInputFunctions();
// Inlines functions.
- inlineDeclaredUdfs();
+ inlineDeclaredUdfs(inlineUdfs);
// Rewrites function names.
// This should be done after inlineDeclaredUdfs() because user-defined
function
@@ -213,7 +215,7 @@
topExpr.accept(groupByVisitor, null);
}
- protected void inlineDeclaredUdfs() throws CompilationException {
+ protected void inlineDeclaredUdfs(boolean inlineUdfs) throws
CompilationException {
List<FunctionSignature> funIds = new ArrayList<FunctionSignature>();
for (FunctionDecl fdecl : declaredFunctions) {
funIds.add(fdecl.getSignature());
@@ -226,7 +228,7 @@
signature ->
FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false)));
}
declaredFunctions.addAll(usedStoredFunctionDecls);
- if (!declaredFunctions.isEmpty()) {
+ if (inlineUdfs && !declaredFunctions.isEmpty()) {
SqlppInlineUdfsVisitor visitor = new
SqlppInlineUdfsVisitor(context,
new SqlppFunctionBodyRewriterFactory() /* the rewriter for
function bodies expressions*/,
declaredFunctions, metadataProvider);
@@ -237,7 +239,8 @@
declaredFunctions.removeAll(usedStoredFunctionDecls);
}
- private Set<FunctionSignature> getFunctionCalls(Expression expression)
throws CompilationException {
+ @Override
+ public Set<CallExpr> getFunctionCalls(Expression expression) throws
CompilationException {
GatherFunctionCalls gfc = new GatherFunctionCalls();
expression.accept(gfc, null);
return gfc.getCalls();
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 368fc2a..cee5edb 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -27,7 +27,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.logging.Logger;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
import org.apache.asterix.common.api.INcApplicationContext;
@@ -271,7 +270,6 @@
}
}
- @SuppressWarnings("unchecked")
@Override
public <T extends IExtensionMetadataEntity> void addEntity(TxnId txnId, T
entity)
throws AlgebricksException, RemoteException {
@@ -283,7 +281,6 @@
addEntity(txnId, entity, tupleTranslator, index);
}
- @SuppressWarnings("unchecked")
@Override
public <T extends IExtensionMetadataEntity> void upsertEntity(TxnId txnId,
T entity)
throws AlgebricksException, RemoteException {
@@ -295,7 +292,6 @@
upsertEntity(txnId, entity, tupleTranslator, index);
}
- @SuppressWarnings("unchecked")
@Override
public <T extends IExtensionMetadataEntity> void deleteEntity(TxnId txnId,
T entity)
throws AlgebricksException, RemoteException {
@@ -307,7 +303,6 @@
deleteEntity(txnId, entity, tupleTranslator, index);
}
- @SuppressWarnings("unchecked")
@Override
public <T extends IExtensionMetadataEntity> List<T> getEntities(TxnId
txnId, IExtensionMetadataSearchKey searchKey)
throws AlgebricksException, RemoteException {
@@ -530,13 +525,22 @@
confirmDataverseCanBeDeleted(txnId, dataverseName);
+ // As a side effect, acquires an S lock on the 'Function' dataset
+ // on behalf of txnId.
+ List<Function> dataverseFunctions = getDataverseFunctions(txnId,
dataverseName);
+ // Drop all functions in this dataverse.
+ for (Function function : dataverseFunctions) {
+ dropFunction(txnId, new FunctionSignature(dataverseName,
function.getName(), function.getArity()),
+ true);
+ }
+
List<Dataset> dataverseDatasets;
Dataset ds;
dataverseDatasets = getDataverseDatasets(txnId, dataverseName);
// Drop all datasets in this dataverse.
for (int i = 0; i < dataverseDatasets.size(); i++) {
ds = dataverseDatasets.get(i);
- dropDataset(txnId, dataverseName, ds.getDatasetName());
+ dropDataset(txnId, dataverseName, ds.getDatasetName(), true);
}
// After dropping datasets, drop datatypes
@@ -547,14 +551,6 @@
// Drop all types in this dataverse.
for (int i = 0; i < dataverseDatatypes.size(); i++) {
forceDropDatatype(txnId, dataverseName,
dataverseDatatypes.get(i).getDatatypeName());
- }
-
- // As a side effect, acquires an S lock on the 'Function' dataset
- // on behalf of txnId.
- List<Function> dataverseFunctions = getDataverseFunctions(txnId,
dataverseName);
- // Drop all functions in this dataverse.
- for (Function function : dataverseFunctions) {
- dropFunction(txnId, new FunctionSignature(dataverseName,
function.getName(), function.getArity()));
}
// As a side effect, acquires an S lock on the 'Adapter' dataset
@@ -612,6 +608,16 @@
@Override
public void dropDataset(TxnId txnId, String dataverseName, String
datasetName)
throws AlgebricksException, RemoteException {
+ dropDataset(txnId, dataverseName, datasetName, false);
+ }
+
+ public void dropDataset(TxnId txnId, String dataverseName, String
datasetName, boolean force)
+ throws AlgebricksException, RemoteException {
+
+ if (!force) {
+ confirmDatasetCanBeDeleted(txnId, dataverseName, datasetName);
+ }
+
Dataset dataset = getDataset(txnId, dataverseName, datasetName);
if (dataset == null) {
throw new AlgebricksException("Cannot drop dataset '" +
datasetName + "' because it doesn't exist.");
@@ -905,6 +911,19 @@
}
}
+ public List<Function> getAllFunctions(TxnId txnId) throws
AlgebricksException, RemoteException {
+ try {
+ ITupleReference searchKey = null;
+ FunctionTupleTranslator tupleReaderWriter =
tupleTranslatorProvider.getFunctionTupleTranslator(false);
+ IValueExtractor<Function> valueExtractor = new
MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Function> results = new ArrayList<>();
+ searchIndex(txnId, MetadataPrimaryIndexes.FUNCTION_DATASET,
searchKey, valueExtractor, results);
+ return results;
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
public List<Datatype> getAllDatatypes(TxnId txnId) throws
AlgebricksException, RemoteException {
try {
ITupleReference searchKey = null;
@@ -933,6 +952,62 @@
throw new AlgebricksException(
"Cannot drop dataverse. Type " + dataverseName + "." +
set.getItemTypeName()
+ " used by dataset " + set.getDataverseName()
+ "." + set.getDatasetName());
+ }
+ }
+
+ // If a function from a DIFFERENT dataverse
+ // uses functions or datatypes from this dataverse
+ // throw an error
+ List<Function> functions = getAllFunctions(txnId);
+ for (Function function : functions) {
+ if (function.getDataverseName().equals(dataverseName)) {
+ continue;
+ }
+ for (List<String> datasetDependency :
function.getDependencies().get(0)) {
+ if (datasetDependency.get(0).equals(dataverseName)) {
+ throw new AlgebricksException("Cannot drop dataverse.
Function " + function.getDataverseName() + "."
+ + function.getName() + "@" + function.getArity() +
" depends on dataset "
+ + datasetDependency.get(0) + "." +
datasetDependency.get(1));
+ }
+ }
+ for (List<String> functionDependency :
function.getDependencies().get(1)) {
+ if (functionDependency.get(0).equals(dataverseName)) {
+ throw new AlgebricksException(
+ "Cannot drop dataverse. Function " +
function.getDataverseName() + "." + function.getName()
+ + "@" + function.getArity() + " depends on
function " + functionDependency.get(0)
+ + "." + functionDependency.get(1) + "@" +
functionDependency.get(2));
+ }
+ }
+ }
+ }
+
+ private void confirmFunctionCanBeDeleted(TxnId txnId, FunctionSignature
signature)
+ throws AlgebricksException, RemoteException {
+ // If any other function uses this function, throw an error
+ List<Function> functions = getAllFunctions(txnId);
+ for (Function function : functions) {
+ for (List<String> functionalDependency :
function.getDependencies().get(1)) {
+ if
(functionalDependency.get(0).equals(signature.getNamespace())
+ &&
functionalDependency.get(1).equals(signature.getName())
+ &&
functionalDependency.get(2).equals(Integer.toString(signature.getArity()))) {
+ throw new AlgebricksException("Cannot drop function " +
signature + " being used by function "
+ + function.getDataverseName() + "." +
function.getName() + "@" + function.getArity());
+ }
+ }
+ }
+ }
+
+ private void confirmDatasetCanBeDeleted(TxnId txnId, String dataverseName,
String datasetName)
+ throws AlgebricksException, RemoteException {
+ // If any function uses this type, throw an error
+ List<Function> functions = getAllFunctions(txnId);
+ for (Function function : functions) {
+ for (List<String> datasetDependency :
function.getDependencies().get(0)) {
+ if (datasetDependency.get(0).equals(dataverseName) &&
datasetDependency.get(1).equals(datasetName)) {
+ throw new AlgebricksException("Cannot drop dataset " +
dataverseName + "." + datasetName
+ + " being used by function " +
function.getDataverseName() + "." + function.getName() + "@"
+ + function.getArity());
+ }
}
}
}
@@ -1126,6 +1201,15 @@
@Override
public void dropFunction(TxnId txnId, FunctionSignature functionSignature)
throws AlgebricksException, RemoteException {
+ dropFunction(txnId, functionSignature, false);
+ }
+
+ private void dropFunction(TxnId txnId, FunctionSignature
functionSignature, boolean force)
+ throws AlgebricksException, RemoteException {
+
+ if (!force) {
+ confirmFunctionCanBeDeleted(txnId, functionSignature);
+ }
Function function = getFunction(txnId, functionSignature);
@@ -1332,7 +1416,6 @@
// TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer
// Hyracks version.
public static ITupleReference createTuple(String... fields) {
- @SuppressWarnings("unchecked")
ISerializerDeserializer<AString> stringSerde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
AMutableString aString = new AMutableString("");
@@ -1839,7 +1922,6 @@
// This method is used to create a search tuple for external data file
since the
// search tuple has an int value
- @SuppressWarnings("unchecked")
public ITupleReference createExternalFileSearchTuple(String dataverseName,
String datasetName, int fileNumber)
throws HyracksDataException {
ISerializerDeserializer<AString> stringSerde =
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index 406b3d6..99bec66 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -158,7 +158,7 @@
public void dropFunction(FunctionSignature signature) {
Function function = new Function(signature.getNamespace(),
signature.getName(), signature.getArity(), null,
- null, null, null, null);
+ null, null, null, null, null);
droppedCache.addFunctionIfNotExists(function);
logAndApply(new MetadataLogicalOperation(function, false));
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
index 2a04b58..df8b7f1 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -48,6 +48,7 @@
public static final String FIELD_NAME_DATAVERSE_NAME = "DataverseName";
public static final String FIELD_NAME_DATA_FORMAT = "DataFormat";
public static final String FIELD_NAME_DEFINITION = "Definition";
+ public static final String FIELD_NAME_DEPENDENCIES = "Dependencies";
public static final String FIELD_NAME_DERIVED = "Derived";
public static final String FIELD_NAME_DESCRIPTION = "Description";
public static final String FIELD_NAME_EXTERNAL_DETAILS = "ExternalDetails";
@@ -321,16 +322,20 @@
public static final int FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX =
5;
public static final int FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX = 6;
public static final int FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX = 7;
+ public static final int FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX
= 8;
public static final ARecordType FUNCTION_RECORDTYPE = createRecordType(
// RecordTypeName
RECORD_NAME_FUNCTION,
// FieldNames
new String[] { FIELD_NAME_DATAVERSE_NAME, FIELD_NAME_NAME,
FIELD_NAME_ARITY, FIELD_NAME_PARAMS,
- FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION,
FIELD_NAME_LANGUAGE, FIELD_NAME_KIND },
+ FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION,
FIELD_NAME_LANGUAGE, FIELD_NAME_KIND,
+ FIELD_NAME_DEPENDENCIES },
// FieldTypes
new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING,
BuiltinType.ASTRING,
new AOrderedListType(BuiltinType.ASTRING, null),
BuiltinType.ASTRING, BuiltinType.ASTRING,
- BuiltinType.ASTRING, BuiltinType.ASTRING },
+ BuiltinType.ASTRING, BuiltinType.ASTRING,
+ new AOrderedListType(new AOrderedListType(new
AOrderedListType(BuiltinType.ASTRING, null), null),
+ null) },
//IsOpen?
true);
//------------------------------------------ Adapter
----------------------------------------//
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index 1d1db37..b01f087 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -36,13 +36,14 @@
private final String name;
private final int arity;
private final List<String> params;
+ private final List<List<List<String>>> dependencies;
private final String body;
private final String returnType;
private final String language;
private final String kind;
public Function(String dataverseName, String functionName, int arity,
List<String> params, String returnType,
- String functionBody, String language, String functionKind) {
+ String functionBody, String language, String functionKind,
List<List<List<String>>> dependencies) {
this.dataverse = dataverseName;
this.name = functionName;
this.params = params;
@@ -51,6 +52,7 @@
this.language = language;
this.kind = functionKind;
this.arity = arity;
+ this.dependencies = dependencies;
}
public String getDataverseName() {
@@ -65,6 +67,10 @@
return params;
}
+ public List<List<List<String>>> getDependencies() {
+ return dependencies;
+ }
+
public String getFunctionBody() {
return body;
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index e15805e..86f221d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -35,6 +35,7 @@
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.base.IACursor;
import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.BuiltinType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -56,7 +57,13 @@
// Payload field containing serialized Function.
public static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3;
- @SuppressWarnings("unchecked")
+ private transient OrderedListBuilder dependenciesListBuilder = new
OrderedListBuilder();
+ private transient OrderedListBuilder dependencyListBuilder = new
OrderedListBuilder();
+ private transient OrderedListBuilder dependencyNameListBuilder = new
OrderedListBuilder();
+ private transient AOrderedListType stringList = new
AOrderedListType(BuiltinType.ASTRING, null);
+ private transient AOrderedListType ListofLists =
+ new AOrderedListType(new AOrderedListType(BuiltinType.ASTRING,
null), null);
+
private ISerializerDeserializer<ARecord> recordSerDes =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(MetadataRecordTypes.FUNCTION_RECORDTYPE);
@@ -104,8 +111,33 @@
String functionKind =
((AString)
functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX))
.getStringValue();
+
+ IACursor dependenciesCursor = ((AOrderedList) functionRecord
+
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX)).getCursor();
+ List<List<List<String>>> dependencies = new ArrayList<>();
+ AOrderedList dependencyList;
+ AOrderedList quilifiedList;
+ int i = 0;
+ while (dependenciesCursor.next()) {
+ dependencies.add(new ArrayList<>());
+ dependencyList = (AOrderedList) dependenciesCursor.get();
+ IACursor qualifiedDependencyCursor = (dependencyList.getCursor());
+ int j = 0;
+ while (qualifiedDependencyCursor.next()) {
+ quilifiedList = (AOrderedList) qualifiedDependencyCursor.get();
+ IACursor qualifiedNameCursor = (quilifiedList.getCursor());
+ dependencies.get(i).add(new ArrayList<>());
+ while (qualifiedNameCursor.next()) {
+ dependencies.get(i).get(j).add(((AString)
qualifiedNameCursor.get()).getStringValue());
+ }
+ j++;
+ }
+ i++;
+
+ }
+
return new Function(dataverseName, functionName,
Integer.parseInt(arity), params, returnType, definition,
- language, functionKind);
+ language, functionKind, dependencies);
}
@@ -185,6 +217,33 @@
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX,
fieldValue);
+ // write field 8
+ dependenciesListBuilder.reset((AOrderedListType)
MetadataRecordTypes.FUNCTION_RECORDTYPE
+
.getFieldTypes()[MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX]);
+ List<List<List<String>>> dependenciesList = function.getDependencies();
+ for (List<List<String>> dependencies : dependenciesList) {
+ dependencyListBuilder.reset(ListofLists);
+ for (List<String> dependency : dependencies) {
+ dependencyNameListBuilder.reset(stringList);
+ for (String subName : dependency) {
+ itemValue.reset();
+ aString.setValue(subName);
+ stringSerde.serialize(aString, itemValue.getDataOutput());
+ dependencyNameListBuilder.addItem(itemValue);
+ }
+ itemValue.reset();
+ dependencyNameListBuilder.write(itemValue.getDataOutput(),
true);
+ dependencyListBuilder.addItem(itemValue);
+
+ }
+ itemValue.reset();
+ dependencyListBuilder.write(itemValue.getDataOutput(), true);
+ dependenciesListBuilder.addItem(itemValue);
+ }
+ fieldValue.reset();
+ dependenciesListBuilder.write(fieldValue.getDataOutput(), true);
+
recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX,
fieldValue);
+
// write record
recordBuilder.write(tupleBuilder.getDataOutput(), true);
tupleBuilder.addFieldEndOffset();
--
To view, visit https://asterix-gerrit.ics.uci.edu/2253
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2f08ff150dfd57432b88381c507814ddb57bd67b
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Steven Jacobs <sjaco...@ucr.edu>
