Repository: asterixdb Updated Branches: refs/heads/master b8cc121cf -> f4e755e4b
[ASTERIXDB-1949][FUN][ING] Function usage check refactoring in feed - user model changes: no - storage format changes: no - interface changes: yes Removed several interefaces for maintaining reference count. Added getFunctions and getFeeds method for MetadataNode. Details: 1. Change the usage check from reference count to scan feed connections in all dataverses. 2. Add test case for drop dataverse which has function that is being used in other dataverse. This operation should be blocked. 3. Fix small bug in apply function metadata which the function dataverse was not recorded properly. Change-Id: I568f7af18e69216ff2a5fd3b623a1667809b2f95 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1787 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> BAD: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: abdullah alamoudi <bamou...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f4e755e4 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f4e755e4 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f4e755e4 Branch: refs/heads/master Commit: f4e755e4b3b8fe853d1ac404697f4a83540f14f0 Parents: b8cc121 Author: Xikui Wang <xkk...@gmail.com> Authored: Fri Jul 28 10:11:12 2017 -0700 Committer: Xikui Wang <xkk...@gmail.com> Committed: Sat Jul 29 10:09:17 2017 -0700 ---------------------------------------------------------------------- .../app/external/ExternalLibraryUtils.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 57 +++++++++++++------- .../metadata/results/basic/meta06/meta06.1.adm | 2 +- .../metadata_datatype/metadata_datatype.1.adm | 2 +- ...taverse-with-function-used-by-feed.1.ddl.aql | 26 +++++++++ ...taverse-with-function-used-by-feed.2.ddl.aql | 39 ++++++++++++++ ...erse-with-function-used-by-feed.3.update.aql | 23 ++++++++ ...taverse-with-function-used-by-feed.4.ddl.aql | 20 +++++++ .../results/feeds/feeds_03/feeds_03.1.adm | 2 +- .../single-line-definition.1.adm | 2 +- .../user-defined-functions/udf28/udf28.1.adm | 2 +- .../src/test/resources/runtimets/testsuite.xml | 6 +++ .../asterix/common/exceptions/ErrorCode.java | 1 + .../main/resources/asx_errormsg/en.properties | 1 + .../functionDataset/functionDataset.1.adm | 16 +++--- .../asterix/metadata/MetadataManager.java | 31 +++++++---- .../apache/asterix/metadata/MetadataNode.java | 48 ++++++++++------- .../metadata/MetadataTransactionContext.java | 2 +- .../asterix/metadata/api/IMetadataManager.java | 6 ++- .../asterix/metadata/api/IMetadataNode.java | 6 ++- .../metadata/bootstrap/MetadataRecordTypes.java | 7 +-- .../metadata/entities/FeedConnection.java | 10 ++++ .../asterix/metadata/entities/Function.java | 16 +----- .../FeedConnectionTupleTranslator.java | 6 +-- .../FunctionTupleTranslator.java | 10 +--- .../functionDataset/functionDataset.1.adm | 16 +++--- 26 files changed, 249 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java index 0e72976..5f86c28 100755 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java @@ -247,7 +247,7 @@ public class ExternalLibraryUtils { } Function f = new Function(dataverse, libraryName + "#" + function.getName().trim(), args.size(), args, function.getReturnType().trim(), function.getDefinition().trim(), - library.getLanguage().trim(), function.getFunctionType().trim(), 0); + library.getLanguage().trim(), function.getFunctionType().trim()); MetadataManager.INSTANCE.addFunction(mdTxnCtx, f); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Installed function: " + libraryName + "#" + function.getName().trim()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 7725936..28cb998 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -1161,6 +1161,16 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new AlgebricksException("There is no dataverse with this name " + dataverseName + "."); } } + // # check whether any function in current dataverse is being used by others + List<Function> functionsInDataverse = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, + dataverseName); + for (Function function : functionsInDataverse) { + if (checkWhetherFunctionIsBeingUsed(mdTxnCtx, function.getDataverseName(), function.getName(), + function.getArity(), dataverseName)) { + throw new MetadataException(ErrorCode.METADATA_DROP_FUCTION_IN_USE, + function.getDataverseName() + "." + function.getName() + "@" + function.getArity()); + } + } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; // # disconnect all feeds from any datasets in the dataverse. @@ -1624,10 +1634,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen if (dv == null) { throw new AlgebricksException("There is no dataverse with this name " + dataverse + "."); } - // If the function body contains function calls, theirs reference count won't be increased. Function function = new Function(dataverse, functionName, cfs.getaAterixFunction().getArity(), cfs.getParamList(), Function.RETURNTYPE_VOID, cfs.getFunctionBody(), Function.LANGUAGE_AQL, - FunctionKind.SCALAR.toString(), 0); + FunctionKind.SCALAR.toString()); MetadataManager.INSTANCE.addFunction(mdTxnCtx, function); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); @@ -1639,6 +1648,27 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } } + protected boolean checkWhetherFunctionIsBeingUsed(MetadataTransactionContext ctx, String dataverseName, + String functionName, int arity, String currentDataverse) throws MetadataException { + List<Dataverse> allDataverses = MetadataManager.INSTANCE.getDataverses(ctx); + for (Dataverse dataverse : allDataverses) { + if (currentDataverse != null && dataverse.getDataverseName().equals(currentDataverse)) { + continue; + } + List<Feed> feeds = MetadataManager.INSTANCE.getFeeds(ctx, dataverse.getDataverseName()); + for (Feed feed : feeds) { + List<FeedConnection> feedConnections = MetadataManager.INSTANCE.getFeedConections(ctx, + dataverse.getDataverseName(), feed.getFeedName()); + for (FeedConnection conn : feedConnections) { + if (conn.containsFunction(dataverseName, functionName, arity)) { + return true; + } + } + } + } + return false; + } + protected void handleFunctionDropStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { FunctionDropStatement stmtDropFunction = (FunctionDropStatement) stmt; FunctionSignature signature = stmtDropFunction.getFunctionSignature(); @@ -1649,12 +1679,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen signature.getNamespace() + "." + signature.getName()); try { Function function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature); - if (function == null) { - if (!stmtDropFunction.getIfExists()) { - throw new AlgebricksException("Unknonw function " + signature); - } - } else if (function.getReferenceCount() != 0) { - throw new AlgebricksException("Function " + signature + " is being used. It cannot be dropped."); + if (function == null && !stmtDropFunction.getIfExists()) { + throw new AlgebricksException("Unknonw function " + signature); + } else if (checkWhetherFunctionIsBeingUsed(mdTxnCtx, signature.getNamespace(), signature.getName(), + signature.getArity(), null)) { + throw new MetadataException(ErrorCode.METADATA_DROP_FUCTION_IN_USE, signature); } else { MetadataManager.INSTANCE.dropFunction(mdTxnCtx, signature); } @@ -2123,13 +2152,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen fc = new FeedConnection(dataverseName, feedName, datasetName, appliedFunctions, policyName, outputType.toString()); MetadataManager.INSTANCE.addFeedConnection(metadataProvider.getMetadataTxnContext(), fc); - // Increase function reference count. - for (FunctionSignature funcSig : appliedFunctions) { - // The function should be cached in Metadata manager, so this operation is not that expensive. - Function func = MetadataManager.INSTANCE.getFunction(mdTxnCtx, funcSig); - func.reference(); - MetadataManager.INSTANCE.updateFunction(mdTxnCtx, func); - } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); if (listener != null) { listener.add(dataset); @@ -2174,11 +2196,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen + cfs.getDatasetName().getValue() + ". Invalid operation!"); } MetadataManager.INSTANCE.dropFeedConnection(mdTxnCtx, dataverseName, feedName, datasetName); - for (FunctionSignature functionSignature : fc.getAppliedFunctions()) { - Function function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, functionSignature); - function.dereference(); - MetadataManager.INSTANCE.updateFunction(mdTxnCtx, function); - } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); if (listener != null) { listener.remove(ds); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm index 2131611..58f5b77 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm +++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm @@ -1 +1 @@ -{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" } +{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm index 650c924..7229aa8 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm +++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm @@ -29,7 +29,7 @@ { "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterConfiguration", "FieldType": "FeedRecordType_AdapterConfiguration", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } { "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedRecordType_AdapterConfiguration_Item" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } { "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false }, { "FieldName": "ReferenceCount", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Sun May 21 09:56:53 PDT 2017" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } { "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Params", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } { "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "SearchKey", "FieldType": "IndexRecordType_SearchKey", "IsNullable": false }, { "FieldName": "IsPrimary", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } { "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "IndexRecordType_SearchKey_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.1.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.1.ddl.aql new file mode 100644 index 0000000..ea461d2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.1.ddl.aql @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse fundv if exists; +create dataverse fundv; +use dataverse fundv; + +create function test_func0($xyz) { + let $tty1 := if ($xyz.followers_count > 25000) then {"popularity":"Good!"} else {"popularity":"Bad!"} + return object_merge($tty1, $xyz) +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.2.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.2.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.2.ddl.aql new file mode 100644 index 0000000..f16a105 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.2.ddl.aql @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse feeddv if exists; +create dataverse feeddv; +use dataverse feeddv; + +create type TwitterUser if not exists as open{ + screen-name: string, + friends_count: int32, + name: string, + followers_count: int32 +}; + +create dataset TwitterUsers(TwitterUser) primary key screen-name; + +create feed UserFeed using socket_adapter +( + ("sockets"="127.0.0.1:10001"), + ("address-type"="IP"), + ("type-name"="TwitterUser"), + ("format"="adm"), + ("upsert-feed"="true") +); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.3.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.3.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.3.update.aql new file mode 100644 index 0000000..5fe29b5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.3.update.aql @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse feeddv; +set wait-for-completion-feed "false"; + +connect feed UserFeed to dataset TwitterUsers apply function fundv.test_func0; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.4.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.4.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.4.ddl.aql new file mode 100644 index 0000000..50e68e1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/drop-dataverse-with-function-used-by-feed/drop-dataverse-with-function-used-by-feed.4.ddl.aql @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +drop dataverse fundv; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_03/feeds_03.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_03/feeds_03.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_03/feeds_03.1.adm index cbde66d..fbd87b6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_03/feeds_03.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_03/feeds_03.1.adm @@ -1 +1 @@ -{ "DataverseName": "feeds", "FeedName": "TweetFeed", "DatasetName": "Tweets", "ReturnType": "TweetType: closed {\n id: string,\n username: string,\n location: string,\n text: string,\n timestamp: string\n}\n", "AppliedFunctions": {{ "feed_processor" }}, "PolicyName": "Basic" } +{ "DataverseName": "feeds", "FeedName": "TweetFeed", "DatasetName": "Tweets", "ReturnType": "TweetType: closed {\n id: string,\n username: string,\n location: string,\n text: string,\n timestamp: string\n}\n", "AppliedFunctions": {{ "feeds.feed_processor" }}, "PolicyName": "Basic" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm index 5545dfb..5ba00ae 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/single-line-definition/single-line-definition.1.adm @@ -1 +1 @@ -{ "DataverseName": "test", "Name": "printName", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "'AsterixDB Shared nothing parallel BDMS'", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" } +{ "DataverseName": "test", "Name": "printName", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "'AsterixDB Shared nothing parallel BDMS'", "Language": "AQL", "Kind": "SCALAR" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm index 2bc8eaf..b106dbe 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf28/udf28.1.adm @@ -1 +1 @@ -{ "DataverseName": "test", "Name": "f1", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "100", "Language": "AQL", "Kind": "SCALAR", "ReferenceCount": "0" } +{ "DataverseName": "test", "Name": "f1", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "100", "Language": "AQL", "Kind": "SCALAR" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index e99686d..433ee4c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -290,6 +290,12 @@ <output-dir compare="Text">drop-function-used-by-feed</output-dir> </compilation-unit> </test-case> + <test-case FilePath="feeds"> + <compilation-unit name="drop-dataverse-with-function-used-by-feed"> + <output-dir compare="Text">drop-dataverse-with-function-used-by-feed</output-dir> + <expected-error>Function fundv.test_func0@1 is being used. It cannot be dropped.</expected-error> + </compilation-unit> + </test-case> </test-group> <test-group name="upsert"> <test-case FilePath="upsert"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index a00ed99..94fe951 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -227,6 +227,7 @@ public class ErrorCode { public static final int ACTIVE_RUNTIME_IS_NOT_REGISTERED = 3106; public static final int ACTIVE_EVENT_HANDLER_ALREADY_SUSPENDED = 3107; public static final int FEED_STOPPED_WHILE_WAITING_FOR_A_NEW_RECORD = 3108; + public static final int METADATA_DROP_FUCTION_IN_USE = 3109; // Lifecycle management errors public static final int DUPLICATE_PARTITION_ID = 4000; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 9ed6aa5..4cc06a6 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -216,6 +216,7 @@ 3106 = %1$s is not registered 3107 = Active Notification Handler is already suspended 3108 = Feed stopped while waiting for a new record +3109 = Function %1$s is being used. It cannot be dropped. # Lifecycle management errors 4000 = Partition id %1$d for node %2$s already in use by node %3$s http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm b/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm index 52fb855..ef563f1 100644 --- a/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm +++ b/asterixdb/asterix-installer/src/test/resources/integrationts/library/results/library-metadata/functionDataset/functionDataset.1.adm @@ -1,8 +1,8 @@ -{ "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } +{ "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java index 5e6d11a..23e6fb0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java @@ -565,17 +565,6 @@ public class MetadataManager implements IMetadataManager { } @Override - public void updateFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException { - try { - metadataNode.updateFunction(mdTxnCtx.getJobId(), function); - } catch (RemoteException e) { - throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e); - } - mdTxnCtx.dropFunction(function); - mdTxnCtx.addFunction(function); - } - - @Override public void addFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException { try { metadataNode.addFunction(mdTxnCtx.getJobId(), function); @@ -637,6 +626,15 @@ public class MetadataManager implements IMetadataManager { } @Override + public List<Function> getFunctions(MetadataTransactionContext ctx, String dataverseName) throws MetadataException { + try { + return metadataNode.getFunctions(ctx.getJobId(), dataverseName); + } catch (RemoteException e) { + throw new MetadataException(e); + } + } + + @Override public void addFeedPolicy(MetadataTransactionContext mdTxnCtx, FeedPolicyEntity feedPolicy) throws MetadataException { try { @@ -797,6 +795,17 @@ public class MetadataManager implements IMetadataManager { } @Override + public List<Feed> getFeeds(MetadataTransactionContext ctx, String dataverse) throws MetadataException { + List<Feed> feeds; + try { + feeds = metadataNode.getFeeds(ctx.getJobId(), dataverse); + } catch (RemoteException e) { + throw new MetadataException(e); + } + return feeds; + } + + @Override public void dropFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException { Feed feed = null; List<FeedConnection> feedConnections = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 265e533..51cf988 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -1162,6 +1162,20 @@ public class MetadataNode implements IMetadataNode { } @Override + public List<Function> getFunctions(JobId jobId, String dataverseName) throws MetadataException, RemoteException { + try { + ITupleReference searchKey = createTuple(dataverseName); + FunctionTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFunctionTupleTranslator(false); + List<Function> results = new ArrayList<>(); + IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter); + searchIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, searchKey, valueExtractor, results); + return results; + } catch (HyracksDataException e) { + throw new MetadataException(e); + } + } + + @Override public void dropFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException, RemoteException { @@ -1741,6 +1755,20 @@ public class MetadataNode implements IMetadataNode { } @Override + public List<Feed> getFeeds(JobId jobId, String dataverse) throws MetadataException, RemoteException { + try { + ITupleReference searchKey = createTuple(dataverse); + FeedTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedTupleTranslator(false); + List<Feed> results = new ArrayList<>(); + IValueExtractor<Feed> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter); + searchIndex(jobId, MetadataPrimaryIndexes.FEED_DATASET, searchKey, valueExtractor, results); + return results; + } catch (HyracksDataException e) { + throw new MetadataException(e); + } + } + + @Override public void dropFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException { try { ITupleReference searchKey = createTuple(dataverse, feedName); @@ -1936,24 +1964,4 @@ public class MetadataNode implements IMetadataNode { throw new MetadataException(e); } } - - @Override - public void updateFunction(JobId jobId, Function function) throws MetadataException, RemoteException { - try { - // remove old function - ITupleReference searchKey; - searchKey = - createTuple(function.getDataverseName(), function.getName(), Integer.toString(function.getArity())); - ITupleReference functionTuple = - getTupleToBeDeleted(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, searchKey); - deleteTupleFromIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple); - // add new function - FunctionTupleTranslator functionTupleTranslator = tupleTranslatorProvider.getFunctionTupleTranslator(true); - functionTuple = functionTupleTranslator.getTupleFromMetadataEntity(function); - insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple); - } catch (HyracksDataException | ACIDException e) { - throw new MetadataException(e); - } - - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java index 0229f39..72285d0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java @@ -158,7 +158,7 @@ public class MetadataTransactionContext extends MetadataCache { public void dropFunction(FunctionSignature signature) { Function function = new Function(signature.getNamespace(), signature.getName(), signature.getArity(), null, - null, null, null, null, 0); + null, null, null, null); droppedCache.addFunctionIfNotExists(function); logAndApply(new MetadataLogicalOperation(function, false)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java index 6ad1a20..88153a3 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java @@ -391,6 +391,8 @@ public interface IMetadataManager extends IMetadataBootstrap { Function getFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature) throws MetadataException; + List<Function> getFunctions(MetadataTransactionContext ctx, String dataverseName) throws MetadataException; + /** * @param ctx * MetadataTransactionContext of an active metadata transaction. @@ -400,8 +402,6 @@ public interface IMetadataManager extends IMetadataBootstrap { */ void dropFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature) throws MetadataException; - void updateFunction(MetadataTransactionContext ctx, Function function) throws MetadataException; - /** * @param mdTxnCtx * MetadataTransactionContext of an active metadata transaction. @@ -488,6 +488,8 @@ public interface IMetadataManager extends IMetadataBootstrap { */ Feed getFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException; + List<Feed> getFeeds(MetadataTransactionContext ctx, String dataverse) throws MetadataException; + /** * @param ctx * @param dataverse http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java index 78360fd..a990d29 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java @@ -403,6 +403,8 @@ public interface IMetadataNode extends Remote, Serializable { */ Function getFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException, RemoteException; + List<Function> getFunctions(JobId jobId, String dataverseName) throws MetadataException, RemoteException; + /** * Deletes a function, acquiring local locks on behalf of the given * transaction id. @@ -430,8 +432,6 @@ public interface IMetadataNode extends Remote, Serializable { */ void addFunction(JobId jobId, Function function) throws MetadataException, RemoteException; - void updateFunction(JobId jobId, Function function) throws MetadataException, RemoteException; - /** * @param ctx * @param dataverseName @@ -539,6 +539,8 @@ public interface IMetadataNode extends Remote, Serializable { */ Feed getFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException; + List<Feed> getFeeds(JobId jobId, String dataverse) throws MetadataException, RemoteException; + /** * @param jobId * @param dataverse http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java index 852c01f..2a04b58 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java @@ -99,7 +99,6 @@ public final class MetadataRecordTypes { public static final String FIELD_NAME_VALUE = "Value"; public static final String FIELD_NAME_WORKING_MEMORY_SIZE = "WorkingMemorySize"; public static final String FIELD_NAME_APPLIED_FUNCTIONS = "AppliedFunctions"; - public static final String FIELD_NAME_REFERENCE_COUNT = "ReferenceCount"; //---------------------------------- Record Types Creation ----------------------------------// //--------------------------------------- Properties ----------------------------------------// @@ -322,18 +321,16 @@ public final class MetadataRecordTypes { public static final int FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX = 5; public static final int FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX = 6; public static final int FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX = 7; - public static final int FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX = 8; public static final ARecordType FUNCTION_RECORDTYPE = createRecordType( // RecordTypeName RECORD_NAME_FUNCTION, // FieldNames new String[] { FIELD_NAME_DATAVERSE_NAME, FIELD_NAME_NAME, FIELD_NAME_ARITY, FIELD_NAME_PARAMS, - FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND, - FIELD_NAME_REFERENCE_COUNT }, + FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND }, // FieldTypes new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING, new AOrderedListType(BuiltinType.ASTRING, null), BuiltinType.ASTRING, BuiltinType.ASTRING, - BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING }, + BuiltinType.ASTRING, BuiltinType.ASTRING }, //IsOpen? true); //------------------------------------------ Adapter ----------------------------------------// http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java index 151c9ca..8e14ee5 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java @@ -112,4 +112,14 @@ public class FeedConnection implements IMetadataEntity<FeedConnection> { public EntityId getFeedId() { return feedId; } + + public boolean containsFunction(String dataverseName, String functionName, int arity) { + for (FunctionSignature signature : this.appliedFunctions) { + if (signature.getNamespace().equals(dataverseName) && signature.getName().equals(functionName) + && signature.getArity() == arity) { + return true; + } + } + return false; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java index b010a8a..7ff423c 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java @@ -19,7 +19,6 @@ package org.apache.asterix.metadata.entities; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.asterix.metadata.MetadataCache; import org.apache.asterix.metadata.api.IMetadataEntity; @@ -40,10 +39,9 @@ public class Function implements IMetadataEntity<Function> { private final String returnType; private final String language; private final String kind; - private AtomicInteger referenceCount; public Function(String dataverseName, String functionName, int arity, List<String> params, String returnType, - String functionBody, String language, String functionKind, int referenceCount) { + String functionBody, String language, String functionKind) { this.dataverse = dataverseName; this.name = functionName; this.params = params; @@ -52,7 +50,6 @@ public class Function implements IMetadataEntity<Function> { this.language = language; this.kind = functionKind; this.arity = arity; - this.referenceCount = new AtomicInteger(referenceCount); } public String getDataverseName() { @@ -97,15 +94,4 @@ public class Function implements IMetadataEntity<Function> { return cache.dropFunction(this); } - public int getReferenceCount() { - return referenceCount.get(); - } - - public int reference() { - return referenceCount.incrementAndGet(); - } - - public int dereference() { - return referenceCount.decrementAndGet(); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java index 2a2387d..ac78b8f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java @@ -95,8 +95,8 @@ public class FeedConnectionTupleTranslator extends AbstractTupleTranslator<FeedC cursor = ((AUnorderedList) feedConnRecord .getValueByPos(MetadataRecordTypes.FEED_CONN_APPLIED_FUNCTIONS_FIELD_INDEX)).getCursor(); while (cursor.next()) { - //TODO: allow different arity - functionSignature = new FunctionSignature(dataverseName, ((AString) cursor.get()).getStringValue(), 1); + String[] functionFullName = ((AString) cursor.get()).getStringValue().split("\\."); + functionSignature = new FunctionSignature(functionFullName[0], functionFullName[1], 1); appliedFunctions.add(functionSignature); } } @@ -177,7 +177,7 @@ public class FeedConnectionTupleTranslator extends AbstractTupleTranslator<FeedC List<FunctionSignature> appliedFunctions = fc.getAppliedFunctions(); for (FunctionSignature af : appliedFunctions) { listEleBuffer.reset(); - aString.setValue(af.getName()); + aString.setValue(af.getNamespace() + "." + af.getName()); stringSerde.serialize(aString, listEleBuffer.getDataOutput()); listBuilder.addItem(listEleBuffer); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java index eed082c..858d443 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java @@ -104,10 +104,8 @@ public class FunctionTupleTranslator extends AbstractTupleTranslator<Function> { String functionKind = ((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX)) .getStringValue(); - String referenceCount = ((AString) functionRecord - .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX)).getStringValue(); return new Function(dataverseName, functionName, Integer.parseInt(arity), params, returnType, definition, - language, functionKind, Integer.parseInt(referenceCount)); + language, functionKind); } @@ -187,12 +185,6 @@ public class FunctionTupleTranslator extends AbstractTupleTranslator<Function> { stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX, fieldValue); - // write field 8 - fieldValue.reset(); - aString.setValue(Integer.toString(function.getReferenceCount())); - stringSerde.serialize(aString, fieldValue.getDataOutput()); - recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_REFERENCE_COUNT_INDEX, fieldValue); - // write record recordBuilder.write(tupleBuilder.getDataOutput(), true); tupleBuilder.addFieldEndOffset(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f4e755e4/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm b/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm index cdf7792..0675316 100644 --- a/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm +++ b/asterixdb/asterix-yarn/src/test/resources/library/results/library-metadata/functionDataset/functionDataset.1.adm @@ -1,8 +1,8 @@ -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } -{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "ReferenceCount": "0" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR" } +{ "DataverseName": "externallibtest", "Name": "asterix-external-data-testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR" }
