Dmitry Lychagin has submitted this change and it was merged. Change subject: [ASTERIXDB-2180][FUN] Prevent dropping of entities used by functions ......................................................................
[ASTERIXDB-2180][FUN] Prevent dropping of entities used by functions Add dependencies to Functional Metadata Check dependencies before dropping datasets or functions Add tests Change-Id: I2f08ff150dfd57432b88381c507814ddb57bd67b Reviewed-on: https://asterix-gerrit.ics.uci.edu/2253 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm M asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.3.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm A asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml M asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java 30 files changed, 875 insertions(+), 185 deletions(-) Approvals: Jenkins: Verified; ; Verified Dmitry Lychagin: Looks good to me, approved Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 1257a61..4428e05 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -180,7 +180,7 @@ } public Pair<IReturningStatement, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions, - MetadataProvider metadataProvider, IReturningStatement q, SessionOutput output) + MetadataProvider metadataProvider, IReturningStatement q, SessionOutput output, boolean inlineUdfs) throws CompilationException { if (q == null) { return null; @@ -193,7 +193,7 @@ printPlanPostfix(output); } IQueryRewriter rw = rewriterFactory.createQueryRewriter(); - rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter())); + rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter()), inlineUdfs); return new Pair<>(q, q.getVarCounter()); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java index b013c60..f2933f4 100755 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java @@ -248,7 +248,7 @@ } Function f = new Function(dataverse, libraryName + "#" + function.getName().trim(), args.size(), args, function.getReturnType().trim(), function.getDefinition().trim(), - library.getLanguage().trim(), function.getFunctionType().trim()); + library.getLanguage().trim(), function.getFunctionType().trim(), null); MetadataManager.INSTANCE.addFunction(mdTxnCtx, f); if (LOGGER.isInfoEnabled()) { LOGGER.info("Installed function: " + libraryName + "#" + function.getName().trim()); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 70f72a5..9b96883 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -124,6 +124,7 @@ import org.apache.asterix.lang.common.statement.WriteStatement; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.struct.VarIdentifier; +import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.lang.common.util.MergePolicyUtils; import org.apache.asterix.lang.sqlpp.rewrites.SqlppRewriterFactory; import org.apache.asterix.metadata.IDatasetDetails; @@ -497,8 +498,8 @@ throw new CompilationException("Unknown compaction policy: " + compactionPolicy); } String compactionPolicyFactoryClassName = compactionPolicyEntity.getClassName(); - ILSMMergePolicyFactory mergePolicyFactory = (ILSMMergePolicyFactory) Class - .forName(compactionPolicyFactoryClassName).newInstance(); + ILSMMergePolicyFactory mergePolicyFactory = + (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance(); if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) { throw new CompilationException("The correlated-prefix merge policy cannot be used with external dataset."); } @@ -585,10 +586,10 @@ } ARecordType metaRecType = (ARecordType) metaItemType; - List<List<String>> partitioningExprs = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()) - .getPartitioningExprs(); - List<Integer> keySourceIndicators = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()) - .getKeySourceIndicators(); + List<List<String>> partitioningExprs = + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(); + List<Integer> keySourceIndicators = + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getKeySourceIndicators(); boolean autogenerated = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated(); ARecordType aRecordType = (ARecordType) itemType; List<IAType> partitioningTypes = ValidateUtil.validatePartitioningExpressions(aRecordType, @@ -613,8 +614,8 @@ String adapter = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getAdapter(); Map<String, String> properties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getProperties(); - datasetDetails = new ExternalDatasetDetails(adapter, properties, new Date(), - TransactionState.COMMIT); + datasetDetails = + new ExternalDatasetDetails(adapter, properties, new Date(), TransactionState.COMMIT); break; default: throw new CompilationException("Unknown datatype " + dd.getDatasetType()); @@ -705,8 +706,8 @@ protected static void validateIfResourceIsActiveInFeed(ICcApplicationContext appCtx, Dataset dataset) throws CompilationException { StringBuilder builder = null; - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); IActiveEntityEventsListener[] listeners = activeEventHandler.getEventListeners(); for (IActiveEntityEventsListener listener : listeners) { if (listener.isEntityUsingDataset(dataset) && listener.isActive()) { @@ -809,8 +810,8 @@ for (Pair<List<String>, IndexedTypeExpression> fieldExpr : stmtCreateIndex.getFieldExprs()) { IAType fieldType = null; - ARecordType subType = KeyFieldTypeUtil.chooseSource(keySourceIndicators, keyIndex, aRecordType, - metaRecordType); + ARecordType subType = + KeyFieldTypeUtil.chooseSource(keySourceIndicators, keyIndex, aRecordType, metaRecordType); boolean isOpen = subType.isOpen(); int i = 0; if (fieldExpr.first.size() > 1 && !isOpen) { @@ -846,8 +847,8 @@ if (stmtCreateIndex.hasMetaField()) { throw new AlgebricksException("Typed open index can only be created on the record part"); } - Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, - fieldExpr.second.getType(), indexName, dataverseName); + Map<TypeSignature, IAType> typeMap = + TypeTranslator.computeTypes(mdTxnCtx, fieldExpr.second.getType(), indexName, dataverseName); TypeSignature typeSignature = new TypeSignature(dataverseName, indexName); fieldType = typeMap.get(typeSignature); overridesFieldTypes = true; @@ -1062,8 +1063,8 @@ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); bActiveTxn = true; try { - JobSpecification jobSpec = ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider, - ds); + JobSpecification jobSpec = + ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider, ds); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; runJob(hcc, jobSpec, jobFlags); @@ -1219,8 +1220,8 @@ } } // # check whether any function in current dataverse is being used by others - List<Function> functionsInDataverse = MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, - dataverseName); + List<Function> functionsInDataverse = + MetadataManager.INSTANCE.getDataverseFunctions(mdTxnCtx, dataverseName); for (Function function : functionsInDataverse) { if (checkWhetherFunctionIsBeingUsed(mdTxnCtx, function.getDataverseName(), function.getName(), function.getArity(), dataverseName)) { @@ -1231,8 +1232,8 @@ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; // # disconnect all feeds from any datasets in the dataverse. - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); IActiveEntityEventsListener[] activeListeners = activeEventHandler.getEventListeners(); for (IActiveEntityEventsListener listener : activeListeners) { EntityId activeEntityId = listener.getEntityId(); @@ -1260,15 +1261,15 @@ String datasetName = dataset.getDatasetName(); DatasetType dsType = dataset.getDatasetType(); if (dsType == DatasetType.INTERNAL) { - List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, - datasetName); + List<Index> indexes = + MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); for (Index index : indexes) { jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, dataset)); } } else { // External dataset - List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, - datasetName); + List<Index> indexes = + MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); for (int k = 0; k < indexes.size(); k++) { if (ExternalIndexingOperations.isFileIndex(indexes.get(k))) { jobsToExecute.add( @@ -1376,8 +1377,8 @@ public static void doDropDataset(String dataverseName, String datasetName, MetadataProvider metadataProvider, boolean ifExists, IHyracksClientConnection hcc, boolean dropCorrespondingNodeGroup) throws Exception { MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS); - MutableObject<MetadataTransactionContext> mdTxnCtx = new MutableObject<>( - MetadataManager.INSTANCE.beginTransaction()); + MutableObject<MetadataTransactionContext> mdTxnCtx = + new MutableObject<>(MetadataManager.INSTANCE.beginTransaction()); MutableBoolean bActiveTxn = new MutableBoolean(true); metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue()); List<JobSpecification> jobsToExecute = new ArrayList<>(); @@ -1452,8 +1453,8 @@ throw new AlgebricksException( "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName); } - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); IActiveEntityEventsListener[] listeners = activeEventHandler.getEventListeners(); StringBuilder builder = null; for (IActiveEntityEventsListener listener : listeners) { @@ -1522,8 +1523,8 @@ } // #. prepare a job to drop the index in NC. jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds)); - List<Index> datasetIndexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, - datasetName); + List<Index> datasetIndexes = + MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); if (datasetIndexes.size() == 2) { dropFilesIndex = true; // only one index + the files index, we need to delete both of the indexes @@ -1701,12 +1702,15 @@ varIds.add(new VarIdentifier(v)); } wrappedQuery.setExternalVars(varIds); - apiFramework.reWriteQuery(declaredFunctions, metadataProvider, wrappedQuery, sessionOutput); + apiFramework.reWriteQuery(declaredFunctions, metadataProvider, wrappedQuery, sessionOutput, false); + + List<List<List<String>>> dependencies = FunctionUtil.getFunctionDependencies( + rewriterFactory.createQueryRewriter(), cfs.getFunctionBodyExpression(), metadataProvider); Function function = new Function(dataverse, functionName, cfs.getFunctionSignature().getArity(), cfs.getParamList(), Function.RETURNTYPE_VOID, cfs.getFunctionBody(), rewriterFactory instanceof SqlppRewriterFactory ? Function.LANGUAGE_SQLPP : Function.LANGUAGE_AQL, - FunctionKind.SCALAR.toString()); + FunctionKind.SCALAR.toString(), dependencies); MetadataManager.INSTANCE.addFunction(mdTxnCtx, function); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); @@ -1778,9 +1782,9 @@ MetadataLockUtil.modifyDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, dataverseName + "." + datasetName); try { - CompiledLoadFromFileStatement cls = new CompiledLoadFromFileStatement(dataverseName, - loadStmt.getDatasetName().getValue(), loadStmt.getAdapter(), loadStmt.getProperties(), - loadStmt.dataIsAlreadySorted()); + CompiledLoadFromFileStatement cls = + new CompiledLoadFromFileStatement(dataverseName, loadStmt.getDatasetName().getValue(), + loadStmt.getAdapter(), loadStmt.getProperties(), loadStmt.dataIsAlreadySorted()); JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionOutput, cls); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; @@ -1897,8 +1901,8 @@ throws RemoteException, AlgebricksException, ACIDException { // Query Rewriting (happens under the same ongoing metadata transaction) - Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, - metadataProvider, query, sessionOutput); + Pair<IReturningStatement, Integer> rewrittenResult = + apiFramework.reWriteQuery(declaredFunctions, metadataProvider, query, sessionOutput, true); // Query Compilation (happens under the same ongoing metadata transaction) return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first, @@ -1911,8 +1915,8 @@ // Insert/upsert statement rewriting (happens under the same ongoing metadata // transaction) - Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(declaredFunctions, - metadataProvider, insertUpsert, sessionOutput); + Pair<IReturningStatement, Integer> rewrittenResult = + apiFramework.reWriteQuery(declaredFunctions, metadataProvider, insertUpsert, sessionOutput, true); InsertStatement rewrittenInsertUpsert = (InsertStatement) rewrittenResult.first; String dataverseName = getActiveDataverse(rewrittenInsertUpsert.getDataverseName()); @@ -1947,8 +1951,8 @@ MetadataLockUtil.createFeedBegin(lockManager, metadataProvider.getLocks(), dataverseName, dataverseName + "." + feedName); try { - Feed feed = MetadataManager.INSTANCE.getFeed(metadataProvider.getMetadataTxnContext(), dataverseName, - feedName); + Feed feed = + MetadataManager.INSTANCE.getFeed(metadataProvider.getMetadataTxnContext(), dataverseName, feedName); if (feed != null) { if (cfs.getIfNotExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); @@ -1983,8 +1987,8 @@ try { mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); - FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE - .getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, policy); + FeedPolicyEntity feedPolicy = + MetadataManager.INSTANCE.getFeedPolicy(metadataProvider.getMetadataTxnContext(), dataverse, policy); if (feedPolicy != null) { if (cfps.getIfNotExists()) { MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); @@ -2062,10 +2066,10 @@ throws Exception { MetadataTransactionContext mdTxnCtx = metadataProvider.getMetadataTxnContext(); EntityId feedId = feed.getFeedId(); - ActiveNotificationHandler activeNotificationHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); - ActiveEntityEventsListener listener = (ActiveEntityEventsListener) activeNotificationHandler - .getListener(feedId); + ActiveNotificationHandler activeNotificationHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); + ActiveEntityEventsListener listener = + (ActiveEntityEventsListener) activeNotificationHandler.getListener(feedId); if (listener != null && listener.getState() != ActivityState.STOPPED) { throw new AlgebricksException("Feed " + feedId + " is currently active and connected to the following dataset(s) \n" + listener.toString()); @@ -2134,15 +2138,15 @@ String fqName = feedConnection.getDataverseName() + "." + feedConnection.getDatasetName(); lockManager.acquireDatasetReadLock(metadataProvider.getLocks(), fqName); } - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); ActiveEntityEventsListener listener = (ActiveEntityEventsListener) activeEventHandler.getListener(entityId); if (listener == null) { // Prepare policy List<Dataset> datasets = new ArrayList<>(); for (FeedConnection connection : feedConnections) { - Dataset ds = metadataProvider.findDataset(connection.getDataverseName(), - connection.getDatasetName()); + Dataset ds = + metadataProvider.findDataset(connection.getDataverseName(), connection.getDatasetName()); datasets.add(ds); } listener = new FeedEventsListener(this, metadataProvider.getApplicationContext(), hcc, entityId, @@ -2167,8 +2171,8 @@ String dataverseName = getActiveDataverse(sfst.getDataverseName()); String feedName = sfst.getFeedName().getValue(); EntityId entityId = new EntityId(Feed.EXTENSION_NAME, dataverseName, feedName); - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); // Obtain runtime info from ActiveListener ActiveEntityEventsListener listener = (ActiveEntityEventsListener) activeEventHandler.getListener(entityId); if (listener == null) { @@ -2194,8 +2198,8 @@ metadataProvider.setMetadataTxnContext(mdTxnCtx); // TODO: Check whether we are connecting a change feed to a non-meta dataset // Check whether feed is alive - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); // Transaction handling MetadataLockUtil.connectFeedBegin(lockManager, metadataProvider.getLocks(), dataverseName, dataverseName + "." + datasetName, dataverseName + "." + feedName); @@ -2248,8 +2252,8 @@ MetadataLockUtil.disconnectFeedBegin(lockManager, metadataProvider.getLocks(), dataverseName, dataverseName + "." + datasetName, dataverseName + "." + cfs.getFeedName()); try { - ActiveNotificationHandler activeEventHandler = (ActiveNotificationHandler) appCtx - .getActiveNotificationHandler(); + ActiveNotificationHandler activeEventHandler = + (ActiveNotificationHandler) appCtx.getActiveNotificationHandler(); // Check whether feed is alive ActiveEntityEventsListener listener = (ActiveEntityEventsListener) activeEventHandler .getListener(new EntityId(Feed.EXTENSION_NAME, dataverseName, feedName)); @@ -2304,8 +2308,8 @@ throw new AlgebricksException( "Cannot compact the extrenal dataset " + datasetName + " because it has no indexes"); } - Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), - dataverseName); + Dataverse dataverse = + MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName); jobsToExecute.add(DatasetUtil.compactDatasetJobSpec(dataverse, datasetName, metadataProvider)); if (ds.getDatasetType() == DatasetType.INTERNAL) { @@ -2435,8 +2439,8 @@ } private void updateJobStats(JobId jobId, Stats stats) { - final IJobManager jobManager = ((ClusterControllerService) appCtx.getServiceContext().getControllerService()) - .getJobManager(); + final IJobManager jobManager = + ((ClusterControllerService) appCtx.getServiceContext().getControllerService()).getJobManager(); final JobRun run = jobManager.get(jobId); if (run == null || run.getStatus() != JobStatus.TERMINATED) { return; @@ -2804,8 +2808,8 @@ String dataverseNameTo = getActiveDataverse(pregelixStmt.getDataverseNameTo()); String datasetNameFrom = pregelixStmt.getDatasetNameFrom().getValue(); String datasetNameTo = pregelixStmt.getDatasetNameTo().getValue(); - String fullyQualifiedDatasetNameTo = DatasetUtil.isFullyQualifiedName(datasetNameTo) ? datasetNameTo - : dataverseNameTo + '.' + datasetNameTo; + String fullyQualifiedDatasetNameTo = + DatasetUtil.isFullyQualifiedName(datasetNameTo) ? datasetNameTo : dataverseNameTo + '.' + datasetNameTo; MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); MetadataLockUtil.insertDeleteUpsertBegin(lockManager, metadataProvider.getLocks(), fullyQualifiedDatasetNameTo); @@ -2888,8 +2892,8 @@ throw new AlgebricksException("Tried to access non-existing dataset: " + datasetNameTo); } // Cleans up the sink dataset -- Drop and then Create. - DropDatasetStatement dropStmt = new DropDatasetStatement(new Identifier(dataverseNameTo), - pregelixStmt.getDatasetNameTo(), true); + DropDatasetStatement dropStmt = + new DropDatasetStatement(new Identifier(dataverseNameTo), pregelixStmt.getDatasetNameTo(), true); this.handleDatasetDropStatement(metadataProvider, dropStmt, hcc, null); IDatasetDetailsDecl idd = new InternalDetailsDecl(toIndex.getKeyFieldNames(), toIndex.getKeyFieldSourceIndicators(), false, null); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 0b3a135..b49bd8f 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -1627,7 +1627,8 @@ public void cleanup(String testCase, List<String> badtestcases) throws Exception { try { ArrayList<String> toBeDropped = new ArrayList<>(); - InputStream resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse` as dv;", + InputStream resultStream = executeQueryService( + "select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName;", getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON); String out = IOUtils.toString(resultStream); ObjectMapper om = new ObjectMapper(); diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm index 58f5b77..8be6cbc 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm +++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/meta06/meta06.1.adm @@ -1 +1 @@ -{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR" } +{ "DataverseName": "testdv", "Name": "fun01", "Arity": "0", "Params": [ ], "ReturnType": "VOID", "Definition": "\"This is an AQL Bodied UDF\"", "Language": "AQL", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm index d92fcd2..822d2c3 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm +++ b/asterixdb/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm @@ -1,66 +1,69 @@ -{ "DataverseName": "Metadata", "DatatypeName": "AnyObject", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicy", "FieldType": "string", "IsNullable": false }, { "FieldName": "Classname", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeDataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetType", "FieldType": "string", "IsNullable": false }, { "FieldName": "GroupName", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicy", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicyProperties", "FieldType": "DatasetRecordType_CompactionPolicyProperties", "IsNullable": false }, { "FieldName": "InternalDetails", "FieldType": "DatasetRecordType_InternalDetails", "IsNullable": true }, { "FieldName": "ExternalDetails", "FieldType": "Datase tRecordType_ExternalDetails", "IsNullable": true }, { "FieldName": "Hints", "FieldType": "DatasetRecordType_Hints", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetId", "FieldType": "int32", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_CompactionPolicyProperties", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_CompactionPolicyProperties_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_CompactionPolicyProperties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string", "IsNullable": false }, { "FieldName": "Properties", "FieldType": "DatasetRecordType_ExternalDetails_Properties", "IsNullable": false }, { "FieldName": "LastRefreshTime", "FieldType": "datetime", "IsNullable": false }, { "FieldName": "TransactionState", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails_Properties", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_ExternalDetails_Properties_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails_Properties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_Hints", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "DatasetRecordType_Hints_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_Hints_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "PartitioningStrategy", "FieldType": "string", "IsNullable": false }, { "FieldName": "PartitioningKey", "FieldType": "DatasetRecordType_InternalDetails_PartitioningKey", "IsNullable": false }, { "FieldName": "PrimaryKey", "FieldType": "DatasetRecordType_InternalDetails_PrimaryKey", "IsNullable": false }, { "FieldName": "Autogenerated", "FieldType": "boolean", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PartitioningKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_InternalDetails_PartitioningKey_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PartitioningKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PrimaryKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_InternalDetails_PrimaryKey_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PrimaryKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Classname", "FieldType": "string", "IsNullable": false }, { "FieldName": "Type", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Derived", "FieldType": "DatatypeRecordType_Derived", "IsNullable": true }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string", "IsNullable": false }, { "FieldName": "IsAnonymous", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Record", "FieldType": "DatatypeRecordType_Derived_Record", "IsNullable": true }, { "FieldName": "UnorderedList", "FieldType": "string", "IsNullable": true }, { "FieldName": "OrderedList", "FieldType": "string", "IsNullable": true } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Fields", "FieldType": "DatatypeRecordType_Derived_Record_Fields", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record_Fields", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatatypeRecordType_Derived_Record_Fields_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record_Fields_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FieldType", "FieldType": "string", "IsNullable": false }, { "FieldName": "IsNullable", "FieldType": "boolean", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DataFormat", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FileNumber", "FieldType": "int32", "IsNullable": false }, { "FieldName": "FileName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FileSize", "FieldType": "int64", "IsNullable": false }, { "FieldName": "FileModTime", "FieldType": "datetime", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "AppliedFunctions", "FieldType": "FeedConnectionRecordType_AppliedFunctions", "IsNullable": false }, { "FieldName": "PolicyName", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType_AppliedFunctions", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "PolicyName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Description", "FieldType": "string", "IsNullable": false }, { "FieldName": "Properties", "FieldType": "FeedPolicyRecordType_Properties", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType_Properties", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedPolicyRecordType_Properties_Item" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType_Properties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterConfiguration", "FieldType": "FeedRecordType_AdapterConfiguration", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Dec 08 08:42:06 UTC 2017" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedRecordType_AdapterConfiguration_Item" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Params", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "SearchKey", "FieldType": "IndexRecordType_SearchKey", "IsNullable": false }, { "FieldName": "IsPrimary", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "IndexRecordType_SearchKey_Item" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:22 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string", "IsNullable": false }, { "FieldName": "NodeNames", "FieldType": "NodeGroupRecordType_NodeNames", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType_NodeNames", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "string" }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "NumberOfCores", "FieldType": "int64", "IsNullable": false }, { "FieldName": "WorkingMemorySize", "FieldType": "int64", "IsNullable": false } ] } }, "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "binary", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "circle", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "date", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "double", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "duration", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "float", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "int16", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "int32", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "int64", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "int8", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "interval", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "line", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "missing", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "null", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "point", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "shortwithouttypeinfo", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "string", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "time", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } -{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Timestamp": "Fri Oct 21 10:29:21 PDT 2016" } +{ "DataverseName": "Metadata", "DatatypeName": "AnyObject", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ ] } }, "Timestamp": "Mon Jan 08 10:27:05 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicy", "FieldType": "string", "IsNullable": false }, { "FieldName": "Classname", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeDataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetType", "FieldType": "string", "IsNullable": false }, { "FieldName": "GroupName", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicy", "FieldType": "string", "IsNullable": false }, { "FieldName": "CompactionPolicyProperties", "FieldType": "DatasetRecordType_CompactionPolicyProperties", "IsNullable": false }, { "FieldName": "InternalDetails", "FieldType": "DatasetRecordType_InternalDetails", "IsNullable": true }, { "FieldName": "ExternalDetails", "FieldType": "Datase tRecordType_ExternalDetails", "IsNullable": true }, { "FieldName": "Hints", "FieldType": "DatasetRecordType_Hints", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetId", "FieldType": "int32", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_CompactionPolicyProperties", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_CompactionPolicyProperties_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_CompactionPolicyProperties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string", "IsNullable": false }, { "FieldName": "Properties", "FieldType": "DatasetRecordType_ExternalDetails_Properties", "IsNullable": false }, { "FieldName": "LastRefreshTime", "FieldType": "datetime", "IsNullable": false }, { "FieldName": "TransactionState", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails_Properties", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_ExternalDetails_Properties_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_ExternalDetails_Properties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_Hints", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "DatasetRecordType_Hints_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_Hints_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "PartitioningStrategy", "FieldType": "string", "IsNullable": false }, { "FieldName": "PartitioningKey", "FieldType": "DatasetRecordType_InternalDetails_PartitioningKey", "IsNullable": false }, { "FieldName": "PrimaryKey", "FieldType": "DatasetRecordType_InternalDetails_PrimaryKey", "IsNullable": false }, { "FieldName": "Autogenerated", "FieldType": "boolean", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PartitioningKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_InternalDetails_PartitioningKey_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PartitioningKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PrimaryKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatasetRecordType_InternalDetails_PrimaryKey_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType_InternalDetails_PrimaryKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Classname", "FieldType": "string", "IsNullable": false }, { "FieldName": "Type", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatatypeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Derived", "FieldType": "DatatypeRecordType_Derived", "IsNullable": true }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string", "IsNullable": false }, { "FieldName": "IsAnonymous", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Record", "FieldType": "DatatypeRecordType_Derived_Record", "IsNullable": true }, { "FieldName": "UnorderedList", "FieldType": "string", "IsNullable": true }, { "FieldName": "OrderedList", "FieldType": "string", "IsNullable": true } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Fields", "FieldType": "DatatypeRecordType_Derived_Record_Fields", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record_Fields", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "DatatypeRecordType_Derived_Record_Fields_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType_Derived_Record_Fields_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FieldType", "FieldType": "string", "IsNullable": false }, { "FieldName": "IsNullable", "FieldType": "boolean", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DataFormat", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FileNumber", "FieldType": "int32", "IsNullable": false }, { "FieldName": "FileName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FileSize", "FieldType": "int64", "IsNullable": false }, { "FieldName": "FileModTime", "FieldType": "datetime", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "AppliedFunctions", "FieldType": "FeedConnectionRecordType_AppliedFunctions", "IsNullable": false }, { "FieldName": "PolicyName", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedConnectionRecordType_AppliedFunctions", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "PolicyName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Description", "FieldType": "string", "IsNullable": false }, { "FieldName": "Properties", "FieldType": "FeedPolicyRecordType_Properties", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType_Properties", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedPolicyRecordType_Properties_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType_Properties_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "FeedName", "FieldType": "string", "IsNullable": false }, { "FieldName": "AdapterConfiguration", "FieldType": "FeedRecordType_AdapterConfiguration", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "FeedRecordType_AdapterConfiguration_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType_AdapterConfiguration_Item", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Value", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Arity", "FieldType": "string", "IsNullable": false }, { "FieldName": "Params", "FieldType": "FunctionRecordType_Params", "IsNullable": false }, { "FieldName": "ReturnType", "FieldType": "string", "IsNullable": false }, { "FieldName": "Definition", "FieldType": "string", "IsNullable": false }, { "FieldName": "Language", "FieldType": "string", "IsNullable": false }, { "FieldName": "Kind", "FieldType": "string", "IsNullable": false }, { "FieldName": "Dependencies", "FieldType": "FunctionRecordType_Dependencies", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Dependencies", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "FunctionRecordType_Dependencies_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Dependencies_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "FunctionRecordType_Dependencies_Item_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Dependencies_Item_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType_Params", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "DatasetName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexName", "FieldType": "string", "IsNullable": false }, { "FieldName": "IndexStructure", "FieldType": "string", "IsNullable": false }, { "FieldName": "SearchKey", "FieldType": "IndexRecordType_SearchKey", "IsNullable": false }, { "FieldName": "IsPrimary", "FieldType": "boolean", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false }, { "FieldName": "PendingOp", "FieldType": "int32", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "IndexRecordType_SearchKey_Item" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType_SearchKey_Item", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "OrderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string", "IsNullable": false }, { "FieldName": "Name", "FieldType": "string", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string", "IsNullable": false }, { "FieldName": "NodeNames", "FieldType": "NodeGroupRecordType_NodeNames", "IsNullable": false }, { "FieldName": "Timestamp", "FieldType": "string", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType_NodeNames", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "UnorderedList": "string" }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string", "IsNullable": false }, { "FieldName": "NumberOfCores", "FieldType": "int64", "IsNullable": false }, { "FieldName": "WorkingMemorySize", "FieldType": "int64", "IsNullable": false } ] } }, "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "binary", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "circle", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "date", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "double", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "duration", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "float", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "int16", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "int32", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "int64", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "int8", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "interval", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "line", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "missing", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "null", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "point", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "shortwithouttypeinfo", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "string", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "time", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } +{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Timestamp": "Mon Jan 08 10:27:04 PST 2018" } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp new file mode 100644 index 0000000..c9639c4 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.1.ddl.sqlpp @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Verify Function Dependency Metadata + * Expected Res : Success + * Date : Dec 15th 2017 + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse B; +create dataverse C; +use C; + +create type TweetMessageType as closed { + tweetid: uuid, + sender_location: point, + send_time: datetime, + referred_topics: {{ string }}, + message_text: string, + countA: int32, + countB: int32 +}; + +create dataset TweetMessages(TweetMessageType) +primary key tweetid autogenerated; + +create function f1(message, text){ + contains(message,text) +}; + +create function f4(){ +(select * from TweetMessages) +}; + +use B; + +create dataset TweetMessages2(C.TweetMessageType) +primary key tweetid autogenerated; + +create function f0(message, text){ + contains(message,text) +}; + +create function C.f2(place, text) { + (select m.message_text + from TweetMessages m + where contains(m.message_text,text) + and spatial_intersect(m.sender_location, place) + and f1(m.message_text,text) + and B.f0(m.message_text,text)) +}; + +create function C.f3(place, text) { + f2(place, text) +}; + +create function f5(place, text){ + (select m.message_text + from TweetMessages2 m, C.TweetMessages m2 + where contains(m.message_text,text) + and spatial_intersect(m.sender_location, place) + and C.f1(m.message_text,text) + and f0(m2.message_text,text)) +}; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp new file mode 100644 index 0000000..e9ab11e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.2.query.sqlpp @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Verify Function Dependency Metadata + * Expected Res : Success + * Date : Dec 15th 2017 + */ + +select f.DataverseName,f.Name,f.Dependencies from Metadata.`Function` f +order by f.DataverseName, f.Name; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.3.ddl.sqlpp new file mode 100644 index 0000000..215118b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/check-dependencies-1/check-dependencies-1.3.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Description : Verify Function Dependency Metadata + * Expected Res : Success + * Date : Dec 15th 2017 + */ + +drop function B.f5@2; +drop function C.f3@2; +drop function C.f2@2; + +drop dataset B.TweetMessages2; + +drop dataverse C; +drop dataverse B; + + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp new file mode 100644 index 0000000..08ad454 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-1/drop-dependency.1.ddl.sqlpp @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse B; +create dataverse C; +use C; + +create function f1(message, text){ + contains(message,text) +}; + +use B; + +create function f0(message, text){ + C.f1(message,text) +}; + +drop dataverse C; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp new file mode 100644 index 0000000..53debdf --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-2/drop-dependency.2.ddl.sqlpp @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse B; +create dataverse C; +use C; + +create type TweetMessageType as closed { + tweetid: uuid, + sender_location: point, + send_time: datetime, + referred_topics: {{ string }}, + message_text: string, + countA: int32, + countB: int32 +}; + +create dataset TweetMessages(TweetMessageType) +primary key tweetid autogenerated; + +create function f1(message, text){ + contains(message,text) +}; + +use B; + +create function f2(place, text){ + (select m.message_text + from C.TweetMessages m) +}; + +drop dataverse C; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp new file mode 100644 index 0000000..7e7b60e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-3/drop-dependency.3.ddl.sqlpp @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse B; +create dataverse C; +use C; + +create function f1(message, text){ + contains(message,text) +}; + +use B; + +create function f0(message, text){ + C.f1(message,text) +}; + +use C; +drop function f1@2; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp new file mode 100644 index 0000000..c972efc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-4/drop-dependency.4.ddl.sqlpp @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse B; +create dataverse C; +use C; + +create type TweetMessageType as closed { + tweetid: uuid, + sender_location: point, + send_time: datetime, + referred_topics: {{ string }}, + message_text: string, + countA: int32, + countB: int32 +}; + +create dataset TweetMessages(TweetMessageType) +primary key tweetid autogenerated; + +create function f1(message, text){ + contains(message,text) +}; + +use B; + +create function f2(place, text){ + (select m.message_text + from C.TweetMessages m) +}; + +use C; +drop dataset TweetMessages; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp new file mode 100644 index 0000000..3a7f4dc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-5/drop-dependency.5.ddl.sqlpp @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse C; +use C; + +create function f1(message, text){ + contains(message,text) +}; + +create function f0(message, text){ + C.f1(message,text) +}; + +drop function f1@2; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp new file mode 100644 index 0000000..3ab7b6f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/user-defined-functions/drop-dependency-6/drop-dependency.6.ddl.sqlpp @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under A + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Try to drop a functional dependency + * Expected Res : Error + */ + +drop dataverse B if exists; +drop dataverse C if exists; +create dataverse C; +use C; + +create type TweetMessageType as closed { + tweetid: uuid, + sender_location: point, + send_time: datetime, + referred_topics: {{ string }}, + message_text: string, + countA: int32, + countB: int32 +}; + +create dataset TweetMessages(TweetMessageType) +primary key tweetid autogenerated; + +create function f1(message, text){ + contains(message,text) +}; + +create function f2(place, text){ + (select m.message_text + from C.TweetMessages m) +}; + +drop dataset TweetMessages; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm index 8f4c581..403237e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/validate-default-library/validate-default-library.1.adm @@ -1,8 +1,8 @@ -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR" } } -{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR" } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#addHashTags", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#addHashTagsInPlace", "Arity": "1", "Params": [ "Tweet" ], "ReturnType": "ProcessedTweet", "Definition": "org.apache.asterix.external.library.AddHashTagsInPlaceFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#allTypes", "Arity": "1", "Params": [ "AllType" ], "ReturnType": "AllType", "Definition": "org.apache.asterix.external.library.AllTypesFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#echoDelay", "Arity": "1", "Params": [ "TweetMessageType" ], "ReturnType": "TweetMessageType", "Definition": "org.apache.asterix.external.library.EchoDelayFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#getCapital", "Arity": "1", "Params": [ "ASTRING" ], "ReturnType": "CountryCapitalType", "Definition": "org.apache.asterix.external.library.CapitalFinderFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#mysum", "Arity": "2", "Params": [ "AINT32", "AINT32" ], "ReturnType": "AINT32", "Definition": "org.apache.asterix.external.library.SumFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#parseTweet", "Arity": "1", "Params": [ "TweetInputType" ], "ReturnType": "TweetOutputType", "Definition": "org.apache.asterix.external.library.ParseTweetFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } +{ "Function": { "DataverseName": "externallibtest", "Name": "testlib#toUpper", "Arity": "1", "Params": [ "TextType" ], "ReturnType": "TextType", "Definition": "org.apache.asterix.external.library.UpperCaseFactory", "Language": "JAVA", "Kind": "SCALAR", "Dependencies": [ [ ], [ ] ] } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm new file mode 100644 index 0000000..50f19bb --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/user-defined-functions/check-dependencies-1/check-dependencies-1.1.adm @@ -0,0 +1,6 @@ +{ "DataverseName": "B", "Name": "f0", "Dependencies": [ [ ], [ ] ] } +{ "DataverseName": "B", "Name": "f5", "Dependencies": [ [ [ "C", "TweetMessages" ], [ "B", "TweetMessages2" ] ], [ [ "C", "f1", "2" ], [ "B", "f0", "2" ] ] ] } +{ "DataverseName": "C", "Name": "f1", "Dependencies": [ [ ], [ ] ] } +{ "DataverseName": "C", "Name": "f2", "Dependencies": [ [ [ "C", "TweetMessages" ] ], [ [ "C", "f1", "2" ], [ "B", "f0", "2" ] ] ] } +{ "DataverseName": "C", "Name": "f3", "Dependencies": [ [ ], [ [ "C", "f2", "2" ] ] ] } +{ "DataverseName": "C", "Name": "f4", "Dependencies": [ [ [ "C", "TweetMessages" ] ], [ ] ] } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index fd47766..14784d6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -7922,7 +7922,7 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="bad-function-ddl-9"> <output-dir compare="Text">bad-function-ddl-9</output-dir> - <expected-error>function experiments.good_function@1 depends upon function experiments.function_that_does_not_exist@0 which is undefined</expected-error> + <expected-error>function experiments.function_that_does_not_exist@0 is not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> @@ -7932,6 +7932,47 @@ </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> + <compilation-unit name="check-dependencies-1"> + <output-dir compare="Text">check-dependencies-1</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-1"> + <output-dir compare="Text">drop-dependency-1</output-dir> + <expected-error>Cannot drop dataverse. Function B.f0@2 depends on function C.f1@2</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-2"> + <output-dir compare="Text">drop-dependency-2</output-dir> + <expected-error>Cannot drop dataverse. Function B.f2@2 depends on dataset C.TweetMessages</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-3"> + <output-dir compare="Text">drop-dependency-3</output-dir> + <expected-error>Cannot drop function C.f1@2 being used by function B.f0@2</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-4"> + <output-dir compare="Text">drop-dependency-4</output-dir> + <expected-error>Cannot drop dataset C.TweetMessages being used by function B.f2@2</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-5"> + <output-dir compare="Text">drop-dependency-5</output-dir> + <expected-error>Cannot drop function C.f1@2 being used by function C.f0@2</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> + <compilation-unit name="drop-dependency-6"> + <output-dir compare="Text">drop-dependency-6</output-dir> + <expected-error>Cannot drop dataset C.TweetMessages being used by function C.f2@2</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="user-defined-functions"> <compilation-unit name="single-line-definition"> <output-dir compare="Text">single-line-definition</output-dir> </compilation-unit> diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java index 977f6bb..6ab69ad 100644 --- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java +++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java @@ -40,6 +40,7 @@ import org.apache.asterix.lang.common.base.IReturningStatement; import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; +import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair; import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; @@ -68,7 +69,8 @@ @Override public void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement topStatement, - MetadataProvider metadataProvider, LangRewritingContext context) throws CompilationException { + MetadataProvider metadataProvider, LangRewritingContext context, boolean inlineUdfs) + throws CompilationException { setup(declaredFunctions, topStatement, metadataProvider, context); if (topStatement.isTopLevel()) { wrapInLets(); @@ -130,7 +132,8 @@ declaredFunctions.removeAll(storedFunctionDecls); } - private Set<FunctionSignature> getFunctionCalls(Expression expression) throws CompilationException { + @Override + public Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException { GatherFunctionCalls gfc = new GatherFunctionCalls(); expression.accept(gfc, null); return gfc.getCalls(); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java index 0c6c04c..f072917 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java @@ -19,8 +19,10 @@ package org.apache.asterix.lang.common.base; import java.util.List; +import java.util.Set; import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.metadata.declared.MetadataProvider; @@ -39,6 +41,13 @@ * @param context, * manages ids of variables and guarantees uniqueness of variables. */ - public void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement topExpr, - MetadataProvider metadataProvider, LangRewritingContext context) throws CompilationException; + void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement topExpr, + MetadataProvider metadataProvider, LangRewritingContext context, boolean inlineUdfs) + throws CompilationException; + + /** + * Find the function calls used by a given expression + */ + Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException; + } diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java index 1ca9316..3571dcf 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java @@ -20,6 +20,7 @@ package org.apache.asterix.lang.common.util; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -27,13 +28,18 @@ import org.apache.asterix.common.functions.FunctionConstants; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.lang.common.base.Expression; +import org.apache.asterix.lang.common.base.IQueryRewriter; +import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.LiteralExpr; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Function; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo; @@ -52,7 +58,7 @@ @FunctionalInterface public interface IFunctionCollector { - Set<FunctionSignature> getFunctionCalls(Expression expression) throws CompilationException; + Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException; } @FunctionalInterface @@ -95,8 +101,9 @@ } String value = metadataProvider.getConfig().get(FunctionUtil.IMPORT_PRIVATE_FUNCTIONS); boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false; - Set<FunctionSignature> functionCalls = functionCollector.getFunctionCalls(expression); - for (FunctionSignature signature : functionCalls) { + Set<CallExpr> functionCalls = functionCollector.getFunctionCalls(expression); + for (CallExpr functionCall : functionCalls) { + FunctionSignature signature = functionCall.getFunctionSignature(); if (declaredFunctions != null && declaredFunctions.contains(signature)) { continue; } @@ -155,6 +162,35 @@ return functionDecls; } + public static List<List<List<String>>> getFunctionDependencies(IQueryRewriter rewriter, Expression expression, + MetadataProvider metadataProvider) throws CompilationException { + Set<CallExpr> functionCalls = rewriter.getFunctionCalls(expression); + //Get the List of used functions and used datasets + List<List<String>> datasourceDependencies = new ArrayList<>(); + List<List<String>> functionDependencies = new ArrayList<>(); + for (CallExpr functionCall : functionCalls) { + FunctionSignature signature = functionCall.getFunctionSignature(); + FunctionIdentifier fid = + new FunctionIdentifier(signature.getNamespace(), signature.getName(), signature.getArity()); + if (fid.equals(BuiltinFunctions.DATASET)) { + Pair<String, String> path = DatasetUtil.getDatasetInfo(metadataProvider, + ((LiteralExpr) functionCall.getExprList().get(0)).getValue().getStringValue()); + datasourceDependencies.add(Arrays.asList(path.first, path.second)); + } + + else if (BuiltinFunctions.isBuiltinCompilerFunction(signature, false)) { + continue; + } else { + functionDependencies.add(Arrays.asList(signature.getNamespace(), signature.getName(), + Integer.toString(signature.getArity()))); + } + } + List<List<List<String>>> dependencies = new ArrayList<>(); + dependencies.add(datasourceDependencies); + dependencies.add(functionDependencies); + return dependencies; + } + private static Function lookupUserDefinedFunctionDecl(MetadataTransactionContext mdTxnCtx, FunctionSignature signature) throws AlgebricksException { if (signature.getNamespace() == null) { diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java index 7d8c11e..8ae67d6 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java @@ -347,7 +347,7 @@ metadataProvider.setDefaultDataverse(fnDataverse); try { IQueryRewriter queryRewriter = rewriterFactory.createQueryRewriter(); - queryRewriter.rewrite(declaredFunctions, wrappedQuery, metadataProvider, context); + queryRewriter.rewrite(declaredFunctions, wrappedQuery, metadataProvider, context, true); return wrappedQuery.getBody(); } finally { metadataProvider.setDefaultDataverse(defaultDataverse); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java index 8842d86..3d149fb 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java @@ -23,7 +23,6 @@ import java.util.Set; import org.apache.asterix.common.exceptions.CompilationException; -import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.clause.GroupbyClause; import org.apache.asterix.lang.common.clause.LetClause; @@ -53,11 +52,11 @@ public class GatherFunctionCallsVisitor extends AbstractQueryExpressionVisitor<Void, Void> { - protected final Set<FunctionSignature> calls = new HashSet<FunctionSignature>(); + protected final Set<CallExpr> calls = new HashSet<>(); @Override public Void visit(CallExpr pf, Void arg) throws CompilationException { - calls.add(pf.getFunctionSignature()); + calls.add(pf); for (Expression e : pf.getExprList()) { e.accept(this, arg); } @@ -202,7 +201,7 @@ return null; } - public Set<FunctionSignature> getCalls() { + public Set<CallExpr> getCalls() { return calls; } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java index a8b3dc6..cd57396 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppFunctionBodyRewriter.java @@ -31,7 +31,7 @@ @Override public void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement topStatement, MetadataProvider metadataProvider, - LangRewritingContext context) throws CompilationException { + LangRewritingContext context, boolean inlineUdfs) throws CompilationException { // Sets up parameters. setup(declaredFunctions, topStatement, metadataProvider, context); @@ -63,6 +63,6 @@ rewriteListInputFunctions(); // Inlines functions recursively. - inlineDeclaredUdfs(); + inlineDeclaredUdfs(inlineUdfs); } } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java index 3e9a873..befa5ab 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java @@ -28,6 +28,7 @@ import org.apache.asterix.lang.common.base.IQueryRewriter; import org.apache.asterix.lang.common.base.IReturningStatement; import org.apache.asterix.lang.common.clause.LetClause; +import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.util.FunctionUtil; @@ -86,7 +87,8 @@ @Override public void rewrite(List<FunctionDecl> declaredFunctions, IReturningStatement topStatement, - MetadataProvider metadataProvider, LangRewritingContext context) throws CompilationException { + MetadataProvider metadataProvider, LangRewritingContext context, boolean inlineUdfs) + throws CompilationException { if (topStatement == null) { return; } @@ -126,7 +128,7 @@ rewriteListInputFunctions(); // Inlines functions. - inlineDeclaredUdfs(); + inlineDeclaredUdfs(inlineUdfs); // Rewrites function names. // This should be done after inlineDeclaredUdfs() because user-defined function @@ -213,7 +215,7 @@ topExpr.accept(groupByVisitor, null); } - protected void inlineDeclaredUdfs() throws CompilationException { + protected void inlineDeclaredUdfs(boolean inlineUdfs) throws CompilationException { List<FunctionSignature> funIds = new ArrayList<FunctionSignature>(); for (FunctionDecl fdecl : declaredFunctions) { funIds.add(fdecl.getSignature()); @@ -226,7 +228,7 @@ signature -> FunctionMapUtil.normalizeBuiltinFunctionSignature(signature, false))); } declaredFunctions.addAll(usedStoredFunctionDecls); - if (!declaredFunctions.isEmpty()) { + if (inlineUdfs && !declaredFunctions.isEmpty()) { SqlppInlineUdfsVisitor visitor = new SqlppInlineUdfsVisitor(context, new SqlppFunctionBodyRewriterFactory() /* the rewriter for function bodies expressions*/, declaredFunctions, metadataProvider); @@ -237,7 +239,8 @@ declaredFunctions.removeAll(usedStoredFunctionDecls); } - private Set<FunctionSignature> getFunctionCalls(Expression expression) throws CompilationException { + @Override + public Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException { GatherFunctionCalls gfc = new GatherFunctionCalls(); expression.accept(gfc, null); return gfc.getCalls(); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index ed3bbe0..6634e51 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -268,7 +268,6 @@ } } - @SuppressWarnings("unchecked") @Override public <T extends IExtensionMetadataEntity> void addEntity(TxnId txnId, T entity) throws AlgebricksException, RemoteException { @@ -280,7 +279,6 @@ addEntity(txnId, entity, tupleTranslator, index); } - @SuppressWarnings("unchecked") @Override public <T extends IExtensionMetadataEntity> void upsertEntity(TxnId txnId, T entity) throws AlgebricksException, RemoteException { @@ -292,7 +290,6 @@ upsertEntity(txnId, entity, tupleTranslator, index); } - @SuppressWarnings("unchecked") @Override public <T extends IExtensionMetadataEntity> void deleteEntity(TxnId txnId, T entity) throws AlgebricksException, RemoteException { @@ -304,7 +301,6 @@ deleteEntity(txnId, entity, tupleTranslator, index); } - @SuppressWarnings("unchecked") @Override public <T extends IExtensionMetadataEntity> List<T> getEntities(TxnId txnId, IExtensionMetadataSearchKey searchKey) throws AlgebricksException, RemoteException { @@ -527,13 +523,22 @@ confirmDataverseCanBeDeleted(txnId, dataverseName); + // As a side effect, acquires an S lock on the 'Function' dataset + // on behalf of txnId. + List<Function> dataverseFunctions = getDataverseFunctions(txnId, dataverseName); + // Drop all functions in this dataverse. + for (Function function : dataverseFunctions) { + dropFunction(txnId, new FunctionSignature(dataverseName, function.getName(), function.getArity()), + true); + } + List<Dataset> dataverseDatasets; Dataset ds; dataverseDatasets = getDataverseDatasets(txnId, dataverseName); // Drop all datasets in this dataverse. for (int i = 0; i < dataverseDatasets.size(); i++) { ds = dataverseDatasets.get(i); - dropDataset(txnId, dataverseName, ds.getDatasetName()); + dropDataset(txnId, dataverseName, ds.getDatasetName(), true); } // After dropping datasets, drop datatypes @@ -544,14 +549,6 @@ // Drop all types in this dataverse. for (int i = 0; i < dataverseDatatypes.size(); i++) { forceDropDatatype(txnId, dataverseName, dataverseDatatypes.get(i).getDatatypeName()); - } - - // As a side effect, acquires an S lock on the 'Function' dataset - // on behalf of txnId. - List<Function> dataverseFunctions = getDataverseFunctions(txnId, dataverseName); - // Drop all functions in this dataverse. - for (Function function : dataverseFunctions) { - dropFunction(txnId, new FunctionSignature(dataverseName, function.getName(), function.getArity())); } // As a side effect, acquires an S lock on the 'Adapter' dataset @@ -609,6 +606,16 @@ @Override public void dropDataset(TxnId txnId, String dataverseName, String datasetName) throws AlgebricksException, RemoteException { + dropDataset(txnId, dataverseName, datasetName, false); + } + + public void dropDataset(TxnId txnId, String dataverseName, String datasetName, boolean force) + throws AlgebricksException, RemoteException { + + if (!force) { + confirmDatasetCanBeDeleted(txnId, dataverseName, datasetName); + } + Dataset dataset = getDataset(txnId, dataverseName, datasetName); if (dataset == null) { throw new AlgebricksException("Cannot drop dataset '" + datasetName + "' because it doesn't exist."); @@ -902,6 +909,19 @@ } } + public List<Function> getAllFunctions(TxnId txnId) throws AlgebricksException, RemoteException { + try { + ITupleReference searchKey = null; + FunctionTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFunctionTupleTranslator(false); + IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter); + List<Function> results = new ArrayList<>(); + searchIndex(txnId, MetadataPrimaryIndexes.FUNCTION_DATASET, searchKey, valueExtractor, results); + return results; + } catch (HyracksDataException e) { + throw new AlgebricksException(e); + } + } + public List<Datatype> getAllDatatypes(TxnId txnId) throws AlgebricksException, RemoteException { try { ITupleReference searchKey = null; @@ -930,6 +950,62 @@ throw new AlgebricksException( "Cannot drop dataverse. Type " + dataverseName + "." + set.getItemTypeName() + " used by dataset " + set.getDataverseName() + "." + set.getDatasetName()); + } + } + + // If a function from a DIFFERENT dataverse + // uses functions or datatypes from this dataverse + // throw an error + List<Function> functions = getAllFunctions(txnId); + for (Function function : functions) { + if (function.getDataverseName().equals(dataverseName)) { + continue; + } + for (List<String> datasetDependency : function.getDependencies().get(0)) { + if (datasetDependency.get(0).equals(dataverseName)) { + throw new AlgebricksException("Cannot drop dataverse. Function " + function.getDataverseName() + "." + + function.getName() + "@" + function.getArity() + " depends on dataset " + + datasetDependency.get(0) + "." + datasetDependency.get(1)); + } + } + for (List<String> functionDependency : function.getDependencies().get(1)) { + if (functionDependency.get(0).equals(dataverseName)) { + throw new AlgebricksException( + "Cannot drop dataverse. Function " + function.getDataverseName() + "." + function.getName() + + "@" + function.getArity() + " depends on function " + functionDependency.get(0) + + "." + functionDependency.get(1) + "@" + functionDependency.get(2)); + } + } + } + } + + private void confirmFunctionCanBeDeleted(TxnId txnId, FunctionSignature signature) + throws AlgebricksException, RemoteException { + // If any other function uses this function, throw an error + List<Function> functions = getAllFunctions(txnId); + for (Function function : functions) { + for (List<String> functionalDependency : function.getDependencies().get(1)) { + if (functionalDependency.get(0).equals(signature.getNamespace()) + && functionalDependency.get(1).equals(signature.getName()) + && functionalDependency.get(2).equals(Integer.toString(signature.getArity()))) { + throw new AlgebricksException("Cannot drop function " + signature + " being used by function " + + function.getDataverseName() + "." + function.getName() + "@" + function.getArity()); + } + } + } + } + + private void confirmDatasetCanBeDeleted(TxnId txnId, String dataverseName, String datasetName) + throws AlgebricksException, RemoteException { + // If any function uses this type, throw an error + List<Function> functions = getAllFunctions(txnId); + for (Function function : functions) { + for (List<String> datasetDependency : function.getDependencies().get(0)) { + if (datasetDependency.get(0).equals(dataverseName) && datasetDependency.get(1).equals(datasetName)) { + throw new AlgebricksException("Cannot drop dataset " + dataverseName + "." + datasetName + + " being used by function " + function.getDataverseName() + "." + function.getName() + "@" + + function.getArity()); + } } } } @@ -1123,6 +1199,15 @@ @Override public void dropFunction(TxnId txnId, FunctionSignature functionSignature) throws AlgebricksException, RemoteException { + dropFunction(txnId, functionSignature, false); + } + + private void dropFunction(TxnId txnId, FunctionSignature functionSignature, boolean force) + throws AlgebricksException, RemoteException { + + if (!force) { + confirmFunctionCanBeDeleted(txnId, functionSignature); + } Function function = getFunction(txnId, functionSignature); @@ -1329,7 +1414,6 @@ // TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer // Hyracks version. public static ITupleReference createTuple(String... fields) { - @SuppressWarnings("unchecked") ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING); AMutableString aString = new AMutableString(""); @@ -1836,7 +1920,6 @@ // This method is used to create a search tuple for external data file since the // search tuple has an int value - @SuppressWarnings("unchecked") public ITupleReference createExternalFileSearchTuple(String dataverseName, String datasetName, int fileNumber) throws HyracksDataException { ISerializerDeserializer<AString> stringSerde = diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java index 841065b..cb67ee5 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java @@ -158,7 +158,7 @@ public void dropFunction(FunctionSignature signature) { Function function = new Function(signature.getNamespace(), signature.getName(), signature.getArity(), null, - null, null, null, null); + null, null, null, null, null); droppedCache.addFunctionIfNotExists(function); logAndApply(new MetadataLogicalOperation(function, false)); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java index e0f925c..e2724e2 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java @@ -48,6 +48,7 @@ public static final String FIELD_NAME_DATAVERSE_NAME = "DataverseName"; public static final String FIELD_NAME_DATA_FORMAT = "DataFormat"; public static final String FIELD_NAME_DEFINITION = "Definition"; + public static final String FIELD_NAME_DEPENDENCIES = "Dependencies"; public static final String FIELD_NAME_DERIVED = "Derived"; public static final String FIELD_NAME_DESCRIPTION = "Description"; public static final String FIELD_NAME_EXTERNAL_DETAILS = "ExternalDetails"; @@ -321,16 +322,20 @@ public static final int FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX = 5; public static final int FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX = 6; public static final int FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX = 7; + public static final int FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX = 8; public static final ARecordType FUNCTION_RECORDTYPE = createRecordType( // RecordTypeName RECORD_NAME_FUNCTION, // FieldNames new String[] { FIELD_NAME_DATAVERSE_NAME, FIELD_NAME_NAME, FIELD_NAME_ARITY, FIELD_NAME_PARAMS, - FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND }, + FIELD_NAME_RETURN_TYPE, FIELD_NAME_DEFINITION, FIELD_NAME_LANGUAGE, FIELD_NAME_KIND, + FIELD_NAME_DEPENDENCIES }, // FieldTypes new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING, new AOrderedListType(BuiltinType.ASTRING, null), BuiltinType.ASTRING, BuiltinType.ASTRING, - BuiltinType.ASTRING, BuiltinType.ASTRING }, + BuiltinType.ASTRING, BuiltinType.ASTRING, + new AOrderedListType(new AOrderedListType(new AOrderedListType(BuiltinType.ASTRING, null), null), + null) }, //IsOpen? true); //------------------------------------------ Adapter ----------------------------------------// diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java index 1d1db37..8bdbaba 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.metadata.entities; +import java.util.ArrayList; import java.util.List; import org.apache.asterix.metadata.MetadataCache; @@ -36,13 +37,14 @@ private final String name; private final int arity; private final List<String> params; + private final List<List<List<String>>> dependencies; private final String body; private final String returnType; private final String language; private final String kind; public Function(String dataverseName, String functionName, int arity, List<String> params, String returnType, - String functionBody, String language, String functionKind) { + String functionBody, String language, String functionKind, List<List<List<String>>> dependencies) { this.dataverse = dataverseName; this.name = functionName; this.params = params; @@ -51,6 +53,13 @@ this.language = language; this.kind = functionKind; this.arity = arity; + if (dependencies == null) { + this.dependencies = new ArrayList<>(); + this.dependencies.add(new ArrayList<>()); + this.dependencies.add(new ArrayList<>()); + } else { + this.dependencies = dependencies; + } } public String getDataverseName() { @@ -65,6 +74,10 @@ return params; } + public List<List<List<String>>> getDependencies() { + return dependencies; + } + public String getFunctionBody() { return body; } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java index e15805e..273c78a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java @@ -35,6 +35,7 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.base.IACursor; import org.apache.asterix.om.types.AOrderedListType; +import org.apache.asterix.om.types.BuiltinType; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -56,7 +57,13 @@ // Payload field containing serialized Function. public static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3; - @SuppressWarnings("unchecked") + private transient OrderedListBuilder dependenciesListBuilder = new OrderedListBuilder(); + private transient OrderedListBuilder dependencyListBuilder = new OrderedListBuilder(); + private transient OrderedListBuilder dependencyNameListBuilder = new OrderedListBuilder(); + private transient AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null); + private transient AOrderedListType ListofLists = + new AOrderedListType(new AOrderedListType(BuiltinType.ASTRING, null), null); + private ISerializerDeserializer<ARecord> recordSerDes = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(MetadataRecordTypes.FUNCTION_RECORDTYPE); @@ -104,8 +111,33 @@ String functionKind = ((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX)) .getStringValue(); + + IACursor dependenciesCursor = ((AOrderedList) functionRecord + .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX)).getCursor(); + List<List<List<String>>> dependencies = new ArrayList<>(); + AOrderedList dependencyList; + AOrderedList qualifiedList; + int i = 0; + while (dependenciesCursor.next()) { + dependencies.add(new ArrayList<>()); + dependencyList = (AOrderedList) dependenciesCursor.get(); + IACursor qualifiedDependencyCursor = dependencyList.getCursor(); + int j = 0; + while (qualifiedDependencyCursor.next()) { + qualifiedList = (AOrderedList) qualifiedDependencyCursor.get(); + IACursor qualifiedNameCursor = qualifiedList.getCursor(); + dependencies.get(i).add(new ArrayList<>()); + while (qualifiedNameCursor.next()) { + dependencies.get(i).get(j).add(((AString) qualifiedNameCursor.get()).getStringValue()); + } + j++; + } + i++; + + } + return new Function(dataverseName, functionName, Integer.parseInt(arity), params, returnType, definition, - language, functionKind); + language, functionKind, dependencies); } @@ -185,6 +217,33 @@ stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX, fieldValue); + // write field 8 + dependenciesListBuilder.reset((AOrderedListType) MetadataRecordTypes.FUNCTION_RECORDTYPE + .getFieldTypes()[MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX]); + List<List<List<String>>> dependenciesList = function.getDependencies(); + for (List<List<String>> dependencies : dependenciesList) { + dependencyListBuilder.reset(ListofLists); + for (List<String> dependency : dependencies) { + dependencyNameListBuilder.reset(stringList); + for (String subName : dependency) { + itemValue.reset(); + aString.setValue(subName); + stringSerde.serialize(aString, itemValue.getDataOutput()); + dependencyNameListBuilder.addItem(itemValue); + } + itemValue.reset(); + dependencyNameListBuilder.write(itemValue.getDataOutput(), true); + dependencyListBuilder.addItem(itemValue); + + } + itemValue.reset(); + dependencyListBuilder.write(itemValue.getDataOutput(), true); + dependenciesListBuilder.addItem(itemValue); + } + fieldValue.reset(); + dependenciesListBuilder.write(fieldValue.getDataOutput(), true); + recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEPENDENCIES_FIELD_INDEX, fieldValue); + // write record recordBuilder.write(tupleBuilder.getDataOutput(), true); tupleBuilder.addFieldEndOffset(); -- To view, visit https://asterix-gerrit.ics.uci.edu/2253 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2f08ff150dfd57432b88381c507814ddb57bd67b Gerrit-PatchSet: 19 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Steven Jacobs <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Dmitry Lychagin <[email protected]> Gerrit-Reviewer: Ildar Absalyamov <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Steven Jacobs <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]>
