>From Murtadha Hubail <[email protected]>: Murtadha Hubail has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212 )
Change subject: [ASTERIXDB-3370][COMP] Collect accessed entities during compilation ...................................................................... [ASTERIXDB-3370][COMP] Collect accessed entities during compilation - user model changes: no - storage format changes: no - interface changes: no Details: - Collect datasets/views/functions/link for copy-to statement - Add visitor to collect accessed datasets/views/functions/copyto Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Jenkins <[email protected]> --- A asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java A asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java 5 files changed, 211 insertions(+), 8 deletions(-) Approvals: Murtadha Hubail: Looks good to me, approved Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 6424280..3e674fd 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -448,16 +448,16 @@ handleLibraryDropStatement(metadataProvider, stmt, hcc, requestParameters); break; case CREATE_SYNONYM: - handleCreateSynonymStatement(metadataProvider, stmt); + handleCreateSynonymStatement(metadataProvider, stmt, requestParameters); break; case SYNONYM_DROP: - handleDropSynonymStatement(metadataProvider, stmt); + handleDropSynonymStatement(metadataProvider, stmt, requestParameters); break; case CREATE_VIEW: handleCreateViewStatement(metadataProvider, stmt, stmtRewriter, requestParameters); break; case VIEW_DROP: - handleViewDropStatement(metadataProvider, stmt); + handleViewDropStatement(metadataProvider, stmt, requestParameters); break; case LOAD: if (stats.getProfileType() == Stats.ProfileType.FULL) { @@ -2963,7 +2963,8 @@ } } - public void handleViewDropStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { + public void handleViewDropStatement(MetadataProvider metadataProvider, Statement stmt, + IRequestParameters requestParameters) throws Exception { ViewDropStatement stmtDrop = (ViewDropStatement) stmt; SourceLocation sourceLoc = stmtDrop.getSourceLocation(); String viewName = stmtDrop.getViewName().getValue(); @@ -2976,14 +2977,14 @@ } lockUtil.dropDatasetBegin(lockManager, metadataProvider.getLocks(), databaseName, dataverseName, viewName); try { - doDropView(metadataProvider, stmtDrop, databaseName, dataverseName, viewName); + doDropView(metadataProvider, stmtDrop, databaseName, dataverseName, viewName, requestParameters); } finally { metadataProvider.getLocks().unlock(); } } protected boolean doDropView(MetadataProvider metadataProvider, ViewDropStatement stmtViewDrop, String databaseName, - DataverseName dataverseName, String viewName) throws Exception { + DataverseName dataverseName, String viewName, IRequestParameters requestParameters) throws Exception { SourceLocation sourceLoc = stmtViewDrop.getSourceLocation(); MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); metadataProvider.setMetadataTxnContext(mdTxnCtx); @@ -3787,7 +3788,8 @@ } } - protected void handleCreateSynonymStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { + protected void handleCreateSynonymStatement(MetadataProvider metadataProvider, Statement stmt, + IRequestParameters requestParameters) throws Exception { CreateSynonymStatement css = (CreateSynonymStatement) stmt; metadataProvider.validateDatabaseObjectName(css.getNamespace(), css.getSynonymName(), css.getSourceLocation()); Namespace stmtActiveNamespace = getActiveNamespace(css.getNamespace()); @@ -3843,7 +3845,8 @@ } } - protected void handleDropSynonymStatement(MetadataProvider metadataProvider, Statement stmt) throws Exception { + protected void handleDropSynonymStatement(MetadataProvider metadataProvider, Statement stmt, + IRequestParameters requestParameters) throws Exception { SynonymDropStatement stmtSynDrop = (SynonymDropStatement) stmt; String synonymName = stmtSynDrop.getSynonymName(); metadataProvider.validateDatabaseObjectName(stmtSynDrop.getNamespace(), synonymName, diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java index cbbb9de..183b5e0 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java @@ -73,6 +73,7 @@ import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupingSetsVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor; +import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppLoadAccessedDataset; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppRightJoinRewriteVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppSpecialFunctionNameRewriteVisitor; import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor; @@ -200,6 +201,9 @@ // Rewrites RIGHT OUTER JOINs into LEFT OUTER JOINs if possible rewriteRightJoins(); + // Load all the accessed datasets + loadAccessedDatasets(); + // Inlines functions and views loadAndInlineUdfsAndViews(); @@ -336,6 +340,11 @@ rewriteTopExpr(visitor, null); } + protected void loadAccessedDatasets() throws CompilationException { + SqlppLoadAccessedDataset visitor = new SqlppLoadAccessedDataset(context); + rewriteTopExpr(visitor, null); + } + protected void loadAndInlineUdfsAndViews() throws CompilationException { Pair<Map<FunctionSignature, FunctionDecl>, Map<DatasetFullyQualifiedName, ViewDecl>> udfAndViewDecls = loadUdfsAndViews(topStatement); diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java new file mode 100644 index 0000000..355b484 --- /dev/null +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.lang.sqlpp.rewrites.visitor; + +import java.util.List; +import java.util.Map; + +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.lang.common.base.Expression; +import org.apache.asterix.lang.common.base.ILangExpression; +import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.rewrites.LangRewritingContext; +import org.apache.asterix.lang.common.statement.CopyToStatement; +import org.apache.asterix.lang.common.statement.ExternalDetailsDecl; +import org.apache.asterix.lang.common.util.ExpressionUtils; +import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor; +import org.apache.asterix.metadata.entities.EntityDetails; +import org.apache.asterix.om.functions.BuiltinFunctions; + +public class SqlppLoadAccessedDataset extends AbstractSqlppSimpleExpressionVisitor { + + protected final LangRewritingContext context; + + public SqlppLoadAccessedDataset(LangRewritingContext context) { + this.context = context; + } + + @Override + public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException { + ExternalDetailsDecl externalDetailsDecl = stmtCopy.getExternalDetailsDecl(); + Map<String, String> properties = externalDetailsDecl.getProperties(); + String databaseName = properties.getOrDefault("database", "Default"); + String dataverseNameString = properties.getOrDefault("dataverse", "Default"); + String linkName = properties.getOrDefault("name", ""); + DataverseName dataverseName; + try { + dataverseName = DataverseName.createFromCanonicalForm(dataverseNameString); + } catch (AsterixException e) { + throw new IllegalStateException(e); + } + + context.getMetadataProvider().addAccessedEntity( + new EntityDetails(databaseName, dataverseName, linkName, EntityDetails.EntityType.LINK)); + return super.visit(stmtCopy, arg); + } + + @Override + public Expression visit(CallExpr expression, ILangExpression arg) { + if (BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier())) { + List<Expression> exprs = expression.getExprList(); + String databaseName, dataverseNameArg, datasetName; + EntityDetails.EntityType entityType; + entityType = exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3))) + ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET; + + databaseName = ExpressionUtils.getStringLiteral(exprs.get(0)); + dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1)); + DataverseName dataverseName; + try { + dataverseName = DataverseName.createFromCanonicalForm(dataverseNameArg); + } catch (AsterixException e) { + throw new IllegalStateException(e); + } + datasetName = ExpressionUtils.getStringLiteral(exprs.get(2)); + + context.getMetadataProvider() + .addAccessedEntity(new EntityDetails(databaseName, dataverseName, datasetName, entityType)); + } else { + FunctionSignature signature = expression.getFunctionSignature(); + String functionName = signature.getName() + "(" + signature.getArity() + ")"; + context.getMetadataProvider().addAccessedEntity(new EntityDetails(signature.getDatabaseName(), + signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION)); + } + return expression; + } +} diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index e861a3f..6ec29da 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -25,10 +25,13 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.asterix.common.api.INamespaceResolver; @@ -76,6 +79,7 @@ import org.apache.asterix.metadata.entities.DatasourceAdapter; import org.apache.asterix.metadata.entities.Datatype; import org.apache.asterix.metadata.entities.Dataverse; +import org.apache.asterix.metadata.entities.EntityDetails; import org.apache.asterix.metadata.entities.Feed; import org.apache.asterix.metadata.entities.FeedConnection; import org.apache.asterix.metadata.entities.FeedPolicyEntity; @@ -200,6 +204,8 @@ private final INamespaceResolver namespaceResolver; private IDataFormat dataFormat = FormatUtils.getDefaultFormat(); + private final Set<EntityDetails> getAccessedEntities; + public static MetadataProvider createWithDefaultNamespace(ICcApplicationContext appCtx) { java.util.function.Function<ICcApplicationContext, IMetadataProvider<?, ?>> factory = ((ICCExtensionManager) appCtx.getExtensionManager()).getMetadataProviderFactory(); @@ -225,6 +231,7 @@ dataPartitioningProvider = (DataPartitioningProvider) appCtx.getDataPartitioningProvider(); locks = new LockList(); config = new HashMap<>(); + getAccessedEntities = new HashSet<>(); setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE); } @@ -1945,6 +1952,14 @@ } } + public void addAccessedEntity(EntityDetails entityDetails) { + getAccessedEntities.add(entityDetails); + } + + public Set<EntityDetails> getGetAccessedEntities() { + return Collections.unmodifiableSet(getAccessedEntities); + } + private void validateDatabaseObjectNameImpl(String name, SourceLocation sourceLoc) throws AlgebricksException { if (name == null || name.isEmpty()) { throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME, sourceLoc, ""); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java new file mode 100644 index 0000000..c030176 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.entities; + +import org.apache.asterix.common.metadata.DataverseName; + +public class EntityDetails { + + public enum EntityType { + DATASET, + VIEW, + FUNCTION, + LINK + } + + private final String databaseName; + private final DataverseName dataverseName; + private final String datasetName; + private final EntityType entityType; + + public EntityDetails(String databaseName, DataverseName dataverseName, String datasetName, EntityType entityType) { + this.databaseName = databaseName; + this.dataverseName = dataverseName; + this.datasetName = datasetName; + this.entityType = entityType; + } + + public String getDatabaseName() { + return databaseName; + } + + public DataverseName getDataverseName() { + return dataverseName; + } + + public String getDatasetName() { + return datasetName; + } + + public EntityType getEntityType() { + return entityType; + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154 Gerrit-Change-Number: 18212 Gerrit-PatchSet: 6 Gerrit-Owner: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-CC: Ali Alsuliman <[email protected]> Gerrit-CC: [email protected] Gerrit-MessageType: merged
