This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new acf4d6338d [ASTERIXDB-3370][COMP] Collect accessed entities during
compilation
acf4d6338d is described below
commit acf4d6338d88e18f7fed4bbbe2ff5e84ceeb8a36
Author: AnkitPrabhu <ankit.pra...@couchbase.com>
AuthorDate: Thu Apr 4 00:18:24 2024 +0530
[ASTERIXDB-3370][COMP] Collect accessed entities during compilation
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Collect datasets/views/functions/link for copy-to statement
- Add visitor to collect accessed datasets/views/functions/copyto
Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhub...@apache.org>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
.../asterix/app/translator/QueryTranslator.java | 19 +++--
.../lang/sqlpp/rewrites/SqlppQueryRewriter.java | 9 ++
.../rewrites/visitor/SqlppLoadAccessedDataset.java | 95 ++++++++++++++++++++++
.../metadata/declared/MetadataProvider.java | 15 ++++
.../asterix/metadata/entities/EntityDetails.java | 59 ++++++++++++++
5 files changed, 189 insertions(+), 8 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 6424280bbd..3e674fd282 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -448,16 +448,16 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
handleLibraryDropStatement(metadataProvider, stmt,
hcc, requestParameters);
break;
case CREATE_SYNONYM:
- handleCreateSynonymStatement(metadataProvider, stmt);
+ handleCreateSynonymStatement(metadataProvider, stmt,
requestParameters);
break;
case SYNONYM_DROP:
- handleDropSynonymStatement(metadataProvider, stmt);
+ handleDropSynonymStatement(metadataProvider, stmt,
requestParameters);
break;
case CREATE_VIEW:
handleCreateViewStatement(metadataProvider, stmt,
stmtRewriter, requestParameters);
break;
case VIEW_DROP:
- handleViewDropStatement(metadataProvider, stmt);
+ handleViewDropStatement(metadataProvider, stmt,
requestParameters);
break;
case LOAD:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
@@ -2963,7 +2963,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt) throws Exception {
+ public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
ViewDropStatement stmtDrop = (ViewDropStatement) stmt;
SourceLocation sourceLoc = stmtDrop.getSourceLocation();
String viewName = stmtDrop.getViewName().getValue();
@@ -2976,14 +2977,14 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
lockUtil.dropDatasetBegin(lockManager, metadataProvider.getLocks(),
databaseName, dataverseName, viewName);
try {
- doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName);
+ doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName, requestParameters);
} finally {
metadataProvider.getLocks().unlock();
}
}
protected boolean doDropView(MetadataProvider metadataProvider,
ViewDropStatement stmtViewDrop, String databaseName,
- DataverseName dataverseName, String viewName) throws Exception {
+ DataverseName dataverseName, String viewName, IRequestParameters
requestParameters) throws Exception {
SourceLocation sourceLoc = stmtViewDrop.getSourceLocation();
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
@@ -3787,7 +3788,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- protected void handleCreateSynonymStatement(MetadataProvider
metadataProvider, Statement stmt) throws Exception {
+ protected void handleCreateSynonymStatement(MetadataProvider
metadataProvider, Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
CreateSynonymStatement css = (CreateSynonymStatement) stmt;
metadataProvider.validateDatabaseObjectName(css.getNamespace(),
css.getSynonymName(), css.getSourceLocation());
Namespace stmtActiveNamespace = getActiveNamespace(css.getNamespace());
@@ -3843,7 +3845,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- protected void handleDropSynonymStatement(MetadataProvider
metadataProvider, Statement stmt) throws Exception {
+ protected void handleDropSynonymStatement(MetadataProvider
metadataProvider, Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
SynonymDropStatement stmtSynDrop = (SynonymDropStatement) stmt;
String synonymName = stmtSynDrop.getSynonymName();
metadataProvider.validateDatabaseObjectName(stmtSynDrop.getNamespace(),
synonymName,
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index cbbb9de985..183b5e06cf 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -73,6 +73,7 @@ import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupByVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupingSetsVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor;
import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppLoadAccessedDataset;
import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppRightJoinRewriteVisitor;
import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppSpecialFunctionNameRewriteVisitor;
import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor;
@@ -200,6 +201,9 @@ public class SqlppQueryRewriter implements IQueryRewriter {
// Rewrites RIGHT OUTER JOINs into LEFT OUTER JOINs if possible
rewriteRightJoins();
+ // Load all the accessed datasets
+ loadAccessedDatasets();
+
// Inlines functions and views
loadAndInlineUdfsAndViews();
@@ -336,6 +340,11 @@ public class SqlppQueryRewriter implements IQueryRewriter {
rewriteTopExpr(visitor, null);
}
+ protected void loadAccessedDatasets() throws CompilationException {
+ SqlppLoadAccessedDataset visitor = new
SqlppLoadAccessedDataset(context);
+ rewriteTopExpr(visitor, null);
+ }
+
protected void loadAndInlineUdfsAndViews() throws CompilationException {
Pair<Map<FunctionSignature, FunctionDecl>,
Map<DatasetFullyQualifiedName, ViewDecl>> udfAndViewDecls =
loadUdfsAndViews(topStatement);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
new file mode 100644
index 0000000000..355b484369
--- /dev/null
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.util.ExpressionUtils;
+import
org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+import org.apache.asterix.metadata.entities.EntityDetails;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+
+public class SqlppLoadAccessedDataset extends
AbstractSqlppSimpleExpressionVisitor {
+
+ protected final LangRewritingContext context;
+
+ public SqlppLoadAccessedDataset(LangRewritingContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
+ ExternalDetailsDecl externalDetailsDecl =
stmtCopy.getExternalDetailsDecl();
+ Map<String, String> properties = externalDetailsDecl.getProperties();
+ String databaseName = properties.getOrDefault("database", "Default");
+ String dataverseNameString = properties.getOrDefault("dataverse",
"Default");
+ String linkName = properties.getOrDefault("name", "");
+ DataverseName dataverseName;
+ try {
+ dataverseName =
DataverseName.createFromCanonicalForm(dataverseNameString);
+ } catch (AsterixException e) {
+ throw new IllegalStateException(e);
+ }
+
+ context.getMetadataProvider().addAccessedEntity(
+ new EntityDetails(databaseName, dataverseName, linkName,
EntityDetails.EntityType.LINK));
+ return super.visit(stmtCopy, arg);
+ }
+
+ @Override
+ public Expression visit(CallExpr expression, ILangExpression arg) {
+ if
(BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier()))
{
+ List<Expression> exprs = expression.getExprList();
+ String databaseName, dataverseNameArg, datasetName;
+ EntityDetails.EntityType entityType;
+ entityType = exprs.size() > 3 &&
Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))
+ ? EntityDetails.EntityType.VIEW :
EntityDetails.EntityType.DATASET;
+
+ databaseName = ExpressionUtils.getStringLiteral(exprs.get(0));
+ dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1));
+ DataverseName dataverseName;
+ try {
+ dataverseName =
DataverseName.createFromCanonicalForm(dataverseNameArg);
+ } catch (AsterixException e) {
+ throw new IllegalStateException(e);
+ }
+ datasetName = ExpressionUtils.getStringLiteral(exprs.get(2));
+
+ context.getMetadataProvider()
+ .addAccessedEntity(new EntityDetails(databaseName,
dataverseName, datasetName, entityType));
+ } else {
+ FunctionSignature signature = expression.getFunctionSignature();
+ String functionName = signature.getName() + "(" +
signature.getArity() + ")";
+ context.getMetadataProvider().addAccessedEntity(new
EntityDetails(signature.getDatabaseName(),
+ signature.getDataverseName(), functionName,
EntityDetails.EntityType.FUNCTION));
+ }
+ return expression;
+ }
+}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e861a3fc5f..6ec29dac2d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -25,10 +25,13 @@ import static
org.apache.asterix.common.utils.IdentifierUtil.dataset;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
import org.apache.asterix.common.api.INamespaceResolver;
@@ -76,6 +79,7 @@ import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.DatasourceAdapter;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.EntityDetails;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.entities.FeedConnection;
import org.apache.asterix.metadata.entities.FeedPolicyEntity;
@@ -200,6 +204,8 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
+ private final Set<EntityDetails> getAccessedEntities;
+
public static MetadataProvider
createWithDefaultNamespace(ICcApplicationContext appCtx) {
java.util.function.Function<ICcApplicationContext,
IMetadataProvider<?, ?>> factory =
((ICCExtensionManager)
appCtx.getExtensionManager()).getMetadataProviderFactory();
@@ -225,6 +231,7 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
dataPartitioningProvider = (DataPartitioningProvider)
appCtx.getDataPartitioningProvider();
locks = new LockList();
config = new HashMap<>();
+ getAccessedEntities = new HashSet<>();
setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE);
}
@@ -1945,6 +1952,14 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
}
}
+ public void addAccessedEntity(EntityDetails entityDetails) {
+ getAccessedEntities.add(entityDetails);
+ }
+
+ public Set<EntityDetails> getGetAccessedEntities() {
+ return Collections.unmodifiableSet(getAccessedEntities);
+ }
+
private void validateDatabaseObjectNameImpl(String name, SourceLocation
sourceLoc) throws AlgebricksException {
if (name == null || name.isEmpty()) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME,
sourceLoc, "");
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
new file mode 100644
index 0000000000..c030176b11
--- /dev/null
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.entities;
+
+import org.apache.asterix.common.metadata.DataverseName;
+
+public class EntityDetails {
+
+ public enum EntityType {
+ DATASET,
+ VIEW,
+ FUNCTION,
+ LINK
+ }
+
+ private final String databaseName;
+ private final DataverseName dataverseName;
+ private final String datasetName;
+ private final EntityType entityType;
+
+ public EntityDetails(String databaseName, DataverseName dataverseName,
String datasetName, EntityType entityType) {
+ this.databaseName = databaseName;
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ this.entityType = entityType;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public EntityType getEntityType() {
+ return entityType;
+ }
+}
