>From Murtadha Hubail <[email protected]>:
Murtadha Hubail has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212 )
Change subject: [ASTERIXDB-3370][COMP] Collect accessed datasets/views during
compilation
......................................................................
[ASTERIXDB-3370][COMP] Collect accessed datasets/views during compilation
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add visitor to collect accessed datasets/views.
Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154
---
A
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M
asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
A
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
5 files changed, 131 insertions(+), 43 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/12/18212/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 6424280..e3d0b38 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -457,7 +457,7 @@
handleCreateViewStatement(metadataProvider, stmt,
stmtRewriter, requestParameters);
break;
case VIEW_DROP:
- handleViewDropStatement(metadataProvider, stmt);
+ handleViewDropStatement(metadataProvider, stmt,
requestParameters);
break;
case LOAD:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
@@ -2963,7 +2963,8 @@
}
}
- public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt) throws Exception {
+ public void handleViewDropStatement(MetadataProvider metadataProvider,
Statement stmt,
+ IRequestParameters requestParameters) throws Exception {
ViewDropStatement stmtDrop = (ViewDropStatement) stmt;
SourceLocation sourceLoc = stmtDrop.getSourceLocation();
String viewName = stmtDrop.getViewName().getValue();
@@ -2976,14 +2977,14 @@
}
lockUtil.dropDatasetBegin(lockManager, metadataProvider.getLocks(),
databaseName, dataverseName, viewName);
try {
- doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName);
+ doDropView(metadataProvider, stmtDrop, databaseName,
dataverseName, viewName, requestParameters);
} finally {
metadataProvider.getLocks().unlock();
}
}
protected boolean doDropView(MetadataProvider metadataProvider,
ViewDropStatement stmtViewDrop, String databaseName,
- DataverseName dataverseName, String viewName) throws Exception {
+ DataverseName dataverseName, String viewName, IRequestParameters
requestParameters) throws Exception {
SourceLocation sourceLoc = stmtViewDrop.getSourceLocation();
MetadataTransactionContext mdTxnCtx =
MetadataManager.INSTANCE.beginTransaction();
metadataProvider.setMetadataTxnContext(mdTxnCtx);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index cbbb9de..804dbf6 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -57,28 +57,7 @@
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.lang.common.util.ViewUtil;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.GenerateColumnNameVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.InlineColumnAliasVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.InlineWithExpressionVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.OperatorExpressionVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SelectExcludeRewriteSugarVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SetOperationVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlCompatRewriteVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppCaseAggregateExtractionVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppCaseExpressionVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppFunctionCallResolverVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGatherFunctionCallsVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupByAggregationSugarVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupByVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupingSetsVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppRightJoinRewriteVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppSpecialFunctionNameRewriteVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowRewriteVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.SubstituteGroupbyExpressionWithVariableVisitor;
-import
org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.*;
import org.apache.asterix.lang.sqlpp.util.SqlppAstPrintUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
@@ -200,6 +179,9 @@
// Rewrites RIGHT OUTER JOINs into LEFT OUTER JOINs if possible
rewriteRightJoins();
+ // Load all the accessed datasets
+ loadAccessedDatasets();
+
// Inlines functions and views
loadAndInlineUdfsAndViews();
@@ -336,6 +318,13 @@
rewriteTopExpr(visitor, null);
}
+ protected void loadAccessedDatasets() throws CompilationException {
+ SqlppLoadAccessedDataset visitor = new
SqlppLoadAccessedDataset(context);
+ if (inlineUdfsAndViews) {
+ rewriteTopExpr(visitor, null);
+ }
+ }
+
protected void loadAndInlineUdfsAndViews() throws CompilationException {
Pair<Map<FunctionSignature, FunctionDecl>,
Map<DatasetFullyQualifiedName, ViewDecl>> udfAndViewDecls =
loadUdfsAndViews(topStatement);
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
new file mode 100644
index 0000000..c063384
--- /dev/null
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
@@ -0,0 +1,43 @@
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.util.ExpressionUtils;
+import
org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+import org.apache.asterix.metadata.entities.EntityDetails;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+
+public class SqlppLoadAccessedDataset extends
AbstractSqlppSimpleExpressionVisitor {
+
+ protected final LangRewritingContext context;
+
+ public SqlppLoadAccessedDataset(LangRewritingContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Expression visit(CallExpr expression, ILangExpression arg) {
+ if
(BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier()))
{
+ List<Expression> exprs = expression.getExprList();
+ String databaseName =
ExpressionUtils.getStringLiteral(exprs.get(0));
+ String dataverseNameArg =
ExpressionUtils.getStringLiteral(exprs.get(1));
+ DataverseName dataverseName = null;
+ try {
+ dataverseName =
DataverseName.createFromCanonicalForm(dataverseNameArg);
+ } catch (AsterixException e) {
+ }
+ String datasetName =
ExpressionUtils.getStringLiteral(exprs.get(2));
+ EntityDetails.EntityType entityType =
Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))
+ ? EntityDetails.EntityType.VIEW :
EntityDetails.EntityType.DATASET;
+ context.getMetadataProvider()
+ .addAccessedEntity(new EntityDetails(databaseName,
dataverseName, datasetName, entityType));
+ }
+ return expression;
+ }
+}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index e861a3f..37d3a7c 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -24,11 +24,7 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
import java.util.stream.Collectors;
import org.apache.asterix.common.api.INamespaceResolver;
@@ -72,18 +68,7 @@
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.api.ICCExtensionManager;
import
org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetCardinalityHint;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.DatasourceAdapter;
-import org.apache.asterix.metadata.entities.Datatype;
-import org.apache.asterix.metadata.entities.Dataverse;
-import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedConnection;
-import org.apache.asterix.metadata.entities.FeedPolicyEntity;
-import org.apache.asterix.metadata.entities.FullTextConfigMetadataEntity;
-import org.apache.asterix.metadata.entities.FullTextFilterMetadataEntity;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.entities.Synonym;
+import org.apache.asterix.metadata.entities.*;
import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.metadata.provider.ExternalWriterProvider;
import org.apache.asterix.metadata.utils.DataPartitioningProvider;
@@ -200,6 +185,8 @@
private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
+ private Set<EntityDetails> getAccessedEntities;
+
public static MetadataProvider
createWithDefaultNamespace(ICcApplicationContext appCtx) {
java.util.function.Function<ICcApplicationContext,
IMetadataProvider<?, ?>> factory =
((ICCExtensionManager)
appCtx.getExtensionManager()).getMetadataProviderFactory();
@@ -225,6 +212,7 @@
dataPartitioningProvider = (DataPartitioningProvider)
appCtx.getDataPartitioningProvider();
locks = new LockList();
config = new HashMap<>();
+ getAccessedEntities = new HashSet<>();
setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE);
}
@@ -1945,6 +1933,14 @@
}
}
+ public void addAccessedEntity(EntityDetails entityDetails) {
+ getAccessedEntities.add(entityDetails);
+ }
+
+ public Set<EntityDetails> getGetAccessedEntities() {
+ return Collections.unmodifiableSet(getAccessedEntities);
+ }
+
private void validateDatabaseObjectNameImpl(String name, SourceLocation
sourceLoc) throws AlgebricksException {
if (name == null || name.isEmpty()) {
throw new AsterixException(ErrorCode.INVALID_DATABASE_OBJECT_NAME,
sourceLoc, "");
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
new file mode 100644
index 0000000..449729b
--- /dev/null
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
@@ -0,0 +1,42 @@
+package org.apache.asterix.metadata.entities;
+
+import org.apache.asterix.common.metadata.DataverseName;
+
+public class EntityDetails {
+
+ public enum EntityType {
+
+ DATASET,
+
+ VIEW;
+ };
+
+ private final String databaseName;
+ private final DataverseName dataverseName;
+ private final String datasetName;
+
+ private final EntityType entityType;
+
+ public EntityDetails(String databaseName, DataverseName dataverseName,
String datasetName, EntityType entityType) {
+ this.databaseName = databaseName;
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ this.entityType = entityType;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public DataverseName getDataverseName() {
+ return dataverseName;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public EntityType getEntityType() {
+ return entityType;
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18212
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Icb2529cd876bbbf40b92635aca0aa9693e9cd154
Gerrit-Change-Number: 18212
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <[email protected]>
Gerrit-MessageType: newchange
