This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9d3f7961ad [ASTERIXDB-3370][COMP] Collect and authorise dependency for
views and functions
9d3f7961ad is described below
commit 9d3f7961ad2de2c22c3c1c3ce744230d666d52d7
Author: AnkitPrabhu <[email protected]>
AuthorDate: Fri Apr 12 11:55:11 2024 +0530
[ASTERIXDB-3370][COMP] Collect and authorise dependency for views and
functions
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Add ensureAuthorised in IReceptionist method to authorise the dependency
- Fix naming for accessed entities
- Add dependencies to accessed entity field
Change-Id: Ic3b1898307f7099dba3ea135db6c2e794de27bde
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18226
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../apache/asterix/translator/Receptionist.java | 7 ++++
.../asterix/app/translator/QueryTranslator.java | 7 +++-
.../apache/asterix/common/api/IReceptionist.java | 11 +++++
.../asterix/lang/common/util/ExpressionUtils.java | 16 +++++++-
.../asterix/lang/common/util/FunctionUtil.java | 8 ++--
.../apache/asterix/lang/common/util/ViewUtil.java | 10 +++--
.../rewrites/visitor/SqlppLoadAccessedDataset.java | 47 ++++++++++------------
.../metadata/declared/MetadataProvider.java | 10 ++---
.../asterix/metadata/entities/EntityDetails.java | 13 +++---
9 files changed, 80 insertions(+), 49 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
index 84bee6a6de..ed68d8bf76 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java
@@ -28,6 +28,7 @@ import org.apache.asterix.common.api.IRequestReference;
import org.apache.asterix.common.api.ISchedulableClientRequest;
import org.apache.asterix.common.api.RequestReference;
import org.apache.http.HttpHeaders;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.util.NetworkUtil;
@@ -54,4 +55,10 @@ public class Receptionist implements IReceptionist {
public void ensureSchedulable(ISchedulableClientRequest
schedulableRequest) throws HyracksDataException {
// currently we don't have any restrictions
}
+
+ @Override
+ public void ensureAuthorized(ICommonRequestParameters requestParameters,
IMetadataProvider metadataProvider)
+ throws HyracksDataException {
+
+ }
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 3e674fd282..e72ab54c8a 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -2935,7 +2935,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
Collections.emptyList());
List<List<DependencyFullyQualifiedName>> dependencies =
- ViewUtil.getViewDependencies(viewDecl, foreignKeys,
queryRewriter);
+ ViewUtil.getViewDependencies(metadataProvider, viewDecl,
foreignKeys, queryRewriter);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters,
metadataProvider);
ViewDetails viewDetails = new ViewDetails(cvs.getViewBody(),
dependencies, cvs.getDefaultNull(),
primaryKeyFields, foreignKeys, datetimeFormat, dateFormat,
timeFormat);
@@ -3238,7 +3239,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
Collections.emptyList());
List<List<DependencyFullyQualifiedName>> dependencies =
- FunctionUtil.getFunctionDependencies(fd,
queryRewriter);
+ FunctionUtil.getFunctionDependencies(metadataProvider,
fd, queryRewriter);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters,
metadataProvider);
newInlineTypes = Collections.emptyMap();
function = new Function(functionSignature, paramNames, null,
null, cfs.getFunctionBody(),
@@ -4221,6 +4223,7 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
clfrqs.setSourceLocation(stmt.getSourceLocation());
JobSpecification jobSpec =
rewriteCompileQuery(hcc, metadataProvider,
clfrqs.getQuery(), clfrqs, stmtParams, null);
+ appCtx.getReceptionist().ensureAuthorized(requestParameters,
metadataProvider);
afterCompile();
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
index 95ed22ea61..d4ba5ca4c6 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.api;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
@@ -47,4 +48,14 @@ public interface IReceptionist {
* @throws HyracksDataException
*/
void ensureSchedulable(ISchedulableClientRequest schedulableRequest)
throws HyracksDataException;
+
+ /**
+ * Ensures a client's request is authorized
+ *
+ * @param requestParameters
+ * @param metadataProvider
+ * @throws HyracksDataException
+ */
+ void ensureAuthorized(ICommonRequestParameters requestParameters,
IMetadataProvider metadataProvider)
+ throws HyracksDataException;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
index 3e91e22b45..c9f8640933 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
@@ -51,6 +51,8 @@ import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.struct.UnaryExprType;
import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.EntityDetails;
import org.apache.asterix.object.base.AdmArrayNode;
import org.apache.asterix.object.base.AdmBigIntNode;
import org.apache.asterix.object.base.AdmBooleanNode;
@@ -241,8 +243,8 @@ public class ExpressionUtils {
}
}
- public static void collectDependencies(Expression expression,
IQueryRewriter rewriter,
- List<DependencyFullyQualifiedName> outDatasetDependencies,
+ public static void collectDependencies(MetadataProvider metadataProvider,
Expression expression,
+ IQueryRewriter rewriter, List<DependencyFullyQualifiedName>
outDatasetDependencies,
List<DependencyFullyQualifiedName> outSynonymDependencies,
List<DependencyFullyQualifiedName> outFunctionDependencies) throws
CompilationException {
// Duplicate elimination
@@ -260,6 +262,13 @@ public class ExpressionUtils {
if (FunctionUtil.isBuiltinDatasetFunction(signature)) {
Triple<DatasetFullyQualifiedName, Boolean,
DatasetFullyQualifiedName> dsArgs =
FunctionUtil.parseDatasetFunctionArguments(functionCall);
+ DatasetFullyQualifiedName
datasetFullyQualifiedName = dsArgs.first;
+ EntityDetails.EntityType entityType =
+ dsArgs.second ?
EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET;
+ metadataProvider
+ .addAccessedEntity(new
EntityDetails(datasetFullyQualifiedName.getDatabaseName(),
+
datasetFullyQualifiedName.getDataverseName(),
+
datasetFullyQualifiedName.getDatasetName(), entityType));
DatasetFullyQualifiedName synonymReference =
dsArgs.third;
if (synonymReference != null) {
// resolved via synonym -> store synonym name
as a dependency
@@ -280,6 +289,9 @@ public class ExpressionUtils {
}
} else {
if (seenFunctions.add(signature)) {
+ String functionName = signature.getName() + "(" +
signature.getArity() + ")";
+ metadataProvider.addAccessedEntity(new
EntityDetails(signature.getDatabaseName(),
+ signature.getDataverseName(),
functionName, EntityDetails.EntityType.FUNCTION));
outFunctionDependencies.add(new
DependencyFullyQualifiedName(signature.getDatabaseName(),
signature.getDataverseName(),
signature.getName(),
Integer.toString(signature.getArity())));
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index ad70fcad81..ffd20899a7 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -228,8 +228,8 @@ public class FunctionUtil {
};
}
- public static List<List<DependencyFullyQualifiedName>>
getFunctionDependencies(FunctionDecl fd,
- IQueryRewriter rewriter) throws CompilationException {
+ public static List<List<DependencyFullyQualifiedName>>
getFunctionDependencies(MetadataProvider metadataProvider,
+ FunctionDecl fd, IQueryRewriter rewriter) throws
CompilationException {
Expression normBody = fd.getNormalizedFuncBody();
if (normBody == null) {
throw new
CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
fd.getSourceLocation(),
@@ -240,8 +240,8 @@ public class FunctionUtil {
List<DependencyFullyQualifiedName> datasetDependencies = new
ArrayList<>();
List<DependencyFullyQualifiedName> synonymDependencies = new
ArrayList<>();
List<DependencyFullyQualifiedName> functionDependencies = new
ArrayList<>();
- ExpressionUtils.collectDependencies(normBody, rewriter,
datasetDependencies, synonymDependencies,
- functionDependencies);
+ ExpressionUtils.collectDependencies(metadataProvider, normBody,
rewriter, datasetDependencies,
+ synonymDependencies, functionDependencies);
List<DependencyFullyQualifiedName> typeDependencies =
Collections.emptyList();
return Function.createDependencies(datasetDependencies,
functionDependencies, typeDependencies,
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
index 23a2d1ab6f..238af43eb0 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java
@@ -43,6 +43,7 @@ import org.apache.asterix.lang.common.literal.StringLiteral;
import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.ViewDetails;
import org.apache.asterix.metadata.utils.TypeUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
@@ -78,8 +79,9 @@ public final class ViewUtil {
}
}
- public static List<List<DependencyFullyQualifiedName>>
getViewDependencies(ViewDecl viewDecl,
- List<ViewDetails.ForeignKey> foreignKeys, IQueryRewriter rewriter)
throws CompilationException {
+ public static List<List<DependencyFullyQualifiedName>>
getViewDependencies(MetadataProvider metadataProvider,
+ ViewDecl viewDecl, List<ViewDetails.ForeignKey> foreignKeys,
IQueryRewriter rewriter)
+ throws CompilationException {
Expression normBody = viewDecl.getNormalizedViewBody();
if (normBody == null) {
throw new
CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE,
viewDecl.getSourceLocation(),
@@ -90,8 +92,8 @@ public final class ViewUtil {
List<DependencyFullyQualifiedName> datasetDependencies = new
ArrayList<>();
List<DependencyFullyQualifiedName> synonymDependencies = new
ArrayList<>();
List<DependencyFullyQualifiedName> functionDependencies = new
ArrayList<>();
- ExpressionUtils.collectDependencies(normBody, rewriter,
datasetDependencies, synonymDependencies,
- functionDependencies);
+ ExpressionUtils.collectDependencies(metadataProvider, normBody,
rewriter, datasetDependencies,
+ synonymDependencies, functionDependencies);
if (foreignKeys != null) {
DatasetFullyQualifiedName viewName = viewDecl.getViewName();
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
index 355b484369..8b44df3d00 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java
@@ -24,13 +24,14 @@ import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
-import org.apache.asterix.lang.common.statement.CopyToStatement;
-import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.ViewDecl;
import org.apache.asterix.lang.common.util.ExpressionUtils;
import
org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
import org.apache.asterix.metadata.entities.EntityDetails;
@@ -45,33 +46,15 @@ public class SqlppLoadAccessedDataset extends
AbstractSqlppSimpleExpressionVisit
}
@Override
- public Expression visit(CopyToStatement stmtCopy, ILangExpression arg)
throws CompilationException {
- ExternalDetailsDecl externalDetailsDecl =
stmtCopy.getExternalDetailsDecl();
- Map<String, String> properties = externalDetailsDecl.getProperties();
- String databaseName = properties.getOrDefault("database", "Default");
- String dataverseNameString = properties.getOrDefault("dataverse",
"Default");
- String linkName = properties.getOrDefault("name", "");
- DataverseName dataverseName;
- try {
- dataverseName =
DataverseName.createFromCanonicalForm(dataverseNameString);
- } catch (AsterixException e) {
- throw new IllegalStateException(e);
- }
-
- context.getMetadataProvider().addAccessedEntity(
- new EntityDetails(databaseName, dataverseName, linkName,
EntityDetails.EntityType.LINK));
- return super.visit(stmtCopy, arg);
+ public Expression visit(CallExpr expression, ILangExpression arg) throws
CompilationException {
+ addAccessedEntities(expression);
+ return super.visit(expression, arg);
}
- @Override
- public Expression visit(CallExpr expression, ILangExpression arg) {
+ private void addAccessedEntities(CallExpr expression) {
if
(BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier()))
{
List<Expression> exprs = expression.getExprList();
String databaseName, dataverseNameArg, datasetName;
- EntityDetails.EntityType entityType;
- entityType = exprs.size() > 3 &&
Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))
- ? EntityDetails.EntityType.VIEW :
EntityDetails.EntityType.DATASET;
-
databaseName = ExpressionUtils.getStringLiteral(exprs.get(0));
dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1));
DataverseName dataverseName;
@@ -82,14 +65,28 @@ public class SqlppLoadAccessedDataset extends
AbstractSqlppSimpleExpressionVisit
}
datasetName = ExpressionUtils.getStringLiteral(exprs.get(2));
+ EntityDetails.EntityType entityType =
EntityDetails.EntityType.DATASET;
+ if (exprs.size() > 3 &&
Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))) {
+ DatasetFullyQualifiedName viewDatasetName =
+ new DatasetFullyQualifiedName(databaseName,
dataverseName, datasetName);
+ Map<DatasetFullyQualifiedName, ViewDecl> declaredViews =
context.getDeclaredViews();
+ if (declaredViews.containsKey(viewDatasetName)) {
+ return;
+ }
+ entityType = EntityDetails.EntityType.VIEW;
+ }
+
context.getMetadataProvider()
.addAccessedEntity(new EntityDetails(databaseName,
dataverseName, datasetName, entityType));
} else {
FunctionSignature signature = expression.getFunctionSignature();
+ Map<FunctionSignature, FunctionDecl> declaredFunctions =
context.getDeclaredFunctions();
+ if (declaredFunctions.containsKey(signature)) {
+ return;
+ }
String functionName = signature.getName() + "(" +
signature.getArity() + ")";
context.getMetadataProvider().addAccessedEntity(new
EntityDetails(signature.getDatabaseName(),
signature.getDataverseName(), functionName,
EntityDetails.EntityType.FUNCTION));
}
- return expression;
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 6ec29dac2d..e0f44ea0ef 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -204,7 +204,7 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
private final INamespaceResolver namespaceResolver;
private IDataFormat dataFormat = FormatUtils.getDefaultFormat();
- private final Set<EntityDetails> getAccessedEntities;
+ private final Set<EntityDetails> accessedEntities;
public static MetadataProvider
createWithDefaultNamespace(ICcApplicationContext appCtx) {
java.util.function.Function<ICcApplicationContext,
IMetadataProvider<?, ?>> factory =
@@ -231,7 +231,7 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
dataPartitioningProvider = (DataPartitioningProvider)
appCtx.getDataPartitioningProvider();
locks = new LockList();
config = new HashMap<>();
- getAccessedEntities = new HashSet<>();
+ accessedEntities = new HashSet<>();
setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE);
}
@@ -1953,11 +1953,11 @@ public class MetadataProvider implements
IMetadataProvider<DataSourceId, String>
}
public void addAccessedEntity(EntityDetails entityDetails) {
- getAccessedEntities.add(entityDetails);
+ accessedEntities.add(entityDetails);
}
- public Set<EntityDetails> getGetAccessedEntities() {
- return Collections.unmodifiableSet(getAccessedEntities);
+ public Set<EntityDetails> getAccessedEntities() {
+ return Collections.unmodifiableSet(accessedEntities);
}
private void validateDatabaseObjectNameImpl(String name, SourceLocation
sourceLoc) throws AlgebricksException {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
index c030176b11..a341a39ecb 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java
@@ -25,19 +25,18 @@ public class EntityDetails {
public enum EntityType {
DATASET,
VIEW,
- FUNCTION,
- LINK
+ FUNCTION
}
private final String databaseName;
private final DataverseName dataverseName;
- private final String datasetName;
+ private final String entityName;
private final EntityType entityType;
- public EntityDetails(String databaseName, DataverseName dataverseName,
String datasetName, EntityType entityType) {
+ public EntityDetails(String databaseName, DataverseName dataverseName,
String entityName, EntityType entityType) {
this.databaseName = databaseName;
this.dataverseName = dataverseName;
- this.datasetName = datasetName;
+ this.entityName = entityName;
this.entityType = entityType;
}
@@ -49,8 +48,8 @@ public class EntityDetails {
return dataverseName;
}
- public String getDatasetName() {
- return datasetName;
+ public String getEntityName() {
+ return entityName;
}
public EntityType getEntityType() {
