>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18226 )
Change subject: [ASTERIXDB-3370][COMP] Collect and authorise dependency for views and functions ...................................................................... [ASTERIXDB-3370][COMP] Collect and authorise dependency for views and functions - user model changes: no - storage format changes: no - interface changes: yes Details: - Add ensureAuthorised in IReceptionist method to authorise the dependency - Fix naming for accessed entities - Add dependencies to accessed entity field Change-Id: Ic3b1898307f7099dba3ea135db6c2e794de27bde Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18226 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java 9 files changed, 103 insertions(+), 49 deletions(-) Approvals: Ali Alsuliman: Looks good to me, approved Jenkins: Verified; Verified diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java index 84bee6a..ed68d8b 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/Receptionist.java @@ -28,6 +28,7 @@ import org.apache.asterix.common.api.ISchedulableClientRequest; import org.apache.asterix.common.api.RequestReference; import org.apache.http.HttpHeaders; +import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.http.api.IServletRequest; import org.apache.hyracks.util.NetworkUtil; @@ -54,4 +55,10 @@ public void ensureSchedulable(ISchedulableClientRequest schedulableRequest) throws HyracksDataException { // currently we don't have any restrictions } + + @Override + public void ensureAuthorized(ICommonRequestParameters requestParameters, IMetadataProvider metadataProvider) + throws HyracksDataException { + + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 3e674fd..e72ab54 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -2935,7 +2935,8 @@ Collections.emptyList()); List<List<DependencyFullyQualifiedName>> dependencies = - ViewUtil.getViewDependencies(viewDecl, foreignKeys, queryRewriter); + ViewUtil.getViewDependencies(metadataProvider, viewDecl, foreignKeys, queryRewriter); + appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider); ViewDetails viewDetails = new ViewDetails(cvs.getViewBody(), dependencies, cvs.getDefaultNull(), primaryKeyFields, foreignKeys, datetimeFormat, dateFormat, timeFormat); @@ -3238,7 +3239,8 @@ Collections.emptyList()); List<List<DependencyFullyQualifiedName>> dependencies = - FunctionUtil.getFunctionDependencies(fd, queryRewriter); + FunctionUtil.getFunctionDependencies(metadataProvider, fd, queryRewriter); + appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider); newInlineTypes = Collections.emptyMap(); function = new Function(functionSignature, paramNames, null, null, cfs.getFunctionBody(), @@ -4221,6 +4223,7 @@ clfrqs.setSourceLocation(stmt.getSourceLocation()); JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs, stmtParams, null); + appCtx.getReceptionist().ensureAuthorized(requestParameters, metadataProvider); afterCompile(); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java index 95ed22e..d4ba5ca 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IReceptionist.java @@ -18,6 +18,7 @@ */ package org.apache.asterix.common.api; +import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.http.api.IServletRequest; @@ -47,4 +48,14 @@ * @throws HyracksDataException */ void ensureSchedulable(ISchedulableClientRequest schedulableRequest) throws HyracksDataException; + + /** + * Ensures a client's request is authorized + * + * @param requestParameters + * @param metadataProvider + * @throws HyracksDataException + */ + void ensureAuthorized(ICommonRequestParameters requestParameters, IMetadataProvider metadataProvider) + throws HyracksDataException; } diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java index 3e91e22..c9f8640 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java @@ -51,6 +51,8 @@ import org.apache.asterix.lang.common.statement.ViewDecl; import org.apache.asterix.lang.common.struct.UnaryExprType; import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor; +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.EntityDetails; import org.apache.asterix.object.base.AdmArrayNode; import org.apache.asterix.object.base.AdmBigIntNode; import org.apache.asterix.object.base.AdmBooleanNode; @@ -241,8 +243,8 @@ } } - public static void collectDependencies(Expression expression, IQueryRewriter rewriter, - List<DependencyFullyQualifiedName> outDatasetDependencies, + public static void collectDependencies(MetadataProvider metadataProvider, Expression expression, + IQueryRewriter rewriter, List<DependencyFullyQualifiedName> outDatasetDependencies, List<DependencyFullyQualifiedName> outSynonymDependencies, List<DependencyFullyQualifiedName> outFunctionDependencies) throws CompilationException { // Duplicate elimination @@ -260,6 +262,13 @@ if (FunctionUtil.isBuiltinDatasetFunction(signature)) { Triple<DatasetFullyQualifiedName, Boolean, DatasetFullyQualifiedName> dsArgs = FunctionUtil.parseDatasetFunctionArguments(functionCall); + DatasetFullyQualifiedName datasetFullyQualifiedName = dsArgs.first; + EntityDetails.EntityType entityType = + dsArgs.second ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET; + metadataProvider + .addAccessedEntity(new EntityDetails(datasetFullyQualifiedName.getDatabaseName(), + datasetFullyQualifiedName.getDataverseName(), + datasetFullyQualifiedName.getDatasetName(), entityType)); DatasetFullyQualifiedName synonymReference = dsArgs.third; if (synonymReference != null) { // resolved via synonym -> store synonym name as a dependency @@ -280,6 +289,9 @@ } } else { if (seenFunctions.add(signature)) { + String functionName = signature.getName() + "(" + signature.getArity() + ")"; + metadataProvider.addAccessedEntity(new EntityDetails(signature.getDatabaseName(), + signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION)); outFunctionDependencies.add(new DependencyFullyQualifiedName(signature.getDatabaseName(), signature.getDataverseName(), signature.getName(), Integer.toString(signature.getArity()))); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java index ad70fca..ffd2089 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java @@ -228,8 +228,8 @@ }; } - public static List<List<DependencyFullyQualifiedName>> getFunctionDependencies(FunctionDecl fd, - IQueryRewriter rewriter) throws CompilationException { + public static List<List<DependencyFullyQualifiedName>> getFunctionDependencies(MetadataProvider metadataProvider, + FunctionDecl fd, IQueryRewriter rewriter) throws CompilationException { Expression normBody = fd.getNormalizedFuncBody(); if (normBody == null) { throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, fd.getSourceLocation(), @@ -240,8 +240,8 @@ List<DependencyFullyQualifiedName> datasetDependencies = new ArrayList<>(); List<DependencyFullyQualifiedName> synonymDependencies = new ArrayList<>(); List<DependencyFullyQualifiedName> functionDependencies = new ArrayList<>(); - ExpressionUtils.collectDependencies(normBody, rewriter, datasetDependencies, synonymDependencies, - functionDependencies); + ExpressionUtils.collectDependencies(metadataProvider, normBody, rewriter, datasetDependencies, + synonymDependencies, functionDependencies); List<DependencyFullyQualifiedName> typeDependencies = Collections.emptyList(); return Function.createDependencies(datasetDependencies, functionDependencies, typeDependencies, diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java index 23a2d1a..238af43 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ViewUtil.java @@ -43,6 +43,7 @@ import org.apache.asterix.lang.common.statement.ViewDecl; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.struct.VarIdentifier; +import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.ViewDetails; import org.apache.asterix.metadata.utils.TypeUtil; import org.apache.asterix.om.functions.BuiltinFunctions; @@ -78,8 +79,9 @@ } } - public static List<List<DependencyFullyQualifiedName>> getViewDependencies(ViewDecl viewDecl, - List<ViewDetails.ForeignKey> foreignKeys, IQueryRewriter rewriter) throws CompilationException { + public static List<List<DependencyFullyQualifiedName>> getViewDependencies(MetadataProvider metadataProvider, + ViewDecl viewDecl, List<ViewDetails.ForeignKey> foreignKeys, IQueryRewriter rewriter) + throws CompilationException { Expression normBody = viewDecl.getNormalizedViewBody(); if (normBody == null) { throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, viewDecl.getSourceLocation(), @@ -90,8 +92,8 @@ List<DependencyFullyQualifiedName> datasetDependencies = new ArrayList<>(); List<DependencyFullyQualifiedName> synonymDependencies = new ArrayList<>(); List<DependencyFullyQualifiedName> functionDependencies = new ArrayList<>(); - ExpressionUtils.collectDependencies(normBody, rewriter, datasetDependencies, synonymDependencies, - functionDependencies); + ExpressionUtils.collectDependencies(metadataProvider, normBody, rewriter, datasetDependencies, + synonymDependencies, functionDependencies); if (foreignKeys != null) { DatasetFullyQualifiedName viewName = viewDecl.getViewName(); diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java index 355b484..8b44df3 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppLoadAccessedDataset.java @@ -24,13 +24,14 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.functions.FunctionSignature; +import org.apache.asterix.common.metadata.DatasetFullyQualifiedName; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.lang.common.base.Expression; import org.apache.asterix.lang.common.base.ILangExpression; import org.apache.asterix.lang.common.expression.CallExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; -import org.apache.asterix.lang.common.statement.CopyToStatement; -import org.apache.asterix.lang.common.statement.ExternalDetailsDecl; +import org.apache.asterix.lang.common.statement.FunctionDecl; +import org.apache.asterix.lang.common.statement.ViewDecl; import org.apache.asterix.lang.common.util.ExpressionUtils; import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor; import org.apache.asterix.metadata.entities.EntityDetails; @@ -45,33 +46,15 @@ } @Override - public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException { - ExternalDetailsDecl externalDetailsDecl = stmtCopy.getExternalDetailsDecl(); - Map<String, String> properties = externalDetailsDecl.getProperties(); - String databaseName = properties.getOrDefault("database", "Default"); - String dataverseNameString = properties.getOrDefault("dataverse", "Default"); - String linkName = properties.getOrDefault("name", ""); - DataverseName dataverseName; - try { - dataverseName = DataverseName.createFromCanonicalForm(dataverseNameString); - } catch (AsterixException e) { - throw new IllegalStateException(e); - } - - context.getMetadataProvider().addAccessedEntity( - new EntityDetails(databaseName, dataverseName, linkName, EntityDetails.EntityType.LINK)); - return super.visit(stmtCopy, arg); + public Expression visit(CallExpr expression, ILangExpression arg) throws CompilationException { + addAccessedEntities(expression); + return super.visit(expression, arg); } - @Override - public Expression visit(CallExpr expression, ILangExpression arg) { + private void addAccessedEntities(CallExpr expression) { if (BuiltinFunctions.DATASET.equals(expression.getFunctionSignature().createFunctionIdentifier())) { List<Expression> exprs = expression.getExprList(); String databaseName, dataverseNameArg, datasetName; - EntityDetails.EntityType entityType; - entityType = exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3))) - ? EntityDetails.EntityType.VIEW : EntityDetails.EntityType.DATASET; - databaseName = ExpressionUtils.getStringLiteral(exprs.get(0)); dataverseNameArg = ExpressionUtils.getStringLiteral(exprs.get(1)); DataverseName dataverseName; @@ -82,14 +65,28 @@ } datasetName = ExpressionUtils.getStringLiteral(exprs.get(2)); + EntityDetails.EntityType entityType = EntityDetails.EntityType.DATASET; + if (exprs.size() > 3 && Boolean.TRUE.equals(ExpressionUtils.getBooleanLiteral(exprs.get(3)))) { + DatasetFullyQualifiedName viewDatasetName = + new DatasetFullyQualifiedName(databaseName, dataverseName, datasetName); + Map<DatasetFullyQualifiedName, ViewDecl> declaredViews = context.getDeclaredViews(); + if (declaredViews.containsKey(viewDatasetName)) { + return; + } + entityType = EntityDetails.EntityType.VIEW; + } + context.getMetadataProvider() .addAccessedEntity(new EntityDetails(databaseName, dataverseName, datasetName, entityType)); } else { FunctionSignature signature = expression.getFunctionSignature(); + Map<FunctionSignature, FunctionDecl> declaredFunctions = context.getDeclaredFunctions(); + if (declaredFunctions.containsKey(signature)) { + return; + } String functionName = signature.getName() + "(" + signature.getArity() + ")"; context.getMetadataProvider().addAccessedEntity(new EntityDetails(signature.getDatabaseName(), signature.getDataverseName(), functionName, EntityDetails.EntityType.FUNCTION)); } - return expression; } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 6ec29da..e0f44ea 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -204,7 +204,7 @@ private final INamespaceResolver namespaceResolver; private IDataFormat dataFormat = FormatUtils.getDefaultFormat(); - private final Set<EntityDetails> getAccessedEntities; + private final Set<EntityDetails> accessedEntities; public static MetadataProvider createWithDefaultNamespace(ICcApplicationContext appCtx) { java.util.function.Function<ICcApplicationContext, IMetadataProvider<?, ?>> factory = @@ -231,7 +231,7 @@ dataPartitioningProvider = (DataPartitioningProvider) appCtx.getDataPartitioningProvider(); locks = new LockList(); config = new HashMap<>(); - getAccessedEntities = new HashSet<>(); + accessedEntities = new HashSet<>(); setDefaultNamespace(MetadataConstants.DEFAULT_NAMESPACE); } @@ -1953,11 +1953,11 @@ } public void addAccessedEntity(EntityDetails entityDetails) { - getAccessedEntities.add(entityDetails); + accessedEntities.add(entityDetails); } - public Set<EntityDetails> getGetAccessedEntities() { - return Collections.unmodifiableSet(getAccessedEntities); + public Set<EntityDetails> getAccessedEntities() { + return Collections.unmodifiableSet(accessedEntities); } private void validateDatabaseObjectNameImpl(String name, SourceLocation sourceLoc) throws AlgebricksException { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java index c030176..a341a39 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/EntityDetails.java @@ -25,19 +25,18 @@ public enum EntityType { DATASET, VIEW, - FUNCTION, - LINK + FUNCTION } private final String databaseName; private final DataverseName dataverseName; - private final String datasetName; + private final String entityName; private final EntityType entityType; - public EntityDetails(String databaseName, DataverseName dataverseName, String datasetName, EntityType entityType) { + public EntityDetails(String databaseName, DataverseName dataverseName, String entityName, EntityType entityType) { this.databaseName = databaseName; this.dataverseName = dataverseName; - this.datasetName = datasetName; + this.entityName = entityName; this.entityType = entityType; } @@ -49,8 +48,8 @@ return dataverseName; } - public String getDatasetName() { - return datasetName; + public String getEntityName() { + return entityName; } public EntityType getEntityType() { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18226 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: Ic3b1898307f7099dba3ea135db6c2e794de27bde Gerrit-Change-Number: 18226 Gerrit-PatchSet: 5 Gerrit-Owner: [email protected] Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-MessageType: merged
