This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 2568b0d29 [KYUUBI #5684][AUTHZ] Support delete/update path-based table
for Delta Lake
2568b0d29 is described below
commit 2568b0d29a8c4bf4b555b80290485bda095b3ffd
Author: zml1206 <[email protected]>
AuthorDate: Tue Nov 14 15:19:45 2023 +0800
[KYUUBI #5684][AUTHZ] Support delete/update path-based table for Delta Lake
### _Why are the changes needed?_
To close #5684 .
Support delete/update path-based table for Delta Lake in Authz plugin.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No.
Closes #5685 from zml1206/KYUUBI-5684.
Closes #5684
d28d33391 [zml1206] Support delete/update path-based table for Delta Lake
in Authz
Authored-by: zml1206 <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
...he.kyuubi.plugin.spark.authz.serde.URIExtractor | 1 +
.../src/main/resources/table_command_spec.json | 38 ++++++----------------
.../plugin/spark/authz/serde/tableExtractors.scala | 5 +--
.../plugin/spark/authz/serde/uriExtractors.scala | 11 +++++++
.../plugin/spark/authz/gen/DeltaCommands.scala | 8 ++---
.../DeltaCatalogRangerSparkExtensionSuite.scala | 22 +++++++++++++
6 files changed, 49 insertions(+), 36 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor
index 631cb7a23..fb869f602 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor
@@ -24,4 +24,5 @@
org.apache.kyuubi.plugin.spark.authz.serde.PropertiesLocationUriExtractor
org.apache.kyuubi.plugin.spark.authz.serde.PropertiesPathUriExtractor
org.apache.kyuubi.plugin.spark.authz.serde.StringSeqURIExtractor
org.apache.kyuubi.plugin.spark.authz.serde.StringURIExtractor
+org.apache.kyuubi.plugin.spark.authz.serde.SubqueryAliasURIExtractor
org.apache.kyuubi.plugin.spark.authz.serde.TableSpecURIExtractor
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index f582d81de..57fa3b267 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -2145,19 +2145,6 @@
}, {
"classname" : "org.apache.spark.sql.delta.commands.DeleteCommand",
"tableDescs" : [ {
- "fieldName" : "catalogTable",
- "fieldExtractor" : "CatalogTableOptionTableExtractor",
- "columnDesc" : null,
- "actionTypeDesc" : {
- "fieldName" : null,
- "fieldExtractor" : null,
- "actionType" : "UPDATE"
- },
- "tableTypeDesc" : null,
- "catalogDesc" : null,
- "isInput" : false,
- "setCurrentDatabaseIfMissing" : false
- }, {
"fieldName" : "target",
"fieldExtractor" : "SubqueryAliasTableExtractor",
"columnDesc" : null,
@@ -2173,7 +2160,11 @@
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
- "uriDescs" : [ ]
+ "uriDescs" : [ {
+ "fieldName" : "target",
+ "fieldExtractor" : "SubqueryAliasURIExtractor",
+ "isInput" : false
+ } ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.MergeIntoCommand",
"tableDescs" : [ {
@@ -2223,19 +2214,6 @@
}, {
"classname" : "org.apache.spark.sql.delta.commands.UpdateCommand",
"tableDescs" : [ {
- "fieldName" : "catalogTable",
- "fieldExtractor" : "CatalogTableOptionTableExtractor",
- "columnDesc" : null,
- "actionTypeDesc" : {
- "fieldName" : null,
- "fieldExtractor" : null,
- "actionType" : "UPDATE"
- },
- "tableTypeDesc" : null,
- "catalogDesc" : null,
- "isInput" : false,
- "setCurrentDatabaseIfMissing" : false
- }, {
"fieldName" : "target",
"fieldExtractor" : "SubqueryAliasTableExtractor",
"columnDesc" : null,
@@ -2251,5 +2229,9 @@
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
- "uriDescs" : [ ]
+ "uriDescs" : [ {
+ "fieldName" : "target",
+ "fieldExtractor" : "SubqueryAliasURIExtractor",
+ "isInput" : false
+ } ]
} ]
\ No newline at end of file
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
index 3e09775db..ce595c66b 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
@@ -256,9 +256,10 @@ class ResolvedIdentifierTableExtractor extends
TableExtractor {
class SubqueryAliasTableExtractor extends TableExtractor {
override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = {
v1.asInstanceOf[SubqueryAlias] match {
- case SubqueryAlias(_, SubqueryAlias(identifier, _)) =>
+ case SubqueryAlias(_, SubqueryAlias(identifier, _))
+ if !isPathIdentifier(identifier.name, spark) =>
lookupExtractor[StringTableExtractor].apply(spark,
identifier.toString())
- case SubqueryAlias(identifier, _) =>
+ case SubqueryAlias(identifier, _) if !isPathIdentifier(identifier.name,
spark) =>
lookupExtractor[StringTableExtractor].apply(spark,
identifier.toString())
}
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala
index 98fbeffde..7d4833516 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.plugin.spark.authz.serde
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat,
CatalogTable}
+import org.apache.spark.sql.catalyst.plans.logical.SubqueryAlias
import org.apache.spark.sql.connector.catalog.Identifier
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
@@ -104,3 +105,13 @@ class IdentifierURIExtractor extends URIExtractor {
case _ => Nil
}
}
+
+class SubqueryAliasURIExtractor extends URIExtractor {
+ override def apply(spark: SparkSession, v1: AnyRef): Seq[Uri] = v1 match {
+ case SubqueryAlias(_, SubqueryAlias(identifier, _))
+ if isPathIdentifier(identifier.name, spark) =>
+ Seq(identifier.name).map(Uri)
+ case SubqueryAlias(identifier, _) if isPathIdentifier(identifier.name,
spark) =>
+ Seq(identifier.name).map(Uri)
+ }
+}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
index 1c121f6eb..82627a0bf 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
@@ -27,15 +27,11 @@ object DeltaCommands extends CommandSpecs[TableCommandSpec]
{
val cmd = "org.apache.spark.sql.delta.commands.DeleteCommand"
val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE))
val tableDesc = TableDesc(
- "catalogTable",
- classOf[CatalogTableOptionTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
- TableCommandSpec(cmd, Seq(tableDesc))
- val targetDesc = TableDesc(
"target",
classOf[SubqueryAliasTableExtractor],
actionTypeDesc = Some(actionTypeDesc))
- TableCommandSpec(cmd, Seq(tableDesc, targetDesc))
+ val uriDescs = Seq(UriDesc("target", classOf[SubqueryAliasURIExtractor]))
+ TableCommandSpec(cmd, Seq(tableDesc), uriDescs = uriDescs)
}
val UpdateCommand = {
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala
index d1f49763c..514b7970b 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala
@@ -335,6 +335,28 @@ class DeltaCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
doAs(admin, sql(createOrReplaceTableSql))
})
}
+
+ test("delete from path-based table") {
+ withTempDir(path => {
+ doAs(admin, sql(createPathBasedTableSql(path)))
+ val deleteFromTableSql = s"DELETE FROM delta.`$path` WHERE birthDate <
'1955-01-01'"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(deleteFromTableSql))
+ }(s"does not have [write] privilege on [[$path, $path/]]")
+ doAs(admin, sql(deleteFromTableSql))
+ })
+ }
+
+ test("update path-based table") {
+ withTempDir(path => {
+ doAs(admin, sql(createPathBasedTableSql(path)))
+ val updateTableSql = s"UPDATE delta.`$path` SET gender = 'Female' WHERE
gender = 'F'"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(updateTableSql))
+ }(s"does not have [write] privilege on [[$path, $path/]]")
+ doAs(admin, sql(updateTableSql))
+ })
+ }
}
object DeltaCatalogRangerSparkExtensionSuite {
