This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new c5854f74b [KYUUBI #5361] [AUTHZ] Support Drop/Truncate Table Commands
for Hudi
c5854f74b is described below
commit c5854f74b2a69cd5f75b13a55a55e41ecf486ebb
Author: Angerszhuuuu <[email protected]>
AuthorDate: Wed Oct 18 13:16:24 2023 +0800
[KYUUBI #5361] [AUTHZ] Support Drop/Truncate Table Commands for Hudi
### _Why are the changes needed?_
To close #5361. Kyuubi authz support hudi drop/repair/truncate table
commands
- DropHoodieTableCommand:
https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/DropHoodieTableCommand.scala
- TruncateHoodieTableCommand:
https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No
Closes #5445 from AngersZhuuuu/KYUUBI-5361.
Closes #5361
ed9d43acd [Angerszhuuuu] update
a08dcaafc [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
372d1fbed [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
e0aa8a783 [Bowen Liang] add positive cases
9daf0b4d0 [Bowen Liang] compact code style
7eb0828d7 [Bowen Liang] Update
extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
81024ac10 [Angerszhuuuu] update
31d440617 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
fae5a64bc [Angerszhuuuu] Merge branch 'master' into KYUUBI-5361
a70617b18 [Angerszhuuuu] [KYUUBI #5361] [AUTHZ] Support
Drop/Repair/Truncate Table Commands for Hudi
Lead-authored-by: Angerszhuuuu <[email protected]>
Co-authored-by: Bowen Liang <[email protected]>
Co-authored-by: Bowen Liang <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../src/main/resources/table_command_spec.json | 35 +++++++++++
.../plugin/spark/authz/gen/HudiCommands.scala | 31 +++++++++-
.../HudiCatalogRangerSparkExtensionSuite.scala | 72 +++++++++++++++++++---
3 files changed, 130 insertions(+), 8 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index 913fef016..abf4c314c 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -1567,6 +1567,24 @@
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ]
+}, {
+ "classname" : "org.apache.spark.sql.hudi.command.DropHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "tableIdentifier",
+ "fieldExtractor" : "TableIdentifierTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : {
+ "fieldName" : "tableIdentifier",
+ "fieldExtractor" : "TableIdentifierTableTypeExtractor",
+ "skipTypes" : [ "TEMP_VIEW" ]
+ },
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "DROPTABLE",
+ "queryDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand",
"tableDescs" : [ {
@@ -1581,4 +1599,21 @@
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ]
+}, {
+ "classname" : "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "tableIdentifier",
+ "fieldExtractor" : "TableIdentifierTableExtractor",
+ "columnDesc" : {
+ "fieldName" : "partitionSpec",
+ "fieldExtractor" : "PartitionOptionColumnExtractor"
+ },
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "TRUNCATETABLE",
+ "queryDescs" : [ ]
} ]
\ No newline at end of file
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
index c4488edbf..a5f65c3d0 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
@@ -100,6 +100,33 @@ object HudiCommands {
TableCommandSpec(cmd, Seq(tableDesc1, tableDesc2), CREATETABLE)
}
+ val DropHoodieTableCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.DropHoodieTableCommand"
+ val tableTypeDesc =
+ TableTypeDesc(
+ "tableIdentifier",
+ classOf[TableIdentifierTableTypeExtractor],
+ Seq(TEMP_VIEW))
+ TableCommandSpec(
+ cmd,
+ Seq(TableDesc(
+ "tableIdentifier",
+ classOf[TableIdentifierTableExtractor],
+ tableTypeDesc = Some(tableTypeDesc))),
+ DROPTABLE)
+ }
+
+ val TruncateHoodieTableCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand"
+ val columnDesc = ColumnDesc("partitionSpec",
classOf[PartitionOptionColumnExtractor])
+ val tableDesc =
+ TableDesc(
+ "tableIdentifier",
+ classOf[TableIdentifierTableExtractor],
+ columnDesc = Some(columnDesc))
+ TableCommandSpec(cmd, Seq(tableDesc), TRUNCATETABLE)
+ }
+
val data: Array[TableCommandSpec] = Array(
AlterHoodieTableAddColumnsCommand,
AlterHoodieTableChangeColumnCommand,
@@ -109,5 +136,7 @@ object HudiCommands {
Spark31AlterTableCommand,
CreateHoodieTableCommand,
CreateHoodieTableAsSelectCommand,
- CreateHoodieTableLikeCommand)
+ CreateHoodieTableLikeCommand,
+ DropHoodieTableCommand,
+ TruncateHoodieTableCommand)
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
index ac1e357a2..fc3ebf4fe 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
@@ -180,7 +180,10 @@ class HudiCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
}
test("CreateHoodieTableLikeCommand") {
- withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1,
"database"))) {
+ withCleanTmpResources(Seq(
+ (s"$namespace1.$table1", "table"),
+ (s"$namespace1.$table2", "table"),
+ (namespace1, "database"))) {
doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
doAs(
admin,
@@ -195,15 +198,70 @@ class HudiCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
|)
|PARTITIONED BY(city)
|""".stripMargin))
- interceptContains[AccessControlException](
+
+ val createTableSql =
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table2
+ |LIKE $namespace1.$table1
+ |USING HUDI
+ |""".stripMargin
+ interceptContains[AccessControlException] {
doAs(
someone,
sql(
- s"""
- |CREATE TABLE IF NOT EXISTS $namespace1.$table2
- |LIKE $namespace1.$table1
- |USING HUDI
- |""".stripMargin)))(s"does not have [select] privilege on
[$namespace1/$table1]")
+ createTableSql))
+ }(s"does not have [select] privilege on [$namespace1/$table1]")
+ doAs(admin, sql(createTableSql))
+ }
+ }
+
+ test("DropHoodieTableCommand") {
+ withCleanTmpResources(Seq((namespace1, "database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name
string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+
+ val dropTableSql = s"DROP TABLE IF EXISTS $namespace1.$table1"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(dropTableSql))
+ }(s"does not have [drop] privilege on [$namespace1/$table1]")
+ doAs(admin, sql(dropTableSql))
+ }
+ }
+
+ test("TruncateHoodieTableCommand") {
+ withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1,
"database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name
string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+
+ val truncateTableSql = s"TRUNCATE TABLE $namespace1.$table1"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(truncateTableSql))
+ }(s"does not have [update] privilege on [$namespace1/$table1]")
+ doAs(admin, sql(truncateTableSql))
}
}
}
