This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new facfe57a4 [KYUUBI #5452][AUTHZ] Support Compaction table commands for
Hudi
facfe57a4 is described below
commit facfe57a4957b6e89bd7d2004924d7f9ff147b08
Author: Angerszhuuuu <[email protected]>
AuthorDate: Wed Oct 18 18:02:29 2023 +0800
[KYUUBI #5452][AUTHZ] Support Compaction table commands for Hudi
### _Why are the changes needed?_
To close #5452
Support Compaction table/path related command. The SQL grammar is
https://github.com/apache/hudi/blob/release-0.14.0/hudi-spark-datasource/hudi-spark/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlCommon.g4
- CompactionHoodieTableCommand
:https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala
- CompactionShowHoodiePathCommand:
https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodiePathCommand.scala
- CompactionShowHoodieTableCommand:
https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No
Closes #5454 from AngersZhuuuu/KYUUBI-5452.
Closes #5452
e42200f7e [Angerszhuuuu] follow comment
4d5139e9a [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
0e7cb924b [Angerszhuuuu] follow comment
e14dc4129 [Angerszhuuuu] [KYUUBI #5452][AUTHZ] Support Compaction
table/path related command
Authored-by: Angerszhuuuu <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../src/main/resources/table_command_spec.json | 37 ++++++++++++++++++++++
.../plugin/spark/authz/gen/HudiCommands.scala | 20 ++++++++++--
.../HudiCatalogRangerSparkExtensionSuite.scala | 31 ++++++++++++++++++
3 files changed, 85 insertions(+), 3 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index abf4c314c..21d647f5d 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -1513,6 +1513,43 @@
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ]
+}, {
+ "classname" :
"org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "table",
+ "fieldExtractor" : "CatalogTableTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ }, {
+ "fieldName" : "table",
+ "fieldExtractor" : "CatalogTableTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : true,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "CREATETABLE",
+ "queryDescs" : [ ]
+}, {
+ "classname" :
"org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "table",
+ "fieldExtractor" : "CatalogTableTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : true,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "SHOW_TBLPROPERTIES",
+ "queryDescs" : [ ]
}, {
"classname" :
"org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand",
"tableDescs" : [ {
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
index a5f65c3d0..72daa89e2 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
@@ -127,16 +127,30 @@ object HudiCommands {
TableCommandSpec(cmd, Seq(tableDesc), TRUNCATETABLE)
}
+ val CompactionHoodieTableCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand"
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ TableCommandSpec(cmd, Seq(tableDesc, tableDesc.copy(isInput = true)),
CREATETABLE)
+ }
+
+ val CompactionShowHoodieTableCommand = {
+ val cmd =
"org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand"
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
isInput = true)
+ TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES)
+ }
+
val data: Array[TableCommandSpec] = Array(
AlterHoodieTableAddColumnsCommand,
AlterHoodieTableChangeColumnCommand,
AlterHoodieTableDropPartitionCommand,
AlterHoodieTableRenameCommand,
AlterTableCommand,
- Spark31AlterTableCommand,
- CreateHoodieTableCommand,
CreateHoodieTableAsSelectCommand,
+ CreateHoodieTableCommand,
CreateHoodieTableLikeCommand,
+ CompactionHoodieTableCommand,
+ CompactionShowHoodieTableCommand,
DropHoodieTableCommand,
- TruncateHoodieTableCommand)
+ TruncateHoodieTableCommand,
+ Spark31AlterTableCommand)
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
index fc3ebf4fe..48af6bf9f 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
@@ -264,4 +264,35 @@ class HudiCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
doAs(admin, sql(truncateTableSql))
}
}
+
+ test("CompactionHoodieTableCommand / CompactionShowHoodieTableCommand") {
+ withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1,
"database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name
string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'mor',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+
+ val compactionTable = s"RUN COMPACTION ON $namespace1.$table1"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(compactionTable))
+ }(s"does not have [select] privilege on [$namespace1/$table1]")
+ doAs(admin, sql(compactionTable))
+
+ val showCompactionTable = s"SHOW COMPACTION ON $namespace1.$table1"
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(showCompactionTable))
+ }(s"does not have [select] privilege on [$namespace1/$table1]")
+ doAs(admin, sql(showCompactionTable))
+ }
+ }
}
