This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 411df6043 [KYUUBI #5360] Support Hudi InsertIntoHoodieTableCommand
411df6043 is described below
commit 411df6043176ffca8673908da902bc9308f6c695
Author: Angerszhuuuu <[email protected]>
AuthorDate: Thu Oct 19 10:13:12 2023 +0800
[KYUUBI #5360] Support Hudi InsertIntoHoodieTableCommand
### _Why are the changes needed?_
To close #5360
Support Compaction table/path related command. The SQL grammar is
https://github.com/apache/hudi/blob/release-0.14.0/hudi-spark-datasource/hudi-spark/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlCommon.g4
- InsertIntoHoodieTableCommand
:https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/InsertIntoHoodieTableCommand.scala
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No
Closes #5456 from AngersZhuuuu/KYUUBI-5360.
Closes #5360
ebeb3f270 [Angerszhuuuu] Update table_command_spec.json
fcdc51880 [Angerszhuuuu] Merge branch 'master' into KYUUBI-5360
d1281ca66 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
5a6ce81fb [Angerszhuuuu] Merge branch 'master' into KYUUBI-5360
ead082e39 [Angerszhuuuu] [KYUUBI #5360 ] Support Hudi
InsertIntoHoodieTableCommand
Authored-by: Angerszhuuuu <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../src/main/resources/table_command_spec.json | 21 ++++++++++
.../plugin/spark/authz/gen/HudiCommands.scala | 11 ++++++
.../HudiCatalogRangerSparkExtensionSuite.scala | 46 ++++++++++++++++++++++
3 files changed, 78 insertions(+)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index 87831119e..2d7199ff9 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -1622,6 +1622,27 @@
} ],
"opType" : "DROPTABLE",
"queryDescs" : [ ]
+}, {
+ "classname" :
"org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "logicalRelation",
+ "fieldExtractor" : "LogicalRelationTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : {
+ "fieldName" : "overwrite",
+ "fieldExtractor" : "OverwriteOrInsertActionTypeExtractor",
+ "actionType" : null
+ },
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "QUERY",
+ "queryDescs" : [ {
+ "fieldName" : "query",
+ "fieldExtractor" : "LogicalPlanQueryExtractor"
+ } ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand",
"tableDescs" : [ {
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
index 3abac4a88..0b19204f5 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
@@ -144,6 +144,16 @@ object HudiCommands {
TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES)
}
+ val InsertIntoHoodieTableCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand"
+ val tableDesc = TableDesc(
+ "logicalRelation",
+ classOf[LogicalRelationTableExtractor],
+ actionTypeDesc =
+ Some(ActionTypeDesc("overwrite",
classOf[OverwriteOrInsertActionTypeExtractor])))
+ TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(QueryDesc("query")))
+ }
+
val data: Array[TableCommandSpec] = Array(
AlterHoodieTableAddColumnsCommand,
AlterHoodieTableChangeColumnCommand,
@@ -156,6 +166,7 @@ object HudiCommands {
CompactionHoodieTableCommand,
CompactionShowHoodieTableCommand,
DropHoodieTableCommand,
+ InsertIntoHoodieTableCommand,
RepairHoodieTableCommand,
TruncateHoodieTableCommand,
Spark31AlterTableCommand)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
index 7d15709f8..e707f0c9e 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
@@ -324,4 +324,50 @@ class HudiCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite {
doAs(admin, sql(showCompactionTable))
}
}
+
+ test("InsertIntoHoodieTableCommand") {
+ withSingleCallEnabled {
+ withCleanTmpResources(Seq(
+ (s"$namespace1.$table1", "table"),
+ (s"$namespace1.$table2", "table"),
+ (namespace1, "database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name
string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table2(id int, name
string, city string)
+ |USING $format
+ |""".stripMargin))
+
+ val insertIntoHoodieTableSql =
+ s"""
+ |INSERT INTO $namespace1.$table1
+ |PARTITION(city = 'hangzhou')
+ |SELECT id, name
+ |FROM $namespace1.$table2
+ |WHERE city = 'hangzhou'
+ |""".stripMargin
+ interceptContains[AccessControlException] {
+ doAs(someone, sql(insertIntoHoodieTableSql))
+ }(s"does not have [select] privilege on " +
+
s"[$namespace1/$table2/id,$namespace1/$table2/name,hudi_ns/$table2/city], " +
+ s"[update] privilege on [$namespace1/$table1]")
+ }
+ }
+ }
}
