This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 810a41c54 [KYUUBI #4871] [AUTHZ] Adapt plan changes for
CreateNamespace and SetCatalogAndNamespace in Spark 3.4
810a41c54 is described below
commit 810a41c54e4012d259302289d6da1fa5ecccf603
Author: liangbowen <[email protected]>
AuthorDate: Fri May 26 09:26:10 2023 +0800
[KYUUBI #4871] [AUTHZ] Adapt plan changes for CreateNamespace and
SetCatalogAndNamespace in Spark 3.4
### _Why are the changes needed?_
- namespace changed from `Seq[String]` to `ResolvedNamespace` in Spark 3.4
- fixing uts in the name of `CreateNamespace` and `Extracting database info
with ResolvedDBObjectNameDatabaseExtractor` w/ Spark 3.4 for CreateNamespace
and SetCatalogAndNamespace commands
Testing by running `build/mvn clean install -pl :kyuubi-spark-authz_2.12
-Pspark-3.4
-Dmaven.plugin.scalatest.exclude.tags=org.apache.kyuubi.tags.IcebergTest`.
Before:
```
- CreateNamespace *** FAILED ***
0 did not equal 1 (V2CommandsPrivilegesSuite.scala:707)
...
- Extracting database info with ResolvedDBObjectNameDatabaseExtractor ***
FAILED ***
java.lang.NullPointerException:
at
org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite.$anonfun$new$24(V2JdbcTableCatalogPrivilegesBuilderSuite.scala:151)
at org.scalatest.Assertions.withClue(Assertions.scala:1065)
at org.scalatest.Assertions.withClue$(Assertions.scala:1052)
at org.scalatest.funsuite.AnyFunSuite.withClue(AnyFunSuite.scala:1564)
at
org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite.$anonfun$new$21(V2JdbcTableCatalogPrivilegesBuilderSuite.scala:150)
at scala.collection.immutable.List.foreach(List.scala:431)
at
org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite.$anonfun$new$20(V2JdbcTableCatalogPrivilegesBuilderSuite.scala:143)
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
```
After:
```
- CreateNamespace
- Extracting database info with ResolvedDBObjectNameDatabaseExtractor
```
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #4871 from bowenliang123/create-ns.
Closes #4871
fa141fb5a [liangbowen] update spec json
2cd18f829 [liangbowen] use meaningful desc names
142224d1e [liangbowen] Adapt changes for CreateNamespace and
SetCatalogAndNamespace
Authored-by: liangbowen <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
.../src/main/resources/database_command_spec.json | 10 ++++++++++
.../plugin/spark/authz/gen/DatabaseCommands.scala | 17 +++++++++++++----
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
index 4eb4b3ef8..c640ed89b 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
@@ -22,6 +22,11 @@
"fieldExtractor" : "CatalogPluginCatalogExtractor"
},
"isInput" : false
+ }, {
+ "fieldName" : "name",
+ "fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
+ "catalogDesc" : null,
+ "isInput" : false
} ],
"opType" : "CREATEDATABASE"
}, {
@@ -45,6 +50,11 @@
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace",
"databaseDescs" : [ {
+ "fieldName" : "child",
+ "fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
+ "catalogDesc" : null,
+ "isInput" : true
+ }, {
"fieldName" : "child",
"fieldExtractor" : "ResolvedDBObjectNameDatabaseExtractor",
"catalogDesc" : null,
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala
index e947579e9..a61c142ed 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala
@@ -58,9 +58,10 @@ object DatabaseCommands {
"namespace",
classOf[StringSeqDatabaseExtractor],
catalogDesc = Some(CatalogDesc()))
+ val databaseDesc3 = DatabaseDesc("name",
classOf[ResolvedNamespaceDatabaseExtractor])
DatabaseCommandSpec(
"org.apache.spark.sql.catalyst.plans.logical.CreateNamespace",
- Seq(databaseDesc1, databaseDesc2),
+ Seq(databaseDesc1, databaseDesc2, databaseDesc3),
CREATEDATABASE)
}
@@ -97,12 +98,12 @@ object DatabaseCommands {
val SetCatalogAndNamespace = {
val cmd =
"org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace"
- val databaseDesc1 =
+ val resolvedDbObjectDatabaseDesc =
DatabaseDesc(
"child",
classOf[ResolvedDBObjectNameDatabaseExtractor],
isInput = true)
- val databaseDesc2 =
+ val stringSeqOptionDatabaseDesc =
DatabaseDesc(
"namespace",
classOf[StringSeqOptionDatabaseExtractor],
@@ -110,7 +111,15 @@ object DatabaseCommands {
fieldName = "catalogName",
fieldExtractor = classOf[StringOptionCatalogExtractor])),
isInput = true)
- DatabaseCommandSpec(cmd, Seq(databaseDesc1, databaseDesc2), SWITCHDATABASE)
+ val resolvedNamespaceDatabaseDesc =
+ DatabaseDesc(
+ "child",
+ classOf[ResolvedNamespaceDatabaseExtractor],
+ isInput = true)
+ DatabaseCommandSpec(
+ cmd,
+ Seq(resolvedNamespaceDatabaseDesc, resolvedDbObjectDatabaseDesc,
stringSeqOptionDatabaseDesc),
+ SWITCHDATABASE)
}
val SetNamespace = {
