This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new c8f4e9c6c [KYUUBI #5255] Add an optional comment field to the authz
specs for better recognition
c8f4e9c6c is described below
commit c8f4e9c6c5e9f9125666ad8b032fb2aaedea762e
Author: davidyuan <[email protected]>
AuthorDate: Mon Nov 27 09:38:56 2023 +0800
[KYUUBI #5255] Add an optional comment field to the authz specs for better
recognition
# :mag: Description
## Issue References ๐
This pull request fixes #5255
## Describe Your Solution ๐ง
Add Optional comment field for desc, so that we can get information when
the command from third party
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
#### Behavior With This Pull Request :tada:
#### Related Unit Tests
---
# Checklists
## ๐ Author Self Checklist
- [x] My code follows the [style
guidelines](https://kyuubi.readthedocs.io/en/master/contributing/code/style.html)
of this project
- [x] I have performed a self-review
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature
works
- [ ] New and existing unit tests pass locally with my changes
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
## ๐ Committer Pre-Merge Checklist
- [x] Pull request title is okay.
- [x] No license issues.
- [x] Milestone correctly set?
- [x] Test coverage is ok
- [x] Assignees are selected.
- [x] Minimum number of approvals
- [x] No changes are requested
**Be nice. Be informative.**
Closes #5706 from davidyuan1223/add_option_comment_for_authz.
Closes #5255
46234e10d [davidyuan] bugfix
b9166bd33 [davidyuan] bugfix
2d4b920db [davidyuan] regenerate json file
a125ee4b8 [david yuan] Merge branch 'master' into
add_option_comment_for_authz
80eb9ff04 [davidyuan] json file bug fix
bc6ec388d [davidyuan] style fix
b2527a558 [davidyuan] Merge remote-tracking branch
'origin/add_option_comment_for_authz' into add_option_comment_for_authz
682727d77 [davidyuan] bugfix
d79a38f12 [david yuan] Merge branch 'master' into
add_option_comment_for_authz
6fed8b0c7 [davidyuan] issue #5255
e48c5a087 [davidyuan] issue #5255
Lead-authored-by: davidyuan <[email protected]>
Co-authored-by: david yuan <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../src/main/resources/database_command_spec.json | 77 ++-
.../src/main/resources/function_command_spec.json | 50 +-
.../src/main/resources/scan_command_spec.json | 41 +-
.../src/main/resources/table_command_spec.json | 764 ++++++++++++++-------
.../plugin/spark/authz/serde/Descriptor.scala | 39 +-
.../plugin/spark/authz/gen/DeltaCommands.scala | 31 +-
.../plugin/spark/authz/gen/HudiCommands.scala | 101 ++-
.../plugin/spark/authz/gen/IcebergCommands.scala | 8 +-
8 files changed, 736 insertions(+), 375 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
index c45e63890..3918186ac 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json
@@ -4,7 +4,8 @@
"fieldName" : "child",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "ALTERDATABASE",
"uriDescs" : [ ]
@@ -14,26 +15,31 @@
"fieldName" : "name",
"fieldExtractor" : "ResolvedDBObjectNameDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "namespace",
"fieldExtractor" : "StringSeqDatabaseExtractor",
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "name",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "CREATEDATABASE",
"uriDescs" : [ {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace",
@@ -41,7 +47,8 @@
"fieldName" : "namespace",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "DESCDATABASE",
"uriDescs" : [ ]
@@ -51,7 +58,8 @@
"fieldName" : "namespace",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "DROPDATABASE",
"uriDescs" : [ ]
@@ -61,20 +69,24 @@
"fieldName" : "child",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
}, {
"fieldName" : "child",
"fieldExtractor" : "ResolvedDBObjectNameDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
}, {
"fieldName" : "namespace",
"fieldExtractor" : "StringSeqOptionDatabaseExtractor",
"catalogDesc" : {
"fieldName" : "catalogName",
- "fieldExtractor" : "StringOptionCatalogExtractor"
+ "fieldExtractor" : "StringOptionCatalogExtractor",
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "SWITCHDATABASE",
"uriDescs" : [ ]
@@ -84,13 +96,15 @@
"fieldName" : "namespace",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "ALTERDATABASE_LOCATION",
"uriDescs" : [ {
"fieldName" : "location",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.SetNamespaceProperties",
@@ -98,7 +112,8 @@
"fieldName" : "namespace",
"fieldExtractor" : "ResolvedNamespaceDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "ALTERDATABASE",
"uriDescs" : [ ]
@@ -108,7 +123,8 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "ALTERDATABASE",
"uriDescs" : [ ]
@@ -118,13 +134,15 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "ALTERDATABASE_LOCATION",
"uriDescs" : [ {
"fieldName" : "location",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.AnalyzeTablesCommand",
@@ -132,7 +150,8 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringOptionDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "ANALYZE_TABLE",
"uriDescs" : [ ]
@@ -142,13 +161,15 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "CREATEDATABASE",
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.DescribeDatabaseCommand",
@@ -156,7 +177,8 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "DESCDATABASE",
"uriDescs" : [ ]
@@ -166,7 +188,8 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "DROPDATABASE",
"uriDescs" : [ ]
@@ -176,7 +199,8 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "SWITCHDATABASE",
"uriDescs" : [ ]
@@ -186,8 +210,9 @@
"fieldName" : "namespace",
"fieldExtractor" : "StringSeqDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "SWITCHDATABASE",
"uriDescs" : [ ]
-} ]
\ No newline at end of file
+} ]
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json
index 0b71245d2..8644f9860 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json
@@ -7,9 +7,11 @@
"functionTypeDesc" : {
"fieldName" : "isTemp",
"fieldExtractor" : "TempMarkerFunctionTypeExtractor",
- "skipTypes" : [ "TEMP" ]
+ "skipTypes" : [ "TEMP" ],
+ "comment" : ""
},
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "functionName",
"fieldExtractor" : "StringFunctionExtractor",
@@ -17,14 +19,17 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringOptionDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
},
"functionTypeDesc" : {
"fieldName" : "isTemp",
"fieldExtractor" : "TempMarkerFunctionTypeExtractor",
- "skipTypes" : [ "TEMP" ]
+ "skipTypes" : [ "TEMP" ],
+ "comment" : ""
},
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "CREATEFUNCTION"
}, {
@@ -36,9 +41,11 @@
"functionTypeDesc" : {
"fieldName" : "info",
"fieldExtractor" : "ExpressionInfoFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
}, {
"fieldName" : "functionName",
"fieldExtractor" : "FunctionIdentifierFunctionExtractor",
@@ -46,9 +53,11 @@
"functionTypeDesc" : {
"fieldName" : "functionName",
"fieldExtractor" : "FunctionIdentifierFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"opType" : "DESCFUNCTION"
}, {
@@ -60,9 +69,11 @@
"functionTypeDesc" : {
"fieldName" : "isTemp",
"fieldExtractor" : "TempMarkerFunctionTypeExtractor",
- "skipTypes" : [ "TEMP" ]
+ "skipTypes" : [ "TEMP" ],
+ "comment" : ""
},
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "functionName",
"fieldExtractor" : "StringFunctionExtractor",
@@ -70,14 +81,17 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringOptionDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
},
"functionTypeDesc" : {
"fieldName" : "isTemp",
"fieldExtractor" : "TempMarkerFunctionTypeExtractor",
- "skipTypes" : [ "TEMP" ]
+ "skipTypes" : [ "TEMP" ],
+ "comment" : ""
},
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "DROPFUNCTION"
}, {
@@ -89,10 +103,12 @@
"fieldName" : "databaseName",
"fieldExtractor" : "StringOptionDatabaseExtractor",
"catalogDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
},
"functionTypeDesc" : null,
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ],
"opType" : "RELOADFUNCTION"
-} ]
\ No newline at end of file
+} ]
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
index 75510a577..ba4d790e8 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
@@ -3,7 +3,8 @@
"scanDescs" : [ {
"fieldName" : "catalogTable",
"fieldExtractor" : "CatalogTableTableExtractor",
- "catalogDesc" : null
+ "catalogDesc" : null,
+ "comment" : ""
} ],
"functionDescs" : [ ],
"uriDescs" : [ ]
@@ -12,7 +13,8 @@
"scanDescs" : [ {
"fieldName" : "tableMeta",
"fieldExtractor" : "CatalogTableTableExtractor",
- "catalogDesc" : null
+ "catalogDesc" : null,
+ "comment" : ""
} ],
"functionDescs" : [ ],
"uriDescs" : [ ]
@@ -21,20 +23,23 @@
"scanDescs" : [ {
"fieldName" : "catalogTable",
"fieldExtractor" : "CatalogTableOptionTableExtractor",
- "catalogDesc" : null
+ "catalogDesc" : null,
+ "comment" : ""
} ],
"functionDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "relation",
"fieldExtractor" : "BaseRelationFileIndexURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation",
"scanDescs" : [ {
"fieldName" : null,
"fieldExtractor" : "DataSourceV2RelationTableExtractor",
- "catalogDesc" : null
+ "catalogDesc" : null,
+ "comment" : ""
} ],
"functionDescs" : [ ],
"uriDescs" : [ ]
@@ -48,9 +53,11 @@
"functionTypeDesc" : {
"fieldName" : "name",
"fieldExtractor" : "FunctionNameFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -63,9 +70,11 @@
"functionTypeDesc" : {
"fieldName" : "name",
"fieldExtractor" : "FunctionNameFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -78,9 +87,11 @@
"functionTypeDesc" : {
"fieldName" : "name",
"fieldExtractor" : "FunctionNameFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -93,9 +104,11 @@
"functionTypeDesc" : {
"fieldName" : "name",
"fieldExtractor" : "FunctionNameFunctionTypeExtractor",
- "skipTypes" : [ "TEMP", "SYSTEM" ]
+ "skipTypes" : [ "TEMP", "SYSTEM" ],
+ "comment" : ""
},
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ],
"uriDescs" : [ ]
-} ]
\ No newline at end of file
+} ]
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index e4fb863c3..027f0a4e0 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -8,7 +8,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDCOLS",
"queryDescs" : [ ],
@@ -23,7 +24,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDPARTS",
"queryDescs" : [ ],
@@ -38,7 +40,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDCOLS",
"queryDescs" : [ ],
@@ -53,7 +56,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -67,22 +71,26 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "INSERT"
+ "actionType" : "INSERT",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "DataSourceV2RelationURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.catalyst.plans.logical.CacheTable",
@@ -90,7 +98,8 @@
"opType" : "CREATEVIEW",
"queryDescs" : [ {
"fieldName" : "table",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -99,7 +108,8 @@
"opType" : "CREATEVIEW",
"queryDescs" : [ {
"fieldName" : "plan",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -112,7 +122,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -127,7 +138,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierTableExtractor",
@@ -136,10 +148,12 @@
"tableTypeDesc" : null,
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "child",
"fieldExtractor" : "ResolvedDbObjectNameTableExtractor",
@@ -148,22 +162,26 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "tableSpec",
"fieldExtractor" : "TableSpecURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelect",
@@ -175,7 +193,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierTableExtractor",
@@ -184,10 +203,12 @@
"tableTypeDesc" : null,
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "name",
"fieldExtractor" : "ResolvedDbObjectNameTableExtractor",
@@ -196,21 +217,25 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "tableSpec",
"fieldExtractor" : "TableSpecURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateV2Table",
@@ -222,21 +247,25 @@
"tableTypeDesc" : null,
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable",
@@ -247,12 +276,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -267,7 +298,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "DESCTABLE",
"queryDescs" : [ ],
@@ -282,7 +314,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDCOLS",
"queryDescs" : [ ],
@@ -297,7 +330,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_DROPPARTS",
"queryDescs" : [ ],
@@ -312,7 +346,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "child",
"fieldExtractor" : "ResolvedTableTableExtractor",
@@ -321,7 +356,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "DROPTABLE",
"queryDescs" : [ ],
@@ -335,17 +371,20 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "sourceTable",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -357,22 +396,26 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "INSERT_OVERWRITE"
+ "actionType" : "INSERT_OVERWRITE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "DataSourceV2RelationURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic",
@@ -383,22 +426,26 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "INSERT_OVERWRITE"
+ "actionType" : "INSERT_OVERWRITE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "DataSourceV2RelationURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.catalyst.plans.logical.RefreshTable",
@@ -410,7 +457,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -425,7 +473,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_RENAMECOL",
"queryDescs" : [ ],
@@ -440,7 +489,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_RENAMEPART",
"queryDescs" : [ ],
@@ -455,7 +505,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
@@ -470,7 +521,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_REPLACECOLS",
"queryDescs" : [ ],
@@ -484,17 +536,20 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -507,7 +562,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierTableExtractor",
@@ -516,10 +572,12 @@
"tableTypeDesc" : null,
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "child",
"fieldExtractor" : "ResolvedDbObjectNameTableExtractor",
@@ -528,22 +586,26 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "tableSpec",
"fieldExtractor" : "TableSpecURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.ReplaceTableAsSelect",
@@ -555,7 +617,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableName",
"fieldExtractor" : "IdentifierTableExtractor",
@@ -564,10 +627,12 @@
"tableTypeDesc" : null,
"catalogDesc" : {
"fieldName" : "catalog",
- "fieldExtractor" : "CatalogPluginCatalogExtractor"
+ "fieldExtractor" : "CatalogPluginCatalogExtractor",
+ "comment" : ""
},
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "name",
"fieldExtractor" : "ResolvedDbObjectNameTableExtractor",
@@ -576,21 +641,25 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "tableSpec",
"fieldExtractor" : "TableSpecURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
}, {
"fieldName" : "properties",
"fieldExtractor" : "PropertiesLocationUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.catalyst.plans.logical.SetTableProperties",
@@ -602,7 +671,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -617,7 +687,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOW_CREATETABLE",
"queryDescs" : [ ],
@@ -632,7 +703,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOW_TBLPROPERTIES",
"queryDescs" : [ ],
@@ -647,7 +719,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_DROPPARTS",
"queryDescs" : [ ],
@@ -662,7 +735,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "TRUNCATETABLE",
"queryDescs" : [ ],
@@ -676,12 +750,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -694,7 +770,8 @@
"uriDescs" : [ {
"fieldName" : "paths",
"fieldExtractor" : "StringSeqURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.AddFileCommand",
@@ -704,7 +781,8 @@
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.AddFilesCommand",
@@ -714,7 +792,8 @@
"uriDescs" : [ {
"fieldName" : "paths",
"fieldExtractor" : "StringSeqURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.AddJarCommand",
@@ -724,7 +803,8 @@
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.AddJarsCommand",
@@ -734,7 +814,8 @@
"uriDescs" : [ {
"fieldName" : "paths",
"fieldExtractor" : "StringSeqURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand",
@@ -743,13 +824,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "colsToAdd",
- "fieldExtractor" : "StructFieldSeqColumnExtractor"
+ "fieldExtractor" : "StructFieldSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDCOLS",
"queryDescs" : [ ],
@@ -761,20 +844,23 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpecsAndLocs",
- "fieldExtractor" : "PartitionLocsSeqColumnExtractor"
+ "fieldExtractor" : "PartitionLocsSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_ADDPARTS",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "partitionSpecsAndLocs",
"fieldExtractor" : "PartitionLocsSeqURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.AlterTableChangeColumnCommand",
@@ -783,13 +869,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "columnName",
- "fieldExtractor" : "StringColumnExtractor"
+ "fieldExtractor" : "StringColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_REPLACECOLS",
"queryDescs" : [ ],
@@ -801,13 +889,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "specs",
- "fieldExtractor" : "PartitionSeqColumnExtractor"
+ "fieldExtractor" : "PartitionSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_DROPPARTS",
"queryDescs" : [ ],
@@ -822,7 +912,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
@@ -837,11 +928,13 @@
"tableTypeDesc" : {
"fieldName" : "oldName",
"fieldExtractor" : "TableIdentifierTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW" ],
+ "comment" : ""
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_RENAME",
"queryDescs" : [ ],
@@ -853,13 +946,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "oldPartition",
- "fieldExtractor" : "PartitionColumnExtractor"
+ "fieldExtractor" : "PartitionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_RENAMEPART",
"queryDescs" : [ ],
@@ -871,13 +966,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partSpec",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_SERDEPROPERTIES",
"queryDescs" : [ ],
@@ -889,20 +986,23 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpec",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_LOCATION",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "location",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.AlterTableSetPropertiesCommand",
@@ -914,7 +1014,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -929,7 +1030,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -944,16 +1046,19 @@
"tableTypeDesc" : {
"fieldName" : "name",
"fieldExtractor" : "TableIdentifierTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW" ],
+ "comment" : ""
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERVIEW_AS",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -966,31 +1071,36 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableIdent",
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "columnNames",
- "fieldExtractor" : "StringSeqColumnExtractor"
+ "fieldExtractor" : "StringSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableIdent",
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "columnNames",
- "fieldExtractor" : "StringSeqOptionColumnExtractor"
+ "fieldExtractor" : "StringSeqOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -1005,19 +1115,22 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableIdent",
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpec",
- "fieldExtractor" : "PartitionColumnExtractor"
+ "fieldExtractor" : "PartitionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -1032,7 +1145,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
}, {
"fieldName" : "tableIdent",
"fieldExtractor" : "TableIdentifierTableExtractor",
@@ -1041,7 +1155,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -1052,7 +1167,8 @@
"opType" : "CREATEVIEW",
"queryDescs" : [ {
"fieldName" : "plan",
- "fieldExtractor" : "LogicalPlanOptionQueryExtractor"
+ "fieldExtractor" : "LogicalPlanOptionQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1065,17 +1181,20 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.CreateDataSourceTableCommand",
@@ -1087,14 +1206,16 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.CreateTableCommand",
@@ -1106,14 +1227,16 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.command.CreateTableLikeCommand",
@@ -1125,7 +1248,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
}, {
"fieldName" : "sourceTable",
"fieldExtractor" : "TableIdentifierTableExtractor",
@@ -1134,14 +1258,16 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "fileFormat",
"fieldExtractor" : "CatalogStorageFormatURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.CreateViewCommand",
@@ -1153,19 +1279,23 @@
"tableTypeDesc" : {
"fieldName" : "viewType",
"fieldExtractor" : "ViewTypeTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW", "GLOBAL_TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW", "GLOBAL_TEMP_VIEW" ],
+ "comment" : ""
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATEVIEW",
"queryDescs" : [ {
"fieldName" : "plan",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
}, {
"fieldName" : "child",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1175,13 +1305,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "colNameParts",
- "fieldExtractor" : "StringSeqLastColumnExtractor"
+ "fieldExtractor" : "StringSeqLastColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "DESCTABLE",
"queryDescs" : [ ],
@@ -1193,13 +1325,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpec",
- "fieldExtractor" : "PartitionColumnExtractor"
+ "fieldExtractor" : "PartitionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "DESCTABLE",
"queryDescs" : [ ],
@@ -1214,11 +1348,13 @@
"tableTypeDesc" : {
"fieldName" : "tableName",
"fieldExtractor" : "TableIdentifierTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW" ],
+ "comment" : ""
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "DROPTABLE",
"queryDescs" : [ ],
@@ -1229,12 +1365,14 @@
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "storage",
"fieldExtractor" : "CatalogStorageFormatURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.LoadDataCommand",
@@ -1243,24 +1381,28 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partition",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : {
"fieldName" : "isOverwrite",
"fieldExtractor" : "OverwriteOrInsertActionTypeExtractor",
- "actionType" : null
+ "actionType" : null,
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "LOAD",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.execution.command.RefreshTableCommand",
@@ -1272,7 +1414,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1287,7 +1430,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
@@ -1302,7 +1446,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOWCOLUMNS",
"queryDescs" : [ ],
@@ -1317,7 +1462,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOW_CREATETABLE",
"queryDescs" : [ ],
@@ -1332,7 +1478,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOW_CREATETABLE",
"queryDescs" : [ ],
@@ -1344,13 +1491,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "spec",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOWPARTITIONS",
"queryDescs" : [ ],
@@ -1365,7 +1514,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "SHOW_TBLPROPERTIES",
"queryDescs" : [ ],
@@ -1377,13 +1527,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpec",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "TRUNCATETABLE",
"queryDescs" : [ ],
@@ -1398,17 +1550,20 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanOptionQueryExtractor"
+ "fieldExtractor" : "LogicalPlanOptionQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "tableDesc",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.execution.datasources.CreateTempViewUsing",
@@ -1425,17 +1580,20 @@
"actionTypeDesc" : {
"fieldName" : "overwrite",
"fieldExtractor" : "OverwriteOrInsertActionTypeExtractor",
- "actionType" : null
+ "actionType" : null,
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1445,22 +1603,26 @@
"fieldExtractor" : "CatalogTableOptionTableExtractor",
"columnDesc" : {
"fieldName" : "outputColumnNames",
- "fieldExtractor" : "StringSeqColumnExtractor"
+ "fieldExtractor" : "StringSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : {
"fieldName" : "mode",
"fieldExtractor" : "SaveModeActionTypeExtractor",
- "actionType" : null
+ "actionType" : null,
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1473,7 +1635,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1484,12 +1647,14 @@
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "options",
"fieldExtractor" : "PropertiesPathUriExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" :
"org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand",
@@ -1498,23 +1663,27 @@
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : {
"fieldName" : "outputColumnNames",
- "fieldExtractor" : "StringSeqColumnExtractor"
+ "fieldExtractor" : "StringSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "tableDesc",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand",
@@ -1522,12 +1691,14 @@
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "storage",
"fieldExtractor" : "CatalogStorageFormatURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveTable",
@@ -1536,22 +1707,26 @@
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : {
"fieldName" : "outputColumnNames",
- "fieldExtractor" : "StringSeqColumnExtractor"
+ "fieldExtractor" : "StringSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : {
"fieldName" : "overwrite",
"fieldExtractor" : "OverwriteOrInsertActionTypeExtractor",
- "actionType" : null
+ "actionType" : null,
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1561,23 +1736,27 @@
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : {
"fieldName" : "outputColumnNames",
- "fieldExtractor" : "StringSeqColumnExtractor"
+ "fieldExtractor" : "StringSeqColumnExtractor",
+ "comment" : ""
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ {
"fieldName" : "tableDesc",
"fieldExtractor" : "CatalogTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : ""
} ]
}, {
"classname" : "org.apache.spark.sql.catalyst.plans.logical.Call",
@@ -1589,7 +1768,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Iceberg"
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -1603,12 +1783,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Iceberg"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1622,17 +1804,20 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Iceberg"
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "sourceTable",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1644,17 +1829,20 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Iceberg"
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "sourceTable",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1666,12 +1854,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Iceberg"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1683,13 +1873,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "colsToAdd",
- "fieldExtractor" : "StructFieldSeqColumnExtractor"
+ "fieldExtractor" : "StructFieldSeqColumnExtractor",
+ "comment" : "Hudi"
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_ADDCOLS",
"queryDescs" : [ ],
@@ -1701,13 +1893,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "columnName",
- "fieldExtractor" : "StringColumnExtractor"
+ "fieldExtractor" : "StringColumnExtractor",
+ "comment" : "Hudi"
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_REPLACECOLS",
"queryDescs" : [ ],
@@ -1719,13 +1913,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpecs",
- "fieldExtractor" : "PartitionSeqColumnExtractor"
+ "fieldExtractor" : "PartitionSeqColumnExtractor",
+ "comment" : "Hudi"
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_DROPPARTS",
"queryDescs" : [ ],
@@ -1740,11 +1936,13 @@
"tableTypeDesc" : {
"fieldName" : "oldName",
"fieldExtractor" : "TableIdentifierTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW" ],
+ "comment" : "Hudi"
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_RENAME",
"queryDescs" : [ ],
@@ -1759,7 +1957,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -1773,12 +1972,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "OTHER"
+ "actionType" : "OTHER",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
}, {
"fieldName" : "clone",
"fieldExtractor" : "HudiCallProcedureOutputTableExtractor",
@@ -1786,12 +1987,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : ""
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : ""
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1804,7 +2007,8 @@
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Hudi"
} ]
}, {
"classname" :
"org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand",
@@ -1816,7 +2020,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
@@ -1829,7 +2034,8 @@
"uriDescs" : [ {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : true
+ "isInput" : true,
+ "comment" : "Hudi"
} ]
}, {
"classname" :
"org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand",
@@ -1841,7 +2047,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "SHOW_TBLPROPERTIES",
"queryDescs" : [ ],
@@ -1856,12 +2063,14 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "CREATETABLE_AS_SELECT",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -1874,7 +2083,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
@@ -1889,7 +2099,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : "Hudi"
}, {
"fieldName" : "sourceTable",
"fieldExtractor" : "TableIdentifierTableExtractor",
@@ -1898,7 +2109,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : true
+ "setCurrentDatabaseIfMissing" : true,
+ "comment" : "Hudi"
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ],
@@ -1913,7 +2125,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "CREATEINDEX",
"queryDescs" : [ ],
@@ -1927,12 +2140,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Hudi"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -1947,11 +2162,13 @@
"tableTypeDesc" : {
"fieldName" : "tableIdentifier",
"fieldExtractor" : "TableIdentifierTableTypeExtractor",
- "skipTypes" : [ "TEMP_VIEW" ]
+ "skipTypes" : [ "TEMP_VIEW" ],
+ "comment" : "Hudi"
},
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "DROPTABLE",
"queryDescs" : [ ],
@@ -1966,7 +2183,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "DROPINDEX",
"queryDescs" : [ ],
@@ -1980,17 +2198,20 @@
"actionTypeDesc" : {
"fieldName" : "overwrite",
"fieldExtractor" : "OverwriteOrInsertActionTypeExtractor",
- "actionType" : null
+ "actionType" : null,
+ "comment" : "Hudi"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "query",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : ""
} ],
"uriDescs" : [ ]
}, {
@@ -2002,17 +2223,20 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Hudi"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "mergeInto",
- "fieldExtractor" : "HudiMergeIntoSourceTableExtractor"
+ "fieldExtractor" : "HudiMergeIntoSourceTableExtractor",
+ "comment" : "Hudi"
} ],
"uriDescs" : [ ]
}, {
@@ -2025,7 +2249,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERINDEX_REBUILD",
"queryDescs" : [ ],
@@ -2040,7 +2265,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : ""
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
@@ -2052,13 +2278,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "specOpt",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : "Hudi"
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "SHOWPARTITIONS",
"queryDescs" : [ ],
@@ -2073,7 +2301,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "SHOWINDEXES",
"queryDescs" : [ ],
@@ -2088,7 +2317,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ],
@@ -2100,13 +2330,15 @@
"fieldExtractor" : "TableIdentifierTableExtractor",
"columnDesc" : {
"fieldName" : "partitionSpec",
- "fieldExtractor" : "PartitionOptionColumnExtractor"
+ "fieldExtractor" : "PartitionOptionColumnExtractor",
+ "comment" : "Hudi"
},
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "TRUNCATETABLE",
"queryDescs" : [ ],
@@ -2120,12 +2352,14 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Hudi"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Hudi"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
@@ -2140,7 +2374,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "table",
"fieldExtractor" : "TableIdentifierOptionTableExtractor",
@@ -2149,22 +2384,26 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "child",
"fieldExtractor" : "ResolvedTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "table",
"fieldExtractor" : "TableIdentifierOptionURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
} ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.DeleteCommand",
@@ -2175,19 +2414,22 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Delta"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "target",
"fieldExtractor" : "SubqueryAliasURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
} ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.MergeIntoCommand",
@@ -2198,22 +2440,26 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Delta"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
} ],
"opType" : "QUERY",
"queryDescs" : [ {
"fieldName" : "source",
- "fieldExtractor" : "LogicalPlanQueryExtractor"
+ "fieldExtractor" : "LogicalPlanQueryExtractor",
+ "comment" : "Delta"
} ],
"uriDescs" : [ {
"fieldName" : "target",
"fieldExtractor" : "SubqueryAliasURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
} ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.OptimizeTableCommand",
@@ -2225,7 +2471,8 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "tableId",
"fieldExtractor" : "TableIdentifierOptionTableExtractor",
@@ -2234,22 +2481,26 @@
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
} ],
"opType" : "ALTERTABLE_COMPACT",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "child",
"fieldExtractor" : "ResolvedTableURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "tableId",
"fieldExtractor" : "TableIdentifierOptionURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
}, {
"fieldName" : "path",
"fieldExtractor" : "StringURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
} ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.UpdateCommand",
@@ -2260,18 +2511,21 @@
"actionTypeDesc" : {
"fieldName" : null,
"fieldExtractor" : null,
- "actionType" : "UPDATE"
+ "actionType" : "UPDATE",
+ "comment" : "Delta"
},
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
- "setCurrentDatabaseIfMissing" : false
+ "setCurrentDatabaseIfMissing" : false,
+ "comment" : "Delta"
} ],
"opType" : "QUERY",
"queryDescs" : [ ],
"uriDescs" : [ {
"fieldName" : "target",
"fieldExtractor" : "SubqueryAliasURIExtractor",
- "isInput" : false
+ "isInput" : false,
+ "comment" : "Delta"
} ]
-} ]
\ No newline at end of file
+} ]
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
index 2ffad1a2f..4c0cf2a14 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
@@ -54,11 +54,15 @@ sealed trait Descriptor {
*/
def extract(v: AnyRef): AnyRef
+ def comment: String
+
final def error(v: AnyRef, e: Throwable): String = {
val resourceName = getClass.getSimpleName.stripSuffix("Desc")
val objectClass = v.getClass.getName
s"[Spark$SPARK_VERSION] failed to get $resourceName from $objectClass by" +
- s" $fieldExtractor/$fieldName, due to ${e.getMessage}"
+ s" $fieldExtractor/$fieldName, " +
+ (if (comment.nonEmpty) s"desc comment: ${comment}") +
+ s"due to ${e.getMessage}"
}
}
@@ -70,7 +74,8 @@ sealed trait Descriptor {
*/
case class ColumnDesc(
fieldName: String,
- fieldExtractor: String) extends Descriptor {
+ fieldExtractor: String,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Seq[String] = {
val columnsVal = invokeAs[AnyRef](v, fieldName)
val columnExtractor = lookupExtractor[ColumnExtractor](fieldExtractor)
@@ -89,7 +94,8 @@ case class DatabaseDesc(
fieldName: String,
fieldExtractor: String,
catalogDesc: Option[CatalogDesc] = None,
- isInput: Boolean = false) extends Descriptor {
+ isInput: Boolean = false,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Database = {
val databaseVal = invokeAs[AnyRef](v, fieldName)
val databaseExtractor = lookupExtractor[DatabaseExtractor](fieldExtractor)
@@ -113,7 +119,8 @@ case class DatabaseDesc(
case class FunctionTypeDesc(
fieldName: String,
fieldExtractor: String,
- skipTypes: Seq[String]) extends Descriptor {
+ skipTypes: Seq[String],
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): FunctionType = {
extract(v, SparkSession.active)
}
@@ -143,7 +150,8 @@ case class FunctionDesc(
fieldExtractor: String,
databaseDesc: Option[DatabaseDesc] = None,
functionTypeDesc: Option[FunctionTypeDesc] = None,
- isInput: Boolean = false) extends Descriptor {
+ isInput: Boolean = false,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Function = {
val functionVal = invokeAs[AnyRef](v, fieldName)
val functionExtractor = lookupExtractor[FunctionExtractor](fieldExtractor)
@@ -168,7 +176,8 @@ case class FunctionDesc(
*/
case class QueryDesc(
fieldName: String,
- fieldExtractor: String = "LogicalPlanQueryExtractor") extends Descriptor {
+ fieldExtractor: String = "LogicalPlanQueryExtractor",
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Option[LogicalPlan] = {
val queryVal = invokeAs[AnyRef](v, fieldName)
val queryExtractor = lookupExtractor[QueryExtractor](fieldExtractor)
@@ -186,7 +195,8 @@ case class QueryDesc(
case class TableTypeDesc(
fieldName: String,
fieldExtractor: String,
- skipTypes: Seq[String]) extends Descriptor {
+ skipTypes: Seq[String],
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): TableType = {
extract(v, SparkSession.active)
}
@@ -224,7 +234,8 @@ case class TableDesc(
tableTypeDesc: Option[TableTypeDesc] = None,
catalogDesc: Option[CatalogDesc] = None,
isInput: Boolean = false,
- setCurrentDatabaseIfMissing: Boolean = false) extends Descriptor {
+ setCurrentDatabaseIfMissing: Boolean = false,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Option[Table] = {
extract(v, SparkSession.active)
}
@@ -254,7 +265,8 @@ case class TableDesc(
case class ActionTypeDesc(
fieldName: String = null,
fieldExtractor: String = null,
- actionType: Option[String] = None) extends Descriptor {
+ actionType: Option[String] = None,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): PrivilegeObjectActionType = {
actionType.map(PrivilegeObjectActionType.withName).getOrElse {
val actionTypeVal = invokeAs[AnyRef](v, fieldName)
@@ -272,7 +284,8 @@ case class ActionTypeDesc(
*/
case class CatalogDesc(
fieldName: String = "catalog",
- fieldExtractor: String = "CatalogPluginCatalogExtractor") extends
Descriptor {
+ fieldExtractor: String = "CatalogPluginCatalogExtractor",
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Option[String] = {
val catalogVal = invokeAs[AnyRef](v, fieldName)
val catalogExtractor = lookupExtractor[CatalogExtractor](fieldExtractor)
@@ -283,7 +296,8 @@ case class CatalogDesc(
case class ScanDesc(
fieldName: String,
fieldExtractor: String,
- catalogDesc: Option[CatalogDesc] = None) extends Descriptor {
+ catalogDesc: Option[CatalogDesc] = None,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Option[Table] = {
extract(v, SparkSession.active)
}
@@ -317,7 +331,8 @@ case class ScanDesc(
case class UriDesc(
fieldName: String,
fieldExtractor: String,
- isInput: Boolean = false) extends Descriptor {
+ isInput: Boolean = false,
+ comment: String = "") extends Descriptor {
override def extract(v: AnyRef): Seq[Uri] = {
extract(v, SparkSession.active)
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
index d40caae7e..12f434a50 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala
@@ -25,12 +25,13 @@ object DeltaCommands extends CommandSpecs[TableCommandSpec]
{
val DeleteCommand = {
val cmd = "org.apache.spark.sql.delta.commands.DeleteCommand"
- val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE))
+ val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE), comment =
"Delta")
val tableDesc = TableDesc(
"target",
classOf[SubqueryAliasTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
- val uriDescs = Seq(UriDesc("target", classOf[SubqueryAliasURIExtractor]))
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Delta")
+ val uriDescs = Seq(UriDesc("target", classOf[SubqueryAliasURIExtractor],
comment = "Delta"))
TableCommandSpec(cmd, Seq(tableDesc), uriDescs = uriDescs)
}
@@ -41,29 +42,31 @@ object DeltaCommands extends CommandSpecs[TableCommandSpec]
{
val MergeIntoCommand = {
val cmd = "org.apache.spark.sql.delta.commands.MergeIntoCommand"
- val queryDesc = QueryDesc("source")
+ val queryDesc = QueryDesc("source", comment = "Delta")
DeleteCommand.copy(classname = cmd, queryDescs = Seq(queryDesc))
}
val OptimizeTableCommand = {
val cmd = "org.apache.spark.sql.delta.commands.OptimizeTableCommand"
- val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor])
- val tableDesc = TableDesc("tableId",
classOf[TableIdentifierOptionTableExtractor])
+ val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor],
comment = "Delta")
+ val tableDesc =
+ TableDesc("tableId", classOf[TableIdentifierOptionTableExtractor],
comment = "Delta")
val uriDescs = Seq(
- UriDesc("child", classOf[ResolvedTableURIExtractor]),
- UriDesc("tableId", classOf[TableIdentifierOptionURIExtractor]),
- UriDesc("path", classOf[StringURIExtractor]))
+ UriDesc("child", classOf[ResolvedTableURIExtractor], comment = "Delta"),
+ UriDesc("tableId", classOf[TableIdentifierOptionURIExtractor], comment =
"Delta"),
+ UriDesc("path", classOf[StringURIExtractor], comment = "Delta"))
TableCommandSpec(cmd, Seq(childDesc, tableDesc), ALTERTABLE_COMPACT,
uriDescs = uriDescs)
}
val VacuumTableCommand = {
val cmd = "io.delta.tables.execution.VacuumTableCommand"
- val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor])
- val tableDesc = TableDesc("table",
classOf[TableIdentifierOptionTableExtractor])
+ val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor],
comment = "Delta")
+ val tableDesc =
+ TableDesc("table", classOf[TableIdentifierOptionTableExtractor], comment
= "Delta")
val uriDescs = Seq(
- UriDesc("child", classOf[ResolvedTableURIExtractor]),
- UriDesc("table", classOf[TableIdentifierOptionURIExtractor]),
- UriDesc("path", classOf[StringURIExtractor]))
+ UriDesc("child", classOf[ResolvedTableURIExtractor], comment = "Delta"),
+ UriDesc("table", classOf[TableIdentifierOptionURIExtractor], comment =
"Delta"),
+ UriDesc("path", classOf[StringURIExtractor], comment = "Delta"))
TableCommandSpec(cmd, Seq(childDesc, tableDesc), MSCK, uriDescs = uriDescs)
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
index 381f8081a..87fc509b5 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
@@ -25,24 +25,38 @@ import
org.apache.kyuubi.plugin.spark.authz.serde.TableType._
object HudiCommands extends CommandSpecs[TableCommandSpec] {
val AlterHoodieTableAddColumnsCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand"
- val columnDesc = ColumnDesc("colsToAdd",
classOf[StructFieldSeqColumnExtractor])
- val tableDesc = TableDesc("tableId",
classOf[TableIdentifierTableExtractor], Some(columnDesc))
+ val columnDesc =
+ ColumnDesc("colsToAdd", classOf[StructFieldSeqColumnExtractor], comment
= "Hudi")
+ val tableDesc = TableDesc(
+ "tableId",
+ classOf[TableIdentifierTableExtractor],
+ Some(columnDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_ADDCOLS)
}
val AlterHoodieTableChangeColumnCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.AlterHoodieTableChangeColumnCommand"
- val columnDesc = ColumnDesc("columnName", classOf[StringColumnExtractor])
+ val columnDesc = ColumnDesc("columnName", classOf[StringColumnExtractor],
comment = "Hudi")
val tableDesc =
- TableDesc("tableIdentifier", classOf[TableIdentifierTableExtractor],
Some(columnDesc))
+ TableDesc(
+ "tableIdentifier",
+ classOf[TableIdentifierTableExtractor],
+ Some(columnDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_REPLACECOLS)
}
val AlterHoodieTableDropPartitionCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.AlterHoodieTableDropPartitionCommand"
- val columnDesc = ColumnDesc("partitionSpecs",
classOf[PartitionSeqColumnExtractor])
+ val columnDesc =
+ ColumnDesc("partitionSpecs", classOf[PartitionSeqColumnExtractor],
comment = "Hudi")
val tableDesc =
- TableDesc("tableIdentifier", classOf[TableIdentifierTableExtractor],
Some(columnDesc))
+ TableDesc(
+ "tableIdentifier",
+ classOf[TableIdentifierTableExtractor],
+ Some(columnDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_DROPPARTS)
}
@@ -52,30 +66,32 @@ object HudiCommands extends CommandSpecs[TableCommandSpec] {
TableTypeDesc(
"oldName",
classOf[TableIdentifierTableTypeExtractor],
- Seq(TEMP_VIEW))
+ Seq(TEMP_VIEW),
+ comment = "Hudi")
val oldTableD = TableDesc(
"oldName",
classOf[TableIdentifierTableExtractor],
- tableTypeDesc = Some(oldTableTableTypeDesc))
+ tableTypeDesc = Some(oldTableTableTypeDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(oldTableD), ALTERTABLE_RENAME)
}
val AlterTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.AlterTableCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
None)
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
None, comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES)
}
val Spark31AlterTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
None)
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
None, comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES)
}
val CreateHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE)
}
@@ -92,12 +108,14 @@ object HudiCommands extends CommandSpecs[TableCommandSpec]
{
val tableDesc1 = TableDesc(
"targetTable",
classOf[TableIdentifierTableExtractor],
- setCurrentDatabaseIfMissing = true)
+ setCurrentDatabaseIfMissing = true,
+ comment = "Hudi")
val tableDesc2 = TableDesc(
"sourceTable",
classOf[TableIdentifierTableExtractor],
isInput = true,
- setCurrentDatabaseIfMissing = true)
+ setCurrentDatabaseIfMissing = true,
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc1, tableDesc2), CREATETABLE)
}
@@ -107,7 +125,8 @@ object HudiCommands extends CommandSpecs[TableCommandSpec] {
TableTypeDesc(
"tableIdentifier",
classOf[TableIdentifierTableTypeExtractor],
- Seq(TEMP_VIEW))
+ Seq(TEMP_VIEW),
+ comment = "Hudi")
TableCommandSpec(
cmd,
Seq(TableDesc(
@@ -124,30 +143,33 @@ object HudiCommands extends
CommandSpecs[TableCommandSpec] {
val TruncateHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand"
- val columnDesc = ColumnDesc("partitionSpec",
classOf[PartitionOptionColumnExtractor])
+ val columnDesc =
+ ColumnDesc("partitionSpec", classOf[PartitionOptionColumnExtractor],
comment = "Hudi")
val tableDesc =
TableDesc(
"tableIdentifier",
classOf[TableIdentifierTableExtractor],
- columnDesc = Some(columnDesc))
+ columnDesc = Some(columnDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), TRUNCATETABLE)
}
val CompactionHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE)
}
val CompactionShowHoodieTableCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
isInput = true)
+ val tableDesc =
+ TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true,
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES)
}
val CompactionHoodiePathCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodiePathCommand"
- val uriDesc = UriDesc("path", classOf[StringURIExtractor])
+ val uriDesc = UriDesc("path", classOf[StringURIExtractor], comment =
"Hudi")
TableCommandSpec(
cmd,
Seq.empty,
@@ -157,31 +179,32 @@ object HudiCommands extends
CommandSpecs[TableCommandSpec] {
val CompactionShowHoodiePathCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.CompactionShowHoodiePathCommand"
- val uriDesc = UriDesc("path", classOf[StringURIExtractor], isInput = true)
+ val uriDesc = UriDesc("path", classOf[StringURIExtractor], isInput = true,
comment = "Hudi")
TableCommandSpec(cmd, Seq.empty, SHOW_TBLPROPERTIES, uriDescs =
Seq(uriDesc))
}
val CreateIndexCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CreateIndexCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), CREATEINDEX)
}
val DropIndexCommand = {
val cmd = "org.apache.spark.sql.hudi.command.DropIndexCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), DROPINDEX)
}
val ShowIndexCommand = {
val cmd = "org.apache.spark.sql.hudi.command.ShowIndexesCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
isInput = true)
+ val tableDesc =
+ TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true,
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), SHOWINDEXES)
}
val RefreshIndexCommand = {
val cmd = "org.apache.spark.sql.hudi.command.RefreshIndexCommand"
- val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), ALTERINDEX_REBUILD)
}
@@ -191,52 +214,62 @@ object HudiCommands extends
CommandSpecs[TableCommandSpec] {
"logicalRelation",
classOf[LogicalRelationTableExtractor],
actionTypeDesc =
- Some(ActionTypeDesc("overwrite",
classOf[OverwriteOrInsertActionTypeExtractor])))
+ Some(ActionTypeDesc(
+ "overwrite",
+ classOf[OverwriteOrInsertActionTypeExtractor],
+ comment = "Hudi")),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(QueryDesc("query")))
}
val ShowHoodieTablePartitionsCommand = {
val cmd =
"org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand"
- val columnDesc = ColumnDesc("specOpt",
classOf[PartitionOptionColumnExtractor])
+ val columnDesc =
+ ColumnDesc("specOpt", classOf[PartitionOptionColumnExtractor], comment =
"Hudi")
val tableDesc = TableDesc(
"tableIdentifier",
classOf[TableIdentifierTableExtractor],
isInput = true,
- columnDesc = Some(columnDesc))
+ columnDesc = Some(columnDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), SHOWPARTITIONS)
}
val DeleteHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand"
- val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE))
+ val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE), comment =
"Hudi")
val tableDesc =
TableDesc(
"dft",
classOf[HudiDataSourceV2RelationTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc))
}
val UpdateHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.UpdateHoodieTableCommand"
- val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE))
+ val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE), comment =
"Hudi")
val tableDesc =
TableDesc(
"ut",
classOf[HudiDataSourceV2RelationTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc))
}
val MergeIntoHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand"
- val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE))
+ val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE), comment =
"Hudi")
val tableDesc =
TableDesc(
"mergeInto",
classOf[HudiMergeIntoTargetTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
- val queryDescs = QueryDesc("mergeInto",
classOf[HudiMergeIntoSourceTableExtractor])
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Hudi")
+ val queryDescs =
+ QueryDesc("mergeInto", classOf[HudiMergeIntoSourceTableExtractor],
comment = "Hudi")
TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryDescs))
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala
index 59f8eb7a6..33e94d718 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala
@@ -30,7 +30,8 @@ object IcebergCommands extends CommandSpecs[TableCommandSpec]
{
TableDesc(
"table",
classOf[DataSourceV2RelationTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Iceberg")
TableCommandSpec(cmd, Seq(tableDesc))
}
@@ -45,14 +46,15 @@ object IcebergCommands extends
CommandSpecs[TableCommandSpec] {
val tableDesc = TableDesc(
"targetTable",
classOf[DataSourceV2RelationTableExtractor],
- actionTypeDesc = Some(actionTypeDesc))
+ actionTypeDesc = Some(actionTypeDesc),
+ comment = "Iceberg")
val queryDesc = QueryDesc("sourceTable")
TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryDesc))
}
val CallProcedure = {
val cmd = "org.apache.spark.sql.catalyst.plans.logical.Call"
- val td = TableDesc("args", classOf[ExpressionSeqTableExtractor])
+ val td = TableDesc("args", classOf[ExpressionSeqTableExtractor], comment =
"Iceberg")
TableCommandSpec(cmd, Seq(td), opType =
OperationType.ALTERTABLE_PROPERTIES)
}
