This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new cb689b66b [KYUUBI #4914] [AUTHZ] Reuse extractor singleton instance
with generalized getter for supported extractor types
cb689b66b is described below
commit cb689b66b1ed745a5e38dea37a2b0f1d9e7f1aca
Author: liangbowen <[email protected]>
AuthorDate: Mon Jun 5 13:11:34 2023 +0800
[KYUUBI #4914] [AUTHZ] Reuse extractor singleton instance with generalized
getter for supported extractor types
### _Why are the changes needed?_
- Reuse extractor singleton instance for less memory footprint, as Authz's
extractors are stateless and ready for sharing
- Reneralized getter crossing supported extractor types
- get extractor by class type
- get extractor by explicit class name
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #4914 from bowenliang123/authz-get-extractor.
Closes #4914
11dde777f [liangbowen] update
77ce00276 [liangbowen] make extractorKey of lookupExtractor not null by
default
5f5b6e580 [liangbowen] Revert "extractorKey: String = null => extractorKey:
Option[String] = None"
400c3b054 [liangbowen] extractorKey: String = null => extractorKey:
Option[String] = None
60acd27ec [liangbowen] rename `getExtractor` to `lookupExtractor`
e6fbb450f [liangbowen] generalize getExtractor for getting instance of
supported types of extractors
Authored-by: liangbowen <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
.../plugin/spark/authz/serde/Descriptor.scala | 33 +++++++------------
.../spark/authz/serde/catalogExtractors.scala | 2 +-
.../spark/authz/serde/databaseExtractors.scala | 4 +--
.../spark/authz/serde/functionTypeExtractors.scala | 4 +--
.../kyuubi/plugin/spark/authz/serde/package.scala | 38 ++++++++++++++++++++++
.../plugin/spark/authz/serde/tableExtractors.scala | 18 +++++-----
6 files changed, 64 insertions(+), 35 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
index b72f3296b..3eb3fecea 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
@@ -23,17 +23,8 @@ import
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType
import
org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType.PrivilegeObjectActionType
-import
org.apache.kyuubi.plugin.spark.authz.serde.ActionTypeExtractor.actionTypeExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.CatalogExtractor.catalogExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.ColumnExtractor.columnExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.DatabaseExtractor.dbExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.functionExtractors
import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType.FunctionType
-import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.functionTypeExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors
-import
org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors
import org.apache.kyuubi.plugin.spark.authz.serde.TableType.TableType
-import
org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors
import org.apache.kyuubi.util.reflect.ReflectUtils._
/**
@@ -82,7 +73,7 @@ case class ColumnDesc(
fieldExtractor: String) extends Descriptor {
override def extract(v: AnyRef): Seq[String] = {
val columnsVal = invoke(v, fieldName)
- val columnExtractor = columnExtractors(fieldExtractor)
+ val columnExtractor = lookupExtractor[ColumnExtractor](fieldExtractor)
columnExtractor(columnsVal)
}
}
@@ -101,7 +92,7 @@ case class DatabaseDesc(
isInput: Boolean = false) extends Descriptor {
override def extract(v: AnyRef): Database = {
val databaseVal = invoke(v, fieldName)
- val databaseExtractor = dbExtractors(fieldExtractor)
+ val databaseExtractor = lookupExtractor[DatabaseExtractor](fieldExtractor)
val db = databaseExtractor(databaseVal)
if (db.catalog.isEmpty && catalogDesc.nonEmpty) {
val maybeCatalog = catalogDesc.get.extract(v)
@@ -129,7 +120,7 @@ case class FunctionTypeDesc(
def extract(v: AnyRef, spark: SparkSession): FunctionType = {
val functionTypeVal = invoke(v, fieldName)
- val functionTypeExtractor = functionTypeExtractors(fieldExtractor)
+ val functionTypeExtractor =
lookupExtractor[FunctionTypeExtractor](fieldExtractor)
functionTypeExtractor(functionTypeVal, spark)
}
@@ -155,7 +146,7 @@ case class FunctionDesc(
isInput: Boolean = false) extends Descriptor {
override def extract(v: AnyRef): Function = {
val functionVal = invoke(v, fieldName)
- val functionExtractor = functionExtractors(fieldExtractor)
+ val functionExtractor = lookupExtractor[FunctionExtractor](fieldExtractor)
var function = functionExtractor(functionVal)
if (function.database.isEmpty) {
val maybeDatabase = databaseDesc.map(_.extract(v))
@@ -180,7 +171,7 @@ case class QueryDesc(
fieldExtractor: String = "LogicalPlanQueryExtractor") extends Descriptor {
override def extract(v: AnyRef): Option[LogicalPlan] = {
val queryVal = invoke(v, fieldName)
- val queryExtractor = queryExtractors(fieldExtractor)
+ val queryExtractor = lookupExtractor[QueryExtractor](fieldExtractor)
queryExtractor(queryVal)
}
}
@@ -202,7 +193,7 @@ case class TableTypeDesc(
def extract(v: AnyRef, spark: SparkSession): TableType = {
val tableTypeVal = invoke(v, fieldName)
- val tableTypeExtractor = tableTypeExtractors(fieldExtractor)
+ val tableTypeExtractor =
lookupExtractor[TableTypeExtractor](fieldExtractor)
tableTypeExtractor(tableTypeVal, spark)
}
@@ -240,7 +231,7 @@ case class TableDesc(
def extract(v: AnyRef, spark: SparkSession): Option[Table] = {
val tableVal = invoke(v, fieldName)
- val tableExtractor = tableExtractors(fieldExtractor)
+ val tableExtractor = lookupExtractor[TableExtractor](fieldExtractor)
val maybeTable = tableExtractor(spark, tableVal)
maybeTable.map { t =>
if (t.catalog.isEmpty && catalogDesc.nonEmpty) {
@@ -267,8 +258,8 @@ case class ActionTypeDesc(
override def extract(v: AnyRef): PrivilegeObjectActionType = {
actionType.map(PrivilegeObjectActionType.withName).getOrElse {
val actionTypeVal = invoke(v, fieldName)
- val extractor = actionTypeExtractors(fieldExtractor)
- extractor(actionTypeVal)
+ val actionTypeExtractor =
lookupExtractor[ActionTypeExtractor](fieldExtractor)
+ actionTypeExtractor(actionTypeVal)
}
}
}
@@ -284,8 +275,8 @@ case class CatalogDesc(
fieldExtractor: String = "CatalogPluginCatalogExtractor") extends
Descriptor {
override def extract(v: AnyRef): Option[String] = {
val catalogVal = invoke(v, fieldName)
- val extractor = catalogExtractors(fieldExtractor)
- extractor(catalogVal)
+ val catalogExtractor = lookupExtractor[CatalogExtractor](fieldExtractor)
+ catalogExtractor(catalogVal)
}
}
@@ -303,7 +294,7 @@ case class ScanDesc(
} else {
invoke(v, fieldName)
}
- val tableExtractor = tableExtractors(fieldExtractor)
+ val tableExtractor = lookupExtractor[TableExtractor](fieldExtractor)
val maybeTable = tableExtractor(spark, tableVal)
maybeTable.map { t =>
if (t.catalog.isEmpty && catalogDesc.nonEmpty) {
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
index f87943943..e48becb32 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
@@ -43,7 +43,7 @@ class CatalogPluginOptionCatalogExtractor extends
CatalogExtractor {
override def apply(v1: AnyRef): Option[String] = {
v1 match {
case Some(catalogPlugin: AnyRef) =>
- new CatalogPluginCatalogExtractor().apply(catalogPlugin)
+ lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogPlugin)
case _ => None
}
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
index d14ba7805..f952c816e 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
@@ -70,7 +70,7 @@ class StringSeqOptionDatabaseExtractor extends
DatabaseExtractor {
class ResolvedNamespaceDatabaseExtractor extends DatabaseExtractor {
override def apply(v1: AnyRef): Database = {
val catalogVal = invoke(v1, "catalog")
- val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal)
+ val catalog =
lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal)
val namespace = getFieldVal[Seq[String]](v1, "namespace")
Database(catalog, quote(namespace))
}
@@ -82,7 +82,7 @@ class ResolvedNamespaceDatabaseExtractor extends
DatabaseExtractor {
class ResolvedDBObjectNameDatabaseExtractor extends DatabaseExtractor {
override def apply(v1: AnyRef): Database = {
val catalogVal = invoke(v1, "catalog")
- val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal)
+ val catalog =
lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal)
val namespace = getFieldVal[Seq[String]](v1, "nameParts")
Database(catalog, quote(namespace))
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
index 4c5e9dc84..b70410342 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
@@ -53,9 +53,9 @@ class TempMarkerFunctionTypeExtractor extends
FunctionTypeExtractor {
*/
class ExpressionInfoFunctionTypeExtractor extends FunctionTypeExtractor {
override def apply(v1: AnyRef, spark: SparkSession): FunctionType = {
- val function = new ExpressionInfoFunctionExtractor().apply(v1)
+ val function = lookupExtractor[ExpressionInfoFunctionExtractor].apply(v1)
val fi = FunctionIdentifier(function.functionName, function.database)
- new FunctionIdentifierFunctionTypeExtractor().apply(fi, spark)
+ lookupExtractor[FunctionIdentifierFunctionTypeExtractor].apply(fi, spark)
}
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
index d9f646900..c992a231a 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
@@ -25,6 +25,15 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.kyuubi.plugin.spark.authz.OperationType.{OperationType,
QUERY}
+import
org.apache.kyuubi.plugin.spark.authz.serde.ActionTypeExtractor.actionTypeExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.CatalogExtractor.catalogExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.ColumnExtractor.columnExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.DatabaseExtractor.dbExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.functionExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.functionTypeExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors
+import
org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors
import org.apache.kyuubi.util.reflect.ReflectUtils._
package object serde {
@@ -82,4 +91,33 @@ package object serde {
.map(s => s.operationType)
.getOrElse(QUERY)
}
+
+ /**
+ * get extractor instance by extractor class name
+ * @param extractorKey explicitly load extractor by its simple class name.
+ * null by default means get extractor by
extractor class.
+ * @param ct class tag of extractor class type
+ * @tparam T extractor class type
+ * @return
+ */
+ def lookupExtractor[T <: Extractor](extractorKey: String)(
+ implicit ct: ClassTag[T]): T = {
+ val extractorClass = ct.runtimeClass
+ val extractors: Map[String, Extractor] = extractorClass match {
+ case c if classOf[CatalogExtractor].isAssignableFrom(c) =>
catalogExtractors
+ case c if classOf[DatabaseExtractor].isAssignableFrom(c) => dbExtractors
+ case c if classOf[TableExtractor].isAssignableFrom(c) => tableExtractors
+ case c if classOf[TableTypeExtractor].isAssignableFrom(c) =>
tableTypeExtractors
+ case c if classOf[ColumnExtractor].isAssignableFrom(c) =>
columnExtractors
+ case c if classOf[QueryExtractor].isAssignableFrom(c) => queryExtractors
+ case c if classOf[FunctionExtractor].isAssignableFrom(c) =>
functionExtractors
+ case c if classOf[FunctionTypeExtractor].isAssignableFrom(c) =>
functionTypeExtractors
+ case c if classOf[ActionTypeExtractor].isAssignableFrom(c) =>
actionTypeExtractors
+ case _ => throw new IllegalArgumentException(s"Unknown extractor type:
$ct")
+ }
+ extractors(extractorKey).asInstanceOf[T]
+ }
+
+ def lookupExtractor[T <: Extractor](implicit ct: ClassTag[T]): T =
+ lookupExtractor[T](ct.runtimeClass.getSimpleName)(ct)
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
index 8743f054d..4579349ee 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
@@ -88,7 +88,7 @@ class CatalogTableTableExtractor extends TableExtractor {
class CatalogTableOptionTableExtractor extends TableExtractor {
override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = {
val catalogTable = v1.asInstanceOf[Option[CatalogTable]]
- catalogTable.flatMap(new CatalogTableTableExtractor().apply(spark, _))
+
catalogTable.flatMap(lookupExtractor[CatalogTableTableExtractor].apply(spark,
_))
}
}
@@ -98,9 +98,9 @@ class CatalogTableOptionTableExtractor extends TableExtractor
{
class ResolvedTableTableExtractor extends TableExtractor {
override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = {
val catalogVal = invoke(v1, "catalog")
- val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal)
+ val catalog =
lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal)
val identifier = invoke(v1, "identifier")
- val maybeTable = new IdentifierTableExtractor().apply(spark, identifier)
+ val maybeTable = lookupExtractor[IdentifierTableExtractor].apply(spark,
identifier)
val maybeOwner = TableExtractor.getOwner(v1)
maybeTable.map(_.copy(catalog = catalog, owner = maybeOwner))
}
@@ -129,10 +129,10 @@ class DataSourceV2RelationTableExtractor extends
TableExtractor {
case Some(v2Relation) =>
val maybeCatalogPlugin = invokeAs[Option[AnyRef]](v2Relation,
"catalog")
val maybeCatalog = maybeCatalogPlugin.flatMap(catalogPlugin =>
- new CatalogPluginCatalogExtractor().apply(catalogPlugin))
+ lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogPlugin))
val maybeIdentifier = invokeAs[Option[AnyRef]](v2Relation,
"identifier")
maybeIdentifier.flatMap { id =>
- val maybeTable = new IdentifierTableExtractor().apply(spark, id)
+ val maybeTable =
lookupExtractor[IdentifierTableExtractor].apply(spark, id)
val maybeOwner = TableExtractor.getOwner(v2Relation)
maybeTable.map(_.copy(catalog = maybeCatalog, owner = maybeOwner))
}
@@ -147,7 +147,7 @@ class LogicalRelationTableExtractor extends TableExtractor {
override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = {
val maybeCatalogTable = invokeAs[Option[AnyRef]](v1, "catalogTable")
maybeCatalogTable.flatMap { ct =>
- new CatalogTableTableExtractor().apply(spark, ct)
+ lookupExtractor[CatalogTableTableExtractor].apply(spark, ct)
}
}
}
@@ -158,7 +158,7 @@ class LogicalRelationTableExtractor extends TableExtractor {
class ResolvedDbObjectNameTableExtractor extends TableExtractor {
override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = {
val catalogVal = invoke(v1, "catalog")
- val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal)
+ val catalog =
lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal)
val nameParts = invokeAs[Seq[String]](v1, "nameParts")
val namespace = nameParts.init.toArray
val table = nameParts.last
@@ -174,9 +174,9 @@ class ResolvedIdentifierTableExtractor extends
TableExtractor {
v1.getClass.getName match {
case "org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier" =>
val catalogVal = invoke(v1, "catalog")
- val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal)
+ val catalog =
lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal)
val identifier = invoke(v1, "identifier")
- val maybeTable = new IdentifierTableExtractor().apply(spark,
identifier)
+ val maybeTable =
lookupExtractor[IdentifierTableExtractor].apply(spark, identifier)
maybeTable.map(_.copy(catalog = catalog))
case _ => None
}
