This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 3abc7ce37 [KYUUBI #4167] [Authz] Introduce function support in
PrivilegeBuilder with Serde layers
3abc7ce37 is described below
commit 3abc7ce3718eb67f0516df6014d4cc453498e642
Author: Deng An <[email protected]>
AuthorDate: Tue Jul 11 23:22:59 2023 +0800
[KYUUBI #4167] [Authz] Introduce function support in PrivilegeBuilder with
Serde layers
### _Why are the changes needed?_
to close #4167 which parent issue is #3632
Introduce HiveFunctionPrivilegeBuilder refactored with Serde layers.
The core logic to this is scanning logical plan by transformAllExpressions
and build function privileges for UDFs.
As a SQL gateway, Kyuubi also needs to support control user UDF usage
behavior.
Therefore, the Spark SQL Authz module needs to add suport for extracting
Privilege Objects for UDF usage in queries from a spark logical plans.
In Spark SQL, the hive permanent UDF is wrapped into the following types
based on different types:
Org.apache.spark.sql.live.HiveSimpleUDF
Org.apache.spark.sql.live.HiveGenericUDF
Org. apache. park. SQL. live. HiveUDAFFunction
Org.apache.spark.sql.live.HiveGenericUDTF
Based on the existing serde module, we can define a ScanSpec to extract
hive permanent udf expression's info, the most important info is the udf type.
And the udf type decide whether to skip the construction of the privilege
object based on whether the function type is a system function or a temporary
function, as we only consider the permanent udf uasge.
So we should define QualifiedNameStringFunctionExtractor and
FunctionNameFunctionTypeExtractor to achieve the goal. And in
`org.apache.kyuubi.plugin.spark.authz.PrivilegesBuilder#buildFunctions`, we
define a method `buildFunctions`, use `transformAllExpressions` to gather all
hive udf expression, and building input privilege objects for thoese permanent
hive udf.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #4168 from
packyan/feature_introduce_hive_function_privileges_builder.
Closes #4167
1276ac7f6 [Deng An] [KYUUBI #4167] [Authz] Introduce function support in
PrivilegeBuilder with Serde layers
Authored-by: Deng An <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
...uubi.plugin.spark.authz.serde.FunctionExtractor | 1 +
....plugin.spark.authz.serde.FunctionTypeExtractor | 1 +
.../src/main/resources/scan_command_spec.json | 66 ++++++-
.../plugin/spark/authz/PrivilegesBuilder.scala | 22 +++
.../plugin/spark/authz/serde/CommandSpec.scala | 16 +-
.../spark/authz/serde/functionExtractors.scala | 22 +++
.../spark/authz/serde/functionTypeExtractors.scala | 38 +++-
.../kyuubi/plugin/spark/authz/serde/package.scala | 18 +-
.../authz/FunctionPrivilegesBuilderSuite.scala | 201 +++++++++++++++++++++
.../kyuubi/plugin/spark/authz/gen/Scans.scala | 28 ++-
10 files changed, 398 insertions(+), 15 deletions(-)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor
index 4686bb033..2facb004a 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor
@@ -17,4 +17,5 @@
org.apache.kyuubi.plugin.spark.authz.serde.ExpressionInfoFunctionExtractor
org.apache.kyuubi.plugin.spark.authz.serde.FunctionIdentifierFunctionExtractor
+org.apache.kyuubi.plugin.spark.authz.serde.QualifiedNameStringFunctionExtractor
org.apache.kyuubi.plugin.spark.authz.serde.StringFunctionExtractor
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor
index 475f47afc..3bb0ee6c2 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor
@@ -17,4 +17,5 @@
org.apache.kyuubi.plugin.spark.authz.serde.ExpressionInfoFunctionTypeExtractor
org.apache.kyuubi.plugin.spark.authz.serde.FunctionIdentifierFunctionTypeExtractor
+org.apache.kyuubi.plugin.spark.authz.serde.FunctionNameFunctionTypeExtractor
org.apache.kyuubi.plugin.spark.authz.serde.TempMarkerFunctionTypeExtractor
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
index 9a6aef4ed..3273ccbea 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
+++
b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json
@@ -4,26 +4,86 @@
"fieldName" : "catalogTable",
"fieldExtractor" : "CatalogTableTableExtractor",
"catalogDesc" : null
- } ]
+ } ],
+ "functionDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.catalyst.catalog.HiveTableRelation",
"scanDescs" : [ {
"fieldName" : "tableMeta",
"fieldExtractor" : "CatalogTableTableExtractor",
"catalogDesc" : null
- } ]
+ } ],
+ "functionDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.execution.datasources.LogicalRelation",
"scanDescs" : [ {
"fieldName" : "catalogTable",
"fieldExtractor" : "CatalogTableOptionTableExtractor",
"catalogDesc" : null
- } ]
+ } ],
+ "functionDescs" : [ ]
}, {
"classname" :
"org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation",
"scanDescs" : [ {
"fieldName" : null,
"fieldExtractor" : "DataSourceV2RelationTableExtractor",
"catalogDesc" : null
+ } ],
+ "functionDescs" : [ ]
+}, {
+ "classname" : "org.apache.spark.sql.hive.HiveGenericUDF",
+ "scanDescs" : [ ],
+ "functionDescs" : [ {
+ "fieldName" : "name",
+ "fieldExtractor" : "QualifiedNameStringFunctionExtractor",
+ "databaseDesc" : null,
+ "functionTypeDesc" : {
+ "fieldName" : "name",
+ "fieldExtractor" : "FunctionNameFunctionTypeExtractor",
+ "skipTypes" : [ "TEMP", "SYSTEM" ]
+ },
+ "isInput" : true
+ } ]
+}, {
+ "classname" : "org.apache.spark.sql.hive.HiveGenericUDTF",
+ "scanDescs" : [ ],
+ "functionDescs" : [ {
+ "fieldName" : "name",
+ "fieldExtractor" : "QualifiedNameStringFunctionExtractor",
+ "databaseDesc" : null,
+ "functionTypeDesc" : {
+ "fieldName" : "name",
+ "fieldExtractor" : "FunctionNameFunctionTypeExtractor",
+ "skipTypes" : [ "TEMP", "SYSTEM" ]
+ },
+ "isInput" : true
+ } ]
+}, {
+ "classname" : "org.apache.spark.sql.hive.HiveSimpleUDF",
+ "scanDescs" : [ ],
+ "functionDescs" : [ {
+ "fieldName" : "name",
+ "fieldExtractor" : "QualifiedNameStringFunctionExtractor",
+ "databaseDesc" : null,
+ "functionTypeDesc" : {
+ "fieldName" : "name",
+ "fieldExtractor" : "FunctionNameFunctionTypeExtractor",
+ "skipTypes" : [ "TEMP", "SYSTEM" ]
+ },
+ "isInput" : true
+ } ]
+}, {
+ "classname" : "org.apache.spark.sql.hive.HiveUDAFFunction",
+ "scanDescs" : [ ],
+ "functionDescs" : [ {
+ "fieldName" : "name",
+ "fieldExtractor" : "QualifiedNameStringFunctionExtractor",
+ "databaseDesc" : null,
+ "functionTypeDesc" : {
+ "fieldName" : "name",
+ "fieldExtractor" : "FunctionNameFunctionTypeExtractor",
+ "skipTypes" : [ "TEMP", "SYSTEM" ]
+ },
+ "isInput" : true
} ]
} ]
\ No newline at end of file
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
index eef89deea..b2fa89909 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
@@ -211,6 +211,28 @@ object PrivilegesBuilder {
type PrivilegesAndOpType = (Seq[PrivilegeObject], Seq[PrivilegeObject],
OperationType)
+ /**
+ * Build input privilege objects from a Spark's LogicalPlan for hive
permanent udf
+ *
+ * @param plan A Spark LogicalPlan
+ */
+ def buildFunctions(
+ plan: LogicalPlan,
+ spark: SparkSession): PrivilegesAndOpType = {
+ val inputObjs = new ArrayBuffer[PrivilegeObject]
+ // TODO: add support for Spark 3.4.x
+ plan transformAllExpressions {
+ case hiveFunction: Expression if isKnownFunction(hiveFunction) =>
+ val functionSpec: ScanSpec = getFunctionSpec(hiveFunction)
+ if
(functionSpec.functionDescs.exists(!_.functionTypeDesc.get.skip(hiveFunction,
spark))) {
+ functionSpec.functions(hiveFunction).foreach(func =>
+ inputObjs += PrivilegeObject(func))
+ }
+ hiveFunction
+ }
+ (inputObjs, Seq.empty, OperationType.QUERY)
+ }
+
/**
* Build input and output privilege objects from a Spark's LogicalPlan
*
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
index e96ef8cbf..32ad30e21 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.plugin.spark.authz.serde
import com.fasterxml.jackson.annotation.JsonIgnore
import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.slf4j.LoggerFactory
@@ -94,7 +95,8 @@ case class TableCommandSpec(
case class ScanSpec(
classname: String,
- scanDescs: Seq[ScanDesc]) extends CommandSpec {
+ scanDescs: Seq[ScanDesc],
+ functionDescs: Seq[FunctionDesc] = Seq.empty) extends CommandSpec {
override def opType: String = OperationType.QUERY.toString
def tables: (LogicalPlan, SparkSession) => Seq[Table] = (plan, spark) => {
scanDescs.flatMap { td =>
@@ -107,4 +109,16 @@ case class ScanSpec(
}
}
}
+
+ def functions: (Expression) => Seq[Function] = (expr) => {
+ functionDescs.flatMap { fd =>
+ try {
+ Some(fd.extract(expr))
+ } catch {
+ case e: Exception =>
+ LOG.debug(fd.error(expr, e))
+ None
+ }
+ }
+ }
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala
index 894a6cb8f..729521200 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala
@@ -20,12 +20,23 @@ package org.apache.kyuubi.plugin.spark.authz.serde
import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
+import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.buildFunctionIdentFromQualifiedName
+
trait FunctionExtractor extends (AnyRef => Function) with Extractor
object FunctionExtractor {
val functionExtractors: Map[String, FunctionExtractor] = {
loadExtractorsToMap[FunctionExtractor]
}
+
+ def buildFunctionIdentFromQualifiedName(qualifiedName: String): (String,
Option[String]) = {
+ val parts: Array[String] = qualifiedName.split("\\.", 2)
+ if (parts.length == 1) {
+ (qualifiedName, None)
+ } else {
+ (parts.last, Some(parts.head))
+ }
+ }
}
/**
@@ -37,6 +48,17 @@ class StringFunctionExtractor extends FunctionExtractor {
}
}
+/**
+ * * String
+ */
+class QualifiedNameStringFunctionExtractor extends FunctionExtractor {
+ override def apply(v1: AnyRef): Function = {
+ val qualifiedName: String = v1.asInstanceOf[String]
+ val (funcName, database) =
buildFunctionIdentFromQualifiedName(qualifiedName)
+ Function(database, funcName)
+ }
+}
+
/**
* org.apache.spark.sql.catalyst.FunctionIdentifier
*/
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
index b70410342..a2c5b427f 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala
@@ -19,8 +19,11 @@ package org.apache.kyuubi.plugin.spark.authz.serde
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.catalog.SessionCatalog
+import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.buildFunctionIdentFromQualifiedName
import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType.{FunctionType,
PERMANENT, SYSTEM, TEMP}
+import
org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.getFunctionType
object FunctionType extends Enumeration {
type FunctionType = Value
@@ -33,6 +36,19 @@ object FunctionTypeExtractor {
val functionTypeExtractors: Map[String, FunctionTypeExtractor] = {
loadExtractorsToMap[FunctionTypeExtractor]
}
+
+ def getFunctionType(fi: FunctionIdentifier, catalog: SessionCatalog):
FunctionType = {
+ fi match {
+ case temp if catalog.isTemporaryFunction(temp) =>
+ TEMP
+ case permanent if catalog.isPersistentFunction(permanent) =>
+ PERMANENT
+ case system if catalog.isRegisteredFunction(system) =>
+ SYSTEM
+ case _ =>
+ TEMP
+ }
+ }
}
/**
@@ -66,14 +82,18 @@ class FunctionIdentifierFunctionTypeExtractor extends
FunctionTypeExtractor {
override def apply(v1: AnyRef, spark: SparkSession): FunctionType = {
val catalog = spark.sessionState.catalog
val fi = v1.asInstanceOf[FunctionIdentifier]
- if (catalog.isTemporaryFunction(fi)) {
- TEMP
- } else if (catalog.isPersistentFunction(fi)) {
- PERMANENT
- } else if (catalog.isRegisteredFunction(fi)) {
- SYSTEM
- } else {
- TEMP
- }
+ getFunctionType(fi, catalog)
+ }
+}
+
+/**
+ * String
+ */
+class FunctionNameFunctionTypeExtractor extends FunctionTypeExtractor {
+ override def apply(v1: AnyRef, spark: SparkSession): FunctionType = {
+ val catalog: SessionCatalog = spark.sessionState.catalog
+ val qualifiedName: String = v1.asInstanceOf[String]
+ val (funcName, database) =
buildFunctionIdentFromQualifiedName(qualifiedName)
+ getFunctionType(FunctionIdentifier(funcName, database), catalog)
}
}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
index c992a231a..6863516b6 100644
---
a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
@@ -72,7 +72,8 @@ package object serde {
final private lazy val SCAN_SPECS: Map[String, ScanSpec] = {
val is =
getClass.getClassLoader.getResourceAsStream("scan_command_spec.json")
mapper.readValue(is, new TypeReference[Array[ScanSpec]] {})
- .map(e => (e.classname, e)).toMap
+ .map(e => (e.classname, e))
+ .filter(t => t._2.scanDescs.nonEmpty).toMap
}
def isKnownScan(r: AnyRef): Boolean = {
@@ -83,6 +84,21 @@ package object serde {
SCAN_SPECS(r.getClass.getName)
}
+ final private lazy val FUNCTION_SPECS: Map[String, ScanSpec] = {
+ val is =
getClass.getClassLoader.getResourceAsStream("scan_command_spec.json")
+ mapper.readValue(is, new TypeReference[Array[ScanSpec]] {})
+ .map(e => (e.classname, e))
+ .filter(t => t._2.functionDescs.nonEmpty).toMap
+ }
+
+ def isKnownFunction(r: AnyRef): Boolean = {
+ FUNCTION_SPECS.contains(r.getClass.getName)
+ }
+
+ def getFunctionSpec(r: AnyRef): ScanSpec = {
+ FUNCTION_SPECS(r.getClass.getName)
+ }
+
def operationType(plan: LogicalPlan): OperationType = {
val classname = plan.getClass.getName
TABLE_COMMAND_SPECS.get(classname)
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala
new file mode 100644
index 000000000..0f261c2dd
--- /dev/null
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.plugin.spark.authz
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+// scalastyle:off
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.kyuubi.plugin.spark.authz.OperationType.QUERY
+import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType
+import
org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.isSparkVersionAtMost
+
+abstract class FunctionPrivilegesBuilderSuite extends AnyFunSuite
+ with SparkSessionProvider with BeforeAndAfterAll with BeforeAndAfterEach {
+ // scalastyle:on
+
+ protected def withTable(t: String)(f: String => Unit): Unit = {
+ try {
+ f(t)
+ } finally {
+ sql(s"DROP TABLE IF EXISTS $t")
+ }
+ }
+
+ protected def withDatabase(t: String)(f: String => Unit): Unit = {
+ try {
+ f(t)
+ } finally {
+ sql(s"DROP DATABASE IF EXISTS $t")
+ }
+ }
+
+ protected def checkColumns(plan: LogicalPlan, cols: Seq[String]): Unit = {
+ val (in, out, _) = PrivilegesBuilder.build(plan, spark)
+ assert(out.isEmpty, "Queries shall not check output privileges")
+ val po = in.head
+ assert(po.actionType === PrivilegeObjectActionType.OTHER)
+ assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW)
+ assert(po.columns === cols)
+ }
+
+ protected def checkColumns(query: String, cols: Seq[String]): Unit = {
+ checkColumns(sql(query).queryExecution.optimizedPlan, cols)
+ }
+
+ protected val reusedDb: String = getClass.getSimpleName
+ protected val reusedDb2: String = getClass.getSimpleName + "2"
+ protected val reusedTable: String = reusedDb + "." + getClass.getSimpleName
+ protected val reusedTableShort: String = reusedTable.split("\\.").last
+ protected val reusedPartTable: String = reusedTable + "_part"
+ protected val reusedPartTableShort: String =
reusedPartTable.split("\\.").last
+ protected val functionCount = 3
+ protected val functionNamePrefix = "kyuubi_fun_"
+ protected val tempFunNamePrefix = "kyuubi_temp_fun_"
+
+ override def beforeAll(): Unit = {
+ sql(s"CREATE DATABASE IF NOT EXISTS $reusedDb")
+ sql(s"CREATE DATABASE IF NOT EXISTS $reusedDb2")
+ sql(s"CREATE TABLE IF NOT EXISTS $reusedTable" +
+ s" (key int, value string) USING parquet")
+ sql(s"CREATE TABLE IF NOT EXISTS $reusedPartTable" +
+ s" (key int, value string, pid string) USING parquet" +
+ s" PARTITIONED BY(pid)")
+ // scalastyle:off
+ (0 until functionCount).foreach { index =>
+ {
+ sql(s"CREATE FUNCTION ${reusedDb}.${functionNamePrefix}${index} AS
'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'")
+ sql(s"CREATE FUNCTION ${reusedDb2}.${functionNamePrefix}${index} AS
'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'")
+ sql(s"CREATE TEMPORARY FUNCTION ${tempFunNamePrefix}${index} AS
'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'")
+ }
+ }
+ sql(s"USE ${reusedDb2}")
+ // scalastyle:on
+ super.beforeAll()
+ }
+
+ override def afterAll(): Unit = {
+ Seq(reusedTable, reusedPartTable).foreach { t =>
+ sql(s"DROP TABLE IF EXISTS $t")
+ }
+
+ Seq(reusedDb, reusedDb2).foreach { db =>
+ (0 until functionCount).foreach { index =>
+ sql(s"DROP FUNCTION ${db}.${functionNamePrefix}${index}")
+ }
+ sql(s"DROP DATABASE IF EXISTS ${db}")
+ }
+
+ spark.stop()
+ super.afterAll()
+ }
+}
+
+class HiveFunctionPrivilegesBuilderSuite extends
FunctionPrivilegesBuilderSuite {
+
+ override protected val catalogImpl: String = "hive"
+
+ test("Function Call Query") {
+ assume(isSparkVersionAtMost("3.3"))
+ val plan = sql(s"SELECT kyuubi_fun_1('data'), " +
+ s"kyuubi_fun_2(value), " +
+ s"${reusedDb}.kyuubi_fun_0(value), " +
+ s"kyuubi_temp_fun_1('data2')," +
+ s"kyuubi_temp_fun_2(key) " +
+ s"FROM $reusedTable").queryExecution.analyzed
+ val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark)
+ assert(inputs.size === 3)
+ inputs.foreach { po =>
+ assert(po.actionType === PrivilegeObjectActionType.OTHER)
+ assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION)
+ assert(po.dbname startsWith reusedDb.toLowerCase)
+ assert(po.objectName startsWith functionNamePrefix.toLowerCase)
+ val accessType = ranger.AccessType(po, QUERY, isInput = true)
+ assert(accessType === AccessType.SELECT)
+ }
+ }
+
+ test("Function Call Query with Quoted Name") {
+ assume(isSparkVersionAtMost("3.3"))
+ val plan = sql(s"SELECT `kyuubi_fun_1`('data'), " +
+ s"`kyuubi_fun_2`(value), " +
+ s"`${reusedDb}`.`kyuubi_fun_0`(value), " +
+ s"`kyuubi_temp_fun_1`('data2')," +
+ s"`kyuubi_temp_fun_2`(key) " +
+ s"FROM $reusedTable").queryExecution.analyzed
+ val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark)
+ assert(inputs.size === 3)
+ inputs.foreach { po =>
+ assert(po.actionType === PrivilegeObjectActionType.OTHER)
+ assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION)
+ assert(po.dbname startsWith reusedDb.toLowerCase)
+ assert(po.objectName startsWith functionNamePrefix.toLowerCase)
+ val accessType = ranger.AccessType(po, QUERY, isInput = true)
+ assert(accessType === AccessType.SELECT)
+ }
+ }
+
+ test("Simple Function Call Query") {
+ assume(isSparkVersionAtMost("3.3"))
+ val plan = sql(s"SELECT kyuubi_fun_1('data'), " +
+ s"kyuubi_fun_0('value'), " +
+ s"${reusedDb}.kyuubi_fun_0('value'), " +
+ s"${reusedDb}.kyuubi_fun_2('value'), " +
+ s"kyuubi_temp_fun_1('data2')," +
+ s"kyuubi_temp_fun_2('key') ").queryExecution.analyzed
+ val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark)
+ assert(inputs.size === 4)
+ inputs.foreach { po =>
+ assert(po.actionType === PrivilegeObjectActionType.OTHER)
+ assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION)
+ assert(po.dbname startsWith reusedDb.toLowerCase)
+ assert(po.objectName startsWith functionNamePrefix.toLowerCase)
+ val accessType = ranger.AccessType(po, QUERY, isInput = true)
+ assert(accessType === AccessType.SELECT)
+ }
+ }
+
+ test("Function Call In CAST Command") {
+ assume(isSparkVersionAtMost("3.3"))
+ val table = "castTable"
+ withTable(table) { table =>
+ val plan = sql(s"CREATE TABLE ${table} " +
+ s"SELECT kyuubi_fun_1('data') col1, " +
+ s"${reusedDb2}.kyuubi_fun_2(value) col2, " +
+ s"kyuubi_fun_0(value) col3, " +
+ s"kyuubi_fun_2('value') col4, " +
+ s"${reusedDb}.kyuubi_fun_2('value') col5, " +
+ s"${reusedDb}.kyuubi_fun_1('value') col6, " +
+ s"kyuubi_temp_fun_1('data2') col7, " +
+ s"kyuubi_temp_fun_2(key) col8 " +
+ s"FROM ${reusedTable} WHERE
${reusedDb2}.kyuubi_fun_1(key)='123'").queryExecution.analyzed
+ val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark)
+ assert(inputs.size === 7)
+ inputs.foreach { po =>
+ assert(po.actionType === PrivilegeObjectActionType.OTHER)
+ assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION)
+ assert(po.dbname startsWith reusedDb.toLowerCase)
+ assert(po.objectName startsWith functionNamePrefix.toLowerCase)
+ val accessType = ranger.AccessType(po, QUERY, isInput = true)
+ assert(accessType === AccessType.SELECT)
+ }
+ }
+ }
+
+}
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala
index 7bd8260bb..b2c1868a2 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala
@@ -18,6 +18,7 @@
package org.apache.kyuubi.plugin.spark.authz.gen
import org.apache.kyuubi.plugin.spark.authz.serde._
+import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType._
object Scans {
@@ -57,9 +58,34 @@ object Scans {
ScanSpec(r, Seq(tableDesc))
}
+ val HiveSimpleUDF = {
+ ScanSpec(
+ "org.apache.spark.sql.hive.HiveSimpleUDF",
+ Seq.empty,
+ Seq(FunctionDesc(
+ "name",
+ classOf[QualifiedNameStringFunctionExtractor],
+ functionTypeDesc = Some(FunctionTypeDesc(
+ "name",
+ classOf[FunctionNameFunctionTypeExtractor],
+ Seq(TEMP, SYSTEM))),
+ isInput = true)))
+ }
+
+ val HiveGenericUDF = HiveSimpleUDF.copy(classname =
"org.apache.spark.sql.hive.HiveGenericUDF")
+
+ val HiveUDAFFunction = HiveSimpleUDF.copy(classname =
+ "org.apache.spark.sql.hive.HiveUDAFFunction")
+
+ val HiveGenericUDTF = HiveSimpleUDF.copy(classname =
"org.apache.spark.sql.hive.HiveGenericUDTF")
+
val data: Array[ScanSpec] = Array(
HiveTableRelation,
LogicalRelation,
DataSourceV2Relation,
- PermanentViewMarker)
+ PermanentViewMarker,
+ HiveSimpleUDF,
+ HiveGenericUDF,
+ HiveUDAFFunction,
+ HiveGenericUDTF)
}
