This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 321e051738bb [SPARK-57448][SQL] Support `spark.sql.reflect.allowList`
to restrict methods invoked by `reflect/java_method`
321e051738bb is described below
commit 321e051738bb7c2d1d719124ca28015329e31346
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Jun 14 23:08:58 2026 -0700
[SPARK-57448][SQL] Support `spark.sql.reflect.allowList` to restrict
methods invoked by `reflect/java_method`
### What changes were proposed in this pull request?
For Apache Spark 4.3.0, this PR aims to add a static SQL config
`spark.sql.reflect.allowList` that restricts which static methods the
`reflect`/`java_method` SQL functions (`CallMethodViaReflection`) may invoke.
Note that this is a new feature for operators of multi-tenant Spark server
service (like `Spark Connect Server` or `Spark Thrift Server`), not a bug fix.
### Why are the changes needed?
`reflect`/`java_method` can invoke arbitrary static Java methods. When
Spark runs as a multi-tenant server accepting SQL from untrusted clients (e.g.
a Spark Connect server or the Thrift JDBC/ODBC server), a client could call
sensitive methods such as `java.lang.System.exit` to kill the server JVM. A
static allow list lets admins restrict callable methods, and it cannot be
relaxed at runtime.
### Does this PR introduce _any_ user-facing change?
No behavior change because the default value of new static SQL config
`spark.sql.reflect.allowList` is empty which means allow all. When set, calls
not matching any pattern fail during analysis:
```
$ spark-sql --conf "spark.sql.reflect.allowList=java\.util\.UUID\..*"
spark-sql> SELECT reflect('java.lang.System', 'exit', 0);
[DATATYPE_MISMATCH.METHOD_NOT_ALLOWED] ... the method exit of the class
java.lang.System does not match the allow list "spark.sql.reflect.allowList"
```
### How was this patch tested?
New unit tests in `CallMethodViaReflectionSuite` (empty / matching /
non-matching / regex, and invalid-pattern rejection). Also ran
`SparkThrowableSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #56509 from dongjoon-hyun/SPARK-57448.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/resources/error/error-conditions.json | 5 ++
.../expressions/CallMethodViaReflection.scala | 20 ++++++++
.../apache/spark/sql/internal/StaticSQLConf.scala | 17 +++++++
.../expressions/CallMethodViaReflectionSuite.scala | 53 +++++++++++++++++++++-
4 files changed, 94 insertions(+), 1 deletion(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index e0bea26dc960..ad6e0dfc4c23 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -1866,6 +1866,11 @@
"Input to the <functionName> should have been two maps with
compatible key types, but it's [<leftType>, <rightType>]."
]
},
+ "METHOD_NOT_ALLOWED" : {
+ "message" : [
+ "the method <methodName> of the class <className> does not match the
allow list <config>."
+ ]
+ },
"NON_FOLDABLE_INPUT" : {
"message" : [
"the input <inputName> should be a foldable <inputType> expression;
however, got <inputExpr>."
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index 94d9b305d3b0..90b9f19cf8d0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -26,6 +26,7 @@ import
org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult
import
org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch,
TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryErrorsBase}
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.internal.types.StringTypeWithCollation
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
@@ -126,6 +127,14 @@ case class CallMethodViaReflection(
unexpectedParameter match {
case Some(mismatch) => mismatch
+ case _ if !methodAllowed =>
+ DataTypeMismatch(
+ errorSubClass = "METHOD_NOT_ALLOWED",
+ messageParameters = Map(
+ "methodName" -> methodName,
+ "className" -> className,
+ "config" -> toSQLConf(StaticSQLConf.REFLECT_ALLOW_LIST.key))
+ )
case _ if !classExists =>
DataTypeMismatch(
errorSubClass = "UNEXPECTED_CLASS_TYPE",
@@ -176,6 +185,17 @@ case class CallMethodViaReflection(
/** Name of the method */
@transient private lazy val methodName =
children(1).eval(null).asInstanceOf[UTF8String].toString
+ /**
+ * True if the canonical `class.method` name fully matches one of the
regular expressions in
+ * the allow list configured by [[StaticSQLConf.REFLECT_ALLOW_LIST]]. An
empty list (the default)
+ * allows every call.
+ */
+ @transient private lazy val methodAllowed: Boolean = {
+ val allowList = SQLConf.get.getConf(StaticSQLConf.REFLECT_ALLOW_LIST)
+ val canonicalName = s"$className.$methodName"
+ allowList.isEmpty || allowList.exists(canonicalName.matches)
+ }
+
/** The reflection method. */
@transient lazy val method: Method = CallMethodViaReflection
.findMethod(className, methodName,
argExprs.map(_.dataType).toImmutableArraySeq).orNull
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index ad9106ee3e06..9a3bc9d69d43 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.internal
import java.util.Locale
import java.util.concurrent.TimeUnit
+import scala.util.Try
+
import org.apache.spark.internal.config.ConfigBindingPolicy
import
org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
import org.apache.spark.util.Utils
@@ -325,4 +327,19 @@ object StaticSQLConf {
.version("4.0.0")
.booleanConf
.createWithDefault(true)
+
+ val REFLECT_ALLOW_LIST = buildStaticConf("spark.sql.reflect.allowList")
+ .doc("A comma-separated allow list of regular expressions matched against
the canonical " +
+ "static method name (in the form `class.method`, e.g.
`java.util.UUID.randomUUID`) " +
+ "that the `reflect` and `java_method` SQL functions are allowed to
invoke. A method is " +
+ "allowed when its `class.method` name fully matches at least one of the
patterns. " +
+ "When the list is empty (the default), all such invocations are
allowed.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .stringConf
+ .toSequence
+ .checkValue(
+ _.forall(pattern => Try(pattern.r).isSuccess),
+ "Every entry must be a valid regular expression.")
+ .createWithDefault(Nil)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
index 995b519bd05d..bd7aed831ae0 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
@@ -19,12 +19,14 @@ package org.apache.spark.sql.catalyst.expressions
import java.sql.Timestamp
-import org.apache.spark.{SPARK_DOC_ROOT, SparkFunSuite}
+import org.apache.spark.{SPARK_DOC_ROOT, SparkFunSuite,
SparkIllegalArgumentException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.expressions.Cast.toSQLType
import
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
+import org.apache.spark.sql.catalyst.util.QuotingUtils.toSQLConf
import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.types._
/** A static class for testing purpose. */
@@ -51,6 +53,17 @@ class CallMethodViaReflectionSuite extends SparkFunSuite
with ExpressionEvalHelp
private val staticClassName =
ReflectStaticClass.getClass.getName.stripSuffix("$")
private val dynamicClassName = classOf[ReflectDynamicClass].getName
+ /**
+ * Runs `f` with the given regular expression patterns set as the reflect
allow list. The config
+ * is static, so it cannot be set via `withSQLConf`; we install a fresh
`SQLConf` instead. Using
+ * `setConfString` keeps the `checkValue` validation in the path.
+ */
+ private def withAllowList(patterns: String*)(f: => Unit): Unit = {
+ val conf = new SQLConf()
+ conf.setConfString(StaticSQLConf.REFLECT_ALLOW_LIST.key,
patterns.mkString(","))
+ SQLConf.withExistingConf(conf)(f)
+ }
+
test("findMethod via reflection for static methods") {
assert(findMethod(staticClassName, "method1", Seq.empty).exists(_.getName
== "method1"))
assert(findMethod(staticClassName, "method2", Seq(IntegerType)).isDefined)
@@ -159,6 +172,44 @@ class CallMethodViaReflectionSuite extends SparkFunSuite
with ExpressionEvalHelp
checkEvaluation(createExpr(staticClassName, "method4", 4, "four"),
"m4four")
}
+ test("SPARK-57448: allow list restricts reflective calls when set") {
+ def methodNotAllowed(methodName: String): DataTypeMismatch =
+ DataTypeMismatch(
+ errorSubClass = "METHOD_NOT_ALLOWED",
+ messageParameters = Map(
+ "methodName" -> methodName,
+ "className" -> staticClassName,
+ "config" -> toSQLConf(StaticSQLConf.REFLECT_ALLOW_LIST.key))
+ )
+
+ // An empty allow list (the default) allows every call.
+ assert(createExpr(staticClassName,
"method1").checkInputDataTypes().isSuccess)
+
+ // When set, only methods whose canonical name matches a pattern are
allowed.
+ withAllowList(s"$staticClassName.method1") {
+ assert(createExpr(staticClassName,
"method1").checkInputDataTypes().isSuccess)
+ assert(createExpr(staticClassName, "method2", 2).checkInputDataTypes() ==
+ methodNotAllowed("method2"))
+ }
+
+ // Regular expression patterns are supported.
+ withAllowList("java\\.util\\..*") {
+ assert(createExpr("java.util.UUID",
"randomUUID").checkInputDataTypes().isSuccess)
+ }
+ }
+
+ test("SPARK-57448: invalid regular expression in the allow list is
rejected") {
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ withAllowList("(unclosed") {}
+ },
+ condition = "INVALID_CONF_VALUE.REQUIREMENT",
+ parameters = Map(
+ "confName" -> StaticSQLConf.REFLECT_ALLOW_LIST.key,
+ "confValue" -> "(unclosed",
+ "confRequirement" -> "Every entry must be a valid regular
expression."))
+ }
+
test("escaping of class and method names") {
GenerateUnsafeProjection.generate(
CallMethodViaReflection(Seq(Literal("\"quote"), Literal("\"quote"),
Literal(null))) :: Nil)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
