This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3161b4309f14 [SPARK-58223][SQL] Fix SparkSession.Builder.classic()
building a Connect session
3161b4309f14 is described below
commit 3161b4309f14c398cd9556498c209b7e5112eedc
Author: YangJie <[email protected]>
AuthorDate: Tue Jul 21 10:38:05 2026 +0800
[SPARK-58223][SQL] Fix SparkSession.Builder.classic() building a Connect
session
### What changes were proposed in this pull request?
`SparkSession.Builder.classic()` passed `CONNECT_COMPANION` to `mode(...)`,
identical to `connect()`:
```scala
def classic(): this.type = mode(CONNECT_COMPANION)
def connect(): this.type = mode(CONNECT_COMPANION)
```
`companion` drives `builder()` (`companion.builder()`), so on a classpath
with both implementations `SparkSession.builder().classic().getOrCreate()`
built a Connect session rather than a Classic one. This changes `classic()` to
pass `CLASSIC_COMPANION` (which is already defined and used by the config-key
path `API_MODE_CLASSIC => CLASSIC_COMPANION`).
It also adds regression coverage:
`SparkSessionBuilderImplementationBindingSuite` gains a shared
`implementationSpecificBuilder` hook and a test asserting the built session is
in the expected package, exercised by both the Classic (`.classic()`) and
Connect (`.connect()`) concrete suites -- the mode selectors previously had no
coverage.
### Why are the changes needed?
`builder().classic()` does not honor its contract: it returns a Connect
session (or fails, if only the classic implementation is available),
contradicting the method name and scaladoc. Introduced by SPARK-49700; present
since 4.1.0.
### Does this PR introduce _any_ user-facing change?
Yes. `SparkSession.builder().classic().getOrCreate()` now builds a Classic
session as documented, instead of a Connect session. This is a bug fix relative
to released 4.1.x.
### How was this patch tested?
New shared test `SPARK-58223: mode selector binds to the correct
implementation` in `SparkSessionBuilderImplementationBindingSuite`, run by both
the Classic and Connect suites. Verified reproduce-then-fix: the Classic case
fails on the unfixed tree (the built session is not in
`org.apache.spark.sql.classic`) and passes after the fix; the classic suite
passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #57379 from LuciferYang/SPARK-classic-builder.
Authored-by: YangJie <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
(cherry picked from commit 55e386d05dcf4b1d004bc52a7ae5d40d8746d963)
Signed-off-by: yangjie01 <[email protected]>
---
.../src/main/scala/org/apache/spark/sql/SparkSession.scala | 2 +-
.../sql/SparkSessionBuilderImplementationBindingSuite.scala | 12 ++++++++++++
.../SparkSessionBuilderImplementationBindingSuite.scala | 3 +++
.../SparkSessionBuilderImplementationBindingSuite.scala | 8 ++++++--
4 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 37d5d85e6344..a17a5dcac9e8 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -913,7 +913,7 @@ object SparkSession extends SparkSessionCompanion {
/**
* Make the builder create a Classic SparkSession.
*/
- def classic(): this.type = mode(CONNECT_COMPANION)
+ def classic(): this.type = mode(CLASSIC_COMPANION)
/**
* Make the builder create a Connect SparkSession.
diff --git
a/sql/api/src/test/scala/org/apache/spark/sql/SparkSessionBuilderImplementationBindingSuite.scala
b/sql/api/src/test/scala/org/apache/spark/sql/SparkSessionBuilderImplementationBindingSuite.scala
index 95c82cbdbdb6..5c9746736e7c 100644
---
a/sql/api/src/test/scala/org/apache/spark/sql/SparkSessionBuilderImplementationBindingSuite.scala
+++
b/sql/api/src/test/scala/org/apache/spark/sql/SparkSessionBuilderImplementationBindingSuite.scala
@@ -36,6 +36,13 @@ trait SparkSessionBuilderImplementationBindingSuite
protected def sparkContext: SparkContext
protected def implementationPackageName: String = getClass.getPackageName
+ /**
+ * A builder whose implementation mode is pinned via the mode selector
(`classic()` /
+ * `connect()`). Each concrete suite overrides this so we verify the
selector binds to the
+ * matching implementation.
+ */
+ protected def implementationSpecificBuilder: SparkSession.Builder
+
private def assertInCorrectPackage[T](obj: T): Unit = {
assert(obj.getClass.getPackageName == implementationPackageName)
}
@@ -72,6 +79,11 @@ trait SparkSessionBuilderImplementationBindingSuite
assert(df.head() == 11)
}
+ test("SPARK-58223: mode selector binds to the correct implementation") {
+ val session = implementationSpecificBuilder.getOrCreate()
+ assertInCorrectPackage(session)
+ }
+
test("emptyDataFrame with Schema") {
val session = SparkSession.builder().getOrCreate()
val schema =
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/SparkSessionBuilderImplementationBindingSuite.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/SparkSessionBuilderImplementationBindingSuite.scala
index 06eb06299f4c..ba47a413cd67 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/SparkSessionBuilderImplementationBindingSuite.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/SparkSessionBuilderImplementationBindingSuite.scala
@@ -33,4 +33,7 @@ class SparkSessionBuilderImplementationBindingSuite
}
override protected def sparkContext: SparkContext = null
+
+ override protected def implementationSpecificBuilder:
sql.SparkSession.Builder =
+ sql.SparkSession.builder().connect()
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/classic/SparkSessionBuilderImplementationBindingSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/classic/SparkSessionBuilderImplementationBindingSuite.scala
index 5bfc60bef2bf..bbdd429b6d61 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/classic/SparkSessionBuilderImplementationBindingSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/classic/SparkSessionBuilderImplementationBindingSuite.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.classic
-import org.apache.spark.sql
+import org.apache.spark.sql.{SparkSession => BaseSparkSession}
import org.apache.spark.sql.test.SharedSparkSession
/**
@@ -24,4 +24,8 @@ import org.apache.spark.sql.test.SharedSparkSession
*/
class SparkSessionBuilderImplementationBindingSuite
extends SharedSparkSession
- with sql.SparkSessionBuilderImplementationBindingSuite
+ with org.apache.spark.sql.SparkSessionBuilderImplementationBindingSuite {
+
+ override protected def implementationSpecificBuilder:
BaseSparkSession.Builder =
+ BaseSparkSession.builder().classic()
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
