This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 68d6fa13e97b [SPARK-54551][CONNECT][TESTS][FOLLOWUP] Redirect Test
Spark connect server output stream to files
68d6fa13e97b is described below
commit 68d6fa13e97b45ee832ded1426d469675e4fdf67
Author: vinodkc <[email protected]>
AuthorDate: Wed Dec 3 07:34:57 2025 -0800
[SPARK-54551][CONNECT][TESTS][FOLLOWUP] Redirect Test Spark connect server
output stream to files
### What changes were proposed in this pull request?
Currently, the Spark Connect test server's stdout and stderr are discarded
when SPARK_DEBUG_SC_JVM_CLIENT=false, making it difficult to debug test
failures.
This PR enables log4j logging for Test Spark Connect server in all test
modes (both debug and non-debug) by always configuring log4j2.properties.
### Why are the changes needed?
When `SPARK_DEBUG_SC_JVM_CLIENT=false`
SparkConnectJdbcDataTypeSuite randomly hangs because the child server
process blocks on write() calls when stdout/stderr pipe buffers fill up.
Without consuming the output, the buffers reach capacity and cause the process
to block indefinitely.
Instead of `Redirect.DISCARD` , redirect the logs into log4j files
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Tested and confirmed that log files are created when
1) `SPARK_DEBUG_SC_JVM_CLIENT=false build/sbt
"connect-client-jdbc/testOnly
org.apache.spark.sql.connect.client.jdbc.SparkConnectJdbcDataTypeSuite"`
OR
2) `SPARK_DEBUG_SC_JVM_CLIENT=true build/sbt "connect-client-jdbc/testOnly
org.apache.spark.sql.connect.client.jdbc.SparkConnectJdbcDataTypeSuite"`
```
In this file
./target/unit-tests.log
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #53275 from vinodkc/br_redirect_stdout_stderr_to_file.
Authored-by: vinodkc <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 0790fffb918e05b891c97359e337833eb0038791)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../sql/connect/test/IntegrationTestUtils.scala | 30 ++++++++++------------
.../sql/connect/test/RemoteSparkSession.scala | 13 +++-------
2 files changed, 17 insertions(+), 26 deletions(-)
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/IntegrationTestUtils.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/IntegrationTestUtils.scala
index 6e20db5d3493..b59d419c8c60 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/IntegrationTestUtils.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/IntegrationTestUtils.scala
@@ -59,23 +59,21 @@ object IntegrationTestUtils {
s"$connectClientHomeDir/target/$scalaDir/test-classes"
}
- private[sql] def debugConfigs: Seq[String] = {
+ private[sql] def log4jConfigs: Seq[String] = {
val log4j2 = s"$connectClientHomeDir/src/test/resources/log4j2.properties"
- if (isDebug) {
- Seq(
- // Enable to see the server plan change log
- // "--conf",
- // "spark.sql.planChangeLog.level=WARN",
-
- // Enable to see the server grpc received
- // "--conf",
- // "spark.connect.grpc.interceptor.classes=" +
- // "org.apache.spark.sql.connect.service.LoggingInterceptor",
-
- // Redirect server log into console
- "--conf",
- s"spark.driver.extraJavaOptions=-Dlog4j.configurationFile=$log4j2")
- } else Seq.empty
+ Seq(
+ // Enable to see the server plan change log
+ // "--conf",
+ // "spark.sql.planChangeLog.level=WARN",
+
+ // Enable to see the server grpc received
+ // "--conf",
+ // "spark.connect.grpc.interceptor.classes=" +
+ // "org.apache.spark.sql.connect.service.LoggingInterceptor",
+
+ // Redirect server log into console
+ "--conf",
+ s"spark.driver.extraJavaOptions=-Dlog4j.configurationFile=$log4j2")
}
// Log server start stop debug info into console
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/RemoteSparkSession.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/RemoteSparkSession.scala
index 6229e6e299db..535c0d2180d4 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/RemoteSparkSession.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/test/RemoteSparkSession.scala
@@ -77,7 +77,7 @@ object SparkConnectServerUtils {
command += "--jars" += catalystTestJar
command += "--conf" += s"spark.connect.grpc.binding.port=$port"
command ++= testConfigs
- command ++= debugConfigs
+ command ++= log4jConfigs
command += connectJar
val cmds = command.result()
debug {
@@ -90,15 +90,8 @@ object SparkConnectServerUtils {
builder.directory(new File(sparkHome))
val environment = builder.environment()
environment.remove("SPARK_DIST_CLASSPATH")
- if (isDebug) {
- builder.redirectError(Redirect.INHERIT)
- builder.redirectOutput(Redirect.INHERIT)
- } else {
- // If output is not consumed, the stdout/stderr pipe buffers will fill
up,
- // causing the server process to block on write() calls
- builder.redirectError(Redirect.DISCARD)
- builder.redirectOutput(Redirect.DISCARD)
- }
+ builder.redirectError(Redirect.INHERIT)
+ builder.redirectOutput(Redirect.INHERIT)
val process = builder.start()
consoleOut = process.getOutputStream
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
