This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9ec85d3c6b30 [SPARK-45239][CONNECT] Reduce default
spark.connect.jvmStacktrace.maxSize
9ec85d3c6b30 is described below
commit 9ec85d3c6b30f73c05f82797a8afc89e2e4be651
Author: Yihong He <[email protected]>
AuthorDate: Sat Oct 7 15:30:12 2023 -0700
[SPARK-45239][CONNECT] Reduce default spark.connect.jvmStacktrace.maxSize
### What changes were proposed in this pull request?
- Reduce default spark.connect.jvmStacktrace.maxSize
### Why are the changes needed?
- `spark.sql.pyspark.jvmStacktrace.enabled` is partially broken (i.e.
hitting the 8K header limit) and has to be disable in some tests
### Does this PR introduce _any_ user-facing change?
- No
### How was this patch tested?
- Existing tests
### Was this patch authored or co-authored using generative AI tooling?
Closes #43017 from heyihong/SPARK-45239.
Authored-by: Yihong He <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/scala/org/apache/spark/sql/connect/config/Connect.scala | 2 +-
python/pyspark/sql/tests/connect/test_connect_basic.py | 5 +++++
python/pyspark/testing/connectutils.py | 3 ---
3 files changed, 6 insertions(+), 4 deletions(-)
diff --git
a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
index 248444e710d2..2b3f218362cd 100644
---
a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
+++
b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
@@ -186,7 +186,7 @@ object Connect {
|""".stripMargin)
.version("3.5.0")
.intConf
- .createWithDefault(2048)
+ .createWithDefault(1024)
val CONNECT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL =
buildStaticConf("spark.connect.copyFromLocalToFs.allowDestLocal")
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py
b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 620c2f9d76ad..16b445644413 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -3340,6 +3340,11 @@ class SparkConnectSessionTests(ReusedConnectTestCase):
)
self.assertTrue("Caused by: java.time.DateTimeException:" in
e.exception.message)
+ def test_not_hitting_netty_header_limit(self):
+ with self.sql_conf({"spark.sql.pyspark.jvmStacktrace.enabled": True}):
+ with self.assertRaises(AnalysisException):
+ self.spark.sql("select " + "test" * 10000).collect()
+
def test_error_stack_trace(self):
with self.sql_conf({"spark.sql.connect.enrichError.enabled": False}):
with self.sql_conf({"spark.sql.pyspark.jvmStacktrace.enabled":
True}):
diff --git a/python/pyspark/testing/connectutils.py
b/python/pyspark/testing/connectutils.py
index 6c89b1025930..6ec0b1ca0976 100644
--- a/python/pyspark/testing/connectutils.py
+++ b/python/pyspark/testing/connectutils.py
@@ -166,9 +166,6 @@ class ReusedConnectTestCase(unittest.TestCase,
SQLTestUtils, PySparkErrorTestUti
Override this in subclasses to supply a more specific conf
"""
conf = SparkConf(loadDefaults=False)
- # Disable JVM stack trace in Spark Connect tests to prevent the
- # HTTP header size from exceeding the maximum allowed size.
- conf.set("spark.sql.pyspark.jvmStacktrace.enabled", "false")
# Make the server terminate reattachable streams every 1 second and
123 bytes,
# to make the tests exercise reattach.
conf.set("spark.connect.execute.reattachable.senderMaxStreamDuration",
"1s")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
