This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e38d06be77e [SPARK-44740][CONNECT][FOLLOW] Fix metadata values in
Python client
e38d06be77e is described below
commit e38d06be77e06864a09a3247f9d1cfd335583be2
Author: Martin Grund <[email protected]>
AuthorDate: Sat Aug 12 08:54:12 2023 +0900
[SPARK-44740][CONNECT][FOLLOW] Fix metadata values in Python client
### What changes were proposed in this pull request?
This is a follow-up PR that avoids putting the session ID into the metadata
of the GRPC channel.
### Why are the changes needed?
Behavhior
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
UT
Closes #42452 from grundprinzip/SPARK-44740_v2.
Authored-by: Martin Grund <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/connect/client/core.py | 1 +
python/pyspark/sql/tests/connect/test_connect_basic.py | 15 +++++++++++++++
2 files changed, 16 insertions(+)
diff --git a/python/pyspark/sql/connect/client/core.py
b/python/pyspark/sql/connect/client/core.py
index 5e6aacf5999..d9917c6ff5f 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -285,6 +285,7 @@ class ChannelBuilder:
ChannelBuilder.PARAM_USE_SSL,
ChannelBuilder.PARAM_USER_ID,
ChannelBuilder.PARAM_USER_AGENT,
+ ChannelBuilder.PARAM_SESSION_ID,
]
]
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py
b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 63b65ecce1a..c1527365fce 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -3528,6 +3528,21 @@ class ChannelBuilderTests(unittest.TestCase):
chan = ChannelBuilder(f"sc://host/;session_id={id}")
self.assertEqual(id, chan.session_id)
+ chan =
ChannelBuilder(f"sc://host/;session_id={id};user_agent=acbd;token=abcd;use_ssl=true")
+ md = chan.metadata()
+ for kv in md:
+ self.assertNotIn(
+ kv[0],
+ [
+ ChannelBuilder.PARAM_SESSION_ID,
+ ChannelBuilder.PARAM_TOKEN,
+ ChannelBuilder.PARAM_USER_ID,
+ ChannelBuilder.PARAM_USER_AGENT,
+ ChannelBuilder.PARAM_USE_SSL,
+ ],
+ "Metadata must not contain fixed params",
+ )
+
with self.assertRaises(ValueError) as ve:
chan = ChannelBuilder("sc://host/;session_id=abcd")
SparkConnectClient(chan)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
