This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 483b80f1886 [SPARK-46024][PYTHON][DOCS] Document parameters and
examples for RuntimeConf get, set and unset
483b80f1886 is described below
commit 483b80f18862799143003d25b6ed511113ccbda4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Tue Nov 21 17:43:15 2023 +0900
[SPARK-46024][PYTHON][DOCS] Document parameters and examples for
RuntimeConf get, set and unset
### What changes were proposed in this pull request?
This PR documents parameters and examples for RuntimeConf get, set and
unset, and let PySpark understands `SparkNoSuchElementException` exception
instance.
### Why are the changes needed?
For users to understand how set, get and unset works.
### Does this PR introduce _any_ user-facing change?
Yes it improves the user-facing documentation.
### How was this patch tested?
Unittest was added, and manually tested.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43927 from HyukjinKwon/SPARK-46024.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/errors/exceptions/captured.py | 8 ++++
python/pyspark/errors/exceptions/connect.py | 14 ++++++
python/pyspark/sql/conf.py | 71 ++++++++++++++++++++++++----
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/python/pyspark/errors/exceptions/captured.py
b/python/pyspark/errors/exceptions/captured.py
index 8eba174eea3..b7afe896553 100644
--- a/python/pyspark/errors/exceptions/captured.py
+++ b/python/pyspark/errors/exceptions/captured.py
@@ -152,6 +152,8 @@ def convert_exception(e: Py4JJavaError) ->
CapturedException:
return SparkRuntimeException(origin=e)
elif is_instance_of(gw, e, "org.apache.spark.SparkUpgradeException"):
return SparkUpgradeException(origin=e)
+ elif is_instance_of(gw, e, "org.apache.spark.SparkNoSuchElementException"):
+ return SparkNoSuchElementException(origin=e)
c: Py4JJavaError = e.getCause()
stacktrace: str = jvm.org.apache.spark.util.Utils.exceptionString(e)
@@ -301,6 +303,12 @@ class SparkUpgradeException(CapturedException,
BaseSparkUpgradeException):
"""
+class SparkNoSuchElementException(CapturedException, BaseUnknownException):
+ """
+ No such element exception.
+ """
+
+
class UnknownException(CapturedException, BaseUnknownException):
"""
None of the above exceptions.
diff --git a/python/pyspark/errors/exceptions/connect.py
b/python/pyspark/errors/exceptions/connect.py
index bb0faa7d344..80bdea542a9 100644
--- a/python/pyspark/errors/exceptions/connect.py
+++ b/python/pyspark/errors/exceptions/connect.py
@@ -177,6 +177,14 @@ def convert_exception(
"\n An exception was thrown from the Python worker. "
"Please see the stack trace below.\n%s" % message
)
+ elif "org.apache.spark.SparkNoSuchElementException" in classes:
+ return SparkNoSuchElementException(
+ message,
+ error_class=error_class,
+ sql_state=sql_state,
+ server_stacktrace=stacktrace,
+ display_server_stacktrace=display_server_stacktrace,
+ )
# Make sure that the generic SparkException is handled last.
elif "org.apache.spark.SparkException" in classes:
return SparkException(
@@ -359,3 +367,9 @@ class SparkUpgradeException(SparkConnectGrpcException,
BaseSparkUpgradeException
class SparkException(SparkConnectGrpcException):
""" """
+
+
+class SparkNoSuchElementException(SparkConnectGrpcException,
BaseSparkUpgradeException):
+ """
+ No such element exception.
+ """
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index 599bb358fb1..b00f534eb48 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -37,17 +37,52 @@ class RuntimeConfig:
"""Create a new RuntimeConfig that wraps the underlying JVM object."""
self._jconf = jconf
- @since(2.0)
def set(self, key: str, value: Union[str, int, bool]) -> None:
- """Sets the given Spark runtime configuration property."""
+ """
+ Sets the given Spark runtime configuration property.
+
+ .. versionadded:: 2.0.0
+
+ Parameters
+ ----------
+ key : str
+ key of the configuration to set.
+ value : str, int, or bool
+ value of the configuration to set.
+
+ Examples
+ --------
+ >>> spark.conf.set("key1", "value1")
+ """
self._jconf.set(key, value)
- @since(2.0)
def get(
self, key: str, default: Union[Optional[str], _NoValueType] = _NoValue
) -> Optional[str]:
- """Returns the value of Spark runtime configuration property for the
given key,
+ """
+ Returns the value of Spark runtime configuration property for the
given key,
assuming it is set.
+
+ .. versionadded:: 2.0.0
+
+ Parameters
+ ----------
+ key : str
+ key of the configuration to get.
+ default : str, optional
+ value of the configuration to get if the key does not exist.
+
+ Returns
+ -------
+ The string value of the configuration set, or None.
+
+ Examples
+ --------
+ >>> spark.conf.get("non-existent-key", "my_default")
+ 'my_default'
+ >>> spark.conf.set("my_key", "my_value")
+ >>> spark.conf.get("my_key")
+ 'my_value'
"""
self._checkType(key, "key")
if default is _NoValue:
@@ -57,9 +92,28 @@ class RuntimeConfig:
self._checkType(default, "default")
return self._jconf.get(key, default)
- @since(2.0)
def unset(self, key: str) -> None:
- """Resets the configuration property for the given key."""
+ """
+ Resets the configuration property for the given key.
+
+ .. versionadded:: 2.0.0
+
+ Parameters
+ ----------
+ key : str
+ key of the configuration to unset.
+
+ Examples
+ --------
+ >>> spark.conf.set("my_key", "my_value")
+ >>> spark.conf.get("my_key")
+ 'my_value'
+ >>> spark.conf.unset("my_key")
+ >>> spark.conf.get("my_key")
+ Traceback (most recent call last):
+ ...
+ pyspark...SparkNoSuchElementException: ... The SQL config "my_key"
cannot be found...
+ """
self._jconf.unset(key)
def _checkType(self, obj: Any, identifier: str) -> None:
@@ -87,9 +141,10 @@ def _test() -> None:
globs = pyspark.sql.conf.__dict__.copy()
spark = SparkSession.builder.master("local[4]").appName("sql.conf
tests").getOrCreate()
- globs["sc"] = spark.sparkContext
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(pyspark.sql.conf,
globs=globs)
+ (failure_count, test_count) = doctest.testmod(
+ pyspark.sql.conf, globs=globs, optionflags=doctest.ELLIPSIS
+ )
spark.stop()
if failure_count:
sys.exit(-1)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
