This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a3a8d1ca668a [SPARK-50715][PYTHON][CONNECT] `SparkSession.Builder`
sets the configs in batch
a3a8d1ca668a is described below
commit a3a8d1ca668aae61f672c84a8873960edb8ab37b
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Jan 3 12:40:07 2025 +0800
[SPARK-50715][PYTHON][CONNECT] `SparkSession.Builder` sets the configs in
batch
### What changes were proposed in this pull request?
`SparkSession.Builder` sets the configs in batch
### Why are the changes needed?
I notice that there are practice workflows with 500+ configs, existing
implementation always set the configs one by one, and thus cause 500+ `Config`
RPCs.
### Does this PR introduce _any_ user-facing change?
No, internal change
### How was this patch tested?
Existing CI
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #49346 from zhengruifeng/py_connect_set_all.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/connect/conf.py | 14 ++
python/pyspark/sql/connect/proto/base_pb2.py | 162 ++++++++++-----------
python/pyspark/sql/connect/proto/base_pb2.pyi | 17 ++-
python/pyspark/sql/connect/session.py | 40 ++---
.../src/main/protobuf/spark/connect/base.proto | 3 +
.../service/SparkConnectConfigHandler.scala | 14 +-
6 files changed, 142 insertions(+), 108 deletions(-)
diff --git a/python/pyspark/sql/connect/conf.py
b/python/pyspark/sql/connect/conf.py
index 1ef72ee3cfa4..84d7ad34fb36 100644
--- a/python/pyspark/sql/connect/conf.py
+++ b/python/pyspark/sql/connect/conf.py
@@ -49,6 +49,20 @@ class RuntimeConf:
set.__doc__ = PySparkRuntimeConfig.set.__doc__
+ def _set_all(self, configs: Dict[str, Union[str, int, bool]], silent:
bool) -> None:
+ conf_list = []
+ for key, value in configs.items():
+ if isinstance(value, bool):
+ value = "true" if value else "false"
+ elif isinstance(value, int):
+ value = str(value)
+ conf_list.append(proto.KeyValue(key=key, value=value))
+ op_set = proto.ConfigRequest.Set(pairs=conf_list, silent=silent)
+ operation = proto.ConfigRequest.Operation(set=op_set)
+ result = self._client.config(operation)
+ for warn in result.warnings:
+ warnings.warn(warn)
+
def get(
self, key: str, default: Union[Optional[str], _NoValueType] = _NoValue
) -> Optional[str]:
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py
b/python/pyspark/sql/connect/proto/base_pb2.py
index 6e946a5bd4ae..64c549ffe956 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -43,7 +43,7 @@ from pyspark.sql.connect.proto import types_pb2 as
spark_dot_connect_dot_types__
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-
b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01
\x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02
\x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17
[...]
+
b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01
\x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02
\x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17
[...]
)
_globals = globals()
@@ -167,91 +167,91 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals["_KEYVALUE"]._serialized_start = 8609
_globals["_KEYVALUE"]._serialized_end = 8674
_globals["_CONFIGREQUEST"]._serialized_start = 8677
- _globals["_CONFIGREQUEST"]._serialized_end = 9836
+ _globals["_CONFIGREQUEST"]._serialized_end = 9876
_globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 8985
_globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9483
_globals["_CONFIGREQUEST_SET"]._serialized_start = 9485
- _globals["_CONFIGREQUEST_SET"]._serialized_end = 9537
- _globals["_CONFIGREQUEST_GET"]._serialized_start = 9539
- _globals["_CONFIGREQUEST_GET"]._serialized_end = 9564
- _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9566
- _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9629
- _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9631
- _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9662
- _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9664
- _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9712
- _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9714
- _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9741
- _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9743
- _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9777
- _globals["_CONFIGRESPONSE"]._serialized_start = 9839
- _globals["_CONFIGRESPONSE"]._serialized_end = 10014
- _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10017
- _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11019
- _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10492
- _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10545
- _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start =
10547
- _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end =
10658
- _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10660
- _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10753
- _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start =
10756
- _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end =
10949
- _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11022
- _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11294
- _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11213
- _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11294
- _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11297
- _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11623
- _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11626
- _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 11978
- _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start =
11821
- _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11936
- _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start =
11938
- _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end =
11978
- _globals["_INTERRUPTREQUEST"]._serialized_start = 11981
- _globals["_INTERRUPTREQUEST"]._serialized_end = 12584
- _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12384
- _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12512
- _globals["_INTERRUPTRESPONSE"]._serialized_start = 12587
- _globals["_INTERRUPTRESPONSE"]._serialized_end = 12731
- _globals["_REATTACHOPTIONS"]._serialized_start = 12733
- _globals["_REATTACHOPTIONS"]._serialized_end = 12786
- _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12789
- _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13195
- _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13198
- _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13783
- _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13652
- _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13664
- _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13666
- _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13713
- _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13786
- _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13951
- _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13954
- _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14166
- _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14168
- _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14276
- _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14279
- _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14611
- _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14614
- _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16169
- _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start
= 14843
- _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end =
15017
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start =
15020
- _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15388
-
_globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start
= 15351
-
_globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end
= 15388
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start =
15391
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end =
15800
+ _globals["_CONFIGREQUEST_SET"]._serialized_end = 9577
+ _globals["_CONFIGREQUEST_GET"]._serialized_start = 9579
+ _globals["_CONFIGREQUEST_GET"]._serialized_end = 9604
+ _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9606
+ _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9669
+ _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9671
+ _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9702
+ _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9704
+ _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9752
+ _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9754
+ _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9781
+ _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9783
+ _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9817
+ _globals["_CONFIGRESPONSE"]._serialized_start = 9879
+ _globals["_CONFIGRESPONSE"]._serialized_end = 10054
+ _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10057
+ _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11059
+ _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10532
+ _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10585
+ _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start =
10587
+ _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end =
10698
+ _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10700
+ _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10793
+ _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start =
10796
+ _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end =
10989
+ _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11062
+ _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11334
+ _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11253
+ _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11334
+ _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11337
+ _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11663
+ _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11666
+ _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 12018
+ _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start =
11861
+ _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11976
+ _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start =
11978
+ _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end =
12018
+ _globals["_INTERRUPTREQUEST"]._serialized_start = 12021
+ _globals["_INTERRUPTREQUEST"]._serialized_end = 12624
+ _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12424
+ _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12552
+ _globals["_INTERRUPTRESPONSE"]._serialized_start = 12627
+ _globals["_INTERRUPTRESPONSE"]._serialized_end = 12771
+ _globals["_REATTACHOPTIONS"]._serialized_start = 12773
+ _globals["_REATTACHOPTIONS"]._serialized_end = 12826
+ _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12829
+ _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13235
+ _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13238
+ _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13823
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13692
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13704
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13706
+ _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13753
+ _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13826
+ _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13991
+ _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13994
+ _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14206
+ _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14208
+ _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14316
+ _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14319
+ _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14651
+ _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14654
+ _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16209
+ _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start
= 14883
+ _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end =
15057
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start =
15060
+ _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15428
+
_globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start
= 15391
+
_globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end
= 15428
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start =
15431
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end =
15840
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._serialized_start = 15702
+ ]._serialized_start = 15742
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._serialized_end = 15770
- _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15803
- _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16150
- _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16171
- _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16261
- _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16264
- _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17210
+ ]._serialized_end = 15810
+ _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15843
+ _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16190
+ _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16211
+ _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16301
+ _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16304
+ _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17250
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi
b/python/pyspark/sql/connect/proto/base_pb2.pyi
index fc3a7e804f27..adea62ed6b80 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/base_pb2.pyi
@@ -1921,17 +1921,32 @@ class ConfigRequest(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
PAIRS_FIELD_NUMBER: builtins.int
+ SILENT_FIELD_NUMBER: builtins.int
@property
def pairs(
self,
) ->
google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___KeyValue]:
"""(Required) The config key-value pairs to set."""
+ silent: builtins.bool
+ """(Optional) Whether to ignore failures."""
def __init__(
self,
*,
pairs: collections.abc.Iterable[global___KeyValue] | None = ...,
+ silent: builtins.bool | None = ...,
) -> None: ...
- def ClearField(self, field_name: typing_extensions.Literal["pairs",
b"pairs"]) -> None: ...
+ def HasField(
+ self, field_name: typing_extensions.Literal["_silent", b"_silent",
"silent", b"silent"]
+ ) -> builtins.bool: ...
+ def ClearField(
+ self,
+ field_name: typing_extensions.Literal[
+ "_silent", b"_silent", "pairs", b"pairs", "silent", b"silent"
+ ],
+ ) -> None: ...
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["_silent", b"_silent"]
+ ) -> typing_extensions.Literal["silent"] | None: ...
class Get(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
diff --git a/python/pyspark/sql/connect/session.py
b/python/pyspark/sql/connect/session.py
index 3f1663d06850..59349a17886b 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -200,34 +200,26 @@ class SparkSession:
for i in range(int(os.environ.get("PYSPARK_REMOTE_INIT_CONF_LEN",
"0"))):
init_opts =
json.loads(os.environ[f"PYSPARK_REMOTE_INIT_CONF_{i}"])
+ # The options are applied after session creation,
+ # so options ["spark.remote", "spark.master"] always take no
effect.
+ invalid_opts = ["spark.remote", "spark.master"]
+
with self._lock:
+ opts = {}
+
+ # Only attempts to set Spark SQL configurations.
+ # If the configurations are static, it might throw an
exception so
+ # simply ignore it for now.
for k, v in init_opts.items():
- # the options are applied after session creation,
- # so following options always take no effect
- if k not in [
- "spark.remote",
- "spark.master",
- ] and k.startswith("spark.sql."):
- # Only attempts to set Spark SQL configurations.
- # If the configurations are static, it might throw an
exception so
- # simply ignore it for now.
- try:
- session.conf.set(k, v)
- except Exception as e:
- logger.warn(f"Failed to set configuration {k} due
to {e}")
+ if k not in invalid_opts and k.startswith("spark.sql."):
+ opts[k] = v
- with self._lock:
for k, v in self._options.items():
- # the options are applied after session creation,
- # so following options always take no effect
- if k not in [
- "spark.remote",
- "spark.master",
- ]:
- try:
- session.conf.set(k, v)
- except Exception as e:
- logger.warn(f"Failed to set configuration {k} due
to {e}")
+ if k not in invalid_opts:
+ opts[k] = v
+
+ if len(opts) > 0:
+ session.conf._set_all(configs=opts, silent=True)
def create(self) -> "SparkSession":
has_channel_builder = self._channel_builder is not None
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto
b/sql/connect/common/src/main/protobuf/spark/connect/base.proto
index c308c7e21b66..74413509ebc1 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto
@@ -525,6 +525,9 @@ message ConfigRequest {
message Set {
// (Required) The config key-value pairs to set.
repeated KeyValue pairs = 1;
+
+ // (Optional) Whether to ignore failures.
+ optional bool silent = 2;
}
message Get {
diff --git
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala
index c5e484e022bc..06bc24b6ccae 100644
---
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala
+++
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala
@@ -73,11 +73,21 @@ class SparkConnectConfigHandler(responseObserver:
StreamObserver[proto.ConfigRes
private def handleSet(
operation: proto.ConfigRequest.Set,
conf: RuntimeConfig): proto.ConfigResponse.Builder = {
+ val silent = operation.hasSilent && operation.getSilent
val builder = proto.ConfigResponse.newBuilder()
operation.getPairsList.asScala.iterator.foreach { pair =>
val (key, value) = SparkConnectConfigHandler.toKeyValue(pair)
- conf.set(key, value.orNull)
- getWarning(key).foreach(builder.addWarnings)
+ try {
+ conf.set(key, value.orNull)
+ getWarning(key).foreach(builder.addWarnings)
+ } catch {
+ case e: Throwable =>
+ if (silent) {
+ builder.addWarnings(s"Failed to set $key to $value due to
${e.getMessage}")
+ } else {
+ throw e
+ }
+ }
}
builder
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
