This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 72f41a4  [SPARK-61060][CONNECT][PYTHON] Vendor pyspark.conf and 
pyspark.taskcontext, and export the names of pyspark, in the Python client as 
PySpark does
72f41a4 is described below

commit 72f41a42f7cb33f9b67081bee50a6df857c54acd
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:21:08 2026 +0900

    [SPARK-61060][CONNECT][PYTHON] Vendor pyspark.conf and pyspark.taskcontext, 
and export the names of pyspark, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61060](https://issues.apache.org/jira/browse/SPARK-61060): 
`pyspark.conf` and `pyspark.taskcontext` are vendored from Apache Spark v4.3.0, 
and `pyspark` exports the names of the PySpark client.
    
    As the PySpark client's `pyspark` of a Connect-only client, `pyspark` now 
exports:
    - the `SparkConf` of `pyspark.conf`, which keeps its settings in Python
    - the `TaskContext`, `BarrierTaskContext` and `BarrierTaskInfo` of 
`pyspark.taskcontext`
    - `Accumulator`, `AccumulatorParam`, `MarshalSerializer`, 
`CPickleSerializer`, `Profiler` and `BasicProfiler`, of their modules
    - `keyword_only`, and a `since` that appends the version to the docstring 
of a function
    
    The names of the classic `SparkContext`, such as `RDD` and `SparkFiles`, 
which need a JVM, are still not exported, as in a Connect-only PySpark client. 
`pyspark.conf` and `pyspark.taskcontext` are pure Python, byte-identical to the 
tag, and the drift check covers them.
    
    ### Why are the changes needed?
    
    The client had neither module, so `from pyspark import TaskContext` raised 
an `ImportError`. Its `SparkConf` was a stub of `set`, `get`, `remove` and a 
`setAll` of a dict. It lacked `setAppName`, `setMaster`, `setIfMissing`, 
`contains`, `toDebugString` and the other methods of PySpark's. The tests of 
the stub now check those of PySpark's `SparkConf` and `since`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `pyspark` has the modules and names of the PySpark 4.3.0 client.
    
    ### How was this patch tested?
    
    - A new live test sets the settings of a `SparkConf` on a session through 
the builder, and runs a UDF of `TaskContext.get().partitionId()`. It passes 
with official PySpark 4.3.0, and fails without this change.
    - New and changed offline tests check the names that `pyspark` exports, the 
`SparkConf` of PySpark, `since` and `keyword_only`.
    - The lib tests of both crates (582 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1908 passed, 8 skipped), and the 
live drop-in and Arrow suites (120 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #268 from HyukjinKwon/SPARK-61060-conf-taskcontext.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 python/pyspark/__init__.py          |  94 +++---
 python/pyspark/conf.py              | 277 ++++++++++++++++++
 python/pyspark/taskcontext.py       | 565 ++++++++++++++++++++++++++++++++++++
 python/tests/test_dropin_e2e.py     |  18 ++
 python/tests/test_dropin_offline.py |  79 +++--
 scripts/check_vendored_upstream.sh  |   4 +
 6 files changed, 972 insertions(+), 65 deletions(-)

diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 20f56a3..0ab67cc 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -13,7 +13,18 @@ from pyspark.version import __version__  # noqa: F401  
(tracks the Spark version
 __rust_client__: bool = True
 __engine__: str = "rust"  # native Rust core via tonic -- not grpcio/py4j
 
-from typing import Callable, TypeVar, Union
+from functools import wraps
+from typing import cast, Any, Callable, TypeVar, Union
+
+# As the PySpark client's, of a Connect-only client: the names of pyspark but 
those of the
+# classic SparkContext (RDD, SparkContext, SparkFiles, ...), which need a JVM.
+from pyspark.conf import SparkConf
+from pyspark.storagelevel import StorageLevel
+from pyspark.accumulators import Accumulator, AccumulatorParam
+from pyspark.serializers import MarshalSerializer, CPickleSerializer
+from pyspark.taskcontext import TaskContext, BarrierTaskContext, 
BarrierTaskInfo
+from pyspark.profiler import Profiler, BasicProfiler
+from pyspark._globals import _NoValue  # noqa: F401
 
 _F = TypeVar("_F", bound=Callable)
 
@@ -21,60 +32,48 @@ _F = TypeVar("_F", bound=Callable)
 def since(version: Union[str, float]) -> Callable[[_F], _F]:
     """
     A decorator that annotates a function to append the version of Spark the 
function was added.
-    For Connect-only clients, this is a no-op that just passes through the 
function.
     """
+    import re
+
+    indent_p = re.compile(r"\n( +)")
+
     def deco(f: _F) -> _F:
-        # For Connect-only, we don't modify docstrings; this is just a 
pass-through decorator
-        return f
-    return deco
+        assert f.__doc__ is not None
 
+        indents = indent_p.findall(f.__doc__)
+        indent = " " * (min(len(m) for m in indents) if indents else 0)
+        f.__doc__ = f.__doc__.rstrip() + "\n\n%s.. versionadded:: %s" % 
(indent, version)
+        return f
 
-from pyspark.sql import (
-    SparkSession,
-    DataFrame,
-    Column,
-    Row,
-)
-from pyspark.storagelevel import StorageLevel
+    return deco
 
-# For compatibility with testing harness, provide a stub SparkConf that works 
with
-# Spark Connect. This is imported by the testing utils but we don't need a full
-# implementation since we're connect-only.
-class SparkConf:
-    """Stub SparkConf for Spark Connect testing.
 
-    For pure Connect mode, we don't need a full SparkConf with JVM support.
-    This stub accepts configuration as upstream would.
+def keyword_only(func: _F) -> _F:
     """
-    def __init__(self, loadDefaults=True, _jvm=None, _jconf=None):
-        self._jconf = None
-        self.settings = {}
+    A decorator that forces keyword arguments in the wrapped method
+    and saves actual input keyword arguments in `_input_kwargs`.
 
-    def set(self, key, value):
-        """Set a configuration value."""
-        self.settings[key] = value
-        return self
-
-    def get(self, key, defaultValue=None):
-        """Get a configuration value."""
-        return self.settings.get(key, defaultValue)
+    Notes
+    -----
+    Should only be used to wrap a method where first arg is `self`
+    """
 
-    def remove(self, key):
-        """Remove a configuration key."""
-        if key in self.settings:
-            del self.settings[key]
-        return self
+    @wraps(func)
+    def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+        if len(args) > 0:
+            raise TypeError("Method %s forces keyword arguments." % 
func.__name__)
+        self._input_kwargs = kwargs
+        return func(self, **kwargs)
 
-    def setAll(self, settings):
-        """Set multiple configuration values."""
-        for key, value in settings.items():
-            self.set(key, value)
-        return self
+    return cast(_F, wrapper)
 
-    def getAll(self):
-        """Get all configuration values."""
-        return list(self.settings.items())
 
+from pyspark.sql import (
+    SparkSession,
+    DataFrame,
+    Column,
+    Row,
+)
 
 __all__ = [
     "SparkSession",
@@ -83,5 +82,14 @@ __all__ = [
     "Row",
     "SparkConf",
     "StorageLevel",
+    "Accumulator",
+    "AccumulatorParam",
+    "MarshalSerializer",
+    "CPickleSerializer",
+    "Profiler",
+    "BasicProfiler",
+    "TaskContext",
+    "BarrierTaskContext",
+    "BarrierTaskInfo",
     "__version__",
 ]
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
new file mode 100644
index 0000000..2f109b0
--- /dev/null
+++ b/python/pyspark/conf.py
@@ -0,0 +1,277 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+__all__ = ["SparkConf"]
+
+import sys
+from typing import Dict, List, Optional, Tuple, cast, overload, TYPE_CHECKING
+
+from pyspark.util import is_remote_only
+from pyspark.errors import PySparkRuntimeError
+
+if TYPE_CHECKING:
+    from py4j.java_gateway import JVMView, JavaObject
+
+
+class SparkConf:
+    """
+    Configuration for a Spark application. Used to set various Spark
+    parameters as key-value pairs.
+
+    Most of the time, you would create a SparkConf object with
+    ``SparkConf()``, which will load values from `spark.*` Java system
+    properties as well. In this case, any parameters you set directly on
+    the :class:`SparkConf` object take priority over system properties.
+
+    For unit tests, you can also call ``SparkConf(false)`` to skip
+    loading external settings and get the same configuration no matter
+    what the system properties are.
+
+    All setter methods in this class support chaining. For example,
+    you can write ``conf.setMaster("local").setAppName("My app")``.
+
+    Parameters
+    ----------
+    loadDefaults : bool
+        whether to load values from Java system properties (True by default)
+    _jvm : class:`py4j.java_gateway.JVMView`
+        internal parameter used to pass a handle to the
+        Java VM; does not need to be set by users
+    _jconf : class:`py4j.java_gateway.JavaObject`
+        Optionally pass in an existing SparkConf handle
+        to use its parameters
+
+    Notes
+    -----
+    Once a SparkConf object is passed to Spark, it is cloned
+    and can no longer be modified by the user.
+
+    Examples
+    --------
+    >>> from pyspark import SparkConf, SparkContext
+    >>> conf = SparkConf()
+    >>> conf.setMaster("local").setAppName("My app")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.master")
+    'local'
+    >>> conf.get("spark.app.name")
+    'My app'
+    >>> sc = SparkContext(conf=conf)
+    >>> sc.master
+    'local'
+    >>> sc.appName
+    'My app'
+    >>> sc.sparkHome is None
+    True
+
+    >>> conf = SparkConf(loadDefaults=False)
+    >>> conf.setSparkHome("/path")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.home")
+    '/path'
+    >>> conf.setExecutorEnv("VAR1", "value1")
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.setExecutorEnv(pairs = [("VAR3", "value3"), ("VAR4", "value4")])
+    <pyspark.conf.SparkConf object at ...>
+    >>> conf.get("spark.executorEnv.VAR1")
+    'value1'
+    >>> print(conf.toDebugString())
+    spark.executorEnv.VAR1=value1
+    spark.executorEnv.VAR3=value3
+    spark.executorEnv.VAR4=value4
+    spark.home=/path
+    >>> for p in sorted(conf.getAll(), key=lambda p: p[0]):
+    ...     print(p)
+    ('spark.executorEnv.VAR1', 'value1')
+    ('spark.executorEnv.VAR3', 'value3')
+    ('spark.executorEnv.VAR4', 'value4')
+    ('spark.home', '/path')
+    >>> conf._jconf.setExecutorEnv("VAR5", "value5")
+    JavaObject id...
+    >>> print(conf.toDebugString())
+    spark.executorEnv.VAR1=value1
+    spark.executorEnv.VAR3=value3
+    spark.executorEnv.VAR4=value4
+    spark.executorEnv.VAR5=value5
+    spark.home=/path
+    """
+
+    _jconf: Optional["JavaObject"]
+    _conf: Optional[Dict[str, str]]
+
+    def __init__(
+        self,
+        loadDefaults: bool = True,
+        _jvm: Optional["JVMView"] = None,
+        _jconf: Optional["JavaObject"] = None,
+    ):
+        """
+        Create a new Spark configuration.
+        """
+        if _jconf:
+            self._jconf = _jconf
+        else:
+            jvm = None
+            if not is_remote_only():
+                from pyspark.core.context import SparkContext
+
+                jvm = _jvm or SparkContext._jvm
+
+            if jvm is not None:
+                # JVM is created, so create self._jconf directly through JVM
+                self._jconf = jvm.SparkConf(loadDefaults)
+                self._conf = None
+            else:
+                # JVM is not created, so store data in self._conf first
+                self._jconf = None
+                self._conf = {}
+
+    def set(self, key: str, value: str) -> "SparkConf":
+        """Set a configuration property."""
+        # Try to set self._jconf first if JVM is created, set self._conf if 
JVM is not created yet.
+        if self._jconf is not None:
+            self._jconf.set(key, str(value))
+        else:
+            assert self._conf is not None
+            self._conf[key] = str(value)
+        return self
+
+    def setIfMissing(self, key: str, value: str) -> "SparkConf":
+        """Set a configuration property, if not already set."""
+        if self.get(key) is None:
+            self.set(key, value)
+        return self
+
+    def setMaster(self, value: str) -> "SparkConf":
+        """Set master URL to connect to."""
+        self.set("spark.master", value)
+        return self
+
+    def setAppName(self, value: str) -> "SparkConf":
+        """Set application name."""
+        self.set("spark.app.name", value)
+        return self
+
+    def setSparkHome(self, value: str) -> "SparkConf":
+        """Set path where Spark is installed on worker nodes."""
+        self.set("spark.home", value)
+        return self
+
+    @overload
+    def setExecutorEnv(self, key: str, value: str) -> "SparkConf": ...
+
+    @overload
+    def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> "SparkConf": 
...
+
+    def setExecutorEnv(
+        self,
+        key: Optional[str] = None,
+        value: Optional[str] = None,
+        pairs: Optional[List[Tuple[str, str]]] = None,
+    ) -> "SparkConf":
+        """Set an environment variable to be passed to executors."""
+        if (key is not None and pairs is not None) or (key is None and pairs 
is None):
+            raise PySparkRuntimeError(
+                errorClass="KEY_VALUE_PAIR_REQUIRED",
+                messageParameters={},
+            )
+        elif key is not None:
+            self.set("spark.executorEnv.{}".format(key), cast(str, value))
+        elif pairs is not None:
+            for k, v in pairs:
+                self.set("spark.executorEnv.{}".format(k), v)
+        return self
+
+    def setAll(self, pairs: List[Tuple[str, str]]) -> "SparkConf":
+        """
+        Set multiple parameters, passed as a list of key-value pairs.
+
+        Parameters
+        ----------
+        pairs : iterable of tuples
+            list of key-value pairs to set
+        """
+        for k, v in pairs:
+            self.set(k, v)
+        return self
+
+    @overload
+    def get(self, key: str) -> Optional[str]: ...
+
+    @overload
+    def get(self, key: str, defaultValue: None) -> Optional[str]: ...
+
+    @overload
+    def get(self, key: str, defaultValue: str) -> str: ...
+
+    def get(self, key: str, defaultValue: Optional[str] = None) -> 
Optional[str]:
+        """Get the configured value for some key, or return a default 
otherwise."""
+        if defaultValue is None:  # Py4J doesn't call the right get() if we 
pass None
+            if self._jconf is not None:
+                if not self._jconf.contains(key):
+                    return None
+                return self._jconf.get(key)
+            else:
+                assert self._conf is not None
+                return self._conf.get(key, None)
+        else:
+            if self._jconf is not None:
+                return self._jconf.get(key, defaultValue)
+            else:
+                assert self._conf is not None
+                return self._conf.get(key, defaultValue)
+
+    def getAll(self) -> List[Tuple[str, str]]:
+        """Get all values as a list of key-value pairs."""
+        if self._jconf is not None:
+            from py4j.java_gateway import JavaObject
+
+            return [(elem._1(), elem._2()) for elem in cast(JavaObject, 
self._jconf).getAll()]
+        else:
+            assert self._conf is not None
+            return list(self._conf.items())
+
+    def contains(self, key: str) -> bool:
+        """Does this configuration contain a given key?"""
+        if self._jconf is not None:
+            return self._jconf.contains(key)
+        else:
+            assert self._conf is not None
+            return key in self._conf
+
+    def toDebugString(self) -> str:
+        """
+        Returns a printable version of the configuration, as a list of
+        key=value pairs, one per line.
+        """
+        if self._jconf is not None:
+            return self._jconf.toDebugString()
+        else:
+            assert self._conf is not None
+            return "\n".join("%s=%s" % (k, v) for k, v in self._conf.items())
+
+
+def _test() -> None:
+    import doctest
+
+    failure_count, test_count = doctest.testmod(optionflags=doctest.ELLIPSIS)
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
new file mode 100644
index 0000000..7e1dd2d
--- /dev/null
+++ b/python/pyspark/taskcontext.py
@@ -0,0 +1,565 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from decimal import Decimal
+from typing import Any, ClassVar, Type, TypeVar, Dict, List, Optional, Union, 
cast
+
+from pyspark.util import local_connect_and_auth
+from pyspark.serializers import read_int, write_int, write_with_length, 
UTF8Deserializer
+from pyspark.errors import PySparkRuntimeError
+from pyspark.resource import ResourceInformation
+
+T = TypeVar("T", bound="TaskContext")
+
+
+class TaskContext:
+    """
+    Contextual information about a task which can be read or mutated during
+    execution. To access the TaskContext for a running task, use:
+    :meth:`TaskContext.get`.
+
+    .. versionadded:: 2.2.0
+
+    Examples
+    --------
+    >>> from pyspark import TaskContext
+
+    Get a task context instance from :class:`RDD`.
+
+    >>> spark.sparkContext.setLocalProperty("key1", "value")
+    >>> taskcontext = spark.sparkContext.parallelize([1]).map(lambda _: 
TaskContext.get()).first()
+    >>> isinstance(taskcontext.attemptNumber(), int)
+    True
+    >>> isinstance(taskcontext.partitionId(), int)
+    True
+    >>> isinstance(taskcontext.stageId(), int)
+    True
+    >>> isinstance(taskcontext.taskAttemptId(), int)
+    True
+    >>> taskcontext.getLocalProperty("key1")
+    'value'
+    >>> isinstance(taskcontext.cpus(), int)
+    True
+
+    Get a task context instance from a dataframe via Python UDF.
+
+    >>> from pyspark.sql import Row
+    >>> from pyspark.sql.functions import udf
+    >>> @udf("STRUCT<anum: INT, partid: INT, stageid: INT, taskaid: INT, prop: 
STRING, cpus: INT>")
+    ... def taskcontext_as_row():
+    ...    taskcontext = TaskContext.get()
+    ...    return Row(
+    ...        anum=taskcontext.attemptNumber(),
+    ...        partid=taskcontext.partitionId(),
+    ...        stageid=taskcontext.stageId(),
+    ...        taskaid=taskcontext.taskAttemptId(),
+    ...        prop=taskcontext.getLocalProperty("key2"),
+    ...        cpus=taskcontext.cpus())
+    ...
+    >>> spark.sparkContext.setLocalProperty("key2", "value")
+    >>> [(anum, partid, stageid, taskaid, prop, cpus)] = (
+    ...     spark.range(1).select(taskcontext_as_row()).first()
+    ... )
+    >>> isinstance(anum, int)
+    True
+    >>> isinstance(partid, int)
+    True
+    >>> isinstance(stageid, int)
+    True
+    >>> isinstance(taskaid, int)
+    True
+    >>> prop
+    'value'
+    >>> isinstance(cpus, int)
+    True
+
+    Get a task context instance from a dataframe via Pandas UDF.
+
+    >>> import pandas as pd  # doctest: +SKIP
+    >>> from pyspark.sql.functions import pandas_udf
+    >>> @pandas_udf("STRUCT<"
+    ...     "anum: INT, partid: INT, stageid: INT, taskaid: INT, prop: STRING, 
cpus: INT>")
+    ... def taskcontext_as_row(_):
+    ...    taskcontext = TaskContext.get()
+    ...    return pd.DataFrame({
+    ...        "anum": [taskcontext.attemptNumber()],
+    ...        "partid": [taskcontext.partitionId()],
+    ...        "stageid": [taskcontext.stageId()],
+    ...        "taskaid": [taskcontext.taskAttemptId()],
+    ...        "prop": [taskcontext.getLocalProperty("key3")],
+    ...        "cpus": [taskcontext.cpus()]
+    ...    })  # doctest: +SKIP
+    ...
+    >>> spark.sparkContext.setLocalProperty("key3", "value")  # doctest: +SKIP
+    >>> [(anum, partid, stageid, taskaid, prop, cpus)] = (
+    ...     spark.range(1).select(taskcontext_as_row("id")).first()
+    ... )  # doctest: +SKIP
+    >>> isinstance(anum, int)
+    True
+    >>> isinstance(partid, int)
+    True
+    >>> isinstance(stageid, int)
+    True
+    >>> isinstance(taskaid, int)
+    True
+    >>> prop
+    'value'
+    >>> isinstance(cpus, int)
+    True
+    """
+
+    _taskContext: ClassVar[Optional["TaskContext"]] = None
+
+    _attemptNumber: Optional[int] = None
+    _partitionId: Optional[int] = None
+    _stageId: Optional[int] = None
+    _taskAttemptId: Optional[int] = None
+    _localProperties: Optional[Dict[str, str]] = None
+    _cpus: Optional[int] = None
+    _cpuAmount: Optional[Decimal] = None
+    _resources: Optional[Dict[str, "ResourceInformation"]] = None
+
+    def __new__(cls: Type["TaskContext"], **kwargs: Any) -> "TaskContext":
+        """
+        Even if users construct :class:`TaskContext` instead of using get, 
give them the singleton.
+        """
+        taskContext = cls._taskContext
+        if taskContext is not None:
+            return taskContext
+        cls._taskContext = taskContext = object.__new__(cls)
+        return taskContext
+
+    def __init__(
+        self,
+        **kwargs: Any,
+    ) -> None:
+        # Set attributes only if they are passed in and not None
+        # The kwargs are auto-mapped to the private attributes of TaskContext
+        # e.g., stageId -> _stageId
+        for key, val in kwargs.items():
+            if not hasattr(self, f"_{key}"):
+                raise TypeError(f"__init__() got an unexpected keyword 
argument '{key}'")
+            if val is not None:
+                setattr(self, f"_{key}", val)
+
+    @classmethod
+    def _getOrCreate(cls: Type["TaskContext"]) -> "TaskContext":
+        """Internal function to get or create global :class:`TaskContext`."""
+        if cls._taskContext is None:
+            cls._taskContext = TaskContext()
+        return cls._taskContext
+
+    @classmethod
+    def _setTaskContext(cls: Type["TaskContext"], taskContext: 
Optional["TaskContext"]) -> None:
+        cls._taskContext = taskContext
+
+    @classmethod
+    def get(cls: Type["TaskContext"]) -> Optional["TaskContext"]:
+        """
+        Return the currently active :class:`TaskContext`. This can be called 
inside of
+        user functions to access contextual information about running tasks.
+
+        Returns
+        -------
+        :class:`TaskContext`, optional
+
+        Notes
+        -----
+        Must be called on the worker, not the driver. Returns ``None`` if not 
initialized.
+        """
+        return cls._taskContext
+
+    @classmethod
+    def from_json(cls: Type[T], json: dict) -> T:
+        """
+        Create a TaskContext from a JSON dictionary.
+        """
+        return cls(
+            stageId=json["stageId"],
+            partitionId=json["partitionId"],
+            attemptNumber=json["attemptNumber"],
+            taskAttemptId=json["taskAttemptId"],
+            cpus=json["cpus"],
+            cpuAmount=Decimal(json["cpuAmount"]) if "cpuAmount" in json else 
None,
+            resources={
+                k: ResourceInformation(v["name"], v["addresses"])
+                for k, v in json["resources"].items()
+            },
+            localProperties=json["localProperties"],
+        )
+
+    def stageId(self) -> int:
+        """
+        The ID of the stage that this task belong to.
+
+        Returns
+        -------
+        int
+            current stage id.
+        """
+        return cast(int, self._stageId)
+
+    def partitionId(self) -> int:
+        """
+        The ID of the RDD partition that is computed by this task.
+
+        Returns
+        -------
+        int
+            current partition id.
+        """
+        return cast(int, self._partitionId)
+
+    def attemptNumber(self) -> int:
+        """
+        How many times this task has been attempted.  The first task attempt 
will be assigned
+        attemptNumber = 0, and subsequent attempts will have increasing 
attempt numbers.
+
+        Returns
+        -------
+        int
+            current attempt number.
+        """
+        return cast(int, self._attemptNumber)
+
+    def taskAttemptId(self) -> int:
+        """
+        An ID that is unique to this task attempt (within the same 
:class:`SparkContext`,
+        no two task attempts will share the same attempt ID).  This is roughly 
equivalent
+        to Hadoop's `TaskAttemptID`.
+
+        Returns
+        -------
+        int
+            current task attempt id.
+        """
+        return cast(int, self._taskAttemptId)
+
+    def getLocalProperty(self, key: str) -> Optional[str]:
+        """
+        Get a local property set upstream in the driver, or None if it is 
missing.
+
+        Parameters
+        ----------
+        key : str
+            the key of the local property to get.
+
+        Returns
+        -------
+        int
+            the value of the local property.
+        """
+        return cast(Dict[str, str], self._localProperties).get(key, None)
+
+    def cpus(self) -> int:
+        """
+        CPUs allocated to the task, rounded up to a whole number when the exact
+        allocation is fractional. Use :meth:`TaskContext.cpuAmount` to get the 
exact,
+        possibly fractional, amount.
+
+        Returns
+        -------
+        int
+            the number of CPUs.
+        """
+        return cast(int, self._cpus)
+
+    def cpuAmount(self) -> Decimal:
+        """
+        The exact amount of CPUs allocated to the task. This can be fractional 
when
+        ``spark.task.cpus`` or the task resource profile requests a fractional 
amount
+        (e.g. 0.5).
+
+        .. versionadded:: 4.3.0
+
+        Returns
+        -------
+        decimal.Decimal
+            the exact, possibly fractional amount of CPUs.
+
+        See Also
+        --------
+        TaskContext.cpus
+        """
+        return cast(Decimal, self._cpuAmount)
+
+    def resources(self) -> Dict[str, "ResourceInformation"]:
+        """
+        Resources allocated to the task. The key is the resource name and the 
value is information
+        about the resource.
+
+        Returns
+        -------
+        dict
+            a dictionary of a string resource name, and 
:class:`ResourceInformation`.
+        """
+        return cast(Dict[str, "ResourceInformation"], self._resources)
+
+
+BARRIER_FUNCTION = 1
+ALL_GATHER_FUNCTION = 2
+
+
+def _load_from_socket(
+    conn_info: Optional[Union[str, int]],
+    auth_secret: Optional[str],
+    function: int,
+    all_gather_message: Optional[str] = None,
+) -> List[str]:
+    """
+    Load data from a given socket, this is a blocking method thus only return 
when the socket
+    connection has been closed.
+    """
+    sockfile, sock = local_connect_and_auth(conn_info, auth_secret)
+
+    # The call may block forever, so no timeout
+    sock.settimeout(None)
+
+    if function == BARRIER_FUNCTION:
+        # Make a barrier() function call.
+        write_int(function, sockfile)
+    elif function == ALL_GATHER_FUNCTION:
+        # Make a all_gather() function call.
+        write_int(function, sockfile)
+        write_with_length(cast(str, all_gather_message).encode("utf-8"), 
sockfile)
+    else:
+        raise ValueError("Unrecognized function type")
+    sockfile.flush()
+
+    # Collect result.
+    len = read_int(sockfile)
+    res = []
+    for i in range(len):
+        res.append(UTF8Deserializer().loads(sockfile))
+
+    # Release resources.
+    sockfile.close()
+    sock.close()
+
+    return res
+
+
+class BarrierTaskContext(TaskContext):
+    """
+    A :class:`TaskContext` with extra contextual info and tooling for tasks in 
a barrier stage.
+    Use :func:`BarrierTaskContext.get` to obtain the barrier context for a 
running barrier task.
+
+    .. versionadded:: 2.4.0
+
+    Notes
+    -----
+    This API is experimental
+
+    Examples
+    --------
+    Set a barrier, and execute it with RDD.
+
+    >>> from pyspark import BarrierTaskContext
+    >>> def block_and_do_something(itr):
+    ...     taskcontext = BarrierTaskContext.get()
+    ...     # Do something.
+    ...
+    ...     # Wait until all tasks finished.
+    ...     taskcontext.barrier()
+    ...
+    ...     return itr
+    ...
+    >>> rdd = spark.sparkContext.parallelize([1])
+    >>> rdd.barrier().mapPartitions(block_and_do_something).collect()
+    [1]
+    """
+
+    _conn_info: ClassVar[Optional[Union[str, int]]] = None
+    _secret: ClassVar[Optional[str]] = None
+
+    @classmethod
+    def _getOrCreate(cls: Type["BarrierTaskContext"]) -> "BarrierTaskContext":
+        """
+        Internal function to get or create global :class:`BarrierTaskContext`. 
We need to make sure
+        :class:`BarrierTaskContext` is returned from here because it is needed 
in python worker
+        reuse scenario, see SPARK-25921 for more details.
+        """
+        if not isinstance(cls._taskContext, BarrierTaskContext):
+            cls._taskContext = object.__new__(cls)
+        return cls._taskContext
+
+    @classmethod
+    def get(cls: Type["BarrierTaskContext"]) -> "BarrierTaskContext":
+        """
+        Return the currently active :class:`BarrierTaskContext`.
+        This can be called inside of user functions to access contextual 
information about
+        running tasks.
+
+        Notes
+        -----
+        Must be called on the worker, not the driver. Returns ``None`` if not 
initialized.
+        An Exception will raise if it is not in a barrier stage.
+
+        This API is experimental
+        """
+        if not isinstance(cls._taskContext, BarrierTaskContext):
+            raise PySparkRuntimeError(
+                errorClass="NOT_IN_BARRIER_STAGE",
+                messageParameters={},
+            )
+        return cls._taskContext
+
+    @classmethod
+    def _initialize(
+        cls: Type["BarrierTaskContext"], conn_info: Optional[Union[str, int]], 
secret: Optional[str]
+    ) -> None:
+        """
+        Initialize :class:`BarrierTaskContext`, other methods within 
:class:`BarrierTaskContext`
+        can only be called after BarrierTaskContext is initialized.
+        """
+        cls._conn_info = conn_info
+        cls._secret = secret
+
+    @classmethod
+    def from_json(cls: Type["BarrierTaskContext"], json: dict) -> 
"BarrierTaskContext":
+        """
+        Create a BarrierTaskContext from a JSON dictionary.
+        """
+        cls._initialize(json["connInfo"], json["secret"])
+        return super().from_json(json)
+
+    def barrier(self) -> None:
+        """
+        Sets a global barrier and waits until all tasks in this stage hit this 
barrier.
+        Similar to `MPI_Barrier` function in MPI, this function blocks until 
all tasks
+        in the same stage have reached this routine.
+
+        .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is experimental
+
+        In a barrier stage, each task much have the same number of `barrier()`
+        calls, in all possible code branches. Otherwise, you may get the job 
hanging
+        or a `SparkException` after timeout.
+        """
+        if self._conn_info is None:
+            raise PySparkRuntimeError(
+                errorClass="CALL_BEFORE_INITIALIZE",
+                messageParameters={
+                    "func_name": "barrier",
+                    "object": "BarrierTaskContext",
+                },
+            )
+        else:
+            _load_from_socket(self._conn_info, self._secret, BARRIER_FUNCTION)
+
+    def allGather(self, message: str = "") -> List[str]:
+        """
+        This function blocks until all tasks in the same stage have reached 
this routine.
+        Each task passes in a message and returns with a list of all the 
messages passed in
+        by each of those tasks.
+
+        .. versionadded:: 3.0.0
+
+        Notes
+        -----
+        This API is experimental
+
+        In a barrier stage, each task much have the same number of `barrier()`
+        calls, in all possible code branches. Otherwise, you may get the job 
hanging
+        or a `SparkException` after timeout.
+        """
+        if not isinstance(message, str):
+            raise TypeError("Argument `message` must be of type `str`")
+        elif self._conn_info is None:
+            raise PySparkRuntimeError(
+                errorClass="CALL_BEFORE_INITIALIZE",
+                messageParameters={
+                    "func_name": "allGather",
+                    "object": "BarrierTaskContext",
+                },
+            )
+        else:
+            return _load_from_socket(self._conn_info, self._secret, 
ALL_GATHER_FUNCTION, message)
+
+    def getTaskInfos(self) -> List["BarrierTaskInfo"]:
+        """
+        Returns :class:`BarrierTaskInfo` for all tasks in this barrier stage,
+        ordered by partition ID.
+
+        .. versionadded:: 2.4.0
+
+        Notes
+        -----
+        This API is experimental
+
+        Examples
+        --------
+        >>> from pyspark import BarrierTaskContext
+        >>> rdd = spark.sparkContext.parallelize([1])
+        >>> barrier_info = rdd.barrier().mapPartitions(
+        ...     lambda _: 
[BarrierTaskContext.get().getTaskInfos()]).collect()[0][0]
+        >>> barrier_info.address
+        '...:...'
+        """
+        if self._conn_info is None:
+            raise PySparkRuntimeError(
+                errorClass="CALL_BEFORE_INITIALIZE",
+                messageParameters={
+                    "func_name": "getTaskInfos",
+                    "object": "BarrierTaskContext",
+                },
+            )
+        else:
+            addresses = cast(Dict[str, str], 
self._localProperties).get("addresses", "")
+            return [BarrierTaskInfo(h.strip()) for h in addresses.split(",")]
+
+
+class BarrierTaskInfo:
+    """
+    Carries all task infos of a barrier task.
+
+    .. versionadded:: 2.4.0
+
+    Attributes
+    ----------
+    address : str
+        The IPv4 address (host:port) of the executor that the barrier task is 
running on
+
+    Notes
+    -----
+    This API is experimental
+    """
+
+    def __init__(self, address: str) -> None:
+        self.address = address
+
+
+def _test() -> None:
+    import doctest
+    import sys
+    from pyspark.sql import SparkSession
+
+    globs = globals().copy()
+    globs["spark"] = (
+        SparkSession.builder.master("local[2]").appName("taskcontext 
tests").getOrCreate()
+    )
+    failure_count, test_count = doctest.testmod(globs=globs, 
optionflags=doctest.ELLIPSIS)
+    globs["spark"].stop()
+
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index ec1a8b5..b58f3f2 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -692,6 +692,24 @@ def test_create_dataframe_from_arrow_table(session, 
schema, expected):
     assert _rows(df) == ARROW_TABLE_ROWS
 
 
+def test_spark_conf_and_task_context_are_those_of_pyspark(session):
+    # As PySpark's: pyspark exports the SparkConf, whose settings the builder 
sets on the
+    # session, and the TaskContext, which a UDF uses on the server.
+    from pyspark import SparkConf, TaskContext
+    from pyspark.sql import functions as F
+
+    key = "spark.sql.test.conf.of.a.spark.conf"
+    conf = SparkConf().set(key, "v").setIfMissing(key, "other")
+    try:
+        assert SparkSession.builder.config(conf=conf).getOrCreate() is session
+        assert session.conf.get(key) == "v"
+    finally:
+        session.conf.unset(key)
+    partition = F.udf(lambda: TaskContext.get().partitionId(), "int")
+    rows = session.range(0, 4, 1, 
2).select(partition().alias("p")).distinct().collect()
+    assert sorted(row.p for row in rows) == [0, 1]
+
+
 def test_add_artifacts_reach_udfs_as_pyspark(session, tmp_path):
     import uuid
     import zipfile
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 1130465..a4eb6c3 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2311,14 +2311,57 @@ with mock_connect_session() as (spark, service):
 """, str(Path(__file__).resolve().parent)], check=True)
 
 
-def test_spark_conf_stub():
+def test_spark_conf_is_that_of_pyspark():
+    # As PySpark's SparkConf of a Connect-only client, which keeps its 
settings in Python.
     from pyspark import SparkConf
-    c = SparkConf().set("a", "1").setAll({"b": "2", "c": "3"})
+    c = SparkConf().set("a", "1").setAll([("b", "2"), ("c", "3")])
     assert c.get("a") == "1"
     assert dict(c.getAll())["b"] == "2"
-    c.remove("a")
-    assert c.get("a", "default") == "default"
-    c.remove("nonexistent")  # removing an absent key is a no-op
+    assert c.contains("b") and not c.contains("z")
+    assert c.setIfMissing("a", "x").get("a") == "1"
+    assert c.setAppName("app").setMaster("local[2]").get("spark.app.name") == 
"app"
+    assert "spark.master=local[2]" in c.toDebugString().splitlines()
+
+
+def test_pyspark_exports_the_names_of_the_pyspark_client():
+    # As the PySpark client's pyspark, of a Connect-only client: its 
SparkConf, TaskContext,
+    # accumulators, serializers, profilers and decorators, of their modules.
+    import pyspark
+    from pyspark.accumulators import Accumulator, AccumulatorParam
+    from pyspark.conf import SparkConf
+    from pyspark.profiler import BasicProfiler, Profiler
+    from pyspark.serializers import CPickleSerializer, MarshalSerializer
+    from pyspark.taskcontext import BarrierTaskContext, BarrierTaskInfo, 
TaskContext
+
+    for name, value in [
+        ("SparkConf", SparkConf),
+        ("TaskContext", TaskContext),
+        ("BarrierTaskContext", BarrierTaskContext),
+        ("BarrierTaskInfo", BarrierTaskInfo),
+        ("Accumulator", Accumulator),
+        ("AccumulatorParam", AccumulatorParam),
+        ("MarshalSerializer", MarshalSerializer),
+        ("CPickleSerializer", CPickleSerializer),
+        ("Profiler", Profiler),
+        ("BasicProfiler", BasicProfiler),
+    ]:
+        assert getattr(pyspark, name) is value, name
+    assert TaskContext.get() is None
+
+    @pyspark.since(4.3)
+    def documented():
+        """A function."""
+
+    assert documented.__doc__.endswith(".. versionadded:: 4.3")
+
+    class Estimator:
+        @pyspark.keyword_only
+        def set(self, a=None):
+            return self._input_kwargs
+
+    assert Estimator().set(a=1) == {"a": 1}
+    with pytest.raises(TypeError, match="forces keyword arguments"):
+        Estimator().set(1)
 
 
 def test_sql_is_remote():
@@ -4337,21 +4380,23 @@ def test_row_length():
 
 
 def test_since_decorator():
-    """Test the @since decorator from pyspark."""
+    """Test the @since decorator from pyspark, which appends the version to 
the docstring."""
     from pyspark import since
 
     @since(3.0)
     def new_function():
+        """A function."""
         return "result"
 
-    # The decorator should be transparent (pass-through)
     assert new_function() == "result"
+    assert new_function.__doc__ == "A function.\n\n.. versionadded:: 3.0"
 
-    @since("4.0")
-    def another_func():
-        return 42
+    # As PySpark's, of a function that has a docstring.
+    with pytest.raises(AssertionError):
 
-    assert another_func() == 42
+        @since("4.0")
+        def another_func():
+            return 42
 
 
 def test_spark_conf_empty_config():
@@ -4362,16 +4407,6 @@ def test_spark_conf_empty_config():
     assert conf.getAll() == []
 
 
-def test_spark_conf_remove_missing():
-    """Test removing a non-existent key from SparkConf."""
-    from pyspark import SparkConf
-
-    conf = SparkConf()
-    # Removing a key that doesn't exist should not raise
-    result = conf.remove("nonexistent")
-    assert result is conf  # Should return self for chaining
-
-
 def test_spark_conf_get_with_default():
     """Test SparkConf.get with default value."""
     from pyspark import SparkConf
@@ -4390,7 +4425,7 @@ def test_spark_conf_chain_operations():
     conf = (SparkConf()
             .set("key1", "value1")
             .set("key2", "value2")
-            .setAll({"key3": "value3", "key4": "value4"}))
+            .setAll([("key3", "value3"), ("key4", "value4")]))
 
     assert conf.get("key1") == "value1"
     assert conf.get("key3") == "value3"
diff --git a/scripts/check_vendored_upstream.sh 
b/scripts/check_vendored_upstream.sh
index 00923bd..875eb6d 100644
--- a/scripts/check_vendored_upstream.sh
+++ b/scripts/check_vendored_upstream.sh
@@ -54,6 +54,10 @@ VENDORED_FILES=(
   # Call sites of the JVM gateway, which pyspark.sql.pandas.conversion imports.
   "traceback_utils.py"
 
+  # The SparkConf and the TaskContext of pyspark, pure Python, which it 
exports.
+  "conf.py"
+  "taskcontext.py"
+
   # Logging
   "logger/__init__.py"
   "logger/logger.py"


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to