This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7080556dd3cd [SPARK-60604][PYTHON] Add sampling profiler for Python
UDFs
7080556dd3cd is described below
commit 7080556dd3cdc00ff6cc1b3fe95731a1248610bb
Author: Tian Gao <[email protected]>
AuthorDate: Thu Nov 5 14:35:04 2026 -0800
[SPARK-60604][PYTHON] Add sampling profiler for Python UDFs
### What changes were proposed in this pull request?
This PR adds a sampling profiler for Python UDFs, enabled with
`spark.sql.pyspark.udf.profiler=sampling`.
- `WorkerSamplingProfiler` (`pyspark/sql/profiler.py`): while the UDF runs,
a background thread wakes up every `samplingInterval` ms, calls
`sys._current_frames()`, and records each thread's call stack into a per-thread
call tree with a sample count on each node. Nodes are keyed by `co_qualname
(filename:firstlineno)`.
- When the task finishes, the result is sent back through the existing V2
profiler accumulator (`SQL_UDF_PROFIER_V2`) under a `"sampling"` key, using a
per-task key (`stage_partition.attempt_taskAttemptId`). `SamplingResultParam`
merges these call trees by recursively summing the counts.
- `ProfilerCollector._sampling_profile_results` gives access to the
collected results.
- A new SQL conf, `spark.sql.pyspark.udf.profiler.samplingInterval` (time
conf in ms, default 100ms), is passed to the Python worker by `PythonUDFRunner`
and `ArrowPythonRunner`.
- `spark.sql.pyspark.udf.profiler` now accepts `sampling` in addition to
`perf` and `memory`.
### Why are the changes needed?
The existing `perf` (cProfile) and `memory` profilers are deterministic.
They add overhead to every function call or line, which can distort results and
slow down heavy UDFs. A sampling profiler adds little, roughly constant
overhead, and gives a call-tree view (usable for flame graphs) of where time is
spent, including in threads the UDF starts.
### Does this PR introduce _any_ user-facing change?
Yes. There is a new value `sampling` for `spark.sql.pyspark.udf.profiler`
and a new conf, `spark.sql.pyspark.udf.profiler.samplingInterval`. This is a
work-in-progress draft: user-facing APIs to show or dump sampling results (like
`spark.profile.show`/`dump`) are not added yet.
### How was this patch tested?
Added `test_sampling_profiler_udf` in
`pyspark.sql.tests.test_udf_profiler`. It checks that no results are collected
when the profiler is disabled, and that with it enabled there is one result per
task and the UDF frame appears in the sampled call tree.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Isaac (PR description)
This pull request and its description were written by Isaac.
Closes #59852 from gaogaotiantian/sampling-profiler.
Authored-by: Tian Gao <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
---
python/pyspark/profiler.py | 23 ++++
python/pyspark/sql/_typing.pyi | 1 +
python/pyspark/sql/profiler.py | 121 ++++++++++++++++++++-
python/pyspark/sql/tests/test_udf_profiler.py | 61 +++++++++++
python/pyspark/worker.py | 24 +++-
python/pyspark/worker_util.py | 4 +
.../org/apache/spark/sql/internal/SQLConf.scala | 17 ++-
.../sql/execution/python/ArrowPythonRunner.scala | 1 +
.../sql/execution/python/PythonUDFRunner.scala | 6 +-
9 files changed, 250 insertions(+), 8 deletions(-)
diff --git a/python/pyspark/profiler.py b/python/pyspark/profiler.py
index b112dee0f4b1..2dd8f7b1ae93 100644
--- a/python/pyspark/profiler.py
+++ b/python/pyspark/profiler.py
@@ -161,6 +161,29 @@ class Profiler:
raise NotImplementedError
+class SamplingResultParam(AccumulatorParam[Dict[str, Any]]):
+ """SamplingResultParam is used to merge sampling results"""
+
+ @staticmethod
+ def zero(value: Dict[str, Any]) -> Dict[str, Any]:
+ return {}
+
+ @staticmethod
+ def addInPlace(value1: Dict[str, Any], value2: Dict[str, Any]) ->
Dict[str, Any]:
+ # Combine recursively
+ for key, value in value2.items():
+ if key not in value1:
+ value1[key] = value
+ else:
+ if isinstance(value, int):
+ value1[key] += value
+ elif isinstance(value, dict):
+ value1[key] = SamplingResultParam.addInPlace(value1[key],
value)
+ else:
+ raise ValueError(f"Unsupported value type: {type(value)}")
+ return value1
+
+
class PStatsParam(AccumulatorParam[Optional[pstats.Stats]]):
"""PStatsParam is used to merge pstats.Stats"""
diff --git a/python/pyspark/sql/_typing.pyi b/python/pyspark/sql/_typing.pyi
index 604e7394aa0e..e5ea7a1e211c 100644
--- a/python/pyspark/sql/_typing.pyi
+++ b/python/pyspark/sql/_typing.pyi
@@ -96,5 +96,6 @@ ProfileResults = Dict[Union[int, str],
Tuple[Optional[pstats.Stats], Optional[Co
class ProfileResult(TypedDict, total=False):
perf: pstats.Stats
memory: CodeMapDict
+ sampling: dict
ProfileResultsV2 = Dict[Union[int, str], ProfileResult]
diff --git a/python/pyspark/sql/profiler.py b/python/pyspark/sql/profiler.py
index 8917e7173e0a..353f463211ce 100644
--- a/python/pyspark/sql/profiler.py
+++ b/python/pyspark/sql/profiler.py
@@ -17,11 +17,13 @@
import cProfile
import os
import pstats
+import sys
+import threading
import warnings
from abc import ABC, abstractmethod
from io import StringIO
from threading import RLock
-from types import CodeType, TracebackType
+from types import CodeType, FrameType, TracebackType
from typing import (
TYPE_CHECKING,
Any,
@@ -42,16 +44,18 @@ from pyspark.accumulators import (
SpecialAccumulatorIds,
_accumulatorRegistry,
)
-from pyspark.errors import PySparkValueError
+from pyspark.errors import PySparkRuntimeError, PySparkValueError
from pyspark.profiler import (
CodeMapDict,
MemoryProfiler,
MemUsageParam,
PStatsParam,
+ SamplingResultParam,
)
if TYPE_CHECKING:
from pyspark.sql._typing import ProfileResults, ProfileResultsV2
+ from pyspark.taskcontext import TaskContext
class _ProfileResultsParam(AccumulatorParam[Optional["ProfileResults"]]):
@@ -119,12 +123,89 @@ class
_ProfileResultsParamV2(AccumulatorParam["ProfileResultsV2"]):
)
if memory is not None:
value1[key]["memory"] = memory
+ sampling = SamplingResultParam.addInPlace(
+ value1[key].get("sampling", {}), result.get("sampling", {})
+ )
+ if sampling:
+ value1[key]["sampling"] = sampling
return value1
ProfileResultsParamV2 = _ProfileResultsParamV2()
+class WorkerSamplingProfiler:
+ """
+ SamplingProfiler is a profiler for sampling profiling.
+ """
+
+ def __init__(
+ self,
+ task_context: "TaskContext",
+ sampling_interval: float = 100,
+ ) -> None:
+ self._callstack_data: dict[int, dict] = {}
+ self._task_context = task_context
+ self._sampling_interval = sampling_interval
+ self._sampling_thread: Optional[threading.Thread] = None
+ self._stop_event = threading.Event()
+
+ def __enter__(self) -> "WorkerSamplingProfiler":
+ self._start_sampling_thread()
+ return self
+
+ def __exit__(
+ self,
+ exc_type: Optional[type[BaseException]],
+ exc_val: Optional[BaseException],
+ exc_tb: Optional[TracebackType],
+ ) -> None:
+ assert self._sampling_thread is not None
+ self._stop_event.set()
+ self._sampling_thread.join()
+ self._sampling_thread = None
+
+ def save(self, accumulator: Accumulator["ProfileResultsV2"]) -> None:
+ accumulator.add({self._get_task_key(): {"sampling": {"callstack":
self._callstack_data}}})
+
+ def _start_sampling_thread(self) -> None:
+ if self._sampling_thread is not None:
+ raise PySparkRuntimeError("Sampling thread already started")
+ self._sampling_thread = threading.Thread(target=self._sample_loop)
+ self._sampling_thread.start()
+
+ def _sample_loop(self) -> None:
+ self_tid = threading.get_ident()
+ while not self._stop_event.wait(self._sampling_interval / 1000):
+ for tid, frame in sys._current_frames().items():
+ if tid != self_tid:
+ self._add_sample(tid, frame)
+
+ def _add_sample(self, tid: int, frame: Optional[FrameType]) -> None:
+ if tid not in self._callstack_data:
+ self._callstack_data[tid] = {}
+ frames: list[FrameType] = []
+ while frame is not None:
+ frames.append(frame)
+ frame = frame.f_back
+
+ d = self._callstack_data[tid]
+ for f in reversed(frames):
+ ident = self._get_ident_from_frame(f)
+ if ident not in d:
+ d[ident] = {"count": 0}
+ d[ident]["count"] += 1
+ d = d[ident]
+
+ def _get_ident_from_frame(self, frame: FrameType) -> str:
+ code = frame.f_code
+ return f"{code.co_qualname} ({code.co_filename}:{code.co_firstlineno})"
+
+ def _get_task_key(self) -> str:
+ tc = self._task_context
+ return
f"{tc.stageId()}_{tc.partitionId()}.{tc.attemptNumber()}_{tc.taskAttemptId()}"
+
+
class WorkerPerfProfiler:
"""
PerfProfiler is a profiler for performance profiling.
@@ -327,6 +408,15 @@ class ProfilerCollector(ABC):
if result.get("memory", None) is not None
}
+ @property
+ def _sampling_profile_results(self) -> Dict[Union[int, str], dict]:
+ with self._lock:
+ return {
+ result_id: result["sampling"]
+ for result_id, result in self._profile_results.items()
+ if result.get("sampling", None) is not None
+ }
+
@property
@abstractmethod
def _profile_results(self) -> "ProfileResultsV2":
@@ -451,6 +541,30 @@ class ProfilerCollector(ABC):
if not self._profile_results[id]:
self._profile_results.pop(id)
+ def clear_sampling_profiles(self, id: Optional[Union[int, str]] = None) ->
None:
+ """
+ Clear the sampling profile results.
+
+ .. versionadded:: 4.4.0
+
+ Parameters
+ ----------
+ id : int or str, optional
+ The result ID whose sampling profile results should be cleared.
+ If not specified, all the results will be cleared.
+ """
+ with self._lock:
+ if id is not None:
+ if id in self._profile_results:
+ self._profile_results[id].pop("sampling", None)
+ if not self._profile_results[id]:
+ self._profile_results.pop(id)
+ else:
+ for id in list(self._profile_results.keys()):
+ self._profile_results[id].pop("sampling", None)
+ if not self._profile_results[id]:
+ self._profile_results.pop(id)
+
class AccumulatorProfilerCollector(ProfilerCollector):
def __init__(self) -> None:
@@ -650,8 +764,9 @@ class Profile:
self.profiler_collector.clear_memory_profiles(id)
elif type == "perf" or type is None:
self.profiler_collector.clear_perf_profiles(id)
- if type is None: # Clear both perf and memory profiles
+ if type is None: # Clear perf, memory and sampling profiles
self.profiler_collector.clear_memory_profiles(id)
+ self.profiler_collector.clear_sampling_profiles(id)
else:
raise PySparkValueError(
errorClass="VALUE_NOT_ALLOWED",
diff --git a/python/pyspark/sql/tests/test_udf_profiler.py
b/python/pyspark/sql/tests/test_udf_profiler.py
index b435574d8887..4f9c9a362452 100644
--- a/python/pyspark/sql/tests/test_udf_profiler.py
+++ b/python/pyspark/sql/tests/test_udf_profiler.py
@@ -762,6 +762,67 @@ class UDFProfiler2TestsMixin:
self.spark.profile.clear()
self.assertEqual(0, len(self.profile_results),
str(list(self.profile_results)))
+ @property
+ def sampling_profile_results(self):
+ return self.spark._profiler_collector._sampling_profile_results
+
+ def test_sampling_profiler_udf(self):
+ import time
+
+ @udf("long")
+ def busy_add1(x):
+ # Sleep so the sampling thread reliably catches this frame on the
stack.
+ time.sleep(0.05)
+ return x + 1
+
+ # Without the conf enabled, no sampling results are collected.
+ self.spark.range(5, numPartitions=1).select(busy_add1("id")).collect()
+ self.assertEqual(0, len(self.sampling_profile_results))
+
+ with self.sql_conf(
+ {
+ "spark.sql.pyspark.udf.profiler": "sampling",
+ "spark.sql.pyspark.udf.profiler.samplingInterval": "10",
+ }
+ ):
+ self.spark.range(10,
numPartitions=2).select(busy_add1("id")).collect()
+
+ results = self.sampling_profile_results
+ # Two partitions -> two tasks -> two call-trees keyed by task.
+ self.assertEqual(2, len(results), str(list(results)))
+
+ # Flatten the per-thread call trees and assert the UDF frame was
sampled.
+ def collect_idents(node, idents):
+ for key, child in node.items():
+ if key == "count":
+ continue
+ idents.append(key)
+ collect_idents(child, idents)
+
+ for result in results.values():
+ idents = []
+ for thread_tree in result["callstack"].values():
+ collect_idents(thread_tree, idents)
+ self.assertTrue(
+ any("busy_add1" in ident for ident in idents),
+ f"UDF frame not found in samples: {idents}",
+ )
+
+ # Clearing a single result removes only that result.
+ ids = list(results)
+ self.spark._profiler_collector.clear_sampling_profiles(ids[0])
+ self.assertEqual([ids[1]], list(self.sampling_profile_results))
+
+ # Clearing other profiler types keeps the sampling results.
+ self.spark.profile.clear(type="perf")
+ self.spark.profile.clear(type="memory")
+ self.assertEqual([ids[1]], list(self.sampling_profile_results))
+
+ # Clearing all profiles removes the sampling results and the emptied
entries.
+ self.spark.profile.clear()
+ self.assertEqual(0, len(self.sampling_profile_results))
+ self.assertEqual(0,
len(self.spark._profiler_collector._profile_results))
+
class UDFProfiler2Tests(UDFProfiler2TestsMixin, ReusedSQLTestCase):
def setUp(self) -> None:
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index f10e0c0dbfad..f56b982199e6 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -31,7 +31,11 @@ import warnings
from typing import Any, BinaryIO, Optional
from pyspark import shuffle
-from pyspark.accumulators import _accumulatorRegistry
+from pyspark.accumulators import (
+ SpecialAccumulatorIds,
+ _accumulatorRegistry,
+ _deserialize_accumulator,
+)
from pyspark.errors import PySparkRuntimeError
from pyspark.eval_handlers._base import EvalTypeHandler, get_eval_type_handler
from pyspark.eval_handlers._typing import UDF, UDTF
@@ -143,7 +147,8 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
memory_limit_mb = int(os.environ.get("PYSPARK_EXECUTOR_MEMORY_MB",
"-1"))
setup_memory_limits(memory_limit_mb)
- TaskContext._setTaskContext(init_info.task_context.to_task_context())
+ task_context = init_info.task_context.to_task_context()
+ TaskContext._setTaskContext(task_context)
shuffle.MemoryBytesSpilled = 0
shuffle.DiskBytesSpilled = 0
@@ -344,9 +349,24 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
serializer._flush_per_batch = True
run_process = pipelined_process if is_pipelined else process
+ sampling_profiler_enabled = runner_conf.udf_profiler == "sampling"
+
with metrics.measure("pythonExecutionDurationMs"), capture_outputs():
if profiler:
profiler.profile(run_process)
+ elif sampling_profiler_enabled:
+ from pyspark.sql.profiler import ProfileResultsParamV2,
WorkerSamplingProfiler
+
+ accumulator = _deserialize_accumulator(
+ SpecialAccumulatorIds.SQL_UDF_PROFIER_V2, {},
ProfileResultsParamV2
+ )
+
+ with WorkerSamplingProfiler(
+
sampling_interval=runner_conf.udf_profiler_sampling_interval,
+ task_context=task_context,
+ ) as profiler:
+ run_process()
+ profiler.save(accumulator)
else:
run_process()
diff --git a/python/pyspark/worker_util.py b/python/pyspark/worker_util.py
index 10507bef0fe1..8b5bb153f634 100644
--- a/python/pyspark/worker_util.py
+++ b/python/pyspark/worker_util.py
@@ -393,6 +393,10 @@ class RunnerConf(Conf):
def udf_profiler(self) -> Optional[str]:
return self.get("spark.sql.pyspark.udf.profiler", None)
+ @property
+ def udf_profiler_sampling_interval(self) -> int:
+ return int(self.get("spark.sql.pyspark.udf.profiler.samplingInterval",
100))
+
@property
def data_source_profiler(self) -> Optional[str]:
return self.get("spark.sql.pyspark.dataSource.profiler", None)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e56f75915c3c..cb04e489b198 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5539,14 +5539,25 @@ object SQLConf {
val PYTHON_UDF_PROFILER =
buildConf("spark.sql.pyspark.udf.profiler")
.doc("Configure the Python/Pandas UDF profiler by enabling or disabling
it " +
- "with the option to choose between \"perf\" and \"memory\" types, " +
+ "with the option to choose between \"perf\", \"memory\" and
\"sampling\" types, " +
"or unsetting the config disables the profiler. This is disabled by
default.")
.version("4.0.0")
.stringConf
.transform(_.toLowerCase(Locale.ROOT))
- .checkValues(Set("perf", "memory"))
+ .checkValues(Set("perf", "memory", "sampling"))
.createOptional
+ val PYTHON_UDF_PROFILER_SAMPLING_INTERVAL =
+ buildConf("spark.sql.pyspark.udf.profiler.samplingInterval")
+ .doc("The interval in milliseconds at which the sampling profiler
samples the call " +
+ "stack. This takes effect only when spark.sql.pyspark.udf.profiler is
set to " +
+ "\"sampling\".")
+ .version("4.4.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .timeConf(TimeUnit.MILLISECONDS)
+ .checkValue(v => v > 0L, "The sampling interval must be at least 1
millisecond.")
+ .createWithDefault(100)
+
val PYTHON_UDF_WORKER_FAULTHANLDER_ENABLED =
buildConf("spark.sql.execution.pyspark.udf.faulthandler.enabled")
.doc(
@@ -10185,6 +10196,8 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def pythonUDFProfiler: Option[String] = getConf(PYTHON_UDF_PROFILER)
+ def pythonUDFProfilerSamplingInterval: Long =
getConf(PYTHON_UDF_PROFILER_SAMPLING_INTERVAL)
+
def pythonDataSourceProfiler: Option[String] =
getConf(PYTHON_DATA_SOURCE_PROFILER)
def pythonUDFWorkerFaulthandlerEnabled: Boolean =
getConf(PYTHON_UDF_WORKER_FAULTHANLDER_ENABLED)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index 2bc5f4648ea6..9096875e1e05 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -193,6 +193,7 @@ object ArrowPythonRunner {
SQLConf.PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE,
SQLConf.PYSPARK_BINARY_AS_BYTES,
SQLConf.PYTHON_UDF_ARROW_WORKER_OUTPUT_BATCH_MAX_BYTES,
+ SQLConf.PYTHON_UDF_PROFILER_SAMPLING_INTERVAL,
// Optional
SQLConf.PYTHON_UDF_ARROW_CONCURRENCY_LEVEL,
SQLConf.PYTHON_UDF_PROFILER,
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
index b66057a50ecc..37d113619542 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonUDFRunner.scala
@@ -54,7 +54,11 @@ abstract class BasePythonUDFRunner(
override def runnerConf: Map[String, String] = {
super.runnerConf ++ SQLConf.get.pythonUDFProfiler.map(p =>
Map(SQLConf.PYTHON_UDF_PROFILER.key -> p)
- ).getOrElse(Map.empty)
+ ).getOrElse(Map.empty) ++
+ Map(
+ SQLConf.PYTHON_UDF_PROFILER_SAMPLING_INTERVAL.key ->
+ SQLConf.get.pythonUDFProfilerSamplingInterval.toString
+ )
}
override val pythonExec: String =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]