This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9614eaacdceb [SPARK-59993][PYTHON] Collect worker and Arrow-optimized
Python UDF timing metrics
9614eaacdceb is described below
commit 9614eaacdceb3cd79494b67f0ed29556e86d5989
Author: Spenser Sun <[email protected]>
AuthorDate: Wed Oct 21 20:19:50 2026 -0700
[SPARK-59993][PYTHON] Collect worker and Arrow-optimized Python UDF timing
metrics
### What changes were proposed in this pull request?
- Reset `WorkerMetrics` per task and decode its numeric report into SQL
metrics.
- For Arrow scalar UDFs, measure five phases: read/deserialization, input
preparation, UDF execution, output preparation, and serialization/write. Count
timed tasks, batches, and pipelined tasks.
- A default-off config gates timing. The Arrow handler measures preparation
and UDF work; one serializer path measures Arrow read/write operations.
### Why are the changes needed?
- `pythonProcessingTime` spans the worker loop, hiding where time is spent.
### Does this PR introduce _any_ user-facing change?
- Yes. Enabling the config adds five durations and two counts to Arrow UDF
SQL metrics. Read/write time includes Arrow IPC and buffered stream calls.
- The start marker counts toward write time; the worker's final output
flush is outside these intervals.
### How was this patch tested?
- On the rebased head, 44 Arrow handler tests (plus four subtests), seven
WorkerMetrics tests, Ruff, format, and mypy passed. Earlier serializer, worker
protocol, and live gate-on Spark checks passed before the conflict-only rebase;
local `sql/compile` was blocked by missing Maven BOMs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-6).
Closes #59246 from Spenserrrr/udf-metrics-collection.
Authored-by: Spenser Sun <[email protected]>
Signed-off-by: Tian Gao <[email protected]>
---
.../org/apache/spark/api/python/PythonRunner.scala | 69 +++--
.../spark/api/python/BasePythonRunnerSuite.scala | 50 ++--
python/pyspark/eval_handlers/_arrow.py | 133 ++++++---
.../tests/test_arrow_eval_type_handlers.py | 115 ++++++++
python/pyspark/sql/pandas/serializers.py | 65 ++++-
.../sql/tests/arrow/test_arrow_python_udf.py | 306 +++++++++++++++++++++
.../pyspark/sql/tests/pandas/test_pipelined_udf.py | 9 +-
python/pyspark/tests/test_worker.py | 92 +++++++
python/pyspark/worker.py | 45 +--
python/pyspark/worker_metrics.py | 134 +++++++++
python/pyspark/worker_util.py | 4 +
.../org/apache/spark/sql/internal/SQLConf.scala | 12 +
.../sql/execution/python/ArrowEvalPythonExec.scala | 20 +-
.../sql/execution/python/ArrowPythonRunner.scala | 1 +
.../sql/execution/python/PythonSQLMetrics.scala | 27 +-
15 files changed, 943 insertions(+), 139 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index b7fc25432971..0be49a5f71f0 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -477,21 +477,12 @@ private[spark] object BasePythonRunner extends Logging {
}
}
- // Values used to update the existing Python SQL metrics and task spill
metrics.
- private[python] case class WorkerMetrics(
- bootTimestampMs: Long,
- initTimestampMs: Long,
- finishTimestampMs: Long,
- pythonExecutionDurationMs: Long,
- memoryBytesSpilled: Long,
- diskBytesSpilled: Long)
-
private lazy val workerMetricsMapper = new ObjectMapper()
.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
- /** Read and validate worker metrics after METRICS_DATA; ignore additional
fields. */
- private[python] def readWorkerMetrics(stream: DataInputStream):
WorkerMetrics = {
+ /** Decode every METRICS_DATA field as a named int64; reject other values. */
+ private[python] def readWorkerMetrics(stream: DataInputStream): Map[String,
Long] = {
val length = stream.readInt()
if (length <= 0) {
throw new SparkException(s"Invalid Python worker report length: $length")
@@ -507,21 +498,13 @@ private[spark] object BasePythonRunner extends Logging {
throw new SparkException("Expected a Python worker JSON object")
}
- def metricValue(name: String): Long = {
- val value = report.get(name)
- if (value == null || !value.isIntegralNumber || !value.canConvertToLong)
{
- throw new SparkException(s"Missing or invalid Python worker metric:
$name")
+ report.fields().asScala.map { entry =>
+ val value = entry.getValue
+ if (!value.isIntegralNumber || !value.canConvertToLong) {
+ throw new SparkException(s"Invalid Python worker metric:
${entry.getKey}")
}
- value.longValue()
- }
-
- WorkerMetrics(
- metricValue("bootTimestampMs"),
- metricValue("initTimestampMs"),
- metricValue("finishTimestampMs"),
- metricValue("pythonExecutionDurationMs"),
- metricValue("memoryBytesSpilled"),
- metricValue("diskBytesSpilled"))
+ entry.getKey -> value.longValue()
+ }.toMap
}
/**
@@ -1155,17 +1138,32 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
protected def handleMetricsData(): Unit = {
val workerMetrics = BasePythonRunner.readWorkerMetrics(stream)
- val bootTime = workerMetrics.bootTimestampMs
- val initTime = workerMetrics.initTimestampMs
- val finishTime = workerMetrics.finishTimestampMs
- val pythonExecutionDurationMs = workerMetrics.pythonExecutionDurationMs
+ def requiredMetric(name: String): Long = workerMetrics.getOrElse(name,
+ throw new SparkException(s"Missing Python worker metric: $name"))
+
+ // Read required inputs before updating any SQL or task metrics.
+ val bootTime = requiredMetric("bootTimestampMs")
+ val initTime = requiredMetric("initTimestampMs")
+ val finishTime = requiredMetric("finishTimestampMs")
+ val pythonExecutionDurationMs =
requiredMetric("pythonExecutionDurationMs")
+ val memoryBytesSpilled = requiredMetric("memoryBytesSpilled")
+ val diskBytesSpilled = requiredMetric("diskBytesSpilled")
+
// A reused Python worker records bootTime before waiting for this task,
so it can precede
// startTime. Use the later timestamp to exclude the worker's idle time
from initialization.
val pythonWorkerInitializationStartTime = math.max(startTime, bootTime)
val boot = pythonWorkerInitializationStartTime - startTime
val init = initTime - pythonWorkerInitializationStartTime
- val finish = finishTime - initTime
val total = finishTime - startTime
+ val finish = finishTime - initTime
+ val calculatedMetrics = Map(
+ "pythonBootTime" -> boot,
+ "pythonInitTime" -> init,
+ "pythonTotalTime" -> total,
+ "pythonProcessingTime" -> pythonExecutionDurationMs)
+ // Select registered report values; calculated timings take precedence
over raw aliases.
+ val sqlMetrics = workerMetrics.filter { case (name, _) =>
metrics.contains(name) } ++
+ calculatedMetrics
// Format data size for readability
val dataKB = totalDataReceived / 1024.0
@@ -1181,12 +1179,11 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
log"finish = ${MDC(TIME, finish)} - " +
log"Batches: ${MDC(COUNT, batchesProcessed)}, Data: ${MDC(SIZE,
dataStr)} - " +
log"${MDC(TASK_NAME, taskIdentifier(context))}")
- metrics.get("pythonBootTime").foreach(_.add(boot))
- metrics.get("pythonInitTime").foreach(_.add(init))
- metrics.get("pythonTotalTime").foreach(_.add(total))
-
metrics.get("pythonProcessingTime").foreach(_.add(pythonExecutionDurationMs))
-
context.taskMetrics().incMemoryBytesSpilled(workerMetrics.memoryBytesSpilled)
- context.taskMetrics().incDiskBytesSpilled(workerMetrics.diskBytesSpilled)
+ sqlMetrics.foreach { case (name, value) =>
+ metrics.get(name).foreach(_.add(value))
+ }
+ context.taskMetrics().incMemoryBytesSpilled(memoryBytesSpilled)
+ context.taskMetrics().incDiskBytesSpilled(diskBytesSpilled)
}
protected def handlePythonException(): PythonException = {
diff --git
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
index 1ea85e2910bb..f4a9d11e967c 100644
---
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
@@ -720,7 +720,13 @@ class BasePythonRunnerSuite extends SparkFunSuite with
SharedSparkContext {
"{\"bootTimestampMs\":1250,\"initTimestampMs\":2500," +
"\"finishTimestampMs\":3750,\"pythonExecutionDurationMs\":42," +
"\"memoryBytesSpilled\":7,\"diskBytesSpilled\":9}"
- private val expectedMetrics = BasePythonRunner.WorkerMetrics(1250L, 2500L,
3750L, 42L, 7L, 9L)
+ private val expectedMetrics = Map(
+ "bootTimestampMs" -> 1250L,
+ "initTimestampMs" -> 2500L,
+ "finishTimestampMs" -> 3750L,
+ "pythonExecutionDurationMs" -> 42L,
+ "memoryBytesSpilled" -> 7L,
+ "diskBytesSpilled" -> 9L)
private def framedMetricsStream(json: String, declaredLength: Option[Int] =
None,
endMarkers: Boolean = true): DataInputStream = {
@@ -745,41 +751,32 @@ class BasePythonRunnerSuite extends SparkFunSuite with
SharedSparkContext {
}
test("SPARK-59773: read worker metrics and leave the following sections
aligned") {
- val withExtra = metricsReport.dropRight(1) +
",\"futureMetric\":{\"value\":[1,2]}}"
+ val withExtra = metricsReport.dropRight(1) +
",\"futureMetric\":17,\"zeroMetric\":0}"
val stream = framedMetricsStream(withExtra)
- assert(BasePythonRunner.readWorkerMetrics(stream) == expectedMetrics)
+ val result = BasePythonRunner.readWorkerMetrics(stream)
+ assert(result == expectedMetrics ++ Map("futureMetric" -> 17L,
"zeroMetric" -> 0L))
checkEndMarkers(stream)
}
- test("SPARK-59773: worker metrics require all current fields") {
- val fields = Seq(
- "bootTimestampMs" -> "\"bootTimestampMs\":1250,",
- "initTimestampMs" -> "\"initTimestampMs\":2500,",
- "finishTimestampMs" -> "\"finishTimestampMs\":3750,",
- "pythonExecutionDurationMs" -> "\"pythonExecutionDurationMs\":42,",
- "memoryBytesSpilled" -> "\"memoryBytesSpilled\":7,",
- "diskBytesSpilled" -> ",\"diskBytesSpilled\":9")
- fields.foreach { case (name, field) =>
- val stream = framedMetricsStream(metricsReport.replace(field, ""))
- val error = intercept[SparkException] {
- BasePythonRunner.readWorkerMetrics(stream)
- }
- assert(error.getMessage.contains(name))
- checkEndMarkers(stream)
- }
+ test("SPARK-59993: decode worker metric names independently of lifecycle and
SQL consumers") {
+ val stream = framedMetricsStream("{\"futureDuration\":42}")
+ val result = BasePythonRunner.readWorkerMetrics(stream)
+ assert(result == Map("futureDuration" -> 42L))
+ checkEndMarkers(stream)
}
- test("SPARK-59773: worker metrics reject non-int64 values") {
+ test("SPARK-59773: worker metric decoder rejects non-int64 values") {
val invalidValues = Seq("null", "true", "\"42\"", "42.0", "[]", "{}",
"9223372036854775808", "-9223372036854775809")
+ val reportWithExtra = metricsReport.dropRight(1) + ",\"futureMetric\":17}"
val fields = Seq("pythonExecutionDurationMs" -> 42,
- "memoryBytesSpilled" -> 7, "diskBytesSpilled" -> 9)
+ "memoryBytesSpilled" -> 7, "diskBytesSpilled" -> 9, "futureMetric" -> 17)
for {
(name, originalValue) <- fields
value <- invalidValues
} {
val field = "\"" + name + "\":"
- val json = metricsReport.replace(field + originalValue, field + value)
+ val json = reportWithExtra.replace(field + originalValue, field + value)
val stream = framedMetricsStream(json)
val error = intercept[SparkException] {
BasePythonRunner.readWorkerMetrics(stream)
@@ -797,8 +794,8 @@ class BasePythonRunnerSuite extends SparkFunSuite with
SharedSparkContext {
.replace("\"diskBytesSpilled\":9", "\"diskBytesSpilled\":" + value)
val stream = framedMetricsStream(json)
assert(BasePythonRunner.readWorkerMetrics(stream) ==
- expectedMetrics.copy(pythonExecutionDurationMs = value,
- memoryBytesSpilled = value, diskBytesSpilled = value))
+ expectedMetrics ++ Map("pythonExecutionDurationMs" -> value,
+ "memoryBytesSpilled" -> value, "diskBytesSpilled" -> value))
checkEndMarkers(stream)
}
}
@@ -830,9 +827,10 @@ class BasePythonRunnerSuite extends SparkFunSuite with
SharedSparkContext {
}
test("SPARK-59773: worker metrics accept a larger report with an additional
field") {
- val json = metricsReport.dropRight(1) + ",\"future\":\"" + ("x" * (70 *
1024)) + "\"}"
+ val json = metricsReport.dropRight(1) + ",\"futureMetric\":17" + (" " *
(70 * 1024)) + "}"
val stream = framedMetricsStream(json)
- assert(BasePythonRunner.readWorkerMetrics(stream) == expectedMetrics)
+ val result = BasePythonRunner.readWorkerMetrics(stream)
+ assert(result == expectedMetrics + ("futureMetric" -> 17L))
checkEndMarkers(stream)
}
}
diff --git a/python/pyspark/eval_handlers/_arrow.py
b/python/pyspark/eval_handlers/_arrow.py
index 2c9acf409235..217135b9577d 100644
--- a/python/pyspark/eval_handlers/_arrow.py
+++ b/python/pyspark/eval_handlers/_arrow.py
@@ -28,6 +28,7 @@ unregistered.
from __future__ import annotations
import itertools
+import time
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
@@ -56,10 +57,12 @@ from pyspark.sql.conversion import (
ArrowTableToRowsConversion,
LocalDataToArrowConversion,
)
+from pyspark.sql.pandas.serializers import ArrowStreamSerializer
from pyspark.sql.pandas.types import to_arrow_schema, to_arrow_type
from pyspark.sql.pandas.utils import require_minimum_pyarrow_version
from pyspark.sql.types import StructField, StructType
from pyspark.util import PythonEvalType, fail_on_stopiteration
+from pyspark.worker_metrics import WorkerMetrics
if TYPE_CHECKING:
import pyarrow as pa
@@ -855,6 +858,7 @@ class
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
),
)
)
+ assert self._udf_infos, "At least one Arrow UDF expected here."
self._col_names = [f"_{i}" for i in range(len(udfs))]
# --- Input preparation ---
@@ -865,10 +869,28 @@ class
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
for f in eval_conf.input_type # type: ignore[union-attr]
]
+ @property
+ def serializer(self) -> ArrowStreamSerializer:
+ return ArrowStreamSerializer(
+ write_start_stream=True,
+ collect_timing=self._runner_conf.detailed_timing_enabled,
+ )
+
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) ->
Iterator[pa.RecordBatch]:
import pyarrow as pa
runner_conf = self._runner_conf
+ metrics = WorkerMetrics()
+ if runner_conf.detailed_timing_enabled:
+ phase_timers = (
+ metrics.measure("pythonInputPreparationTime"),
+ metrics.measure("pythonUDFExecutionTime"),
+ metrics.measure("pythonOutputPreparationTime"),
+ )
+ metrics.set("pythonNumTimingReports", 1)
+ metrics.set("pythonNumTimedBatches", 0)
+ else:
+ phase_timers = None
@fail_on_stopiteration
def _evaluate_batch_udf(udf_func: Any, rows: list[tuple]) -> list[Any]:
@@ -879,42 +901,77 @@ class
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
with
ThreadPoolExecutor(max_workers=runner_conf.arrow_concurrency_level) as pool:
return list(pool.map(lambda row: udf_func(*row), rows))
- for input_batch in data:
- num_rows = input_batch.num_rows
-
- # --- Input: Arrow -> Python columns ---
- columns = [
- (
- [conv(v) for v in
ArrowTableToRowsConversion._to_pylist(col)]
- if conv is not None
- else ArrowTableToRowsConversion._to_pylist(col)
- )
- for col, conv in zip(input_batch.itercolumns(),
self._arrow_to_py_converters)
- ]
- if not columns:
- columns = [[_NoValue] * num_rows]
-
- # --- Process: evaluate each UDF row-by-row ---
- output_arrays = []
- for udf_func, offsets, zero_arg, arrow_return_type, result_conv in
self._udf_infos:
- rows = (
- [() for _ in range(num_rows)]
- if zero_arg
- else list(zip(*[columns[o] for o in offsets]))
- )
- results = _evaluate_batch_udf(udf_func, rows)
- verify_result_row_count(len(results), num_rows)
-
- # --- Output: Python -> Arrow ---
- converted = (
- [result_conv(r) for r in results] if result_conv is not
None else results
- )
- try:
- arr = pa.array(converted, type=arrow_return_type)
- except pa.lib.ArrowInvalid:
- arr = pa.array(converted).cast(
- target_type=arrow_return_type,
safe=runner_conf.safecheck
+ # Keep phase totals and the batch count local, then publish them once
per task.
+ input_duration_ns = 0
+ udf_duration_ns = 0
+ output_duration_ns = 0
+ num_timed_batches = 0
+ clock = time.perf_counter_ns
+ num_udfs = len(self._udf_infos)
+ try:
+ for input_batch in data:
+ if phase_timers is not None:
+ num_timed_batches += 1
+ input_start_ns = clock()
+ num_rows = input_batch.num_rows
+
+ # --- Input: Arrow -> Python columns ---
+ columns = [
+ (
+ [conv(v) for v in
ArrowTableToRowsConversion._to_pylist(col)]
+ if conv is not None
+ else ArrowTableToRowsConversion._to_pylist(col)
)
- output_arrays.append(arr)
-
- yield pa.RecordBatch.from_arrays(output_arrays, self._col_names)
+ for col, conv in zip(input_batch.itercolumns(),
self._arrow_to_py_converters)
+ ]
+ if not columns:
+ columns = [[_NoValue] * num_rows]
+
+ # --- Process: evaluate each UDF row-by-row ---
+ output_arrays: list[pa.Array] = []
+ for udf_func, offsets, zero_arg, arrow_return_type,
result_conv in self._udf_infos:
+ if phase_timers is not None and output_arrays:
+ # Reuse the batch start for the first UDF, avoiding
another clock read.
+ # Later UDFs start input after the loop transition.
+ input_start_ns = clock()
+ rows = (
+ [() for _ in range(num_rows)]
+ if zero_arg
+ else list(zip(*[columns[o] for o in offsets]))
+ )
+ if phase_timers is not None:
+ udf_start_ns = clock()
+ input_duration_ns += udf_start_ns - input_start_ns
+ results = _evaluate_batch_udf(udf_func, rows)
+ if phase_timers is not None:
+ output_start_ns = clock()
+ udf_duration_ns += output_start_ns - udf_start_ns
+ verify_result_row_count(len(results), num_rows)
+
+ # --- Output: Python -> Arrow ---
+ converted = (
+ [result_conv(r) for r in results] if result_conv is
not None else results
+ )
+ try:
+ arr = pa.array(converted, type=arrow_return_type)
+ except pa.lib.ArrowInvalid:
+ arr = pa.array(converted).cast(
+ target_type=arrow_return_type,
safe=runner_conf.safecheck
+ )
+ output_arrays.append(arr)
+ # All UDF results for this batch are ready; time its
assembly as output.
+ if len(output_arrays) == num_udfs:
+ output_batch =
pa.RecordBatch.from_arrays(output_arrays, self._col_names)
+ if phase_timers is not None:
+ output_duration_ns += clock() - output_start_ns
+
+ yield output_batch
+ del output_batch
+ finally:
+ # Publish the count and completed intervals if the generator
closes early or raises.
+ if phase_timers is not None:
+ metrics.set("pythonNumTimedBatches", num_timed_batches)
+ input_timer, udf_timer, output_timer = phase_timers
+ input_timer.add_duration_ns(input_duration_ns)
+ udf_timer.add_duration_ns(udf_duration_ns)
+ output_timer.add_duration_ns(output_duration_ns)
diff --git
a/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
b/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
index 490f0cac683e..b62283a20c0f 100644
--- a/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
+++ b/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
@@ -24,6 +24,7 @@ same way: ``run(0, <input>)`` and assert on the output
batches.
import os
import unittest
+from io import BytesIO
from unittest.mock import patch
from pyspark.errors import PySparkRuntimeError
@@ -33,6 +34,7 @@ from pyspark.sql.pandas.serializers import
ArrowStreamCoGroupSerializer, ArrowSt
from pyspark.sql.types import DoubleType, LongType, StringType, StructField,
StructType
from pyspark.testing.utils import PySparkBaseTestCase, have_pyarrow,
pyarrow_requirement_message
from pyspark.util import PythonEvalType
+from pyspark.worker_metrics import WorkerMetrics
with patch.dict(os.environ, {"SPARK_PYTHON_RUNTIME": "PYTHON_WORKER"}):
from pyspark.worker_util import EvalConf, RunnerConf
@@ -201,6 +203,119 @@ class
ArrowBatchedUDFArrowHandlerTests(ArrowEvalTypeHandlerTestCase):
handler = self._handler((lambda a: a, [0], {}, LongType()))
self.assertIsInstance(handler.serializer, ArrowStreamSerializer)
+ def test_detailed_timing_uses_handler_gate(self):
+ metrics = WorkerMetrics()
+ self.addCleanup(metrics.reset)
+ timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+ for enabled in (False, True):
+ with self.subTest(enabled=enabled):
+ metrics.reset()
+ handler = self._handler(
+ (lambda a: a + 1, [0], {}, LongType()),
+ (lambda a, b: a * b, [0, 1], {}, LongType()),
+ runner_conf=RunnerConf({timing_conf:
str(enabled).lower()}),
+ )
+ self.assertEqual(handler.serializer._collect_timing, enabled)
+ output = list(
+ handler.run(
+ 0,
+ iter([self._batch(a=[1, 2], b=[10, 20]),
self._batch(a=[3], b=[4])]),
+ )
+ )
+ self.assertEqual([batch.num_rows for batch in output], [2, 1])
+ report = metrics.to_dict()
+ if enabled:
+ self.assertEqual(report["pythonNumTimingReports"], 1)
+ self.assertEqual(report["pythonNumTimedBatches"], 2)
+ for name in (
+ "pythonInputPreparationTime",
+ "pythonUDFExecutionTime",
+ "pythonOutputPreparationTime",
+ ):
+ self.assertIn(name, report)
+ else:
+ self.assertNotIn("pythonNumTimingReports", report)
+ self.assertNotIn("pythonInputPreparationTime", report)
+
+ def test_phase_timing_boundaries(self):
+ metrics = WorkerMetrics()
+ metrics.reset()
+ self.addCleanup(metrics.reset)
+ clock_ns = 0
+ udf_rows = 0
+ input_ns = 0
+ output_ns = 0
+
+ class TimedBatch:
+ def __init__(self, batch):
+ self.batch = batch
+
+ @property
+ def num_rows(self):
+ return self.batch.num_rows
+
+ def itercolumns(self):
+ nonlocal clock_ns, input_ns
+ clock_ns += 3_000_000
+ input_ns += 3_000_000
+ return self.batch.itercolumns()
+
+ class TimedInt:
+ def __init__(self, value):
+ self.value = value
+
+ def __index__(self):
+ nonlocal clock_ns, output_ns
+ clock_ns += 5_000_000
+ output_ns += 5_000_000
+ return self.value
+
+ def udf(value):
+ nonlocal clock_ns, udf_rows
+ clock_ns += 100_000_000
+ udf_rows += 1
+ return TimedInt(value + 1)
+
+ class OutputStream(BytesIO):
+ write_ns = 0
+ rows_at_write = None
+
+ def write(self, data):
+ nonlocal clock_ns
+ if self.rows_at_write is None:
+ self.rows_at_write = udf_rows
+ clock_ns += 5_000_000
+ self.write_ns += 5_000_000
+ return super().write(data)
+
+ handler = self._handler(
+ (udf, [0], {}, LongType()),
+ runner_conf=RunnerConf(
+ {"spark.sql.execution.pythonUDF.detailedTiming.enabled":
"true"}
+ ),
+ )
+ output = OutputStream()
+ batches = [self._batch(a=[1, 2], b=[10, 20]), self._batch(a=[3],
b=[30])]
+
+ def input_batches():
+ nonlocal clock_ns
+ for batch in batches:
+ clock_ns += 100_000_000
+ yield TimedBatch(batch)
+
+ with patch("pyspark.eval_handlers._arrow.time.perf_counter_ns",
new=lambda: clock_ns):
+ handler.serializer.dump_stream(handler.run(0, input_batches()),
output)
+
+ report = metrics.to_dict()
+ self.assertEqual(output.rows_at_write, 2)
+ self.assertEqual(report["pythonNumTimedBatches"], 2)
+ self.assertGreater(input_ns, 0)
+ self.assertEqual(report["pythonInputPreparationTime"], input_ns //
1_000_000)
+ self.assertEqual(report["pythonUDFExecutionTime"], 300)
+ self.assertGreater(output_ns, 0)
+ self.assertEqual(report["pythonOutputPreparationTime"], output_ns //
1_000_000)
+ self.assertEqual(report["pythonDataWriteTime"], output.write_ns //
1_000_000)
+
def test_invokes_udf_per_row(self):
handler = self._handler(
(lambda a: a + 1, [0], {}, LongType()),
diff --git a/python/pyspark/sql/pandas/serializers.py
b/python/pyspark/sql/pandas/serializers.py
index b829e4a8636f..0ddd17e55015 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -19,6 +19,7 @@
Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for
more details.
"""
+import time
from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Tuple
from pyspark.errors import PySparkRuntimeError, PySparkValueError
@@ -28,6 +29,7 @@ from pyspark.serializers import (
read_int,
write_int,
)
+from pyspark.worker_metrics import WorkerMetrics
if TYPE_CHECKING:
import pyarrow as pa
@@ -51,12 +53,27 @@ class ArrowStreamSerializer(Serializer):
write_start_stream : bool
If True, writes the START_ARROW_STREAM marker before the first
output batch. Default False.
+ collect_timing : bool
+ Record combined Arrow IPC and stream I/O time in worker metrics.
+ Enable only for UDF paths that report the corresponding SQL metrics.
+
+ Read timing covers opening the stream and fetching batches, including end
of stream.
+ Write timing covers the start marker, writer setup, batch writes, existing
per-batch
+ flushes, and close. Producer/consumer work between batches, other task
protocol, and
+ the worker owner's final flush are outside these intervals.
"""
- def __init__(self, write_start_stream: bool = False, flush_per_batch: bool
= False) -> None:
+ def __init__(
+ self,
+ write_start_stream: bool = False,
+ flush_per_batch: bool = False,
+ *,
+ collect_timing: bool = False,
+ ) -> None:
super().__init__()
self._write_start_stream: bool = write_start_stream
self._flush_per_batch: bool = flush_per_batch
+ self._collect_timing = collect_timing
def dump_stream(self, iterator: Iterable["pa.RecordBatch"], stream:
IO[bytes]) -> None:
"""Optionally prepend START_ARROW_STREAM, then write batches."""
@@ -65,9 +82,17 @@ class ArrowStreamSerializer(Serializer):
iterator = self._write_stream_start(iterator, stream)
import pyarrow as pa
+ write_timer = (
+ WorkerMetrics().measure("pythonDataWriteTime") if
self._collect_timing else None
+ )
+ clock = time.perf_counter_ns
+ write_duration_ns = 0
writer = None
try:
for batch in iterator:
+ # The batch is already produced; time Arrow serialization and
writing below.
+ if write_timer is not None:
+ start_ns = clock()
if writer is None:
writer = pa.RecordBatchStreamWriter(stream, batch.schema)
writer.write_batch(batch)
@@ -75,17 +100,44 @@ class ArrowStreamSerializer(Serializer):
# while still sending input, rather than buffering all output.
if self._flush_per_batch:
stream.flush()
+ if write_timer is not None:
+ write_duration_ns += clock() - start_ns
finally:
if writer is not None:
+ if write_timer is not None:
+ start_ns = clock()
writer.close()
+ if write_timer is not None:
+ write_duration_ns += clock() - start_ns
+ if write_timer is not None:
+ write_timer.add_duration_ns(write_duration_ns)
def load_stream(self, stream: IO[bytes]) -> Iterator["pa.RecordBatch"]:
"""Load batches from a plain Arrow stream."""
import pyarrow as pa
- reader = pa.ipc.open_stream(stream)
- for batch in reader:
- yield batch
+ read_timer = WorkerMetrics().measure("pythonDataReadTime") if
self._collect_timing else None
+ clock = time.perf_counter_ns
+ read_duration_ns = 0
+ try:
+ if read_timer is not None:
+ start_ns = clock()
+ reader = pa.ipc.open_stream(stream)
+ if read_timer is not None:
+ read_duration_ns += clock() - start_ns
+ while True:
+ if read_timer is not None:
+ start_ns = clock()
+ batch = next(reader, None)
+ if read_timer is not None:
+ read_duration_ns += clock() - start_ns
+ if batch is None:
+ return
+ # Stop timing before the consumer runs, including on early
generator close.
+ yield batch
+ finally:
+ if read_timer is not None:
+ read_timer.add_duration_ns(read_duration_ns)
def _write_stream_start(
self, batch_iterator: Iterator["pa.RecordBatch"], stream: IO[bytes]
@@ -99,7 +151,12 @@ class ArrowStreamSerializer(Serializer):
# Signal the JVM after the first batch succeeds, so errors during
# batch creation can be reported before the Arrow stream starts.
+ if self._collect_timing:
+ start_ns = time.perf_counter_ns()
write_int(SpecialLengths.START_ARROW_STREAM, stream)
+ if self._collect_timing:
+ duration_ns = time.perf_counter_ns() - start_ns
+
WorkerMetrics().measure("pythonDataWriteTime").add_duration_ns(duration_ns)
yield from itertools.chain([first], batch_iterator)
def __repr__(self) -> str:
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
index 417f1082cdc4..361ea9676ab7 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
@@ -15,13 +15,19 @@
# limitations under the License.
#
+import time
import unittest
from decimal import Decimal
+from io import BufferedReader, BufferedWriter, BytesIO
+from itertools import product
+from unittest.mock import patch
from pyspark.errors import AnalysisException, PySparkNotImplementedError,
PythonException
from pyspark.loose_version import LooseVersion
+from pyspark.serializers import read_int
from pyspark.sql import Row
from pyspark.sql.functions import col, udf
+from pyspark.sql.pandas.serializers import ArrowStreamSerializer,
SpecialLengths
from pyspark.sql.tests.test_udf import BaseUDFTestsMixin
from pyspark.sql.types import (
ArrayType,
@@ -43,6 +49,7 @@ from pyspark.testing.utils import (
pyarrow_requirement_message,
)
from pyspark.util import PythonEvalType
+from pyspark.worker_metrics import WorkerMetrics
if have_pandas:
import pandas as pd
@@ -543,6 +550,305 @@ class
ArrowPythonUDFNonLegacyTests(ArrowPythonUDFNonLegacyTestsMixin, ReusedSQLT
finally:
super().tearDownClass()
+ def test_phase_metrics(self):
+ @udf("long", useArrow=True)
+ def measured(value):
+ time.sleep(0.002)
+ return value + 1
+
+ constant = udf(lambda: 7, "long", useArrow=True)
+ phase_names = (
+ "pythonDataReadTime",
+ "pythonInputPreparationTime",
+ "pythonUDFExecutionTime",
+ "pythonOutputPreparationTime",
+ "pythonDataWriteTime",
+ )
+ with self.sql_conf(
+ {
+ "spark.sql.execution.arrow.maxRecordsPerBatch": "2",
+ "spark.sql.adaptive.enabled": "false",
+ "spark.sql.execution.pythonUDF.detailedTiming.enabled": "true",
+ }
+ ):
+ # Exercise sequential and concurrent evaluation, then an empty
worker task.
+ for num_rows, concurrency in ((5, 0), (3, 2), (0, 0)):
+ concurrency_conf = (
+ {"spark.sql.execution.pythonUDF.arrow.concurrency.level":
str(concurrency)}
+ if concurrency
+ else {}
+ )
+ with (
+ self.subTest(num_rows=num_rows, concurrency=concurrency),
+ self.sql_conf(concurrency_conf),
+ ):
+ if num_rows:
+ data = self.spark.range(num_rows, numPartitions=1)
+ else:
+ data =
self.spark.createDataFrame(self.sc.parallelize([], 1), "id long")
+ result = data.select(measured("id"), constant())
+ self.assertEqual(
+ [tuple(row) for row in result.collect()],
+ [(value + 1, 7) for value in range(num_rows)],
+ )
+ plan = result._jdf.queryExecution().executedPlan()
+ while plan.nodeName() != "ArrowEvalPython":
+ self.assertEqual(plan.children().size(), 1)
+ plan = plan.children().apply(0)
+ metrics = plan.metrics()
+
self.assertEqual(metrics.apply("pythonNumPipelinedTasks").value(), 0)
+
self.assertEqual(metrics.apply("pythonNumTimingReports").value(), 1)
+ self.assertEqual(
+ metrics.apply("pythonNumTimedBatches").value(),
(num_rows + 1) // 2
+ )
+ phases = [metrics.apply(name).value() for name in
phase_names]
+ self.assertTrue(all(value >= 0 for value in phases))
+ if num_rows:
+ for name in phase_names:
+ self.assertFalse(metrics.apply(name).isZero(),
name)
+ if num_rows and concurrency == 0:
+ self.assertGreaterEqual(phases[2], 2 * num_rows)
+ elif num_rows:
+ self.assertGreater(phases[2], 0)
+ else:
+ self.assertEqual(phases[1:4], [0, 0, 0])
+ self.assertLessEqual(sum(phases),
metrics.apply("pythonProcessingTime").value())
+
+ def test_phase_metrics_disabled_by_default(self):
+ add_one = udf(lambda value: value + 1, "long", useArrow=True)
+
+ def run():
+ result = self.spark.range(2, numPartitions=1).select(add_one("id"))
+ self.assertEqual([row[0] for row in result.collect()], [1, 2])
+ plan = result._jdf.queryExecution().executedPlan()
+ while plan.nodeName() != "ArrowEvalPython":
+ self.assertEqual(plan.children().size(), 1)
+ plan = plan.children().apply(0)
+ return plan.metrics()
+
+ timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+ with self.sql_conf({"spark.sql.adaptive.enabled": "false"}):
+ with self.sql_conf({timing_conf: "true"}):
+
self.assertEqual(run().apply("pythonNumTimingReports").value(), 1)
+ self.assertEqual(self.spark.conf.get(timing_conf), "false")
+ metrics = run()
+
+ detailed_metrics = (
+ "pythonDataReadTime",
+ "pythonInputPreparationTime",
+ "pythonUDFExecutionTime",
+ "pythonOutputPreparationTime",
+ "pythonDataWriteTime",
+ "pythonNumTimingReports",
+ "pythonNumTimedBatches",
+ )
+ for name in detailed_metrics:
+ self.assertFalse(metrics.contains(name), name)
+ self.assertTrue(metrics.contains("pythonProcessingTime"))
+ self.assertTrue(metrics.contains("pythonNumPipelinedTasks"))
+
+ def test_phase_metrics_on_materialized_plan_after_config_change(self):
+ add_one = udf(lambda value: value + 1, "long", useArrow=True)
+ timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+
+ with self.sql_conf({"spark.sql.adaptive.enabled": "false"}):
+ with self.sql_conf({timing_conf: "true"}):
+ result = self.spark.range(2,
numPartitions=1).select(add_one("id"))
+ plan = result._jdf.queryExecution().executedPlan()
+ while plan.nodeName() != "ArrowEvalPython":
+ self.assertEqual(plan.children().size(), 1)
+ plan = plan.children().apply(0)
+ metrics = plan.metrics()
+ self.assertTrue(metrics.contains("pythonNumTimingReports"))
+
+ self.assertEqual(self.spark.conf.get(timing_conf), "false")
+ self.assertEqual([row[0] for row in result.collect()], [1, 2])
+ self.assertEqual(metrics.apply("pythonNumTimingReports").value(),
1)
+ self.assertEqual(metrics.apply("pythonNumTimedBatches").value(), 1)
+
+
[email protected](not have_pyarrow, pyarrow_requirement_message)
+class ArrowStreamTimingTests(unittest.TestCase):
+ def test_timing_excludes_producer_consumer_and_owner_flush(self):
+ import pyarrow as pa
+
+ metrics = WorkerMetrics()
+ self.addCleanup(metrics.reset)
+ batch = pa.record_batch([[1, 2]], names=["value"])
+ clock_ns = 0
+
+ def clock():
+ return clock_ns
+
+ class DelayedStream(BytesIO):
+ read_ns = 0
+ write_ns = 0
+
+ def read(self, size=-1):
+ nonlocal clock_ns
+ clock_ns += 3_000_000
+ self.read_ns += 3_000_000
+ return super().read(size)
+
+ def write(self, data):
+ nonlocal clock_ns
+ clock_ns += 5_000_000
+ self.write_ns += 5_000_000
+ return super().write(data)
+
+ def flush(self):
+ nonlocal clock_ns
+ clock_ns += 7_000_000
+ self.write_ns += 7_000_000
+ return super().flush()
+
+ def produce():
+ nonlocal clock_ns
+ for _ in range(2):
+ clock_ns += 100_000_000
+ yield batch
+ clock_ns += 100_000_000
+
+ for write_start_stream, flush_per_batch, close_early in
product((False, True), repeat=3):
+ with self.subTest(
+ write_start_stream=write_start_stream,
+ flush_per_batch=flush_per_batch,
+ close_early=close_early,
+ ):
+ metrics.reset()
+ serializer = ArrowStreamSerializer(
+ write_start_stream=write_start_stream,
+ flush_per_batch=flush_per_batch,
+ collect_timing=True,
+ )
+ with patch(
+ "pyspark.sql.pandas.serializers.time.perf_counter_ns",
side_effect=clock
+ ):
+ output = DelayedStream()
+ serializer.dump_stream(produce(), output)
+ # Include the start marker and writer close, but not the
owner's flush.
+ write_ns = output.write_ns
+ output.flush()
+ source = DelayedStream(output.getvalue())
+ if write_start_stream:
+ self.assertEqual(read_int(source),
SpecialLengths.START_ARROW_STREAM)
+ source.read_ns = 0
+ reader = serializer.load_stream(source)
+ for actual in reader:
+ clock_ns += 100_000_000
+ self.assertTrue(actual.equals(batch))
+ if close_early:
+ reader.close()
+ break
+
+ # Only stream operations advance the clock inside the Arrow
intervals.
+ report = metrics.to_dict()
+ self.assertEqual(
+ report,
+ {
+ "pythonDataReadTime": source.read_ns // 1_000_000,
+ "pythonDataWriteTime": write_ns // 1_000_000,
+ },
+ )
+
+ def test_disabled_timing_does_not_read_clock(self):
+ import pyarrow as pa
+
+ metrics = WorkerMetrics()
+ metrics.reset()
+ self.addCleanup(metrics.reset)
+ batch = pa.record_batch([[1, 2]], names=["value"])
+ serializer = ArrowStreamSerializer(write_start_stream=True)
+ with patch(
+ "pyspark.sql.pandas.serializers.time.perf_counter_ns",
+ side_effect=AssertionError("timing is disabled"),
+ ):
+ stream = BytesIO()
+ serializer.dump_stream(iter([batch]), stream)
+ stream.seek(0)
+ self.assertEqual(read_int(stream),
SpecialLengths.START_ARROW_STREAM)
+
self.assertTrue(list(serializer.load_stream(stream))[0].equals(batch))
+ self.assertEqual(metrics.to_dict(), {})
+
+ def test_timing_preserves_stream_operations(self):
+ import pyarrow as pa
+
+ metrics = WorkerMetrics()
+ self.addCleanup(metrics.reset)
+ batch = pa.record_batch([[1, 2]], names=["value"])
+
+ def run(timed, buffer_size, write_start_stream, flush_per_batch,
num_batches, fail):
+ metrics.reset()
+ events = []
+
+ class RecordedStream(BytesIO):
+ def __init__(self, data, mode):
+ super().__init__(data)
+ self.mode = mode
+
+ def readable(self):
+ return self.mode == "rb"
+
+ def writable(self):
+ return self.mode == "wb"
+
+ def seekable(self):
+ return False
+
+ def readinto(self, data):
+ events.append(("readinto", len(data)))
+ return super().readinto(data)
+
+ def write(self, data):
+ events.append(("write", bytes(data)))
+ return super().write(data)
+
+ def flush(self):
+ events.append(("flush",))
+ return super().flush()
+
+ def produce():
+ for _ in range(num_batches):
+ events.append(("produce",))
+ yield batch
+ if fail:
+ raise RuntimeError("batch producer failed")
+
+ raw_output = RecordedStream(b"", "wb")
+ stream = BufferedWriter(raw_output, buffer_size)
+ serializer = ArrowStreamSerializer(
+ write_start_stream=write_start_stream,
+ flush_per_batch=flush_per_batch,
+ collect_timing=timed,
+ )
+ if fail:
+ with self.assertRaisesRegex(RuntimeError, "batch producer
failed"):
+ serializer.dump_stream(produce(), stream)
+ else:
+ serializer.dump_stream(produce(), stream)
+ stream.flush()
+ data = raw_output.getvalue()
+ raw_input = RecordedStream(data, "rb")
+ reader = BufferedReader(raw_input, buffer_size)
+ if num_batches:
+ if write_start_stream:
+ self.assertEqual(read_int(reader),
SpecialLengths.START_ARROW_STREAM)
+ for actual in serializer.load_stream(reader):
+ events.append(("consume",))
+ self.assertTrue(actual.equals(batch))
+ self.assertFalse(stream.closed)
+ reader.close()
+ stream.close()
+ return data, events
+
+ for buffer_size in (8, 64, 65536):
+ for write_start_stream in (False, True):
+ for flush_per_batch in (False, True):
+ for num_batches, fail in ((0, False), (0, True), (2,
False), (2, True)):
+ case = (buffer_size, write_start_stream,
flush_per_batch, num_batches, fail)
+ with self.subTest(case=case):
+ self.assertEqual(run(False, *case), run(True,
*case))
+
if __name__ == "__main__":
from pyspark.testing import main
diff --git a/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
b/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
index ad194070b6d8..005184ff46fc 100644
--- a/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
+++ b/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
@@ -70,8 +70,13 @@ class PipelinedUDFTests(ReusedSQLTestCase):
flag = os.environ.get("SPARK_PIPELINED_UDF", "not_set")
return pd.Series([flag] * len(x))
- result = self.spark.range(1).select(check_env(col("id"))).first()[0]
- self.assertEqual(result, "1", "JVM should set SPARK_PIPELINED_UDF=1")
+ result = self.spark.range(1,
numPartitions=1).select(check_env(col("id")))
+ self.assertEqual(result.collect()[0][0], "1", "JVM should set
SPARK_PIPELINED_UDF=1")
+ plan = result._jdf.queryExecution().executedPlan()
+ while plan.nodeName() != "ArrowEvalPython":
+ self.assertEqual(plan.children().size(), 1)
+ plan = plan.children().apply(0)
+
self.assertEqual(plan.metrics().apply("pythonNumPipelinedTasks").value(), 1)
def test_scalar_arrow_udf(self):
"""Basic scalar Arrow UDF with pipelined mode."""
diff --git a/python/pyspark/tests/test_worker.py
b/python/pyspark/tests/test_worker.py
index a7468ff0ff9e..48d2fc48e9f9 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -21,6 +21,7 @@ import tempfile
import threading
import time
import unittest
+from unittest.mock import patch
has_resource_module = True
try:
@@ -32,6 +33,97 @@ from py4j.protocol import Py4JJavaError
from pyspark import SparkConf, SparkContext
from pyspark.testing.utils import QuietTest, ReusedPySparkTestCase, eventually
+from pyspark.worker_metrics import WorkerMetrics
+
+
+class WorkerMetricsTests(unittest.TestCase):
+ def setUp(self):
+ metrics = WorkerMetrics()
+ metrics.reset()
+ self.addCleanup(metrics.reset)
+
+ def test_constructor_reuses_collector_without_reset(self):
+ metrics = WorkerMetrics()
+ metrics.set("bootTimestampMs", 123)
+ metrics.increment("rows", 2)
+ self.assertIs(WorkerMetrics(), metrics)
+ self.assertEqual(WorkerMetrics().to_dict(), {"bootTimestampMs": 123,
"rows": 2})
+
+ def test_reset_isolates_new_task_from_existing_timer(self):
+ metrics = WorkerMetrics()
+ metrics.set("previousTaskRows", 10)
+ previous_timer = metrics.measure("previousTaskDuration")
+ with patch(
+ "pyspark.worker_metrics.time.perf_counter_ns",
+ side_effect=[0, 5_000_000, 10_000_000, 12_000_000],
+ ):
+ with previous_timer:
+ metrics.reset()
+ metrics.increment("currentTaskRows")
+ with metrics.measure("currentTaskDuration"):
+ pass
+ previous_timer.add_duration_ns(3_000_000)
+ self.assertIs(WorkerMetrics(), metrics)
+ self.assertEqual(metrics.to_dict(), {"currentTaskRows": 1,
"currentTaskDuration": 2})
+
+ def test_set_current_timestamp_uses_epoch_milliseconds(self):
+ metrics = WorkerMetrics()
+ with patch("pyspark.worker_metrics.time.time_ns",
return_value=1_700_000_000_123_456_789):
+ metrics.set_current_timestamp("bootTimestampMs")
+ self.assertEqual(metrics.to_dict(), {"bootTimestampMs":
1_700_000_000_123})
+
+ def test_accumulate_before_converting_to_milliseconds(self):
+ metrics = WorkerMetrics()
+ # Control the clock to retain fractional milliseconds and exclude gaps
between blocks.
+ with patch(
+ "pyspark.worker_metrics.time.perf_counter_ns",
+ side_effect=[10_000_000, 10_600_000, 100_000_000, 100_700_000],
+ ):
+ with metrics.measure("duration"):
+ pass
+ self.assertEqual(metrics.to_dict(), {"duration": 0})
+ with metrics.measure("duration"):
+ pass
+ self.assertEqual(metrics.to_dict(), {"duration": 1})
+
+ def test_nested_timings_are_inclusive(self):
+ metrics = WorkerMetrics()
+ with patch(
+ "pyspark.worker_metrics.time.perf_counter_ns",
+ side_effect=[0, 1_000_000, 3_000_000, 5_000_000],
+ ):
+ with metrics.measure("outer"):
+ with metrics.measure("inner"):
+ pass
+ self.assertEqual(metrics.to_dict(), {"outer": 5, "inner": 2})
+
+ def test_reuse_timer_scope(self):
+ metrics = WorkerMetrics()
+ timer = metrics.measure("duration")
+ with patch(
+ "pyspark.worker_metrics.time.perf_counter_ns",
+ side_effect=[0, 2_000_000, 3_000_000, 4_000_000],
+ ):
+ with timer:
+ with self.assertRaisesRegex(RuntimeError, "already running"):
+ with timer:
+ pass
+ with timer:
+ pass
+ self.assertEqual(metrics.to_dict(), {"duration": 3})
+
+ def test_accumulate_and_propagate_exception(self):
+ metrics = WorkerMetrics()
+ with patch(
+ "pyspark.worker_metrics.time.perf_counter_ns",
+ side_effect=[0, 2_000_000, 3_000_000, 4_000_000],
+ ):
+ with self.assertRaisesRegex(ValueError, "worker failure"):
+ with metrics.measure("duration"):
+ raise ValueError("worker failure")
+ with metrics.measure("duration"):
+ pass
+ self.assertEqual(metrics.to_dict(), {"duration": 3})
class WorkerTests(ReusedPySparkTestCase):
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index c76dd0c08f93..f9e305707b23 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -24,7 +24,6 @@ import inspect
import json
import os
import sys
-import time
import warnings
from typing import BinaryIO
@@ -61,6 +60,7 @@ from pyspark.util import (
with_faulthandler,
)
from pyspark.worker_message import UDTFInfo, WorkerInitInfo
+from pyspark.worker_metrics import WorkerMetrics
from pyspark.worker_util import (
EvalConf,
RunnerConf,
@@ -75,21 +75,9 @@ from pyspark.worker_util import (
)
-def report_metrics(
- outfile, boot, init, finish, execution_duration_ms, memory_bytes_spilled,
disk_bytes_spilled
-):
+def report_metrics(outfile: BinaryIO, metrics: dict[str, int]) -> None:
"""Write the worker metrics as a length-prefixed JSON report after
METRICS_DATA."""
- payload = json.dumps(
- {
- "bootTimestampMs": int(1000 * boot),
- "initTimestampMs": int(1000 * init),
- "finishTimestampMs": int(1000 * finish),
- "pythonExecutionDurationMs": execution_duration_ms,
- "memoryBytesSpilled": memory_bytes_spilled,
- "diskBytesSpilled": disk_bytes_spilled,
- },
- separators=(",", ":"),
- ).encode("utf-8")
+ payload = json.dumps(metrics, separators=(",", ":")).encode("utf-8")
write_int(SpecialLengths.METRICS_DATA, outfile)
write_with_length(payload, outfile)
@@ -380,7 +368,9 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
Initialization -> Processing -> Finish/Cleanup
"""
try:
- boot_time = time.time()
+ metrics = WorkerMetrics()
+ metrics.reset()
+ metrics.set_current_timestamp("bootTimestampMs")
# Initialization
init_message = message_receiver.get_init_message()
init_info = WorkerInitInfo.from_stream(init_message)
@@ -436,7 +426,7 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
split_index = init_info.split_index
del init_info
- init_time = time.time()
+ metrics.set_current_timestamp("initTimestampMs")
# Processing
@@ -543,17 +533,17 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
)
is_pipelined = os.environ.get("SPARK_PIPELINED_UDF") == "1"
+ # Pipelined reads can overlap UDF work, so mark these task reports for
filtering.
+ metrics.set("pythonNumPipelinedTasks", int(is_pipelined))
if is_pipelined and hasattr(serializer, "_flush_per_batch"):
serializer._flush_per_batch = True
run_process = pipelined_process if is_pipelined else process
- execution_start_time = time.time()
- with capture_outputs():
+ with metrics.measure("pythonExecutionDurationMs"), capture_outputs():
if profiler:
profiler.profile(run_process)
else:
run_process()
- execution_duration_ms = int(1000 * (time.time() -
execution_start_time))
# Cleanup
# Reset task context to None. This is a guard code to avoid residual
context when worker
@@ -563,16 +553,11 @@ def invoke_udf(message_receiver: SparkMessageReceiver,
outfile: BinaryIO):
except BaseException as e:
handle_worker_exception(e, outfile)
sys.exit(-1)
- finish_time = time.time()
- report_metrics(
- outfile,
- boot_time,
- init_time,
- finish_time,
- execution_duration_ms,
- shuffle.MemoryBytesSpilled,
- shuffle.DiskBytesSpilled,
- )
+ metrics.set_current_timestamp("finishTimestampMs")
+ # Spill totals are byte counts already expressed in their reporting unit.
+ metrics.set("memoryBytesSpilled", shuffle.MemoryBytesSpilled)
+ metrics.set("diskBytesSpilled", shuffle.DiskBytesSpilled)
+ report_metrics(outfile, metrics.to_dict())
# Mark the beginning of the accumulators section of the output
write_int(SpecialLengths.END_OF_DATA_SECTION, outfile)
diff --git a/python/pyspark/worker_metrics.py b/python/pyspark/worker_metrics.py
new file mode 100644
index 000000000000..356e390bf74c
--- /dev/null
+++ b/python/pyspark/worker_metrics.py
@@ -0,0 +1,134 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""Shared metrics and reusable timers for a Python worker process."""
+
+import threading
+import time
+from typing import Any, ClassVar, Optional
+
+
+class _WorkerTimer:
+ """Measure one named duration and add each interval to the collector's
task total.
+
+ The collector owns the duration dictionary. This scope keeps a reference
to it and a
+ start time for its current with block. Nested scopes measure inclusive
time.
+
+ Repeatedly entering and exiting this context is costly: every interval
reads the
+ clock twice and updates the duration dictionary. In a hot batch loop,
accumulate
+ locally and call add_duration_ns once instead of entering this context per
batch.
+
+ Pipelined reads can overlap UDF work on another thread. An additive phase
breakdown
+ for that mode would require tracking those overlapping intervals.
+ """
+
+ def __init__(self, shared_duration_totals_ns: dict[str, int], metric_name:
str) -> None:
+ """Bind one metric name to the collector's shared duration totals."""
+ # Keep the collector's dictionary by reference so updates reach its
task totals.
+ self._shared_duration_totals_ns = shared_duration_totals_ns
+ self._metric_name = metric_name
+ self._block_start_ns: Optional[int] = None
+
+ def __enter__(self) -> None:
+ """Start the clock when Python enters the with block."""
+ # Re-entering this instance would overwrite an unfinished
measurement's start time.
+ if self._block_start_ns is not None:
+ raise RuntimeError("Worker timer is already running")
+ self._block_start_ns = time.perf_counter_ns()
+
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+ """Accumulate elapsed time when Python leaves the block, including on
an exception."""
+ assert self._block_start_ns is not None
+ elapsed_ns = time.perf_counter_ns() - self._block_start_ns
+ self._block_start_ns = None
+ self._shared_duration_totals_ns[self._metric_name] += elapsed_ns
+ # Python supplies the exception arguments; returning None lets the
error propagate.
+
+ def add_duration_ns(self, duration_ns: int) -> None:
+ """Flush a duration accumulated in local variables across a hot loop.
+
+ This updates the task-bound dictionary once when the loop ends, rather
than
+ calling a timer scope and updating the dictionary for every batch.
+ """
+ self._shared_duration_totals_ns[self._metric_name] += duration_ns
+
+
+class WorkerMetrics:
+ """One collector per Python worker process, reset by the worker before
each task.
+
+ set stores a supplied value; increment adds to a counter.
+ set_current_timestamp reads the clock and stores epoch milliseconds.
+ measure creates timer scopes that accumulate nanoseconds in a separate
shared dictionary.
+ to_dict combines both dictionaries, converting only durations to
milliseconds.
+ """
+
+ _instance: ClassVar[Optional["WorkerMetrics"]] = None
+ _instance_lock = threading.Lock()
+
+ def __new__(cls) -> "WorkerMetrics":
+ """Return the shared collector without resetting an active task."""
+ instance = cls._instance
+ if instance is None:
+ with cls._instance_lock:
+ instance = cls._instance
+ if instance is None:
+ instance = super().__new__(cls)
+ instance.reset()
+ cls._instance = instance
+ return instance
+
+ def reset(self) -> None:
+ """Start fresh task state while keeping the singleton's identity."""
+ # Counters, spill byte counts, and epoch timestamps need no conversion
at report time.
+ self._values_in_report_units: dict[str, int] = {}
+ # Existing timers keep the old dictionary. A fresh dictionary keeps
their
+ # measurements out of the next task's report.
+ self._duration_totals_ns: dict[str, int] = {}
+
+ def set_current_timestamp(self, name: str) -> None:
+ """Read the current clock and store its epoch milliseconds under
name."""
+ self.set(name, time.time_ns() // 1_000_000)
+
+ def set(self, name: str, value: int) -> None:
+ """Store the supplied value unchanged, replacing any earlier value."""
+ self._values_in_report_units[name] = value
+
+ def increment(self, name: str, value: int = 1) -> None:
+ """Add to a counter in its reporting unit, starting from zero.
+
+ The default increment is one.
+ """
+ self._values_in_report_units[name] =
self._values_in_report_units.get(name, 0) + value
+
+ def measure(self, name: str) -> _WorkerTimer:
+ """Create a task-bound duration handle for with or add_duration_ns."""
+ # Register zero so an empty supported task can still report this
timing.
+ self._duration_totals_ns.setdefault(name, 0)
+ return _WorkerTimer(self._duration_totals_ns, name)
+
+ def to_dict(self) -> dict[str, int]:
+ """Build the numeric report dictionary consumed by report_metrics.
+
+ Counters, bytes, and epoch timestamps already have their reporting
units. Only the
+ accumulated duration totals require conversion from nanoseconds to
milliseconds.
+ """
+ # Round once per task, preserving sub-millisecond contributions from
individual batches.
+ return {
+ **self._values_in_report_units,
+ **{name: duration // 1_000_000 for name, duration in
self._duration_totals_ns.items()},
+ }
diff --git a/python/pyspark/worker_util.py b/python/pyspark/worker_util.py
index f02f0cd8dc44..10507bef0fe1 100644
--- a/python/pyspark/worker_util.py
+++ b/python/pyspark/worker_util.py
@@ -319,6 +319,10 @@ class RunnerConf(Conf):
== "true"
)
+ @property
+ def detailed_timing_enabled(self) -> bool:
+ return
self.get("spark.sql.execution.pythonUDF.detailedTiming.enabled", "false") ==
"true"
+
@property
def use_legacy_pandas_udtf_conversion(self) -> bool:
return (
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cb60d8905455..3b9d0d1f6ae0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5801,6 +5801,15 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
+ val PYTHON_UDF_DETAILED_TIMING_ENABLED =
+ buildConf("spark.sql.execution.pythonUDF.detailedTiming.enabled")
+ .internal()
+ .doc("When true, collect detailed timing metrics for supported Python
UDFs.")
+ .version("4.4.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .booleanConf
+ .createWithDefault(false)
+
val PYTHON_UDF_ARROW_ENABLED =
buildConf("spark.sql.execution.pythonUDF.arrow.enabled")
.doc("Enable Arrow optimization in regular Python UDFs. This
optimization " +
@@ -10001,6 +10010,9 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def pythonWorkerLoggingEnabled: Boolean =
getConf(PYTHON_WORKER_LOGGING_ENABLED)
+ def pythonUDFDetailedTimingEnabled: Boolean =
+ getConf(PYTHON_UDF_DETAILED_TIMING_ENABLED)
+
def pythonUDFArrowConcurrencyLevel: Option[Int] =
getConf(PYTHON_UDF_ARROW_CONCURRENCY_LEVEL)
def pythonUDFArrowFallbackOnUDT: Boolean =
getConf(PYTHON_UDF_ARROW_FALLBACK_ON_UDT)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
index 6e65dc032459..ddfa84466272 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.execution.python.EvalPythonExec.ArgumentMetadata
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{StructType, UserDefinedType}
import org.apache.spark.sql.types.DataType.equalsIgnoreCompatibleCollation
import org.apache.spark.sql.vectorized.ColumnarBatch
@@ -89,6 +90,21 @@ case class ArrowEvalPythonExec(
throw SparkException.internalError(s"Unexpected eval type $evalType")
}
+ override protected def additionalMetrics: Map[String, SQLMetric] = {
+ if (evalType == PythonEvalType.SQL_ARROW_BATCHED_UDF &&
+ !conf.legacyPandasConversionUDF &&
conf.pythonUDFDetailedTimingEnabled) {
+ createPythonPhaseMetrics()
+ } else {
+ Map.empty
+ }
+ }
+
+ // SQL metrics are fixed with the plan; use that decision if the session
setting changes later.
+ private def pythonRunnerConfMap: Map[String, String] =
+ ArrowPythonRunner.getPythonRunnerConfMap(conf).updated(
+ SQLConf.PYTHON_UDF_DETAILED_TIMING_ENABLED.key,
+ pythonMetrics.contains("pythonNumTimingReports").toString)
+
private[this] val jobArtifactUUID =
JobArtifactSet.getCurrentJobArtifactState.map(_.uuid)
private[this] val sessionUUID = {
Option(session).collect {
@@ -145,7 +161,7 @@ case class ArrowEvalPythonExec(
evalType,
conf.sessionLocalTimeZone,
conf.arrowUseLargeVarTypes,
- ArrowPythonRunner.getPythonRunnerConfMap(conf),
+ pythonRunnerConfMap,
pythonMetrics,
jobArtifactUUID,
sessionUUID)
@@ -159,7 +175,7 @@ case class ArrowEvalPythonExec(
evalType,
conf.sessionLocalTimeZone,
conf.arrowUseLargeVarTypes,
- ArrowPythonRunner.getPythonRunnerConfMap(conf),
+ pythonRunnerConfMap,
pythonMetrics,
jobArtifactUUID,
sessionUUID)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index dedd8b5cbdf6..2bc5f4648ea6 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -187,6 +187,7 @@ object ArrowPythonRunner {
SQLConf.ARROW_EXECUTION_USE_LARGE_VAR_TYPES,
SQLConf.PYTHON_TABLE_UDF_LEGACY_PANDAS_CONVERSION_ENABLED,
SQLConf.PYTHON_UDF_LEGACY_PANDAS_CONVERSION_ENABLED,
+ SQLConf.PYTHON_UDF_DETAILED_TIMING_ENABLED,
SQLConf.PYTHON_UDF_MAP_IN_BATCH_LEGACY_ACCEPT_ANY_ITERABLE_ENABLED,
SQLConf.PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED,
SQLConf.PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE,
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
index 3f3bb8df2a01..0d3c78e424b6 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
@@ -42,6 +42,14 @@ trait PythonSQLMetrics { self: SparkPlan =>
// empty so the shared trait stays minimal; see [[PythonPickleBatchMetrics]].
protected def additionalMetrics: Map[String, SQLMetric] = Map.empty
+ protected def createPythonPhaseMetrics(): Map[String, SQLMetric] = {
+ PythonSQLMetrics.pythonPhaseTimingMetricsDesc.map { case (k, v) =>
+ k -> SQLMetrics.createTimingMetric(sparkContext, v)
+ } ++ PythonSQLMetrics.pythonPhaseCountMetricsDesc.map { case (k, v) =>
+ k -> SQLMetrics.createMetric(sparkContext, v)
+ }
+ }
+
override lazy val metrics: Map[String, SQLMetric] = pythonMetrics
}
@@ -63,7 +71,24 @@ object PythonSQLMetrics {
}
val pythonOtherMetricsDesc: Map[String, String] = {
- Map("pythonNumRowsReceived" -> "number of output rows")
+ Map(
+ "pythonNumRowsReceived" -> "number of output rows",
+ "pythonNumPipelinedTasks" -> "number of pipelined Python worker tasks")
+ }
+
+ val pythonPhaseTimingMetricsDesc: Map[String, String] = {
+ Map(
+ "pythonDataReadTime" -> "time to read and deserialize Arrow data",
+ "pythonInputPreparationTime" -> "time to prepare Python UDF input",
+ "pythonUDFExecutionTime" -> "time to evaluate Python UDFs",
+ "pythonOutputPreparationTime" -> "time to prepare Python UDF output",
+ "pythonDataWriteTime" -> "time to serialize and write Arrow data")
+ }
+
+ val pythonPhaseCountMetricsDesc: Map[String, String] = {
+ Map(
+ "pythonNumTimingReports" -> "number of Python tasks with detailed
timing",
+ "pythonNumTimedBatches" -> "number of Python batches with detailed
timing")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]