This is an automated email from the ASF dual-hosted git repository.

gaogaotiantian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 9614eaacdceb [SPARK-59993][PYTHON] Collect worker and Arrow-optimized 
Python UDF timing metrics
9614eaacdceb is described below

commit 9614eaacdceb3cd79494b67f0ed29556e86d5989
Author: Spenser Sun <[email protected]>
AuthorDate: Wed Oct 21 20:19:50 2026 -0700

    [SPARK-59993][PYTHON] Collect worker and Arrow-optimized Python UDF timing 
metrics
    
    ### What changes were proposed in this pull request?
    
    - Reset `WorkerMetrics` per task and decode its numeric report into SQL 
metrics.
    - For Arrow scalar UDFs, measure five phases: read/deserialization, input 
preparation, UDF execution, output preparation, and serialization/write. Count 
timed tasks, batches, and pipelined tasks.
    - A default-off config gates timing. The Arrow handler measures preparation 
and UDF work; one serializer path measures Arrow read/write operations.
    
    ### Why are the changes needed?
    
    - `pythonProcessingTime` spans the worker loop, hiding where time is spent.
    
    ### Does this PR introduce _any_ user-facing change?
    
    - Yes. Enabling the config adds five durations and two counts to Arrow UDF 
SQL metrics. Read/write time includes Arrow IPC and buffered stream calls.
    - The start marker counts toward write time; the worker's final output 
flush is outside these intervals.
    
    ### How was this patch tested?
    
    - On the rebased head, 44 Arrow handler tests (plus four subtests), seven 
WorkerMetrics tests, Ruff, format, and mypy passed. Earlier serializer, worker 
protocol, and live gate-on Spark checks passed before the conflict-only rebase; 
local `sql/compile` was blocked by missing Maven BOMs.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-6).
    
    Closes #59246 from Spenserrrr/udf-metrics-collection.
    
    Authored-by: Spenser Sun <[email protected]>
    Signed-off-by: Tian Gao <[email protected]>
---
 .../org/apache/spark/api/python/PythonRunner.scala |  69 +++--
 .../spark/api/python/BasePythonRunnerSuite.scala   |  50 ++--
 python/pyspark/eval_handlers/_arrow.py             | 133 ++++++---
 .../tests/test_arrow_eval_type_handlers.py         | 115 ++++++++
 python/pyspark/sql/pandas/serializers.py           |  65 ++++-
 .../sql/tests/arrow/test_arrow_python_udf.py       | 306 +++++++++++++++++++++
 .../pyspark/sql/tests/pandas/test_pipelined_udf.py |   9 +-
 python/pyspark/tests/test_worker.py                |  92 +++++++
 python/pyspark/worker.py                           |  45 +--
 python/pyspark/worker_metrics.py                   | 134 +++++++++
 python/pyspark/worker_util.py                      |   4 +
 .../org/apache/spark/sql/internal/SQLConf.scala    |  12 +
 .../sql/execution/python/ArrowEvalPythonExec.scala |  20 +-
 .../sql/execution/python/ArrowPythonRunner.scala   |   1 +
 .../sql/execution/python/PythonSQLMetrics.scala    |  27 +-
 15 files changed, 943 insertions(+), 139 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala 
b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index b7fc25432971..0be49a5f71f0 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -477,21 +477,12 @@ private[spark] object BasePythonRunner extends Logging {
     }
   }
 
-  // Values used to update the existing Python SQL metrics and task spill 
metrics.
-  private[python] case class WorkerMetrics(
-      bootTimestampMs: Long,
-      initTimestampMs: Long,
-      finishTimestampMs: Long,
-      pythonExecutionDurationMs: Long,
-      memoryBytesSpilled: Long,
-      diskBytesSpilled: Long)
-
   private lazy val workerMetricsMapper = new ObjectMapper()
     .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
     .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
 
-  /** Read and validate worker metrics after METRICS_DATA; ignore additional 
fields. */
-  private[python] def readWorkerMetrics(stream: DataInputStream): 
WorkerMetrics = {
+  /** Decode every METRICS_DATA field as a named int64; reject other values. */
+  private[python] def readWorkerMetrics(stream: DataInputStream): Map[String, 
Long] = {
     val length = stream.readInt()
     if (length <= 0) {
       throw new SparkException(s"Invalid Python worker report length: $length")
@@ -507,21 +498,13 @@ private[spark] object BasePythonRunner extends Logging {
       throw new SparkException("Expected a Python worker JSON object")
     }
 
-    def metricValue(name: String): Long = {
-      val value = report.get(name)
-      if (value == null || !value.isIntegralNumber || !value.canConvertToLong) 
{
-        throw new SparkException(s"Missing or invalid Python worker metric: 
$name")
+    report.fields().asScala.map { entry =>
+      val value = entry.getValue
+      if (!value.isIntegralNumber || !value.canConvertToLong) {
+        throw new SparkException(s"Invalid Python worker metric: 
${entry.getKey}")
       }
-      value.longValue()
-    }
-
-    WorkerMetrics(
-      metricValue("bootTimestampMs"),
-      metricValue("initTimestampMs"),
-      metricValue("finishTimestampMs"),
-      metricValue("pythonExecutionDurationMs"),
-      metricValue("memoryBytesSpilled"),
-      metricValue("diskBytesSpilled"))
+      entry.getKey -> value.longValue()
+    }.toMap
   }
 
   /**
@@ -1155,17 +1138,32 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
 
     protected def handleMetricsData(): Unit = {
       val workerMetrics = BasePythonRunner.readWorkerMetrics(stream)
-      val bootTime = workerMetrics.bootTimestampMs
-      val initTime = workerMetrics.initTimestampMs
-      val finishTime = workerMetrics.finishTimestampMs
-      val pythonExecutionDurationMs = workerMetrics.pythonExecutionDurationMs
+      def requiredMetric(name: String): Long = workerMetrics.getOrElse(name,
+        throw new SparkException(s"Missing Python worker metric: $name"))
+
+      // Read required inputs before updating any SQL or task metrics.
+      val bootTime = requiredMetric("bootTimestampMs")
+      val initTime = requiredMetric("initTimestampMs")
+      val finishTime = requiredMetric("finishTimestampMs")
+      val pythonExecutionDurationMs = 
requiredMetric("pythonExecutionDurationMs")
+      val memoryBytesSpilled = requiredMetric("memoryBytesSpilled")
+      val diskBytesSpilled = requiredMetric("diskBytesSpilled")
+
       // A reused Python worker records bootTime before waiting for this task, 
so it can precede
       // startTime. Use the later timestamp to exclude the worker's idle time 
from initialization.
       val pythonWorkerInitializationStartTime = math.max(startTime, bootTime)
       val boot = pythonWorkerInitializationStartTime - startTime
       val init = initTime - pythonWorkerInitializationStartTime
-      val finish = finishTime - initTime
       val total = finishTime - startTime
+      val finish = finishTime - initTime
+      val calculatedMetrics = Map(
+        "pythonBootTime" -> boot,
+        "pythonInitTime" -> init,
+        "pythonTotalTime" -> total,
+        "pythonProcessingTime" -> pythonExecutionDurationMs)
+      // Select registered report values; calculated timings take precedence 
over raw aliases.
+      val sqlMetrics = workerMetrics.filter { case (name, _) => 
metrics.contains(name) } ++
+        calculatedMetrics
 
       // Format data size for readability
       val dataKB = totalDataReceived / 1024.0
@@ -1181,12 +1179,11 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
         log"finish = ${MDC(TIME, finish)} - " +
         log"Batches: ${MDC(COUNT, batchesProcessed)}, Data: ${MDC(SIZE, 
dataStr)} - " +
         log"${MDC(TASK_NAME, taskIdentifier(context))}")
-      metrics.get("pythonBootTime").foreach(_.add(boot))
-      metrics.get("pythonInitTime").foreach(_.add(init))
-      metrics.get("pythonTotalTime").foreach(_.add(total))
-      
metrics.get("pythonProcessingTime").foreach(_.add(pythonExecutionDurationMs))
-      
context.taskMetrics().incMemoryBytesSpilled(workerMetrics.memoryBytesSpilled)
-      context.taskMetrics().incDiskBytesSpilled(workerMetrics.diskBytesSpilled)
+      sqlMetrics.foreach { case (name, value) =>
+        metrics.get(name).foreach(_.add(value))
+      }
+      context.taskMetrics().incMemoryBytesSpilled(memoryBytesSpilled)
+      context.taskMetrics().incDiskBytesSpilled(diskBytesSpilled)
     }
 
     protected def handlePythonException(): PythonException = {
diff --git 
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala 
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
index 1ea85e2910bb..f4a9d11e967c 100644
--- 
a/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/api/python/BasePythonRunnerSuite.scala
@@ -720,7 +720,13 @@ class BasePythonRunnerSuite extends SparkFunSuite with 
SharedSparkContext {
     "{\"bootTimestampMs\":1250,\"initTimestampMs\":2500," +
       "\"finishTimestampMs\":3750,\"pythonExecutionDurationMs\":42," +
       "\"memoryBytesSpilled\":7,\"diskBytesSpilled\":9}"
-  private val expectedMetrics = BasePythonRunner.WorkerMetrics(1250L, 2500L, 
3750L, 42L, 7L, 9L)
+  private val expectedMetrics = Map(
+    "bootTimestampMs" -> 1250L,
+    "initTimestampMs" -> 2500L,
+    "finishTimestampMs" -> 3750L,
+    "pythonExecutionDurationMs" -> 42L,
+    "memoryBytesSpilled" -> 7L,
+    "diskBytesSpilled" -> 9L)
 
   private def framedMetricsStream(json: String, declaredLength: Option[Int] = 
None,
       endMarkers: Boolean = true): DataInputStream = {
@@ -745,41 +751,32 @@ class BasePythonRunnerSuite extends SparkFunSuite with 
SharedSparkContext {
   }
 
   test("SPARK-59773: read worker metrics and leave the following sections 
aligned") {
-    val withExtra = metricsReport.dropRight(1) + 
",\"futureMetric\":{\"value\":[1,2]}}"
+    val withExtra = metricsReport.dropRight(1) + 
",\"futureMetric\":17,\"zeroMetric\":0}"
     val stream = framedMetricsStream(withExtra)
-    assert(BasePythonRunner.readWorkerMetrics(stream) == expectedMetrics)
+    val result = BasePythonRunner.readWorkerMetrics(stream)
+    assert(result == expectedMetrics ++ Map("futureMetric" -> 17L, 
"zeroMetric" -> 0L))
     checkEndMarkers(stream)
   }
 
-  test("SPARK-59773: worker metrics require all current fields") {
-    val fields = Seq(
-      "bootTimestampMs" -> "\"bootTimestampMs\":1250,",
-      "initTimestampMs" -> "\"initTimestampMs\":2500,",
-      "finishTimestampMs" -> "\"finishTimestampMs\":3750,",
-      "pythonExecutionDurationMs" -> "\"pythonExecutionDurationMs\":42,",
-      "memoryBytesSpilled" -> "\"memoryBytesSpilled\":7,",
-      "diskBytesSpilled" -> ",\"diskBytesSpilled\":9")
-    fields.foreach { case (name, field) =>
-      val stream = framedMetricsStream(metricsReport.replace(field, ""))
-      val error = intercept[SparkException] {
-        BasePythonRunner.readWorkerMetrics(stream)
-      }
-      assert(error.getMessage.contains(name))
-      checkEndMarkers(stream)
-    }
+  test("SPARK-59993: decode worker metric names independently of lifecycle and 
SQL consumers") {
+    val stream = framedMetricsStream("{\"futureDuration\":42}")
+    val result = BasePythonRunner.readWorkerMetrics(stream)
+    assert(result == Map("futureDuration" -> 42L))
+    checkEndMarkers(stream)
   }
 
-  test("SPARK-59773: worker metrics reject non-int64 values") {
+  test("SPARK-59773: worker metric decoder rejects non-int64 values") {
     val invalidValues = Seq("null", "true", "\"42\"", "42.0", "[]", "{}",
       "9223372036854775808", "-9223372036854775809")
+    val reportWithExtra = metricsReport.dropRight(1) + ",\"futureMetric\":17}"
     val fields = Seq("pythonExecutionDurationMs" -> 42,
-      "memoryBytesSpilled" -> 7, "diskBytesSpilled" -> 9)
+      "memoryBytesSpilled" -> 7, "diskBytesSpilled" -> 9, "futureMetric" -> 17)
     for {
       (name, originalValue) <- fields
       value <- invalidValues
     } {
       val field = "\"" + name + "\":"
-      val json = metricsReport.replace(field + originalValue, field + value)
+      val json = reportWithExtra.replace(field + originalValue, field + value)
       val stream = framedMetricsStream(json)
       val error = intercept[SparkException] {
         BasePythonRunner.readWorkerMetrics(stream)
@@ -797,8 +794,8 @@ class BasePythonRunnerSuite extends SparkFunSuite with 
SharedSparkContext {
         .replace("\"diskBytesSpilled\":9", "\"diskBytesSpilled\":" + value)
       val stream = framedMetricsStream(json)
       assert(BasePythonRunner.readWorkerMetrics(stream) ==
-        expectedMetrics.copy(pythonExecutionDurationMs = value,
-          memoryBytesSpilled = value, diskBytesSpilled = value))
+        expectedMetrics ++ Map("pythonExecutionDurationMs" -> value,
+          "memoryBytesSpilled" -> value, "diskBytesSpilled" -> value))
       checkEndMarkers(stream)
     }
   }
@@ -830,9 +827,10 @@ class BasePythonRunnerSuite extends SparkFunSuite with 
SharedSparkContext {
   }
 
   test("SPARK-59773: worker metrics accept a larger report with an additional 
field") {
-    val json = metricsReport.dropRight(1) + ",\"future\":\"" + ("x" * (70 * 
1024)) + "\"}"
+    val json = metricsReport.dropRight(1) + ",\"futureMetric\":17" + (" " * 
(70 * 1024)) + "}"
     val stream = framedMetricsStream(json)
-    assert(BasePythonRunner.readWorkerMetrics(stream) == expectedMetrics)
+    val result = BasePythonRunner.readWorkerMetrics(stream)
+    assert(result == expectedMetrics + ("futureMetric" -> 17L))
     checkEndMarkers(stream)
   }
 }
diff --git a/python/pyspark/eval_handlers/_arrow.py 
b/python/pyspark/eval_handlers/_arrow.py
index 2c9acf409235..217135b9577d 100644
--- a/python/pyspark/eval_handlers/_arrow.py
+++ b/python/pyspark/eval_handlers/_arrow.py
@@ -28,6 +28,7 @@ unregistered.
 from __future__ import annotations
 
 import itertools
+import time
 from collections.abc import Iterator
 from typing import TYPE_CHECKING, Any
 
@@ -56,10 +57,12 @@ from pyspark.sql.conversion import (
     ArrowTableToRowsConversion,
     LocalDataToArrowConversion,
 )
+from pyspark.sql.pandas.serializers import ArrowStreamSerializer
 from pyspark.sql.pandas.types import to_arrow_schema, to_arrow_type
 from pyspark.sql.pandas.utils import require_minimum_pyarrow_version
 from pyspark.sql.types import StructField, StructType
 from pyspark.util import PythonEvalType, fail_on_stopiteration
+from pyspark.worker_metrics import WorkerMetrics
 
 if TYPE_CHECKING:
     import pyarrow as pa
@@ -855,6 +858,7 @@ class 
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
                     ),
                 )
             )
+        assert self._udf_infos, "At least one Arrow UDF expected here."
         self._col_names = [f"_{i}" for i in range(len(udfs))]
 
         # --- Input preparation ---
@@ -865,10 +869,28 @@ class 
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
             for f in eval_conf.input_type  # type: ignore[union-attr]
         ]
 
+    @property
+    def serializer(self) -> ArrowStreamSerializer:
+        return ArrowStreamSerializer(
+            write_start_stream=True,
+            collect_timing=self._runner_conf.detailed_timing_enabled,
+        )
+
     def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> 
Iterator[pa.RecordBatch]:
         import pyarrow as pa
 
         runner_conf = self._runner_conf
+        metrics = WorkerMetrics()
+        if runner_conf.detailed_timing_enabled:
+            phase_timers = (
+                metrics.measure("pythonInputPreparationTime"),
+                metrics.measure("pythonUDFExecutionTime"),
+                metrics.measure("pythonOutputPreparationTime"),
+            )
+            metrics.set("pythonNumTimingReports", 1)
+            metrics.set("pythonNumTimedBatches", 0)
+        else:
+            phase_timers = None
 
         @fail_on_stopiteration
         def _evaluate_batch_udf(udf_func: Any, rows: list[tuple]) -> list[Any]:
@@ -879,42 +901,77 @@ class 
ArrowBatchedUDFArrowHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
             with 
ThreadPoolExecutor(max_workers=runner_conf.arrow_concurrency_level) as pool:
                 return list(pool.map(lambda row: udf_func(*row), rows))
 
-        for input_batch in data:
-            num_rows = input_batch.num_rows
-
-            # --- Input: Arrow -> Python columns ---
-            columns = [
-                (
-                    [conv(v) for v in 
ArrowTableToRowsConversion._to_pylist(col)]
-                    if conv is not None
-                    else ArrowTableToRowsConversion._to_pylist(col)
-                )
-                for col, conv in zip(input_batch.itercolumns(), 
self._arrow_to_py_converters)
-            ]
-            if not columns:
-                columns = [[_NoValue] * num_rows]
-
-            # --- Process: evaluate each UDF row-by-row ---
-            output_arrays = []
-            for udf_func, offsets, zero_arg, arrow_return_type, result_conv in 
self._udf_infos:
-                rows = (
-                    [() for _ in range(num_rows)]
-                    if zero_arg
-                    else list(zip(*[columns[o] for o in offsets]))
-                )
-                results = _evaluate_batch_udf(udf_func, rows)
-                verify_result_row_count(len(results), num_rows)
-
-                # --- Output: Python -> Arrow ---
-                converted = (
-                    [result_conv(r) for r in results] if result_conv is not 
None else results
-                )
-                try:
-                    arr = pa.array(converted, type=arrow_return_type)
-                except pa.lib.ArrowInvalid:
-                    arr = pa.array(converted).cast(
-                        target_type=arrow_return_type, 
safe=runner_conf.safecheck
+        # Keep phase totals and the batch count local, then publish them once 
per task.
+        input_duration_ns = 0
+        udf_duration_ns = 0
+        output_duration_ns = 0
+        num_timed_batches = 0
+        clock = time.perf_counter_ns
+        num_udfs = len(self._udf_infos)
+        try:
+            for input_batch in data:
+                if phase_timers is not None:
+                    num_timed_batches += 1
+                    input_start_ns = clock()
+                num_rows = input_batch.num_rows
+
+                # --- Input: Arrow -> Python columns ---
+                columns = [
+                    (
+                        [conv(v) for v in 
ArrowTableToRowsConversion._to_pylist(col)]
+                        if conv is not None
+                        else ArrowTableToRowsConversion._to_pylist(col)
                     )
-                output_arrays.append(arr)
-
-            yield pa.RecordBatch.from_arrays(output_arrays, self._col_names)
+                    for col, conv in zip(input_batch.itercolumns(), 
self._arrow_to_py_converters)
+                ]
+                if not columns:
+                    columns = [[_NoValue] * num_rows]
+
+                # --- Process: evaluate each UDF row-by-row ---
+                output_arrays: list[pa.Array] = []
+                for udf_func, offsets, zero_arg, arrow_return_type, 
result_conv in self._udf_infos:
+                    if phase_timers is not None and output_arrays:
+                        # Reuse the batch start for the first UDF, avoiding 
another clock read.
+                        # Later UDFs start input after the loop transition.
+                        input_start_ns = clock()
+                    rows = (
+                        [() for _ in range(num_rows)]
+                        if zero_arg
+                        else list(zip(*[columns[o] for o in offsets]))
+                    )
+                    if phase_timers is not None:
+                        udf_start_ns = clock()
+                        input_duration_ns += udf_start_ns - input_start_ns
+                    results = _evaluate_batch_udf(udf_func, rows)
+                    if phase_timers is not None:
+                        output_start_ns = clock()
+                        udf_duration_ns += output_start_ns - udf_start_ns
+                    verify_result_row_count(len(results), num_rows)
+
+                    # --- Output: Python -> Arrow ---
+                    converted = (
+                        [result_conv(r) for r in results] if result_conv is 
not None else results
+                    )
+                    try:
+                        arr = pa.array(converted, type=arrow_return_type)
+                    except pa.lib.ArrowInvalid:
+                        arr = pa.array(converted).cast(
+                            target_type=arrow_return_type, 
safe=runner_conf.safecheck
+                        )
+                    output_arrays.append(arr)
+                    # All UDF results for this batch are ready; time its 
assembly as output.
+                    if len(output_arrays) == num_udfs:
+                        output_batch = 
pa.RecordBatch.from_arrays(output_arrays, self._col_names)
+                    if phase_timers is not None:
+                        output_duration_ns += clock() - output_start_ns
+
+                yield output_batch
+                del output_batch
+        finally:
+            # Publish the count and completed intervals if the generator 
closes early or raises.
+            if phase_timers is not None:
+                metrics.set("pythonNumTimedBatches", num_timed_batches)
+                input_timer, udf_timer, output_timer = phase_timers
+                input_timer.add_duration_ns(input_duration_ns)
+                udf_timer.add_duration_ns(udf_duration_ns)
+                output_timer.add_duration_ns(output_duration_ns)
diff --git 
a/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py 
b/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
index 490f0cac683e..b62283a20c0f 100644
--- a/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
+++ b/python/pyspark/eval_handlers/tests/test_arrow_eval_type_handlers.py
@@ -24,6 +24,7 @@ same way: ``run(0, <input>)`` and assert on the output 
batches.
 
 import os
 import unittest
+from io import BytesIO
 from unittest.mock import patch
 
 from pyspark.errors import PySparkRuntimeError
@@ -33,6 +34,7 @@ from pyspark.sql.pandas.serializers import 
ArrowStreamCoGroupSerializer, ArrowSt
 from pyspark.sql.types import DoubleType, LongType, StringType, StructField, 
StructType
 from pyspark.testing.utils import PySparkBaseTestCase, have_pyarrow, 
pyarrow_requirement_message
 from pyspark.util import PythonEvalType
+from pyspark.worker_metrics import WorkerMetrics
 
 with patch.dict(os.environ, {"SPARK_PYTHON_RUNTIME": "PYTHON_WORKER"}):
     from pyspark.worker_util import EvalConf, RunnerConf
@@ -201,6 +203,119 @@ class 
ArrowBatchedUDFArrowHandlerTests(ArrowEvalTypeHandlerTestCase):
         handler = self._handler((lambda a: a, [0], {}, LongType()))
         self.assertIsInstance(handler.serializer, ArrowStreamSerializer)
 
+    def test_detailed_timing_uses_handler_gate(self):
+        metrics = WorkerMetrics()
+        self.addCleanup(metrics.reset)
+        timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+        for enabled in (False, True):
+            with self.subTest(enabled=enabled):
+                metrics.reset()
+                handler = self._handler(
+                    (lambda a: a + 1, [0], {}, LongType()),
+                    (lambda a, b: a * b, [0, 1], {}, LongType()),
+                    runner_conf=RunnerConf({timing_conf: 
str(enabled).lower()}),
+                )
+                self.assertEqual(handler.serializer._collect_timing, enabled)
+                output = list(
+                    handler.run(
+                        0,
+                        iter([self._batch(a=[1, 2], b=[10, 20]), 
self._batch(a=[3], b=[4])]),
+                    )
+                )
+                self.assertEqual([batch.num_rows for batch in output], [2, 1])
+                report = metrics.to_dict()
+                if enabled:
+                    self.assertEqual(report["pythonNumTimingReports"], 1)
+                    self.assertEqual(report["pythonNumTimedBatches"], 2)
+                    for name in (
+                        "pythonInputPreparationTime",
+                        "pythonUDFExecutionTime",
+                        "pythonOutputPreparationTime",
+                    ):
+                        self.assertIn(name, report)
+                else:
+                    self.assertNotIn("pythonNumTimingReports", report)
+                    self.assertNotIn("pythonInputPreparationTime", report)
+
+    def test_phase_timing_boundaries(self):
+        metrics = WorkerMetrics()
+        metrics.reset()
+        self.addCleanup(metrics.reset)
+        clock_ns = 0
+        udf_rows = 0
+        input_ns = 0
+        output_ns = 0
+
+        class TimedBatch:
+            def __init__(self, batch):
+                self.batch = batch
+
+            @property
+            def num_rows(self):
+                return self.batch.num_rows
+
+            def itercolumns(self):
+                nonlocal clock_ns, input_ns
+                clock_ns += 3_000_000
+                input_ns += 3_000_000
+                return self.batch.itercolumns()
+
+        class TimedInt:
+            def __init__(self, value):
+                self.value = value
+
+            def __index__(self):
+                nonlocal clock_ns, output_ns
+                clock_ns += 5_000_000
+                output_ns += 5_000_000
+                return self.value
+
+        def udf(value):
+            nonlocal clock_ns, udf_rows
+            clock_ns += 100_000_000
+            udf_rows += 1
+            return TimedInt(value + 1)
+
+        class OutputStream(BytesIO):
+            write_ns = 0
+            rows_at_write = None
+
+            def write(self, data):
+                nonlocal clock_ns
+                if self.rows_at_write is None:
+                    self.rows_at_write = udf_rows
+                clock_ns += 5_000_000
+                self.write_ns += 5_000_000
+                return super().write(data)
+
+        handler = self._handler(
+            (udf, [0], {}, LongType()),
+            runner_conf=RunnerConf(
+                {"spark.sql.execution.pythonUDF.detailedTiming.enabled": 
"true"}
+            ),
+        )
+        output = OutputStream()
+        batches = [self._batch(a=[1, 2], b=[10, 20]), self._batch(a=[3], 
b=[30])]
+
+        def input_batches():
+            nonlocal clock_ns
+            for batch in batches:
+                clock_ns += 100_000_000
+                yield TimedBatch(batch)
+
+        with patch("pyspark.eval_handlers._arrow.time.perf_counter_ns", 
new=lambda: clock_ns):
+            handler.serializer.dump_stream(handler.run(0, input_batches()), 
output)
+
+        report = metrics.to_dict()
+        self.assertEqual(output.rows_at_write, 2)
+        self.assertEqual(report["pythonNumTimedBatches"], 2)
+        self.assertGreater(input_ns, 0)
+        self.assertEqual(report["pythonInputPreparationTime"], input_ns // 
1_000_000)
+        self.assertEqual(report["pythonUDFExecutionTime"], 300)
+        self.assertGreater(output_ns, 0)
+        self.assertEqual(report["pythonOutputPreparationTime"], output_ns // 
1_000_000)
+        self.assertEqual(report["pythonDataWriteTime"], output.write_ns // 
1_000_000)
+
     def test_invokes_udf_per_row(self):
         handler = self._handler(
             (lambda a: a + 1, [0], {}, LongType()),
diff --git a/python/pyspark/sql/pandas/serializers.py 
b/python/pyspark/sql/pandas/serializers.py
index b829e4a8636f..0ddd17e55015 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -19,6 +19,7 @@
 Serializers for PyArrow and pandas conversions. See `pyspark.serializers` for 
more details.
 """
 
+import time
 from typing import IO, TYPE_CHECKING, Iterable, Iterator, List, Tuple
 
 from pyspark.errors import PySparkRuntimeError, PySparkValueError
@@ -28,6 +29,7 @@ from pyspark.serializers import (
     read_int,
     write_int,
 )
+from pyspark.worker_metrics import WorkerMetrics
 
 if TYPE_CHECKING:
     import pyarrow as pa
@@ -51,12 +53,27 @@ class ArrowStreamSerializer(Serializer):
     write_start_stream : bool
         If True, writes the START_ARROW_STREAM marker before the first
         output batch. Default False.
+    collect_timing : bool
+        Record combined Arrow IPC and stream I/O time in worker metrics.
+        Enable only for UDF paths that report the corresponding SQL metrics.
+
+    Read timing covers opening the stream and fetching batches, including end 
of stream.
+    Write timing covers the start marker, writer setup, batch writes, existing 
per-batch
+    flushes, and close. Producer/consumer work between batches, other task 
protocol, and
+    the worker owner's final flush are outside these intervals.
     """
 
-    def __init__(self, write_start_stream: bool = False, flush_per_batch: bool 
= False) -> None:
+    def __init__(
+        self,
+        write_start_stream: bool = False,
+        flush_per_batch: bool = False,
+        *,
+        collect_timing: bool = False,
+    ) -> None:
         super().__init__()
         self._write_start_stream: bool = write_start_stream
         self._flush_per_batch: bool = flush_per_batch
+        self._collect_timing = collect_timing
 
     def dump_stream(self, iterator: Iterable["pa.RecordBatch"], stream: 
IO[bytes]) -> None:
         """Optionally prepend START_ARROW_STREAM, then write batches."""
@@ -65,9 +82,17 @@ class ArrowStreamSerializer(Serializer):
             iterator = self._write_stream_start(iterator, stream)
         import pyarrow as pa
 
+        write_timer = (
+            WorkerMetrics().measure("pythonDataWriteTime") if 
self._collect_timing else None
+        )
+        clock = time.perf_counter_ns
+        write_duration_ns = 0
         writer = None
         try:
             for batch in iterator:
+                # The batch is already produced; time Arrow serialization and 
writing below.
+                if write_timer is not None:
+                    start_ns = clock()
                 if writer is None:
                     writer = pa.RecordBatchStreamWriter(stream, batch.schema)
                 writer.write_batch(batch)
@@ -75,17 +100,44 @@ class ArrowStreamSerializer(Serializer):
                 # while still sending input, rather than buffering all output.
                 if self._flush_per_batch:
                     stream.flush()
+                if write_timer is not None:
+                    write_duration_ns += clock() - start_ns
         finally:
             if writer is not None:
+                if write_timer is not None:
+                    start_ns = clock()
                 writer.close()
+                if write_timer is not None:
+                    write_duration_ns += clock() - start_ns
+            if write_timer is not None:
+                write_timer.add_duration_ns(write_duration_ns)
 
     def load_stream(self, stream: IO[bytes]) -> Iterator["pa.RecordBatch"]:
         """Load batches from a plain Arrow stream."""
         import pyarrow as pa
 
-        reader = pa.ipc.open_stream(stream)
-        for batch in reader:
-            yield batch
+        read_timer = WorkerMetrics().measure("pythonDataReadTime") if 
self._collect_timing else None
+        clock = time.perf_counter_ns
+        read_duration_ns = 0
+        try:
+            if read_timer is not None:
+                start_ns = clock()
+            reader = pa.ipc.open_stream(stream)
+            if read_timer is not None:
+                read_duration_ns += clock() - start_ns
+            while True:
+                if read_timer is not None:
+                    start_ns = clock()
+                batch = next(reader, None)
+                if read_timer is not None:
+                    read_duration_ns += clock() - start_ns
+                if batch is None:
+                    return
+                # Stop timing before the consumer runs, including on early 
generator close.
+                yield batch
+        finally:
+            if read_timer is not None:
+                read_timer.add_duration_ns(read_duration_ns)
 
     def _write_stream_start(
         self, batch_iterator: Iterator["pa.RecordBatch"], stream: IO[bytes]
@@ -99,7 +151,12 @@ class ArrowStreamSerializer(Serializer):
 
         # Signal the JVM after the first batch succeeds, so errors during
         # batch creation can be reported before the Arrow stream starts.
+        if self._collect_timing:
+            start_ns = time.perf_counter_ns()
         write_int(SpecialLengths.START_ARROW_STREAM, stream)
+        if self._collect_timing:
+            duration_ns = time.perf_counter_ns() - start_ns
+            
WorkerMetrics().measure("pythonDataWriteTime").add_duration_ns(duration_ns)
         yield from itertools.chain([first], batch_iterator)
 
     def __repr__(self) -> str:
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py 
b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
index 417f1082cdc4..361ea9676ab7 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_python_udf.py
@@ -15,13 +15,19 @@
 # limitations under the License.
 #
 
+import time
 import unittest
 from decimal import Decimal
+from io import BufferedReader, BufferedWriter, BytesIO
+from itertools import product
+from unittest.mock import patch
 
 from pyspark.errors import AnalysisException, PySparkNotImplementedError, 
PythonException
 from pyspark.loose_version import LooseVersion
+from pyspark.serializers import read_int
 from pyspark.sql import Row
 from pyspark.sql.functions import col, udf
+from pyspark.sql.pandas.serializers import ArrowStreamSerializer, 
SpecialLengths
 from pyspark.sql.tests.test_udf import BaseUDFTestsMixin
 from pyspark.sql.types import (
     ArrayType,
@@ -43,6 +49,7 @@ from pyspark.testing.utils import (
     pyarrow_requirement_message,
 )
 from pyspark.util import PythonEvalType
+from pyspark.worker_metrics import WorkerMetrics
 
 if have_pandas:
     import pandas as pd
@@ -543,6 +550,305 @@ class 
ArrowPythonUDFNonLegacyTests(ArrowPythonUDFNonLegacyTestsMixin, ReusedSQLT
         finally:
             super().tearDownClass()
 
+    def test_phase_metrics(self):
+        @udf("long", useArrow=True)
+        def measured(value):
+            time.sleep(0.002)
+            return value + 1
+
+        constant = udf(lambda: 7, "long", useArrow=True)
+        phase_names = (
+            "pythonDataReadTime",
+            "pythonInputPreparationTime",
+            "pythonUDFExecutionTime",
+            "pythonOutputPreparationTime",
+            "pythonDataWriteTime",
+        )
+        with self.sql_conf(
+            {
+                "spark.sql.execution.arrow.maxRecordsPerBatch": "2",
+                "spark.sql.adaptive.enabled": "false",
+                "spark.sql.execution.pythonUDF.detailedTiming.enabled": "true",
+            }
+        ):
+            # Exercise sequential and concurrent evaluation, then an empty 
worker task.
+            for num_rows, concurrency in ((5, 0), (3, 2), (0, 0)):
+                concurrency_conf = (
+                    {"spark.sql.execution.pythonUDF.arrow.concurrency.level": 
str(concurrency)}
+                    if concurrency
+                    else {}
+                )
+                with (
+                    self.subTest(num_rows=num_rows, concurrency=concurrency),
+                    self.sql_conf(concurrency_conf),
+                ):
+                    if num_rows:
+                        data = self.spark.range(num_rows, numPartitions=1)
+                    else:
+                        data = 
self.spark.createDataFrame(self.sc.parallelize([], 1), "id long")
+                    result = data.select(measured("id"), constant())
+                    self.assertEqual(
+                        [tuple(row) for row in result.collect()],
+                        [(value + 1, 7) for value in range(num_rows)],
+                    )
+                    plan = result._jdf.queryExecution().executedPlan()
+                    while plan.nodeName() != "ArrowEvalPython":
+                        self.assertEqual(plan.children().size(), 1)
+                        plan = plan.children().apply(0)
+                    metrics = plan.metrics()
+                    
self.assertEqual(metrics.apply("pythonNumPipelinedTasks").value(), 0)
+                    
self.assertEqual(metrics.apply("pythonNumTimingReports").value(), 1)
+                    self.assertEqual(
+                        metrics.apply("pythonNumTimedBatches").value(), 
(num_rows + 1) // 2
+                    )
+                    phases = [metrics.apply(name).value() for name in 
phase_names]
+                    self.assertTrue(all(value >= 0 for value in phases))
+                    if num_rows:
+                        for name in phase_names:
+                            self.assertFalse(metrics.apply(name).isZero(), 
name)
+                    if num_rows and concurrency == 0:
+                        self.assertGreaterEqual(phases[2], 2 * num_rows)
+                    elif num_rows:
+                        self.assertGreater(phases[2], 0)
+                    else:
+                        self.assertEqual(phases[1:4], [0, 0, 0])
+                    self.assertLessEqual(sum(phases), 
metrics.apply("pythonProcessingTime").value())
+
+    def test_phase_metrics_disabled_by_default(self):
+        add_one = udf(lambda value: value + 1, "long", useArrow=True)
+
+        def run():
+            result = self.spark.range(2, numPartitions=1).select(add_one("id"))
+            self.assertEqual([row[0] for row in result.collect()], [1, 2])
+            plan = result._jdf.queryExecution().executedPlan()
+            while plan.nodeName() != "ArrowEvalPython":
+                self.assertEqual(plan.children().size(), 1)
+                plan = plan.children().apply(0)
+            return plan.metrics()
+
+        timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+        with self.sql_conf({"spark.sql.adaptive.enabled": "false"}):
+            with self.sql_conf({timing_conf: "true"}):
+                
self.assertEqual(run().apply("pythonNumTimingReports").value(), 1)
+            self.assertEqual(self.spark.conf.get(timing_conf), "false")
+            metrics = run()
+
+        detailed_metrics = (
+            "pythonDataReadTime",
+            "pythonInputPreparationTime",
+            "pythonUDFExecutionTime",
+            "pythonOutputPreparationTime",
+            "pythonDataWriteTime",
+            "pythonNumTimingReports",
+            "pythonNumTimedBatches",
+        )
+        for name in detailed_metrics:
+            self.assertFalse(metrics.contains(name), name)
+        self.assertTrue(metrics.contains("pythonProcessingTime"))
+        self.assertTrue(metrics.contains("pythonNumPipelinedTasks"))
+
+    def test_phase_metrics_on_materialized_plan_after_config_change(self):
+        add_one = udf(lambda value: value + 1, "long", useArrow=True)
+        timing_conf = "spark.sql.execution.pythonUDF.detailedTiming.enabled"
+
+        with self.sql_conf({"spark.sql.adaptive.enabled": "false"}):
+            with self.sql_conf({timing_conf: "true"}):
+                result = self.spark.range(2, 
numPartitions=1).select(add_one("id"))
+                plan = result._jdf.queryExecution().executedPlan()
+                while plan.nodeName() != "ArrowEvalPython":
+                    self.assertEqual(plan.children().size(), 1)
+                    plan = plan.children().apply(0)
+                metrics = plan.metrics()
+                self.assertTrue(metrics.contains("pythonNumTimingReports"))
+
+            self.assertEqual(self.spark.conf.get(timing_conf), "false")
+            self.assertEqual([row[0] for row in result.collect()], [1, 2])
+            self.assertEqual(metrics.apply("pythonNumTimingReports").value(), 
1)
+            self.assertEqual(metrics.apply("pythonNumTimedBatches").value(), 1)
+
+
[email protected](not have_pyarrow, pyarrow_requirement_message)
+class ArrowStreamTimingTests(unittest.TestCase):
+    def test_timing_excludes_producer_consumer_and_owner_flush(self):
+        import pyarrow as pa
+
+        metrics = WorkerMetrics()
+        self.addCleanup(metrics.reset)
+        batch = pa.record_batch([[1, 2]], names=["value"])
+        clock_ns = 0
+
+        def clock():
+            return clock_ns
+
+        class DelayedStream(BytesIO):
+            read_ns = 0
+            write_ns = 0
+
+            def read(self, size=-1):
+                nonlocal clock_ns
+                clock_ns += 3_000_000
+                self.read_ns += 3_000_000
+                return super().read(size)
+
+            def write(self, data):
+                nonlocal clock_ns
+                clock_ns += 5_000_000
+                self.write_ns += 5_000_000
+                return super().write(data)
+
+            def flush(self):
+                nonlocal clock_ns
+                clock_ns += 7_000_000
+                self.write_ns += 7_000_000
+                return super().flush()
+
+        def produce():
+            nonlocal clock_ns
+            for _ in range(2):
+                clock_ns += 100_000_000
+                yield batch
+            clock_ns += 100_000_000
+
+        for write_start_stream, flush_per_batch, close_early in 
product((False, True), repeat=3):
+            with self.subTest(
+                write_start_stream=write_start_stream,
+                flush_per_batch=flush_per_batch,
+                close_early=close_early,
+            ):
+                metrics.reset()
+                serializer = ArrowStreamSerializer(
+                    write_start_stream=write_start_stream,
+                    flush_per_batch=flush_per_batch,
+                    collect_timing=True,
+                )
+                with patch(
+                    "pyspark.sql.pandas.serializers.time.perf_counter_ns", 
side_effect=clock
+                ):
+                    output = DelayedStream()
+                    serializer.dump_stream(produce(), output)
+                    # Include the start marker and writer close, but not the 
owner's flush.
+                    write_ns = output.write_ns
+                    output.flush()
+                    source = DelayedStream(output.getvalue())
+                    if write_start_stream:
+                        self.assertEqual(read_int(source), 
SpecialLengths.START_ARROW_STREAM)
+                    source.read_ns = 0
+                    reader = serializer.load_stream(source)
+                    for actual in reader:
+                        clock_ns += 100_000_000
+                        self.assertTrue(actual.equals(batch))
+                        if close_early:
+                            reader.close()
+                            break
+
+                # Only stream operations advance the clock inside the Arrow 
intervals.
+                report = metrics.to_dict()
+                self.assertEqual(
+                    report,
+                    {
+                        "pythonDataReadTime": source.read_ns // 1_000_000,
+                        "pythonDataWriteTime": write_ns // 1_000_000,
+                    },
+                )
+
+    def test_disabled_timing_does_not_read_clock(self):
+        import pyarrow as pa
+
+        metrics = WorkerMetrics()
+        metrics.reset()
+        self.addCleanup(metrics.reset)
+        batch = pa.record_batch([[1, 2]], names=["value"])
+        serializer = ArrowStreamSerializer(write_start_stream=True)
+        with patch(
+            "pyspark.sql.pandas.serializers.time.perf_counter_ns",
+            side_effect=AssertionError("timing is disabled"),
+        ):
+            stream = BytesIO()
+            serializer.dump_stream(iter([batch]), stream)
+            stream.seek(0)
+            self.assertEqual(read_int(stream), 
SpecialLengths.START_ARROW_STREAM)
+            
self.assertTrue(list(serializer.load_stream(stream))[0].equals(batch))
+        self.assertEqual(metrics.to_dict(), {})
+
+    def test_timing_preserves_stream_operations(self):
+        import pyarrow as pa
+
+        metrics = WorkerMetrics()
+        self.addCleanup(metrics.reset)
+        batch = pa.record_batch([[1, 2]], names=["value"])
+
+        def run(timed, buffer_size, write_start_stream, flush_per_batch, 
num_batches, fail):
+            metrics.reset()
+            events = []
+
+            class RecordedStream(BytesIO):
+                def __init__(self, data, mode):
+                    super().__init__(data)
+                    self.mode = mode
+
+                def readable(self):
+                    return self.mode == "rb"
+
+                def writable(self):
+                    return self.mode == "wb"
+
+                def seekable(self):
+                    return False
+
+                def readinto(self, data):
+                    events.append(("readinto", len(data)))
+                    return super().readinto(data)
+
+                def write(self, data):
+                    events.append(("write", bytes(data)))
+                    return super().write(data)
+
+                def flush(self):
+                    events.append(("flush",))
+                    return super().flush()
+
+            def produce():
+                for _ in range(num_batches):
+                    events.append(("produce",))
+                    yield batch
+                if fail:
+                    raise RuntimeError("batch producer failed")
+
+            raw_output = RecordedStream(b"", "wb")
+            stream = BufferedWriter(raw_output, buffer_size)
+            serializer = ArrowStreamSerializer(
+                write_start_stream=write_start_stream,
+                flush_per_batch=flush_per_batch,
+                collect_timing=timed,
+            )
+            if fail:
+                with self.assertRaisesRegex(RuntimeError, "batch producer 
failed"):
+                    serializer.dump_stream(produce(), stream)
+            else:
+                serializer.dump_stream(produce(), stream)
+            stream.flush()
+            data = raw_output.getvalue()
+            raw_input = RecordedStream(data, "rb")
+            reader = BufferedReader(raw_input, buffer_size)
+            if num_batches:
+                if write_start_stream:
+                    self.assertEqual(read_int(reader), 
SpecialLengths.START_ARROW_STREAM)
+                for actual in serializer.load_stream(reader):
+                    events.append(("consume",))
+                    self.assertTrue(actual.equals(batch))
+            self.assertFalse(stream.closed)
+            reader.close()
+            stream.close()
+            return data, events
+
+        for buffer_size in (8, 64, 65536):
+            for write_start_stream in (False, True):
+                for flush_per_batch in (False, True):
+                    for num_batches, fail in ((0, False), (0, True), (2, 
False), (2, True)):
+                        case = (buffer_size, write_start_stream, 
flush_per_batch, num_batches, fail)
+                        with self.subTest(case=case):
+                            self.assertEqual(run(False, *case), run(True, 
*case))
+
 
 if __name__ == "__main__":
     from pyspark.testing import main
diff --git a/python/pyspark/sql/tests/pandas/test_pipelined_udf.py 
b/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
index ad194070b6d8..005184ff46fc 100644
--- a/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
+++ b/python/pyspark/sql/tests/pandas/test_pipelined_udf.py
@@ -70,8 +70,13 @@ class PipelinedUDFTests(ReusedSQLTestCase):
             flag = os.environ.get("SPARK_PIPELINED_UDF", "not_set")
             return pd.Series([flag] * len(x))
 
-        result = self.spark.range(1).select(check_env(col("id"))).first()[0]
-        self.assertEqual(result, "1", "JVM should set SPARK_PIPELINED_UDF=1")
+        result = self.spark.range(1, 
numPartitions=1).select(check_env(col("id")))
+        self.assertEqual(result.collect()[0][0], "1", "JVM should set 
SPARK_PIPELINED_UDF=1")
+        plan = result._jdf.queryExecution().executedPlan()
+        while plan.nodeName() != "ArrowEvalPython":
+            self.assertEqual(plan.children().size(), 1)
+            plan = plan.children().apply(0)
+        
self.assertEqual(plan.metrics().apply("pythonNumPipelinedTasks").value(), 1)
 
     def test_scalar_arrow_udf(self):
         """Basic scalar Arrow UDF with pipelined mode."""
diff --git a/python/pyspark/tests/test_worker.py 
b/python/pyspark/tests/test_worker.py
index a7468ff0ff9e..48d2fc48e9f9 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -21,6 +21,7 @@ import tempfile
 import threading
 import time
 import unittest
+from unittest.mock import patch
 
 has_resource_module = True
 try:
@@ -32,6 +33,97 @@ from py4j.protocol import Py4JJavaError
 
 from pyspark import SparkConf, SparkContext
 from pyspark.testing.utils import QuietTest, ReusedPySparkTestCase, eventually
+from pyspark.worker_metrics import WorkerMetrics
+
+
+class WorkerMetricsTests(unittest.TestCase):
+    def setUp(self):
+        metrics = WorkerMetrics()
+        metrics.reset()
+        self.addCleanup(metrics.reset)
+
+    def test_constructor_reuses_collector_without_reset(self):
+        metrics = WorkerMetrics()
+        metrics.set("bootTimestampMs", 123)
+        metrics.increment("rows", 2)
+        self.assertIs(WorkerMetrics(), metrics)
+        self.assertEqual(WorkerMetrics().to_dict(), {"bootTimestampMs": 123, 
"rows": 2})
+
+    def test_reset_isolates_new_task_from_existing_timer(self):
+        metrics = WorkerMetrics()
+        metrics.set("previousTaskRows", 10)
+        previous_timer = metrics.measure("previousTaskDuration")
+        with patch(
+            "pyspark.worker_metrics.time.perf_counter_ns",
+            side_effect=[0, 5_000_000, 10_000_000, 12_000_000],
+        ):
+            with previous_timer:
+                metrics.reset()
+                metrics.increment("currentTaskRows")
+            with metrics.measure("currentTaskDuration"):
+                pass
+        previous_timer.add_duration_ns(3_000_000)
+        self.assertIs(WorkerMetrics(), metrics)
+        self.assertEqual(metrics.to_dict(), {"currentTaskRows": 1, 
"currentTaskDuration": 2})
+
+    def test_set_current_timestamp_uses_epoch_milliseconds(self):
+        metrics = WorkerMetrics()
+        with patch("pyspark.worker_metrics.time.time_ns", 
return_value=1_700_000_000_123_456_789):
+            metrics.set_current_timestamp("bootTimestampMs")
+        self.assertEqual(metrics.to_dict(), {"bootTimestampMs": 
1_700_000_000_123})
+
+    def test_accumulate_before_converting_to_milliseconds(self):
+        metrics = WorkerMetrics()
+        # Control the clock to retain fractional milliseconds and exclude gaps 
between blocks.
+        with patch(
+            "pyspark.worker_metrics.time.perf_counter_ns",
+            side_effect=[10_000_000, 10_600_000, 100_000_000, 100_700_000],
+        ):
+            with metrics.measure("duration"):
+                pass
+            self.assertEqual(metrics.to_dict(), {"duration": 0})
+            with metrics.measure("duration"):
+                pass
+        self.assertEqual(metrics.to_dict(), {"duration": 1})
+
+    def test_nested_timings_are_inclusive(self):
+        metrics = WorkerMetrics()
+        with patch(
+            "pyspark.worker_metrics.time.perf_counter_ns",
+            side_effect=[0, 1_000_000, 3_000_000, 5_000_000],
+        ):
+            with metrics.measure("outer"):
+                with metrics.measure("inner"):
+                    pass
+        self.assertEqual(metrics.to_dict(), {"outer": 5, "inner": 2})
+
+    def test_reuse_timer_scope(self):
+        metrics = WorkerMetrics()
+        timer = metrics.measure("duration")
+        with patch(
+            "pyspark.worker_metrics.time.perf_counter_ns",
+            side_effect=[0, 2_000_000, 3_000_000, 4_000_000],
+        ):
+            with timer:
+                with self.assertRaisesRegex(RuntimeError, "already running"):
+                    with timer:
+                        pass
+            with timer:
+                pass
+        self.assertEqual(metrics.to_dict(), {"duration": 3})
+
+    def test_accumulate_and_propagate_exception(self):
+        metrics = WorkerMetrics()
+        with patch(
+            "pyspark.worker_metrics.time.perf_counter_ns",
+            side_effect=[0, 2_000_000, 3_000_000, 4_000_000],
+        ):
+            with self.assertRaisesRegex(ValueError, "worker failure"):
+                with metrics.measure("duration"):
+                    raise ValueError("worker failure")
+            with metrics.measure("duration"):
+                pass
+        self.assertEqual(metrics.to_dict(), {"duration": 3})
 
 
 class WorkerTests(ReusedPySparkTestCase):
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index c76dd0c08f93..f9e305707b23 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -24,7 +24,6 @@ import inspect
 import json
 import os
 import sys
-import time
 import warnings
 from typing import BinaryIO
 
@@ -61,6 +60,7 @@ from pyspark.util import (
     with_faulthandler,
 )
 from pyspark.worker_message import UDTFInfo, WorkerInitInfo
+from pyspark.worker_metrics import WorkerMetrics
 from pyspark.worker_util import (
     EvalConf,
     RunnerConf,
@@ -75,21 +75,9 @@ from pyspark.worker_util import (
 )
 
 
-def report_metrics(
-    outfile, boot, init, finish, execution_duration_ms, memory_bytes_spilled, 
disk_bytes_spilled
-):
+def report_metrics(outfile: BinaryIO, metrics: dict[str, int]) -> None:
     """Write the worker metrics as a length-prefixed JSON report after 
METRICS_DATA."""
-    payload = json.dumps(
-        {
-            "bootTimestampMs": int(1000 * boot),
-            "initTimestampMs": int(1000 * init),
-            "finishTimestampMs": int(1000 * finish),
-            "pythonExecutionDurationMs": execution_duration_ms,
-            "memoryBytesSpilled": memory_bytes_spilled,
-            "diskBytesSpilled": disk_bytes_spilled,
-        },
-        separators=(",", ":"),
-    ).encode("utf-8")
+    payload = json.dumps(metrics, separators=(",", ":")).encode("utf-8")
     write_int(SpecialLengths.METRICS_DATA, outfile)
     write_with_length(payload, outfile)
 
@@ -380,7 +368,9 @@ def invoke_udf(message_receiver: SparkMessageReceiver, 
outfile: BinaryIO):
     Initialization -> Processing -> Finish/Cleanup
     """
     try:
-        boot_time = time.time()
+        metrics = WorkerMetrics()
+        metrics.reset()
+        metrics.set_current_timestamp("bootTimestampMs")
         # Initialization
         init_message = message_receiver.get_init_message()
         init_info = WorkerInitInfo.from_stream(init_message)
@@ -436,7 +426,7 @@ def invoke_udf(message_receiver: SparkMessageReceiver, 
outfile: BinaryIO):
         split_index = init_info.split_index
         del init_info
 
-        init_time = time.time()
+        metrics.set_current_timestamp("initTimestampMs")
 
         # Processing
 
@@ -543,17 +533,17 @@ def invoke_udf(message_receiver: SparkMessageReceiver, 
outfile: BinaryIO):
                     )
 
         is_pipelined = os.environ.get("SPARK_PIPELINED_UDF") == "1"
+        # Pipelined reads can overlap UDF work, so mark these task reports for 
filtering.
+        metrics.set("pythonNumPipelinedTasks", int(is_pipelined))
         if is_pipelined and hasattr(serializer, "_flush_per_batch"):
             serializer._flush_per_batch = True
         run_process = pipelined_process if is_pipelined else process
 
-        execution_start_time = time.time()
-        with capture_outputs():
+        with metrics.measure("pythonExecutionDurationMs"), capture_outputs():
             if profiler:
                 profiler.profile(run_process)
             else:
                 run_process()
-        execution_duration_ms = int(1000 * (time.time() - 
execution_start_time))
 
         # Cleanup
         # Reset task context to None. This is a guard code to avoid residual 
context when worker
@@ -563,16 +553,11 @@ def invoke_udf(message_receiver: SparkMessageReceiver, 
outfile: BinaryIO):
     except BaseException as e:
         handle_worker_exception(e, outfile)
         sys.exit(-1)
-    finish_time = time.time()
-    report_metrics(
-        outfile,
-        boot_time,
-        init_time,
-        finish_time,
-        execution_duration_ms,
-        shuffle.MemoryBytesSpilled,
-        shuffle.DiskBytesSpilled,
-    )
+    metrics.set_current_timestamp("finishTimestampMs")
+    # Spill totals are byte counts already expressed in their reporting unit.
+    metrics.set("memoryBytesSpilled", shuffle.MemoryBytesSpilled)
+    metrics.set("diskBytesSpilled", shuffle.DiskBytesSpilled)
+    report_metrics(outfile, metrics.to_dict())
 
     # Mark the beginning of the accumulators section of the output
     write_int(SpecialLengths.END_OF_DATA_SECTION, outfile)
diff --git a/python/pyspark/worker_metrics.py b/python/pyspark/worker_metrics.py
new file mode 100644
index 000000000000..356e390bf74c
--- /dev/null
+++ b/python/pyspark/worker_metrics.py
@@ -0,0 +1,134 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""Shared metrics and reusable timers for a Python worker process."""
+
+import threading
+import time
+from typing import Any, ClassVar, Optional
+
+
+class _WorkerTimer:
+    """Measure one named duration and add each interval to the collector's 
task total.
+
+    The collector owns the duration dictionary. This scope keeps a reference 
to it and a
+    start time for its current with block. Nested scopes measure inclusive 
time.
+
+    Repeatedly entering and exiting this context is costly: every interval 
reads the
+    clock twice and updates the duration dictionary. In a hot batch loop, 
accumulate
+    locally and call add_duration_ns once instead of entering this context per 
batch.
+
+    Pipelined reads can overlap UDF work on another thread. An additive phase 
breakdown
+    for that mode would require tracking those overlapping intervals.
+    """
+
+    def __init__(self, shared_duration_totals_ns: dict[str, int], metric_name: 
str) -> None:
+        """Bind one metric name to the collector's shared duration totals."""
+        # Keep the collector's dictionary by reference so updates reach its 
task totals.
+        self._shared_duration_totals_ns = shared_duration_totals_ns
+        self._metric_name = metric_name
+        self._block_start_ns: Optional[int] = None
+
+    def __enter__(self) -> None:
+        """Start the clock when Python enters the with block."""
+        # Re-entering this instance would overwrite an unfinished 
measurement's start time.
+        if self._block_start_ns is not None:
+            raise RuntimeError("Worker timer is already running")
+        self._block_start_ns = time.perf_counter_ns()
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        """Accumulate elapsed time when Python leaves the block, including on 
an exception."""
+        assert self._block_start_ns is not None
+        elapsed_ns = time.perf_counter_ns() - self._block_start_ns
+        self._block_start_ns = None
+        self._shared_duration_totals_ns[self._metric_name] += elapsed_ns
+        # Python supplies the exception arguments; returning None lets the 
error propagate.
+
+    def add_duration_ns(self, duration_ns: int) -> None:
+        """Flush a duration accumulated in local variables across a hot loop.
+
+        This updates the task-bound dictionary once when the loop ends, rather 
than
+        calling a timer scope and updating the dictionary for every batch.
+        """
+        self._shared_duration_totals_ns[self._metric_name] += duration_ns
+
+
+class WorkerMetrics:
+    """One collector per Python worker process, reset by the worker before 
each task.
+
+    set stores a supplied value; increment adds to a counter.
+    set_current_timestamp reads the clock and stores epoch milliseconds.
+    measure creates timer scopes that accumulate nanoseconds in a separate 
shared dictionary.
+    to_dict combines both dictionaries, converting only durations to 
milliseconds.
+    """
+
+    _instance: ClassVar[Optional["WorkerMetrics"]] = None
+    _instance_lock = threading.Lock()
+
+    def __new__(cls) -> "WorkerMetrics":
+        """Return the shared collector without resetting an active task."""
+        instance = cls._instance
+        if instance is None:
+            with cls._instance_lock:
+                instance = cls._instance
+                if instance is None:
+                    instance = super().__new__(cls)
+                    instance.reset()
+                    cls._instance = instance
+        return instance
+
+    def reset(self) -> None:
+        """Start fresh task state while keeping the singleton's identity."""
+        # Counters, spill byte counts, and epoch timestamps need no conversion 
at report time.
+        self._values_in_report_units: dict[str, int] = {}
+        # Existing timers keep the old dictionary. A fresh dictionary keeps 
their
+        # measurements out of the next task's report.
+        self._duration_totals_ns: dict[str, int] = {}
+
+    def set_current_timestamp(self, name: str) -> None:
+        """Read the current clock and store its epoch milliseconds under 
name."""
+        self.set(name, time.time_ns() // 1_000_000)
+
+    def set(self, name: str, value: int) -> None:
+        """Store the supplied value unchanged, replacing any earlier value."""
+        self._values_in_report_units[name] = value
+
+    def increment(self, name: str, value: int = 1) -> None:
+        """Add to a counter in its reporting unit, starting from zero.
+
+        The default increment is one.
+        """
+        self._values_in_report_units[name] = 
self._values_in_report_units.get(name, 0) + value
+
+    def measure(self, name: str) -> _WorkerTimer:
+        """Create a task-bound duration handle for with or add_duration_ns."""
+        # Register zero so an empty supported task can still report this 
timing.
+        self._duration_totals_ns.setdefault(name, 0)
+        return _WorkerTimer(self._duration_totals_ns, name)
+
+    def to_dict(self) -> dict[str, int]:
+        """Build the numeric report dictionary consumed by report_metrics.
+
+        Counters, bytes, and epoch timestamps already have their reporting 
units. Only the
+        accumulated duration totals require conversion from nanoseconds to 
milliseconds.
+        """
+        # Round once per task, preserving sub-millisecond contributions from 
individual batches.
+        return {
+            **self._values_in_report_units,
+            **{name: duration // 1_000_000 for name, duration in 
self._duration_totals_ns.items()},
+        }
diff --git a/python/pyspark/worker_util.py b/python/pyspark/worker_util.py
index f02f0cd8dc44..10507bef0fe1 100644
--- a/python/pyspark/worker_util.py
+++ b/python/pyspark/worker_util.py
@@ -319,6 +319,10 @@ class RunnerConf(Conf):
             == "true"
         )
 
+    @property
+    def detailed_timing_enabled(self) -> bool:
+        return 
self.get("spark.sql.execution.pythonUDF.detailedTiming.enabled", "false") == 
"true"
+
     @property
     def use_legacy_pandas_udtf_conversion(self) -> bool:
         return (
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cb60d8905455..3b9d0d1f6ae0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5801,6 +5801,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val PYTHON_UDF_DETAILED_TIMING_ENABLED =
+    buildConf("spark.sql.execution.pythonUDF.detailedTiming.enabled")
+      .internal()
+      .doc("When true, collect detailed timing metrics for supported Python 
UDFs.")
+      .version("4.4.0")
+      .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+      .booleanConf
+      .createWithDefault(false)
+
   val PYTHON_UDF_ARROW_ENABLED =
     buildConf("spark.sql.execution.pythonUDF.arrow.enabled")
       .doc("Enable Arrow optimization in regular Python UDFs. This 
optimization " +
@@ -10001,6 +10010,9 @@ class SQLConf extends Serializable with Logging with 
SqlApiConf {
 
   def pythonWorkerLoggingEnabled: Boolean = 
getConf(PYTHON_WORKER_LOGGING_ENABLED)
 
+  def pythonUDFDetailedTimingEnabled: Boolean =
+    getConf(PYTHON_UDF_DETAILED_TIMING_ENABLED)
+
   def pythonUDFArrowConcurrencyLevel: Option[Int] = 
getConf(PYTHON_UDF_ARROW_CONCURRENCY_LEVEL)
 
   def pythonUDFArrowFallbackOnUDT: Boolean = 
getConf(PYTHON_UDF_ARROW_FALLBACK_ON_UDT)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
index 6e65dc032459..ddfa84466272 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowEvalPythonExec.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.python.EvalPythonExec.ArgumentMetadata
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{StructType, UserDefinedType}
 import org.apache.spark.sql.types.DataType.equalsIgnoreCompatibleCollation
 import org.apache.spark.sql.vectorized.ColumnarBatch
@@ -89,6 +90,21 @@ case class ArrowEvalPythonExec(
     throw SparkException.internalError(s"Unexpected eval type $evalType")
   }
 
+  override protected def additionalMetrics: Map[String, SQLMetric] = {
+    if (evalType == PythonEvalType.SQL_ARROW_BATCHED_UDF &&
+        !conf.legacyPandasConversionUDF && 
conf.pythonUDFDetailedTimingEnabled) {
+      createPythonPhaseMetrics()
+    } else {
+      Map.empty
+    }
+  }
+
+  // SQL metrics are fixed with the plan; use that decision if the session 
setting changes later.
+  private def pythonRunnerConfMap: Map[String, String] =
+    ArrowPythonRunner.getPythonRunnerConfMap(conf).updated(
+      SQLConf.PYTHON_UDF_DETAILED_TIMING_ENABLED.key,
+      pythonMetrics.contains("pythonNumTimingReports").toString)
+
   private[this] val jobArtifactUUID = 
JobArtifactSet.getCurrentJobArtifactState.map(_.uuid)
   private[this] val sessionUUID = {
     Option(session).collect {
@@ -145,7 +161,7 @@ case class ArrowEvalPythonExec(
       evalType,
       conf.sessionLocalTimeZone,
       conf.arrowUseLargeVarTypes,
-      ArrowPythonRunner.getPythonRunnerConfMap(conf),
+      pythonRunnerConfMap,
       pythonMetrics,
       jobArtifactUUID,
       sessionUUID)
@@ -159,7 +175,7 @@ case class ArrowEvalPythonExec(
       evalType,
       conf.sessionLocalTimeZone,
       conf.arrowUseLargeVarTypes,
-      ArrowPythonRunner.getPythonRunnerConfMap(conf),
+      pythonRunnerConfMap,
       pythonMetrics,
       jobArtifactUUID,
       sessionUUID)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
index dedd8b5cbdf6..2bc5f4648ea6 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ArrowPythonRunner.scala
@@ -187,6 +187,7 @@ object ArrowPythonRunner {
       SQLConf.ARROW_EXECUTION_USE_LARGE_VAR_TYPES,
       SQLConf.PYTHON_TABLE_UDF_LEGACY_PANDAS_CONVERSION_ENABLED,
       SQLConf.PYTHON_UDF_LEGACY_PANDAS_CONVERSION_ENABLED,
+      SQLConf.PYTHON_UDF_DETAILED_TIMING_ENABLED,
       SQLConf.PYTHON_UDF_MAP_IN_BATCH_LEGACY_ACCEPT_ANY_ITERABLE_ENABLED,
       SQLConf.PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED,
       SQLConf.PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE,
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
index 3f3bb8df2a01..0d3c78e424b6 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonSQLMetrics.scala
@@ -42,6 +42,14 @@ trait PythonSQLMetrics { self: SparkPlan =>
   // empty so the shared trait stays minimal; see [[PythonPickleBatchMetrics]].
   protected def additionalMetrics: Map[String, SQLMetric] = Map.empty
 
+  protected def createPythonPhaseMetrics(): Map[String, SQLMetric] = {
+    PythonSQLMetrics.pythonPhaseTimingMetricsDesc.map { case (k, v) =>
+      k -> SQLMetrics.createTimingMetric(sparkContext, v)
+    } ++ PythonSQLMetrics.pythonPhaseCountMetricsDesc.map { case (k, v) =>
+      k -> SQLMetrics.createMetric(sparkContext, v)
+    }
+  }
+
   override lazy val metrics: Map[String, SQLMetric] = pythonMetrics
 }
 
@@ -63,7 +71,24 @@ object PythonSQLMetrics {
   }
 
   val pythonOtherMetricsDesc: Map[String, String] = {
-    Map("pythonNumRowsReceived" -> "number of output rows")
+    Map(
+      "pythonNumRowsReceived" -> "number of output rows",
+      "pythonNumPipelinedTasks" -> "number of pipelined Python worker tasks")
+  }
+
+  val pythonPhaseTimingMetricsDesc: Map[String, String] = {
+    Map(
+      "pythonDataReadTime" -> "time to read and deserialize Arrow data",
+      "pythonInputPreparationTime" -> "time to prepare Python UDF input",
+      "pythonUDFExecutionTime" -> "time to evaluate Python UDFs",
+      "pythonOutputPreparationTime" -> "time to prepare Python UDF output",
+      "pythonDataWriteTime" -> "time to serialize and write Arrow data")
+  }
+
+  val pythonPhaseCountMetricsDesc: Map[String, String] = {
+    Map(
+      "pythonNumTimingReports" -> "number of Python tasks with detailed 
timing",
+      "pythonNumTimedBatches" -> "number of Python batches with detailed 
timing")
   }
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to