This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 98e2b9b  [SPARK-61085][CONNECT][PYTHON] Collect the results of the UDF 
profilers, and show them with spark.profile, in the Python client as PySpark 
does
98e2b9b is described below

commit 98e2b9b526b4c26b116f1de0b68b0dc39c6abf6c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:10:54 2026 +0900

    [SPARK-61085][CONNECT][PYTHON] Collect the results of the UDF profilers, 
and show them with spark.profile, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61085](https://issues.apache.org/jira/browse/SPARK-61085): 
`spark.profile` now collects and shows the results of the UDF profilers, as 
PySpark Connect does.
    
    As PySpark Connect's:
    - `spark.profile` is a `pyspark.sql.profiler.Profile` of the session's 
`ConnectProfilerCollector`, also exposed as `spark._profiler_collector`.
    - The client updates the collector from the `__python_accumulator__` 
observed metrics of each execution. Each value is a pickled `(id, update)` of 
`specialAccumulatorSer`. The updates of the UDF profiler (`SQL_UDF_PROFIER_V2`, 
enabled by `spark.sql.pyspark.udf.profiler`) update the collector. A value that 
does not unpickle is logged as a warning and skipped.
    - So `spark.profile.show`, `dump`, `render` and `clear` show, write and 
clear the perf and memory profiles of the UDFs.
    
    `pyspark.sql.connect.profiler` is vendored from Apache Spark v4.3.0, and 
the drift check covers it.
    
    The core hands each pickled update of the Python accumulators to the 
handler of the new `SparkSession::set_python_accumulator_handler`. It no longer 
keeps the observed metrics of each execution as text in a `ProfilerCollector`; 
that type and `SparkSession::profiler()` are removed.
    
    ### Why are the changes needed?
    
    `spark.profile` returned a `ProfilerCollector` of the client's own, which 
never held a UDF profile:
    - it stored the observed metrics of each plan as text instead
    - its `show()` returned a str rather than printing the profiles
    - `dump(path)` and `show(type="perf")` raised `TypeError`s
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `spark.profile` collects and shows the UDF profiles of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test enables the perf UDF profiler, runs a UDF, shows its 
profile with `spark.profile`, and clears it. It passes with official PySpark 
4.3.0 and fails without this change.
    - A new offline test sends the pickled updates of the UDF profiler, of 
another accumulator, and a value that does not unpickle, from a mock Connect 
server. It checks the profiles of the collector, the warning, `show` and 
`clear`.
    - A new lib test checks that the handler takes the updates of the Python 
accumulators, and not the metrics of observations.
    - On a local server, the profile of a UDF is shown, dumped and cleared as 
with official PySpark 4.3.0.
    - The lib tests of both crates (584 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2040 passed, 12 skipped), and the 
live drop-in and Arrow suites (145 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #294 from HyukjinKwon/SPARK-61085-udf-profiler.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/lib.rs               |   2 -
 crates/pyspark-rs/src/profiler.rs          | 112 ++++---
 crates/pyspark-rs/src/session.rs           |  33 +-
 crates/spark-connect/src/dataframe.rs      |   4 +-
 crates/spark-connect/src/lib.rs            |   1 -
 crates/spark-connect/src/profiler.rs       | 473 ++++-------------------------
 crates/spark-connect/src/session.rs        |  35 ++-
 crates/spark-connect/tests/e2e_coverage.rs |   1 -
 python/pyspark/sql/connect/profiler.py     |  41 +++
 python/tests/test_dropin_e2e.py            |  25 ++
 python/tests/test_dropin_offline.py        |  51 ++++
 scripts/check_vendored_upstream.sh         |   3 +
 12 files changed, 303 insertions(+), 478 deletions(-)

diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index be4b978..6a97253 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -69,7 +69,6 @@ use dataframe::{
 };
 use datasource::PyDataSourceRegistration;
 use group::{PyCoGroupedData, PyGroupedData};
-use profiler::PyProfilerCollector;
 use readwriter::PyDataFrameReader;
 use resource::{
     PyExecutorResourceRequests, PyResourceProfile, PyResourceProfileBuilder, 
PyTaskResourceRequests,
@@ -169,7 +168,6 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> 
PyResult<()> {
 
     // Data source and profiler classes
     m.add_class::<PyDataSourceRegistration>()?;
-    m.add_class::<PyProfilerCollector>()?;
 
     // Register DataType classes
     m.add_class::<PyDataType>()?;
diff --git a/crates/pyspark-rs/src/profiler.rs 
b/crates/pyspark-rs/src/profiler.rs
index 652fde8..c25e5e7 100644
--- a/crates/pyspark-rs/src/profiler.rs
+++ b/crates/pyspark-rs/src/profiler.rs
@@ -1,49 +1,85 @@
-//! PyO3 wrapper for the Spark profiler collector.
+//! The UDF profiles of the Python client, as PySpark Connect's client 
collects them: a
+//! `ConnectProfilerCollector` of each session, which the pickled updates of 
the Python
+//! accumulators of its executions update, and of which `spark.profile` is a 
`Profile`.
 
-use pyo3::prelude::*;
-use spark_connect::profiler::ProfilerCollector;
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
 
-use crate::errors::ResultExt;
+use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+use spark_connect::SparkSession;
 
-/// Python wrapper for the ProfilerCollector.
-///
-/// Exposed as `SparkSession.profile` to access profiler results.
-#[pyclass(name = "ProfilerCollector", module = "pyspark.sql.profiler")]
-pub struct PyProfilerCollector {
-    pub(crate) inner: Arc<ProfilerCollector>,
+/// The `ConnectProfilerCollector` of each session of the client, by its 
session ID: every
+/// Python object of a session has its profiles.
+fn registry() -> &'static Mutex<HashMap<String, Py<PyAny>>> {
+    static REGISTRY: OnceLock<Mutex<HashMap<String, Py<PyAny>>>> = 
OnceLock::new();
+    REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
 }
 
-impl PyProfilerCollector {
-    pub fn new(inner: Arc<ProfilerCollector>) -> Self {
-        PyProfilerCollector { inner }
+/// The `ConnectProfilerCollector` of `session`, as the `_profiler_collector` 
of PySpark's
+/// client: the first time, it sets the handler of the core session that 
updates it.
+pub(crate) fn profiler_collector(py: Python<'_>, session: &SparkSession) -> 
PyResult<Py<PyAny>> {
+    let mut registry = registry().lock().unwrap_or_else(|e| e.into_inner());
+    let id = session.session_id().to_string();
+    if let Some(collector) = registry.get(&id) {
+        return Ok(collector.clone_ref(py));
     }
+    let collector = py
+        .import("pyspark.sql.connect.profiler")?
+        .getattr("ConnectProfilerCollector")?
+        .call0()?
+        .unbind();
+    let shared = collector.clone_ref(py);
+    session.set_python_accumulator_handler(move |update| {
+        Python::attach(|py| {
+            if let Err(error) = update_collector(py, shared.bind(py), update) {
+                error.write_unraisable(py, None);
+            }
+        })
+    });
+    registry.insert(id, collector.clone_ref(py));
+    Ok(collector)
 }
 
-#[pymethods]
-impl PyProfilerCollector {
-    /// Show the profile results for a given profiler ID, or all profiles if 
ID is None.
-    ///
-    /// Returns a formatted string representation of the profile data.
-    #[pyo3(signature = (id=None))]
-    fn show(&self, id: Option<i64>) -> String {
-        self.inner.show(id)
-    }
-
-    /// Write profile results to a file.
-    ///
-    /// For a given profiler ID (or all profiles if ID is None), writes the 
profile data to the
-    /// specified file path.
-    #[pyo3(signature = (id=None, path="."))]
-    fn dump(&self, py: Python<'_>, id: Option<i64>, path: &str) -> 
PyResult<()> {
-        // Release the GIL during the file I/O operation
-        py.detach(|| self.inner.dump(id, path)).to_pyerr()?;
-        Ok(())
-    }
+/// Forget the profiles of a stopped session.
+pub(crate) fn forget_session(session: &SparkSession) {
+    registry()
+        .lock()
+        .unwrap_or_else(|e| e.into_inner())
+        .remove(session.session_id());
+}
 
-    /// Clear collected profile results for a given ID, or all profiles if ID 
is None.
-    #[pyo3(signature = (id=None))]
-    fn clear(&self, id: Option<i64>) {
-        self.inner.clear(id);
+/// Update `collector` with a pickled update of a Python accumulator, as 
PySpark Connect's
+/// client: `(id, update)` of `specialAccumulatorSer`, of which those of the 
UDF profiler
+/// (`SQL_UDF_PROFIER_V2`) update the collector, and the warning of one that 
does not unpickle.
+fn update_collector(py: Python<'_>, collector: &Bound<'_, PyAny>, update: 
&[u8]) -> PyResult<()> {
+    let accumulators = py.import("pyspark.accumulators")?;
+    let unpickled = accumulators
+        .getattr("specialAccumulatorSer")?
+        .call_method1("loads", (PyBytes::new(py, update),));
+    let unpickled = match unpickled {
+        Ok(unpickled) => unpickled,
+        Err(error) if error.is_instance(py, 
&py.import("pickle")?.getattr("UnpicklingError")?) => {
+            // As PySpark's: an unexpected class or function in the metric is 
ignored.
+            py.import("logging")?
+                .call_method1("getLogger", 
("pyspark.sql.connect.client.core",))?
+                .call_method1(
+                    "warning",
+                    (format!(
+                        "Error unpickling accumulator metric: {}",
+                        error.value(py)
+                    ),),
+                )?;
+            return Ok(());
+        }
+        Err(error) => return Err(error),
+    };
+    let (id, update): (Bound<'_, PyAny>, Bound<'_, PyAny>) = 
unpickled.extract()?;
+    let profiler_v2 = accumulators
+        .getattr("SpecialAccumulatorIds")?
+        .getattr("SQL_UDF_PROFIER_V2")?;
+    if id.eq(profiler_v2)? {
+        collector.call_method1("_update", (update,))?;
     }
+    Ok(())
 }
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index a9ef96d..8007ee5 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -12,7 +12,6 @@ use crate::catalog::PyCatalog;
 use crate::dataframe::PyDataFrame;
 use crate::datasource::PyDataSourceRegistration;
 use crate::errors::ResultExt;
-use crate::profiler::PyProfilerCollector;
 use crate::resource::PyResourceProfile;
 use crate::streaming::{PyDataStreamReader, PyStreamingQueryManager};
 
@@ -482,8 +481,14 @@ pub struct PySparkSession {
 
 impl PySparkSession {
     pub fn new(session: SparkSession) -> Self {
-        // The progress of the session, for the progress bars of its 
executions.
-        Python::attach(|py| crate::progress::session_progress(py, &session));
+        // The progress of the session, for the progress bars of its 
executions, and its
+        // profiles, which the updates of its first execution update too.
+        Python::attach(|py| {
+            crate::progress::session_progress(py, &session);
+            if let Err(error) = crate::profiler::profiler_collector(py, 
&session) {
+                error.write_unraisable(py, None);
+            }
+        });
         PySparkSession { session }
     }
 
@@ -1471,6 +1476,7 @@ impl PySparkSession {
     /// active session and the cached `SQLContext` if they belong to this 
session.
     fn stop(&self, py: Python<'_>) -> PyResult<()> {
         crate::progress::forget_session(&self.session);
+        crate::profiler::forget_session(&self.session);
         if let Err(e) = py.detach(|| self.session.stop()) {
             let logger = py
                 .import("logging")?
@@ -1753,13 +1759,22 @@ impl PySparkSession {
         crate::progress::session_progress(py, &self.session).clear(py)
     }
 
-    /// Get the profiler collector for this session.
-    ///
-    /// Mirrors `SparkSession.profile`. Profile data is accumulated across 
query
-    /// executions and can be shown, dumped, or cleared via the returned 
collector.
+    /// The UDF profiles of the session, as PySpark Connect's `spark.profile`: 
a `Profile` of its
+    /// `ConnectProfilerCollector`, of `show`, `dump`, `render` and `clear`.
     #[getter]
-    fn profile(&self) -> PyProfilerCollector {
-        PyProfilerCollector::new(self.session.profiler())
+    fn profile(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        let collector = crate::profiler::profiler_collector(py, 
&self.session)?;
+        Ok(py
+            .import("pyspark.sql.profiler")?
+            .getattr("Profile")?
+            .call1((collector,))?
+            .unbind())
+    }
+
+    /// The `ConnectProfilerCollector` of the session, as PySpark Connect's.
+    #[getter(_profiler_collector)]
+    fn profiler_collector(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+        crate::profiler::profiler_collector(py, &self.session)
     }
 
     /// Get the data source registration accessor.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index ea078c0..838789b 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -2258,8 +2258,8 @@ fn capture_execution(
     if !resp.observed_metrics.is_empty() {
         let metrics = std::mem::take(&mut resp.observed_metrics);
         set_observations(session, &metrics);
-        // Feed observed metrics to the profiler collector
-        session.profiler().accumulate_observed_metrics(&metrics);
+        // The updates of the Python accumulators, such as the results of the 
UDF profilers.
+        session.python_accumulators().accumulate(&metrics);
         info.observed_metrics.extend(metrics);
     }
 }
diff --git a/crates/spark-connect/src/lib.rs b/crates/spark-connect/src/lib.rs
index ad07a2d..c5a8b85 100644
--- a/crates/spark-connect/src/lib.rs
+++ b/crates/spark-connect/src/lib.rs
@@ -50,7 +50,6 @@ pub use expression::{
 pub use group::{CoGroupedData, GroupedData};
 pub use merge::MergeIntoWriter;
 pub use observation::Observation;
-pub use profiler::ProfilerCollector;
 pub use progress::{ProgressUpdate, StageInfo};
 pub use readwriter::{
     DataFrameReader, DataFrameWriter, DataFrameWriterV2, ReadType, SaveMode, 
TableSaveMethod,
diff --git a/crates/spark-connect/src/profiler.rs 
b/crates/spark-connect/src/profiler.rs
index 8957e67..06e7e91 100644
--- a/crates/spark-connect/src/profiler.rs
+++ b/crates/spark-connect/src/profiler.rs
@@ -1,436 +1,91 @@
-//! Client-side profiler collector for UDF and plan profiling results.
-//!
-//! Mirrors `pyspark.sql.connect.profiler.ConnectProfilerCollector`. 
Accumulates profile
-//! results observed across multiple `ExecutePlanResponse`s during the session 
and provides
-//! methods to show, dump, and clear profile data.
-//!
-//! Profile data is populated by the server only when UDF profiling is enabled 
via
-//! `spark.python.profile*` or `spark.sql.pyspark.udf.profiler` configuration.
+//! The updates of the Python accumulators of the executions of a session, as 
PySpark Connect's
+//! client receives them: the server sends each pickled `(id, update)` in the
+//! `__python_accumulator__` observed metrics of an execution, of which the 
UDF profilers
+//! (`spark.sql.pyspark.udf.profiler`) send their results. PySpark's client 
unpickles them with
+//! `specialAccumulatorSer` into its `ConnectProfilerCollector`, of 
`spark.profile`.
 
-use std::collections::HashMap;
-use std::sync::Mutex;
+use std::sync::{Arc, Mutex};
 
-use spark_connect_core::error::Result;
 use spark_connect_proto as proto;
 
-/// Format a protobuf Literal value as a string.
-pub fn format_literal(literal: &proto::expression::Literal) -> String {
-    use proto::expression::literal::LiteralType;
+/// The name of the observed metrics of the updates of the Python accumulators.
+pub const PYTHON_ACCUMULATOR_METRICS: &str = "__python_accumulator__";
 
-    match &literal.literal_type {
-        Some(LiteralType::Null(_)) => "null".to_string(),
-        Some(LiteralType::Boolean(b)) => b.to_string(),
-        Some(LiteralType::Byte(b)) => b.to_string(),
-        Some(LiteralType::Short(s)) => s.to_string(),
-        Some(LiteralType::Integer(i)) => i.to_string(),
-        Some(LiteralType::Long(l)) => l.to_string(),
-        Some(LiteralType::Float(f)) => f.to_string(),
-        Some(LiteralType::Double(d)) => d.to_string(),
-        Some(LiteralType::Decimal(_)) => "decimal".to_string(),
-        Some(LiteralType::String(s)) => s.clone(),
-        Some(LiteralType::Binary(b)) => format!("<binary: {} bytes>", b.len()),
-        Some(LiteralType::CalendarInterval(_)) => 
"calendar_interval".to_string(),
-        Some(LiteralType::YearMonthInterval(_)) => 
"year_month_interval".to_string(),
-        Some(LiteralType::DayTimeInterval(_)) => 
"day_time_interval".to_string(),
-        Some(LiteralType::Date(_)) => "date".to_string(),
-        Some(LiteralType::Timestamp(_)) => "timestamp".to_string(),
-        Some(LiteralType::TimestampNtz(_)) => "timestamp_ntz".to_string(),
-        Some(LiteralType::Time(_)) => "time".to_string(),
-        Some(LiteralType::TimestampNtzNanos(_)) => 
"timestamp_ntz_nanos".to_string(),
-        Some(LiteralType::TimestampLtzNanos(_)) => 
"timestamp_ltz_nanos".to_string(),
-        Some(LiteralType::Map(_)) => "map".to_string(),
-        Some(LiteralType::Array(_)) => "array".to_string(),
-        Some(LiteralType::Struct(_)) => "struct".to_string(),
-        Some(LiteralType::SpecializedArray(_)) => 
"specialized_array".to_string(),
-        None => "unknown".to_string(),
-    }
-}
+/// A handler of the pickled `(id, update)` of each update of the Python 
accumulators.
+pub type PythonAccumulatorHandler = Arc<dyn Fn(&[u8]) + Send + Sync>;
 
-/// Stores collected profile data for a single profiler ID.
-#[derive(Debug, Clone)]
-struct ProfileResult {
-    /// The accumulated profile data (typically a string representation).
-    pub data: String,
-    /// Metadata about the profile (e.g., profile type, timestamp). Collected 
from the
-    /// profiler responses (see `collect_profiles`) for parity with pyspark; 
retained for
-    /// a future `show()`/`dump()` that surfaces it, hence not read yet.
-    #[allow(dead_code)]
-    pub metadata: HashMap<String, String>,
+/// The handler of the updates of the Python accumulators of a session, shared 
by its clones.
+#[derive(Clone, Default)]
+pub(crate) struct PythonAccumulators {
+    handler: Arc<Mutex<Option<PythonAccumulatorHandler>>>,
 }
 
-/// Client-side collector for accumulated profiler results.
-///
-/// Accumulates UDF and plan profile results across multiple query executions
-/// and provides access via `show()`, `dump()`, and `clear()` methods.
-#[derive(Debug, Clone)]
-pub struct ProfilerCollector {
-    /// Profile results keyed by profiler ID.
-    profiles: std::sync::Arc<Mutex<HashMap<i64, ProfileResult>>>,
-}
-
-impl ProfilerCollector {
-    /// Create a new profiler collector.
-    pub fn new() -> Self {
-        ProfilerCollector {
-            profiles: std::sync::Arc::new(Mutex::new(HashMap::new())),
-        }
-    }
-
-    /// Accumulate a profile result. Called internally during query execution.
-    pub(crate) fn accumulate_profile(
-        &self,
-        id: i64,
-        data: String,
-        metadata: HashMap<String, String>,
-    ) {
-        let mut profiles = self.profiles.lock().unwrap();
-        profiles.insert(id, ProfileResult { data, metadata });
-    }
-
-    /// Accumulate observed metrics from a server response.
-    pub(crate) fn accumulate_observed_metrics(
-        &self,
-        metrics: &[proto::execute_plan_response::ObservedMetrics],
-    ) {
-        for metric in metrics {
-            // Use the metric name and plan_id as profile identifier
-            let id = metric.plan_id;
-            let mut data = String::new();
-            let mut metadata = HashMap::new();
-
-            // Pair up keys and values if available
-            let num_values = metric.values.len();
-            for i in 0..num_values {
-                let key = if i < metric.keys.len() {
-                    metric.keys[i].clone()
-                } else {
-                    format!("value_{}", i)
-                };
-
-                // Try to extract value as string from the Literal
-                let value_str = format_literal(&metric.values[i]);
-
-                metadata.insert(key.clone(), value_str.clone());
-                // Accumulate all metric values into data
-                if !data.is_empty() {
-                    data.push('\n');
-                }
-                data.push_str(&format!("{}: {}", key, value_str));
-            }
-
-            // Include the metric name and plan_id in the data
-            if !data.is_empty() {
-                data.insert_str(0, &format!("name: {}, plan_id: {}\n", 
metric.name, id));
-            }
-
-            // Store the profile result if we have any data or metadata
-            if !data.is_empty() || !metadata.is_empty() {
-                self.accumulate_profile(id, data, metadata);
-            }
-        }
+impl PythonAccumulators {
+    pub(crate) fn set_handler(&self, handler: PythonAccumulatorHandler) {
+        *self.handler.lock().unwrap() = Some(handler);
     }
 
-    /// Show the profile results for a given profiler ID, or all profiles if 
ID is None.
-    ///
-    /// Returns a formatted string representation of the profile data.
-    pub fn show(&self, id: Option<i64>) -> String {
-        let profiles = self.profiles.lock().unwrap();
-
-        match id {
-            Some(profile_id) => profiles
-                .get(&profile_id)
-                .map(|p| p.data.clone())
-                .unwrap_or_else(|| format!("No profile data for id: {}", 
profile_id)),
-            None => {
-                // Show all profiles
-                if profiles.is_empty() {
-                    "No profile data collected".to_string()
-                } else {
-                    let mut result = String::new();
-                    for (id, profile) in profiles.iter() {
-                        result.push_str(&format!("=== Profile {} ===\n", id));
-                        result.push_str(&profile.data);
-                        result.push('\n');
-                    }
-                    result
+    /// Hand each pickled update of the `__python_accumulator__` metrics of 
`metrics` to the
+    /// handler, as PySpark's client takes each of their values.
+    pub(crate) fn accumulate(&self, metrics: 
&[proto::execute_plan_response::ObservedMetrics]) {
+        let Some(handler) = self.handler.lock().unwrap().clone() else {
+            return;
+        };
+        for metric in metrics
+            .iter()
+            .filter(|metric| metric.name == PYTHON_ACCUMULATOR_METRICS)
+        {
+            for value in &metric.values {
+                if let 
Some(proto::expression::literal::LiteralType::Binary(update)) =
+                    &value.literal_type
+                {
+                    handler(update);
                 }
             }
         }
     }
-
-    /// Write profile results to a file.
-    ///
-    /// For a given profiler ID (or all profiles if ID is None), writes the 
profile data to the
-    /// specified file path.
-    pub fn dump(&self, id: Option<i64>, path: &str) -> Result<()> {
-        let data = self.show(id);
-
-        std::fs::write(path, data).map_err(|e| {
-            spark_connect_core::error::SparkError::connect_msg(format!(
-                "Failed to write profile to {}: {}",
-                path, e
-            ))
-        })?;
-
-        Ok(())
-    }
-
-    /// Clear collected profile results for a given ID, or all profiles if ID 
is None.
-    pub fn clear(&self, id: Option<i64>) {
-        let mut profiles = self.profiles.lock().unwrap();
-
-        match id {
-            Some(profile_id) => {
-                profiles.remove(&profile_id);
-            }
-            None => {
-                profiles.clear();
-            }
-        }
-    }
-
-    /// Get the number of collected profiles.
-    #[cfg(test)]
-    pub(crate) fn count(&self) -> usize {
-        self.profiles.lock().unwrap().len()
-    }
-}
-
-impl Default for ProfilerCollector {
-    fn default() -> Self {
-        Self::new()
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    #[test]
-    fn test_profiler_collector_accumulate_and_show() {
-        let collector = ProfilerCollector::new();
-
-        // Accumulate a profile
-        let mut metadata = HashMap::new();
-        metadata.insert("type".to_string(), "udf".to_string());
-        collector.accumulate_profile(1, "profile data 1".to_string(), 
metadata);
-
-        // Show profile by ID
-        let result = collector.show(Some(1));
-        assert_eq!(result, "profile data 1");
-
-        // Show all profiles
-        let all = collector.show(None);
-        assert!(all.contains("profile data 1"));
-        assert!(all.contains("=== Profile 1 ==="));
-    }
-
-    #[test]
-    fn test_profiler_collector_show_empty() {
-        let collector = ProfilerCollector::new();
-
-        let result = collector.show(None);
-        assert_eq!(result, "No profile data collected");
-
-        let result = collector.show(Some(99));
-        assert_eq!(result, "No profile data for id: 99");
-    }
-
-    #[test]
-    fn test_profiler_collector_clear() {
-        let collector = ProfilerCollector::new();
-
-        let metadata = HashMap::new();
-        collector.accumulate_profile(1, "profile 1".to_string(), 
metadata.clone());
-        collector.accumulate_profile(2, "profile 2".to_string(), metadata);
-
-        assert_eq!(collector.count(), 2);
-
-        // Clear one profile
-        collector.clear(Some(1));
-        assert_eq!(collector.count(), 1);
-
-        // Clear all
-        collector.clear(None);
-        assert_eq!(collector.count(), 0);
-    }
-
-    #[test]
-    fn test_profiler_collector_dump_and_read() {
-        let collector = ProfilerCollector::new();
-        let temp_dir = std::env::temp_dir();
-        let path = temp_dir.join("test_profile_dump.txt");
-
-        let metadata = HashMap::new();
-        collector.accumulate_profile(1, "test profile data".to_string(), 
metadata);
-
-        // Dump to file
-        let result = collector.dump(Some(1), path.to_str().unwrap());
-        assert!(result.is_ok());
-
-        // Read and verify
-        let contents = std::fs::read_to_string(&path).unwrap();
-        assert_eq!(contents, "test profile data");
-
-        // Cleanup
-        let _ = std::fs::remove_file(path);
-    }
-
-    #[test]
-    fn test_profiler_collector_accumulate_observed_metrics() {
-        let collector = ProfilerCollector::new();
-
-        // Create a mock observed metrics
-        let mut metric = 
proto::execute_plan_response::ObservedMetrics::default();
-        metric.name = "udf_profile".to_string();
-        metric.plan_id = 1;
-        metric.keys = vec!["udf_time_ms".to_string(), 
"python_calls".to_string()];
-
-        // Create literal values
-        metric.values.push(proto::expression::Literal {
-            literal_type: 
Some(proto::expression::literal::LiteralType::Long(150)),
-            data_type: None,
-        });
-        metric.values.push(proto::expression::Literal {
-            literal_type: 
Some(proto::expression::literal::LiteralType::Long(42)),
-            data_type: None,
-        });
-
-        collector.accumulate_observed_metrics(&[metric]);
-
-        // Verify the profile was accumulated
-        let result = collector.show(Some(1));
-        assert!(result.contains("udf_time_ms: 150"));
-        assert!(result.contains("python_calls: 42"));
-    }
-
-    #[test]
-    fn format_literal_covers_all_variants() {
-        use proto::expression::literal::{self, LiteralType};
-        let lit = |lt: LiteralType| proto::expression::Literal {
-            literal_type: Some(lt),
-            data_type: None,
-        };
-        assert_eq!(
-            
format_literal(&lit(LiteralType::Null(proto::DataType::default()))),
-            "null"
-        );
-        assert_eq!(format_literal(&lit(LiteralType::Boolean(true))), "true");
-        assert_eq!(format_literal(&lit(LiteralType::Byte(1))), "1");
-        assert_eq!(format_literal(&lit(LiteralType::Short(2))), "2");
-        assert_eq!(format_literal(&lit(LiteralType::Integer(3))), "3");
-        assert_eq!(format_literal(&lit(LiteralType::Long(4))), "4");
-        assert_eq!(format_literal(&lit(LiteralType::Float(1.5))), "1.5");
-        assert_eq!(format_literal(&lit(LiteralType::Double(2.5))), "2.5");
-        assert_eq!(
-            
format_literal(&lit(LiteralType::Decimal(literal::Decimal::default()))),
-            "decimal"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::String("hi".to_string()))),
-            "hi"
-        );
-        assert!(
-            format_literal(&lit(LiteralType::Binary(vec![1u8, 2, 
3].into()))).contains("3 bytes")
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::CalendarInterval(
-                literal::CalendarInterval::default()
-            ))),
-            "calendar_interval"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::YearMonthInterval(0))),
-            "year_month_interval"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::DayTimeInterval(0))),
-            "day_time_interval"
-        );
-        assert_eq!(format_literal(&lit(LiteralType::Date(0))), "date");
-        assert_eq!(format_literal(&lit(LiteralType::Timestamp(0))), 
"timestamp");
-        assert_eq!(
-            format_literal(&lit(LiteralType::TimestampNtz(0))),
-            "timestamp_ntz"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::Time(literal::Time::default()))),
-            "time"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::TimestampNtzNanos(
-                literal::TimestampNtzNanos::default()
-            ))),
-            "timestamp_ntz_nanos"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::TimestampLtzNanos(
-                literal::TimestampLtzNanos::default()
-            ))),
-            "timestamp_ltz_nanos"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::Map(literal::Map::default()))),
-            "map"
-        );
-        assert_eq!(
-            
format_literal(&lit(LiteralType::Array(literal::Array::default()))),
-            "array"
-        );
-        assert_eq!(
-            
format_literal(&lit(LiteralType::Struct(literal::Struct::default()))),
-            "struct"
-        );
-        assert_eq!(
-            format_literal(&lit(LiteralType::SpecializedArray(
-                literal::SpecializedArray::default()
-            ))),
-            "specialized_array"
-        );
-        // literal_type == None falls through to "unknown".
-        assert_eq!(
-            format_literal(&proto::expression::Literal {
-                literal_type: None,
-                data_type: None,
-            }),
-            "unknown"
-        );
-    }
-
-    #[test]
-    fn accumulate_observed_metrics_more_values_than_keys() {
-        // values longer than keys hits the `value_{i}` fallback key branch.
-        let collector = ProfilerCollector::new();
-        let mut metric = 
proto::execute_plan_response::ObservedMetrics::default();
-        metric.name = "m".to_string();
-        metric.plan_id = 7;
-        metric.keys = vec!["a".to_string()];
-        metric.values.push(proto::expression::Literal {
-            literal_type: 
Some(proto::expression::literal::LiteralType::Long(1)),
-            data_type: None,
-        });
-        metric.values.push(proto::expression::Literal {
-            literal_type: 
Some(proto::expression::literal::LiteralType::Long(2)),
-            data_type: None,
-        });
-        collector.accumulate_observed_metrics(&[metric]);
-        let shown = collector.show(Some(7));
-        assert!(shown.contains("a: 1"));
-        assert!(shown.contains("value_1: 2"));
+    fn metrics(
+        name: &str,
+        values: Vec<proto::expression::Literal>,
+    ) -> proto::execute_plan_response::ObservedMetrics {
+        proto::execute_plan_response::ObservedMetrics {
+            name: name.to_string(),
+            values,
+            ..Default::default()
+        }
     }
 
-    #[test]
-    fn dump_to_unwritable_path_errors() {
-        let collector = ProfilerCollector::new();
-        collector.accumulate_profile(1, "data".to_string(), HashMap::new());
-        // A path under a non-existent directory cannot be written -> Err 
branch.
-        let res = collector.dump(Some(1), 
"/nonexistent_dir_zzz_12345/profile.txt");
-        assert!(res.is_err());
+    fn binary(bytes: &[u8]) -> proto::expression::Literal {
+        proto::expression::Literal {
+            literal_type: Some(proto::expression::literal::LiteralType::Binary(
+                bytes.to_vec().into(),
+            )),
+            ..Default::default()
+        }
     }
 
+    /// The handler takes each binary value of the `__python_accumulator__` 
metrics, and none of
+    /// the metrics of observations.
     #[test]
-    fn default_constructs_empty_collector() {
-        let collector = ProfilerCollector::default();
-        assert_eq!(collector.count(), 0);
+    fn the_handler_takes_the_updates_of_the_python_accumulators() {
+        let accumulators = PythonAccumulators::default();
+        // Without a handler, the updates are not kept.
+        accumulators.accumulate(&[metrics(PYTHON_ACCUMULATOR_METRICS, 
vec![binary(b"a")])]);
+        let updates = Arc::new(Mutex::new(Vec::new()));
+        let seen = Arc::clone(&updates);
+        accumulators.set_handler(Arc::new(move |update: &[u8]| {
+            seen.lock().unwrap().push(update.to_vec());
+        }));
+        accumulators.accumulate(&[
+            metrics(PYTHON_ACCUMULATOR_METRICS, vec![binary(b"b"), 
binary(b"c")]),
+            metrics("observation", vec![binary(b"d")]),
+        ]);
+        assert_eq!(*updates.lock().unwrap(), vec![b"b".to_vec(), 
b"c".to_vec()]);
     }
 }
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index a938f37..2932827 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -13,7 +13,7 @@ use spark_connect_proto as proto;
 
 use crate::dataframe::DataFrame;
 use crate::plan::{LogicalPlan, PlanIds};
-use crate::profiler::ProfilerCollector;
+use crate::profiler::PythonAccumulators;
 use crate::progress::{ExecutionStream, ProgressTracker, ProgressUpdate};
 use crate::row::{Row, Value};
 use crate::types::DataType;
@@ -89,8 +89,8 @@ pub struct SparkSession {
     progress_handlers: Arc<Mutex<Vec<(u64, ProgressHandler)>>>,
     /// Monotonic id source for `register_progress_handler`.
     progress_handler_id: Arc<AtomicU64>,
-    /// Profiler collector for accumulating profile results across executions.
-    profiler: Arc<ProfilerCollector>,
+    /// The handler of the updates of the Python accumulators of its 
executions.
+    python_accumulators: PythonAccumulators,
     /// Whether `stop()` has been called on this session (shared across 
clones).
     /// Backs `is_stopped`.
     stopped: Arc<AtomicBool>,
@@ -108,7 +108,7 @@ impl SparkSession {
             last_execution: Arc::new(Mutex::new(None)),
             progress_handlers: Arc::new(Mutex::new(Vec::new())),
             progress_handler_id: Arc::new(AtomicU64::new(0)),
-            profiler: Arc::new(ProfilerCollector::new()),
+            python_accumulators: PythonAccumulators::default(),
             stopped: Arc::new(AtomicBool::new(false)),
             server_version: Arc::new(Mutex::new(None)),
         }
@@ -248,8 +248,8 @@ impl SparkSession {
     /// Raw observed metrics from the most recent execution only.
     ///
     /// This is a low-level snapshot of the last execution's observed metrics. 
For
-    /// the profiler surface that mirrors `SparkSession.profile` (results 
accumulated
-    /// across executions, with show/dump/clear), use 
[`SparkSession::profiler`].
+    /// the UDF profiles of `SparkSession.profile`, whose results the server 
sends as the updates
+    /// of Python accumulators, see 
[`SparkSession::set_python_accumulator_handler`].
     pub fn profile(&self) -> 
Vec<proto::execute_plan_response::ObservedMetrics> {
         self.last_execution_info()
             .map(|i| i.observed_metrics)
@@ -273,7 +273,7 @@ impl SparkSession {
             last_execution: Arc::new(Mutex::new(None)),
             progress_handlers: Arc::new(Mutex::new(Vec::new())),
             progress_handler_id: Arc::new(AtomicU64::new(0)),
-            profiler: Arc::new(ProfilerCollector::new()),
+            python_accumulators: PythonAccumulators::default(),
             stopped: Arc::new(AtomicBool::new(false)),
             server_version: Arc::new(Mutex::new(None)),
         }
@@ -944,14 +944,17 @@ impl SparkSession {
         )
     }
 
-    /// Get the profiler collector for this session.
-    ///
-    /// Mirrors the client-visible surface of `SparkSession.profile`. Profile 
data is
-    /// accumulated across query executions and can be shown, dumped, or 
cleared via the
-    /// returned collector. Profile data is populated by the server only when 
UDF profiling
-    /// is enabled via `spark.python.profile*` or 
`spark.sql.pyspark.udf.profiler` configuration.
-    pub fn profiler(&self) -> Arc<ProfilerCollector> {
-        Arc::clone(&self.profiler)
+    /// Set the handler of the updates of the Python accumulators of the 
executions of this
+    /// session: each pickled `(id, update)` of their `__python_accumulator__` 
observed metrics,
+    /// of which the UDF profilers send their results, as PySpark Connect's 
client unpickles each
+    /// into its `ConnectProfilerCollector`.
+    pub fn set_python_accumulator_handler(&self, handler: impl Fn(&[u8]) + 
Send + Sync + 'static) {
+        self.python_accumulators.set_handler(Arc::new(handler));
+    }
+
+    /// The handler of the updates of the Python accumulators, of the 
executions.
+    pub(crate) fn python_accumulators(&self) -> &PythonAccumulators {
+        &self.python_accumulators
     }
 
     /// Stop this Spark session.
@@ -981,7 +984,7 @@ impl Clone for SparkSession {
             last_execution: Arc::clone(&self.last_execution),
             progress_handlers: Arc::clone(&self.progress_handlers),
             progress_handler_id: Arc::clone(&self.progress_handler_id),
-            profiler: Arc::clone(&self.profiler),
+            python_accumulators: self.python_accumulators.clone(),
             stopped: Arc::clone(&self.stopped),
             server_version: Arc::clone(&self.server_version),
         }
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index ed61e12..bd342a1 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -743,7 +743,6 @@ fn session_extras_surface() {
     let s = session();
     let _ = s.session_id();
     let _ = s.profile();
-    let _ = s.profiler();
     let _ = s.read(); // batch DataFrameReader
     assert_eq!(s.empty_data_frame().unwrap().count().unwrap(), 0);
     let ns = s.new_session();
diff --git a/python/pyspark/sql/connect/profiler.py 
b/python/pyspark/sql/connect/profiler.py
new file mode 100644
index 0000000..8c14f6a
--- /dev/null
+++ b/python/pyspark/sql/connect/profiler.py
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import TYPE_CHECKING
+
+from pyspark.sql.profiler import ProfilerCollector, ProfileResultsParamV2
+
+if TYPE_CHECKING:
+    from pyspark.sql._typing import ProfileResultsV2
+
+
+class ConnectProfilerCollector(ProfilerCollector):
+    """
+    ProfilerCollector for Spark Connect.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._value = ProfileResultsParamV2.zero({})
+
+    @property
+    def _profile_results(self) -> "ProfileResultsV2":
+        with self._lock:
+            return self._value if self._value is not None else {}
+
+    def _update(self, update: "ProfileResultsV2") -> None:
+        with self._lock:
+            self._value = 
ProfileResultsParamV2.addInPlace(self._profile_results, update)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 5a17747..328547a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2487,6 +2487,31 @@ def 
test_stream_writer_starts_a_query_of_its_real_time_trigger_as_pyspark(sessio
         "STREAMING_REAL_TIME_MODE.OPERATOR_OR_SINK_NOT_IN_ALLOWLIST")
 
 
+def test_udf_profiles_are_shown_by_spark_profile_as_pyspark(session, capsys):
+    # As PySpark Connect's: with the UDF profiler, spark.profile shows the 
profile of each UDF
+    # that an execution ran, which clear removes.
+    from pyspark.sql.functions import udf
+
+    session.profile.clear()
+    session.conf.set("spark.sql.pyspark.udf.profiler", "perf")
+    try:
+        @udf("long")
+        def plus_one(x):
+            return x + 1
+
+        session.range(4).select(plus_one("id")).collect()
+    finally:
+        session.conf.unset("spark.sql.pyspark.udf.profiler")
+    capsys.readouterr()
+    session.profile.show(type="perf")
+    assert "Profile of UDF<id=" in capsys.readouterr().out
+    session.profile.clear()
+    session.profile.show()
+    assert capsys.readouterr().out == ""
+
+
+
+
 def test_create_dataframe_raises_merge_errors_as_pyspark(session):
     from pyspark.errors import PySparkException
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b7f41ee..4d9a75d 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7244,6 +7244,57 @@ def 
test_schema_cache_random_split_and_observe_reuse_warm_schema(mock_connect):
     assert service.calls == 1
 
 
+def test_udf_profiles_are_collected_from_the_python_accumulators_as_pyspark(
+        mock_connect, capsys, caplog):
+    # As PySpark Connect's: spark.profile is a Profile of a 
ConnectProfilerCollector, which the
+    # pickled updates of the UDF profiler, in the __python_accumulator__ 
observed metrics of an
+    # execution, update; the updates of other accumulators are ignored, and 
the warning of one
+    # that does not unpickle is logged.
+    import cProfile
+    import pstats
+
+    from pyspark.accumulators import SpecialAccumulatorIds, 
specialAccumulatorSer
+    from pyspark.sql.connect.profiler import ConnectProfilerCollector
+    from pyspark.sql.profiler import Profile
+
+    spark, service = mock_connect
+    profiler = cProfile.Profile()
+    profiler.enable()
+    sum(range(10))
+    profiler.disable()
+    # As the profiler of the Python worker sends its stats.
+    stats = pstats.Stats(profiler, stream=None)
+    stats.stream = None
+
+    def accumulator_metrics(*updates):
+        # ExecutePlanResponse.observed_metrics=6 -> ObservedMetrics name=1, 
values=2
+        # (Expression.Literal binary=2).
+        observed = message_field(1, "__python_accumulator__")
+        for update in updates:
+            observed += message_field(2, message_field(2, update))
+        return message_field(6, observed)
+
+    service.execute_payload = accumulator_metrics(
+        specialAccumulatorSer.dumps(
+            (SpecialAccumulatorIds.SQL_UDF_PROFIER_V2, {7: {"perf": stats}})),
+        specialAccumulatorSer.dumps((SpecialAccumulatorIds.SQL_UDF_PROFIER, 
{8: (stats, None)})),
+        b"not a pickle",
+    )
+    with caplog.at_level("WARNING"):
+        spark.range(1).collect()
+    assert isinstance(spark.profile, Profile)
+    assert isinstance(spark._profiler_collector, ConnectProfilerCollector)
+    assert list(spark._profiler_collector._profile_results) == [7]
+    assert "Error unpickling accumulator metric" in caplog.text
+    capsys.readouterr()
+    spark.profile.show(type="perf")
+    assert "Profile of UDF<id=7>" in capsys.readouterr().out
+    spark.profile.clear()
+    assert spark._profiler_collector._profile_results == {}
+
+
+
+
 def test_schema_cache_cold_children_keep_independent_caches(mock_connect):
     spark, service = mock_connect
     df = spark.range(10)
diff --git a/scripts/check_vendored_upstream.sh 
b/scripts/check_vendored_upstream.sh
index cfdf7ce..84cff3c 100644
--- a/scripts/check_vendored_upstream.sh
+++ b/scripts/check_vendored_upstream.sh
@@ -93,6 +93,9 @@ VENDORED_FILES=(
   "accumulators.py"
   "memory_profiler_ext.py"
   "profiler.py"
+  # The ConnectProfilerCollector of spark.profile, of the UDF profiles that 
the client takes
+  # from the updates of the Python accumulators of its executions.
+  "sql/connect/profiler.py"
   "worker_util.py"
   "instrumentation_utils.py"
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to