This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new e052b4e [SPARK-61013][CONNECT][PYTHON] Set the results of the
Observations of observe as PySpark does
e052b4e is described below
commit e052b4ece053265dcf9021d74e2276a26ea57646
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:04:04 2026 +0900
[SPARK-61013][CONNECT][PYTHON] Set the results of the Observations of
observe as PySpark does
### What changes were proposed in this pull request?
[SPARK-61013](https://issues.apache.org/jira/browse/SPARK-61013): the
`Observation`s of `df.observe` now receive their metrics, as in PySpark.
After `df.observe(o, count(lit(1)).alias("c")).collect()`, `o.get` is
`{'c': 3}`, as in PySpark. The same holds for `count()`, `toPandas()`,
`show()`, `toArrow()`, `toLocalIterator()` and writes, of the DataFrame or of a
DataFrame of it.
The server reports the metrics of each `CollectMetrics` relation by its
plan ID. So two Observations of the same name on two DataFrames each receive
their own. `get` raises the error of collecting the metrics, which the server
reports in its `ObservedMetrics`, as PySpark's `convert_observation_errors`
makes it.
The client also checks the arguments as PySpark does:
- An `Observation` is used once (`REUSE_OBSERVATION`).
- `get` raises `NO_OBSERVE_BEFORE_GET` before it is used, and returns `{}`
until an action runs.
- Its name, a random UUID by default, is a non-empty string
(`NOT_EXPECTED_TYPE`, `VALUE_NOT_NON_EMPTY_STR`).
- A streaming DataFrame is not observed (`UNSUPPORTED_OPERATION`);
`observe` asks the server whether the DataFrame is streaming, as
`df.isStreaming` does.
- `DataFrame.observe` raises `CANNOT_BE_EMPTY` without exprs, and
`NOT_EXPECTED_TYPE` for exprs that are not Columns and for an observation that
is neither an `Observation` nor a string.
The core `Observation` keeps the shared state of the observation, which
`DataFrame::observe_with` observes with, and its `get` returns the metrics.
### Why are the changes needed?
`Observation.get` was always `{}`: it never received the observed metrics,
and none of these errors were raised.
### Does this PR introduce _any_ user-facing change?
Yes. Observations receive their metrics, and raise the errors of PySpark
4.3.0.
### How was this patch tested?
- A new live test observes metrics through `collect`, a write and `count`,
of two Observations of one name, and an observation error. It passes with
official PySpark 4.3.0, and fails without this change.
- A new offline test checks the arguments and the errors of `Observation`
and `DataFrame.observe` against a mock Connect server.
- New lib tests check the observations that observed metrics set, by the
plan IDs of their relations, the single use of an observation, and the errors
of observations.
- The lib tests of both crates (574 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #242 from HyukjinKwon/SPARK-61013-observation.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 67 +++++++++---
crates/pyspark-rs/src/observation.rs | 90 ++++++++++++----
crates/spark-connect-core/src/error.rs | 74 +++++++++++++
crates/spark-connect/src/dataframe.rs | 147 ++++++++++++++++++++++++++
crates/spark-connect/src/observation.rs | 177 +++++++++++++++++++++++++++++---
crates/spark-connect/src/plan.rs | 17 ++-
python/tests/test_dropin_e2e.py | 34 ++++++
python/tests/test_dropin_offline.py | 39 +++++++
8 files changed, 594 insertions(+), 51 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index f6518ba..6c7061b 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1264,24 +1264,65 @@ impl PyDataFrame {
)
}
- /// Observe named metrics on the stream/df. `observation` is an Observation
- /// object or a name string; the remaining args are aggregate metric
columns.
- /// Mirrors `DataFrame.observe(observation, *exprs)`.
+ /// Observe metrics, as PySpark's `DataFrame.observe(observation,
*exprs)`: the Columns
+ /// `exprs`, which an Observation receives, or which are observed by a
name.
#[pyo3(signature = (observation, *exprs))]
fn observe(
&self,
+ py: Python<'_>,
observation: &Bound<'_, PyAny>,
exprs: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- let name =
- if let Ok(obs) =
observation.extract::<PyRef<crate::observation::PyObservation>>() {
- obs.inner.name().to_string()
- } else {
- observation.extract::<String>()?
- };
- let columns = to_column_list(exprs)?;
- let e: Vec<_> = columns.iter().map(|c|
c.expression().clone()).collect();
- Ok(PyDataFrame::new(self.dataframe.observe(&name, e)))
+ if exprs.is_empty() {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_BE_EMPTY",
+ &[("item", "exprs")],
+ ));
+ }
+ let mut columns = Vec::with_capacity(exprs.len());
+ for expr in &exprs {
+ match expr.extract::<PyRef<'_, PyColumn>>() {
+ Ok(column) => columns.push(column.column.expression().clone()),
+ Err(_) => {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", "exprs"),
+ ("expected_type", "list[Column]"),
+ ("arg_type", "tuple"),
+ ],
+ ))
+ }
+ }
+ }
+ if let Ok(observation) =
+ observation.extract::<PyRef<'_,
crate::observation::PyObservation>>()
+ {
+ let observation = &observation.inner;
+ return py
+ .detach(|| self.dataframe.observe_with(observation, columns))
+ .to_pyerr()
+ .map(PyDataFrame::new);
+ }
+ if observation.is_instance_of::<pyo3::types::PyString>() {
+ let name: String = observation.extract()?;
+ return Ok(PyDataFrame::new(self.dataframe.observe(&name,
columns)));
+ }
+ let type_name = observation.get_type().name()?.to_string();
+ Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", "observation"),
+ ("expected_type", "Observation or str"),
+ ("arg_type", &type_name),
+ ],
+ ))
}
/// Register as a temporary view (session-scoped).
@@ -3495,7 +3536,7 @@ fn execution_metrics<'py>(
/// `data_type` is the literal's type when the enclosing literal carries it; a
struct's
/// field names come from it, else from the literal's own `data_type`, else
from the
/// deprecated `struct_type`.
-fn literal_to_py(
+pub(crate) fn literal_to_py(
py: Python<'_>,
literal: &spark_connect_proto::expression::Literal,
data_type: Option<&spark_connect_proto::DataType>,
diff --git a/crates/pyspark-rs/src/observation.rs
b/crates/pyspark-rs/src/observation.rs
index 9ec3538..b7c058f 100644
--- a/crates/pyspark-rs/src/observation.rs
+++ b/crates/pyspark-rs/src/observation.rs
@@ -1,13 +1,14 @@
//! PyO3 wrapper for `spark_connect::observation::Observation`
(`pyspark.sql.Observation`).
-use std::sync::atomic::{AtomicU64, Ordering};
-
use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyString};
use spark_connect::observation::Observation;
-static OBS_COUNTER: AtomicU64 = AtomicU64::new(0);
+use crate::column::{raise_pyspark, PyColumn};
+use crate::dataframe::PyDataFrame;
+use crate::errors::ResultExt;
-/// Python wrapper for a named metric Observation.
+/// Python wrapper for an Observation of metrics, as PySpark Connect's
`Observation`.
#[pyclass(name = "Observation", module = "pyspark.sql.observation")]
pub struct PyObservation {
pub(crate) inner: Observation,
@@ -15,28 +16,77 @@ pub struct PyObservation {
#[pymethods]
impl PyObservation {
- /// `Observation(name=None)` - an unnamed observation gets an
auto-generated name,
- /// matching pyspark.
+ /// `Observation(name=None)`, as PySpark's: the name is a non-empty
string, and an
+ /// observation without one is of a random UUID once it is used.
#[new]
#[pyo3(signature = (name=None))]
- fn new(name: Option<String>) -> Self {
- let n = name.unwrap_or_else(|| {
- format!("observation_{}", OBS_COUNTER.fetch_add(1,
Ordering::SeqCst))
- });
- PyObservation {
- inner: Observation::new(&n),
- }
+ fn new(py: Python<'_>, name: Option<&Bound<'_, PyAny>>) -> PyResult<Self> {
+ let inner = match name {
+ None => Observation::unnamed(),
+ Some(name) if !name.is_instance_of::<PyString>() => {
+ let type_name = name.get_type().name()?.to_string();
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", "name"),
+ ("expected_type", "str"),
+ ("arg_type", &type_name),
+ ],
+ ));
+ }
+ Some(name) => {
+ let name: String = name.extract()?;
+ if name.is_empty() {
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "VALUE_NOT_NON_EMPTY_STR",
+ &[("arg_name", "name"), ("arg_value", &name)],
+ ));
+ }
+ Observation::new(&name)
+ }
+ };
+ Ok(PyObservation { inner })
}
- /// The observation name.
- #[getter]
- fn name(&self) -> String {
- self.inner.name().to_string()
+ /// The name of the observation, as PySpark's `_name`: None for an unnamed
observation
+ /// before it is used.
+ #[getter(_name)]
+ fn name(&self) -> Option<String> {
+ self.inner.name()
}
- /// The observed metrics (a dict), available after the observed action
runs.
+ /// The observed metrics, as PySpark's `Observation.get`: a dict of the
metrics that the
+ /// actions run so far observed, by their names, and empty before an
action runs.
#[getter]
- fn get(&self) -> std::collections::HashMap<String, String> {
- self.inner.get()
+ fn get<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyDict>> {
+ let metrics = self.inner.get().to_pyerr()?;
+ let result = PyDict::new(py);
+ for (name, value) in metrics {
+ result.set_item(name, crate::dataframe::literal_to_py(py, &value,
None)?)?;
+ }
+ Ok(result)
+ }
+
+ /// Observe the metrics of the Columns `exprs` on `df` with this
observation, as PySpark's
+ /// `Observation._on`, which `DataFrame.observe` of an Observation calls.
+ #[pyo3(signature = (df, *exprs))]
+ fn _on(
+ &self,
+ py: Python<'_>,
+ df: PyRef<'_, PyDataFrame>,
+ exprs: Vec<PyRef<'_, PyColumn>>,
+ ) -> PyResult<PyDataFrame> {
+ let exprs: Vec<_> = exprs
+ .iter()
+ .map(|c| c.column.expression().clone())
+ .collect();
+ let (dataframe, observation) = (df.dataframe.clone(), &self.inner);
+ py.detach(|| dataframe.observe_with(observation, exprs))
+ .to_pyerr()
+ .map(PyDataFrame::new)
}
}
diff --git a/crates/spark-connect-core/src/error.rs
b/crates/spark-connect-core/src/error.rs
index b14c1a2..791813a 100644
--- a/crates/spark-connect-core/src/error.rs
+++ b/crates/spark-connect-core/src/error.rs
@@ -465,6 +465,45 @@ impl SparkError {
err
}
+ /// The error of collecting the metrics of an Observation, which the
server reports in
+ /// its `ObservedMetrics`, as PySpark's `convert_observation_errors` makes
it: the root
+ /// error of `errors`, as a server error of the details of
`FetchErrorDetails`.
+ pub fn from_observation_errors(
+ root_error_idx: i32,
+ errors: Vec<spark_connect_proto::fetch_error_details_response::Error>,
+ ) -> Self {
+ let Some(root) = usize::try_from(root_error_idx)
+ .ok()
+ .and_then(|index| errors.get(index))
+ else {
+ return Self::connect_msg("Observation error: invalid
root_error_idx");
+ };
+ let throwable = root.spark_throwable.clone().unwrap_or_default();
+ let mut err = Self {
+ kind: SparkErrorKind::Unknown,
+ error_class: throwable.error_class.clone().unwrap_or_default(),
+ params: BTreeMap::new(),
+ message: root.message.clone(),
+ sql_state: throwable.sql_state.clone(),
+ contexts: Vec::new(),
+ server_stacktrace: None,
+ grpc_code: None,
+ grpc_details: None,
+ operation_id: None,
+ server: Some(Box::new(ServerErrorInfo {
+ classes: root.error_type_hierarchy.clone(),
+ ..Default::default()
+ })),
+ };
+ err.kind = err.server_kind();
+ let details = spark_connect_proto::FetchErrorDetailsResponse {
+ root_error_idx: Some(root_error_idx),
+ errors,
+ ..Default::default()
+ };
+ err.with_details(Some(details), false)
+ }
+
/// Completes an error that the server raised, as
/// `pyspark.errors.exceptions.connect._convert_exception` does: with the
error's details
/// as `FetchErrorDetails` returned them, if it did, and whether the
message shows the
@@ -945,6 +984,41 @@ mod tests {
assert_eq!(from_status.operation_id, None);
}
+ #[test]
+ fn observation_errors_are_their_root_errors_as_pyspark() {
+ use spark_connect_proto::fetch_error_details_response::{Error,
SparkThrowable};
+ let error = |message: &str, class: &str| Error {
+ error_type_hierarchy: vec![class.to_string(),
"java.lang.Throwable".to_string()],
+ message: message.to_string(),
+ spark_throwable: Some(SparkThrowable {
+ error_class: Some("USER_RAISED_EXCEPTION".to_string()),
+ sql_state: Some("P0001".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ let errors = vec![
+ error("cause", "java.lang.Exception"),
+ error(
+ "[USER_RAISED_EXCEPTION] test error",
+ "org.apache.spark.SparkRuntimeException",
+ ),
+ ];
+ let err = SparkError::from_observation_errors(1, errors.clone());
+ assert_eq!(err.kind, SparkErrorKind::SparkRuntime);
+ assert_eq!(
+ err.get_condition().as_deref(),
+ Some("USER_RAISED_EXCEPTION")
+ );
+ assert_eq!(err.get_sql_state().as_deref(), Some("P0001"));
+ assert!(err.message().contains("test error"));
+ // As PySpark's convert_observation_errors, of an index out of the
errors.
+ for index in [-1, 2] {
+ let err = SparkError::from_observation_errors(index,
errors.clone());
+ assert_eq!(err.message(), "Observation error: invalid
root_error_idx");
+ }
+ }
+
#[test]
fn test_render_simple_message() {
let mut params = BTreeMap::new();
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 7d32d57..55a3762 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1437,6 +1437,48 @@ impl DataFrame {
input: Arc::clone(&self.plan),
name: name.to_string(),
exprs,
+ observation: None,
+ };
+ self.with_cached_schema(plan)
+ }
+
+ /// Observe the metrics of `exprs` with `observation`, mirroring
+ /// `pyspark.sql.DataFrame.observe` of an `Observation`, as
`Observation._on` does: by
+ /// a `CollectMetrics` of the name of the observation, a random UUID for
an unnamed one,
+ /// whose metrics each action that runs the DataFrame sets in the
observation.
+ /// `REUSE_OBSERVATION` for an observation used already, and
`UNSUPPORTED_OPERATION` for a
+ /// streaming DataFrame.
+ pub fn observe_with(
+ &self,
+ observation: &crate::observation::Observation,
+ exprs: Vec<Expression>,
+ ) -> Result<DataFrame> {
+ let name = observation.name_to_observe()?;
+ if self.is_streaming()? {
+ return Err(SparkError::classed(
+ spark_connect_core::error::SparkErrorKind::IllegalArgument,
+ "UNSUPPORTED_OPERATION",
+ &[("operation", "Streaming DataFrame with Observation")],
+ ));
+ }
+ Ok(self.observed_with(observation, name, exprs))
+ }
+
+ /// The DataFrame of the `CollectMetrics` of `observation`, which starts,
of `exprs` by
+ /// `name`, as `observe_with` makes it once it has checked the observation
and the
+ /// DataFrame.
+ pub(crate) fn observed_with(
+ &self,
+ observation: &crate::observation::Observation,
+ name: String,
+ exprs: Vec<Expression>,
+ ) -> DataFrame {
+ observation.start();
+ let plan = LogicalPlan::Observe {
+ input: Arc::clone(&self.plan),
+ name,
+ exprs,
+ observation: Some(observation.clone()),
};
self.with_cached_schema(plan)
}
@@ -2210,6 +2252,7 @@ fn capture_execution(
}
if !resp.observed_metrics.is_empty() {
let metrics = std::mem::take(&mut resp.observed_metrics);
+ set_observations(session, &metrics);
// Feed observed metrics to the profiler collector
session.profiler().accumulate_observed_metrics(&metrics);
info.observed_metrics.extend(metrics);
@@ -2221,6 +2264,38 @@ fn capture_execution(
}
}
+/// Set the metrics that the server observed in the Observations of their
relations, as
+/// PySpark's client sets them in the Observations of the plan: those of each
relation, which
+/// the server reports by its plan ID, or the error of collecting them.
+fn set_observations(
+ session: &SparkSession,
+ metrics: &[proto::execute_plan_response::ObservedMetrics],
+) {
+ for observed in metrics {
+ let Some(node) = session.plan_ids().node_of(observed.plan_id) else {
+ continue;
+ };
+ let LogicalPlan::Observe {
+ name,
+ observation: Some(observation),
+ ..
+ } = node.as_ref()
+ else {
+ continue;
+ };
+ if *name != observed.name {
+ continue;
+ }
+ match observed.root_error_idx {
+ Some(root_error_idx) =>
observation.set_error(SparkError::from_observation_errors(
+ root_error_idx,
+ observed.errors.clone(),
+ )),
+ None => observation.update(&observed.keys, &observed.values),
+ }
+ }
+}
+
/// Decode an Arrow batch into rows.
pub(crate) fn decode_arrow_batch(
batch: &proto::execute_plan_response::ArrowBatch,
@@ -3857,6 +3932,78 @@ mod plan_construction_tests {
assert_eq!(joined.plan.to_proto(), expected);
}
+ #[test]
+ fn observed_metrics_set_the_observations_of_their_relations() {
+ use crate::observation::Observation;
+ use proto::expression::literal::LiteralType;
+ let spark = session();
+ let literal = |value: i64| proto::expression::Literal {
+ literal_type: Some(LiteralType::Long(value)),
+ ..Default::default()
+ };
+ // Two observations of one name, on two DataFrames, as PySpark allows.
+ let (first, second) = (Observation::new("m"), Observation::new("m"));
+ let count = || {
+ vec![crate::functions::count(crate::column::lit(1))
+ .expression()
+ .clone()]
+ };
+ // As observe_with, without its request of whether the DataFrame is
streaming.
+ let observe = |df: DataFrame, observation: &Observation| {
+ df.observed_with(observation,
observation.name_to_observe().unwrap(), count())
+ };
+ let df1 = observe(spark.range(3).unwrap(), &first);
+ let df2 = observe(spark.range(5).unwrap(), &second);
+ assert!(first.get().unwrap().is_empty() &&
second.get().unwrap().is_empty());
+ let err = df1.observe_with(&first, count()).err().unwrap();
+ assert_eq!(err.get_condition().as_deref(), Some("REUSE_OBSERVATION"));
+
+ let observed = |df: &DataFrame, value: i64|
proto::execute_plan_response::ObservedMetrics {
+ name: "m".to_string(),
+ values: vec![literal(value)],
+ keys: vec!["c".to_string()],
+ plan_id: spark.plan_ids().id_of(&df.plan),
+ ..Default::default()
+ };
+ let mut resp = proto::ExecutePlanResponse {
+ observed_metrics: vec![observed(&df1, 3)],
+ ..Default::default()
+ };
+ let mut info = ExecutionInfo::default();
+ capture_execution(&mut resp, &mut info, &spark);
+ let value = |obs: &Observation| -> Vec<(String, Option<LiteralType>)> {
+ obs.get()
+ .unwrap()
+ .into_iter()
+ .map(|(key, value)| (key, value.literal_type))
+ .collect()
+ };
+ assert_eq!(
+ value(&first),
+ vec![("c".to_string(), Some(LiteralType::Long(3)))]
+ );
+ assert!(value(&second).is_empty());
+ // The metrics of a relation by another name, or of no relation, set
nothing.
+ let mut other = observed(&df2, 5);
+ other.name = "other".to_string();
+ let mut unknown = observed(&df2, 5);
+ unknown.plan_id = -1;
+ resp.observed_metrics = vec![other, unknown];
+ capture_execution(&mut resp, &mut info, &spark);
+ assert!(value(&second).is_empty());
+ // An error of collecting the metrics is what get returns.
+ let mut failed = observed(&df2, 5);
+ failed.root_error_idx = Some(0);
+ resp.observed_metrics = vec![failed];
+ capture_execution(&mut resp, &mut info, &spark);
+ assert_eq!(
+ second.get().unwrap_err().message(),
+ "Observation error: invalid root_error_idx"
+ );
+ // That a streaming DataFrame is not observed, which the server tells,
the live tests
+ // check.
+ }
+
#[test]
fn derived_dataframes_keep_the_plan_ids_of_their_inputs() {
use crate::column::lit;
diff --git a/crates/spark-connect/src/observation.rs
b/crates/spark-connect/src/observation.rs
index 47691f6..96664bc 100644
--- a/crates/spark-connect/src/observation.rs
+++ b/crates/spark-connect/src/observation.rs
@@ -1,45 +1,188 @@
//! Observation implementation for collecting metrics.
//!
-//! Mirroring `pyspark.sql.observation.Observation`.
+//! Mirrors `pyspark.sql.connect.observation.Observation`.
-use std::collections::HashMap;
+use std::sync::{Arc, Mutex, MutexGuard};
-/// An Observation for collecting metrics from a DataFrame.
+use spark_connect_core::error::{Result, SparkError, SparkErrorKind};
+use spark_connect_proto as proto;
+
+/// Metrics that the server observes on a DataFrame, mirroring
`pyspark.sql.Observation`.
///
-/// Mirrors `pyspark.sql.observation.Observation`.
+/// [`crate::DataFrame::observe_with`] observes them, by a `CollectMetrics`
relation of the
+/// name of the observation. Each action that runs the DataFrame, or a
DataFrame of it, sets
+/// them as the server observed them, which [`Observation::get`] returns. A
clone shares the
+/// state of the observation.
+#[derive(Clone, Default)]
pub struct Observation {
- name: String,
- metrics: HashMap<String, String>,
+ state: Arc<Mutex<ObservationState>>,
+}
+
+#[derive(Default)]
+struct ObservationState {
+ name: Option<String>,
+ /// None before the observation is used; then the metrics observed so far,
in order.
+ result: Option<Vec<(String, proto::expression::Literal)>>,
+ /// The error of collecting the metrics, which the server reported.
+ error: Option<SparkError>,
+}
+
+impl std::fmt::Debug for Observation {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Observation")
+ .field("name", &self.lock().name)
+ .finish()
+ }
}
impl Observation {
- /// Create a new Observation with a name.
+ /// An observation of a name.
pub fn new(name: &str) -> Self {
Observation {
- name: name.to_string(),
- metrics: HashMap::new(),
+ state: Arc::new(Mutex::new(ObservationState {
+ name: Some(name.to_string()),
+ ..Default::default()
+ })),
+ }
+ }
+
+ /// An observation without a name, which is a random UUID once it is used,
as PySpark's
+ /// `Observation()`.
+ pub fn unnamed() -> Self {
+ Self::default()
+ }
+
+ /// The name of the observation; None for an unnamed observation before it
is used.
+ pub fn name(&self) -> Option<String> {
+ self.lock().name.clone()
+ }
+
+ /// The observed metrics, mirroring `Observation.get`: those that the
actions run so far
+ /// observed, by their names, in order, and none before an action runs.
The error of
+ /// collecting them, if the server reported one, and
`NO_OBSERVE_BEFORE_GET` before the
+ /// observation is used.
+ pub fn get(&self) -> Result<Vec<(String, proto::expression::Literal)>> {
+ let state = self.lock();
+ if let Some(error) = &state.error {
+ return Err(error.clone());
}
+ state.result.clone().ok_or_else(|| {
+ SparkError::classed(SparkErrorKind::AssertionError,
"NO_OBSERVE_BEFORE_GET", &[])
+ })
+ }
+
+ /// Whether `self` and `other` are the same observation, of which one is a
clone.
+ pub fn same(&self, other: &Observation) -> bool {
+ Arc::ptr_eq(&self.state, &other.state)
}
- /// Get the name of this Observation.
- pub fn name(&self) -> &str {
- &self.name
+ /// The name to observe the metrics by, as PySpark's `Observation._on`
before it
+ /// observes them: `REUSE_OBSERVATION` for an observation used already,
and the name,
+ /// a random UUID that it keeps for an unnamed observation.
+ pub(crate) fn name_to_observe(&self) -> Result<String> {
+ let mut state = self.lock();
+ if state.result.is_some() {
+ return Err(SparkError::classed(
+ SparkErrorKind::AssertionError,
+ "REUSE_OBSERVATION",
+ &[],
+ ));
+ }
+ Ok(state
+ .name
+ .get_or_insert_with(|| uuid::Uuid::new_v4().to_string())
+ .clone())
}
- /// Get metrics from this Observation.
- pub fn get(&self) -> HashMap<String, String> {
- self.metrics.clone()
+ /// Mark the observation used, with no metrics until an action observes
them.
+ pub(crate) fn start(&self) {
+ self.lock().result = Some(Vec::new());
+ }
+
+ /// Set the metrics that an action observed, as PySpark's client updates
the result of
+ /// the observation with them: a metric replaces the value of a metric of
its name.
+ pub(crate) fn update(&self, keys: &[String], values:
&[proto::expression::Literal]) {
+ let mut state = self.lock();
+ let Some(result) = state.result.as_mut() else {
+ return;
+ };
+ for (key, value) in keys.iter().zip(values) {
+ match result.iter_mut().find(|(name, _)| name == key) {
+ Some((_, existing)) => *existing = value.clone(),
+ None => result.push((key.clone(), value.clone())),
+ }
+ }
+ }
+
+ /// Set the error of collecting the metrics, which `get` returns.
+ pub(crate) fn set_error(&self, error: SparkError) {
+ self.lock().error = Some(error);
+ }
+
+ fn lock(&self) -> MutexGuard<'_, ObservationState> {
+ self.state.lock().unwrap_or_else(|e| e.into_inner())
}
}
#[cfg(test)]
mod tests {
use super::*;
+ use proto::expression::literal::LiteralType;
+
+ fn long(value: i64) -> proto::expression::Literal {
+ proto::expression::Literal {
+ literal_type: Some(LiteralType::Long(value)),
+ ..Default::default()
+ }
+ }
#[test]
fn test_observation_creation() {
let obs = Observation::new("test_obs");
- assert_eq!(obs.name(), "test_obs");
- assert!(obs.get().is_empty());
+ assert_eq!(obs.name().as_deref(), Some("test_obs"));
+ // As PySpark's: an observation that is not used has no metrics to get.
+ let err = obs.get().unwrap_err();
+ assert_eq!(
+ err.get_condition().as_deref(),
+ Some("NO_OBSERVE_BEFORE_GET")
+ );
+ }
+
+ #[test]
+ fn an_observation_is_used_once_and_updated_by_each_action() {
+ let obs = Observation::unnamed();
+ assert_eq!(obs.name(), None);
+ let name = obs.name_to_observe().unwrap();
+ assert_eq!(name.len(), 36);
+ assert_eq!(obs.name(), Some(name.clone()));
+ // Its name is kept until it is used.
+ assert_eq!(obs.name_to_observe().unwrap(), name);
+ obs.start();
+ assert!(obs.get().unwrap().is_empty());
+ let err = obs.name_to_observe().unwrap_err();
+ assert_eq!(err.get_condition().as_deref(), Some("REUSE_OBSERVATION"));
+
+ let clone = obs.clone();
+ assert!(clone.same(&obs) && !Observation::unnamed().same(&obs));
+ clone.update(&["a".to_string(), "b".to_string()], &[long(1), long(2)]);
+ obs.update(&["b".to_string(), "c".to_string()], &[long(3), long(4)]);
+ let names: Vec<(String, Option<LiteralType>)> = obs
+ .get()
+ .unwrap()
+ .into_iter()
+ .map(|(name, value)| (name, value.literal_type))
+ .collect();
+ assert_eq!(
+ names,
+ vec![
+ ("a".to_string(), Some(LiteralType::Long(1))),
+ ("b".to_string(), Some(LiteralType::Long(3))),
+ ("c".to_string(), Some(LiteralType::Long(4))),
+ ]
+ );
+
+ obs.set_error(SparkError::connect_msg("failed"));
+ assert!(obs.get().unwrap_err().message().contains("failed"));
+ assert!(format!("{obs:?}").contains(&name));
}
}
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index f90cd38..8f4ba30 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -325,6 +325,9 @@ pub enum LogicalPlan {
input: Arc<LogicalPlan>,
name: String,
exprs: Vec<Expression>,
+ /// The Observation that the metrics that the server observes for this
relation set,
+ /// as PySpark's `CollectMetrics` of an `Observation`.
+ observation: Option<crate::observation::Observation>,
},
/// UnresolvedTableValuedFunction: a TVF like explode, inline, range.
UnresolvedTableValuedFunction {
@@ -506,6 +509,16 @@ impl PlanIds {
id
}
+ /// The live node of the plan ID `id`, as the server reports a relation by
its plan ID.
+ pub(crate) fn node_of(&self, id: i64) -> Option<Arc<LogicalPlan>> {
+ let nodes = self.nodes.lock().unwrap_or_else(|e| e.into_inner());
+ nodes
+ .ids
+ .values()
+ .find(|(_, node_id)| *node_id == id)
+ .and_then(|(node, _)| node.upgrade())
+ }
+
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.nodes.lock().unwrap().ids.len()
@@ -1379,7 +1392,9 @@ impl LogicalPlan {
relation.rel_type =
Some(proto::relation::RelType::SampleBy(Box::new(stat)));
}
- LogicalPlan::Observe { input, name, exprs } => {
+ LogicalPlan::Observe {
+ input, name, exprs, ..
+ } => {
let mut collect_metrics = proto::CollectMetrics::default();
collect_metrics.input = Some(Box::new(relation_of(input,
plan_ids)));
collect_metrics.name = name.clone();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 710e58f..2b27cee 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1492,6 +1492,40 @@ def
test_decimal_literals_are_of_the_types_of_pyspark(session):
)
+def test_observations_receive_the_observed_metrics_as_pyspark(session):
+ # As PySpark's: each action that runs a DataFrame of an observation sets
in it the metrics
+ # that the server observed, for each observation of a name on its own, and
get raises the
+ # error of collecting them.
+ from pyspark.errors import PySparkException
+ from pyspark.sql import Observation
+
+ first, second = Observation("m"), Observation("m")
+ df1 = session.range(3).observe(
+ first, F.count(F.lit(1)).alias("c"), F.sum("id").alias("s"),
F.lit(None).alias("n")
+ )
+ df2 = session.range(5).observe(second, F.count(F.lit(1)).alias("c"))
+ assert first.get == {} and second.get == {}
+ assert df1.collect() == [Row(id=0), Row(id=1), Row(id=2)]
+ assert first.get == {"c": 3, "s": 3, "n": None} and second.get == {}
+ df2.select("*").write.format("noop").mode("overwrite").save()
+ assert second.get == {"c": 5}
+ assert df1.where("id > 0").count() == 2 and first.get["c"] == 3
+
+ failing = Observation("failing")
+ df = session.range(2).observe(failing, F.raise_error(F.lit("test
error")).alias("e"))
+ assert df.collect() == [Row(id=0), Row(id=1)]
+ with pytest.raises(PySparkException, match="test error"):
+ failing.get
+
+ # A streaming DataFrame, which the server tells, is not observed.
+ from pyspark.errors import IllegalArgumentException
+
+ stream = session.readStream.format("rate").load()
+ with pytest.raises(IllegalArgumentException) as error:
+ stream.observe(Observation(), F.count(F.lit(1)).alias("c"))
+ assert error.value.getCondition() == "UNSUPPORTED_OPERATION"
+
+
def test_ml_vector_functions_as_pyspark(session):
from pyspark.ml.functions import array_to_vector, vector_to_array
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index af59898..5da39f8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4211,6 +4211,45 @@ def test_row_hash_matches_tuple():
assert hash(Row(a=Row(b=1))) == hash(((1,),))
+def test_observations_check_their_arguments_as_pyspark(mock_connect):
+ # As PySpark's Observation and DataFrame.observe, which check their
arguments on the
+ # client, and ask the server whether the DataFrame is streaming, as
`df.isStreaming`.
+ from pyspark.errors import PySparkAssertionError, PySparkTypeError,
PySparkValueError
+ from pyspark.sql import Observation
+ from pyspark.sql import functions as F
+
+ spark, _ = mock_connect
+ count = F.count(F.lit(1)).alias("c")
+ df = spark.range(1)
+ for call, error_type, condition, parameters in (
+ (lambda: Observation(1), PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": "name", "expected_type": "str", "arg_type": "int"}),
+ (lambda: Observation(""), PySparkValueError, "VALUE_NOT_NON_EMPTY_STR",
+ {"arg_name": "name", "arg_value": ""}),
+ (lambda: Observation("o").get, PySparkAssertionError,
"NO_OBSERVE_BEFORE_GET", {}),
+ (lambda: df.observe(Observation("o")), PySparkValueError,
"CANNOT_BE_EMPTY",
+ {"item": "exprs"}),
+ (lambda: df.observe("o", "id"), PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": "exprs", "expected_type": "list[Column]", "arg_type":
"tuple"}),
+ (lambda: df.observe(1, count), PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": "observation", "expected_type": "Observation or str",
"arg_type": "int"}),
+ ):
+ with pytest.raises(error_type) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+ # An unnamed observation is of a random UUID once it is used, and is used
once; it has no
+ # metrics until an action runs.
+ observation = Observation()
+ assert observation._name is None
+ df.observe(observation, count)
+ assert len(observation._name) == 36 and observation.get == {}
+ for observe in (lambda: df.observe(observation, count), lambda:
observation._on(df, count)):
+ with pytest.raises(PySparkAssertionError) as error:
+ observe()
+ assert error.value.getCondition() == "REUSE_OBSERVATION"
+
+
def test_row_usable_in_set_and_dict():
"""Equal Rows collapse in a set and work as dict keys."""
from pyspark.sql import Row
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]