This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 617fa14 [SPARK-60677][CONNECT] Cache and propagate DataFrame schemas
617fa14 is described below
commit 617fa1481b3e6fc7b825928130f280506328c9b1
Author: DB Tsai <[email protected]>
AuthorDate: Fri Oct 30 09:28:51 2026 -0700
[SPARK-60677][CONNECT] Cache and propagate DataFrame schemas
### What changes were proposed in this pull request?
Cache resolved schemas across metadata reads and exact clones; failures
remain retryable. Propagate snapshots through selected schema-preserving
transformations, following Spark 4.2; set operations require exactly equal
schemas from the same session. `to(schema)` and stateless partition/group
pandas/Arrow UDFs use declared struct schemas.
Propagation never analyzes parents or waits for in-flight analysis. Other
plans resolve independently. Python metadata waits release the GIL, and the
shared cache recovers defensively after Rust panics.
### Why are the changes needed?
Avoid redundant analysis RPCs while allowing Python threads to progress
during metadata waits.
### Does this PR introduce _any_ user-facing change?
Yes. Metadata reads reuse resolved schemas. Concurrent readers share one
analysis request; a stalled request blocks those readers. External table/view
changes can leave cached metadata stale; recreate the DataFrame from its source
to refresh it.
### How was this patch tested?
- 398 Rust library tests and 318 CI-selected offline Python tests passed.
Regression checks reject the prior build and a blocking propagation mutation.
- 19 live cases matched reference PySpark on Spark 4.2, comparing schemas,
metadata, columns, dtypes, and collected rows.
- Release/WASM integration compilation, formatting, and wheel build passed.
Advisory Clippy reports the same 923 diagnostics as the base.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #120 from dbtsai/codex/spark-60677-cache-schema.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: DB Tsai <[email protected]>
Co-authored-by: Isaac <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 28 ++-
crates/spark-connect/src/dataframe.rs | 342 +++++++++++++++++++++++++++++-----
crates/spark-connect/src/group.rs | 44 ++++-
python/tests/_mock_connect_server.py | 160 ++++++++++++++++
python/tests/test_dropin_offline.py | 332 ++++++++++++++++++++++++++++++++-
5 files changed, 838 insertions(+), 68 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 5ae82b0..d74891d 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1298,8 +1298,8 @@ impl PyDataFrame {
/// Column names (a property, mirroring `DataFrame.columns`).
#[getter]
- fn columns(&self) -> PyResult<Vec<String>> {
- self.dataframe.columns().to_pyerr()
+ fn columns(&self, py: Python<'_>) -> PyResult<Vec<String>> {
+ py.detach(|| self.dataframe.columns().to_pyerr())
}
/// Get the last `num` rows as a list of Rows (eager, like pyspark `tail`).
@@ -1370,20 +1370,16 @@ impl PyDataFrame {
/// Get the schema as a list of (name, type) tuples.
#[getter]
- fn dtypes(&self) -> PyResult<Vec<(String, String)>> {
- let schema = self.dataframe.schema().to_pyerr()?;
- match schema {
- spark_connect::types::DataType::Struct { fields } => {
- let dtypes = fields
- .iter()
- .map(|f| (f.name.clone(), f.data_type.simple_string()))
- .collect();
- Ok(dtypes)
- }
- _ => Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "Schema is not a struct type",
- )),
- }
+ fn dtypes(&self, py: Python<'_>) -> PyResult<Vec<(String, String)>> {
+ let spark_connect::types::DataType::Struct { fields } =
+ py.detach(|| self.dataframe.schema().to_pyerr())?
+ else {
+ unreachable!("schema() validates struct responses")
+ };
+ Ok(fields
+ .into_iter()
+ .map(|field| (field.name, field.data_type.simple_string()))
+ .collect())
}
/// Additional set operation: unionAll (alias for union).
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index ea0e2c6..4ce3898 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -3,7 +3,7 @@
//! Provides transformations and actions for working with distributed data.
use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use spark_connect_core::client::ReattachableResponseStream;
use spark_connect_core::error::{Result, SparkError};
@@ -26,6 +26,10 @@ use crate::udf::CommonInlineUserDefinedFunctionExpression;
pub struct DataFrame {
pub(crate) session: SparkSession,
pub(crate) plan: Arc<LogicalPlan>,
+ // Exact clones share this cache. Derived plans can snapshot an already
+ // resolved schema, but have their own cache for subsequent analysis.
+ // The resolved value is immutable, so propagation need not copy wide
schemas.
+ schema_cache: Arc<Mutex<Option<Arc<DataType>>>>,
}
/// Iterator over rows from a DataFrame, yielded lazily as the server streams
results.
@@ -203,10 +207,61 @@ fn spawn_prefetch(
impl DataFrame {
/// Create a new DataFrame.
pub(crate) fn new(session: SparkSession, plan: LogicalPlan) -> Self {
+ Self::new_with_schema(session, plan, None)
+ }
+
+ /// Seed declared struct output schemas without analyzing the plan.
+ pub(crate) fn new_with_schema(
+ session: SparkSession,
+ plan: LogicalPlan,
+ schema: Option<DataType>,
+ ) -> Self {
+ Self::new_with_cached_schema(
+ session,
+ plan,
+ schema
+ .filter(|schema| matches!(schema, DataType::Struct { .. }))
+ .map(Arc::new),
+ )
+ }
+
+ fn new_with_cached_schema(
+ session: SparkSession,
+ plan: LogicalPlan,
+ schema: Option<Arc<DataType>>,
+ ) -> Self {
DataFrame {
session,
plan: Arc::new(plan),
+ schema_cache: Arc::new(Mutex::new(schema)),
+ }
+ }
+
+ fn cached_schema(&self) -> Option<Arc<DataType>> {
+ // Propagation snapshots resolved metadata; it must not wait for an
+ // in-flight analysis. Python transformations can still hold the GIL.
+ self.schema_cache.try_lock().ok()?.clone()
+ }
+
+ /// Selected schema-preserving transformations snapshot the resolved
schema,
+ /// mirroring Spark 4.2 `_cached_schema` propagation. Other plans resolve
independently.
+ fn with_cached_schema(&self, plan: LogicalPlan) -> Self {
+ Self::new_with_cached_schema(self.session.clone(), plan,
self.cached_schema())
+ }
+
+ /// Set operations propagate only exactly equal cached schemas from the
same session.
+ fn with_merged_schema(&self, other: &DataFrame, plan: LogicalPlan) -> Self
{
+ // A right-hand plan from another session is analyzed in our session;
+ // its previously resolved schema is not evidence about that
resolution.
+ if self.session.client().session_id() !=
other.session.client().session_id() {
+ return Self::new(self.session.clone(), plan);
}
+ // Snapshot separately, including for self-unions; never lock both
inputs.
+ let schema = match (self.cached_schema(), other.cached_schema()) {
+ (Some(left), Some(right)) if left == right => Some(left),
+ _ => None,
+ };
+ Self::new_with_cached_schema(self.session.clone(), plan, schema)
}
/// Get the underlying logical plan.
@@ -231,7 +286,7 @@ impl DataFrame {
input: Arc::clone(&self.plan),
condition,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Alias for filter().
@@ -300,7 +355,7 @@ impl DataFrame {
input: Arc::clone(&self.plan),
limit: n,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Skip the first n rows.
@@ -329,7 +384,7 @@ impl DataFrame {
column_names: vec![],
within_watermark: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Remove duplicate rows, optionally on specific columns.
@@ -345,7 +400,7 @@ impl DataFrame {
column_names: cols,
within_watermark: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Sort rows.
@@ -355,7 +410,7 @@ impl DataFrame {
order: columns,
is_global: true,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Alias for sort().
@@ -451,7 +506,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Union by name.
@@ -471,7 +526,7 @@ impl DataFrame {
by_name: true,
allow_missing_columns,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Intersect with another DataFrame.
@@ -484,7 +539,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Subtract (except) another DataFrame.
@@ -497,7 +552,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Repartition.
@@ -507,7 +562,7 @@ impl DataFrame {
num_partitions,
shuffle: true,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Coalesce.
@@ -517,7 +572,7 @@ impl DataFrame {
num_partitions,
shuffle: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Add a hint.
@@ -531,7 +586,7 @@ impl DataFrame {
name: name.to_string(),
parameters: parameters.into_iter().map(Into::into).collect(),
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Marks a DataFrame as eligible for broadcast join (smaller table).
@@ -556,7 +611,7 @@ impl DataFrame {
input: Arc::clone(&self.plan),
alias: alias.to_string(),
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Map over each partition with a pandas UDF (`DataFrame.mapInPandas`).
@@ -586,12 +641,13 @@ impl DataFrame {
func: CommonInlineUserDefinedFunctionExpression,
is_barrier: bool,
) -> DataFrame {
+ let schema = func.python_udf.output_type.clone();
let plan = LogicalPlan::MapPartitions {
input: Arc::clone(&self.plan),
func,
is_barrier,
};
- DataFrame::new(self.session.clone(), plan)
+ Self::new_with_schema(self.session.clone(), plan, Some(schema))
}
/// Apply a function to each row for its side effects
(`DataFrame.foreach`).
@@ -629,7 +685,7 @@ impl DataFrame {
with_replacement,
seed,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Group by columns for aggregation.
@@ -769,14 +825,48 @@ impl DataFrame {
}
/// Get the schema of this DataFrame.
+ ///
+ /// The first successful lookup is cached and shared with exact clones.
+ /// Concurrent lookups on the same DataFrame or its clones wait for a
single
+ /// analysis RPC, so a stalled analysis blocks those metadata readers.
+ /// Changes to external table or view metadata do not invalidate this
cache;
+ /// recreate the DataFrame from its source to resolve the schema again.
+ /// Each call returns an independent value, and failed lookups can be
retried.
pub fn schema(&self) -> Result<DataType> {
+ // Keep the lock during analysis so concurrent accesses issue only one
+ // successful RPC. Errors leave the cache empty and can be retried.
+ let mut cached = self
+ .schema_cache
+ .lock()
+ // Unlike the other cache locks, this guard spans a blocking RPC.
+ // Recover defensively from Rust callers that panic during analysis
+ // (for example, by calling the sync API inside a Tokio runtime).
+ // The slot contains only None or a complete schema, so it is safe.
+ .unwrap_or_else(|poisoned| {
+ self.schema_cache.clear_poison();
+ poisoned.into_inner()
+ });
+ if let Some(schema) = cached.clone() {
+ // Copy the schema after unlocking so propagation can snapshot it.
+ drop(cached);
+ return Ok(schema.as_ref().clone());
+ }
let request = self.build_analyze_request()?;
let response = block_on(self.session.client().analyze_plan(request))?;
if let Some(proto::analyze_plan_response::Result::Schema(schema)) =
response.result {
- Ok(DataType::from_proto(&schema.schema.ok_or_else(|| {
- SparkError::connect_msg("Schema is missing")
- })?)?)
+ let schema = DataType::from_proto(
+ &schema
+ .schema
+ .ok_or_else(|| SparkError::connect_msg("Schema is
missing"))?,
+ )?;
+ if !matches!(&schema, DataType::Struct { .. }) {
+ return Err(SparkError::connect_msg("Schema is not a struct
type"));
+ }
+ let schema = Arc::new(schema);
+ *cached = Some(Arc::clone(&schema));
+ drop(cached);
+ Ok(schema.as_ref().clone())
} else {
Err(SparkError::connect_msg(
"Schema analyze failed: no schema in response",
@@ -806,11 +896,10 @@ impl DataFrame {
/// Get column names.
pub fn columns(&self) -> Result<Vec<String>> {
- let schema = self.schema()?;
- match schema {
- DataType::Struct { fields } => Ok(fields.iter().map(|f|
f.name.clone()).collect()),
- _ => Err(SparkError::connect_msg("Schema is not a struct type")),
- }
+ let DataType::Struct { fields } = self.schema()? else {
+ unreachable!("schema() validates struct responses")
+ };
+ Ok(fields.into_iter().map(|field| field.name).collect())
}
/// Build an ExecutePlanRequest from the logical plan.
@@ -1089,7 +1178,7 @@ impl DataFrame {
num_partitions: Some(num_partitions),
partition_exprs: columns,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Repartition into `num_partitions` by hashing the given column
expressions.
@@ -1104,7 +1193,7 @@ impl DataFrame {
num_partitions,
expressions: columns,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Alias for to_df().
@@ -1161,7 +1250,7 @@ impl DataFrame {
name: name.to_string(),
exprs,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Get stat functions.
@@ -1356,7 +1445,7 @@ impl DataFrame {
order: columns,
is_global: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_cached_schema(plan)
}
/// Drop duplicates within a watermark.
@@ -1400,7 +1489,7 @@ impl DataFrame {
with_replacement: false,
seed,
};
- results.push(DataFrame::new(self.session.clone(), plan));
+ results.push(self.with_cached_schema(plan));
cumulative = upper;
}
@@ -1441,17 +1530,13 @@ impl DataFrame {
/// Get dtypes (column names and types).
pub fn dtypes(&self) -> Result<Vec<(String, String)>> {
- let schema = self.schema()?;
- match schema {
- DataType::Struct { fields } => {
- let dtypes = fields
- .iter()
- .map(|f| (f.name.clone(), f.data_type.to_string()))
- .collect();
- Ok(dtypes)
- }
- _ => Err(SparkError::connect_msg("Schema is not a struct type")),
- }
+ let DataType::Struct { fields } = self.schema()? else {
+ unreachable!("schema() validates struct responses")
+ };
+ Ok(fields
+ .into_iter()
+ .map(|field| (field.name, field.data_type.to_string()))
+ .collect())
}
/// Compute the server-side semantic hash of this DataFrame's logical plan,
@@ -1539,7 +1624,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Except all rows.
@@ -1552,7 +1637,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Intersect all rows.
@@ -1565,7 +1650,7 @@ impl DataFrame {
by_name: false,
allow_missing_columns: false,
};
- DataFrame::new(self.session.clone(), plan)
+ self.with_merged_schema(other, plan)
}
/// Unpivot columns (like melt).
@@ -1699,9 +1784,9 @@ impl DataFrame {
pub fn to(&self, schema: DataType) -> DataFrame {
let plan = LogicalPlan::ToSchema {
input: Arc::clone(&self.plan),
- schema,
+ schema: schema.clone(),
};
- DataFrame::new(self.session.clone(), plan)
+ Self::new_with_schema(self.session.clone(), plan, Some(schema))
}
/// Check if the DataFrame exists (is not empty).
@@ -1746,10 +1831,7 @@ impl DataFrame {
left: Arc::clone(&self.plan),
right: Arc::clone(&other.plan),
};
- Ok(DataFrame {
- plan: Arc::new(plan),
- session: self.session.clone(),
- })
+ Ok(DataFrame::new(self.session.clone(), plan))
}
/// Register this DataFrame as a temporary table (deprecated - use
createTempView).
@@ -2850,6 +2932,168 @@ mod cache_tests {
use super::*;
use prost::Message;
+ #[test]
+ fn schema_cache_shares_exact_clones_but_not_changed_plans_or_new_sources()
{
+ let spark = SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .unwrap();
+ let df = spark.range(3).unwrap();
+ let cloned_before_analysis = df.clone();
+ let schema = DataType::Struct { fields: vec![] };
+ *df.schema_cache.lock().unwrap() = Some(Arc::new(schema.clone()));
+
+ // These accesses succeed without a server, including a clone created
+ // before the schema was populated.
+ assert_eq!(cloned_before_analysis.schema().unwrap(), schema);
+ assert_eq!(df.clone().schema().unwrap(), schema);
+ assert!(Arc::ptr_eq(
+ &df.schema_cache,
+ &cloned_before_analysis.schema_cache
+ ));
+ for derived in [
+ df.select(["id"]),
+ df.zip(&df).unwrap(),
+ spark.range(3).unwrap(),
+ ] {
+ assert!(!Arc::ptr_eq(&df.schema_cache, &derived.schema_cache));
+ assert!(derived.schema_cache.lock().unwrap().is_none());
+ }
+ }
+
+ #[test]
+ fn schema_propagation_snapshots_without_waiting_for_analysis() {
+ let spark = SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .unwrap();
+ let df = spark.range(3).unwrap();
+ let early_child = df.limit(1);
+ let schema = DataType::Struct { fields: vec![] };
+ *df.schema_cache.lock().unwrap() = Some(Arc::new(schema.clone()));
+ let warm_child = df.limit(1);
+ assert_eq!(warm_child.schema().unwrap(), schema);
+ assert!(!Arc::ptr_eq(&df.schema_cache, &warm_child.schema_cache));
+ assert!(early_child.cached_schema().is_none());
+
+ // Even when an input lock is held, constructing unary and binary plans
+ // must finish. Release it before joining so a regression fails
boundedly.
+ let guard = df.schema_cache.lock().unwrap();
+ let input = df.clone();
+ let (tx, rx) = std::sync::mpsc::channel();
+ let worker = std::thread::spawn(move || {
+ tx.send((input.limit(1), input.union(&input))).unwrap();
+ });
+ let result = rx.recv_timeout(std::time::Duration::from_secs(5));
+ drop(guard);
+ worker.join().unwrap();
+ let (unary, binary) = result.expect("plan construction must not wait
for schema analysis");
+ assert!(unary.cached_schema().is_none());
+ assert!(binary.cached_schema().is_none());
+ }
+
+ #[test]
+ fn schema_propagation_merges_only_exact_schemas_from_the_same_session() {
+ let spark = SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .unwrap();
+ let df = spark.range(3).unwrap();
+ let schema = serde_json::json!({
+ "type": "struct",
+ "fields": [{"name": "id", "type": "long", "nullable": true,
+ "metadata": {"note": "original"}}]
+ });
+ let left = df.to(DataType::from_json(&schema).unwrap());
+ assert_eq!(left.union(&left).schema().unwrap(),
left.schema().unwrap());
+ let mut changes = vec![];
+ for (key, value) in [
+ ("name", serde_json::json!("renamed")),
+ ("type", serde_json::json!("double")),
+ ("nullable", serde_json::json!(false)),
+ ("metadata", serde_json::json!({"note": "changed"})),
+ ] {
+ let mut changed = schema.clone();
+ changed["fields"][0][key] = value;
+ changes.push(DataType::from_json(&changed).unwrap());
+ }
+ for changed in changes {
+ assert!(left.union(&df.to(changed)).cached_schema().is_none());
+ }
+ assert!(left.union(&df).cached_schema().is_none());
+ let other_session = spark.new_session();
+ let other = other_session.range(3).unwrap().to(left.schema().unwrap());
+ assert!(left.union(&other).cached_schema().is_none());
+ }
+
+ #[test]
+ fn schema_propagation_seeds_only_struct_output_schemas() {
+ use crate::udf::PythonUDFPayload;
+
+ let spark = SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .unwrap();
+ let df = spark.range(3).unwrap();
+ let schema = DataType::Struct { fields: vec![] };
+ let udf = |output_type| {
+ CommonInlineUserDefinedFunctionExpression::new(
+ "map".to_string(),
+ true,
+ vec![],
+ PythonUDFPayload::new(output_type, 205, vec![],
"3.12".to_string()),
+ )
+ };
+ for output in [
+ df.to(schema.clone()),
+ df.map_in_pandas(udf(schema.clone()), false),
+ df.map_in_arrow(udf(schema.clone()), false),
+ ] {
+ assert_eq!(output.schema().unwrap(), schema);
+ }
+ assert!(df.to(DataType::Long).cached_schema().is_none());
+ assert!(df
+ .map_in_arrow(udf(DataType::Long), false)
+ .cached_schema()
+ .is_none());
+ }
+
+ #[test]
+ fn schema_cache_recovers_after_analysis_panics() {
+ let spark = SparkSession::builder()
+ .remote("sc://localhost:15002")
+ .get_or_create()
+ .unwrap();
+ let df = spark.range(3).unwrap();
+ let cloned = df.clone();
+ let runtime = tokio::runtime::Runtime::new().unwrap();
+
+ // Calling the synchronous API inside a runtime panics during analysis.
+ // A clone must still attempt analysis afterward, rather than returning
+ // a permanent cache-poisoning error.
+ for frame in [&df, &cloned] {
+ let result = runtime.block_on(async {
+ std::panic::catch_unwind(std::panic::AssertUnwindSafe(||
frame.schema()))
+ });
+ assert!(result.is_err());
+ // Poisoning proves the lock spans the analysis boundary without
+ // relying on thread scheduling to produce contended cache misses.
+ assert!(
+ frame.schema_cache.is_poisoned(),
+ "schema analysis must hold the shared cache lock"
+ );
+ }
+ let mut cached = df.schema_cache.lock().unwrap_err().into_inner();
+ assert!(cached.is_none());
+ let schema = DataType::Struct { fields: vec![] };
+ *cached = Some(Arc::new(schema.clone()));
+ drop(cached);
+ assert!(df.schema_cache.is_poisoned());
+ assert_eq!(df.schema().unwrap(), schema);
+ assert!(!df.schema_cache.is_poisoned());
+ assert_eq!(cloned.schema().unwrap(), schema);
+ }
+
#[test]
fn cache_default_is_memory_and_disk_deser() {
let sl = DataFrame::memory_and_disk_deser();
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index 0047a67..54c9eea 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -141,6 +141,7 @@ impl GroupedData {
}
fn group_map(&self, func: CommonInlineUserDefinedFunctionExpression) ->
DataFrame {
+ let schema = func.python_udf.output_type.clone();
let plan = LogicalPlan::GroupMap {
input: Arc::clone(&self.dataframe.plan),
grouping_expressions: self.grouping_expressions(),
@@ -154,7 +155,7 @@ impl GroupedData {
state_schema: None,
transform_with_state_info: None,
};
- DataFrame::new(self.dataframe.session.clone(), plan)
+ DataFrame::new_with_schema(self.dataframe.session.clone(), plan,
Some(schema))
}
/// Apply a stateful pandas UDF to each group
(`GroupedData.applyInPandasWithState`).
@@ -525,6 +526,7 @@ impl CoGroupedData {
}
fn cogroup_map(&self, func: CommonInlineUserDefinedFunctionExpression) ->
DataFrame {
+ let schema = func.python_udf.output_type.clone();
let plan = LogicalPlan::CoGroupMap {
input: Arc::clone(&self.left.dataframe.plan),
input_grouping_expressions: self.left.grouping_expressions(),
@@ -532,7 +534,7 @@ impl CoGroupedData {
other_grouping_expressions: self.right.grouping_expressions(),
func,
};
- DataFrame::new(self.left.dataframe.session.clone(), plan)
+ DataFrame::new_with_schema(self.left.dataframe.session.clone(), plan,
Some(schema))
}
}
@@ -571,6 +573,44 @@ mod tests {
assert!(Arc::ptr_eq(input, &df.plan));
}
+ #[test]
+ fn grouped_udfs_reuse_declared_output_schema() {
+ use crate::udf::{eval_type, PythonUDFPayload};
+
+ let spark = session();
+ let grouped = spark.range(10).unwrap().group_by(Vec::<Column>::new());
+ let cogrouped = grouped.cogroup(&grouped);
+ let schema = DataType::from_json(&serde_json::json!({
+ "type": "struct",
+ "fields": [{
+ "name": "result",
+ "type": "long",
+ "nullable": false,
+ "metadata": {"description": "declared output"}
+ }]
+ }))
+ .unwrap();
+ let udf = |eval_type| {
+ CommonInlineUserDefinedFunctionExpression::new(
+ "output".to_string(),
+ true,
+ vec![],
+ PythonUDFPayload::new(schema.clone(), eval_type, vec![],
"3.12".to_string()),
+ )
+ };
+
+ // These accesses must work without a server and use the supplied
output
+ // schema, which differs from the unresolved input's `id` column.
+ for result in [
+
grouped.apply_in_pandas(udf(eval_type::SQL_GROUPED_MAP_PANDAS_UDF)),
+ grouped.apply_in_arrow(udf(eval_type::SQL_GROUPED_MAP_ARROW_UDF)),
+
cogrouped.apply_in_pandas(udf(eval_type::SQL_COGROUPED_MAP_PANDAS_UDF)),
+
cogrouped.apply_in_arrow(udf(eval_type::SQL_COGROUPED_MAP_ARROW_UDF)),
+ ] {
+ assert_eq!(result.schema().unwrap(), schema);
+ }
+ }
+
#[test]
fn group_count_plan() {
let spark = session();
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
new file mode 100644
index 0000000..555f829
--- /dev/null
+++ b/python/tests/_mock_connect_server.py
@@ -0,0 +1,160 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Small, separately hosted Spark Connect service for transport-level tests."""
+
+import multiprocessing
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from threading import Event
+
+
+# The drop-in package does not ship generated service protobuf classes, so
these
+# encoders provide just the messages needed by this test server.
+def varint(value):
+ result = bytearray()
+ while value > 127:
+ result.append((value & 127) | 128)
+ value >>= 7
+ result.append(value)
+ return bytes(result)
+
+
+def message_field(field, value):
+ if isinstance(value, str):
+ value = value.encode()
+ return varint(field * 8 + 2) + varint(len(value)) + value
+
+
+def struct_schema(name="id"):
+ # types.proto: DataType.long=7, .struct=21; Struct.fields=1;
+ # StructField: name=1, data_type=2, nullable=3, metadata=4.
+ field = (
+ message_field(1, name)
+ + message_field(2, message_field(7, b""))
+ + b"\x18\x01"
+ + message_field(4, '{"note":"original"}')
+ )
+ return message_field(21, message_field(1, field))
+
+
+def serve(connection):
+ import grpc
+
+ # A separate interpreter lets tests detect clients retaining the GIL while
+ # they wait for a response.
+ class Service:
+ calls = 0
+ data_type = struct_schema()
+ failure = None
+ block_response = False
+ response_gate = Event()
+
+ def analyze(self, request, context):
+ self.calls += 1
+ if self.block_response and not self.response_gate.wait(5):
+ context.abort(
+ grpc.StatusCode.DEADLINE_EXCEEDED,
+ "Python thread could not run while metadata lookup waited",
+ )
+ if self.failure == "rpc":
+ context.abort(grpc.StatusCode.INVALID_ARGUMENT, "schema
unavailable")
+ if self.failure == "no-result":
+ return b""
+ if self.failure == "missing-schema":
+ return message_field(2, b"")
+ if self.failure == "invalid-type":
+ return message_field(2, message_field(1, b""))
+ # base.proto: AnalyzePlanResponse.schema=2, Schema.schema=1.
+ return message_field(2, message_field(1, self.data_type))
+
+ service = Service()
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ server = grpc.server(executor)
+ handlers = {
+ "AnalyzePlan":
grpc.unary_unary_rpc_method_handler(service.analyze),
+ "ReleaseSession": grpc.unary_unary_rpc_method_handler(lambda
request, context: b""),
+ }
+ server.add_generic_rpc_handlers(
+
(grpc.method_handlers_generic_handler("spark.connect.SparkConnectService",
handlers),)
+ )
+ port = server.add_insecure_port("127.0.0.1:0")
+ server.start()
+ connection.send(port)
+ try:
+ while True:
+ command = connection.recv()
+ if command[0] == "stop":
+ break
+ if command[0] == "get":
+ connection.send(getattr(service, command[1]))
+ elif command[0] == "set":
+ setattr(service, command[1], command[2])
+ connection.send(None)
+ elif command[0] == "release":
+ service.response_gate.set()
+ connection.send(None)
+ finally:
+ server.stop(0).wait()
+
+
+class RemoteService:
+ def __init__(self, connection):
+ object.__setattr__(self, "connection", connection)
+
+ def __getattr__(self, name):
+ self.connection.send(("get", name))
+ assert self.connection.poll(10), "Mock server did not respond"
+ return self.connection.recv()
+
+ def __setattr__(self, name, value):
+ self.connection.send(("set", name, value))
+ assert self.connection.poll(10), "Mock server did not respond"
+ self.connection.recv()
+
+ def release_response(self):
+ self.connection.send(("release",))
+ assert self.connection.poll(10), "Mock server did not respond"
+ self.connection.recv()
+
+
+@contextmanager
+def mock_connect_session():
+ from pyspark.sql import SparkSession
+
+ context = multiprocessing.get_context("spawn")
+ parent, child = context.Pipe()
+ process = context.Process(target=serve, args=(child,))
+ process.start()
+ spark = None
+ try:
+ assert parent.poll(20), "Mock server did not start"
+ port = parent.recv()
+ spark =
SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate()
+ yield spark, RemoteService(parent)
+ finally:
+ try:
+ if spark is not None:
+ spark.stop()
+ finally:
+ if process.is_alive():
+ parent.send(("stop",))
+ process.join(10)
+ if process.is_alive():
+ process.terminate()
+ process.join(10)
+ parent.close()
+ child.close()
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 67ccaa4..9988f65 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7,10 +7,14 @@ that require a live server live in test_dropin_e2e.py and are
gated on SPARK_REM
"""
import pickle
+import time
+from concurrent.futures import ThreadPoolExecutor
+from threading import Barrier
import pytest
-from pyspark.sql import types as T
+from _mock_connect_server import message_field, mock_connect_session,
struct_schema
+from pyspark.sql import SparkSession, types as T
# ---------------------------------------------------------------------------
types
@@ -2755,3 +2759,329 @@ for schema in schemas:
else:
raise AssertionError("expected a width validation error")
"""], check=True)
+
+
+# ---------------------------------------------------------------------------
schema cache
+
+
[email protected]
+def mock_connect():
+ pytest.importorskip("grpc")
+ with mock_connect_session() as session:
+ yield session
+
+
[email protected]("first", ["schema", "columns", "dtypes"])
+def test_schema_cache_one_analysis_shared_by_metadata_accessors(mock_connect,
first):
+ spark, service = mock_connect
+ df = spark.range(10)
+ assert service.calls == 0
+ getattr(df, first)
+ assert service.calls == 1
+ for _ in range(3):
+ schema = df.schema
+ assert isinstance(schema, T.StructType)
+ expected = T.StructType([T.StructField("id", T.LongType(), True,
{"note": "original"})])
+ assert schema.jsonValue() == expected.jsonValue()
+ assert df.columns == ["id"]
+ assert df.dtypes == [("id", "bigint")]
+ assert service.calls == 1
+
+
+def test_schema_cache_returned_schema_and_lists_are_independent(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ expected_json = df.schema.jsonValue()
+ modified = df.schema
+ modified.add("extra", T.StringType())
+ assert len(modified.fields) == 2
+ columns = df.columns
+ columns.append("extra")
+ dtypes = df.dtypes
+ dtypes.clear()
+ assert df.schema.jsonValue() == expected_json
+ assert df.columns == ["id"]
+ assert df.dtypes == [("id", "bigint")]
+ assert service.calls == 1
+
+
+def
test_schema_cache_derived_and_new_plans_resolve_independently(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ assert df.columns == ["id"]
+ derived = df.selectExpr("id AS renamed")
+ service.data_type = struct_schema("renamed")
+ assert derived.columns == ["renamed"]
+ assert service.calls == 2
+ assert df.columns == ["id"]
+ # Rebuilding the same source must observe updated server metadata, while
the
+ # original immutable DataFrame retains its already-resolved schema.
+ fresh = spark.range(10)
+ assert fresh.columns == ["renamed"]
+ other_session = spark.newSession()
+ try:
+ assert other_session.range(10).columns == ["renamed"]
+ finally:
+ other_session.stop()
+ assert service.calls == 4
+ assert df.columns == ["id"]
+
+
[email protected]("transform", [
+ pytest.param(lambda df: df.filter("id > 0"), id="filter"),
+ pytest.param(lambda df: df.limit(2), id="limit"),
+ pytest.param(lambda df: df.alias("source"), id="alias"),
+ pytest.param(lambda df: df.distinct(), id="distinct"),
+ pytest.param(lambda df: df.dropDuplicates(["id"]), id="dropDuplicates"),
+ pytest.param(lambda df: df.coalesce(1), id="coalesce"),
+ pytest.param(lambda df: df.repartition(2), id="repartition"),
+ pytest.param(lambda df: df.repartition(2, "id"),
id="repartition-expression"),
+ pytest.param(lambda df: df.repartition("id"), id="repartition-column"),
+ pytest.param(lambda df: df.repartitionByRange(2, "id"),
id="repartitionByRange"),
+ pytest.param(lambda df: df.repartitionById(2, "id"), id="repartitionById"),
+ pytest.param(lambda df: df.sort("id"), id="sort"),
+ pytest.param(lambda df: df.sortWithinPartitions("id"),
id="sortWithinPartitions"),
+ pytest.param(lambda df: df.sample(fraction=0.5, seed=1), id="sample"),
+ pytest.param(lambda df: df.hint("broadcast"), id="hint"),
+])
+def test_schema_cache_preserving_transforms_reuse_warm_schema(mock_connect,
transform):
+ spark, service = mock_connect
+ df = spark.range(10)
+ expected = df.schema.jsonValue()
+ assert service.calls == 1
+ # A new analysis would return different metadata, making accidental misses
+ # observable as well as countable.
+ service.data_type = struct_schema("unexpected")
+ derived = transform(df)
+ assert derived.schema.jsonValue() == expected
+ assert derived.limit(1).columns == ["id"]
+ assert service.calls == 1
+
+
+def test_schema_cache_random_split_and_observe_reuse_warm_schema(mock_connect):
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ df = spark.range(10)
+ assert df.columns == ["id"]
+ service.data_type = struct_schema("unexpected")
+ for split in df.randomSplit([1.0, 2.0], seed=1):
+ assert split.columns == ["id"]
+ observed = df.observe("rows", F.count("*").alias("count"))
+ assert observed.columns == ["id"]
+ assert service.calls == 1
+
+
+def test_schema_cache_cold_children_keep_independent_caches(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ before_lookup = df.limit(1)
+ service.block_response = True
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ lookup = pool.submit(lambda: df.schema)
+ try:
+ for _ in range(500):
+ if service.calls:
+ break
+ time.sleep(0.01)
+ else:
+ raise AssertionError("Metadata request never reached the
server")
+ # Constructing a child also runs with the GIL. It must not wait for
+ # the in-flight cache miss, or this thread cannot release the
server.
+ during_lookup = df.filter("id > 0")
+ assert not lookup.done()
+ finally:
+ service.release_response()
+ assert lookup.result(timeout=10).fieldNames() == ["id"]
+ assert service.calls == 1
+ # Neither child picks up a cache populated after the child was constructed.
+ for calls, child in enumerate((before_lookup, during_lookup), start=2):
+ service.data_type = struct_schema(f"child_{calls}")
+ assert child.columns == [f"child_{calls}"]
+ assert service.calls == calls
+ assert df.columns == ["id"]
+
+
[email protected]("transform", [
+ pytest.param(lambda df: df.selectExpr("id AS renamed"), id="select"),
+ pytest.param(lambda df: df.zipWithIndex(), id="zipWithIndex"),
+ # Reference PySpark does not propagate through offset, despite its
unchanged
+ # output columns. Match its explicit propagation rules rather than
guessing.
+ pytest.param(lambda df: df.offset(1), id="offset"),
+])
+def test_schema_cache_other_transforms_analyze_independently(mock_connect,
transform):
+ spark, service = mock_connect
+ df = spark.range(10)
+ assert df.columns == ["id"]
+ derived = transform(df)
+ assert service.calls == 1
+ service.data_type = struct_schema("derived")
+ assert derived.columns == ["derived"]
+ assert service.calls == 2
+ assert df.columns == ["id"]
+
+
[email protected]("operation", [
+ "union", "unionAll", "unionByName", "subtract", "exceptAll", "intersect",
"intersectAll",
+])
+def test_schema_cache_set_operations_require_equal_warm_schemas(mock_connect,
operation):
+ spark, service = mock_connect
+ left, right = spark.range(10), spark.range(10)
+ assert left.columns == right.columns == ["id"]
+ assert service.calls == 2
+ equal = getattr(left, operation)(right)
+ assert equal.columns == ["id"]
+ assert service.calls == 2
+
+ cold = spark.range(10)
+ missing = getattr(left, operation)(cold)
+ assert service.calls == 2
+ service.data_type = struct_schema("resolved")
+ assert missing.columns == ["resolved"]
+ assert service.calls == 3
+
+ different = spark.range(10)
+ assert different.columns == ["resolved"]
+ assert service.calls == 4
+ mismatched = getattr(left, operation)(different)
+ service.data_type = struct_schema("coerced")
+ assert mismatched.columns == ["coerced"]
+ assert service.calls == 5
+ assert left.columns == ["id"]
+
+
+def test_schema_cache_to_uses_supplied_struct_schema(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ target = T.StructType([T.StructField("converted", T.StringType(), True)])
+ expected = target.jsonValue()
+ converted = df.to(target)
+ target.add("extra", T.LongType())
+ assert converted.schema.jsonValue() == expected
+ assert converted.limit(1).dtypes == [("converted", "string")]
+ assert service.calls == 0
+ assert df.columns == ["id"]
+ assert service.calls == 1
+
+
[email protected]("method", ["mapInPandas", "mapInArrow"])
+def test_schema_cache_map_uses_declared_struct_schema(mock_connect, method):
+ spark, service = mock_connect
+ target = T.StructType([
+ T.StructField("output", T.StringType(), True, {"note": "declared"}),
+ ])
+ mapped = getattr(spark.range(10), method)(lambda batches: batches,
schema=target)
+ # The binding resolves the input columns while building the UDF. Reading
its
+ # declared output schema must not issue another analysis request.
+ assert service.calls == 1
+ service.data_type = struct_schema("unexpected")
+ assert mapped.schema.jsonValue() == target.jsonValue()
+ assert mapped.columns == ["output"]
+ assert mapped.dtypes == [("output", "string")]
+ assert service.calls == 1
+
+
[email protected]("failure", ["rpc", "no-result", "missing-schema",
"invalid-type"])
+def test_schema_cache_failures_are_not_cached(mock_connect, failure):
+ spark, service = mock_connect
+ df = spark.range(10)
+ service.failure = failure
+ for count in (1, 2):
+ message = {
+ "rpc": "schema unavailable",
+ "no-result": "no schema in response",
+ "missing-schema": "Schema is missing",
+ "invalid-type": "DataType kind not set",
+ }[failure]
+ with pytest.raises(Exception, match=message):
+ _ = df.schema
+ assert service.calls == count
+ service.failure = None
+ assert df.columns == ["id"]
+ assert df.dtypes == [("id", "bigint")]
+ assert service.calls == 3
+
+
+def test_schema_cache_non_struct_responses_are_not_cached(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ service.data_type = message_field(7, b"")
+ for count in (1, 2):
+ with pytest.raises(Exception, match="Schema is not a struct type"):
+ _ = df.schema
+ assert service.calls == count
+ service.data_type = struct_schema()
+ assert df.columns == ["id"]
+ assert df.dtypes == [("id", "bigint")]
+ assert service.calls == 3
+
+
+def
test_schema_cache_concurrent_metadata_access_issues_one_analysis(mock_connect):
+ spark, service = mock_connect
+ df = spark.range(10)
+ accessors = ("schema", "columns", "dtypes") * 3
+ start = Barrier(len(accessors) + 1)
+ service.block_response = True
+
+ def read_metadata(accessor):
+ start.wait(timeout=10)
+ return getattr(df, accessor)
+
+ with ThreadPoolExecutor(max_workers=len(accessors)) as pool:
+ pending = [pool.submit(read_metadata, accessor) for accessor in
accessors]
+ try:
+ start.wait(timeout=10)
+ for _ in range(500):
+ if service.calls:
+ break
+ time.sleep(0.01)
+ else:
+ raise AssertionError("Metadata request never reached the
server")
+ # Keep the first RPC in flight while the other workers attempt
their
+ # cold lookups. An implementation without miss synchronization must
+ # not pass just because the first response arrives immediately.
+ time.sleep(0.1)
+ assert not any(future.done() for future in pending)
+ finally:
+ service.release_response()
+ results = [future.result(timeout=10) for future in pending]
+ schemas = []
+ for accessor, result in zip(accessors, results):
+ if accessor == "schema":
+ schemas.append(result)
+ elif accessor == "columns":
+ assert result == ["id"]
+ else:
+ assert result == [("id", "bigint")]
+ assert all(schema.jsonValue() == schemas[0].jsonValue() for schema in
schemas)
+ assert len({id(schema) for schema in schemas}) == len(schemas)
+ assert service.calls == 1
+
+
[email protected]("accessor", ["schema", "columns", "dtypes"])
+def test_schema_cache_metadata_wait_releases_gil(mock_connect, accessor):
+ spark, service = mock_connect
+ df = spark.range(10)
+ service.block_response = True
+
+ def allow_response_after_request():
+ for _ in range(500):
+ if service.calls:
+ service.release_response()
+ return
+ time.sleep(0.01)
+ raise AssertionError("Metadata request never reached the server")
+
+ # This Python thread must run while the main thread waits for the RPC. The
+ # server bounds its wait, so retaining the GIL fails instead of
deadlocking.
+ with ThreadPoolExecutor(max_workers=1) as pool:
+ allowed = pool.submit(allow_response_after_request)
+ result = getattr(df, accessor)
+ allowed.result(timeout=10)
+ if accessor == "schema":
+ assert isinstance(result, T.StructType)
+ elif accessor == "columns":
+ assert result == ["id"]
+ else:
+ assert result == [("id", "bigint")]
+ assert service.calls == 1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]