This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 45bfe1a [SPARK-60739][CONNECT] Fix truncated results and unreleased
executions in the reattachable stream, and four drop-in parity gaps
45bfe1a is described below
commit 45bfe1a3bc3813f3da2b936bbb30aff0548eb6f5
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Wed Nov 4 09:57:28 2026 +0900
[SPARK-60739][CONNECT] Fix truncated results and unreleased executions in
the reattachable stream, and four drop-in parity gaps
### What changes were proposed in this pull request?
The Rust client's reattachable stream now reattaches when the server ends a
response stream before the result is complete, and releases executions with
`ReleaseExecute`, as the reference client does. This also fixes four drop-in
gaps against PySpark 4.3.0: `StructType.add()` fields in
`mapInArrow`/`mapInPandas` schemas, `repartitionById(n, "col")`, `conf.get(key,
None)`, and `createDataFrame(pyarrow.Table)`.
### Why are the changes needed?
Results were silently truncated whenever the server ended a stream early
(by default after 2 minutes or 1 GiB), and the server kept every finished
execution until it timed out.
### Does this PR introduce _any_ user-facing change?
Yes, these behaviors now match PySpark 4.3.0.
### How was this patch tested?
New Rust e2e and Python live-server tests, which fail on master, pass here
against a Spark 4.3.0 server; CI's e2e servers now use a 1 MiB
`senderMaxStreamSize` to exercise the reattach path.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #141 from HyukjinKwon/SPARK-60739-reattach-release-dropin-parity.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.github/workflows/build_python_connect.yml | 10 +-
.github/workflows/coverage.yml | 3 +
crates/pyspark-rs/src/column.rs | 2 +-
crates/pyspark-rs/src/conf.rs | 35 +++--
crates/pyspark-rs/src/dataframe.rs | 11 +-
crates/pyspark-rs/src/session.rs | 130 ++++++++++++++++++-
crates/pyspark-rs/src/types.rs | 4 +
crates/spark-connect-core/src/client.rs | 160 ++++++++++++++++++-----
crates/spark-connect-core/src/reattach.rs | 86 ++++++++++---
crates/spark-connect/src/conf.rs | 6 +
crates/spark-connect/src/session.rs | 28 ++++
crates/spark-connect/tests/e2e_integration.rs | 141 ++++++++++++++++++++
dev/design/architecture.md | 3 +
docs/architecture.md | 10 +-
python/tests/test_dropin_e2e.py | 179 ++++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 10 ++
16 files changed, 752 insertions(+), 66 deletions(-)
diff --git a/.github/workflows/build_python_connect.yml
b/.github/workflows/build_python_connect.yml
index b03271e..14833b2 100644
--- a/.github/workflows/build_python_connect.yml
+++ b/.github/workflows/build_python_connect.yml
@@ -213,12 +213,14 @@ jobs:
--exclude
"test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \
2>&1 | tee /tmp/parity_ours.log
- - name: Native Arrow handoff smoke tests (against the live server)
+ - name: Drop-in live-server tests (native Arrow handoff and API
behaviors)
+ # The parity gate above runs the reference pyspark over our transport,
so the
+ # drop-in's own Rust-backed classes are only exercised against a
server here.
working-directory: python
env:
SPARK_REMOTE: sc://localhost:15002
PYTHONPATH: ${{ github.workspace }}/python
- run: python -m pytest tests/test_arrow_e2e.py -q
+ run: python -m pytest tests/test_arrow_e2e.py tests/test_dropin_e2e.py
-q
- name: Restart Connect server in local-cluster mode
# Resource profiles and artifact distribution across executors can
only be
@@ -366,9 +368,13 @@ jobs:
PYSPARK_DRIVER_PYTHON: python
run: |
SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}"
+ # A small senderMaxStreamSize makes the server end the response
stream of every
+ # result over 1 MiB before it is complete, so that the client's
reattach path is
+ # exercised (by default it only is for results over 1 GiB or 2
minutes).
"${SPARK_HOME}/sbin/start-connect-server.sh" \
--driver-memory 3g \
--conf spark.sql.session.localRelationSizeLimit=1073741824 \
+ --conf
spark.connect.execute.reattachable.senderMaxStreamSize=1048576 \
> /tmp/connect-server.log 2>&1 &
for i in $(seq 1 90); do
if (echo > /dev/tcp/localhost/15002) 2>/dev/null; then
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index e766676..15747ed 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -124,9 +124,12 @@ jobs:
run: |
SPARK_HOME="${{ steps.spark-download.outputs.SPARK_HOME }}"
echo "Starting Connect server on sc://localhost:15002"
+ # As for the behavioral e2e tests: a small senderMaxStreamSize
exercises the
+ # client's reattach path on every result over 1 MiB.
"${SPARK_HOME}/sbin/start-connect-server.sh" \
--driver-memory 3g \
--conf spark.sql.session.localRelationSizeLimit=1073741824 \
+ --conf
spark.connect.execute.reattachable.senderMaxStreamSize=1048576 \
> /tmp/connect-server.log 2>&1 &
for i in $(seq 1 90); do
if (echo > /dev/tcp/localhost/15002) 2>/dev/null; then
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index 48280d9..b1ee90a 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -8,7 +8,7 @@ use crate::functions::to_column;
/// Raise a `pyspark.errors` exception (`class_name`) carrying the given
`error_class`
/// condition and message parameters, mirroring the reference Column
guard-rails.
-fn raise_pyspark(
+pub(crate) fn raise_pyspark(
py: Python<'_>,
class_name: &str,
error_class: &str,
diff --git a/crates/pyspark-rs/src/conf.rs b/crates/pyspark-rs/src/conf.rs
index 999959b..f3eb8dd 100644
--- a/crates/pyspark-rs/src/conf.rs
+++ b/crates/pyspark-rs/src/conf.rs
@@ -17,6 +17,25 @@ impl PyRuntimeConf {
}
}
+/// The `default` argument of `RuntimeConf.get`, where pyspark tells an
omitted default apart
+/// from None: `get(key, None)` returns None for an unset key, while
`get(key)` raises.
+pub enum ConfDefault {
+ Omitted,
+ Given(Option<String>),
+}
+
+impl<'a, 'py> FromPyObject<'a, 'py> for ConfDefault {
+ type Error = PyErr;
+
+ fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
+ if obj.is_none() {
+ Ok(ConfDefault::Given(None))
+ } else {
+ Ok(ConfDefault::Given(Some(obj.extract()?)))
+ }
+ }
+}
+
// Every method round-trips to the server
(`block_on(client.set_config/get_configs)`
// in the core), so each releases the GIL across the RPC via `py.detach`;
otherwise a
// Python thread stays blocked for the whole call.
@@ -30,16 +49,16 @@ impl PyRuntimeConf {
py.detach(|| self.conf.set(key, &v)).to_pyerr()
}
- /// Get a configuration value. With a `default`, an unknown key returns
the default
- /// (server `GetWithDefault`) rather than raising; without one, an unknown
key raises,
- /// mirroring `RuntimeConfig.get`.
- #[pyo3(signature = (key, default=None))]
- fn get(&self, py: Python<'_>, key: &str, default: Option<String>) ->
PyResult<Option<String>> {
+ /// Get a configuration value. With a `default`, which may be None, an
unknown key returns
+ /// the default (server `GetWithDefault`) rather than raising; without
one, an unknown key
+ /// raises, mirroring `RuntimeConfig.get`.
+ #[pyo3(signature = (key, default=ConfDefault::Omitted))]
+ fn get(&self, py: Python<'_>, key: &str, default: ConfDefault) ->
PyResult<Option<String>> {
match default {
- Some(d) => py
- .detach(|| self.conf.get_with_default(key, Some(&d)))
+ ConfDefault::Given(d) => py
+ .detach(|| self.conf.get_with_default(key, d.as_deref()))
.to_pyerr(),
- None => py.detach(|| self.conf.get(key)).to_pyerr(),
+ ConfDefault::Omitted => py.detach(||
self.conf.get(key)).to_pyerr(),
}
}
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 9c01358..cf32ab2 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -400,7 +400,16 @@ impl PyDataFrame {
"numPartitions must be positive, got {numPartitions}"
)));
}
- let col = to_column(partitionIdCol)?;
+ // A `ColumnOrName`, as in pyspark: a string is a column name, not a
literal.
+ let col = if let Ok(c) = partitionIdCol.extract::<Bound<'_,
PyColumn>>() {
+ c.borrow().column.clone()
+ } else if let Ok(name) = partitionIdCol.extract::<String>() {
+ spark_connect::functions::col(&name)
+ } else {
+ return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
+ "partitionIdCol should be a Column or a column name (str)",
+ ));
+ };
Ok(PyDataFrame::new(
self.dataframe.repartition_by_id(numPartitions, col),
))
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 2bf4bc3..f81bde1 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -278,6 +278,129 @@ impl PySparkSession {
pub fn new(session: SparkSession) -> Self {
PySparkSession { session }
}
+
+ /// `createDataFrame` of a pyarrow Table, as the reference client does it:
the column
+ /// names come from a list `schema` (extra columns are named `_N`), or
else from the table;
+ /// the Spark types come from a `StructType` (or DDL) `schema`, or else
from the Arrow
+ /// types. The table is cast to the Arrow form of those types, with its
timestamps
+ /// localized to the session time zone, and sent as an Arrow IPC stream.
+ fn create_dataframe_from_arrow(
+ &self,
+ py: Python<'_>,
+ table: &Bound<'_, PyAny>,
+ schema: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ let mut names: Option<Vec<String>> = None;
+ let mut user_schema: Option<DataType> = None;
+ match schema {
+ None => names = Some(table.getattr("column_names")?.extract()?),
+ Some(s) => {
+ if let Ok(mut given) = s.extract::<Vec<String>>() {
+ let num_columns: usize =
table.getattr("num_columns")?.extract()?;
+ given.extend((given.len()..num_columns).map(|i|
format!("_{}", i + 1)));
+ names = Some(given);
+ } else {
+ user_schema = Some(crate::types::py_to_data_type(s)?);
+ }
+ }
+ }
+ // As the reference does for any empty input: a relation of the user's
schema, which
+ // a list of names does not give.
+ if table.len()? == 0 {
+ return match user_schema {
+ Some(dt) => Ok(PyDataFrame::new(
+ py.detach(|| self.session.create_dataframe(vec![], dt))
+ .to_pyerr()?,
+ )),
+ None => Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_INFER_EMPTY_SCHEMA",
+ &[],
+ )),
+ };
+ }
+
+ let conf = self.session.conf();
+ let configs = py
+ .detach(|| {
+ conf.get_many(&[
+ "spark.sql.timestampType",
+ "spark.sql.session.timeZone",
+ "spark.sql.execution.arrow.useLargeVarTypes",
+ ])
+ })
+ .to_pyerr()?;
+ let config = |i: usize| configs.get(i).cloned().flatten();
+ let prefer_timestamp_ntz = config(0).as_deref() ==
Some("TIMESTAMP_NTZ");
+ let timezone = config(1);
+ let prefers_large_types = config(2).as_deref() == Some("true");
+
+ let types = py.import("pyspark.sql.pandas.types")?;
+ let spark_schema = match &user_schema {
+ Some(dt @ DataType::Struct { .. }) => {
+ crate::types::data_type_to_py(py, dt)?.into_bound(py)
+ }
+ _ => types.call_method1(
+ "from_arrow_schema",
+ (table.getattr("schema")?, prefer_timestamp_ntz),
+ )?,
+ };
+ let kwargs = PyDict::new(py);
+ kwargs.set_item("error_on_duplicated_field_names_in_struct", true)?;
+ kwargs.set_item("timezone", "UTC")?;
+ kwargs.set_item("prefers_large_types", prefers_large_types)?;
+ let arrow_schema = types.call_method("to_arrow_schema",
(&spark_schema,), Some(&kwargs))?;
+ let converted = types
+ .call_method1(
+ "_check_arrow_table_timestamps_localize",
+ (table, &spark_schema, true, timezone),
+ )?
+ .call_method1("cast", (arrow_schema,))?
+ .call_method1("rename_columns",
(spark_schema.getattr("names")?,))?;
+
+ let pa = py.import("pyarrow")?;
+ let sink = pa.call_method0("BufferOutputStream")?;
+ let writer = pa
+ .getattr("ipc")?
+ .call_method1("new_stream", (&sink,
converted.getattr("schema")?))?;
+ writer.call_method1("write_table", (&converted,))?;
+ writer.call_method0("close")?;
+ let stream =
sink.call_method0("getvalue")?.call_method0("to_pybytes")?;
+ let data = stream.cast::<pyo3::types::PyBytes>()?.as_bytes().to_vec();
+
+ // The reference sends the user's schema, or none for the server to
read the Arrow
+ // one; the plan always carries a schema, so the Arrow one is sent
explicitly.
+ let relation_schema = match user_schema {
+ Some(dt) => dt,
+ None => crate::types::py_to_data_type(&spark_schema)?,
+ };
+ let df = self
+ .session
+ .create_dataframe_from_arrow(data, relation_schema);
+ // Like the reference, apply the column names with toDF.
+ Ok(PyDataFrame::new(match names {
+ Some(names) if !names.is_empty() => {
+ df.to_df(names.iter().map(String::as_str).collect())
+ }
+ _ => df,
+ }))
+ }
+}
+
+/// Whether `obj` is an instance of `module.class`, without importing `module`
when it is not
+/// loaded yet: an instance can only exist once its module was imported.
+fn is_loaded_instance(
+ py: Python<'_>,
+ obj: &Bound<'_, PyAny>,
+ module: &str,
+ class: &str,
+) -> PyResult<bool> {
+ let modules = py.import("sys")?.getattr("modules")?;
+ match modules.cast::<PyDict>()?.get_item(module)? {
+ Some(m) => obj.is_instance(&m.getattr(class)?),
+ None => Ok(false),
+ }
}
#[pymethods]
@@ -362,7 +485,8 @@ impl PySparkSession {
Ok(PyDataFrame::new(df))
}
- /// Create a DataFrame from a list of tuples/lists and an optional schema.
+ /// Create a DataFrame from a list of tuples/lists and an optional schema.
A pandas
+ /// DataFrame or a pyarrow Table is accepted too, as in pyspark.
///
/// If schema is a list of strings, those are used as column names (all
types inferred as string).
/// If schema is None, columns are named col0, col1, etc. and all types
are string.
@@ -384,6 +508,10 @@ impl PySparkSession {
// signature parity. See PARITY_GAP_ANALYSIS.md.
let _ = (samplingRatio, verifySchema);
+ if is_loaded_instance(py, data, "pyarrow", "Table")? {
+ return self.create_dataframe_from_arrow(py, data, schema);
+ }
+
// Accept a pandas DataFrame (as pyspark does): convert it to a list
of row lists
// with NaN/NaT mapped to None, and take its column names as the
default schema.
// pandas-on-Spark always passes an explicit `schema`, so this mainly
needs to
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 4aab99a..761c1cb 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1704,6 +1704,10 @@ impl PyStructType {
{
let mut me = slf.borrow_mut();
me.fields.push(new_field);
+ // The base `DataType` is what `str()` and the schemas of
`mapInArrow` and
+ // friends read, so it must follow the fields.
+ let fields = me.fields.clone();
+ me.as_super().inner = DataType::Struct { fields };
}
Ok(slf)
}
diff --git a/crates/spark-connect-core/src/client.rs
b/crates/spark-connect-core/src/client.rs
index 31820ed..0a71747 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -4,7 +4,8 @@
use std::collections::BTreeMap;
use std::str::FromStr;
-use tonic::metadata::MetadataValue;
+use std::time::Duration;
+use tonic::metadata::{MetadataMap, MetadataValue};
use tonic::transport::{Channel, ClientTlsConfig};
use tonic::{Request, Response, Streaming};
use uuid::Uuid;
@@ -341,8 +342,10 @@ impl SparkConnectClient {
/// Execute a plan as a reattachable stream: retries the initial call and,
if the
/// response stream drops mid-flight (e.g. `UNAVAILABLE` /
`INVALID_CURSOR.DISCONNECTED`)
- /// before `ResultComplete`, transparently resumes it with
`ReattachExecute` from the
- /// last observed `response_id`. Mirrors the reference reattachable
execute path.
+ /// or the server ends it before `ResultComplete`, transparently resumes
it with
+ /// `ReattachExecute` from the last observed `response_id`. The stream
releases the
+ /// execution on the server with `ReleaseExecute` (see
[`ReattachableResponseStream`]).
+ /// Mirrors the reference reattachable execute path.
pub async fn execute_plan_reattachable(
&self,
request: ExecutePlanRequest,
@@ -368,6 +371,7 @@ impl SparkConnectClient {
iter,
stream,
done: false,
+ released: false,
})
}
@@ -867,9 +871,13 @@ impl SparkConnectClient {
/// A reattachable, retrying response stream for a reattachable ExecutePlan.
///
-/// Drives the gRPC response stream and, on a transient mid-stream disconnect
-/// before `ResultComplete`, resumes it via `ReattachExecute` from the last
-/// observed `response_id` (with backoff). Mirrors
+/// Drives the gRPC response stream and resumes it via `ReattachExecute` from
the last
+/// observed `response_id` (with backoff) whenever it ends before
`ResultComplete`: on a
+/// transient mid-stream disconnect, and when the server completes the stream
on purpose
+/// (after `spark.connect.execute.reattachable.senderMaxStreamDuration` or
+/// `senderMaxStreamSize`). It releases each response once it is read, and the
whole
+/// execution once the result is complete, the stream fails, or it is dropped,
so that the
+/// server does not keep them until the execution is abandoned. Mirrors
///
`pyspark.sql.connect.client.reattach.ExecutePlanResponseReattachableIterator`.
pub struct ReattachableResponseStream {
stub: SparkConnectServiceClient<Channel>,
@@ -881,11 +889,14 @@ pub struct ReattachableResponseStream {
iter: ExecutePlanResponseReattachableIterator,
stream: Streaming<ExecutePlanResponse>,
done: bool,
+ /// Whether `ReleaseAll` was sent, after which nothing else is released.
+ released: bool,
}
impl ReattachableResponseStream {
- fn attach_metadata<T>(&self, req: &mut Request<T>) {
- let metadata = req.metadata_mut();
+ /// The gRPC metadata of every request about this execution.
+ fn metadata_map(&self) -> MetadataMap {
+ let mut metadata = MetadataMap::new();
if let Ok(v) = MetadataValue::from_str(&self.user_agent) {
let _ = metadata.insert("user-agent", v);
}
@@ -901,9 +912,15 @@ impl ReattachableResponseStream {
}
}
}
+ metadata
+ }
+
+ fn attach_metadata<T>(&self, req: &mut Request<T>) {
+ *req.metadata_mut() = self.metadata_map();
}
- /// Next response, transparently reattaching on a transient mid-stream
drop.
+ /// Next response, transparently reattaching whenever the stream ends
before
+ /// `ResultComplete`.
pub async fn message(&mut self) -> Result<Option<ExecutePlanResponse>> {
if self.done {
return Ok(None);
@@ -914,46 +931,123 @@ impl ReattachableResponseStream {
self.iter.set_last_response_id(&resp).await;
if self.iter.is_completed().await {
self.done = true;
+ self.release_all();
+ } else if !resp.response_id.is_empty() {
+ self.release_until(&resp.response_id);
}
return Ok(Some(resp));
}
- Ok(None) => {
- self.done = true;
- return Ok(None);
- }
+ // The server completes a reattachable stream before
`ResultComplete` on
+ // purpose, after `senderMaxStreamDuration` or
`senderMaxStreamSize`: the rest
+ // of the result is read by reattaching.
+ Ok(None) => self.reattach().await?,
Err(status) => {
// Only reattach a still-running execution on a retriable
drop.
if self.iter.is_completed().await ||
!self.retry_policy.can_retry(&status) {
- return Err(SparkError::from_grpc_status(status));
+ return
Err(self.fail(SparkError::from_grpc_status(status)));
}
- loop {
- let reattach =
self.iter.create_reattach_request().await;
- let mut req = Request::new(reattach);
- self.attach_metadata(&mut req);
- match self.stub.reattach_execute(req).await {
- Ok(r) => {
- self.stream = r.into_inner();
- break;
- }
- Err(e) => {
- if self.retry_policy.can_retry(&e) {
- if let Some(w) =
self.retry_state.next_attempt(None) {
-
tokio::time::sleep(std::time::Duration::from_millis(w))
- .await;
- continue;
- }
- }
- return Err(SparkError::from_grpc_status(e));
+ self.reattach().await?;
+ }
+ }
+ }
+ }
+
+ /// Resumes the execution from the last response read, retrying transient
failures.
+ async fn reattach(&mut self) -> Result<()> {
+ loop {
+ let mut req =
Request::new(self.iter.create_reattach_request().await);
+ self.attach_metadata(&mut req);
+ match self.stub.reattach_execute(req).await {
+ Ok(r) => {
+ self.stream = r.into_inner();
+ return Ok(());
+ }
+ Err(e) => {
+ if self.retry_policy.can_retry(&e) {
+ if let Some(w) = self.retry_state.next_attempt(None) {
+ tokio::time::sleep(Duration::from_millis(w)).await;
+ continue;
+ }
+ }
+ return Err(self.fail(SparkError::from_grpc_status(e)));
+ }
+ }
+ }
+ }
+
+ /// Ends the stream with `error`, releasing the execution as the reference
client does
+ /// when an execution fails.
+ fn fail(&mut self, error: SparkError) -> SparkError {
+ self.done = true;
+ self.release_all();
+ error
+ }
+
+ /// Releases the responses up to and including `response_id`, which were
read.
+ fn release_until(&self, response_id: &str) {
+ if !self.released {
+ self.spawn_release(self.iter.release_until_request(response_id));
+ }
+ }
+
+ /// Releases the whole execution.
+ fn release_all(&mut self) {
+ if !self.released {
+ self.released = true;
+ self.spawn_release(self.iter.release_all_request());
+ }
+ }
+
+ /// Sends `request` in the background, so that reading is not delayed,
retrying under the
+ /// client's retry policy. Like the reference client, a failure is only
logged: the server
+ /// removes an execution that is never released once it has been detached
for
+ /// `spark.connect.execute.manager.detachedTimeout`.
+ fn spawn_release(&self, request: ReleaseExecuteRequest) {
+ let mut stub = self.stub.clone();
+ let metadata = self.metadata_map();
+ let policy = self.retry_policy.clone();
+ let release = async move {
+ let mut state = RetryPolicyState::new(policy.clone());
+ loop {
+ let mut req = Request::new(request.clone());
+ *req.metadata_mut() = metadata.clone();
+ match stub.release_execute(req).await {
+ Ok(_) => return,
+ Err(status) => {
+ if policy.can_retry(&status) {
+ if let Some(w) = state.next_attempt(None) {
+
tokio::time::sleep(Duration::from_millis(w)).await;
+ continue;
}
}
+ tracing::warn!(
+ "ReleaseExecute of operation {} failed: {}",
+ request.operation_id,
+ status
+ );
+ return;
}
- // Resume reading on the freshly reattached stream.
}
}
+ };
+ // A stream is read inside a runtime, but it can be dropped outside of
any, for
+ // example after the `block_on` that read it.
+ match tokio::runtime::Handle::try_current() {
+ Ok(handle) => drop(handle.spawn(release)),
+ Err(_) => drop(crate::runtime::get_runtime().spawn(release)),
}
}
}
+impl Drop for ReattachableResponseStream {
+ /// A stream dropped before its result is complete, for example by a
caller that stopped
+ /// reading, still holds its execution on the server: release it, as the
reference client
+ /// does when its iterator is closed.
+ fn drop(&mut self) {
+ self.release_all();
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/spark-connect-core/src/reattach.rs
b/crates/spark-connect-core/src/reattach.rs
index a5e139d..d5c2028 100644
--- a/crates/spark-connect-core/src/reattach.rs
+++ b/crates/spark-connect-core/src/reattach.rs
@@ -125,29 +125,47 @@ impl ExecutePlanResponseReattachableIterator {
/// Create a ReleaseExecuteRequest to release buffered responses.
pub async fn create_release_request(&self, release_all: bool) ->
ReleaseExecuteRequest {
- let mut release = ReleaseExecuteRequest {
+ if release_all {
+ self.release_all_request()
+ } else if let Some(id) = self.last_response_id().await {
+ self.release_until_request(&id)
+ } else {
+ self.release_request(None)
+ }
+ }
+
+ /// A ReleaseExecuteRequest that releases the whole execution.
+ pub fn release_all_request(&self) -> ReleaseExecuteRequest {
+ self.release_request(Some(
+ spark_connect_proto::release_execute_request::Release::ReleaseAll(
+ spark_connect_proto::release_execute_request::ReleaseAll {},
+ ),
+ ))
+ }
+
+ /// A ReleaseExecuteRequest that releases the responses up to and
including `response_id`.
+ pub fn release_until_request(&self, response_id: &str) ->
ReleaseExecuteRequest {
+ self.release_request(Some(
+
spark_connect_proto::release_execute_request::Release::ReleaseUntil(
+ spark_connect_proto::release_execute_request::ReleaseUntil {
+ response_id: response_id.to_string(),
+ },
+ ),
+ ))
+ }
+
+ fn release_request(
+ &self,
+ release: Option<spark_connect_proto::release_execute_request::Release>,
+ ) -> ReleaseExecuteRequest {
+ ReleaseExecuteRequest {
session_id: self.request.session_id.clone(),
user_context: self.request.user_context.clone(),
operation_id: self.operation_id.clone(),
client_type: self.request.client_type.clone(),
+ release,
..Default::default()
- };
-
- if release_all {
- release.release = Some(
-
spark_connect_proto::release_execute_request::Release::ReleaseAll(
- spark_connect_proto::release_execute_request::ReleaseAll
{},
- ),
- );
- } else if let Some(id) = self.last_response_id().await {
- release.release = Some(
-
spark_connect_proto::release_execute_request::Release::ReleaseUntil(
- spark_connect_proto::release_execute_request::ReleaseUntil
{ response_id: id },
- ),
- );
}
-
- release
}
}
@@ -286,6 +304,40 @@ mod tests {
));
}
+ #[test]
+ fn test_release_requests_without_awaiting() {
+ let request = ExecutePlanRequest {
+ session_id: "session-123".to_string(),
+ operation_id: Some("op-456".to_string()),
+ ..Default::default()
+ };
+
+ let iter = ExecutePlanResponseReattachableIterator::new(request);
+
+ let all = iter.release_all_request();
+ assert_eq!(all.session_id, "session-123");
+ assert_eq!(all.operation_id, "op-456".to_string());
+ assert!(matches!(
+ &all.release,
+
Some(spark_connect_proto::release_execute_request::Release::ReleaseAll(_))
+ ));
+
+ let until = iter.release_until_request("response-789");
+ assert_eq!(until.operation_id, "op-456".to_string());
+ assert!(matches!(
+ &until.release,
+
Some(spark_connect_proto::release_execute_request::Release::ReleaseUntil(r))
+ if r.response_id == "response-789"
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_create_release_request_until_without_response() {
+ // Nothing has been read yet, so there is nothing to release until.
+ let iter =
ExecutePlanResponseReattachableIterator::new(ExecutePlanRequest::default());
+ assert!(iter.create_release_request(false).await.release.is_none());
+ }
+
#[tokio::test]
async fn test_state_machine_progression() {
let request = ExecutePlanRequest::default();
diff --git a/crates/spark-connect/src/conf.rs b/crates/spark-connect/src/conf.rs
index d43d611..d90907c 100644
--- a/crates/spark-connect/src/conf.rs
+++ b/crates/spark-connect/src/conf.rs
@@ -70,6 +70,12 @@ impl RuntimeConf {
Ok(results.into_iter().next().flatten())
}
+ /// Get several configuration values in one request, in the order of
`keys`; an unknown
+ /// key errors as in `get`. Mirrors the reference client's
`get_config_dict`.
+ pub fn get_many(&self, keys: &[&str]) -> Result<Vec<Option<String>>> {
+ block_on(self.client.get_configs(keys))
+ }
+
/// Get all configuration values as a HashMap.
///
/// # Example
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 7eb29ac..1ca69c6 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -290,6 +290,19 @@ impl SparkSession {
Ok(DataFrame::new(self.clone(), plan))
}
+ /// Create a DataFrame from rows that are already encoded as an Arrow IPC
stream, whose
+ /// Arrow schema matches `schema`.
+ ///
+ /// Mirrors `pyspark.sql.SparkSession.createDataFrame(pyarrow.Table)`,
which sends the
+ /// table as an Arrow IPC stream in a `LocalRelation`.
+ pub fn create_dataframe_from_arrow(&self, data: Vec<u8>, schema: DataType)
-> DataFrame {
+ let plan = LogicalPlan::LocalRelation {
+ schema,
+ data: Some(data),
+ };
+ DataFrame::new(self.clone(), plan)
+ }
+
/// Create a DataFrameReader for reading data from various sources.
///
/// Mirrors `pyspark.sql.SparkSession.read`.
@@ -1230,6 +1243,21 @@ mod tests {
assert!(!spark.is_stopped());
}
+ #[test]
+ fn create_dataframe_from_arrow_sends_the_stream_and_schema() {
+ let schema = DataType::Struct {
+ fields: vec![struct_field("id", DataType::Long)],
+ };
+ let df = session().create_dataframe_from_arrow(vec![1, 2, 3],
schema.clone());
+ match df.plan().to_proto().rel_type {
+ Some(proto::relation::RelType::LocalRelation(local)) => {
+ assert_eq!(local.data.as_deref(), Some(&[1u8, 2, 3][..]));
+ assert_eq!(local.schema, Some(schema.json()));
+ }
+ other => panic!("expected LocalRelation, got {other:?}"),
+ }
+ }
+
#[test]
fn session_id_is_set() {
let spark = session();
diff --git a/crates/spark-connect/tests/e2e_integration.rs
b/crates/spark-connect/tests/e2e_integration.rs
index 2c137cc..9b1e048 100644
--- a/crates/spark-connect/tests/e2e_integration.rs
+++ b/crates/spark-connect/tests/e2e_integration.rs
@@ -6,15 +6,80 @@
//! Run with: `SPARK_REMOTE=sc://localhost:15002 cargo test --test
e2e_integration`
//! Or simply skip with: `cargo test --test e2e_integration` (will skip all
tests)
+use std::time::Duration;
+
use spark_connect::column::{col, lit};
use spark_connect::row::Value;
use spark_connect::session::SparkSession;
use spark_connect::types::DataType;
+use spark_connect_core::runtime::block_on;
+use spark_connect_core::{ChannelBuilder, SparkConnectClient};
+use spark_connect_proto as proto;
fn should_run() -> bool {
std::env::var("SPARK_REMOTE").is_ok()
}
+fn remote_url() -> String {
+ std::env::var("SPARK_REMOTE").unwrap_or_else(|_|
"sc://localhost:15002".to_string())
+}
+
+async fn connect_client() -> SparkConnectClient {
+ let builder = ChannelBuilder::parse(&remote_url()).expect("Failed to parse
SPARK_REMOTE");
+ SparkConnectClient::connect(&builder)
+ .await
+ .expect("Failed to connect")
+}
+
+/// A request to run `query`, with an operation ID of its own, which it
returns as well.
+fn sql_execute_request(
+ client: &SparkConnectClient,
+ query: &str,
+) -> (proto::ExecutePlanRequest, String) {
+ let operation_id = uuid::Uuid::new_v4().to_string();
+ let mut relation = proto::Relation::default();
+ relation.rel_type = Some(proto::relation::RelType::Sql(proto::Sql {
+ query: query.to_string(),
+ ..Default::default()
+ }));
+ let mut plan = proto::Plan::default();
+ plan.op_type = Some(proto::plan::OpType::Root(relation));
+ let request = proto::ExecutePlanRequest {
+ session_id: client.session_id().to_string(),
+ user_context: Some(proto::UserContext::default()),
+ operation_id: Some(operation_id.clone()),
+ plan: Some(plan),
+ ..Default::default()
+ };
+ (request, operation_id)
+}
+
+/// Waits until the server no longer knows `operation_id`, as the client
releases executions
+/// in the background: `ReattachExecute` of a released execution fails with
+/// `OPERATION_NOT_FOUND`, while it replays the responses of a kept one.
+async fn assert_released(client: &SparkConnectClient, operation_id: &str) {
+ for _ in 0..100 {
+ let request = proto::ReattachExecuteRequest {
+ session_id: client.session_id().to_string(),
+ user_context: Some(proto::UserContext::default()),
+ operation_id: operation_id.to_string(),
+ ..Default::default()
+ };
+ let error = match client.reattach_execute(request).await {
+ Err(e) => e.to_string(),
+ Ok(mut stream) => match stream.message().await {
+ Err(status) => status.message().to_string(),
+ Ok(_) => String::new(),
+ },
+ };
+ if error.contains("OPERATION_NOT_FOUND") {
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ panic!("The server still has operation {operation_id}");
+}
+
#[test]
fn test_range_collect() {
if !should_run() {
@@ -479,6 +544,82 @@ fn test_collect_batch_larger_than_grpc_default_cap() {
);
}
+/// The server ends the response stream of a reattachable execution before the
result is
+/// complete after
`spark.connect.execute.reattachable.senderMaxStreamDuration` or
+/// `senderMaxStreamSize`, and the client reads the rest by reattaching. CI's
server sets a
+/// small `senderMaxStreamSize`, so that this result takes several streams.
+#[test]
+fn test_collect_reads_a_result_across_stream_ends() {
+ if !should_run() {
+ println!(
+ "Skipping test_collect_reads_a_result_across_stream_ends - set
SPARK_REMOTE to run"
+ );
+ return;
+ }
+ let session = SparkSession::builder()
+ .remote(&remote_url())
+ .get_or_create()
+ .expect("Failed to create session");
+
+ let n: i64 = 300_000;
+ let rows = session
+ .range(n)
+ .expect("Failed to create range DataFrame")
+ .collect()
+ .expect("Failed to collect rows");
+
+ assert_eq!(rows.len() as i64, n);
+ let sum: i64 = rows
+ .iter()
+ .map(|row| match row.get(0) {
+ Some(Value::Long(id)) => *id,
+ other => panic!("Expected Long value, got {:?}", other),
+ })
+ .sum();
+ assert_eq!(sum, n * (n - 1) / 2);
+}
+
+/// Once its result is complete, the client releases an execution, so that the
server does not
+/// keep it, with its request and buffered responses, until it is abandoned.
+#[test]
+fn test_completed_execution_is_released() {
+ if !should_run() {
+ println!("Skipping test_completed_execution_is_released - set
SPARK_REMOTE to run");
+ return;
+ }
+ block_on(async {
+ let client = connect_client().await;
+ let (request, operation_id) = sql_execute_request(&client, "SELECT 1");
+ let mut stream = client
+ .execute_plan_reattachable(request)
+ .await
+ .expect("Failed to execute");
+ while stream.message().await.expect("Failed to read").is_some() {}
+ assert_released(&client, &operation_id).await;
+ });
+}
+
+/// A stream dropped before its result is complete releases its execution as
well.
+#[test]
+fn test_dropped_stream_releases_its_execution() {
+ if !should_run() {
+ println!("Skipping test_dropped_stream_releases_its_execution - set
SPARK_REMOTE to run");
+ return;
+ }
+ block_on(async {
+ let client = connect_client().await;
+ let (request, operation_id) =
+ sql_execute_request(&client, "SELECT id FROM range(100000000)");
+ let mut stream = client
+ .execute_plan_reattachable(request)
+ .await
+ .expect("Failed to execute");
+ stream.message().await.expect("Failed to read");
+ drop(stream);
+ assert_released(&client, &operation_id).await;
+ });
+}
+
#[test]
fn test_day_time_interval_real_server() {
if !should_run() {
diff --git a/dev/design/architecture.md b/dev/design/architecture.md
index 7df0659..c7e6f97 100644
--- a/dev/design/architecture.md
+++ b/dev/design/architecture.md
@@ -86,6 +86,9 @@ PyArrow's pandas conversion, which may copy.
Scalar input conversion and row materialization stay in `pyspark-rs`.
`collect()`
still uses the native client's owned rows; `createDataFrame(pandas)` still
materializes Python rows. Neither path is claimed to be zero-copy.
+`createDataFrame(pyarrow.Table)` converts the table with PyArrow as the
reference
+client does (timestamp localization, a cast to the target schema) and sends it
as an
+Arrow IPC stream.
The Arrow handoff has been tested locally with PyArrow 10.0.0 (Python 3.9)
and 26.0.0 (Python 3.12). CI installs the latest PyArrow; it does not pin or
diff --git a/docs/architecture.md b/docs/architecture.md
index c4b5380..020361d 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -102,9 +102,13 @@ sequenceDiagram
```
Transformations are **lazy** - they only build up the plan. Only an *action*
-triggers `ExecutePlan`. If the response stream breaks mid-flight, the core's
-reattachable-execute iterator resumes from the last response it saw, so long
-results survive transient disconnects.
+triggers `ExecutePlan`. If the response stream breaks mid-flight, or the server
+ends it before the result is complete (as it does after
+`spark.connect.execute.reattachable.senderMaxStreamDuration` or
+`senderMaxStreamSize`), the core's reattachable-execute iterator resumes from
the
+last response it saw, so long results survive transient disconnects. Like the
+reference client, it releases each response it has read, and the whole
execution
+once the result is complete, so the server does not keep them.
### Python Arrow table handoff
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
new file mode 100644
index 0000000..2f72056
--- /dev/null
+++ b/python/tests/test_dropin_e2e.py
@@ -0,0 +1,179 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Drop-in behaviors that need a live Spark Connect server. The official tests
run the
+reference pyspark over our transport rather than these Rust-backed classes, so
the expected
+values here are those of the reference client; the tests pass with it as well.
+"""
+
+import os
+from datetime import date, datetime
+
+import pyarrow as pa
+import pytest
+from pyspark.errors import PySparkValueError
+from pyspark.sql import SparkSession
+from pyspark.sql import functions as F
+from pyspark.sql.types import (
+ ArrayType,
+ DateType,
+ DoubleType,
+ IntegerType,
+ LongType,
+ StringType,
+ StructField,
+ StructType,
+ TimestampType,
+)
+
+
[email protected]
+def session():
+ remote = os.environ.get("SPARK_REMOTE")
+ if not remote:
+ pytest.skip("SPARK_REMOTE is required for live-server tests")
+ session = SparkSession.builder.remote(remote).getOrCreate()
+ try:
+ yield session
+ finally:
+ session.stop()
+
+
+def test_conf_get_with_a_none_default(session):
+ key = "spark.connect.rust.test.unset"
+ assert session.conf.get(key, None) is None
+ assert session.conf.get(key, "x") == "x"
+ with pytest.raises(Exception, match="SQL_CONF_NOT_FOUND"):
+ session.conf.get(key)
+
+
[email protected]("by_name", [True, False])
+def test_repartition_by_id_takes_a_column_name(session, by_name):
+ df = session.range(12).withColumn("p", (F.col("id") % 3).cast("int"))
+ partition_id_col = "p" if by_name else F.col("p")
+ rows = (
+ df.repartitionById(3, partition_id_col)
+ .select("p", F.spark_partition_id().alias("pid"))
+ .collect()
+ )
+ assert {(r.p, r.pid) for r in rows} == {(0, 0), (1, 1), (2, 2)}
+
+
[email protected](
+ "schema",
+ [
+ StructType().add("a", LongType()).add("b", LongType()),
+ StructType([StructField("a", LongType())]).add("b", LongType()),
+ ],
+)
+def test_map_in_arrow_and_pandas_with_a_struct_type_built_with_add(session,
schema):
+ df = session.range(3).selectExpr("id AS a", "id * 2 AS b")
+ assert df.mapInArrow(lambda batches: batches, schema).columns == ["a", "b"]
+ rows = df.mapInPandas(lambda frames: frames, schema).orderBy("a").collect()
+ assert [tuple(r) for r in rows] == [(0, 0), (1, 2), (2, 4)]
+
+
+def _arrow_table():
+ return pa.table(
+ {
+ "a": pa.array([1, 2, None], pa.int64()),
+ "b": pa.array(["x", None, "z"]),
+ "ts": pa.array(
+ [datetime(2020, 1, 1, 12), None, datetime(2021, 6, 1)],
pa.timestamp("us")
+ ),
+ "tz": pa.array(
+ [datetime(2020, 1, 1, 12), None, datetime(2021, 6, 1)],
+ pa.timestamp("us", tz="America/Los_Angeles"),
+ ),
+ "d": pa.array([date(2020, 1, 1), None, date(2021, 1, 1)]),
+ "f": pa.array([1.5, None, 2.5]),
+ "l": pa.array([[1, 2], None, []], pa.list_(pa.int32())),
+ }
+ )
+
+
+def _rows(df):
+ # Timestamps as strings in the session time zone, so that the client's
does not matter.
+ columns = [
+ F.col(f.name).cast("string") if isinstance(f.dataType, TimestampType)
else F.col(f.name)
+ for f in df.schema.fields
+ ]
+ return [tuple(r) for r in df.select(*columns).collect()]
+
+
+ARROW_TABLE_ROWS = [
+ (1, "x", "2020-01-01 12:00:00", "2020-01-01 12:00:00", date(2020, 1, 1),
1.5, [1, 2]),
+ (2, None, None, None, None, None, None),
+ (None, "z", "2021-06-01 00:00:00", "2021-06-01 00:00:00", date(2021, 1,
1), 2.5, []),
+]
+
+
[email protected](
+ "schema, expected",
+ [
+ (None,
"struct<a:bigint,b:string,ts:timestamp,tz:timestamp,d:date,f:double,l:array<int>>"),
+ (
+ ["x", "y"],
+
"struct<x:bigint,y:string,_3:timestamp,_4:timestamp,_5:date,_6:double,_7:array<int>>",
+ ),
+ (
+ "a int, b string, ts timestamp, tz timestamp, d date, f double, l
array<int>",
+
"struct<a:int,b:string,ts:timestamp,tz:timestamp,d:date,f:double,l:array<int>>",
+ ),
+ (
+ StructType(
+ [
+ StructField("a", IntegerType()),
+ StructField("b", StringType()),
+ StructField("ts", TimestampType()),
+ StructField("tz", TimestampType()),
+ StructField("d", DateType()),
+ StructField("f", DoubleType()),
+ StructField("l", ArrayType(IntegerType())),
+ ]
+ ),
+
"struct<a:int,b:string,ts:timestamp,tz:timestamp,d:date,f:double,l:array<int>>",
+ ),
+ ],
+)
+def test_create_dataframe_from_arrow_table(session, schema, expected):
+ session.conf.set("spark.sql.session.timeZone", "UTC")
+ if schema is None:
+ df = session.createDataFrame(_arrow_table())
+ else:
+ df = session.createDataFrame(_arrow_table(), schema)
+ assert df.schema.simpleString() == expected
+ assert _rows(df) == ARROW_TABLE_ROWS
+
+
+def test_create_dataframe_from_empty_arrow_table(session):
+ empty = pa.table({"a": pa.array([], pa.int64()), "b": pa.array([],
pa.string())})
+ df = session.createDataFrame(empty, "a long, b string")
+ assert df.schema.simpleString() == "struct<a:bigint,b:string>"
+ assert df.count() == 0
+ # Without types, an empty table has no schema to infer, as an empty list
does not.
+ for args in ((empty,), (empty, ["x", "y"])):
+ with pytest.raises(PySparkValueError) as error:
+ session.createDataFrame(*args)
+ assert error.value.getCondition() == "CANNOT_INFER_EMPTY_SCHEMA"
+
+
+def test_collect_reads_a_result_across_stream_ends(session):
+ # Takes several response streams when the server's senderMaxStreamSize is
small.
+ n = 300_000
+ assert len(session.range(n).collect()) == n
+ assert session.range(n).toArrow().num_rows == n
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 97f32b8..2a027bb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -412,6 +412,16 @@ def test_structtype_add_reflected_in_json():
assert st.fieldNames() == ["a", "b"]
+def test_structtype_add_reflected_in_the_base_data_type():
+ # Regression: add() must update the base DataType too, which str() and the
schemas of
+ # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype).
+ built = T.StructType([T.StructField("a", T.LongType()), T.StructField("b",
T.LongType())])
+ assert str(T.StructType().add("a", T.LongType()).add("b", T.LongType()))
== str(built)
+ st = T.StructType([T.StructField("a", T.LongType())])
+ st.add("b", T.LongType())
+ assert str(st) == str(built) == "struct<a:bigint,b:bigint>"
+
+
def test_datatype_object_model():
# json / jsonValue / typeName / simpleString / needConversion match
official pyspark.
assert T.IntegerType().typeName() == "integer"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]