This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new c2d0654  [SPARK-60960][CONNECT][PYTHON] Keep executionInfo per 
DataFrame, and the operation id of every ExecutePlan error, as PySpark does
c2d0654 is described below

commit c2d0654a22ca6b6b9e1f4712009d15798fa41598
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 14:45:23 2026 +0900

    [SPARK-60960][CONNECT][PYTHON] Keep executionInfo per DataFrame, and the 
operation id of every ExecutePlan error, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60960](https://issues.apache.org/jira/browse/SPARK-60960) has two 
parts.
    
    **`executionInfo` per DataFrame.** `DataFrame.executionInfo` is now the 
information of the last execution of that DataFrame, and `None` before one, as 
PySpark's `DataFrame._execution_info` is. Before, it was the session's last 
execution, of any DataFrame.
    
    - These set it, as in PySpark: `collect`, the Arrow and pandas collects, 
`write`, `writeTo` and `mergeInto`, the temp views, and the checkpoints.
    - An action that runs a derived DataFrame, such as `count`, `show`, `first` 
or `take`, sets that DataFrame's, as in PySpark, not the original's.
    - Exact clones of a DataFrame share the information, and a DataFrame of 
another plan has its own.
    - The session still records its last execution, for `profile()`.
    
    **Operation ids on errors.** The catalog operations and 
`writeStream.start()` now run on the reattachable path, as the other 
`ExecutePlan` requests do. So their errors carry the operation id of the 
request, as PySpark's do. `start()` now also sends the session's tags.
    
    ### Why are the changes needed?
    
    - For `a = spark.range(3); a.collect(); b = spark.range(5)`, 
`b.executionInfo` gave the information of `a`'s execution, where PySpark gives 
`None`. Any DataFrame showed the metrics and operation id of the last execution 
in the session, also across threads.
    - `spark.catalog.setCurrentDatabase("nope")` raised an `AnalysisException` 
whose `operation_id` was `None`, and so did a failing `writeStream.start()`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `executionInfo` and these errors behave as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, a probe of 13 cases gives the same 
results as official PySpark 4.3.0: `executionInfo` before and after `collect`, 
on other and derived DataFrames, after `count`, `show`, `first` and `take`, 
temp views, `toArrow`, `toPandas`, a write and distinct operation ids, and the 
two errors. Before this change, six of them differed.
    - New live tests check those cases and errors. They also pass with official 
PySpark 4.3.0, and fail without this change.
    - The offline tests of `executionInfo` read it from the DataFrame that ran, 
and check that another one has none.
    - The lib tests and every integration test, e2e included, against the local 
server (647 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (687 passed, 9 skipped), and the live drop-in and Arrow suites (72 
passed) and live cases (10 passed). Clippy reports no diagnostics in the 
changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #193 from HyukjinKwon/SPARK-60960-execution-info.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/catalog.rs    | 22 +++----------
 crates/spark-connect/src/dataframe.rs  | 56 ++++++++++++++++++++++++--------
 crates/spark-connect/src/merge.rs      | 44 ++++++++++++++++++--------
 crates/spark-connect/src/readwriter.rs | 58 ++++++++++++++++++++++++----------
 crates/spark-connect/src/streaming.rs  | 35 ++++++--------------
 python/tests/test_dropin_e2e.py        | 32 +++++++++++++++++++
 python/tests/test_dropin_offline.py    | 12 ++++---
 7 files changed, 168 insertions(+), 91 deletions(-)

diff --git a/crates/spark-connect/src/catalog.rs 
b/crates/spark-connect/src/catalog.rs
index f6a3201..8915923 100644
--- a/crates/spark-connect/src/catalog.rs
+++ b/crates/spark-connect/src/catalog.rs
@@ -1029,29 +1029,17 @@ impl Catalog {
 
     fn execute_catalog(&self, catalog: &proto::Catalog) -> Result<Vec<Row>> {
         let request = self.build_execute_catalog_request(catalog)?;
-        let mut stream = 
block_on(self.session.client().execute_plan(request))?;
-
+        // On the reattachable path, as PySpark's client runs a catalog 
operation, so that an
+        // error carries the operation id of the request.
+        let mut stream = 
block_on(self.session.client().execute_plan_reattachable(request))?;
         let mut rows = vec![];
-
-        loop {
-            let client = self.session.client();
-            let resp = block_on(async {
-                match stream.message().await {
-                    Ok(resp) => Ok(resp),
-                    Err(status) => Err(client.error_from_status(status).await),
-                }
-            })?;
-            let Some(resp) = resp else {
-                break;
-            };
+        while let Some(resp) = block_on(stream.message())? {
             if let 
Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) =
                 resp.response_type
             {
-                let batch_rows = decode_arrow_batch(&batch)?;
-                rows.extend(batch_rows);
+                rows.extend(decode_arrow_batch(&batch)?);
             }
         }
-
         Ok(rows)
     }
 
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 7e64cf2..7ddf16e 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -30,8 +30,15 @@ pub struct DataFrame {
     // resolved schema, but have their own cache for subsequent analysis.
     // The resolved value is immutable, so propagation need not copy wide 
schemas.
     schema_cache: Arc<Mutex<Option<Arc<DataType>>>>,
+    // The information of the last execution of this DataFrame, as PySpark's
+    // `DataFrame._execution_info`: exact clones share it, and a derived 
DataFrame, of
+    // another plan, has its own, which is empty until it executes.
+    execution: ExecutionSlot,
 }
 
+/// Where a DataFrame keeps the information of its last execution.
+pub(crate) type ExecutionSlot = Arc<Mutex<Option<ExecutionInfo>>>;
+
 /// Iterator over rows from a DataFrame, yielded lazily as the server streams 
results.
 ///
 /// Returned by `DataFrame::to_local_iterator()`, this iterator consumes the 
ExecutePlan
@@ -234,9 +241,17 @@ impl DataFrame {
             session,
             plan: Arc::new(plan),
             schema_cache: Arc::new(Mutex::new(schema)),
+            execution: Arc::new(Mutex::new(None)),
         }
     }
 
+    /// Keep `info`, the information of an execution of this DataFrame, and 
record it as
+    /// the session's most recent one.
+    pub(crate) fn record_execution(&self, info: ExecutionInfo) {
+        *self.execution.lock().unwrap() = Some(info.clone());
+        self.session.record_execution(info);
+    }
+
     fn cached_schema(&self) -> Option<Arc<DataType>> {
         // Propagation snapshots resolved metadata; it must not wait for an
         // in-flight analysis. Python transformations can still hold the GIL.
@@ -717,7 +732,7 @@ impl DataFrame {
                 rows.extend(batch_rows);
             }
         }
-        self.session.record_execution(info);
+        self.record_execution(info);
 
         Ok(rows)
     }
@@ -743,13 +758,13 @@ impl DataFrame {
         ))
     }
 
-    /// Execution metrics collected during the most recent action on this
-    /// DataFrame's session. Mirrors `pyspark.sql.DataFrame.executionInfo`.
-    ///
-    /// The metrics reflect the session's most recent execution; call this 
right
-    /// after an action (e.g. `collect`/`count`/`show`).
+    /// The information of the last execution of this DataFrame. Mirrors
+    /// `pyspark.sql.DataFrame.executionInfo`: set by `collect`, the Arrow and 
pandas
+    /// collects, the writes, the temp views and the checkpoints of this 
DataFrame, and empty
+    /// before one. An action that runs a derived DataFrame, such as `count` 
or `show`, sets
+    /// that one's, as in PySpark.
     pub fn execution_info(&self) -> Result<ExecutionInfo> {
-        self.session.last_execution_info().ok_or_else(|| {
+        self.execution.lock().unwrap().clone().ok_or_else(|| {
             SparkError::connect_msg("no execution info available; run an 
action first")
         })
     }
@@ -778,7 +793,7 @@ impl DataFrame {
                 batches.extend(record_batches);
             }
         }
-        self.session.record_execution(info);
+        self.record_execution(info);
 
         Ok(batches)
     }
@@ -947,6 +962,7 @@ impl DataFrame {
     /// Mirrors `pyspark.sql.DataFrame.write`.
     pub fn write(&self) -> crate::readwriter::DataFrameWriter {
         crate::readwriter::DataFrameWriter::new(self.session.clone(), 
Arc::clone(&self.plan))
+            .recording_into(Arc::clone(&self.execution))
     }
 
     /// Create a [`DataFrameWriterV2`](crate::readwriter::DataFrameWriterV2) 
for
@@ -959,6 +975,7 @@ impl DataFrame {
             Arc::clone(&self.plan),
             table_name,
         )
+        .recording_into(Arc::clone(&self.execution))
     }
 
     /// Merge a set of updates, insertions, and deletions into a target table.
@@ -973,6 +990,7 @@ impl DataFrame {
             table.to_string(),
             condition,
         )
+        .recording_into(Arc::clone(&self.execution))
     }
 
     /// Create a DataStreamWriter for writing this streaming DataFrame to 
various sinks.
@@ -1055,10 +1073,11 @@ impl DataFrame {
         cmd.relation = Some(self.relation());
         cmd.local = local;
         cmd.eager = eager;
-        let responses = execute_command_collect(
+        let (responses, info) = execute_command_with_info(
             &self.session,
             proto::command::CommandType::CheckpointCommand(cmd),
         )?;
+        self.record_execution(info);
         for resp in &responses {
             if let 
Some(proto::execute_plan_response::ResponseType::CheckpointCommandResult(res)) =
                 &resp.response_type
@@ -1107,10 +1126,12 @@ impl DataFrame {
         cmd.name = name.to_string();
         cmd.is_global = global;
         cmd.replace = replace;
-        execute_command(
+        let (_, info) = execute_command_with_info(
             &self.session,
             proto::command::CommandType::CreateDataframeView(cmd),
-        )
+        )?;
+        self.record_execution(info);
+        Ok(())
     }
 
     /// Print the execution plan to the console. Mirrors 
`pyspark.sql.DataFrame.explain`
@@ -1881,6 +1902,15 @@ pub(crate) fn execute_command_collect(
     session: &SparkSession,
     command_type: proto::command::CommandType,
 ) -> Result<Vec<proto::ExecutePlanResponse>> {
+    execute_command_with_info(session, command_type).map(|(responses, _)| 
responses)
+}
+
+/// Like `execute_command_collect`, and also returns the information of the 
execution, which
+/// it records as the session's most recent one, for a DataFrame that keeps 
its own.
+pub(crate) fn execute_command_with_info(
+    session: &SparkSession,
+    command_type: proto::command::CommandType,
+) -> Result<(Vec<proto::ExecutePlanResponse>, ExecutionInfo)> {
     let mut command = proto::Command::default();
     command.command_type = Some(command_type);
 
@@ -1902,8 +1932,8 @@ pub(crate) fn execute_command_collect(
         capture_execution(&mut resp, &mut info, session);
         responses.push(resp);
     }
-    session.record_execution(info);
-    Ok(responses)
+    session.record_execution(info.clone());
+    Ok((responses, info))
 }
 
 /// Pull execution metrics/observed-metrics off a response and fire progress
diff --git a/crates/spark-connect/src/merge.rs 
b/crates/spark-connect/src/merge.rs
index e03d914..e7ad891 100644
--- a/crates/spark-connect/src/merge.rs
+++ b/crates/spark-connect/src/merge.rs
@@ -10,7 +10,7 @@ use spark_connect_core::error::Result;
 use spark_connect_proto as proto;
 
 use crate::column::Column;
-use crate::dataframe::{build_input_relation, execute_command};
+use crate::dataframe::build_input_relation;
 use crate::plan::LogicalPlan;
 use crate::session::SparkSession;
 
@@ -48,6 +48,7 @@ pub struct MergeIntoWriter {
     matched_actions: Vec<proto::Expression>,
     not_matched_actions: Vec<proto::Expression>,
     not_matched_by_source_actions: Vec<proto::Expression>,
+    execution: Option<crate::dataframe::ExecutionSlot>,
 }
 
 impl MergeIntoWriter {
@@ -71,9 +72,26 @@ impl MergeIntoWriter {
             matched_actions: Vec::new(),
             not_matched_actions: Vec::new(),
             not_matched_by_source_actions: Vec::new(),
+            execution: None,
         }
     }
 
+    /// Keep the information of this merge's execution as the last execution 
of its
+    /// DataFrame, as PySpark's MergeIntoWriter reports it.
+    pub(crate) fn recording_into(mut self, execution: 
crate::dataframe::ExecutionSlot) -> Self {
+        self.execution = Some(execution);
+        self
+    }
+
+    /// Run the command of the merge, keeping its execution as its DataFrame's 
last one.
+    fn run(&self, command_type: proto::command::CommandType) -> Result<()> {
+        let (_, info) = 
crate::dataframe::execute_command_with_info(&self.session, command_type)?;
+        if let Some(execution) = &self.execution {
+            *execution.lock().unwrap() = Some(info);
+        }
+        Ok(())
+    }
+
     /// Begin a `WHEN MATCHED [AND condition]` clause.
     pub fn when_matched(self, condition: Option<Column>) -> WhenMatched {
         WhenMatched {
@@ -105,19 +123,17 @@ impl MergeIntoWriter {
     }
 
     /// Execute the merge.
-    pub fn merge(self) -> Result<()> {
-        let mut cmd = proto::MergeIntoTableCommand::default();
-        cmd.target_table_name = self.target_table;
-        cmd.source_table_plan = Some(build_input_relation(&self.source_plan, 
&self.session));
-        cmd.merge_condition = Some(self.condition.to_proto());
-        cmd.match_actions = self.matched_actions;
-        cmd.not_matched_actions = self.not_matched_actions;
-        cmd.not_matched_by_source_actions = self.not_matched_by_source_actions;
-        cmd.with_schema_evolution = self.schema_evolution;
-        execute_command(
-            &self.session,
-            proto::command::CommandType::MergeIntoTableCommand(cmd),
-        )
+    pub fn merge(mut self) -> Result<()> {
+        let cmd = proto::MergeIntoTableCommand {
+            target_table_name: std::mem::take(&mut self.target_table),
+            source_table_plan: Some(build_input_relation(&self.source_plan, 
&self.session)),
+            merge_condition: Some(self.condition.to_proto()),
+            match_actions: std::mem::take(&mut self.matched_actions),
+            not_matched_actions: std::mem::take(&mut self.not_matched_actions),
+            not_matched_by_source_actions: std::mem::take(&mut 
self.not_matched_by_source_actions),
+            with_schema_evolution: self.schema_evolution,
+        };
+        self.run(proto::command::CommandType::MergeIntoTableCommand(cmd))
     }
 }
 
diff --git a/crates/spark-connect/src/readwriter.rs 
b/crates/spark-connect/src/readwriter.rs
index ad08c94..5891641 100644
--- a/crates/spark-connect/src/readwriter.rs
+++ b/crates/spark-connect/src/readwriter.rs
@@ -9,7 +9,7 @@ use spark_connect_core::error::Result;
 use spark_connect_proto as proto;
 
 use crate::column::Column;
-use crate::dataframe::{build_input_relation, execute_command, DataFrame};
+use crate::dataframe::{build_input_relation, DataFrame};
 use crate::plan::LogicalPlan;
 use crate::session::SparkSession;
 
@@ -358,6 +358,7 @@ pub struct DataFrameWriter {
     bucket_cols: Vec<String>,
     sort_cols: Vec<String>,
     num_buckets: Option<i32>,
+    execution: Option<crate::dataframe::ExecutionSlot>,
 }
 
 impl DataFrameWriter {
@@ -379,9 +380,26 @@ impl DataFrameWriter {
             bucket_cols: vec![],
             sort_cols: vec![],
             num_buckets: None,
+            execution: None,
         }
     }
 
+    /// Keep the information of this write's execution as the last execution 
of its
+    /// DataFrame, as PySpark's writers report it to theirs.
+    pub(crate) fn recording_into(mut self, execution: 
crate::dataframe::ExecutionSlot) -> Self {
+        self.execution = Some(execution);
+        self
+    }
+
+    /// Run the command of the write, keeping its execution as its DataFrame's 
last one.
+    fn run(&self, command_type: proto::command::CommandType) -> Result<()> {
+        let (_, info) = 
crate::dataframe::execute_command_with_info(&self.session, command_type)?;
+        if let Some(execution) = &self.execution {
+            *execution.lock().unwrap() = Some(info);
+        }
+        Ok(())
+    }
+
     /// Cluster the output by the given columns (liquid clustering).
     pub fn cluster_by<S: Into<String>>(mut self, cols: impl IntoIterator<Item 
= S>) -> Self {
         self.cluster_cols = cols.into_iter().map(Into::into).collect();
@@ -466,20 +484,14 @@ impl DataFrameWriter {
         table.save_method = method.to_proto();
         let op =
             
self.build_write_operation(Some(proto::write_operation::SaveType::Table(table)))?;
-        execute_command(
-            &self.session,
-            proto::command::CommandType::WriteOperation(op),
-        )
+        self.run(proto::command::CommandType::WriteOperation(op))
     }
 
     /// Save the DataFrame to a file path (or to a path-less sink such as 
`noop`).
     pub fn save(self, path: Option<&str>) -> Result<()> {
         let save_type = path.map(|p| 
proto::write_operation::SaveType::Path(p.to_string()));
         let op = self.build_write_operation(save_type)?;
-        execute_command(
-            &self.session,
-            proto::command::CommandType::WriteOperation(op),
-        )
+        self.run(proto::command::CommandType::WriteOperation(op))
     }
 
     /// Save the DataFrame as a managed table.
@@ -542,6 +554,7 @@ pub struct DataFrameWriterV2 {
     table_properties: HashMap<String, String>,
     partition_cols: Vec<Column>,
     cluster_cols: Vec<String>,
+    execution: Option<crate::dataframe::ExecutionSlot>,
 }
 
 impl DataFrameWriterV2 {
@@ -565,7 +578,24 @@ impl DataFrameWriterV2 {
             table_properties: HashMap::new(),
             partition_cols: vec![],
             cluster_cols: vec![],
+            execution: None,
+        }
+    }
+
+    /// Keep the information of this write's execution as the last execution 
of its
+    /// DataFrame, as PySpark's writers report it to theirs.
+    pub(crate) fn recording_into(mut self, execution: 
crate::dataframe::ExecutionSlot) -> Self {
+        self.execution = Some(execution);
+        self
+    }
+
+    /// Run the command of the write, keeping its execution as its DataFrame's 
last one.
+    fn run(&self, command_type: proto::command::CommandType) -> Result<()> {
+        let (_, info) = 
crate::dataframe::execute_command_with_info(&self.session, command_type)?;
+        if let Some(execution) = &self.execution {
+            *execution.lock().unwrap() = Some(info);
         }
+        Ok(())
     }
 
     /// Cluster the output table by the given columns (liquid clustering).
@@ -626,10 +656,7 @@ impl DataFrameWriterV2 {
 
     fn execute(self, mode: proto::write_operation_v2::Mode) -> Result<()> {
         let op = self.build_operation(mode, None)?;
-        execute_command(
-            &self.session,
-            proto::command::CommandType::WriteOperationV2(op),
-        )
+        self.run(proto::command::CommandType::WriteOperationV2(op))
     }
 
     /// Create a new table from the DataFrame.
@@ -658,10 +685,7 @@ impl DataFrameWriterV2 {
             proto::write_operation_v2::Mode::Overwrite,
             Some(condition.to_proto()),
         )?;
-        execute_command(
-            &self.session,
-            proto::command::CommandType::WriteOperationV2(op),
-        )
+        self.run(proto::command::CommandType::WriteOperationV2(op))
     }
 
     /// Overwrite all partitions touched by the DataFrame (dynamic overwrite).
diff --git a/crates/spark-connect/src/streaming.rs 
b/crates/spark-connect/src/streaming.rs
index 18b5494..a5b4b79 100644
--- a/crates/spark-connect/src/streaming.rs
+++ b/crates/spark-connect/src/streaming.rs
@@ -457,35 +457,18 @@ impl DataStreamWriter {
             write_op.foreach_writer = Some(foreach_func);
         }
 
-        // Build the plan with the WriteStreamOperationStart command
-        let mut plan = proto::Plan::default();
-        let mut cmd = proto::Command::default();
-        cmd.command_type = 
Some(proto::command::CommandType::WriteStreamOperationStart(
-            write_op,
-        ));
-        plan.op_type = Some(proto::plan::OpType::Command(cmd));
-
-        // Create ExecutePlanRequest
-        let request = proto::ExecutePlanRequest {
-            session_id: self.session.client().session_id().to_string(),
-            user_context: Some(proto::UserContext::default()),
-            plan: Some(plan),
-            ..Default::default()
-        };
-
-        // Execute the plan and read the WriteStreamOperationStartResult, 
which carries
-        // the server-assigned query id / run id / name.
-        let mut response_stream = 
block_on(self.session.client().execute_plan(request))?;
+        // Run the WriteStreamOperationStart command, as other commands are, 
on the
+        // reattachable path with the session's tags, so that an error carries 
the operation
+        // id of the request, as in PySpark. Its 
WriteStreamOperationStartResult carries the
+        // server-assigned query id / run id / name.
+        let responses = crate::dataframe::execute_command_collect(
+            &self.session,
+            proto::command::CommandType::WriteStreamOperationStart(write_op),
+        )?;
         let mut query_id = String::new();
         let mut run_id = String::new();
         let mut name = self.query_name.clone();
-        let client = self.session.client();
-        while let Some(resp) = block_on(async {
-            match response_stream.message().await {
-                Ok(resp) => Ok(resp),
-                Err(status) => Err(client.error_from_status(status).await),
-            }
-        })? {
+        for resp in responses {
             if let Some(
                 
proto::execute_plan_response::ResponseType::WriteStreamOperationStartResult(res),
             ) = resp.response_type
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index bc0bc9b..15ab7cf 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -273,6 +273,38 @@ def test_add_artifacts_reach_udfs_as_pyspark(session, 
tmp_path):
     assert session.range(1).select(read("id")).first()[0] == "from the 
module|file content|inside"
 
 
+def test_execution_info_is_that_of_each_dataframe_as_pyspark(session):
+    a = session.range(3)
+    assert a.executionInfo is None
+    a.collect()
+    assert a.executionInfo is not None
+    # Another DataFrame, and one that an action runs derived from it, have 
none.
+    b = session.range(5)
+    b.count()
+    b.first()
+    assert b.executionInfo is None and a.select("id").executionInfo is None
+    for run in (
+        lambda df: df.toArrow(),
+        lambda df: df.toPandas(),
+        lambda df: df.createOrReplaceTempView("execution_info_view"),
+        lambda df: df.write.mode("overwrite").format("noop").save(),
+    ):
+        df = session.range(2)
+        run(df)
+        assert df.executionInfo is not None
+        assert df.executionInfo.operation_id != a.executionInfo.operation_id
+
+
+def test_catalog_and_stream_start_errors_carry_their_operation_id(session):
+    for run in (
+        lambda: session.catalog.setCurrentDatabase("no_such_database_60960"),
+        lambda: 
session.readStream.format("rate").load().writeStream.format("no_such_sink_60960").start(),
+    ):
+        with pytest.raises(Exception) as error:
+            run()
+        assert isinstance(error.value.operation_id, str) and 
error.value.operation_id
+
+
 def test_create_dataframe_from_empty_arrow_table(session):
     empty = pa.table({"a": pa.array([], pa.int64()), "b": pa.array([], 
pa.string())})
     df = session.createDataFrame(empty, "a long, b string")
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index d7d7296..849835e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4774,8 +4774,11 @@ def 
test_execution_info_decodes_observed_metrics_and_operation_id(mock_connect):
     service.execute_payload = message_field(6, observed)
 
     assert spark.range(1).executionInfo is None
-    spark.range(1).collect()
-    info = spark.range(1).executionInfo
+    df = spark.range(1)
+    df.collect()
+    info = df.executionInfo
+    # The information is that of the DataFrame that ran, as in PySpark.
+    assert spark.range(1).executionInfo is None
     assert info.operation_id == 
_request_operation_id(service.execute_requests[-1])
     ((name, pairs),) = info.flows
     assert name == "o"
@@ -4794,8 +4797,9 @@ def 
test_execution_info_unsupported_literal_raises_on_flows(mock_connect):
     # Literal.calendar_interval=19 (months=1): the reference rejects it in 
_to_value.
     observed = message_field(1, "o") + message_field(2, message_field(19, 
varint_field(1, 2)))
     service.execute_payload = message_field(6, observed)
-    spark.range(1).collect()
-    info = spark.range(1).executionInfo
+    df = spark.range(1)
+    df.collect()
+    info = df.executionInfo
     assert info.operation_id == 
_request_operation_id(service.execute_requests[-1])
     with pytest.raises(PySparkTypeError, match="UNSUPPORTED_LITERAL"):
         info.flows


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to