This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7135f819 [SPARK-61171][CONNECT][PYTHON] Let the objects of the Python
client be referred to weakly and take attributes, as those of PySpark
7135f819 is described below
commit 7135f8194ca03d19c851db57ca14e9557920ef91
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 17:27:09 2026 +0900
[SPARK-61171][CONNECT][PYTHON] Let the objects of the Python client be
referred to weakly and take attributes, as those of PySpark
### What changes were proposed in this pull request?
[SPARK-61171](https://issues.apache.org/jira/browse/SPARK-61171): PySpark's
classes of Spark Connect are Python classes. Their objects can be referred to
weakly (`weakref.ref`, `WeakKeyDictionary`, `WeakValueDictionary`) and take
attributes of their own, as libraries that cache by weak reference or tag a
DataFrame do.
The public classes of the client's Rust extension now support both too,
with PyO3's `weakref` and `dict`, 31 classes:
- `SparkSession`, `DataFrame`, `Column`, `GroupedData`,
`PandasCogroupedOps`, `WindowSpec`
- `DataFrameReader`, `DataFrameWriter`, `DataFrameWriterV2`,
`MergeIntoWriter`
- `DataStreamReader`, `DataStreamWriter`, `StreamingQuery`,
`StreamingQueryManager`
- `Catalog`, `RuntimeConf`, `DataFrameNaFunctions`, `DataFrameStatFunctions`
- `TableValuedFunction`, `TableArg`, `DataSourceRegistration`, `Observation`
- `DataType`, whose subclasses, the data types, inherit them, and
`StructField`
- `VariantVal`, `Geometry`, `Geography`
- `ExecutorResourceRequests`, `TaskResourceRequests`,
`ResourceProfileBuilder`, `ResourceProfile`
An attribute that an object takes is found before the `__getattr__` of
`DataFrame` and `Column`, which give columns and fields, as with PySpark's.
### Why are the changes needed?
They refused both:
```
TypeError: cannot create weak reference to 'DataFrame' object
AttributeError: 'pyspark.sql.connect.dataframe.DataFrame' object has no
attribute '...'
```
### Does this PR introduce _any_ user-facing change?
Yes. The objects of these classes can be referred to weakly and take
attributes, as with PySpark 4.3.0.
### How was this patch tested?
- A new offline test refers weakly to an object of each of 22 of these
classes, and sets, reads and deletes an attribute on it. It fails without this
change, and its body passes with official PySpark 4.3.0.
- The weak references and attributes of the main classes against a local
server are those of official PySpark 4.3.0, where only the `UDFRegistration` of
the client supported both before this change.
- The lib tests of both crates (597 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2223 passed, 8 skipped), and the
live drop-in and Arrow suites (206 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #360 from HyukjinKwon/SPARK-61171-weakref-dict.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/catalog.rs | 2 +-
crates/pyspark-rs/src/column.rs | 2 +-
crates/pyspark-rs/src/conf.rs | 7 ++++++-
crates/pyspark-rs/src/dataframe.rs | 31 ++++++++++++++++++++++++++-----
crates/pyspark-rs/src/datasource.rs | 7 ++++++-
crates/pyspark-rs/src/group.rs | 9 +++++++--
crates/pyspark-rs/src/observation.rs | 7 ++++++-
crates/pyspark-rs/src/readwriter.rs | 7 ++++++-
crates/pyspark-rs/src/resource.rs | 28 ++++++++++++++++++++++++----
crates/pyspark-rs/src/session.rs | 2 +-
crates/pyspark-rs/src/stat.rs | 7 ++++++-
crates/pyspark-rs/src/streaming.rs | 28 ++++++++++++++++++++++++----
crates/pyspark-rs/src/tablearg.rs | 2 +-
crates/pyspark-rs/src/tvf.rs | 7 ++++++-
crates/pyspark-rs/src/types.rs | 10 ++++++++--
crates/pyspark-rs/src/values.rs | 6 +++---
crates/pyspark-rs/src/window.rs | 8 +++++++-
python/tests/test_dropin_offline.py | 25 +++++++++++++++++++++++++
18 files changed, 164 insertions(+), 31 deletions(-)
diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index 754fa327..80680ba5 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -168,7 +168,7 @@ fn named_tuples<T: NamedTuple>(py: Python<'_>, results:
Vec<T>) -> PyResult<Vec<
}
/// Python wrapper for Spark Catalog.
-#[pyclass(name = "Catalog", module = "pyspark.sql.catalog")]
+#[pyclass(name = "Catalog", module = "pyspark.sql.catalog", weakref, dict)]
pub struct PyCatalog {
pub(crate) catalog: Catalog,
}
diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs
index e948a22b..6617f730 100644
--- a/crates/pyspark-rs/src/column.rs
+++ b/crates/pyspark-rs/src/column.rs
@@ -78,7 +78,7 @@ pub(crate) fn column_reference_name(column: PyRef<'_,
PyColumn>) -> Option<Strin
}
/// Python wrapper for a Spark Column.
-#[pyclass(name = "Column", module = "pyspark.sql.connect.column")]
+#[pyclass(name = "Column", module = "pyspark.sql.connect.column", weakref,
dict)]
pub struct PyColumn {
pub(crate) column: Column,
}
diff --git a/crates/pyspark-rs/src/conf.rs b/crates/pyspark-rs/src/conf.rs
index 156a9216..7591dba1 100644
--- a/crates/pyspark-rs/src/conf.rs
+++ b/crates/pyspark-rs/src/conf.rs
@@ -6,7 +6,12 @@ use spark_connect::conf::RuntimeConf;
use crate::errors::ResultExt;
/// Python wrapper for the session runtime configuration.
-#[pyclass(name = "RuntimeConf", module = "pyspark.sql.connect.conf")]
+#[pyclass(
+ name = "RuntimeConf",
+ module = "pyspark.sql.connect.conf",
+ weakref,
+ dict
+)]
pub struct PyRuntimeConf {
conf: RuntimeConf,
}
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 88141df3..4a578dcf 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -43,7 +43,13 @@ fn flatten_str_cols(cols: Vec<Bound<'_, PyAny>>) ->
PyResult<Vec<String>> {
}
/// Python wrapper for a Spark DataFrame.
-#[pyclass(name = "DataFrame", module = "pyspark.sql.connect.dataframe",
subclass)]
+#[pyclass(
+ name = "DataFrame",
+ module = "pyspark.sql.connect.dataframe",
+ subclass,
+ weakref,
+ dict
+)]
pub struct PyDataFrame {
pub(crate) dataframe: DataFrame,
/// Whether `_repr_html_` was called, as PySpark's `_support_repr_html`:
then `__repr__` is
@@ -2721,7 +2727,12 @@ impl PyDataFrame {
}
/// Python wrapper for `DataFrameNaFunctions` (`df.na`).
-#[pyclass(name = "DataFrameNaFunctions", module = "pyspark.sql.dataframe")]
+#[pyclass(
+ name = "DataFrameNaFunctions",
+ module = "pyspark.sql.dataframe",
+ weakref,
+ dict
+)]
pub struct PyDataFrameNaFunctions {
df: spark_connect::dataframe::DataFrame,
/// The DataFrame of `df.na`, as PySpark's `DataFrameNaFunctions.df`.
@@ -3002,7 +3013,7 @@ pub(crate) fn column_or_name(col: &Bound<'_, PyAny>) ->
PyResult<spark_connect::
/// (`whenMatched().updateAll()`, ...) change this object and return it, and
`merge`
/// leaves it usable. The core writer is a consuming builder, so the wrapped
value is
/// replaced in place and `merge` runs on a clone.
-#[pyclass(name = "MergeIntoWriter", module = "pyspark.sql.merge")]
+#[pyclass(name = "MergeIntoWriter", module = "pyspark.sql.merge", weakref,
dict)]
pub struct PyMergeIntoWriter {
inner: WriterState<spark_connect::MergeIntoWriter>,
}
@@ -3186,7 +3197,12 @@ impl PyWhenNotMatchedBySource {
}
/// Python wrapper for the V1 `DataFrameWriter` (`df.write`).
-#[pyclass(name = "DataFrameWriter", module = "pyspark.sql.readwriter")]
+#[pyclass(
+ name = "DataFrameWriter",
+ module = "pyspark.sql.readwriter",
+ weakref,
+ dict
+)]
pub struct PyDataFrameWriter {
inner: WriterState<spark_connect::readwriter::DataFrameWriter>,
}
@@ -3627,7 +3643,12 @@ impl PyDataFrameWriter {
}
/// Python wrapper for the V2 `DataFrameWriter` (`df.writeTo`).
-#[pyclass(name = "DataFrameWriterV2", module = "pyspark.sql.readwriter")]
+#[pyclass(
+ name = "DataFrameWriterV2",
+ module = "pyspark.sql.readwriter",
+ weakref,
+ dict
+)]
pub struct PyDataFrameWriterV2 {
inner: WriterState<spark_connect::readwriter::DataFrameWriterV2>,
}
diff --git a/crates/pyspark-rs/src/datasource.rs
b/crates/pyspark-rs/src/datasource.rs
index d6b81042..521d921c 100644
--- a/crates/pyspark-rs/src/datasource.rs
+++ b/crates/pyspark-rs/src/datasource.rs
@@ -12,7 +12,12 @@ use crate::session::PySparkSession;
/// Python accessor for data source registration.
///
/// Exposed as `SparkSession.dataSource` to register custom data sources.
-#[pyclass(name = "DataSourceRegistration", module = "pyspark.sql.datasource")]
+#[pyclass(
+ name = "DataSourceRegistration",
+ module = "pyspark.sql.datasource",
+ weakref,
+ dict
+)]
pub struct PyDataSourceRegistration {
pub(crate) session: PySparkSession,
}
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 827cca95..da2da7d2 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -29,7 +29,7 @@ fn check_same_session(
}
/// Python wrapper for Spark GroupedData.
-#[pyclass(name = "GroupedData", module = "pyspark.sql.group")]
+#[pyclass(name = "GroupedData", module = "pyspark.sql.group", weakref, dict)]
pub struct PyGroupedData {
pub(crate) grouped_data: RustGroupedData,
}
@@ -501,7 +501,12 @@ fn twus_func<'py>(
}
/// Python wrapper for cogrouped pandas/arrow ops
(`pyspark.sql.group.PandasCogroupedOps`).
-#[pyclass(name = "PandasCogroupedOps", module =
"pyspark.sql.pandas.group_ops")]
+#[pyclass(
+ name = "PandasCogroupedOps",
+ module = "pyspark.sql.pandas.group_ops",
+ weakref,
+ dict
+)]
pub struct PyCoGroupedData {
pub(crate) inner: RustCoGroupedData,
}
diff --git a/crates/pyspark-rs/src/observation.rs
b/crates/pyspark-rs/src/observation.rs
index b7c058f7..9166847d 100644
--- a/crates/pyspark-rs/src/observation.rs
+++ b/crates/pyspark-rs/src/observation.rs
@@ -9,7 +9,12 @@ use crate::dataframe::PyDataFrame;
use crate::errors::ResultExt;
/// Python wrapper for an Observation of metrics, as PySpark Connect's
`Observation`.
-#[pyclass(name = "Observation", module = "pyspark.sql.observation")]
+#[pyclass(
+ name = "Observation",
+ module = "pyspark.sql.observation",
+ weakref,
+ dict
+)]
pub struct PyObservation {
pub(crate) inner: Observation,
}
diff --git a/crates/pyspark-rs/src/readwriter.rs
b/crates/pyspark-rs/src/readwriter.rs
index ae37bec0..cd8d26fa 100644
--- a/crates/pyspark-rs/src/readwriter.rs
+++ b/crates/pyspark-rs/src/readwriter.rs
@@ -15,7 +15,12 @@ use crate::errors::ResultExt;
/// and return it, and a reader can be used again after a read. The core
reader is a
/// consuming builder, so the setters replace the wrapped value in place and
return
/// the same Python object, and the terminal reads build their DataFrame from
a clone.
-#[pyclass(name = "DataFrameReader", module = "pyspark.sql.readwriter")]
+#[pyclass(
+ name = "DataFrameReader",
+ module = "pyspark.sql.readwriter",
+ weakref,
+ dict
+)]
pub struct PyDataFrameReader {
inner: DataFrameReader,
}
diff --git a/crates/pyspark-rs/src/resource.rs
b/crates/pyspark-rs/src/resource.rs
index 425e9f0e..ca6d24fc 100644
--- a/crates/pyspark-rs/src/resource.rs
+++ b/crates/pyspark-rs/src/resource.rs
@@ -6,7 +6,12 @@ use spark_connect::resource::{
};
/// Python wrapper for ExecutorResourceRequests.
-#[pyclass(name = "ExecutorResourceRequests", module =
"pyspark.resource.profile")]
+#[pyclass(
+ name = "ExecutorResourceRequests",
+ module = "pyspark.resource.profile",
+ weakref,
+ dict
+)]
pub struct PyExecutorResourceRequests {
pub(crate) inner: ExecutorResourceRequests,
}
@@ -61,7 +66,12 @@ impl PyExecutorResourceRequests {
}
/// Python wrapper for TaskResourceRequests.
-#[pyclass(name = "TaskResourceRequests", module = "pyspark.resource.profile")]
+#[pyclass(
+ name = "TaskResourceRequests",
+ module = "pyspark.resource.profile",
+ weakref,
+ dict
+)]
pub struct PyTaskResourceRequests {
pub(crate) inner: TaskResourceRequests,
}
@@ -92,7 +102,12 @@ impl PyTaskResourceRequests {
}
/// Python wrapper for ResourceProfileBuilder.
-#[pyclass(name = "ResourceProfileBuilder", module =
"pyspark.resource.profile")]
+#[pyclass(
+ name = "ResourceProfileBuilder",
+ module = "pyspark.resource.profile",
+ weakref,
+ dict
+)]
pub struct PyResourceProfileBuilder {
pub(crate) inner: ResourceProfileBuilder,
}
@@ -133,7 +148,12 @@ impl PyResourceProfileBuilder {
}
/// Python wrapper for ResourceProfile.
-#[pyclass(name = "ResourceProfile", module = "pyspark.resource.profile")]
+#[pyclass(
+ name = "ResourceProfile",
+ module = "pyspark.resource.profile",
+ weakref,
+ dict
+)]
pub struct PyResourceProfile {
pub(crate) inner: ResourceProfile,
}
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index f9fd5f36..94aa92dd 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -566,7 +566,7 @@ fn tag_of(py: Python<'_>, tag: &Bound<'_, PyAny>) ->
PyResult<String> {
tag.extract()
}
-#[pyclass(name = "SparkSession", module = "pyspark.sql.session")]
+#[pyclass(name = "SparkSession", module = "pyspark.sql.session", weakref,
dict)]
pub struct PySparkSession {
pub(crate) session: SparkSession,
/// The `StreamingQueryManager` of `spark.streams`, made at its first
access.
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index 445d293f..d95f58da 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -6,7 +6,12 @@ use spark_connect::group::StatFunctions;
use crate::dataframe::PyDataFrame;
/// Python wrapper for DataFrame statistical functions.
-#[pyclass(name = "DataFrameStatFunctions", module = "pyspark.sql.dataframe")]
+#[pyclass(
+ name = "DataFrameStatFunctions",
+ module = "pyspark.sql.dataframe",
+ weakref,
+ dict
+)]
pub struct PyStatFunctions {
stat: StatFunctions,
/// The DataFrame of `df.stat`, as PySpark's `DataFrameStatFunctions.df`.
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index 3a7c7df6..4fe8afae 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -140,7 +140,12 @@ fn query_name_of(name: &Bound<'_, PyAny>) ->
PyResult<String> {
/// Like PySpark's reader, the setters change this object and return it, and
the
/// reader can be used again after a read: the core reader is a consuming
builder, so
/// the setters replace the wrapped value in place and the reads work on a
clone.
-#[pyclass(name = "DataStreamReader", module =
"pyspark.sql.streaming.readwriter")]
+#[pyclass(
+ name = "DataStreamReader",
+ module = "pyspark.sql.streaming.readwriter",
+ weakref,
+ dict
+)]
pub struct PyDataStreamReader {
inner: DataStreamReader,
}
@@ -607,7 +612,12 @@ impl PyTrigger {
}
/// Python wrapper for DataStreamWriter.
-#[pyclass(name = "DataStreamWriter", module =
"pyspark.sql.streaming.readwriter")]
+#[pyclass(
+ name = "DataStreamWriter",
+ module = "pyspark.sql.streaming.readwriter",
+ weakref,
+ dict
+)]
pub struct PyDataStreamWriter {
inner: WriterState<DataStreamWriter>,
}
@@ -1002,7 +1012,12 @@ fn streaming_function_command(
}
/// Python wrapper for StreamingQuery.
-#[pyclass(name = "StreamingQuery", module = "pyspark.sql.streaming.query")]
+#[pyclass(
+ name = "StreamingQuery",
+ module = "pyspark.sql.streaming.query",
+ weakref,
+ dict
+)]
pub struct PyStreamingQuery {
inner: StreamingQuery,
}
@@ -1125,7 +1140,12 @@ impl PyStreamingQuery {
}
/// Python wrapper for StreamingQueryManager, of the listener bus of its
session.
-#[pyclass(name = "StreamingQueryManager", module =
"pyspark.sql.streaming.query")]
+#[pyclass(
+ name = "StreamingQueryManager",
+ module = "pyspark.sql.streaming.query",
+ weakref,
+ dict
+)]
pub struct PyStreamingQueryManager {
inner: StreamingQueryManager,
/// The session, which the listeners get as their `spark`.
diff --git a/crates/pyspark-rs/src/tablearg.rs
b/crates/pyspark-rs/src/tablearg.rs
index 0723d3f1..ef2df060 100644
--- a/crates/pyspark-rs/src/tablearg.rs
+++ b/crates/pyspark-rs/src/tablearg.rs
@@ -9,7 +9,7 @@ use crate::errors::ResultExt;
/// A table argument for a table-valued function, mirroring
/// `pyspark.sql.connect.table_arg.TableArg`. Built via `df.asTable()`, then
optionally
/// `partitionBy`/`orderBy`/`withSinglePartition` (each returns a new
TableArg).
-#[pyclass(name = "TableArg", module = "pyspark.sql.table_arg")]
+#[pyclass(name = "TableArg", module = "pyspark.sql.table_arg", weakref, dict)]
pub struct PyTableArg {
pub(crate) inner: TableArg,
}
diff --git a/crates/pyspark-rs/src/tvf.rs b/crates/pyspark-rs/src/tvf.rs
index 844a1408..85c3b420 100644
--- a/crates/pyspark-rs/src/tvf.rs
+++ b/crates/pyspark-rs/src/tvf.rs
@@ -8,7 +8,12 @@ use crate::dataframe::PyDataFrame;
use crate::errors::ResultExt;
/// Python wrapper for the table-valued-function namespace (`spark.tvf`).
-#[pyclass(name = "TableValuedFunction", module = "pyspark.sql.tvf")]
+#[pyclass(
+ name = "TableValuedFunction",
+ module = "pyspark.sql.tvf",
+ weakref,
+ dict
+)]
pub struct PyTableValuedFunction {
inner: TableValuedFunction,
}
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 1d479945..3688cd9d 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -62,7 +62,13 @@ fn lowered_udts(py: Python<'_>) -> &Bound<'_,
pyo3::types::PyDict> {
/// (mirroring pyspark's `DataType`); concrete type classes extend it
(directly or via
/// the intermediate abstract bases below), so `isinstance(dt, DataType)` and
the object
/// model (simpleString/typeName/json/...) are inherited from here via
`self.inner`.
-#[pyclass(subclass, name = "DataType", module = "pyspark.sql.types")]
+#[pyclass(
+ subclass,
+ name = "DataType",
+ module = "pyspark.sql.types",
+ weakref,
+ dict
+)]
pub struct PyDataType {
pub(crate) inner: DataType,
}
@@ -1859,7 +1865,7 @@ impl PyMapType {
}
}
-#[pyclass(name = "StructField", module = "pyspark.sql.types")]
+#[pyclass(name = "StructField", module = "pyspark.sql.types", weakref, dict)]
pub struct PyStructField {
pub(crate) field: StructField,
/// The metadata dict, which PySpark's `StructField` keeps as it is: an
edit of it in place
diff --git a/crates/pyspark-rs/src/values.rs b/crates/pyspark-rs/src/values.rs
index 92ab34e1..1af1a40d 100644
--- a/crates/pyspark-rs/src/values.rs
+++ b/crates/pyspark-rs/src/values.rs
@@ -6,7 +6,7 @@ use pyo3::types::PyBytes;
/// A Variant value (the `value`/`metadata` binary components). Mirrors
/// `pyspark.sql.types.VariantVal`.
-#[pyclass(name = "VariantVal", module = "pyspark.sql.types")]
+#[pyclass(name = "VariantVal", module = "pyspark.sql.types", weakref, dict)]
pub struct PyVariantVal {
pub(crate) value: Vec<u8>,
pub(crate) metadata: Vec<u8>,
@@ -91,7 +91,7 @@ impl PyVariantVal {
}
/// A Geography value (WKB bytes + SRID). Mirrors
`pyspark.sql.types.Geography`.
-#[pyclass(name = "Geography", module = "pyspark.sql.types")]
+#[pyclass(name = "Geography", module = "pyspark.sql.types", weakref, dict)]
pub struct PyGeography {
pub(crate) wkb: Vec<u8>,
pub(crate) srid: i32,
@@ -135,7 +135,7 @@ impl PyGeography {
}
/// A Geometry value (WKB bytes + SRID). Mirrors `pyspark.sql.types.Geometry`.
-#[pyclass(name = "Geometry", module = "pyspark.sql.types")]
+#[pyclass(name = "Geometry", module = "pyspark.sql.types", weakref, dict)]
pub struct PyGeometry {
pub(crate) wkb: Vec<u8>,
pub(crate) srid: i32,
diff --git a/crates/pyspark-rs/src/window.rs b/crates/pyspark-rs/src/window.rs
index a4fb5776..fff2d9ba 100644
--- a/crates/pyspark-rs/src/window.rs
+++ b/crates/pyspark-rs/src/window.rs
@@ -166,7 +166,13 @@ impl PyFrameBound {
}
/// Python wrapper for a WindowSpec.
-#[pyclass(name = "WindowSpec", module = "pyspark.sql.window", from_py_object)]
+#[pyclass(
+ name = "WindowSpec",
+ module = "pyspark.sql.window",
+ from_py_object,
+ weakref,
+ dict
+)]
#[derive(Clone)]
pub struct PyWindowSpec {
pub(crate) spec: WindowSpec,
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 799cfc4f..abdebcb8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7869,6 +7869,31 @@ def exact_date_capture_input():
child.close()
+def
test_the_objects_of_the_client_take_weak_references_and_attributes_as_pyspark(mock_connect):
+ # As PySpark's, whose classes are Python classes: the objects of the
client can be referred
+ # to weakly and take attributes of their own, as libraries that cache by
weak reference or
+ # tag a DataFrame do.
+ import weakref
+
+ from pyspark.sql import Observation, Window
+ from pyspark.sql import functions as F
+
+ spark, _ = mock_connect
+ df = spark.range(3)
+ for obj in [
+ spark, df, F.col("a"), df.groupBy("id"), Window.partitionBy("id"),
spark.read, df.write,
+ df.writeTo("t"), df.mergeInto("t", F.lit(True)), spark.readStream,
df.na, df.stat,
+ spark.catalog, spark.conf, spark.tvf, spark.dataSource,
Observation("o"),
+ T.StructType([T.StructField("a", T.LongType())]), T.StructField("a",
T.LongType()),
+ T.LongType(), T.ArrayType(T.StringType()), T.DecimalType(5, 2),
+ ]:
+ reference = weakref.ref(obj)
+ assert reference() is obj, type(obj)
+ obj._tagged_by_test = type(obj).__name__
+ assert obj._tagged_by_test == type(obj).__name__
+ del obj._tagged_by_test
+
+
def
test_year_month_and_calendar_intervals_raise_as_pyspark_for_local_rows(mock_connect):
# As PySpark's to_arrow_schema, which has no Arrow type of these
intervals: the values
# and the None values of such a column raise
UNSUPPORTED_DATA_TYPE_FOR_ARROW_CONVERSION,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]