This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 83d7446 [SPARK-61091][CONNECT][PYTHON] Return the NamedTuples of
pyspark.sql.catalog from Catalog, and register functions with registerFunction,
in the Python client as PySpark does
83d7446 is described below
commit 83d74462ec41012993176cf3af99017576688eb8
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 15:38:07 2026 +0900
[SPARK-61091][CONNECT][PYTHON] Return the NamedTuples of
pyspark.sql.catalog from Catalog, and register functions with registerFunction,
in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61091](https://issues.apache.org/jira/browse/SPARK-61091): the
methods of `spark.catalog` now return the NamedTuples of `pyspark.sql.catalog`,
and `registerFunction` registers functions, as PySpark does.
As PySpark's:
- the methods of the catalog return the NamedTuples of
`pyspark.sql.catalog`: `CatalogMetadata`, `Database`, `Table` (with its
`database` property), `Column`, `Function` and `TablePartition`. They are
tuples, so `_fields`, `_asdict`, indexing, unpacking and tuple equality work:
```python
>>> spark.catalog.getTable("default.t")
Table(name='t', catalog='spark_catalog', namespace=['default'],
description=None, tableType='MANAGED', isTemporary=False)
```
- `Catalog.registerFunction(name, f, returnType)` issues its
`FutureWarning` ("Deprecated in 2.3.0. Use spark.udf.register instead.") and
registers the function with `spark.udf.register`.
The NamedTuples are PySpark's definitions, in the shim
`pyspark.sql.catalog`, and the Rust methods build them. The Rust classes of the
results are removed. The core `Catalog` gains `spark_session()`.
### Why are the changes needed?
- The client returned classes of its own, which were not tuples: they had
no `_fields`, and their repr differed.
- The class of a column was named `CatalogColumn`.
- `registerFunction` raised `NOT_EXPECTED_TYPE` for `evalType`, because it
passed its arguments to `UserDefinedFunction` out of order.
### Does this PR introduce _any_ user-facing change?
Yes. The catalog returns the results of PySpark 4.3.0, and
`registerFunction` works.
### How was this patch tested?
- A new live test checks the `Table`, the `Column`s, the `Database` and the
`Function` of a table and a function, as tuples, and registers a function with
`registerFunction`, including its warning. It passes with official PySpark
4.3.0 and fails without this change.
- The offline test of the result classes now checks that they are the
NamedTuples of PySpark, with their fields and the `database` of a `Table`.
- On a local server, 45 cases of the catalog give the same results as
official PySpark 4.3.0. The exceptions are the messages of the `TypeError`s for
non-str names, which are protobuf's in PySpark. The reprs of the results, and
the warning of `registerFunction`, are the same too.
- The lib tests of both crates (592 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2041 passed, 12 skipped), and the
live drop-in and Arrow suites (145 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #295 from HyukjinKwon/SPARK-61091-catalog-namedtuples.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/catalog.rs | 406 +++++++++++++-----------------------
crates/pyspark-rs/src/lib.rs | 11 +-
crates/spark-connect/src/catalog.rs | 5 +
python/pyspark/sql/catalog.py | 73 +++++--
python/tests/test_dropin_e2e.py | 29 +++
python/tests/test_dropin_offline.py | 17 +-
6 files changed, 249 insertions(+), 292 deletions(-)
diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index a251a06..d587ee6 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -1,6 +1,7 @@
//! PyO3 wrapper for spark_connect::catalog::Catalog.
use pyo3::prelude::*;
+use pyo3::types::PyDict;
use spark_connect::catalog::{
Catalog, CatalogMetadata, Column, Database, Function, Table,
TablePartition,
};
@@ -74,203 +75,98 @@ fn table_options(
Ok(map)
}
-/// `pyspark.sql.catalog.CatalogMetadata`.
-#[pyclass(name = "CatalogMetadata", module = "pyspark.sql.catalog")]
-pub struct PyCatalogMetadata {
- inner: CatalogMetadata,
-}
-#[pymethods]
-impl PyCatalogMetadata {
- #[getter]
- fn name(&self) -> String {
- self.inner.name.clone()
- }
- #[getter]
- fn description(&self) -> Option<String> {
- self.inner.description.clone()
- }
- fn __repr__(&self) -> String {
- format!(
- "CatalogMetadata(name='{}', description={:?})",
- self.inner.name, self.inner.description
- )
+/// The NamedTuple of `pyspark.sql.catalog` of a result of the catalog, as
PySpark's `Catalog`
+/// returns: `CatalogMetadata`, `Database`, `Table`, `Column`, `Function` or
`TablePartition`.
+trait NamedTuple {
+ /// The name of the NamedTuple, and its fields.
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)>;
+
+ fn named_tuple(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
+ let (name, fields) = self.fields(py)?;
+ Ok(py
+ .import("pyspark.sql.catalog")?
+ .getattr(name)?
+ .call((), Some(&fields))?
+ .unbind())
}
}
-/// `pyspark.sql.catalog.Database`.
-#[pyclass(name = "Database", module = "pyspark.sql.catalog")]
-pub struct PyDatabase {
- inner: Database,
-}
-#[pymethods]
-impl PyDatabase {
- #[getter]
- fn name(&self) -> String {
- self.inner.name.clone()
- }
- #[getter]
- fn catalog(&self) -> Option<String> {
- self.inner.catalog.clone()
- }
- #[getter]
- fn description(&self) -> Option<String> {
- self.inner.description.clone()
- }
- #[getter]
- fn locationUri(&self) -> String {
- self.inner.location_uri.clone()
- }
- fn __repr__(&self) -> String {
- format!("Database(name='{}')", self.inner.name)
+impl NamedTuple for CatalogMetadata {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("name", &self.name)?;
+ fields.set_item("description", &self.description)?;
+ Ok(("CatalogMetadata", fields))
}
}
-/// `pyspark.sql.catalog.Table`.
-#[pyclass(name = "Table", module = "pyspark.sql.catalog")]
-pub struct PyTable {
- inner: Table,
-}
-#[pymethods]
-impl PyTable {
- #[getter]
- fn name(&self) -> String {
- self.inner.name.clone()
- }
- #[getter]
- fn catalog(&self) -> Option<String> {
- self.inner.catalog.clone()
- }
- #[getter]
- fn namespace(&self) -> Option<Vec<String>> {
- self.inner.namespace.clone()
- }
- #[getter]
- fn description(&self) -> Option<String> {
- self.inner.description.clone()
- }
- #[getter]
- fn tableType(&self) -> String {
- self.inner.table_type.clone()
- }
- #[getter]
- fn isTemporary(&self) -> bool {
- self.inner.is_temporary
- }
- /// The single-element namespace as a database name, else None (mirrors
`Table.database`).
- #[getter]
- fn database(&self) -> Option<String> {
- match &self.inner.namespace {
- Some(ns) if ns.len() == 1 => Some(ns[0].clone()),
- _ => None,
- }
- }
- fn __repr__(&self) -> String {
- format!(
- "Table(name='{}', tableType='{}')",
- self.inner.name, self.inner.table_type
- )
+impl NamedTuple for Database {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("name", &self.name)?;
+ fields.set_item("catalog", &self.catalog)?;
+ fields.set_item("description", &self.description)?;
+ fields.set_item("locationUri", &self.location_uri)?;
+ Ok(("Database", fields))
}
}
-/// `pyspark.sql.catalog.Column` (exposed as `CatalogColumn` in the extension
to avoid
-/// colliding with the expression `Column`; re-exported as `Column` by
pyspark.sql.catalog).
-#[pyclass(name = "CatalogColumn", module = "pyspark.sql.catalog")]
-pub struct PyCatalogColumn {
- inner: Column,
-}
-#[pymethods]
-impl PyCatalogColumn {
- #[getter]
- fn name(&self) -> String {
- self.inner.name.clone()
- }
- #[getter]
- fn description(&self) -> Option<String> {
- self.inner.description.clone()
- }
- #[getter]
- fn dataType(&self) -> String {
- self.inner.data_type.clone()
- }
- #[getter]
- fn nullable(&self) -> bool {
- self.inner.nullable
- }
- #[getter]
- fn isPartition(&self) -> bool {
- self.inner.is_partition
- }
- #[getter]
- fn isBucket(&self) -> bool {
- self.inner.is_bucket
- }
- #[getter]
- fn isCluster(&self) -> bool {
- self.inner.is_cluster
- }
- fn __repr__(&self) -> String {
- format!(
- "Column(name='{}', dataType='{}')",
- self.inner.name, self.inner.data_type
- )
+impl NamedTuple for Table {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("name", &self.name)?;
+ fields.set_item("catalog", &self.catalog)?;
+ fields.set_item("namespace", &self.namespace)?;
+ fields.set_item("description", &self.description)?;
+ fields.set_item("tableType", &self.table_type)?;
+ fields.set_item("isTemporary", self.is_temporary)?;
+ Ok(("Table", fields))
}
}
-/// `pyspark.sql.catalog.Function`.
-#[pyclass(name = "Function", module = "pyspark.sql.catalog")]
-pub struct PyFunction {
- inner: Function,
-}
-#[pymethods]
-impl PyFunction {
- #[getter]
- fn name(&self) -> String {
- self.inner.name.clone()
- }
- #[getter]
- fn catalog(&self) -> Option<String> {
- self.inner.catalog.clone()
- }
- #[getter]
- fn namespace(&self) -> Option<Vec<String>> {
- self.inner.namespace.clone()
- }
- #[getter]
- fn description(&self) -> Option<String> {
- self.inner.description.clone()
- }
- #[getter]
- fn className(&self) -> String {
- self.inner.class_name.clone()
- }
- #[getter]
- fn isTemporary(&self) -> bool {
- self.inner.is_temporary
- }
- fn __repr__(&self) -> String {
- format!(
- "Function(name='{}', className='{}')",
- self.inner.name, self.inner.class_name
- )
+impl NamedTuple for Column {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("name", &self.name)?;
+ fields.set_item("description", &self.description)?;
+ fields.set_item("dataType", &self.data_type)?;
+ fields.set_item("nullable", self.nullable)?;
+ fields.set_item("isPartition", self.is_partition)?;
+ fields.set_item("isBucket", self.is_bucket)?;
+ fields.set_item("isCluster", self.is_cluster)?;
+ Ok(("Column", fields))
}
}
-/// `pyspark.sql.catalog.TablePartition`.
-#[pyclass(name = "TablePartition", module = "pyspark.sql.catalog")]
-pub struct PyTablePartition {
- inner: TablePartition,
-}
-#[pymethods]
-impl PyTablePartition {
- #[getter]
- fn partition(&self) -> String {
- self.inner.partition.clone()
+impl NamedTuple for Function {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("name", &self.name)?;
+ fields.set_item("catalog", &self.catalog)?;
+ fields.set_item("namespace", &self.namespace)?;
+ fields.set_item("description", &self.description)?;
+ fields.set_item("className", &self.class_name)?;
+ fields.set_item("isTemporary", self.is_temporary)?;
+ Ok(("Function", fields))
}
- fn __repr__(&self) -> String {
- format!("TablePartition(partition='{}')", self.inner.partition)
+}
+
+impl NamedTuple for TablePartition {
+ fn fields<'py>(&self, py: Python<'py>) -> PyResult<(&'static str,
Bound<'py, PyDict>)> {
+ let fields = PyDict::new(py);
+ fields.set_item("partition", &self.partition)?;
+ Ok(("TablePartition", fields))
}
}
+/// The NamedTuples of `results`.
+fn named_tuples<T: NamedTuple>(py: Python<'_>, results: Vec<T>) ->
PyResult<Vec<Py<PyAny>>> {
+ results
+ .iter()
+ .map(|result| result.named_tuple(py))
+ .collect()
+}
+
/// Python wrapper for Spark Catalog.
#[pyclass(name = "Catalog", module = "pyspark.sql.catalog")]
pub struct PyCatalog {
@@ -309,18 +205,13 @@ impl PyCatalog {
/// List catalogs. Mirrors `Catalog.listCatalogs(pattern=None) ->
List[CatalogMetadata]`.
#[pyo3(signature = (pattern=None))]
- fn listCatalogs(
- &self,
- py: Python<'_>,
- pattern: Option<&str>,
- ) -> PyResult<Vec<PyCatalogMetadata>> {
- Ok(self
- .detached(py, |catalog| {
+ fn listCatalogs(&self, py: Python<'_>, pattern: Option<&str>) ->
PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog.list_catalogs_typed(pattern).map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyCatalogMetadata { inner })
- .collect())
+ })?,
+ )
}
/// Get current database.
@@ -337,14 +228,13 @@ impl PyCatalog {
/// List databases. Mirrors `Catalog.listDatabases(pattern=None) ->
List[Database]`.
#[pyo3(signature = (pattern=None))]
- fn listDatabases(&self, py: Python<'_>, pattern: Option<&str>) ->
PyResult<Vec<PyDatabase>> {
- Ok(self
- .detached(py, |catalog| {
+ fn listDatabases(&self, py: Python<'_>, pattern: Option<&str>) ->
PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog.list_databases_typed(pattern).map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyDatabase { inner })
- .collect())
+ })?,
+ )
}
/// Check if database exists.
@@ -355,12 +245,11 @@ impl PyCatalog {
}
/// Get database info. Mirrors `Catalog.getDatabase(dbName) -> Database`.
- fn getDatabase(&self, py: Python<'_>, dbName: &str) ->
PyResult<PyDatabase> {
- Ok(PyDatabase {
- inner: self.detached(py, |catalog| {
- catalog.get_database_typed(dbName).map_err(Box::new)
- })?,
- })
+ fn getDatabase(&self, py: Python<'_>, dbName: &str) -> PyResult<Py<PyAny>>
{
+ self.detached(py, |catalog| {
+ catalog.get_database_typed(dbName).map_err(Box::new)
+ })?
+ .named_tuple(py)
}
/// List tables. Mirrors `Catalog.listTables(dbName=None, pattern=None) ->
List[Table]`.
@@ -371,14 +260,13 @@ impl PyCatalog {
py: Python<'_>,
dbName: Option<&str>,
pattern: Option<&str>,
- ) -> PyResult<Vec<PyTable>> {
- Ok(self
- .detached(py, |catalog| {
+ ) -> PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog.list_tables_typed(dbName, pattern).map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyTable { inner })
- .collect())
+ })?,
+ )
}
/// Check if table exists. Mirrors `Catalog.tableExists(tableName,
dbName=None) -> bool`.
@@ -391,12 +279,11 @@ impl PyCatalog {
}
/// Get table info. Mirrors `Catalog.getTable(tableName) -> Table`.
- fn getTable(&self, py: Python<'_>, tableName: &str) -> PyResult<PyTable> {
- Ok(PyTable {
- inner: self.detached(py, |catalog| {
- catalog.get_table_typed(tableName).map_err(Box::new)
- })?,
- })
+ fn getTable(&self, py: Python<'_>, tableName: &str) -> PyResult<Py<PyAny>>
{
+ self.detached(py, |catalog| {
+ catalog.get_table_typed(tableName).map_err(Box::new)
+ })?
+ .named_tuple(py)
}
/// List functions. Mirrors `Catalog.listFunctions(dbName=None,
pattern=None) -> List[Function]`.
@@ -407,16 +294,15 @@ impl PyCatalog {
py: Python<'_>,
dbName: Option<&str>,
pattern: Option<&str>,
- ) -> PyResult<Vec<PyFunction>> {
- Ok(self
- .detached(py, |catalog| {
+ ) -> PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog
.list_functions_typed(dbName, pattern)
.map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyFunction { inner })
- .collect())
+ })?,
+ )
}
/// Check if function exists. Mirrors
`Catalog.functionExists(functionName, dbName=None) -> bool`.
@@ -434,12 +320,11 @@ impl PyCatalog {
}
/// Get function info. Mirrors `Catalog.getFunction(functionName) ->
Function`.
- fn getFunction(&self, py: Python<'_>, functionName: &str) ->
PyResult<PyFunction> {
- Ok(PyFunction {
- inner: self.detached(py, |catalog| {
- catalog.get_function_typed(functionName).map_err(Box::new)
- })?,
- })
+ fn getFunction(&self, py: Python<'_>, functionName: &str) ->
PyResult<Py<PyAny>> {
+ self.detached(py, |catalog| {
+ catalog.get_function_typed(functionName).map_err(Box::new)
+ })?
+ .named_tuple(py)
}
/// List columns of a table. Mirrors `Catalog.listColumns(tableName,
dbName=None) -> List[Column]`.
@@ -450,17 +335,16 @@ impl PyCatalog {
py: Python<'_>,
tableName: &str,
dbName: Option<&str>,
- ) -> PyResult<Vec<PyCatalogColumn>> {
+ ) -> PyResult<Vec<Py<PyAny>>> {
warn_db_name(py, "listColumns", dbName)?;
- Ok(self
- .detached(py, |catalog| {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog
.list_columns_typed(tableName, dbName)
.map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyCatalogColumn { inner })
- .collect())
+ })?,
+ )
}
/// Cache a table. Mirrors `Catalog.cacheTable(tableName,
storageLevel=None) -> None`.
@@ -602,18 +486,20 @@ impl PyCatalog {
f: Bound<'a, PyAny>,
returnType: Option<Bound<'a, PyAny>>,
) -> PyResult<Bound<'a, PyAny>> {
- let udf_cls = py
- .import("pyspark.sql.udf")?
- .getattr("UserDefinedFunction")?;
- let rt: Bound<'a, PyAny> = match returnType {
- Some(t) => t,
- None => py
- .import("pyspark.sql.types")?
- .getattr("StringType")?
- .call0()?,
- };
- // UserDefinedFunction(func, returnType, evalType=100, name=name)
- udf_cls.call1((f, rt, 100, name))
+ let warnings = py.import("warnings")?;
+ let future_warning = py.import("builtins")?.getattr("FutureWarning")?;
+ warnings.call_method1(
+ "warn",
+ (
+ "Deprecated in 2.3.0. Use spark.udf.register instead.",
+ future_warning,
+ ),
+ )?;
+ let session =
crate::session::PySparkSession::new(self.catalog.spark_session().clone());
+ Py::new(py, session)?
+ .bind(py)
+ .getattr("udf")?
+ .call_method1("register", (name, f, returnType))
}
/// Create a database.
@@ -728,14 +614,13 @@ impl PyCatalog {
/// List the partitions of a table. Mirrors `Catalog.listPartitions ->
List[TablePartition]`.
#[allow(non_snake_case)]
- fn listPartitions(&self, py: Python<'_>, tableName: &str) ->
PyResult<Vec<PyTablePartition>> {
- Ok(self
- .detached(py, |catalog| {
+ fn listPartitions(&self, py: Python<'_>, tableName: &str) ->
PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog.list_partitions_typed(tableName).map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyTablePartition { inner })
- .collect())
+ })?,
+ )
}
/// List the views in a database. Mirrors `Catalog.listViews ->
List[Table]`.
@@ -746,14 +631,13 @@ impl PyCatalog {
py: Python<'_>,
dbName: Option<&str>,
pattern: Option<&str>,
- ) -> PyResult<Vec<PyTable>> {
- Ok(self
- .detached(py, |catalog| {
+ ) -> PyResult<Vec<Py<PyAny>>> {
+ named_tuples(
+ py,
+ self.detached(py, |catalog| {
catalog.list_views_typed(dbName, pattern).map_err(Box::new)
- })?
- .into_iter()
- .map(|inner| PyTable { inner })
- .collect())
+ })?,
+ )
}
fn __repr__(&self) -> String {
diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index a2adc3b..be4b978 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -59,10 +59,7 @@ mod udtf_analyze;
mod values;
mod window;
-use catalog::{
- PyCatalog, PyCatalogColumn, PyCatalogMetadata, PyDatabase, PyFunction,
PyTable,
- PyTablePartition,
-};
+use catalog::PyCatalog;
use column::PyColumn;
use conf::PyRuntimeConf;
use dataframe::{
@@ -121,12 +118,6 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) ->
PyResult<()> {
m.add_class::<transport::ResponseStream>()?;
m.add("RustRpcError", m.py().get_type::<transport::RustRpcError>())?;
m.add_class::<PyCatalog>()?;
- m.add_class::<PyCatalogMetadata>()?;
- m.add_class::<PyDatabase>()?;
- m.add_class::<PyTable>()?;
- m.add_class::<PyCatalogColumn>()?;
- m.add_class::<PyFunction>()?;
- m.add_class::<PyTablePartition>()?;
m.add_class::<eval_type::PyPythonEvalType>()?;
m.add_class::<PyRuntimeConf>()?;
m.add_class::<PyDataFrameReader>()?;
diff --git a/crates/spark-connect/src/catalog.rs
b/crates/spark-connect/src/catalog.rs
index b5a1320..3c02f90 100644
--- a/crates/spark-connect/src/catalog.rs
+++ b/crates/spark-connect/src/catalog.rs
@@ -134,6 +134,11 @@ impl Catalog {
Catalog { session }
}
+ /// The session of this catalog, as PySpark's `Catalog._sparkSession`.
+ pub fn spark_session(&self) -> &SparkSession {
+ &self.session
+ }
+
/// Get the current catalog.
///
/// Mirrors `pyspark.sql.connect.catalog.Catalog.currentCatalog`.
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index 60fe40c..4c58e32 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -16,20 +16,65 @@
#
"""Catalog + its metadata result classes, mirroring ``pyspark.sql.catalog``.
-The Catalog and the metadata result classes (Database/Table/Column/Function/
-CatalogMetadata/TablePartition) are all implemented natively in the Rust core
and
-re-exported here (``CatalogColumn`` is exposed as ``Column`` to avoid
colliding with
-the expression ``Column`` in the extension module)."""
-
-from pyspark._pyspark import (
- Catalog,
- CatalogMetadata,
- Database,
- Table,
- CatalogColumn as Column,
- Function,
- TablePartition,
-)
+The Catalog is implemented natively in the Rust core and re-exported here. Its
methods return
+the NamedTuples of PySpark's ``pyspark.sql.catalog`` below (Database, Table,
Column, Function,
+CatalogMetadata and TablePartition), which are tuples, as PySpark's are."""
+
+from typing import List, NamedTuple, Optional
+
+from pyspark._pyspark import Catalog
+
+
+class CatalogMetadata(NamedTuple):
+ name: str
+ description: Optional[str]
+
+
+class Database(NamedTuple):
+ name: str
+ catalog: Optional[str]
+ description: Optional[str]
+ locationUri: str
+
+
+class Table(NamedTuple):
+ name: str
+ catalog: Optional[str]
+ namespace: Optional[List[str]]
+ description: Optional[str]
+ tableType: str
+ isTemporary: bool
+
+ @property
+ def database(self) -> Optional[str]:
+ if self.namespace is not None and len(self.namespace) == 1:
+ return self.namespace[0]
+ else:
+ return None
+
+
+class Column(NamedTuple):
+ name: str
+ description: Optional[str]
+ dataType: str
+ nullable: bool
+ isPartition: bool
+ isBucket: bool
+ isCluster: bool
+
+
+class Function(NamedTuple):
+ name: str
+ catalog: Optional[str]
+ namespace: Optional[List[str]]
+ description: Optional[str]
+ className: str
+ isTemporary: bool
+
+
+class TablePartition(NamedTuple):
+ partition: str
+
__all__ = [
"Catalog",
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a5c0d8a..306f50b 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2513,6 +2513,35 @@ def
test_create_dataframe_converts_numpy_scalars_as_pyspark(session):
assert [tuple(r) for r in df.collect()] == [(1, 0.5, True, 1)]
+def test_catalogs_return_the_named_tuples_of_pyspark(session):
+ # As PySpark's: the methods of the catalog return the NamedTuples of
pyspark.sql.catalog,
+ # and registerFunction registers a UDF with spark.udf.register, with its
FutureWarning.
+ from pyspark.sql.catalog import Column, Database, Function, Table
+
+ session.sql("drop table if exists default.catalog_tuples_t")
+ session.sql("create table default.catalog_tuples_t (a int, b string) using
parquet")
+ try:
+ table = session.catalog.getTable("default.catalog_tuples_t")
+ assert isinstance(table, Table)
+ assert table == (
+ "catalog_tuples_t", "spark_catalog", ["default"], None, "MANAGED",
False)
+ assert table._asdict()["tableType"] == "MANAGED" and table.database ==
"default"
+ assert session.catalog.listColumns("default.catalog_tuples_t") == [
+ Column("a", None, "int", True, False, False, False),
+ Column("b", None, "string", True, False, False, False),
+ ]
+ assert isinstance(session.catalog.getDatabase("default"), Database)
+ name, *_ = session.catalog.getFunction("abs")
+ assert name == "abs" and
isinstance(session.catalog.getFunction("abs"), Function)
+ with pytest.warns(FutureWarning, match="Use spark.udf.register
instead"):
+ session.catalog.registerFunction("catalog_tuples_plus_one", lambda
x: x + 1, "int")
+ assert session.sql("select catalog_tuples_plus_one(1) as v").first().v
== 2
+ finally:
+ session.sql("drop table if exists default.catalog_tuples_t")
+
+
+
+
def
test_create_dataframe_infers_by_the_session_configurations_as_pyspark(session):
naive = datetime(2020, 1, 2, 3, 4, 5)
cases = [
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 13d6de9..4730128 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2092,17 +2092,20 @@ def test_udf_udtf_determinism_and_extras():
def test_catalog_result_classes():
- # The catalog metadata result classes are Rust-backed and importable; the
Table
- # `database` property derives from a single-element namespace.
+ # As PySpark's: the catalog metadata result classes are the NamedTuples of
+ # pyspark.sql.catalog, and the Table `database` property derives from a
single-element
+ # namespace.
from pyspark.sql.catalog import (
Catalog, CatalogMetadata, Database, Table, Column, Function,
TablePartition,
)
- import pyspark._pyspark as p
- # Rust-backed, but reporting the reference `pyspark.sql.catalog` module
path
- # (SPARK-59117), not "builtins".
- for cls in (CatalogMetadata, Database, Table, Function, TablePartition):
+ assert Catalog.__module__ == "pyspark.sql.catalog"
+ for cls in (CatalogMetadata, Database, Table, Column, Function,
TablePartition):
assert cls.__module__ == "pyspark.sql.catalog", f"{cls.__name__}
module = {cls.__module__}"
- assert Column is p.CatalogColumn
+ assert issubclass(cls, tuple)
+ assert Column._fields == (
+ "name", "description", "dataType", "nullable", "isPartition",
"isBucket", "isCluster")
+ assert Table("t", None, ["db"], None, "MANAGED", False).database == "db"
+ assert Table("t", None, ["a", "b"], None, "MANAGED", False).database is
None
def test_udtf_analyze_result_family():
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]