This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 5ee9519  [SPARK-61054][CONNECT][PYTHON] Take the arguments of 
spark.range and of the table-valued functions of spark.tvf in the Python client 
as PySpark does
5ee9519 is described below

commit 5ee9519220dff8c2cb233565cb5fa2085790d956
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:32:17 2026 +0900

    [SPARK-61054][CONNECT][PYTHON] Take the arguments of spark.range and of the 
table-valued functions of spark.tvf in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61054](https://issues.apache.org/jira/browse/SPARK-61054): 
`spark.range`, `spark.tvf.range` and the table-valued functions of `spark.tvf` 
now take their arguments as PySpark Connect does.
    
    - `spark.range(start, end, step, numPartitions)` and `spark.tvf.range` take 
the `int()` of each argument, of `numPartitions` first, such as 
`spark.range("3")` and `spark.range(2.7)`. A value out of the range of its 
proto field raises the `ValueError` of protobuf, "Value out of range: ...". The 
client raises it when the DataFrame is made; PySpark raises it when it sends 
the plan.
    - The functions of `spark.tvf`, such as `explode`, `inline`, `json_tuple` 
and `stack`, take Columns or column names, as `F._to_col` does, else 
`NOT_EXPECTED_TYPE` of `col`.
    - `json_tuple` asks for fields before it takes its arguments.
    
    ### Why are the changes needed?
    
    The client raised `TypeError`s for arguments of `range` that are not ints. 
It took the arguments of `spark.tvf` as literals:
    - `spark.tvf.json_tuple(json, "a")` took the string "a" rather than the 
column `a`
    - `spark.tvf.explode([1, 2])` exploded an array of the list rather than 
raising
    - `spark.tvf.stack(2, ...)` took an int rather than raising
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These take the arguments of PySpark 4.3.0, and raise its errors.
    
    ### How was this patch tested?
    
    - A new live test checks the rows of `range` of a str and of floats, the 
errors of `range`, the column of a str argument of `json_tuple`, and the 
`NOT_EXPECTED_TYPE` of a list. It passes with official PySpark 4.3.0, and fails 
without this change.
    - A new offline test checks the `Range` and the arguments of the 
table-valued functions that these send to a mock Connect server, and their 
errors.
    - On a local server, 30 cases of `spark.range` and `spark.tvf.range`, and 
23 of `spark.tvf`, give the rows and errors of official PySpark 4.3.0. One 
`TypeError` of `int()` names the class of the client's Column in full.
    - The lib tests of both crates (574 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1813 passed, 8 skipped), and the 
live drop-in and Arrow suites (112 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #252 from HyukjinKwon/SPARK-61054-tvf-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  |  2 +-
 crates/pyspark-rs/src/session.rs    | 72 +++++++++++++++++++++++++++++++------
 crates/pyspark-rs/src/tvf.rs        | 55 ++++++++++++++++++----------
 python/tests/test_dropin_e2e.py     | 22 ++++++++++++
 python/tests/test_dropin_offline.py | 58 ++++++++++++++++++++++++++++++
 5 files changed, 180 insertions(+), 29 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 235bd46..8e7b9bd 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2561,7 +2561,7 @@ fn unpivot_columns(cols: &Bound<'_, PyAny>) -> 
PyResult<Vec<spark_connect::colum
 }
 
 /// A column, or a column by its name, as PySpark's `_to_col`.
-fn column_or_name(col: &Bound<'_, PyAny>) -> 
PyResult<spark_connect::column::Column> {
+pub(crate) fn column_or_name(col: &Bound<'_, PyAny>) -> 
PyResult<spark_connect::column::Column> {
     if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
         return Ok(column.column.clone());
     }
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index b8bcb16..1c843aa 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -854,22 +854,20 @@ impl PySparkSession {
     ///
     /// Mirrors `pyspark.sql.SparkSession.range(start, end=None, step=1, 
numPartitions=None)`:
     /// called with a single positional arg it means `range(0, start)`.
-    #[pyo3(signature = (start, end=None, step=1, numPartitions=None))]
+    /// Each argument is taken by its `int()`, as PySpark's `range` takes them.
+    #[pyo3(signature = (start, end=None, step=RangeStep::Omitted, 
numPartitions=None))]
     #[allow(non_snake_case)]
     fn range(
         &self,
-        start: i64,
-        end: Option<i64>,
-        step: i64,
-        numPartitions: Option<i32>,
+        start: &Bound<'_, PyAny>,
+        end: Option<&Bound<'_, PyAny>>,
+        step: RangeStep,
+        numPartitions: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let (start, end) = match end {
-            Some(e) => (start, e),
-            None => (0, start),
-        };
+        let (start, end, step, num_partitions) = range_arguments(start, end, 
step, numPartitions)?;
         let df = self
             .session
-            .range_full(start, end, step, numPartitions)
+            .range_full(start, end, step, num_partitions)
             .to_pyerr()?;
         Ok(PyDataFrame::new(df))
     }
@@ -2277,3 +2275,57 @@ fn decimal_text_digits(value: &str) -> Option<(i64, 
i64)> {
     let digits = (integer.len() + fraction.len() - leading_zeros).max(1) as 
i64;
     Some((digits, exponent - fraction.len() as i64))
 }
+
+/// The `step` of `range`, which PySpark defaults to 1, told apart from None, 
whose `int()`
+/// raises.
+pub(crate) enum RangeStep {
+    Omitted,
+    Given(Py<PyAny>),
+}
+
+impl<'a, 'py> FromPyObject<'a, 'py> for RangeStep {
+    type Error = PyErr;
+
+    fn extract(obj: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
+        Ok(RangeStep::Given(obj.to_owned().unbind()))
+    }
+}
+
+/// The start, end, step and number of partitions of `spark.range` and 
`spark.tvf.range`, as
+/// PySpark's `range` takes them: the `int()` of each, of the number of 
partitions first, with
+/// `range(n)` from 0 to `n`. A value out of the range of its proto field is 
the `ValueError` of
+/// protobuf.
+pub(crate) fn range_arguments(
+    start: &Bound<'_, PyAny>,
+    end: Option<&Bound<'_, PyAny>>,
+    step: RangeStep,
+    num_partitions: Option<&Bound<'_, PyAny>>,
+) -> PyResult<(i64, i64, i64, Option<i32>)> {
+    let py = start.py();
+    let int = |value: &Bound<'_, PyAny>| -> PyResult<i64> {
+        let int = py.get_type::<pyo3::types::PyInt>().call1((value,))?;
+        int.extract::<i64>().map_err(|_| value_out_of_range(&int))
+    };
+    let num_partitions = match num_partitions {
+        None => None,
+        Some(value) => {
+            let int = py.get_type::<pyo3::types::PyInt>().call1((value,))?;
+            Some(int.extract::<i32>().map_err(|_| value_out_of_range(&int))?)
+        }
+    };
+    let (start, end) = match end {
+        Some(end) => (int(start)?, int(end)?),
+        None => (0, int(start)?),
+    };
+    let step = match step {
+        RangeStep::Omitted => 1,
+        RangeStep::Given(step) => int(step.bind(py))?,
+    };
+    Ok((start, end, step, num_partitions))
+}
+
+/// The `ValueError` of protobuf for an int out of the range of its field.
+fn value_out_of_range(int: &Bound<'_, PyAny>) -> PyErr {
+    let shown = int.str().map(|text| text.to_string()).unwrap_or_default();
+    pyo3::exceptions::PyValueError::new_err(format!("Value out of range: 
{shown}"))
+}
diff --git a/crates/pyspark-rs/src/tvf.rs b/crates/pyspark-rs/src/tvf.rs
index 523e4bd..844a140 100644
--- a/crates/pyspark-rs/src/tvf.rs
+++ b/crates/pyspark-rs/src/tvf.rs
@@ -3,9 +3,9 @@
 use pyo3::prelude::*;
 use spark_connect::tvf::TableValuedFunction;
 
+use crate::dataframe::column_or_name;
 use crate::dataframe::PyDataFrame;
 use crate::errors::ResultExt;
-use crate::functions::to_column;
 
 /// Python wrapper for the table-valued-function namespace (`spark.tvf`).
 #[pyclass(name = "TableValuedFunction", module = "pyspark.sql.tvf")]
@@ -19,30 +19,40 @@ impl PyTableValuedFunction {
     }
 }
 
+/// The arguments of a table-valued function, as PySpark's `_fn` takes them: 
Columns or column
+/// names, as `F._to_col` does, else `NOT_EXPECTED_TYPE` of `col`.
 fn cols(items: Vec<Bound<'_, PyAny>>) -> 
PyResult<Vec<spark_connect::column::Column>> {
-    items.iter().map(to_column).collect()
+    items.iter().map(column_or_name).collect()
 }
 
 #[pymethods]
 impl PyTableValuedFunction {
-    #[pyo3(signature = (start, end=None, step=1, numPartitions=None))]
+    /// Each argument is taken by its `int()`, as PySpark's `spark.tvf.range` 
takes them, which
+    /// is `spark.range`.
+    #[pyo3(signature = (start, end=None, 
step=crate::session::RangeStep::Omitted, numPartitions=None))]
     #[allow(non_snake_case)]
     fn range(
         &self,
         py: Python<'_>,
-        start: i64,
-        end: Option<i64>,
-        step: i64,
-        numPartitions: Option<i32>,
+        start: &Bound<'_, PyAny>,
+        end: Option<&Bound<'_, PyAny>>,
+        step: crate::session::RangeStep,
+        numPartitions: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
+        let (start, end, step, num_partitions) =
+            crate::session::range_arguments(start, end, step, numPartitions)?;
         let df = py
-            .detach(|| self.inner.range(start, end, step, numPartitions))
+            .detach(|| {
+                self.inner
+                    .range(start, Some(end), step, num_partitions)
+                    .map_err(Box::new)
+            })
             .to_pyerr()?;
         Ok(PyDataFrame::new(df))
     }
 
     fn explode(&self, py: Python<'_>, collection: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
-        let c = to_column(collection)?;
+        let c = column_or_name(collection)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.explode(&c)).to_pyerr()?,
         ))
@@ -54,14 +64,14 @@ impl PyTableValuedFunction {
         py: Python<'_>,
         collection: &Bound<'_, PyAny>,
     ) -> PyResult<PyDataFrame> {
-        let c = to_column(collection)?;
+        let c = column_or_name(collection)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.explode_outer(&c)).to_pyerr()?,
         ))
     }
 
     fn inline(&self, py: Python<'_>, input: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
-        let c = to_column(input)?;
+        let c = column_or_name(input)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.inline(&c)).to_pyerr()?,
         ))
@@ -69,7 +79,7 @@ impl PyTableValuedFunction {
 
     #[pyo3(name = "inline_outer")]
     fn inline_outer(&self, py: Python<'_>, input: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
-        let c = to_column(input)?;
+        let c = column_or_name(input)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.inline_outer(&c)).to_pyerr()?,
         ))
@@ -82,7 +92,16 @@ impl PyTableValuedFunction {
         input: &Bound<'_, PyAny>,
         fields: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let inp = to_column(input)?;
+        // As PySpark's, which asks for fields before it takes the arguments.
+        if fields.is_empty() {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "CANNOT_BE_EMPTY",
+                &[("item", "field")],
+            ));
+        }
+        let inp = column_or_name(input)?;
         let fs = cols(fields)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.json_tuple(&inp, fs)).to_pyerr()?,
@@ -90,7 +109,7 @@ impl PyTableValuedFunction {
     }
 
     fn posexplode(&self, py: Python<'_>, collection: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
-        let c = to_column(collection)?;
+        let c = column_or_name(collection)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.posexplode(&c)).to_pyerr()?,
         ))
@@ -102,7 +121,7 @@ impl PyTableValuedFunction {
         py: Python<'_>,
         collection: &Bound<'_, PyAny>,
     ) -> PyResult<PyDataFrame> {
-        let c = to_column(collection)?;
+        let c = column_or_name(collection)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.posexplode_outer(&c)).to_pyerr()?,
         ))
@@ -115,7 +134,7 @@ impl PyTableValuedFunction {
         n: &Bound<'_, PyAny>,
         fields: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        let nc = to_column(n)?;
+        let nc = column_or_name(n)?;
         let fs = cols(fields)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.stack(&nc, fs)).to_pyerr()?,
@@ -137,7 +156,7 @@ impl PyTableValuedFunction {
 
     #[pyo3(name = "variant_explode")]
     fn variant_explode(&self, py: Python<'_>, input: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
-        let c = to_column(input)?;
+        let c = column_or_name(input)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.variant_explode(&c)).to_pyerr()?,
         ))
@@ -149,7 +168,7 @@ impl PyTableValuedFunction {
         py: Python<'_>,
         input: &Bound<'_, PyAny>,
     ) -> PyResult<PyDataFrame> {
-        let c = to_column(input)?;
+        let c = column_or_name(input)?;
         Ok(PyDataFrame::new(
             py.detach(|| self.inner.variant_explode_outer(&c))
                 .to_pyerr()?,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 66125a4..e95056f 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1443,6 +1443,28 @@ def 
test_plans_name_the_udfs_of_map_functions_after_the_functions_as_pyspark(ses
     assert "FlatMapGroupsInPandas [id#" in out and "], my_grouped_func(" in out
 
 
+def 
test_range_and_table_valued_functions_take_their_arguments_as_pyspark(session):
+    # As PySpark's: range takes the int() of its arguments, and the 
table-valued functions
+    # take Columns or column names, so a str is a column, else 
NOT_EXPECTED_TYPE of col.
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    assert [row.id for row in session.range("3").collect()] == [0, 1, 2]
+    assert [row.id for row in session.tvf.range(0, 6.5, 2.5, "2").collect()] 
== [0, 2, 4]
+    with pytest.raises(ValueError, match="invalid literal"):
+        session.range("x").collect()
+    with pytest.raises(ValueError, match="Value out of range"):
+        session.tvf.range(2**63).collect()
+    with pytest.raises(AnalysisException) as e:
+        session.tvf.json_tuple(F.lit('{"a": 1}'), "a").collect()
+    assert e.value.getCondition() == "UNRESOLVED_COLUMN.WITHOUT_SUGGESTION"
+    with pytest.raises(PySparkTypeError) as e:
+        session.tvf.explode([1, 2])
+    assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+    rows = session.tvf.json_tuple(F.lit('{"a": 1}'), F.lit("a")).collect()
+    assert [tuple(row) for row in rows] == [("1",)]
+
+
 def test_sql_runs_statements_eagerly(session):
     # sql() sends the statement at once: a command takes effect without an 
action, and the
     # DataFrame of its result is the local relation the server returned.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 29b338c..f9070a7 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3210,6 +3210,64 @@ def 
test_replace_and_unpivot_check_their_arguments_as_pyspark():
     ]
 
 
+def 
test_range_and_table_valued_functions_take_their_arguments_as_pyspark(mock_connect):
+    # As PySpark's: spark.range and spark.tvf.range take the int() of their 
arguments, and the
+    # table-valued functions of spark.tvf take Columns or column names, as 
F._to_col does.
+    import re
+
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+
+    def sent():
+        # ExecutePlanRequest.plan=3, Plan.root=1.
+        return 
parse_message(parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0])
+
+    def sent_range():
+        # Relation.range=15: Range.end=2, step=3, num_partitions=4.
+        fields = parse_message(sent()[15][0])
+        return {field: fields[field] for field in (2, 3, 4) if field in fields}
+
+    spark.range("3").collect()
+    assert sent_range() == {2: [3], 3: [1]}
+    spark.tvf.range(0, 6.5, 2.5, "2").collect()
+    assert sent_range() == {2: [6], 3: [2], 4: [2]}
+    for args, error, message in [
+        (("x",), ValueError, "invalid literal for int()"),
+        ((None,), TypeError, "int() argument must be"),
+        ((0, 5, None), TypeError, "int() argument must be"),
+        ((2**63,), ValueError, "Value out of range: 9223372036854775808"),
+        ((0, 3, 1, 2**31), ValueError, "Value out of range: 2147483648"),
+    ]:
+        for range_ in (spark.range, spark.tvf.range):
+            with pytest.raises(error, match=re.escape(message)):
+                range_(*args)
+    spark.tvf.json_tuple(F.lit('{"a": 1}'), "a").collect()
+    # Relation.unresolved_table_valued_function=43: arguments=2, of the column 
`a`, an
+    # Expression.unresolved_attribute=2 of 
UnresolvedAttribute.unparsed_identifier=1.
+    field = parse_message(parse_message(sent()[43][0])[2][1])
+    assert parse_message(field[2][0])[1] == [b"a"]
+    tvf = spark.tvf
+    for function, args, arg_type in [
+        (tvf.explode, ([1, 2],), "list"),
+        (tvf.inline, (1,), "int"),
+        (tvf.stack, (2, F.lit(1)), "int"),
+        (tvf.json_tuple, (F.lit("{}"), 1), "int"),
+    ]:
+        with pytest.raises(PySparkTypeError) as e:
+            function(*args)
+        assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert e.value.getMessageParameters() == {
+            "arg_name": "col",
+            "expected_type": "Column or str",
+            "arg_type": arg_type,
+        }
+    with pytest.raises(PySparkValueError) as e:
+        tvf.json_tuple(1)
+    assert e.value.getCondition() == "CANNOT_BE_EMPTY"
+
+
 def test_functions_replace():
     """Test replace function."""
     from pyspark.sql import functions as F


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to