This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new dd00c18  [SPARK-61124][CONNECT][PYTHON] Check the arguments of 
nearestByJoin, and take them, in the Python client as PySpark does
dd00c18 is described below

commit dd00c1893f3badd2a397ec86e7cab9d85dfa9cc9
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 06:24:22 2026 +0900

    [SPARK-61124][CONNECT][PYTHON] Check the arguments of nearestByJoin, and 
take them, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61124](https://issues.apache.org/jira/browse/SPARK-61124): 
`DataFrame.nearestByJoin` checks its arguments, and takes them, as PySpark's.
    
    PySpark 4.3.0's `nearestByJoin(other, rankingExpression, numResults, mode, 
direction, *, joinType="inner")` checks its arguments before it builds the 
plan, as the Scala `Dataset.validateNearestByJoinArgs` does. It raises an 
`AnalysisException`:
    - `NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE`, for a `numResults` that is 
not between 1 and 100000
    - `NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE`, for a `joinType` other than 
`inner`, `left` or `leftouter` (case- and underscore-insensitive)
    - `NEAREST_BY_JOIN.UNSUPPORTED_MODE`, for a `mode` other than `approx` or 
`exact`
    - `NEAREST_BY_JOIN.UNSUPPORTED_DIRECTION`, for a `direction` other than 
`distance` or `similarity`
    
    The client now does the same. `numResults`, `mode` and `direction` have no 
defaults, `joinType` is keyword-only, and `numResults` is taken as 
`int(numResults)`.
    
    ### Why are the changes needed?
    
    The client checked none of these, so an invalid argument failed only when 
the query ran, if at all. Its `numResults`, `mode` and `direction` defaulted to 
`1`, `"brute"` and `"asc"`, values that PySpark rejects.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Invalid arguments raise as with PySpark 4.3.0, and the arguments 
without defaults in PySpark have none here.
    
    ### How was this patch tested?
    
    - A new offline test checks:
      - the error class and parameters of each invalid argument
      - valid arguments of other cases and a string `numResults`
      - the `TypeError` of missing arguments and of a positional `joinType`
    
      It passes with official PySpark 4.3.0.
    - The existing session test now passes the required arguments.
    - Apache Spark's `test_parity_nearest_by_join.py` passes on this client 
against a local server, all 14 tests. Before this change, its five 
`test_invalid_*` tests failed.
    - The lib tests of both crates (590 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2179 passed, 8 skipped), and the 
live drop-in and Arrow suites (174 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #322 from HyukjinKwon/SPARK-61124-nearest-by-join-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs     | 65 +++++++++++++++++++++++++++++++---
 dev/parity/4.3.0-introspection-diff.md |  6 ++--
 python/tests/test_dropin_offline.py    | 34 +++++++++++++++++-
 3 files changed, 97 insertions(+), 8 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index e7178b9..4b70df5 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2573,22 +2573,79 @@ impl PyDataFrame {
         Ok(())
     }
 
-    /// `DataFrame.nearestByJoin`.
-    #[pyo3(name = "nearestByJoin", signature = (other, rankingExpression, 
numResults = 1, mode = "brute", direction = "asc", joinType = "inner"))]
+    /// `DataFrame.nearestByJoin(other, rankingExpression, numResults, mode, 
direction, *,
+    /// joinType="inner")`, as PySpark's, which checks its arguments as the 
Scala
+    /// `Dataset.validateNearestByJoinArgs` does: an `AnalysisException` of
+    /// `NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE`, `UNSUPPORTED_JOIN_TYPE`, 
`UNSUPPORTED_MODE` or
+    /// `UNSUPPORTED_DIRECTION`.
+    #[pyo3(name = "nearestByJoin", signature = (other, rankingExpression, 
numResults, mode, direction, *, joinType = "inner"))]
+    #[allow(clippy::too_many_arguments)]
     fn nearest_by_join(
         &self,
+        py: Python<'_>,
         other: &Bound<'_, PyAny>,
         rankingExpression: &PyColumn,
-        numResults: i32,
+        numResults: &Bound<'_, PyAny>,
         mode: &str,
         direction: &str,
         joinType: &str,
     ) -> PyResult<PyDataFrame> {
+        const MAX_NUM_RESULTS: i64 = 100_000;
+        const MODES: [&str; 2] = ["approx", "exact"];
+        const DIRECTIONS: [&str; 2] = ["distance", "similarity"];
         let other = self.same_session(other)?;
+        // As PySpark's `int(numResults)`.
+        let num_results = py
+            .import("builtins")?
+            .getattr("int")?
+            .call1((numResults,))?;
+        let analysis = |class: &str, params: &[(&str, &str)]| {
+            crate::column::raise_pyspark(py, "AnalysisException", class, 
params)
+        };
+        let in_range = num_results
+            .extract::<i64>()
+            .is_ok_and(|n| (1..=MAX_NUM_RESULTS).contains(&n));
+        if !in_range {
+            return Err(analysis(
+                "NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE",
+                &[
+                    ("numResults", &num_results.str()?.to_cow()?),
+                    ("min", "1"),
+                    ("max", &MAX_NUM_RESULTS.to_string()),
+                ],
+            ));
+        }
+        if !["inner", "leftouter", "left"]
+            .contains(&joinType.to_lowercase().replace('_', "").as_str())
+        {
+            return Err(analysis(
+                "NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE",
+                &[
+                    ("joinType", joinType),
+                    ("supported", "'INNER', 'LEFT OUTER'"),
+                ],
+            ));
+        }
+        let supported = |values: &[&str]| format!("'{}'", values.join("', '"));
+        if !MODES.contains(&mode.to_lowercase().as_str()) {
+            return Err(analysis(
+                "NEAREST_BY_JOIN.UNSUPPORTED_MODE",
+                &[("mode", mode), ("supported", &supported(&MODES))],
+            ));
+        }
+        if !DIRECTIONS.contains(&direction.to_lowercase().as_str()) {
+            return Err(analysis(
+                "NEAREST_BY_JOIN.UNSUPPORTED_DIRECTION",
+                &[
+                    ("direction", direction),
+                    ("supported", &supported(&DIRECTIONS)),
+                ],
+            ));
+        }
         Ok(PyDataFrame::new(self.dataframe.nearest_by_join(
             &other.dataframe,
             rankingExpression.column.clone(),
-            numResults,
+            num_results.extract::<i32>()?,
             mode,
             direction,
             joinType,
diff --git a/dev/parity/4.3.0-introspection-diff.md 
b/dev/parity/4.3.0-introspection-diff.md
index 1512008..ab1471c 100644
--- a/dev/parity/4.3.0-introspection-diff.md
+++ b/dev/parity/4.3.0-introspection-diff.md
@@ -48,9 +48,9 @@ Fixed here: `SparkSession.Builder.appName/master/remote` 
param names now match p
 
 Remaining, intentionally benign: `Builder.config` accepts an extra optional 
`conf`;
 `emptyDataFrame`, `DataFrameReader.text`, `freqItems`, `persist`, `replace`, 
`melt`,
-`unpivot`, `nearestByJoin` use `None`-sentinel defaults (resolved to the 
documented value
-internally) or default args where the reference marks them required — every 
valid
-reference call is accepted.
+`unpivot` use `None`-sentinel defaults (resolved to the documented value 
internally) or
+default args where the reference marks them required — every valid reference 
call is accepted.
+`nearestByJoin` takes its arguments as the reference does, and checks them 
(SPARK-61124).
 
 ## Full-surface sweep (beyond the 12 core classes)
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 2609f54..9ce3d67 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9147,6 +9147,37 @@ def _offline_session():
     return SparkSession.builder.remote("sc://localhost:1").getOrCreate()
 
 
+def test_nearest_by_join_checks_its_arguments_as_pyspark():
+    # As PySpark's nearestByJoin, as the Scala 
Dataset.validateNearestByJoinArgs: an
+    # AnalysisException of NEAREST_BY_JOIN of the number of results, the join 
type, the mode and
+    # the direction, before the plan; numResults, mode and direction have no 
defaults, and
+    # joinType is keyword-only.
+    from pyspark.errors import AnalysisException
+    from pyspark.sql import functions as F
+
+    spark = _offline_session()
+    a, b = spark.range(1), spark.range(1)
+    ranking = F.lit(1)
+    for kwargs, condition, params in [
+        (dict(numResults=0), "NUM_RESULTS_OUT_OF_RANGE", {"numResults": "0", 
"min": "1", "max": "100000"}),
+        (dict(numResults=100001), "NUM_RESULTS_OUT_OF_RANGE", {"numResults": 
"100001", "min": "1", "max": "100000"}),
+        (dict(joinType="right"), "UNSUPPORTED_JOIN_TYPE", {"joinType": 
"right", "supported": "'INNER', 'LEFT OUTER'"}),
+        (dict(mode="brute"), "UNSUPPORTED_MODE", {"mode": "brute", 
"supported": "'approx', 'exact'"}),
+        (dict(direction="asc"), "UNSUPPORTED_DIRECTION", {"direction": "asc", 
"supported": "'distance', 'similarity'"}),
+    ]:
+        args = {"numResults": 1, "mode": "approx", "direction": "distance", 
**kwargs}
+        with pytest.raises(AnalysisException) as error:
+            a.nearestByJoin(b, ranking, **args)
+        assert error.value.getCondition() == "NEAREST_BY_JOIN." + condition
+        assert error.value.getMessageParameters() == params
+    for join_type in ["inner", "LEFT_OUTER", "left", "leftouter"]:
+        a.nearestByJoin(b, ranking, "2", "EXACT", "Similarity", 
joinType=join_type)
+    with pytest.raises(TypeError):
+        a.nearestByJoin(b, ranking)
+    with pytest.raises(TypeError):
+        a.nearestByJoin(b, ranking, 1, "approx", "distance", "inner")
+
+
 def test_dataframes_of_other_sessions_raise_session_not_same_as_pyspark():
     # As PySpark's `_check_same_session`, of the DataFrames that the methods 
take, before any
     # request: SESSION_NOT_SAME for a DataFrame of another session, and for 
what is not one.
@@ -9159,7 +9190,8 @@ def 
test_dataframes_of_other_sessions_raise_session_not_same_as_pyspark():
         lambda: a.join(b), lambda: a.crossJoin(b), lambda: a.lateralJoin(b),
         lambda: a.union(b), lambda: a.unionAll(b), lambda: a.unionByName(b),
         lambda: a.intersect(b), lambda: a.intersectAll(b), lambda: 
a.subtract(b),
-        lambda: a.exceptAll(b), lambda: a.zip(b), lambda: a.nearestByJoin(b, 
F.lit(1)),
+        lambda: a.exceptAll(b), lambda: a.zip(b),
+        lambda: a.nearestByJoin(b, F.lit(1), 1, "approx", "distance"),
         lambda: a.sameSemantics(b), lambda: a.union(1), lambda: a.join("id"),
         lambda: a.groupBy("id").cogroup(b.groupBy("id")),
     ):


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to