This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new dd00c18 [SPARK-61124][CONNECT][PYTHON] Check the arguments of
nearestByJoin, and take them, in the Python client as PySpark does
dd00c18 is described below
commit dd00c1893f3badd2a397ec86e7cab9d85dfa9cc9
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 06:24:22 2026 +0900
[SPARK-61124][CONNECT][PYTHON] Check the arguments of nearestByJoin, and
take them, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61124](https://issues.apache.org/jira/browse/SPARK-61124):
`DataFrame.nearestByJoin` checks its arguments, and takes them, as PySpark's.
PySpark 4.3.0's `nearestByJoin(other, rankingExpression, numResults, mode,
direction, *, joinType="inner")` checks its arguments before it builds the
plan, as the Scala `Dataset.validateNearestByJoinArgs` does. It raises an
`AnalysisException`:
- `NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE`, for a `numResults` that is
not between 1 and 100000
- `NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE`, for a `joinType` other than
`inner`, `left` or `leftouter` (case- and underscore-insensitive)
- `NEAREST_BY_JOIN.UNSUPPORTED_MODE`, for a `mode` other than `approx` or
`exact`
- `NEAREST_BY_JOIN.UNSUPPORTED_DIRECTION`, for a `direction` other than
`distance` or `similarity`
The client now does the same. `numResults`, `mode` and `direction` have no
defaults, `joinType` is keyword-only, and `numResults` is taken as
`int(numResults)`.
### Why are the changes needed?
The client checked none of these, so an invalid argument failed only when
the query ran, if at all. Its `numResults`, `mode` and `direction` defaulted to
`1`, `"brute"` and `"asc"`, values that PySpark rejects.
### Does this PR introduce _any_ user-facing change?
Yes. Invalid arguments raise as with PySpark 4.3.0, and the arguments
without defaults in PySpark have none here.
### How was this patch tested?
- A new offline test checks:
- the error class and parameters of each invalid argument
- valid arguments of other cases and a string `numResults`
- the `TypeError` of missing arguments and of a positional `joinType`
It passes with official PySpark 4.3.0.
- The existing session test now passes the required arguments.
- Apache Spark's `test_parity_nearest_by_join.py` passes on this client
against a local server, all 14 tests. Before this change, its five
`test_invalid_*` tests failed.
- The lib tests of both crates (590 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2179 passed, 8 skipped), and the
live drop-in and Arrow suites (174 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #322 from HyukjinKwon/SPARK-61124-nearest-by-join-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 65 +++++++++++++++++++++++++++++++---
dev/parity/4.3.0-introspection-diff.md | 6 ++--
python/tests/test_dropin_offline.py | 34 +++++++++++++++++-
3 files changed, 97 insertions(+), 8 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index e7178b9..4b70df5 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2573,22 +2573,79 @@ impl PyDataFrame {
Ok(())
}
- /// `DataFrame.nearestByJoin`.
- #[pyo3(name = "nearestByJoin", signature = (other, rankingExpression,
numResults = 1, mode = "brute", direction = "asc", joinType = "inner"))]
+ /// `DataFrame.nearestByJoin(other, rankingExpression, numResults, mode,
direction, *,
+ /// joinType="inner")`, as PySpark's, which checks its arguments as the
Scala
+ /// `Dataset.validateNearestByJoinArgs` does: an `AnalysisException` of
+ /// `NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE`, `UNSUPPORTED_JOIN_TYPE`,
`UNSUPPORTED_MODE` or
+ /// `UNSUPPORTED_DIRECTION`.
+ #[pyo3(name = "nearestByJoin", signature = (other, rankingExpression,
numResults, mode, direction, *, joinType = "inner"))]
+ #[allow(clippy::too_many_arguments)]
fn nearest_by_join(
&self,
+ py: Python<'_>,
other: &Bound<'_, PyAny>,
rankingExpression: &PyColumn,
- numResults: i32,
+ numResults: &Bound<'_, PyAny>,
mode: &str,
direction: &str,
joinType: &str,
) -> PyResult<PyDataFrame> {
+ const MAX_NUM_RESULTS: i64 = 100_000;
+ const MODES: [&str; 2] = ["approx", "exact"];
+ const DIRECTIONS: [&str; 2] = ["distance", "similarity"];
let other = self.same_session(other)?;
+ // As PySpark's `int(numResults)`.
+ let num_results = py
+ .import("builtins")?
+ .getattr("int")?
+ .call1((numResults,))?;
+ let analysis = |class: &str, params: &[(&str, &str)]| {
+ crate::column::raise_pyspark(py, "AnalysisException", class,
params)
+ };
+ let in_range = num_results
+ .extract::<i64>()
+ .is_ok_and(|n| (1..=MAX_NUM_RESULTS).contains(&n));
+ if !in_range {
+ return Err(analysis(
+ "NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE",
+ &[
+ ("numResults", &num_results.str()?.to_cow()?),
+ ("min", "1"),
+ ("max", &MAX_NUM_RESULTS.to_string()),
+ ],
+ ));
+ }
+ if !["inner", "leftouter", "left"]
+ .contains(&joinType.to_lowercase().replace('_', "").as_str())
+ {
+ return Err(analysis(
+ "NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE",
+ &[
+ ("joinType", joinType),
+ ("supported", "'INNER', 'LEFT OUTER'"),
+ ],
+ ));
+ }
+ let supported = |values: &[&str]| format!("'{}'", values.join("', '"));
+ if !MODES.contains(&mode.to_lowercase().as_str()) {
+ return Err(analysis(
+ "NEAREST_BY_JOIN.UNSUPPORTED_MODE",
+ &[("mode", mode), ("supported", &supported(&MODES))],
+ ));
+ }
+ if !DIRECTIONS.contains(&direction.to_lowercase().as_str()) {
+ return Err(analysis(
+ "NEAREST_BY_JOIN.UNSUPPORTED_DIRECTION",
+ &[
+ ("direction", direction),
+ ("supported", &supported(&DIRECTIONS)),
+ ],
+ ));
+ }
Ok(PyDataFrame::new(self.dataframe.nearest_by_join(
&other.dataframe,
rankingExpression.column.clone(),
- numResults,
+ num_results.extract::<i32>()?,
mode,
direction,
joinType,
diff --git a/dev/parity/4.3.0-introspection-diff.md
b/dev/parity/4.3.0-introspection-diff.md
index 1512008..ab1471c 100644
--- a/dev/parity/4.3.0-introspection-diff.md
+++ b/dev/parity/4.3.0-introspection-diff.md
@@ -48,9 +48,9 @@ Fixed here: `SparkSession.Builder.appName/master/remote`
param names now match p
Remaining, intentionally benign: `Builder.config` accepts an extra optional
`conf`;
`emptyDataFrame`, `DataFrameReader.text`, `freqItems`, `persist`, `replace`,
`melt`,
-`unpivot`, `nearestByJoin` use `None`-sentinel defaults (resolved to the
documented value
-internally) or default args where the reference marks them required — every
valid
-reference call is accepted.
+`unpivot` use `None`-sentinel defaults (resolved to the documented value
internally) or
+default args where the reference marks them required — every valid reference
call is accepted.
+`nearestByJoin` takes its arguments as the reference does, and checks them
(SPARK-61124).
## Full-surface sweep (beyond the 12 core classes)
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 2609f54..9ce3d67 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9147,6 +9147,37 @@ def _offline_session():
return SparkSession.builder.remote("sc://localhost:1").getOrCreate()
+def test_nearest_by_join_checks_its_arguments_as_pyspark():
+ # As PySpark's nearestByJoin, as the Scala
Dataset.validateNearestByJoinArgs: an
+ # AnalysisException of NEAREST_BY_JOIN of the number of results, the join
type, the mode and
+ # the direction, before the plan; numResults, mode and direction have no
defaults, and
+ # joinType is keyword-only.
+ from pyspark.errors import AnalysisException
+ from pyspark.sql import functions as F
+
+ spark = _offline_session()
+ a, b = spark.range(1), spark.range(1)
+ ranking = F.lit(1)
+ for kwargs, condition, params in [
+ (dict(numResults=0), "NUM_RESULTS_OUT_OF_RANGE", {"numResults": "0",
"min": "1", "max": "100000"}),
+ (dict(numResults=100001), "NUM_RESULTS_OUT_OF_RANGE", {"numResults":
"100001", "min": "1", "max": "100000"}),
+ (dict(joinType="right"), "UNSUPPORTED_JOIN_TYPE", {"joinType":
"right", "supported": "'INNER', 'LEFT OUTER'"}),
+ (dict(mode="brute"), "UNSUPPORTED_MODE", {"mode": "brute",
"supported": "'approx', 'exact'"}),
+ (dict(direction="asc"), "UNSUPPORTED_DIRECTION", {"direction": "asc",
"supported": "'distance', 'similarity'"}),
+ ]:
+ args = {"numResults": 1, "mode": "approx", "direction": "distance",
**kwargs}
+ with pytest.raises(AnalysisException) as error:
+ a.nearestByJoin(b, ranking, **args)
+ assert error.value.getCondition() == "NEAREST_BY_JOIN." + condition
+ assert error.value.getMessageParameters() == params
+ for join_type in ["inner", "LEFT_OUTER", "left", "leftouter"]:
+ a.nearestByJoin(b, ranking, "2", "EXACT", "Similarity",
joinType=join_type)
+ with pytest.raises(TypeError):
+ a.nearestByJoin(b, ranking)
+ with pytest.raises(TypeError):
+ a.nearestByJoin(b, ranking, 1, "approx", "distance", "inner")
+
+
def test_dataframes_of_other_sessions_raise_session_not_same_as_pyspark():
# As PySpark's `_check_same_session`, of the DataFrames that the methods
take, before any
# request: SESSION_NOT_SAME for a DataFrame of another session, and for
what is not one.
@@ -9159,7 +9190,8 @@ def
test_dataframes_of_other_sessions_raise_session_not_same_as_pyspark():
lambda: a.join(b), lambda: a.crossJoin(b), lambda: a.lateralJoin(b),
lambda: a.union(b), lambda: a.unionAll(b), lambda: a.unionByName(b),
lambda: a.intersect(b), lambda: a.intersectAll(b), lambda:
a.subtract(b),
- lambda: a.exceptAll(b), lambda: a.zip(b), lambda: a.nearestByJoin(b,
F.lit(1)),
+ lambda: a.exceptAll(b), lambda: a.zip(b),
+ lambda: a.nearestByJoin(b, F.lit(1), 1, "approx", "distance"),
lambda: a.sameSemantics(b), lambda: a.union(1), lambda: a.join("id"),
lambda: a.groupBy("id").cogroup(b.groupBy("id")),
):
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]