This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new db6c0c5  [SPARK-60990][CONNECT][PYTHON] Sort nulls first in 
Window.orderBy of a column as PySpark does
db6c0c5 is described below

commit db6c0c5bdcc4239e46402f0720b3d5e6bd3cb0e5
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 14:07:23 2026 +0900

    [SPARK-60990][CONNECT][PYTHON] Sort nulls first in Window.orderBy of a 
column as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60990](https://issues.apache.org/jira/browse/SPARK-60990): 
`Window.orderBy` and `WindowSpec.orderBy` now order by a column name, or a 
Column that is not a sort order, ascending with nulls first. PySpark's 
`SortOrder(expr)` defaults to that, and Spark sorts ascending that way. Before, 
such a column was ordered ascending with nulls last. A Column that is a sort 
order, such as `F.col("o").desc()`, is unchanged.
    
    ### Why are the changes needed?
    
    Window functions over a column with nulls gave other results than in 
PySpark. For example, over the values NULL, 1 and 2 of `o`:
    
    ```python
    df.select("o", F.row_number().over(Window.partitionBy("g").orderBy("o")))
    ```
    
    This gives the rows (NULL, 1), (1, 2), (2, 3) in PySpark 4.3.0, and gave 
(1, 1), (2, 2), (NULL, 3) here. `rank`, `lag`, running aggregates and the other 
functions over such a window differed the same way. `DataFrame.orderBy` and 
`sort` were already right.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, window functions over a column with nulls give the same results as in 
PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, `row_number` over windows ordered by 
a name, a Column, a spec ordered twice and a descending Column gives the same 
rows as official PySpark 4.3.0. Before this change, three of the four differed.
    - A new live test checks those windows. It also passes with official 
PySpark 4.3.0, and fails without this change.
    - The lib tests and every integration test, e2e included, against the local 
server (647 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (676 passed, 9 skipped), and the live drop-in and Arrow suites (70 
passed) and live cases (10 passed). Clippy reports no diagnostics in the 
changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #191 from HyukjinKwon/SPARK-60990-window-nulls-first.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/window.rs | 11 ++++-------
 python/tests/test_dropin_e2e.py | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/crates/pyspark-rs/src/window.rs b/crates/pyspark-rs/src/window.rs
index 1ff756b..09b2b70 100644
--- a/crates/pyspark-rs/src/window.rs
+++ b/crates/pyspark-rs/src/window.rs
@@ -1,7 +1,7 @@
 //! PyO3 wrappers for window functions and specifications.
 
 use pyo3::prelude::*;
-use spark_connect::expression::{Expression, ExpressionNode, NullOrdering, 
SortOrder};
+use spark_connect::expression::{Expression, ExpressionNode, SortOrder};
 use spark_connect::window::{FrameBound, WindowSpec};
 
 use crate::dataframe::to_column_list;
@@ -16,18 +16,15 @@ fn to_partition_exprs(cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<Vec<Expression>>
 }
 
 /// Build sort orders from ColumnOrName args; a bare string becomes an 
ascending
-/// SortOrder over that column (not a string literal).
+/// SortOrder over that column (not a string literal). A column that is not a 
sort order is
+/// ascending with nulls first, as `SortOrder(expr)` of PySpark defaults to.
 fn to_sort_orders(cols: Vec<Bound<'_, PyAny>>) -> PyResult<Vec<SortOrder>> {
     let mut sort_orders = Vec::new();
     for column in to_column_list(cols)? {
         if let ExpressionNode::SortOrder(sort) = column.expression().node() {
             sort_orders.push((**sort).clone());
         } else {
-            sort_orders.push(SortOrder {
-                child: column.expression().clone(),
-                ascending: true,
-                null_ordering: NullOrdering::Last,
-            });
+            
sort_orders.push(SortOrder::asc_nulls_first(column.expression().clone()));
         }
     }
     Ok(sort_orders)
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index b153d0e..4de0fb8 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -638,6 +638,21 @@ def 
test_column_alias_names_every_column_and_keeps_its_metadata(session):
     assert df.schema.fields[0].metadata == {"k": 1, "n": [1, 2], "s": "v"}
 
 
+def test_window_order_by_a_column_sorts_nulls_first_as_pyspark(session):
+    from pyspark.sql import Window
+
+    df = session.sql("SELECT * FROM VALUES (1, CAST(NULL AS INT)), (1, 2), (1, 
1) AS t(g, o)")
+    # A column that is not a sort order is ascending with nulls first, as 
SortOrder(expr).
+    for window, expected in (
+        (Window.partitionBy("g").orderBy("o"), [(None, 1), (1, 2), (2, 3)]),
+        (Window.partitionBy("g").orderBy(F.col("o")), [(None, 1), (1, 2), (2, 
3)]),
+        (Window.orderBy("o").partitionBy("g").orderBy("o"), [(None, 1), (1, 
2), (2, 3)]),
+        (Window.partitionBy("g").orderBy(F.col("o").desc()), [(2, 1), (1, 2), 
(None, 3)]),
+    ):
+        rows = df.select("o", 
F.row_number().over(window).alias("rn")).orderBy("rn").collect()
+        assert [(r.o, r.rn) for r in rows] == expected
+
+
 def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     row = session.range(1).selectExpr(
         "map(2, 'b', 1, 'a') AS ints",


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to