This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 361653d  [SPARK-61012][CONNECT][PYTHON] Sample by the fractions of 
sampleBy, and take a Column, as PySpark does
361653d is described below

commit 361653d47ae954a14bb6a43db522b66bac403087
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:31:19 2026 +0900

    [SPARK-61012][CONNECT][PYTHON] Sample by the fractions of sampleBy, and 
take a Column, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61012](https://issues.apache.org/jira/browse/SPARK-61012): 
`sampleBy` now sends the fraction of each stratum, and takes a column name or a 
Column, as PySpark Connect's does.
    
    - The `StatSampleBy` relation has the fraction of each stratum and the 
column as an expression.
    - `df.sampleBy(col, fractions, seed)` and `df.stat.sampleBy(...)` raise 
`NOT_EXPECTED_TYPE` for a column that is neither a name nor a Column, and for 
fractions that are not a dict. They raise `DISALLOWED_TYPE_FOR_CONTAINER` for a 
stratum that is not a float, an int or a string.
    
    In the Rust API, `StatFunctions::sample_by` takes the column as anything 
that converts into a Column.
    
    ### Why are the changes needed?
    
    The strata were sent without their fractions, so every fraction was 0 and 
the sample was empty. `df.sampleBy("k", {"a": 1.0, "b": 0.0}, seed=1)` returned 
no rows, where PySpark 4.3.0 returns the rows of `"a"`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `sampleBy` samples by its fractions, takes a Column, and raises 
PySpark's errors for arguments of other types.
    
    ### How was this patch tested?
    
    - A new live test samples by the fractions of the strata, by a column name 
and by a Column. It passes with official PySpark 4.3.0, and fails without this 
change.
    - A new offline test checks the errors of the arguments.
    - The plan coverage test checks the fractions of the relation.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (671 passed, and 129 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped), 
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #218 from HyukjinKwon/SPARK-61012-fix.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs                | 13 ++--
 crates/pyspark-rs/src/stat.rs                     | 82 ++++++++++++++++++++---
 crates/spark-connect/src/group.rs                 |  6 +-
 crates/spark-connect/src/plan.rs                  | 32 ++++-----
 crates/spark-connect/tests/coverage_plan_extra.rs | 17 ++++-
 python/tests/test_dropin_e2e.py                   | 12 ++++
 python/tests/test_dropin_offline.py               | 21 ++++++
 7 files changed, 140 insertions(+), 43 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 7bff99a..a0f59d0 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -8,7 +8,6 @@ use spark_connect::udf::{eval_type, 
CommonInlineUserDefinedFunctionExpression, P
 
 use crate::column::PyColumn;
 use crate::errors::ResultExt;
-use crate::functions::to_column;
 use crate::group::PyGroupedData;
 use crate::row::{FieldConverters, PyRow};
 use crate::streaming::PyDataStreamWriter;
@@ -979,17 +978,13 @@ impl PyDataFrame {
     #[pyo3(name = "sampleBy", signature = (col, fractions, seed=None))]
     fn sample_by(
         &self,
-        col: &str,
-        fractions: &Bound<'_, PyDict>,
+        col: &Bound<'_, PyAny>,
+        fractions: &Bound<'_, PyAny>,
         seed: Option<i64>,
     ) -> PyResult<PyDataFrame> {
-        let mut fr = Vec::with_capacity(fractions.len());
-        for (k, v) in fractions.iter() {
-            let key = to_column(&k)?.expression().clone();
-            fr.push((key, v.extract::<f64>()?));
-        }
+        let (col, fractions) = crate::stat::sample_by_arguments(col, 
fractions)?;
         Ok(PyDataFrame::new(
-            self.dataframe.stat().sample_by(col, fr, seed),
+            self.dataframe.stat().sample_by(col, fractions, seed),
         ))
     }
 
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index edb3716..ca26489 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -62,16 +62,12 @@ impl PyStatFunctions {
     #[pyo3(name = "sampleBy", signature = (col, fractions, seed=None))]
     fn sample_by(
         &self,
-        col: &str,
-        fractions: &Bound<'_, pyo3::types::PyDict>,
+        col: &Bound<'_, PyAny>,
+        fractions: &Bound<'_, PyAny>,
         seed: Option<i64>,
     ) -> PyResult<PyDataFrame> {
-        let mut fr = Vec::with_capacity(fractions.len());
-        for (k, v) in fractions.iter() {
-            let key = crate::functions::to_column(&k)?.expression().clone();
-            fr.push((key, v.extract::<f64>()?));
-        }
-        Ok(PyDataFrame::new(self.stat.sample_by(col, fr, seed)))
+        let (col, fractions) = sample_by_arguments(col, fractions)?;
+        Ok(PyDataFrame::new(self.stat.sample_by(col, fractions, seed)))
     }
 
     /// Approximate quantiles of a column at the given probabilities.
@@ -106,3 +102,73 @@ impl PyStatFunctions {
         Ok(quantiles)
     }
 }
+
+/// The column and the fractions of `sampleBy`, as PySpark Connect's 
`DataFrame.sampleBy` takes
+/// them: a column name or a Column, and a dict of the fraction of each 
stratum, a float, an
+/// int or a string.
+pub(crate) fn sample_by_arguments(
+    col: &Bound<'_, PyAny>,
+    fractions: &Bound<'_, PyAny>,
+) -> PyResult<(
+    spark_connect::column::Column,
+    Vec<(spark_connect::expression::Expression, f64)>,
+)> {
+    use crate::column::{raise_pyspark, PyColumn};
+    use pyo3::types::{PyDict, PyFloat, PyInt, PyString};
+
+    let py = col.py();
+    let type_name =
+        |obj: &Bound<'_, PyAny>| -> PyResult<String> { 
Ok(obj.get_type().name()?.to_string()) };
+    let col = if let Ok(name) = col.cast::<PyString>() {
+        spark_connect::functions::col(&name.to_cow()?)
+    } else if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+        column.column.clone()
+    } else {
+        return Err(raise_pyspark(
+            py,
+            "PySparkTypeError",
+            "NOT_EXPECTED_TYPE",
+            &[
+                ("expected_type", "Column or str"),
+                ("arg_name", "col"),
+                ("arg_type", &type_name(col)?),
+            ],
+        ));
+    };
+    let Ok(fractions) = fractions.cast::<PyDict>() else {
+        return Err(raise_pyspark(
+            py,
+            "PySparkTypeError",
+            "NOT_EXPECTED_TYPE",
+            &[
+                ("expected_type", "dict"),
+                ("arg_name", "fractions"),
+                ("arg_type", &type_name(fractions)?),
+            ],
+        ));
+    };
+    let mut strata = Vec::with_capacity(fractions.len());
+    for (key, fraction) in fractions.iter() {
+        if !(key.is_instance_of::<PyFloat>()
+            || key.is_instance_of::<PyInt>()
+            || key.is_instance_of::<PyString>())
+        {
+            return Err(raise_pyspark(
+                py,
+                "PySparkTypeError",
+                "DISALLOWED_TYPE_FOR_CONTAINER",
+                &[
+                    ("arg_name", "fractions"),
+                    ("arg_type", "dict"),
+                    ("allowed_types", "float, int, str"),
+                    ("item_type", &type_name(&key)?),
+                ],
+            ));
+        }
+        let stratum = crate::functions::to_column(&key)?.expression().clone();
+        // As PySpark's float(v).
+        let fraction: f64 = 
py.get_type::<PyFloat>().call1((fraction,))?.extract()?;
+        strata.push((stratum, fraction));
+    }
+    Ok((col, strata))
+}
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index f4619fb..b96c8cc 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -446,13 +446,13 @@ impl StatFunctions {
     /// Sample by values in a column.
     pub fn sample_by(
         &self,
-        col: &str,
+        col: impl Into<Column>,
         fractions: Vec<(Expression, f64)>,
         seed: Option<i64>,
     ) -> DataFrame {
         let plan = LogicalPlan::StatSampleBy {
             input: Arc::clone(&self.dataframe.plan),
-            col: col.to_string(),
+            col: col.into().expression().clone(),
             fractions,
             seed,
         };
@@ -823,7 +823,7 @@ mod tests {
                 fractions,
                 ..
             } => {
-                assert_eq!(col, "category");
+                assert_eq!(*col, Expression::column("category"));
                 assert_eq!(*seed, Some(42));
                 assert_eq!(fractions.len(), 2);
             }
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index fbc33a7..573026c 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -312,7 +312,7 @@ pub enum LogicalPlan {
     /// StatSampleBy: `df.stat.sampleBy(col, fractions, seed)`.
     StatSampleBy {
         input: Arc<LogicalPlan>,
-        col: String,
+        col: Expression,
         fractions: Vec<(Expression, f64)>,
         seed: Option<i64>,
     },
@@ -1361,25 +1361,17 @@ impl LogicalPlan {
             } => {
                 let mut stat = proto::StatSampleBy::default();
                 stat.input = Some(Box::new(relation_of(input, plan_ids)));
-                // col field expects an Expression; create one from the column 
name
-                let mut col_expr = proto::Expression::default();
-                col_expr.expr_type = 
Some(proto::expression::ExprType::UnresolvedAttribute(
-                    proto::expression::UnresolvedAttribute {
-                        unparsed_identifier: col.clone(),
-                        plan_id: None,
-                        is_metadata_column: None,
-                    },
-                ));
-                stat.col = Some(col_expr);
-                for (expr, _frac) in fractions {
-                    // fractions expect Expression.Literal type; create 
Fraction proto
-                    let mut fraction = 
proto::stat_sample_by::Fraction::default();
-                    let expr_proto = expr.to_proto();
-                    // Extract the Literal from the Expression proto
-                    if let Some(proto::expression::ExprType::Literal(lit)) = 
expr_proto.expr_type {
-                        fraction.stratum = Some(lit);
-                    }
-                    stat.fractions.push(fraction);
+                stat.col = Some(col.to_proto());
+                // As PySpark's: each stratum, a literal, with its fraction.
+                for (expr, fraction) in fractions {
+                    let stratum = match expr.to_proto().expr_type {
+                        Some(proto::expression::ExprType::Literal(lit)) => 
Some(lit),
+                        _ => None,
+                    };
+                    stat.fractions.push(proto::stat_sample_by::Fraction {
+                        stratum,
+                        fraction: *fraction,
+                    });
                 }
                 if let Some(s) = seed {
                     stat.seed = Some(*s);
diff --git a/crates/spark-connect/tests/coverage_plan_extra.rs 
b/crates/spark-connect/tests/coverage_plan_extra.rs
index 81784f9..21ed11a 100644
--- a/crates/spark-connect/tests/coverage_plan_extra.rs
+++ b/crates/spark-connect/tests/coverage_plan_extra.rs
@@ -132,17 +132,28 @@ fn read_datasource_to_proto() {
     assert!(p.to_proto().rel_type.is_some());
 }
 
-/// `StatSampleBy::to_proto`: builds the col expression, fraction literals, 
and seed.
+/// `StatSampleBy::to_proto`: builds the col expression, the strata with their 
fractions, as
+/// PySpark sends them, and seed.
 #[test]
 fn stat_sample_by_to_proto() {
     let p = LogicalPlan::StatSampleBy {
         input: Arc::new(base()),
-        col: "k".to_string(),
+        col: spark_connect::expression::Expression::column("k"),
         fractions: vec![
             (lit(0).expression().clone(), 0.1),
             (lit(1).expression().clone(), 0.9),
         ],
         seed: Some(7),
     };
-    assert!(p.to_proto().rel_type.is_some());
+    let Some(spark_connect_proto::relation::RelType::SampleBy(stat)) = 
p.to_proto().rel_type else {
+        panic!("expected StatSampleBy");
+    };
+    let fractions: Vec<f64> = stat.fractions.iter().map(|f| 
f.fraction).collect();
+    assert_eq!(fractions, [0.1, 0.9]);
+    assert!(stat.fractions.iter().all(|f| f.stratum.is_some()));
+    assert_eq!(
+        stat.col,
+        Some(spark_connect::column::col("k").expression().to_proto())
+    );
+    assert_eq!(stat.seed, Some(7));
 }
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 2847def..0745cb7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1048,6 +1048,18 @@ def 
test_to_pandas_converts_the_columns_as_pyspark(session):
         session.conf.unset("spark.sql.session.timeZone")
 
 
+def test_sample_by_samples_the_fractions_of_the_strata_as_pyspark(session):
+    # As PySpark's sampleBy: each stratum by its fraction, of a column name or 
a Column.
+    df = session.createDataFrame([(1, "a"), (2, "b"), (3, "a")], "id int, k 
string")
+    fractions = {"a": 1.0, "b": 0.0}
+    for sampled in (
+        df.sampleBy("k", fractions, seed=1),
+        df.sampleBy(F.col("k"), fractions, seed=1),
+        df.stat.sampleBy("k", fractions, seed=1),
+    ):
+        assert sorted(r.id for r in sampled.collect()) == [1, 3]
+
+
 def test_window_order_by_a_column_sorts_nulls_first_as_pyspark(session):
     from pyspark.sql import Window
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index d3ae644..a85b979 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2572,6 +2572,27 @@ def test_functions_null_and_misc():
     assert type(F.covar_samp(c, c)).__name__ == "Column"
 
 
+def test_sample_by_checks_its_arguments_as_pyspark():
+    # As PySpark's sampleBy: a column name or a Column, a dict of fractions, 
and strata of
+    # floats, ints and strings.
+    from pyspark.errors import PySparkTypeError
+
+    df = _offline_session().range(1)
+    for call, condition, parameters in (
+        (lambda: df.sampleBy(1, {"a": 1.0}), "NOT_EXPECTED_TYPE",
+         {"expected_type": "Column or str", "arg_name": "col", "arg_type": 
"int"}),
+        (lambda: df.stat.sampleBy("k", [0.5]), "NOT_EXPECTED_TYPE",
+         {"expected_type": "dict", "arg_name": "fractions", "arg_type": 
"list"}),
+        (lambda: df.sampleBy("k", {(1, 2): 1.0}), 
"DISALLOWED_TYPE_FOR_CONTAINER",
+         {"arg_name": "fractions", "arg_type": "dict", "allowed_types": 
"float, int, str",
+          "item_type": "tuple"}),
+    ):
+        with pytest.raises(PySparkTypeError) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+
+
 def test_functions_bitwise():
     """Test bitwise functions."""
     from pyspark.sql import functions as F


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to