This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 361653d [SPARK-61012][CONNECT][PYTHON] Sample by the fractions of
sampleBy, and take a Column, as PySpark does
361653d is described below
commit 361653d47ae954a14bb6a43db522b66bac403087
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 01:31:19 2026 +0900
[SPARK-61012][CONNECT][PYTHON] Sample by the fractions of sampleBy, and
take a Column, as PySpark does
### What changes were proposed in this pull request?
[SPARK-61012](https://issues.apache.org/jira/browse/SPARK-61012):
`sampleBy` now sends the fraction of each stratum, and takes a column name or a
Column, as PySpark Connect's does.
- The `StatSampleBy` relation has the fraction of each stratum and the
column as an expression.
- `df.sampleBy(col, fractions, seed)` and `df.stat.sampleBy(...)` raise
`NOT_EXPECTED_TYPE` for a column that is neither a name nor a Column, and for
fractions that are not a dict. They raise `DISALLOWED_TYPE_FOR_CONTAINER` for a
stratum that is not a float, an int or a string.
In the Rust API, `StatFunctions::sample_by` takes the column as anything
that converts into a Column.
### Why are the changes needed?
The strata were sent without their fractions, so every fraction was 0 and
the sample was empty. `df.sampleBy("k", {"a": 1.0, "b": 0.0}, seed=1)` returned
no rows, where PySpark 4.3.0 returns the rows of `"a"`.
### Does this PR introduce _any_ user-facing change?
Yes. `sampleBy` samples by its fractions, takes a Column, and raises
PySpark's errors for arguments of other types.
### How was this patch tested?
- A new live test samples by the fractions of the strata, by a column name
and by a Column. It passes with official PySpark 4.3.0, and fails without this
change.
- A new offline test checks the errors of the arguments.
- The plan coverage test checks the fractions of the relation.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (671 passed, and 129 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1625 passed, 8 skipped),
and the live drop-in and Arrow suites (95 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #218 from HyukjinKwon/SPARK-61012-fix.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 13 ++--
crates/pyspark-rs/src/stat.rs | 82 ++++++++++++++++++++---
crates/spark-connect/src/group.rs | 6 +-
crates/spark-connect/src/plan.rs | 32 ++++-----
crates/spark-connect/tests/coverage_plan_extra.rs | 17 ++++-
python/tests/test_dropin_e2e.py | 12 ++++
python/tests/test_dropin_offline.py | 21 ++++++
7 files changed, 140 insertions(+), 43 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 7bff99a..a0f59d0 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -8,7 +8,6 @@ use spark_connect::udf::{eval_type,
CommonInlineUserDefinedFunctionExpression, P
use crate::column::PyColumn;
use crate::errors::ResultExt;
-use crate::functions::to_column;
use crate::group::PyGroupedData;
use crate::row::{FieldConverters, PyRow};
use crate::streaming::PyDataStreamWriter;
@@ -979,17 +978,13 @@ impl PyDataFrame {
#[pyo3(name = "sampleBy", signature = (col, fractions, seed=None))]
fn sample_by(
&self,
- col: &str,
- fractions: &Bound<'_, PyDict>,
+ col: &Bound<'_, PyAny>,
+ fractions: &Bound<'_, PyAny>,
seed: Option<i64>,
) -> PyResult<PyDataFrame> {
- let mut fr = Vec::with_capacity(fractions.len());
- for (k, v) in fractions.iter() {
- let key = to_column(&k)?.expression().clone();
- fr.push((key, v.extract::<f64>()?));
- }
+ let (col, fractions) = crate::stat::sample_by_arguments(col,
fractions)?;
Ok(PyDataFrame::new(
- self.dataframe.stat().sample_by(col, fr, seed),
+ self.dataframe.stat().sample_by(col, fractions, seed),
))
}
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index edb3716..ca26489 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -62,16 +62,12 @@ impl PyStatFunctions {
#[pyo3(name = "sampleBy", signature = (col, fractions, seed=None))]
fn sample_by(
&self,
- col: &str,
- fractions: &Bound<'_, pyo3::types::PyDict>,
+ col: &Bound<'_, PyAny>,
+ fractions: &Bound<'_, PyAny>,
seed: Option<i64>,
) -> PyResult<PyDataFrame> {
- let mut fr = Vec::with_capacity(fractions.len());
- for (k, v) in fractions.iter() {
- let key = crate::functions::to_column(&k)?.expression().clone();
- fr.push((key, v.extract::<f64>()?));
- }
- Ok(PyDataFrame::new(self.stat.sample_by(col, fr, seed)))
+ let (col, fractions) = sample_by_arguments(col, fractions)?;
+ Ok(PyDataFrame::new(self.stat.sample_by(col, fractions, seed)))
}
/// Approximate quantiles of a column at the given probabilities.
@@ -106,3 +102,73 @@ impl PyStatFunctions {
Ok(quantiles)
}
}
+
+/// The column and the fractions of `sampleBy`, as PySpark Connect's
`DataFrame.sampleBy` takes
+/// them: a column name or a Column, and a dict of the fraction of each
stratum, a float, an
+/// int or a string.
+pub(crate) fn sample_by_arguments(
+ col: &Bound<'_, PyAny>,
+ fractions: &Bound<'_, PyAny>,
+) -> PyResult<(
+ spark_connect::column::Column,
+ Vec<(spark_connect::expression::Expression, f64)>,
+)> {
+ use crate::column::{raise_pyspark, PyColumn};
+ use pyo3::types::{PyDict, PyFloat, PyInt, PyString};
+
+ let py = col.py();
+ let type_name =
+ |obj: &Bound<'_, PyAny>| -> PyResult<String> {
Ok(obj.get_type().name()?.to_string()) };
+ let col = if let Ok(name) = col.cast::<PyString>() {
+ spark_connect::functions::col(&name.to_cow()?)
+ } else if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+ column.column.clone()
+ } else {
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "Column or str"),
+ ("arg_name", "col"),
+ ("arg_type", &type_name(col)?),
+ ],
+ ));
+ };
+ let Ok(fractions) = fractions.cast::<PyDict>() else {
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "dict"),
+ ("arg_name", "fractions"),
+ ("arg_type", &type_name(fractions)?),
+ ],
+ ));
+ };
+ let mut strata = Vec::with_capacity(fractions.len());
+ for (key, fraction) in fractions.iter() {
+ if !(key.is_instance_of::<PyFloat>()
+ || key.is_instance_of::<PyInt>()
+ || key.is_instance_of::<PyString>())
+ {
+ return Err(raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "DISALLOWED_TYPE_FOR_CONTAINER",
+ &[
+ ("arg_name", "fractions"),
+ ("arg_type", "dict"),
+ ("allowed_types", "float, int, str"),
+ ("item_type", &type_name(&key)?),
+ ],
+ ));
+ }
+ let stratum = crate::functions::to_column(&key)?.expression().clone();
+ // As PySpark's float(v).
+ let fraction: f64 =
py.get_type::<PyFloat>().call1((fraction,))?.extract()?;
+ strata.push((stratum, fraction));
+ }
+ Ok((col, strata))
+}
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index f4619fb..b96c8cc 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -446,13 +446,13 @@ impl StatFunctions {
/// Sample by values in a column.
pub fn sample_by(
&self,
- col: &str,
+ col: impl Into<Column>,
fractions: Vec<(Expression, f64)>,
seed: Option<i64>,
) -> DataFrame {
let plan = LogicalPlan::StatSampleBy {
input: Arc::clone(&self.dataframe.plan),
- col: col.to_string(),
+ col: col.into().expression().clone(),
fractions,
seed,
};
@@ -823,7 +823,7 @@ mod tests {
fractions,
..
} => {
- assert_eq!(col, "category");
+ assert_eq!(*col, Expression::column("category"));
assert_eq!(*seed, Some(42));
assert_eq!(fractions.len(), 2);
}
diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs
index fbc33a7..573026c 100644
--- a/crates/spark-connect/src/plan.rs
+++ b/crates/spark-connect/src/plan.rs
@@ -312,7 +312,7 @@ pub enum LogicalPlan {
/// StatSampleBy: `df.stat.sampleBy(col, fractions, seed)`.
StatSampleBy {
input: Arc<LogicalPlan>,
- col: String,
+ col: Expression,
fractions: Vec<(Expression, f64)>,
seed: Option<i64>,
},
@@ -1361,25 +1361,17 @@ impl LogicalPlan {
} => {
let mut stat = proto::StatSampleBy::default();
stat.input = Some(Box::new(relation_of(input, plan_ids)));
- // col field expects an Expression; create one from the column
name
- let mut col_expr = proto::Expression::default();
- col_expr.expr_type =
Some(proto::expression::ExprType::UnresolvedAttribute(
- proto::expression::UnresolvedAttribute {
- unparsed_identifier: col.clone(),
- plan_id: None,
- is_metadata_column: None,
- },
- ));
- stat.col = Some(col_expr);
- for (expr, _frac) in fractions {
- // fractions expect Expression.Literal type; create
Fraction proto
- let mut fraction =
proto::stat_sample_by::Fraction::default();
- let expr_proto = expr.to_proto();
- // Extract the Literal from the Expression proto
- if let Some(proto::expression::ExprType::Literal(lit)) =
expr_proto.expr_type {
- fraction.stratum = Some(lit);
- }
- stat.fractions.push(fraction);
+ stat.col = Some(col.to_proto());
+ // As PySpark's: each stratum, a literal, with its fraction.
+ for (expr, fraction) in fractions {
+ let stratum = match expr.to_proto().expr_type {
+ Some(proto::expression::ExprType::Literal(lit)) =>
Some(lit),
+ _ => None,
+ };
+ stat.fractions.push(proto::stat_sample_by::Fraction {
+ stratum,
+ fraction: *fraction,
+ });
}
if let Some(s) = seed {
stat.seed = Some(*s);
diff --git a/crates/spark-connect/tests/coverage_plan_extra.rs
b/crates/spark-connect/tests/coverage_plan_extra.rs
index 81784f9..21ed11a 100644
--- a/crates/spark-connect/tests/coverage_plan_extra.rs
+++ b/crates/spark-connect/tests/coverage_plan_extra.rs
@@ -132,17 +132,28 @@ fn read_datasource_to_proto() {
assert!(p.to_proto().rel_type.is_some());
}
-/// `StatSampleBy::to_proto`: builds the col expression, fraction literals,
and seed.
+/// `StatSampleBy::to_proto`: builds the col expression, the strata with their
fractions, as
+/// PySpark sends them, and seed.
#[test]
fn stat_sample_by_to_proto() {
let p = LogicalPlan::StatSampleBy {
input: Arc::new(base()),
- col: "k".to_string(),
+ col: spark_connect::expression::Expression::column("k"),
fractions: vec![
(lit(0).expression().clone(), 0.1),
(lit(1).expression().clone(), 0.9),
],
seed: Some(7),
};
- assert!(p.to_proto().rel_type.is_some());
+ let Some(spark_connect_proto::relation::RelType::SampleBy(stat)) =
p.to_proto().rel_type else {
+ panic!("expected StatSampleBy");
+ };
+ let fractions: Vec<f64> = stat.fractions.iter().map(|f|
f.fraction).collect();
+ assert_eq!(fractions, [0.1, 0.9]);
+ assert!(stat.fractions.iter().all(|f| f.stratum.is_some()));
+ assert_eq!(
+ stat.col,
+ Some(spark_connect::column::col("k").expression().to_proto())
+ );
+ assert_eq!(stat.seed, Some(7));
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 2847def..0745cb7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1048,6 +1048,18 @@ def
test_to_pandas_converts_the_columns_as_pyspark(session):
session.conf.unset("spark.sql.session.timeZone")
+def test_sample_by_samples_the_fractions_of_the_strata_as_pyspark(session):
+ # As PySpark's sampleBy: each stratum by its fraction, of a column name or
a Column.
+ df = session.createDataFrame([(1, "a"), (2, "b"), (3, "a")], "id int, k
string")
+ fractions = {"a": 1.0, "b": 0.0}
+ for sampled in (
+ df.sampleBy("k", fractions, seed=1),
+ df.sampleBy(F.col("k"), fractions, seed=1),
+ df.stat.sampleBy("k", fractions, seed=1),
+ ):
+ assert sorted(r.id for r in sampled.collect()) == [1, 3]
+
+
def test_window_order_by_a_column_sorts_nulls_first_as_pyspark(session):
from pyspark.sql import Window
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index d3ae644..a85b979 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2572,6 +2572,27 @@ def test_functions_null_and_misc():
assert type(F.covar_samp(c, c)).__name__ == "Column"
+def test_sample_by_checks_its_arguments_as_pyspark():
+ # As PySpark's sampleBy: a column name or a Column, a dict of fractions,
and strata of
+ # floats, ints and strings.
+ from pyspark.errors import PySparkTypeError
+
+ df = _offline_session().range(1)
+ for call, condition, parameters in (
+ (lambda: df.sampleBy(1, {"a": 1.0}), "NOT_EXPECTED_TYPE",
+ {"expected_type": "Column or str", "arg_name": "col", "arg_type":
"int"}),
+ (lambda: df.stat.sampleBy("k", [0.5]), "NOT_EXPECTED_TYPE",
+ {"expected_type": "dict", "arg_name": "fractions", "arg_type":
"list"}),
+ (lambda: df.sampleBy("k", {(1, 2): 1.0}),
"DISALLOWED_TYPE_FOR_CONTAINER",
+ {"arg_name": "fractions", "arg_type": "dict", "allowed_types":
"float, int, str",
+ "item_type": "tuple"}),
+ ):
+ with pytest.raises(PySparkTypeError) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+
+
def test_functions_bitwise():
"""Test bitwise functions."""
from pyspark.sql import functions as F
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]