This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 21b0899 [SPARK-61066][CONNECT][PYTHON] Raise the errors of join types
and dict aggregations, and return the status and progress of a streaming query,
as PySpark does
21b0899 is described below
commit 21b08990ec95cd585c5ef823ab41d35fa8198248
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:14:35 2026 +0900
[SPARK-61066][CONNECT][PYTHON] Raise the errors of join types and dict
aggregations, and return the status and progress of a streaming query, as
PySpark does
### What changes were proposed in this pull request?
- `join` and `lateralJoin` normalize `how` as official's `plan.Join` and
`plan.LateralJoin` do (lowercased, without underscores, `None` as inner) and
raise `AnalysisException` `UNSUPPORTED_JOIN_TYPE` for the others, when `join`
is called, as official does (`crates/pyspark-rs/src/dataframe.rs`).
`lateralJoin` accepts only inner, left outer and cross, as official. `on`
follows `plan.Join`: a list of Columns joins on their conjunction, which failed
before, and the other forms raise offic [...]
- `DataFrame.agg({})` raises `AssertionError: exprs should not be empty`;
dict aggregations raise official's errors for a non-str column
(`AttributeError`) or function (`AssertionError`), in a helper that
`DataFrame.agg` and `GroupedData.agg` share (`crates/pyspark-rs/src/group.rs`).
- `StreamingQuery.status` returns official's dict (`message`,
`isDataAvailable`, `isTriggerActive`), and `lastProgress` and `recentProgress`
return `StreamingQueryProgress` objects instead of JSON strings
(`crates/pyspark-rs/src/streaming.rs`).
### Why are the changes needed?
[SPARK-61066](https://issues.apache.org/jira/browse/SPARK-61066): against a
local Spark 4.3.0 server:
| Code | Current master | This change and PySpark 4.3.0 |
| --- | --- | --- |
| `df.join(df, "id", "foo")` | `ValueError: Invalid join type: foo` |
`AnalysisException [UNSUPPORTED_JOIN_TYPE]` |
| `df.join(df2, [a.x == b.y, a.x > -1])` | `TypeError` | joins on the
conjunction |
| `df.agg({})` | `[Row()]` | `AssertionError: exprs should not be empty` |
| `df.agg({"id": ["max"]})` | `TypeError` | `AssertionError` |
| `q.status["isDataAvailable"]` | `TypeError: ... not subscriptable` | a
bool |
| `q.lastProgress` | a JSON string | `StreamingQueryProgress` |
### Does this PR introduce _any_ user-facing change?
Yes, these APIs behave as in PySpark.
### How was this patch tested?
- New offline tests in `python/tests/test_dropin_offline.py` for the join
types sent and raised, the forms of `on`, and the dict aggregations, and live
tests in `python/tests/test_dropin_e2e.py` for an unsupported join type and the
status and progress of a streaming query.
- The offline suite (992 passed, 8 skipped) and the live join, aggregation
and streaming tests.
An `on` of a dict or tuple raises official's `AttributeError` at `join`,
where official raises it at `collect`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #257 from HyukjinKwon/SPARK-61066-join-agg-status.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 190 ++++++++++++++++++++++++++----------
crates/pyspark-rs/src/group.rs | 53 +++++++---
crates/pyspark-rs/src/streaming.rs | 27 ++++-
python/tests/test_dropin_e2e.py | 44 +++++++++
python/tests/test_dropin_offline.py | 111 +++++++++++++++++++++
5 files changed, 351 insertions(+), 74 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 8e7b9bd..77bbe83 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -426,22 +426,71 @@ pub(crate) fn to_column_list(
Ok(cols)
}
-/// Map a pyspark `how` string to a JoinType (shared by join / lateralJoin).
-fn parse_join_type(how: &str) -> PyResult<JoinType> {
- Ok(match how.to_lowercase().as_str() {
- "inner" => JoinType::Inner,
- "left" | "leftouter" | "left_outer" => JoinType::LeftOuter,
- "right" | "rightouter" | "right_outer" => JoinType::RightOuter,
- "outer" | "full" | "fullouter" | "full_outer" => JoinType::FullOuter,
- "cross" => JoinType::Cross,
- "left_semi" | "leftsemi" | "semi" => JoinType::LeftSemi,
- "left_anti" | "leftanti" | "anti" => JoinType::LeftAnti,
- other => {
- return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(format!(
- "Invalid join type: {other}"
- )))
- }
- })
+/// The join types that `DataFrame.join` accepts, as the message of its
`UNSUPPORTED_JOIN_TYPE`.
+const JOIN_TYPES_SUPPORTED: &str = "'inner', 'outer', 'full', 'fullouter',
'full_outer', 'leftouter', 'left', 'left_outer', 'rightouter', 'right',
'right_outer', 'leftsemi', 'left_semi', 'semi', 'leftanti', 'left_anti',
'anti', 'cross'";
+
+/// The join types that `DataFrame.lateralJoin` accepts, as the message of its
+/// `UNSUPPORTED_JOIN_TYPE`.
+const LATERAL_JOIN_TYPES_SUPPORTED: &str = "'inner', 'leftouter', 'left',
'left_outer', 'cross'";
+
+/// Map a pyspark `how` to a JoinType, as the connect client's `plan.Join` (or
`plan.LateralJoin`
+/// when `lateral`) maps it. The `how` is lowercased with its underscores
removed, and no `how`
+/// is an inner join. One that is not accepted raises the `AnalysisException`
of
+/// `UNSUPPORTED_JOIN_TYPE` that PySpark raises locally, naming the normalized
`how` (or the
+/// `str()` of a `how` that is not a str).
+fn parse_join_type(
+ py: Python<'_>,
+ how: Option<&Bound<'_, PyAny>>,
+ lateral: bool,
+) -> PyResult<JoinType> {
+ let Some(how) = how.filter(|how| !how.is_none()) else {
+ return Ok(JoinType::Inner);
+ };
+ let normalized = how
+ .extract::<String>()
+ .ok()
+ .map(|how| how.to_lowercase().replace('_', ""));
+ let join_type = match normalized.as_deref() {
+ Some("inner") => Some(JoinType::Inner),
+ Some("cross") => Some(JoinType::Cross),
+ Some("left" | "leftouter") => Some(JoinType::LeftOuter),
+ Some("right" | "rightouter") if !lateral => Some(JoinType::RightOuter),
+ Some("outer" | "full" | "fullouter") if !lateral =>
Some(JoinType::FullOuter),
+ Some("leftsemi" | "semi") if !lateral => Some(JoinType::LeftSemi),
+ Some("leftanti" | "anti") if !lateral => Some(JoinType::LeftAnti),
+ _ => None,
+ };
+ if let Some(join_type) = join_type {
+ return Ok(join_type);
+ }
+ let typ = match normalized {
+ Some(normalized) => normalized,
+ None => how.str()?.to_string(),
+ };
+ let supported = if lateral {
+ LATERAL_JOIN_TYPES_SUPPORTED
+ } else {
+ JOIN_TYPES_SUPPORTED
+ };
+ Err(crate::column::raise_pyspark(
+ py,
+ "AnalysisException",
+ "UNSUPPORTED_JOIN_TYPE",
+ &[("typ", typ.as_str()), ("supported", supported)],
+ ))
+}
+
+/// The `AttributeError` that PySpark's join raises for an `on` that is
neither a name, a
+/// condition nor a list of them: the plan reads `to_plan` of it.
+fn no_to_plan(obj: &Bound<'_, PyAny>) -> PyErr {
+ let typ = obj
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ pyo3::exceptions::PyAttributeError::new_err(format!(
+ "'{typ}' object has no attribute 'to_plan'"
+ ))
}
/// Resolve a filter/where condition: a Column, or a SQL string parsed via
`expr`.
@@ -693,52 +742,83 @@ impl PyDataFrame {
/// Join with another DataFrame.
///
- /// `on` accepts a Column condition, a column-name string, or a list of
- /// column-name strings (name-based "using" join), mirroring
- /// `pyspark.sql.DataFrame.join(other, on=None, how=None)` (default inner).
+ /// `on` is a Column condition, a column name, or a list of column names
(a "using"
+ /// join) or of Column conditions (joined on their conjunction), as the
connect client's
+ /// `plan.Join` takes it. Mirrors `pyspark.sql.DataFrame.join(other,
on=None, how=None)`
+ /// (default inner), including its errors.
#[pyo3(signature = (other, on=None, how=None))]
fn join(
&self,
+ py: Python<'_>,
other: &Bound<'_, PyAny>,
on: Option<Bound<'_, PyAny>>,
- how: Option<&str>,
+ how: Option<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
+ // As PySpark's `_check_same_session`, first.
let other = self.same_session(other)?;
- let join_type = parse_join_type(how.unwrap_or("inner"))?;
-
- // Resolve `on`: None | Column | str | list[str].
- if let Some(bound) = on {
- if let Ok(col) = bound.extract::<PyRef<PyColumn>>() {
- let on_col = Some(col.column.clone());
- return Ok(PyDataFrame::new(self.dataframe.join(
- &other.dataframe,
- on_col,
- join_type,
- )));
+ // As PySpark's `plan.Join`, `on` is iterated before `how` is checked,
so an `on`
+ // that is not iterable is the error that is raised first.
+ if let Some(on) = &on {
+ if !on.is_instance_of::<PyColumn>() &&
!on.is_instance_of::<pyo3::types::PyString>() {
+ on.try_iter()?;
}
- if let Ok(name) = bound.extract::<String>() {
- return Ok(PyDataFrame::new(self.dataframe.join_using(
+ }
+ let join_type = parse_join_type(py, how.as_ref(), false)?;
+ let Some(on) = on else {
+ return Ok(PyDataFrame::new(self.dataframe.join(
+ &other.dataframe,
+ None,
+ join_type,
+ )));
+ };
+ if let Ok(col) = on.extract::<PyRef<PyColumn>>() {
+ return Ok(PyDataFrame::new(self.dataframe.join(
+ &other.dataframe,
+ Some(col.column.clone()),
+ join_type,
+ )));
+ }
+ if let Ok(name) = on.extract::<String>() {
+ return Ok(PyDataFrame::new(self.dataframe.join_using(
+ &other.dataframe,
+ vec![name],
+ join_type,
+ )));
+ }
+ if let Ok(list) = on.cast::<PyList>() {
+ // An empty list is no condition at all.
+ let Some(first) = list.iter().next() else {
+ return Ok(PyDataFrame::new(self.dataframe.join(
&other.dataframe,
- vec![name],
+ None,
join_type,
)));
- }
- if let Ok(names) = bound.extract::<Vec<String>>() {
+ };
+ if first.is_instance_of::<pyo3::types::PyString>() {
+ let names = list.extract::<Vec<String>>()?;
return Ok(PyDataFrame::new(self.dataframe.join_using(
&other.dataframe,
names,
join_type,
)));
}
- return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "join 'on' must be a Column, a str, or a list of str",
- ));
+ let mut condition: Option<spark_connect::column::Column> = None;
+ for item in list.iter() {
+ let Ok(col) = item.extract::<PyRef<PyColumn>>() else {
+ return Err(no_to_plan(&item));
+ };
+ condition = Some(match condition {
+ None => col.column.clone(),
+ Some(left) => left.and(col.column.clone()),
+ });
+ }
+ return Ok(PyDataFrame::new(self.dataframe.join(
+ &other.dataframe,
+ condition,
+ join_type,
+ )));
}
- Ok(PyDataFrame::new(self.dataframe.join(
- &other.dataframe,
- None,
- join_type,
- )))
+ Err(no_to_plan(&on))
}
/// Cross join.
@@ -749,17 +829,19 @@ impl PyDataFrame {
))
}
- /// Lateral (correlated) join. `on` is an optional Column condition; `how`
- /// mirrors `join` (default inner). Mirrors `DataFrame.lateralJoin`.
+ /// Lateral (correlated) join. `on` is an optional Column condition; `how`
is inner by
+ /// default and takes the join types of a lateral join. Mirrors
`DataFrame.lateralJoin`.
#[pyo3(signature = (other, on=None, how=None))]
fn lateralJoin(
&self,
+ py: Python<'_>,
other: &Bound<'_, PyAny>,
on: Option<PyRef<PyColumn>>,
- how: Option<&str>,
+ how: Option<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
+ // As PySpark's `_check_same_session`, first.
let other = self.same_session(other)?;
- let join_type = parse_join_type(how.unwrap_or("inner"))?;
+ let join_type = parse_join_type(py, how.as_ref(), true)?;
let on_col = on.map(|c| c.column.clone());
Ok(PyDataFrame::new(self.dataframe.lateral_join(
&other.dataframe,
@@ -1748,14 +1830,14 @@ impl PyDataFrame {
// `groupBy().agg`, else Columns, which it asserts.
if let [only] = exprs.as_slice() {
if let Ok(dict) = only.cast::<PyDict>() {
- let mut expressions = Vec::with_capacity(dict.len());
- for (column, function) in dict.iter() {
- let column =
spark_connect::functions::col(&column.extract::<String>()?);
-
expressions.push(spark_connect::expression::Expression::function(
- function.extract::<String>()?,
- vec![column.expression().clone()],
+ // As PySpark's, whose `groupBy().agg()` of none of the
entries asserts that
+ // it has expressions.
+ if dict.is_empty() {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "exprs should not be empty",
));
}
+ let expressions = crate::group::dict_aggregates(&dict)?;
return Ok(PyDataFrame::new(self.dataframe.agg(expressions)));
}
}
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 30c19bd..fefe83c 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -40,6 +40,42 @@ impl PyGroupedData {
}
}
+/// The aggregates of the dict form `agg({column: function})`, one
`function(column)` per entry
+/// in order, as PySpark builds them (`F._invoke_function(function,
F.col(column))`). A column
+/// that is not a str raises the `AttributeError` of `F.col`, and a function
that is not a str
+/// the `AssertionError` of `_invoke_function`.
+pub(crate) fn dict_aggregates(
+ dict: &Bound<'_, pyo3::types::PyDict>,
+) -> PyResult<Vec<spark_connect::expression::Expression>> {
+ let mut agg_exprs = Vec::with_capacity(dict.len());
+ for (column, function) in dict.iter() {
+ let Ok(column) = column.extract::<String>() else {
+ let typ = column
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ return Err(pyo3::exceptions::PyAttributeError::new_err(format!(
+ "'{typ}' object has no attribute 'endswith'"
+ )));
+ };
+ let Ok(function) = function.extract::<String>() else {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+ };
+ // As PySpark's `agg` of a dict, by `col`, of its origin.
+ let _origin = crate::origin::with_origin("col");
+ let c = spark_connect::functions::col(&column);
+ // Build func(col) as an unresolved function resolved server-side, so
aggregate names
+ // (max/min/sum/avg/count/...) all work regardless of whether they are
in the generated
+ // dispatch.
+ agg_exprs.push(spark_connect::expression::Expression::function(
+ function,
+ vec![c.expression().clone()],
+ ));
+ }
+ Ok(agg_exprs)
+}
+
impl PyGroupedData {
/// `sum`, `avg`, `mean`, `min` and `max`, as PySpark's `_numeric_agg`: of
column names,
/// which it asserts, every numeric column without any, and
`NOT_NUMERIC_COLUMNS` for a
@@ -80,22 +116,7 @@ impl PyGroupedData {
// Dict form: {column_name: aggregate_function_name} ->
func(col(name)) per entry.
if exprs.len() == 1 {
if let Ok(dict) = exprs[0].cast::<pyo3::types::PyDict>() {
- let mut agg_exprs = vec![];
- for (k, v) in dict.iter() {
- let col_name: String = k.extract()?;
- let func_name: String = v.extract()?;
- // As PySpark's `agg` of a dict, by `col`, of its origin.
- let _origin = crate::origin::with_origin("col");
- let c = spark_connect::functions::col(&col_name);
- // Build func(col) as an unresolved function resolved
server-side, so
- // aggregate names (max/min/sum/avg/count/...) all work
regardless of
- // whether they are in the generated dispatch.
- let agg_expr =
spark_connect::expression::Expression::function(
- func_name,
- vec![c.expression().clone()],
- );
- agg_exprs.push(agg_expr);
- }
+ let agg_exprs = dict_aggregates(&dict)?;
let df = self.grouped_data.agg(agg_exprs);
return Ok(PyDataFrame::new(df));
}
diff --git a/crates/pyspark-rs/src/streaming.rs
b/crates/pyspark-rs/src/streaming.rs
index 9eb03e2..a7f1659 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -852,6 +852,15 @@ impl PyDataStreamWriter {
}
}
+/// The progress of a query from its JSON, as PySpark's
+/// `StreamingQueryProgress.fromJson(json.loads(progress))`.
+fn progress_from_json<'py>(py: Python<'py>, json: &str) -> PyResult<Bound<'py,
PyAny>> {
+ let loaded = py.import("json")?.call_method1("loads", (json,))?;
+ py.import("pyspark.sql.streaming.listener")?
+ .getattr("StreamingQueryProgress")?
+ .call_method1("fromJson", (loaded,))
+}
+
/// The timeout of `awaitTermination` and `awaitAnyTermination`, in seconds,
as PySpark's take
/// it: an int or a float, which must be positive, else `VALUE_NOT_POSITIVE`
of the name of its
/// type.
@@ -979,14 +988,24 @@ impl PyStreamingQuery {
self.inner.await_termination(timeout).to_pyerr()
}
+ /// The last progress of the query, as PySpark's `StreamingQueryProgress`,
or None.
#[getter]
- fn lastProgress(&self) -> PyResult<Option<String>> {
- self.inner.last_progress().to_pyerr()
+ fn lastProgress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py,
PyAny>>> {
+ match self.inner.last_progress().to_pyerr()? {
+ Some(json) => progress_from_json(py, &json).map(Some),
+ None => Ok(None),
+ }
}
+ /// The recent progress of the query, each as PySpark's
`StreamingQueryProgress`.
#[getter]
- fn recentProgress(&self) -> PyResult<Vec<String>> {
- self.inner.recent_progress().to_pyerr()
+ fn recentProgress<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py,
PyAny>>> {
+ self.inner
+ .recent_progress()
+ .to_pyerr()?
+ .iter()
+ .map(|json| progress_from_json(py, json))
+ .collect()
}
fn processAllAvailable(&self) -> PyResult<()> {
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a75331e..a40ce9c 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -21,6 +21,7 @@ values here are those of the reference client; the tests pass
with it as well.
"""
import os
+import time
from collections import namedtuple
from datetime import date, datetime
@@ -206,6 +207,49 @@ def
test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
assert repr(df) == "DataFrame[id: bigint]"
+def test_join_with_an_unsupported_join_type_raises_as_pyspark(session):
+ # As PySpark: the join type is checked when the join is built, which
raises the
+ # AnalysisException of UNSUPPORTED_JOIN_TYPE, naming the type as it is
normalized.
+ df = session.range(3)
+ with pytest.raises(AnalysisException) as error:
+ df.join(df, "id", "foo").collect()
+ assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+ assert error.value.getMessageParameters()["typ"] == "foo"
+ assert sorted(row.id for row in df.join(df, "id", "left_semi").collect())
== [0, 1, 2]
+
+
+def test_streaming_query_status_and_progress_are_as_pyspark(session):
+ # As PySpark: the status is a dict of message, isDataAvailable and
isTriggerActive, and the
+ # progress is a StreamingQueryProgress, a dict, of the query.
+ from pyspark.sql.streaming.listener import StreamingQueryProgress
+
+ query = (
+ session.readStream.format("rate")
+ .load()
+ .writeStream.format("memory")
+ .queryName("fug_status_query")
+ .start()
+ )
+ try:
+ # Wait, bounded, for the first micro-batch, rather than
processAllAvailable.
+ deadline = time.monotonic() + 60
+ while query.lastProgress is None and time.monotonic() < deadline:
+ time.sleep(0.5)
+ status = query.status
+ assert isinstance(status, dict)
+ assert list(status) == ["message", "isDataAvailable",
"isTriggerActive"]
+ assert isinstance(status["message"], str)
+ assert isinstance(status["isDataAvailable"], bool)
+ assert isinstance(status["isTriggerActive"], bool)
+ last = query.lastProgress
+ assert isinstance(last, StreamingQueryProgress)
+ assert last["name"] == "fug_status_query"
+ recent = query.recentProgress
+ assert recent and all(isinstance(p, StreamingQueryProgress) for p in
recent)
+ finally:
+ query.stop()
+
+
def test_replace_and_unpivot_take_their_arguments_as_pyspark(session):
# As PySpark's: replace sends each value as a literal of its type, and
None as a null, over
# a subset of a column name or a list or tuple of them. unpivot and melt
take a column or a
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 4817877..c3d366b 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9463,6 +9463,117 @@ def
test_positional_indexing_positional_propagates_schema_failure(positional_ind
assert service.calls == 1
+# ---------------------------------------------------------------------------
join types, empty aggregations and the status of a streaming query
+
[email protected](("how", "join_type"), [
+ (None, 1), ("inner", 1), ("INNER", 1), ("in_ner", 1),
+ ("outer", 2), ("full", 2), ("fullouter", 2), ("full_outer", 2),
("FULL_OUTER", 2),
+ ("leftouter", 3), ("left", 3), ("left_outer", 3),
+ ("rightouter", 4), ("right", 4), ("right_outer", 4),
+ ("leftanti", 5), ("left_anti", 5), ("anti", 5),
+ ("leftsemi", 6), ("left_semi", 6), ("semi", 6),
+ ("cross", 7),
+])
+def test_join_types_are_sent_as_pyspark_join_types(mock_connect, how,
join_type):
+ # As PySpark's plan.Join: how is lowercased without its underscores, and
no how is inner.
+ spark, service = mock_connect
+ df = spark.range(2)
+ df.join(df, "id", how).collect()
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.join=5,
Join.join_type=4.
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ assert parse_message(parse_message(plan)[5][0])[4] == [join_type]
+
+
+def
test_join_on_lists_join_on_their_conjunction_or_names_as_pyspark(mock_connect):
+ # As PySpark's plan.Join: a list of conditions joins on their conjunction,
a list of names
+ # is a using join, and an empty list is no condition at all.
+ spark, service = mock_connect
+ left, right = spark.range(2), spark.range(3)
+
+ def sent_join():
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.join=5.
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ return parse_message(parse_message(plan)[5][0])
+
+ left.join(right, [left.id == right.id, right.id > 0]).collect()
+ join = sent_join()
+ # Join.join_condition=3 is the conjunction: an UnresolvedFunction "and"
(Expression=3).
+ assert 5 not in join
+ assert parse_message(parse_message(join[3][0])[3][0])[1] == [b"and"]
+
+ left.join(right, ["id"]).collect()
+ join = sent_join()
+ # Join.using_columns=5.
+ assert 3 not in join and join[5] == [b"id"]
+
+ left.join(right, []).collect()
+ join = sent_join()
+ assert 3 not in join and 5 not in join
+
+
[email protected](("how", "typ"), [
+ ("foo", "foo"), ("", ""), ("left semi", "left semi"), ("LEFT_SEMI_X",
"leftsemix"), (1, "1"),
+])
+def test_join_types_that_are_not_accepted_raise_as_pyspark(offline_spark, how,
typ):
+ # As PySpark's plan.Join: the AnalysisException of UNSUPPORTED_JOIN_TYPE,
raised locally,
+ # which names the type as it is normalized (or its str() when it is not a
str).
+ from pyspark.errors import AnalysisException
+
+ df = offline_spark.range(3)
+ with pytest.raises(AnalysisException) as error:
+ df.join(df, "id", how)
+ assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+ assert error.value.getMessageParameters()["typ"] == typ
+
+
[email protected]("how", ["full", "rightouter", "semi", "outer", "foo"])
+def
test_lateral_join_types_that_are_not_accepted_raise_as_pyspark(offline_spark,
how):
+ # As PySpark's plan.LateralJoin, which accepts inner, leftouter, left and
cross only.
+ from pyspark.errors import AnalysisException
+
+ df = offline_spark.range(3)
+ with pytest.raises(AnalysisException) as error:
+ df.lateralJoin(df, None, how)
+ assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+ assert error.value.getMessageParameters()["typ"] == how
+ assert df.lateralJoin(df, None, "left_outer") is not None
+
+
+def
test_join_on_arguments_that_are_neither_names_nor_conditions_raise_as_pyspark(offline_spark):
+ # As PySpark's plan.Join: an on that is not iterable is a TypeError when
the join is built;
+ # one that is iterable, but not a name, a condition or a list of them,
fails on reading
+ # the to_plan of it.
+ df = offline_spark.range(3)
+ with pytest.raises(TypeError, match="'int' object is not iterable"):
+ df.join(df, 1)
+ with pytest.raises(AttributeError, match="'dict' object has no attribute
'to_plan'"):
+ df.join(df, {"id": 1}).collect()
+ with pytest.raises(AttributeError, match="'tuple' object has no attribute
'to_plan'"):
+ df.join(df, ("id",)).collect()
+
+
+def
test_dict_aggregations_that_are_empty_or_not_str_raise_as_pyspark(offline_spark):
+ # As PySpark's DataFrame.agg: a dict with no entries is the aggregation of
no expressions,
+ # which asserts it has some. GroupedData.agg takes the same dict as no
aggregation at all.
+ # A column that is not a str is the AttributeError of F.col, and a
function that is not a
+ # str the AssertionError of _invoke_function, which has no message.
+ df = offline_spark.range(3)
+ with pytest.raises(AssertionError) as error:
+ df.agg({})
+ assert str(error.value) == "exprs should not be empty"
+ assert df.groupBy().agg({}) is not None
+ for call in (lambda: df.agg({"id": ["max"]}), lambda: df.agg({"id": None}),
+ lambda: df.agg({"x": 1}), lambda: df.groupBy().agg({"id":
["max"]}),
+ lambda: df.groupBy().agg({"id": None})):
+ with pytest.raises(AssertionError) as error:
+ call()
+ assert str(error.value) == ""
+ with pytest.raises(AttributeError, match="'int' object has no attribute
'endswith'"):
+ df.agg({1: "max"})
+ with pytest.raises(AttributeError, match="'NoneType' object has no
attribute 'endswith'"):
+ df.groupBy().agg({None: "max"})
+
+
# ------------------------------------------------- reader/writer in-place
setters
@pytest.fixture
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]