This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 21b0899  [SPARK-61066][CONNECT][PYTHON] Raise the errors of join types 
and dict aggregations, and return the status and progress of a streaming query, 
as PySpark does
21b0899 is described below

commit 21b08990ec95cd585c5ef823ab41d35fa8198248
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 13:14:35 2026 +0900

    [SPARK-61066][CONNECT][PYTHON] Raise the errors of join types and dict 
aggregations, and return the status and progress of a streaming query, as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    - `join` and `lateralJoin` normalize `how` as official's `plan.Join` and 
`plan.LateralJoin` do (lowercased, without underscores, `None` as inner) and 
raise `AnalysisException` `UNSUPPORTED_JOIN_TYPE` for the others, when `join` 
is called, as official does (`crates/pyspark-rs/src/dataframe.rs`). 
`lateralJoin` accepts only inner, left outer and cross, as official. `on` 
follows `plan.Join`: a list of Columns joins on their conjunction, which failed 
before, and the other forms raise offic [...]
    - `DataFrame.agg({})` raises `AssertionError: exprs should not be empty`; 
dict aggregations raise official's errors for a non-str column 
(`AttributeError`) or function (`AssertionError`), in a helper that 
`DataFrame.agg` and `GroupedData.agg` share (`crates/pyspark-rs/src/group.rs`).
    - `StreamingQuery.status` returns official's dict (`message`, 
`isDataAvailable`, `isTriggerActive`), and `lastProgress` and `recentProgress` 
return `StreamingQueryProgress` objects instead of JSON strings 
(`crates/pyspark-rs/src/streaming.rs`).
    
    ### Why are the changes needed?
    
    [SPARK-61066](https://issues.apache.org/jira/browse/SPARK-61066): against a 
local Spark 4.3.0 server:
    
    | Code | Current master | This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `df.join(df, "id", "foo")` | `ValueError: Invalid join type: foo` | 
`AnalysisException [UNSUPPORTED_JOIN_TYPE]` |
    | `df.join(df2, [a.x == b.y, a.x > -1])` | `TypeError` | joins on the 
conjunction |
    | `df.agg({})` | `[Row()]` | `AssertionError: exprs should not be empty` |
    | `df.agg({"id": ["max"]})` | `TypeError` | `AssertionError` |
    | `q.status["isDataAvailable"]` | `TypeError: ... not subscriptable` | a 
bool |
    | `q.lastProgress` | a JSON string | `StreamingQueryProgress` |
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these APIs behave as in PySpark.
    
    ### How was this patch tested?
    
    - New offline tests in `python/tests/test_dropin_offline.py` for the join 
types sent and raised, the forms of `on`, and the dict aggregations, and live 
tests in `python/tests/test_dropin_e2e.py` for an unsupported join type and the 
status and progress of a streaming query.
    - The offline suite (992 passed, 8 skipped) and the live join, aggregation 
and streaming tests.
    
    An `on` of a dict or tuple raises official's `AttributeError` at `join`, 
where official raises it at `collect`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #257 from HyukjinKwon/SPARK-61066-join-agg-status.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 190 ++++++++++++++++++++++++++----------
 crates/pyspark-rs/src/group.rs      |  53 +++++++---
 crates/pyspark-rs/src/streaming.rs  |  27 ++++-
 python/tests/test_dropin_e2e.py     |  44 +++++++++
 python/tests/test_dropin_offline.py | 111 +++++++++++++++++++++
 5 files changed, 351 insertions(+), 74 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 8e7b9bd..77bbe83 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -426,22 +426,71 @@ pub(crate) fn to_column_list(
     Ok(cols)
 }
 
-/// Map a pyspark `how` string to a JoinType (shared by join / lateralJoin).
-fn parse_join_type(how: &str) -> PyResult<JoinType> {
-    Ok(match how.to_lowercase().as_str() {
-        "inner" => JoinType::Inner,
-        "left" | "leftouter" | "left_outer" => JoinType::LeftOuter,
-        "right" | "rightouter" | "right_outer" => JoinType::RightOuter,
-        "outer" | "full" | "fullouter" | "full_outer" => JoinType::FullOuter,
-        "cross" => JoinType::Cross,
-        "left_semi" | "leftsemi" | "semi" => JoinType::LeftSemi,
-        "left_anti" | "leftanti" | "anti" => JoinType::LeftAnti,
-        other => {
-            return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(format!(
-                "Invalid join type: {other}"
-            )))
-        }
-    })
+/// The join types that `DataFrame.join` accepts, as the message of its 
`UNSUPPORTED_JOIN_TYPE`.
+const JOIN_TYPES_SUPPORTED: &str = "'inner', 'outer', 'full', 'fullouter', 
'full_outer', 'leftouter', 'left', 'left_outer', 'rightouter', 'right', 
'right_outer', 'leftsemi', 'left_semi', 'semi', 'leftanti', 'left_anti', 
'anti', 'cross'";
+
+/// The join types that `DataFrame.lateralJoin` accepts, as the message of its
+/// `UNSUPPORTED_JOIN_TYPE`.
+const LATERAL_JOIN_TYPES_SUPPORTED: &str = "'inner', 'leftouter', 'left', 
'left_outer', 'cross'";
+
+/// Map a pyspark `how` to a JoinType, as the connect client's `plan.Join` (or 
`plan.LateralJoin`
+/// when `lateral`) maps it. The `how` is lowercased with its underscores 
removed, and no `how`
+/// is an inner join. One that is not accepted raises the `AnalysisException` 
of
+/// `UNSUPPORTED_JOIN_TYPE` that PySpark raises locally, naming the normalized 
`how` (or the
+/// `str()` of a `how` that is not a str).
+fn parse_join_type(
+    py: Python<'_>,
+    how: Option<&Bound<'_, PyAny>>,
+    lateral: bool,
+) -> PyResult<JoinType> {
+    let Some(how) = how.filter(|how| !how.is_none()) else {
+        return Ok(JoinType::Inner);
+    };
+    let normalized = how
+        .extract::<String>()
+        .ok()
+        .map(|how| how.to_lowercase().replace('_', ""));
+    let join_type = match normalized.as_deref() {
+        Some("inner") => Some(JoinType::Inner),
+        Some("cross") => Some(JoinType::Cross),
+        Some("left" | "leftouter") => Some(JoinType::LeftOuter),
+        Some("right" | "rightouter") if !lateral => Some(JoinType::RightOuter),
+        Some("outer" | "full" | "fullouter") if !lateral => 
Some(JoinType::FullOuter),
+        Some("leftsemi" | "semi") if !lateral => Some(JoinType::LeftSemi),
+        Some("leftanti" | "anti") if !lateral => Some(JoinType::LeftAnti),
+        _ => None,
+    };
+    if let Some(join_type) = join_type {
+        return Ok(join_type);
+    }
+    let typ = match normalized {
+        Some(normalized) => normalized,
+        None => how.str()?.to_string(),
+    };
+    let supported = if lateral {
+        LATERAL_JOIN_TYPES_SUPPORTED
+    } else {
+        JOIN_TYPES_SUPPORTED
+    };
+    Err(crate::column::raise_pyspark(
+        py,
+        "AnalysisException",
+        "UNSUPPORTED_JOIN_TYPE",
+        &[("typ", typ.as_str()), ("supported", supported)],
+    ))
+}
+
+/// The `AttributeError` that PySpark's join raises for an `on` that is 
neither a name, a
+/// condition nor a list of them: the plan reads `to_plan` of it.
+fn no_to_plan(obj: &Bound<'_, PyAny>) -> PyErr {
+    let typ = obj
+        .get_type()
+        .name()
+        .map(|name| name.to_string())
+        .unwrap_or_default();
+    pyo3::exceptions::PyAttributeError::new_err(format!(
+        "'{typ}' object has no attribute 'to_plan'"
+    ))
 }
 
 /// Resolve a filter/where condition: a Column, or a SQL string parsed via 
`expr`.
@@ -693,52 +742,83 @@ impl PyDataFrame {
 
     /// Join with another DataFrame.
     ///
-    /// `on` accepts a Column condition, a column-name string, or a list of
-    /// column-name strings (name-based "using" join), mirroring
-    /// `pyspark.sql.DataFrame.join(other, on=None, how=None)` (default inner).
+    /// `on` is a Column condition, a column name, or a list of column names 
(a "using"
+    /// join) or of Column conditions (joined on their conjunction), as the 
connect client's
+    /// `plan.Join` takes it. Mirrors `pyspark.sql.DataFrame.join(other, 
on=None, how=None)`
+    /// (default inner), including its errors.
     #[pyo3(signature = (other, on=None, how=None))]
     fn join(
         &self,
+        py: Python<'_>,
         other: &Bound<'_, PyAny>,
         on: Option<Bound<'_, PyAny>>,
-        how: Option<&str>,
+        how: Option<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
+        // As PySpark's `_check_same_session`, first.
         let other = self.same_session(other)?;
-        let join_type = parse_join_type(how.unwrap_or("inner"))?;
-
-        // Resolve `on`: None | Column | str | list[str].
-        if let Some(bound) = on {
-            if let Ok(col) = bound.extract::<PyRef<PyColumn>>() {
-                let on_col = Some(col.column.clone());
-                return Ok(PyDataFrame::new(self.dataframe.join(
-                    &other.dataframe,
-                    on_col,
-                    join_type,
-                )));
+        // As PySpark's `plan.Join`, `on` is iterated before `how` is checked, 
so an `on`
+        // that is not iterable is the error that is raised first.
+        if let Some(on) = &on {
+            if !on.is_instance_of::<PyColumn>() && 
!on.is_instance_of::<pyo3::types::PyString>() {
+                on.try_iter()?;
             }
-            if let Ok(name) = bound.extract::<String>() {
-                return Ok(PyDataFrame::new(self.dataframe.join_using(
+        }
+        let join_type = parse_join_type(py, how.as_ref(), false)?;
+        let Some(on) = on else {
+            return Ok(PyDataFrame::new(self.dataframe.join(
+                &other.dataframe,
+                None,
+                join_type,
+            )));
+        };
+        if let Ok(col) = on.extract::<PyRef<PyColumn>>() {
+            return Ok(PyDataFrame::new(self.dataframe.join(
+                &other.dataframe,
+                Some(col.column.clone()),
+                join_type,
+            )));
+        }
+        if let Ok(name) = on.extract::<String>() {
+            return Ok(PyDataFrame::new(self.dataframe.join_using(
+                &other.dataframe,
+                vec![name],
+                join_type,
+            )));
+        }
+        if let Ok(list) = on.cast::<PyList>() {
+            // An empty list is no condition at all.
+            let Some(first) = list.iter().next() else {
+                return Ok(PyDataFrame::new(self.dataframe.join(
                     &other.dataframe,
-                    vec![name],
+                    None,
                     join_type,
                 )));
-            }
-            if let Ok(names) = bound.extract::<Vec<String>>() {
+            };
+            if first.is_instance_of::<pyo3::types::PyString>() {
+                let names = list.extract::<Vec<String>>()?;
                 return Ok(PyDataFrame::new(self.dataframe.join_using(
                     &other.dataframe,
                     names,
                     join_type,
                 )));
             }
-            return Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-                "join 'on' must be a Column, a str, or a list of str",
-            ));
+            let mut condition: Option<spark_connect::column::Column> = None;
+            for item in list.iter() {
+                let Ok(col) = item.extract::<PyRef<PyColumn>>() else {
+                    return Err(no_to_plan(&item));
+                };
+                condition = Some(match condition {
+                    None => col.column.clone(),
+                    Some(left) => left.and(col.column.clone()),
+                });
+            }
+            return Ok(PyDataFrame::new(self.dataframe.join(
+                &other.dataframe,
+                condition,
+                join_type,
+            )));
         }
-        Ok(PyDataFrame::new(self.dataframe.join(
-            &other.dataframe,
-            None,
-            join_type,
-        )))
+        Err(no_to_plan(&on))
     }
 
     /// Cross join.
@@ -749,17 +829,19 @@ impl PyDataFrame {
         ))
     }
 
-    /// Lateral (correlated) join. `on` is an optional Column condition; `how`
-    /// mirrors `join` (default inner). Mirrors `DataFrame.lateralJoin`.
+    /// Lateral (correlated) join. `on` is an optional Column condition; `how` 
is inner by
+    /// default and takes the join types of a lateral join. Mirrors 
`DataFrame.lateralJoin`.
     #[pyo3(signature = (other, on=None, how=None))]
     fn lateralJoin(
         &self,
+        py: Python<'_>,
         other: &Bound<'_, PyAny>,
         on: Option<PyRef<PyColumn>>,
-        how: Option<&str>,
+        how: Option<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
+        // As PySpark's `_check_same_session`, first.
         let other = self.same_session(other)?;
-        let join_type = parse_join_type(how.unwrap_or("inner"))?;
+        let join_type = parse_join_type(py, how.as_ref(), true)?;
         let on_col = on.map(|c| c.column.clone());
         Ok(PyDataFrame::new(self.dataframe.lateral_join(
             &other.dataframe,
@@ -1748,14 +1830,14 @@ impl PyDataFrame {
         // `groupBy().agg`, else Columns, which it asserts.
         if let [only] = exprs.as_slice() {
             if let Ok(dict) = only.cast::<PyDict>() {
-                let mut expressions = Vec::with_capacity(dict.len());
-                for (column, function) in dict.iter() {
-                    let column = 
spark_connect::functions::col(&column.extract::<String>()?);
-                    
expressions.push(spark_connect::expression::Expression::function(
-                        function.extract::<String>()?,
-                        vec![column.expression().clone()],
+                // As PySpark's, whose `groupBy().agg()` of none of the 
entries asserts that
+                // it has expressions.
+                if dict.is_empty() {
+                    return Err(pyo3::exceptions::PyAssertionError::new_err(
+                        "exprs should not be empty",
                     ));
                 }
+                let expressions = crate::group::dict_aggregates(&dict)?;
                 return Ok(PyDataFrame::new(self.dataframe.agg(expressions)));
             }
         }
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 30c19bd..fefe83c 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -40,6 +40,42 @@ impl PyGroupedData {
     }
 }
 
+/// The aggregates of the dict form `agg({column: function})`, one 
`function(column)` per entry
+/// in order, as PySpark builds them (`F._invoke_function(function, 
F.col(column))`). A column
+/// that is not a str raises the `AttributeError` of `F.col`, and a function 
that is not a str
+/// the `AssertionError` of `_invoke_function`.
+pub(crate) fn dict_aggregates(
+    dict: &Bound<'_, pyo3::types::PyDict>,
+) -> PyResult<Vec<spark_connect::expression::Expression>> {
+    let mut agg_exprs = Vec::with_capacity(dict.len());
+    for (column, function) in dict.iter() {
+        let Ok(column) = column.extract::<String>() else {
+            let typ = column
+                .get_type()
+                .name()
+                .map(|name| name.to_string())
+                .unwrap_or_default();
+            return Err(pyo3::exceptions::PyAttributeError::new_err(format!(
+                "'{typ}' object has no attribute 'endswith'"
+            )));
+        };
+        let Ok(function) = function.extract::<String>() else {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+        };
+        // As PySpark's `agg` of a dict, by `col`, of its origin.
+        let _origin = crate::origin::with_origin("col");
+        let c = spark_connect::functions::col(&column);
+        // Build func(col) as an unresolved function resolved server-side, so 
aggregate names
+        // (max/min/sum/avg/count/...) all work regardless of whether they are 
in the generated
+        // dispatch.
+        agg_exprs.push(spark_connect::expression::Expression::function(
+            function,
+            vec![c.expression().clone()],
+        ));
+    }
+    Ok(agg_exprs)
+}
+
 impl PyGroupedData {
     /// `sum`, `avg`, `mean`, `min` and `max`, as PySpark's `_numeric_agg`: of 
column names,
     /// which it asserts, every numeric column without any, and 
`NOT_NUMERIC_COLUMNS` for a
@@ -80,22 +116,7 @@ impl PyGroupedData {
         // Dict form: {column_name: aggregate_function_name} -> 
func(col(name)) per entry.
         if exprs.len() == 1 {
             if let Ok(dict) = exprs[0].cast::<pyo3::types::PyDict>() {
-                let mut agg_exprs = vec![];
-                for (k, v) in dict.iter() {
-                    let col_name: String = k.extract()?;
-                    let func_name: String = v.extract()?;
-                    // As PySpark's `agg` of a dict, by `col`, of its origin.
-                    let _origin = crate::origin::with_origin("col");
-                    let c = spark_connect::functions::col(&col_name);
-                    // Build func(col) as an unresolved function resolved 
server-side, so
-                    // aggregate names (max/min/sum/avg/count/...) all work 
regardless of
-                    // whether they are in the generated dispatch.
-                    let agg_expr = 
spark_connect::expression::Expression::function(
-                        func_name,
-                        vec![c.expression().clone()],
-                    );
-                    agg_exprs.push(agg_expr);
-                }
+                let agg_exprs = dict_aggregates(&dict)?;
                 let df = self.grouped_data.agg(agg_exprs);
                 return Ok(PyDataFrame::new(df));
             }
diff --git a/crates/pyspark-rs/src/streaming.rs 
b/crates/pyspark-rs/src/streaming.rs
index 9eb03e2..a7f1659 100644
--- a/crates/pyspark-rs/src/streaming.rs
+++ b/crates/pyspark-rs/src/streaming.rs
@@ -852,6 +852,15 @@ impl PyDataStreamWriter {
     }
 }
 
+/// The progress of a query from its JSON, as PySpark's
+/// `StreamingQueryProgress.fromJson(json.loads(progress))`.
+fn progress_from_json<'py>(py: Python<'py>, json: &str) -> PyResult<Bound<'py, 
PyAny>> {
+    let loaded = py.import("json")?.call_method1("loads", (json,))?;
+    py.import("pyspark.sql.streaming.listener")?
+        .getattr("StreamingQueryProgress")?
+        .call_method1("fromJson", (loaded,))
+}
+
 /// The timeout of `awaitTermination` and `awaitAnyTermination`, in seconds, 
as PySpark's take
 /// it: an int or a float, which must be positive, else `VALUE_NOT_POSITIVE` 
of the name of its
 /// type.
@@ -979,14 +988,24 @@ impl PyStreamingQuery {
         self.inner.await_termination(timeout).to_pyerr()
     }
 
+    /// The last progress of the query, as PySpark's `StreamingQueryProgress`, 
or None.
     #[getter]
-    fn lastProgress(&self) -> PyResult<Option<String>> {
-        self.inner.last_progress().to_pyerr()
+    fn lastProgress<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, 
PyAny>>> {
+        match self.inner.last_progress().to_pyerr()? {
+            Some(json) => progress_from_json(py, &json).map(Some),
+            None => Ok(None),
+        }
     }
 
+    /// The recent progress of the query, each as PySpark's 
`StreamingQueryProgress`.
     #[getter]
-    fn recentProgress(&self) -> PyResult<Vec<String>> {
-        self.inner.recent_progress().to_pyerr()
+    fn recentProgress<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, 
PyAny>>> {
+        self.inner
+            .recent_progress()
+            .to_pyerr()?
+            .iter()
+            .map(|json| progress_from_json(py, json))
+            .collect()
     }
 
     fn processAllAvailable(&self) -> PyResult<()> {
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index a75331e..a40ce9c 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -21,6 +21,7 @@ values here are those of the reference client; the tests pass 
with it as well.
 """
 
 import os
+import time
 from collections import namedtuple
 from datetime import date, datetime
 
@@ -206,6 +207,49 @@ def 
test_repr_shows_the_rows_with_eager_evaluation_as_pyspark(session):
     assert repr(df) == "DataFrame[id: bigint]"
 
 
+def test_join_with_an_unsupported_join_type_raises_as_pyspark(session):
+    # As PySpark: the join type is checked when the join is built, which 
raises the
+    # AnalysisException of UNSUPPORTED_JOIN_TYPE, naming the type as it is 
normalized.
+    df = session.range(3)
+    with pytest.raises(AnalysisException) as error:
+        df.join(df, "id", "foo").collect()
+    assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+    assert error.value.getMessageParameters()["typ"] == "foo"
+    assert sorted(row.id for row in df.join(df, "id", "left_semi").collect()) 
== [0, 1, 2]
+
+
+def test_streaming_query_status_and_progress_are_as_pyspark(session):
+    # As PySpark: the status is a dict of message, isDataAvailable and 
isTriggerActive, and the
+    # progress is a StreamingQueryProgress, a dict, of the query.
+    from pyspark.sql.streaming.listener import StreamingQueryProgress
+
+    query = (
+        session.readStream.format("rate")
+        .load()
+        .writeStream.format("memory")
+        .queryName("fug_status_query")
+        .start()
+    )
+    try:
+        # Wait, bounded, for the first micro-batch, rather than 
processAllAvailable.
+        deadline = time.monotonic() + 60
+        while query.lastProgress is None and time.monotonic() < deadline:
+            time.sleep(0.5)
+        status = query.status
+        assert isinstance(status, dict)
+        assert list(status) == ["message", "isDataAvailable", 
"isTriggerActive"]
+        assert isinstance(status["message"], str)
+        assert isinstance(status["isDataAvailable"], bool)
+        assert isinstance(status["isTriggerActive"], bool)
+        last = query.lastProgress
+        assert isinstance(last, StreamingQueryProgress)
+        assert last["name"] == "fug_status_query"
+        recent = query.recentProgress
+        assert recent and all(isinstance(p, StreamingQueryProgress) for p in 
recent)
+    finally:
+        query.stop()
+
+
 def test_replace_and_unpivot_take_their_arguments_as_pyspark(session):
     # As PySpark's: replace sends each value as a literal of its type, and 
None as a null, over
     # a subset of a column name or a list or tuple of them. unpivot and melt 
take a column or a
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 4817877..c3d366b 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -9463,6 +9463,117 @@ def 
test_positional_indexing_positional_propagates_schema_failure(positional_ind
     assert service.calls == 1
 
 
+# --------------------------------------------------------------------------- 
join types, empty aggregations and the status of a streaming query
+
[email protected](("how", "join_type"), [
+    (None, 1), ("inner", 1), ("INNER", 1), ("in_ner", 1),
+    ("outer", 2), ("full", 2), ("fullouter", 2), ("full_outer", 2), 
("FULL_OUTER", 2),
+    ("leftouter", 3), ("left", 3), ("left_outer", 3),
+    ("rightouter", 4), ("right", 4), ("right_outer", 4),
+    ("leftanti", 5), ("left_anti", 5), ("anti", 5),
+    ("leftsemi", 6), ("left_semi", 6), ("semi", 6),
+    ("cross", 7),
+])
+def test_join_types_are_sent_as_pyspark_join_types(mock_connect, how, 
join_type):
+    # As PySpark's plan.Join: how is lowercased without its underscores, and 
no how is inner.
+    spark, service = mock_connect
+    df = spark.range(2)
+    df.join(df, "id", how).collect()
+    # ExecutePlanRequest.plan=3, Plan.root=1, Relation.join=5, 
Join.join_type=4.
+    plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+    assert parse_message(parse_message(plan)[5][0])[4] == [join_type]
+
+
+def 
test_join_on_lists_join_on_their_conjunction_or_names_as_pyspark(mock_connect):
+    # As PySpark's plan.Join: a list of conditions joins on their conjunction, 
a list of names
+    # is a using join, and an empty list is no condition at all.
+    spark, service = mock_connect
+    left, right = spark.range(2), spark.range(3)
+
+    def sent_join():
+        # ExecutePlanRequest.plan=3, Plan.root=1, Relation.join=5.
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        return parse_message(parse_message(plan)[5][0])
+
+    left.join(right, [left.id == right.id, right.id > 0]).collect()
+    join = sent_join()
+    # Join.join_condition=3 is the conjunction: an UnresolvedFunction "and" 
(Expression=3).
+    assert 5 not in join
+    assert parse_message(parse_message(join[3][0])[3][0])[1] == [b"and"]
+
+    left.join(right, ["id"]).collect()
+    join = sent_join()
+    # Join.using_columns=5.
+    assert 3 not in join and join[5] == [b"id"]
+
+    left.join(right, []).collect()
+    join = sent_join()
+    assert 3 not in join and 5 not in join
+
+
[email protected](("how", "typ"), [
+    ("foo", "foo"), ("", ""), ("left semi", "left semi"), ("LEFT_SEMI_X", 
"leftsemix"), (1, "1"),
+])
+def test_join_types_that_are_not_accepted_raise_as_pyspark(offline_spark, how, 
typ):
+    # As PySpark's plan.Join: the AnalysisException of UNSUPPORTED_JOIN_TYPE, 
raised locally,
+    # which names the type as it is normalized (or its str() when it is not a 
str).
+    from pyspark.errors import AnalysisException
+
+    df = offline_spark.range(3)
+    with pytest.raises(AnalysisException) as error:
+        df.join(df, "id", how)
+    assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+    assert error.value.getMessageParameters()["typ"] == typ
+
+
[email protected]("how", ["full", "rightouter", "semi", "outer", "foo"])
+def 
test_lateral_join_types_that_are_not_accepted_raise_as_pyspark(offline_spark, 
how):
+    # As PySpark's plan.LateralJoin, which accepts inner, leftouter, left and 
cross only.
+    from pyspark.errors import AnalysisException
+
+    df = offline_spark.range(3)
+    with pytest.raises(AnalysisException) as error:
+        df.lateralJoin(df, None, how)
+    assert error.value.getCondition() == "UNSUPPORTED_JOIN_TYPE"
+    assert error.value.getMessageParameters()["typ"] == how
+    assert df.lateralJoin(df, None, "left_outer") is not None
+
+
+def 
test_join_on_arguments_that_are_neither_names_nor_conditions_raise_as_pyspark(offline_spark):
+    # As PySpark's plan.Join: an on that is not iterable is a TypeError when 
the join is built;
+    # one that is iterable, but not a name, a condition or a list of them, 
fails on reading
+    # the to_plan of it.
+    df = offline_spark.range(3)
+    with pytest.raises(TypeError, match="'int' object is not iterable"):
+        df.join(df, 1)
+    with pytest.raises(AttributeError, match="'dict' object has no attribute 
'to_plan'"):
+        df.join(df, {"id": 1}).collect()
+    with pytest.raises(AttributeError, match="'tuple' object has no attribute 
'to_plan'"):
+        df.join(df, ("id",)).collect()
+
+
+def 
test_dict_aggregations_that_are_empty_or_not_str_raise_as_pyspark(offline_spark):
+    # As PySpark's DataFrame.agg: a dict with no entries is the aggregation of 
no expressions,
+    # which asserts it has some. GroupedData.agg takes the same dict as no 
aggregation at all.
+    # A column that is not a str is the AttributeError of F.col, and a 
function that is not a
+    # str the AssertionError of _invoke_function, which has no message.
+    df = offline_spark.range(3)
+    with pytest.raises(AssertionError) as error:
+        df.agg({})
+    assert str(error.value) == "exprs should not be empty"
+    assert df.groupBy().agg({}) is not None
+    for call in (lambda: df.agg({"id": ["max"]}), lambda: df.agg({"id": None}),
+                 lambda: df.agg({"x": 1}), lambda: df.groupBy().agg({"id": 
["max"]}),
+                 lambda: df.groupBy().agg({"id": None})):
+        with pytest.raises(AssertionError) as error:
+            call()
+        assert str(error.value) == ""
+    with pytest.raises(AttributeError, match="'int' object has no attribute 
'endswith'"):
+        df.agg({1: "max"})
+    with pytest.raises(AttributeError, match="'NoneType' object has no 
attribute 'endswith'"):
+        df.groupBy().agg({None: "max"})
+
+
 # ------------------------------------------------- reader/writer in-place 
setters
 
 @pytest.fixture


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to