HyukjinKwon commented on a change in pull request #34114:
URL: https://github.com/apache/spark/pull/34114#discussion_r718157230
##########
File path: python/pyspark/pandas/base.py
##########
@@ -394,7 +394,17 @@ def __abs__(self: IndexOpsLike) -> IndexOpsLike:
# comparison operators
def __eq__(self, other: Any) -> SeriesOrIndex: # type: ignore[override]
- return self._dtype_op.eq(self, other)
+ if isinstance(other, (list, tuple)):
+ if len(self) != len(other):
+ raise ValueError("Lengths must be equal")
+ name = self._internal.spark_column_name_for(self.spark.column)
+ other = ps.Series(other, name=name)
Review comment:
Can we do this via `transform(array_sort(collect_list( ... index and
column ...)), lambda c: c.getItem[0]) == functions.lit(other)`? `other` is in
memory so the size of the list is expected to be small. I think this is fine to
collect all values of the column into single value from `collect_list`.
##########
File path: python/pyspark/pandas/tests/test_ops_on_diff_frames.py
##########
@@ -1830,6 +1830,47 @@ def _test_cov(self, pser1, pser2):
pscov = psser1.cov(psser2, min_periods=3)
self.assert_eq(pcov, pscov, almost=True)
+ def test_series_eq(self):
+ pser = pd.Series([1, 2, 3, 4, 5, 6], name="x")
+ psser = ps.from_pandas(pser)
+
+ # other = Series
+ pandas_other = pd.Series([np.nan, 1, 3, 4, np.nan, 6], name="x")
+ pandas_on_spark_other = ps.from_pandas(pandas_other)
+ self.assert_eq(pser.eq(pandas_other),
psser.eq(pandas_on_spark_other).sort_index())
+ self.assert_eq(pser == pandas_other, (psser ==
pandas_on_spark_other).sort_index())
+
+ # other = Series with different Index
+ pandas_other = pd.Series(
+ [np.nan, 1, 3, 4, np.nan, 6], index=[10, 20, 30, 40, 50, 60],
name="x"
+ )
+ pandas_on_spark_other = ps.from_pandas(pandas_other)
+ self.assert_eq(pser.eq(pandas_other),
psser.eq(pandas_on_spark_other).sort_index())
+
+ # other = Index
+ pandas_other = pd.Index([np.nan, 1, 3, 4, np.nan, 6], name="x")
+ pandas_on_spark_other = ps.from_pandas(pandas_other)
+ self.assert_eq(pser.eq(pandas_other),
psser.eq(pandas_on_spark_other).sort_index())
+ self.assert_eq(pser == pandas_other, (psser ==
pandas_on_spark_other).sort_index())
+
+ # other = list
+ other = [np.nan, 1, 3, 4, np.nan, 6]
Review comment:
How does this work without setting `compute.ops_on_diff_frames`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
