xinrong-databricks commented on a change in pull request #32611:
URL: https://github.com/apache/spark/pull/32611#discussion_r638166139
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py
##########
@@ -33,107 +35,148 @@ def pser(self):
def psser(self):
return ps.from_pandas(self.pser)
+ @property
+ def float_pser(self):
+ return pd.Series([1, 2, 3], dtype=float)
+
+ @property
+ def float_psser(self):
+ return ps.from_pandas(self.float_pser)
+
def test_add(self):
- self.assertRaises(TypeError, lambda: self.psser + 1)
- self.assertRaises(TypeError, lambda: self.psser + 0.1)
+ self.assert_eq(self.pser + 1, self.psser + 1)
+ self.assert_eq(self.pser + 0.1, self.psser + 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ for pser, psser in self.numeric_pser_psser_pairs:
+ self.assert_eq(self.pser + pser, (self.psser +
psser).sort_index())
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser + psser)
def test_sub(self):
- self.assertRaises(TypeError, lambda: self.psser - 1)
- self.assertRaises(TypeError, lambda: self.psser - 0.1)
+ self.assert_eq(self.pser - 1, self.psser - 1)
+ self.assert_eq(self.pser - 0.1, self.psser - 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ for pser, psser in self.numeric_pser_psser_pairs:
+ self.assert_eq(self.pser - pser, (self.psser -
psser).sort_index())
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser - psser)
def test_mul(self):
- self.assertRaises(TypeError, lambda: self.psser * 1)
- self.assertRaises(TypeError, lambda: self.psser * 0.1)
+ self.assert_eq(self.pser * 1, self.psser * 1)
+ self.assert_eq(self.pser * 0.1, self.psser * 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ for pser, psser in self.numeric_pser_psser_pairs:
+ self.assert_eq(self.pser * pser, (self.psser *
psser).sort_index())
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser * psser)
def test_truediv(self):
- self.assertRaises(TypeError, lambda: self.psser / 1)
- self.assertRaises(TypeError, lambda: self.psser / 0.1)
+ self.assert_eq(self.pser / 1, self.psser / 1)
+ self.assert_eq(self.pser / 0.1, self.psser / 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ self.assert_eq(
+ self.pser / self.float_pser, (self.psser /
self.float_psser).sort_index())
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser / psser)
def test_floordiv(self):
- self.assertRaises(TypeError, lambda: self.psser // 1)
- self.assertRaises(TypeError, lambda: self.psser // 0.1)
+ # float is always returned in pandas-on-Spark
+ self.assert_eq((self.pser // 1).astype("float"), self.psser // 1)
+ # in pandas, 1 // 0.1 = 9.0; in pandas-on-Spark, 1 // 0.1 = 10.0
+ # self.assert_eq(self.pser // 0.1, self.psser // 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ self.assert_eq(
+ self.pser // self.float_pser, (self.psser //
self.float_psser).sort_index()
+ )
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser // psser)
def test_mod(self):
- self.assertRaises(TypeError, lambda: self.psser % 1)
- self.assertRaises(TypeError, lambda: self.psser % 0.1)
+ self.assert_eq(self.pser % 1, self.psser % 1)
+ self.assert_eq(self.pser % 0.1, self.psser % 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ for pser, psser in self.numeric_pser_psser_pairs:
+ self.assert_eq(self.pser % pser, (self.psser %
psser).sort_index())
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser % psser)
def test_pow(self):
- self.assertRaises(TypeError, lambda: self.psser ** 1)
- self.assertRaises(TypeError, lambda: self.psser ** 0.1)
+ # float is always returned in pandas-on-Spark
+ self.assert_eq((self.pser ** 1).astype("float"), self.psser ** 1)
+ self.assert_eq(self.pser ** 0.1, self.psser ** 0.1)
with option_context("compute.ops_on_diff_frames", True):
- for psser in self.pssers:
+ self.assert_eq(
+ self.pser ** self.float_pser, (self.psser **
self.float_psser).sort_index()
+ )
+
+ for psser in self.non_numeric_pssers.values():
self.assertRaises(TypeError, lambda: self.psser ** psser)
def test_radd(self):
- self.assertRaises(TypeError, lambda: 1 + self.psser)
- self.assertRaises(TypeError, lambda: 0.1 + self.psser)
+ self.assert_eq(1 + self.pser, 1 + self.psser)
+ self.assert_eq(0.1 + self.pser, 0.1 + self.psser)
self.assertRaises(TypeError, lambda: "x" + self.psser)
self.assertRaises(TypeError, lambda: datetime.date(1994, 1, 1) +
self.psser)
self.assertRaises(TypeError, lambda: datetime.datetime(1994, 1, 1) +
self.psser)
def test_rsub(self):
- self.assertRaises(TypeError, lambda: 1 - self.psser)
- self.assertRaises(TypeError, lambda: 0.1 - self.psser)
+ self.assert_eq(1 - self.pser, 1 - self.psser)
+ self.assert_eq(0.1 - self.pser, 0.1 - self.psser)
self.assertRaises(TypeError, lambda: "x" - self.psser)
self.assertRaises(TypeError, lambda: datetime.date(1994, 1, 1) -
self.psser)
self.assertRaises(TypeError, lambda: datetime.datetime(1994, 1, 1) -
self.psser)
def test_rmul(self):
- self.assertRaises(TypeError, lambda: 1 * self.psser)
- self.assertRaises(TypeError, lambda: 0.1 * self.psser)
+ self.assert_eq(1 * self.pser, 1 * self.psser)
+ self.assert_eq(0.1 * self.pser, 0.1 * self.psser)
self.assertRaises(TypeError, lambda: "x" * self.psser)
self.assertRaises(TypeError, lambda: datetime.date(1994, 1, 1) *
self.psser)
self.assertRaises(TypeError, lambda: datetime.datetime(1994, 1, 1) *
self.psser)
def test_rtruediv(self):
- self.assertRaises(TypeError, lambda: 1 / self.psser)
- self.assertRaises(TypeError, lambda: 0.1 / self.psser)
+ self.assert_eq(1 / self.pser, 1 / self.psser)
+ self.assert_eq(0.1 / self.pser, 0.1 / self.psser)
self.assertRaises(TypeError, lambda: "x" / self.psser)
self.assertRaises(TypeError, lambda: datetime.date(1994, 1, 1) /
self.psser)
self.assertRaises(TypeError, lambda: datetime.datetime(1994, 1, 1) /
self.psser)
def test_rfloordiv(self):
- self.assertRaises(TypeError, lambda: 1 // self.psser)
- self.assertRaises(TypeError, lambda: 0.1 // self.psser)
+ if LooseVersion(pd.__version__) >= LooseVersion("0.25.3"):
+ self.assert_eq(1 // self.pser, 1 // self.psser)
+ self.assert_eq(0.1 // self.pser, 0.1 // self.psser)
Review comment:
`else` case is added.
##########
File path: python/pyspark/pandas/tests/data_type_ops/test_num_ops.py
##########
@@ -101,7 +133,13 @@ def test_floordiv(self):
self.assertRaises(TypeError, lambda: psser //
self.non_numeric_pssers["date"])
self.assertRaises(
TypeError, lambda: psser //
self.non_numeric_pssers["categorical"])
- self.assertRaises(TypeError, lambda: psser //
self.non_numeric_pssers["bool"])
+ if LooseVersion(pd.__version__) >= LooseVersion("0.25.3"):
+ self.assert_eq(
+ (self.float_psser //
self.non_numeric_pssers["bool"]).sort_index(),
+ self.float_pser // self.non_numeric_psers["bool"],
+ )
Review comment:
`else` case is added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
