This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 90f6bab05780 [SPARK-57368][PYTHON][ML][TEST] Fix assertTrue misuse in
PySpark tests
90f6bab05780 is described below
commit 90f6bab057800d99b9ea4faf09a0025427442e42
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Jun 11 20:48:24 2026 +0800
[SPARK-57368][PYTHON][ML][TEST] Fix assertTrue misuse in PySpark tests
### What changes were proposed in this pull request?
Replace `self.assertTrue(a, b)` with `self.assertEqual(a, b)` in cases
where the second argument is an expected value, not a failure message.
`unittest.TestCase.assertTrue(expr, msg=None)` treats the second argument
as an optional failure message, so `assertTrue(a, b)` only checks that `a` is
truthy — it never compares `a` against `b`. These tests were silently passing
regardless of the actual value.
Fixed in three files:
- `sql/tests/test_functions.py`: `assertTrue(row[1], 1)` /
`assertTrue(row[2], 1)` — crosstab result checks
- `ml/tests/test_linalg.py`: repr/str checks for dense and sparse matrices,
norm value checks, UDT schema and matrix equality checks
- `mllib/tests/test_linalg.py`: same patterns (mirrors the `ml` file)
### Why are the changes needed?
These are silent test bugs: the assertions always pass as long as the first
argument is truthy, which it always is (non-empty strings, non-zero numbers,
matrix objects). The expected values in the second argument were never
verified. `assertEqual` makes the tests actually check what they intended.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Test-only change. The assertions being fixed already passed (because they
only checked truthiness), and after the fix they continue to pass because the
actual values match the expected values.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (claude-sonnet-4-6)
Closes #56428 from zhengruifeng/fix-assertTrue-dev2.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/ml/tests/test_linalg.py | 54 +++++++++++++++---------------
python/pyspark/mllib/tests/test_linalg.py | 54 +++++++++++++++---------------
python/pyspark/sql/tests/test_functions.py | 4 +--
3 files changed, 56 insertions(+), 56 deletions(-)
diff --git a/python/pyspark/ml/tests/test_linalg.py
b/python/pyspark/ml/tests/test_linalg.py
index 21e588298b77..4d5752cecd67 100644
--- a/python/pyspark/ml/tests/test_linalg.py
+++ b/python/pyspark/ml/tests/test_linalg.py
@@ -193,23 +193,23 @@ class VectorTests(MLlibTestCase):
def test_repr_dense_matrix(self):
mat = DenseMatrix(3, 2, [0, 1, 4, 6, 8, 10])
- self.assertTrue(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
+ self.assertEqual(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
mat = DenseMatrix(3, 2, [0, 1, 4, 6, 8, 10], True)
- self.assertTrue(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
+ self.assertEqual(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], True)")
mat = DenseMatrix(6, 3, zeros(18))
- self.assertTrue(
+ self.assertEqual(
repr(mat),
- "DenseMatrix(6, 3, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ..., \
- 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], False)",
+ "DenseMatrix(6, 3, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ..., "
+ "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], False)",
)
def test_repr_sparse_matrix(self):
sm1t = SparseMatrix(
3, 4, [0, 2, 3, 5], [0, 1, 2, 0, 2], [3.0, 2.0, 4.0, 9.0, 8.0],
isTransposed=True
)
- self.assertTrue(
+ self.assertEqual(
repr(sm1t),
"SparseMatrix(3, 4, [0, 2, 3, 5], [0, 1, 2, 0, 2], [3.0, 2.0, 4.0,
9.0, 8.0], True)",
)
@@ -217,27 +217,27 @@ class VectorTests(MLlibTestCase):
indices = tile(arange(6), 3)
values = ones(18)
sm = SparseMatrix(6, 3, [0, 6, 12, 18], indices, values)
- self.assertTrue(
+ self.assertEqual(
repr(sm),
- "SparseMatrix(6, 3, [0, 6, 12, 18], \
- [0, 1, 2, 3, 4, 5, 0, 1, ..., 4, 5, 0, 1, 2, 3, 4, 5], \
- [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ..., \
- 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], False)",
+ "SparseMatrix(6, 3, [0, 6, 12, 18], "
+ "[0, 1, 2, 3, 4, 5, 0, 1, ..., 4, 5, 0, 1, 2, 3, 4, 5], "
+ "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ..., "
+ "1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], False)",
)
- self.assertTrue(
+ self.assertEqual(
str(sm),
- "6 X 3 CSCMatrix\n\
- (0,0) 1.0\n(1,0) 1.0\n(2,0) 1.0\n(3,0) 1.0\n(4,0) 1.0\n(5,0) 1.0\n\
- (0,1) 1.0\n(1,1) 1.0\n(2,1) 1.0\n(3,1) 1.0\n(4,1) 1.0\n(5,1) 1.0\n\
- (0,2) 1.0\n(1,2) 1.0\n(2,2) 1.0\n(3,2) 1.0\n..\n..",
+ "6 X 3 CSCMatrix\n"
+ "(0,0) 1.0\n(1,0) 1.0\n(2,0) 1.0\n(3,0) 1.0\n(4,0) 1.0\n(5,0)
1.0\n"
+ "(0,1) 1.0\n(1,1) 1.0\n(2,1) 1.0\n(3,1) 1.0\n(4,1) 1.0\n(5,1)
1.0\n"
+ "(0,2) 1.0\n(1,2) 1.0\n(2,2) 1.0\n(3,2) 1.0\n..\n..",
)
sm = SparseMatrix(1, 18, zeros(19), [], [])
- self.assertTrue(
+ self.assertEqual(
repr(sm),
- "SparseMatrix(1, 18, \
- [0, 0, 0, 0, 0, 0, 0, 0, ..., 0, 0, 0, 0, 0, 0, 0, 0], [], [],
False)",
+ "SparseMatrix(1, 18, "
+ "[0, 0, 0, 0, 0, 0, 0, 0, ..., 0, 0, 0, 0, 0, 0, 0, 0], [], [],
False)",
)
def test_sparse_matrix(self):
@@ -248,7 +248,7 @@ class VectorTests(MLlibTestCase):
self.assertEqual(sm1.colPtrs.tolist(), [0, 2, 2, 4, 4])
self.assertEqual(sm1.rowIndices.tolist(), [1, 2, 1, 2])
self.assertEqual(sm1.values.tolist(), [1.0, 2.0, 4.0, 5.0])
- self.assertTrue(
+ self.assertEqual(
repr(sm1),
"SparseMatrix(3, 4, [0, 2, 2, 4, 4], [1, 2, 1, 2], [1.0, 2.0, 4.0,
5.0], False)",
)
@@ -307,12 +307,12 @@ class VectorTests(MLlibTestCase):
def test_norms(self):
a = DenseVector([0, 2, 3, -1])
self.assertAlmostEqual(a.norm(2), 3.742, 3)
- self.assertTrue(a.norm(1), 6)
- self.assertTrue(a.norm(inf), 3)
+ self.assertEqual(a.norm(1), 6)
+ self.assertEqual(a.norm(inf), 3)
a = SparseVector(4, [0, 2], [3, -4])
self.assertAlmostEqual(a.norm(2), 5)
- self.assertTrue(a.norm(1), 7)
- self.assertTrue(a.norm(inf), 4)
+ self.assertEqual(a.norm(1), 7)
+ self.assertEqual(a.norm(inf), 4)
tmp = SparseVector(4, [0, 2], [3, 0])
self.assertEqual(tmp.numNonzeros(), 1)
@@ -385,14 +385,14 @@ class MatrixUDTTests(MLlibTestCase):
rdd = self.sc.parallelize([("dense", self.dm1), ("sparse", self.sm1)])
df = rdd.toDF()
schema = df.schema
- self.assertTrue(schema.fields[1].dataType, self.udt)
+ self.assertEqual(schema.fields[1].dataType, self.udt)
matrices = df.rdd.map(lambda x: x._2).collect()
self.assertEqual(len(matrices), 2)
for m in matrices:
if isinstance(m, DenseMatrix):
- self.assertTrue(m, self.dm1)
+ self.assertEqual(m, self.dm1)
elif isinstance(m, SparseMatrix):
- self.assertTrue(m, self.sm1)
+ self.assertEqual(m, self.sm1)
else:
raise ValueError("Expected a matrix but got type %r" % type(m))
diff --git a/python/pyspark/mllib/tests/test_linalg.py
b/python/pyspark/mllib/tests/test_linalg.py
index d2cc27464f13..36a1e30c0910 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -199,23 +199,23 @@ class VectorTests(MLlibTestCase):
def test_repr_dense_matrix(self):
mat = DenseMatrix(3, 2, [0, 1, 4, 6, 8, 10])
- self.assertTrue(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
+ self.assertEqual(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
mat = DenseMatrix(3, 2, [0, 1, 4, 6, 8, 10], True)
- self.assertTrue(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], False)")
+ self.assertEqual(repr(mat), "DenseMatrix(3, 2, [0.0, 1.0, 4.0, 6.0,
8.0, 10.0], True)")
mat = DenseMatrix(6, 3, zeros(18))
- self.assertTrue(
+ self.assertEqual(
repr(mat),
- "DenseMatrix(6, 3, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ..., \
- 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], False)",
+ "DenseMatrix(6, 3, [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ..., "
+ "0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], False)",
)
def test_repr_sparse_matrix(self):
sm1t = SparseMatrix(
3, 4, [0, 2, 3, 5], [0, 1, 2, 0, 2], [3.0, 2.0, 4.0, 9.0, 8.0],
isTransposed=True
)
- self.assertTrue(
+ self.assertEqual(
repr(sm1t),
"SparseMatrix(3, 4, [0, 2, 3, 5], [0, 1, 2, 0, 2], [3.0, 2.0, 4.0,
9.0, 8.0], True)",
)
@@ -223,27 +223,27 @@ class VectorTests(MLlibTestCase):
indices = tile(arange(6), 3)
values = ones(18)
sm = SparseMatrix(6, 3, [0, 6, 12, 18], indices, values)
- self.assertTrue(
+ self.assertEqual(
repr(sm),
- "SparseMatrix(6, 3, [0, 6, 12, 18], \
- [0, 1, 2, 3, 4, 5, 0, 1, ..., 4, 5, 0, 1, 2, 3, 4, 5], \
- [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ..., \
- 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], False)",
+ "SparseMatrix(6, 3, [0, 6, 12, 18], "
+ "[0, 1, 2, 3, 4, 5, 0, 1, ..., 4, 5, 0, 1, 2, 3, 4, 5], "
+ "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, ..., "
+ "1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], False)",
)
- self.assertTrue(
+ self.assertEqual(
str(sm),
- "6 X 3 CSCMatrix\n\
- (0,0) 1.0\n(1,0) 1.0\n(2,0) 1.0\n(3,0) 1.0\n(4,0) 1.0\n(5,0) 1.0\n\
- (0,1) 1.0\n(1,1) 1.0\n(2,1) 1.0\n(3,1) 1.0\n(4,1) 1.0\n(5,1) 1.0\n\
- (0,2) 1.0\n(1,2) 1.0\n(2,2) 1.0\n(3,2) 1.0\n..\n..",
+ "6 X 3 CSCMatrix\n"
+ "(0,0) 1.0\n(1,0) 1.0\n(2,0) 1.0\n(3,0) 1.0\n(4,0) 1.0\n(5,0)
1.0\n"
+ "(0,1) 1.0\n(1,1) 1.0\n(2,1) 1.0\n(3,1) 1.0\n(4,1) 1.0\n(5,1)
1.0\n"
+ "(0,2) 1.0\n(1,2) 1.0\n(2,2) 1.0\n(3,2) 1.0\n..\n..",
)
sm = SparseMatrix(1, 18, zeros(19), [], [])
- self.assertTrue(
+ self.assertEqual(
repr(sm),
- "SparseMatrix(1, 18, \
- [0, 0, 0, 0, 0, 0, 0, 0, ..., 0, 0, 0, 0, 0, 0, 0, 0], [], [],
False)",
+ "SparseMatrix(1, 18, "
+ "[0, 0, 0, 0, 0, 0, 0, 0, ..., 0, 0, 0, 0, 0, 0, 0, 0], [], [],
False)",
)
def test_sparse_matrix(self):
@@ -254,7 +254,7 @@ class VectorTests(MLlibTestCase):
self.assertEqual(sm1.colPtrs.tolist(), [0, 2, 2, 4, 4])
self.assertEqual(sm1.rowIndices.tolist(), [1, 2, 1, 2])
self.assertEqual(sm1.values.tolist(), [1.0, 2.0, 4.0, 5.0])
- self.assertTrue(
+ self.assertEqual(
repr(sm1),
"SparseMatrix(3, 4, [0, 2, 2, 4, 4], [1, 2, 1, 2], [1.0, 2.0, 4.0,
5.0], False)",
)
@@ -329,12 +329,12 @@ class VectorTests(MLlibTestCase):
def test_norms(self):
a = DenseVector([0, 2, 3, -1])
self.assertAlmostEqual(a.norm(2), 3.742, 3)
- self.assertTrue(a.norm(1), 6)
- self.assertTrue(a.norm(inf), 3)
+ self.assertEqual(a.norm(1), 6)
+ self.assertEqual(a.norm(inf), 3)
a = SparseVector(4, [0, 2], [3, -4])
self.assertAlmostEqual(a.norm(2), 5)
- self.assertTrue(a.norm(1), 7)
- self.assertTrue(a.norm(inf), 4)
+ self.assertEqual(a.norm(1), 7)
+ self.assertEqual(a.norm(inf), 4)
tmp = SparseVector(4, [0, 2], [3, 0])
self.assertEqual(tmp.numNonzeros(), 1)
@@ -487,14 +487,14 @@ class MatrixUDTTests(MLlibTestCase):
rdd = self.sc.parallelize([("dense", self.dm1), ("sparse", self.sm1)])
df = rdd.toDF()
schema = df.schema
- self.assertTrue(schema.fields[1].dataType, self.udt)
+ self.assertEqual(schema.fields[1].dataType, self.udt)
matrices = df.rdd.map(lambda x: x._2).collect()
self.assertEqual(len(matrices), 2)
for m in matrices:
if isinstance(m, DenseMatrix):
- self.assertTrue(m, self.dm1)
+ self.assertEqual(m, self.dm1)
elif isinstance(m, SparseMatrix):
- self.assertTrue(m, self.sm1)
+ self.assertEqual(m, self.sm1)
else:
raise ValueError("Expected a matrix but got type %r" % type(m))
diff --git a/python/pyspark/sql/tests/test_functions.py
b/python/pyspark/sql/tests/test_functions.py
index a7163326f2be..4edb467d2c2b 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -301,8 +301,8 @@ class FunctionsTestsMixin:
ct = sorted(ct, key=lambda x: x[0])
for i, row in enumerate(ct):
self.assertEqual(row[0], str(i))
- self.assertTrue(row[1], 1)
- self.assertTrue(row[2], 1)
+ self.assertEqual(row[1], 1)
+ self.assertEqual(row[2], 1)
def test_math_functions(self):
df = self.spark.createDataFrame([Row(a=i, b=2 * i) for i in range(10)])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
