This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0447c1aa45 [VL] Include the first data column in the row-based shuffle
checksum (#13553)
0447c1aa45 is described below
commit 0447c1aa45f3a5c96cf921490bbea8d1f1dd1bfe
Author: YangJie <[email protected]>
AuthorDate: Wed Nov 11 04:22:01 2026 -0500
[VL] Include the first data column in the row-based shuffle checksum
(#13553)
---
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 20 ++++---------
.../sql/gluten/GlutenRowBasedChecksumSuite.scala | 34 ++++++++++++++++++++++
.../sql/gluten/GlutenRowBasedChecksumSuite.scala | 34 ++++++++++++++++++++++
3 files changed, 73 insertions(+), 15 deletions(-)
diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index 242683450c..877b63995e 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -1655,21 +1655,11 @@ void
VeloxHashShuffleWriter::computeRowBasedChecksums(const facebook::velox::Row
auto numRows = rv.size();
VELOX_DCHECK(rv.nulls() == nullptr, "RowVector with top-level nulls not
supported for checksum");
- // Get the RowVector to serialize (strip pid column if present).
- facebook::velox::RowVectorPtr dataVector;
- if (partitioner_->hasPid()) {
- // Strip the first column (partition id).
- auto rowType = std::dynamic_pointer_cast<const
facebook::velox::RowType>(rv.type());
- std::vector<std::string> names(rowType->names().begin() + 1,
rowType->names().end());
- std::vector<facebook::velox::TypePtr> types(rowType->children().begin() +
1, rowType->children().end());
- std::vector<facebook::velox::VectorPtr> children(rv.children().begin() +
1, rv.children().end());
- auto dataType = facebook::velox::ROW(std::move(names), std::move(types));
- dataVector =
- std::make_shared<facebook::velox::RowVector>(rv.pool(), dataType,
nullptr, numRows, std::move(children));
- } else {
- auto rowType = std::dynamic_pointer_cast<const
facebook::velox::RowType>(rv.type());
- dataVector = std::make_shared<facebook::velox::RowVector>(rv.pool(),
rowType, nullptr, numRows, rv.children());
- }
+ // Every doSplit caller has already removed the partition id column, so rv
holds only data
+ // columns.
+ auto rowType = std::dynamic_pointer_cast<const
facebook::velox::RowType>(rv.type());
+ facebook::velox::RowVectorPtr dataVector =
+ std::make_shared<facebook::velox::RowVector>(rv.pool(), rowType,
nullptr, numRows, rv.children());
facebook::velox::row::UnsafeRowFast fast(dataVector);
auto dataType = std::dynamic_pointer_cast<const
facebook::velox::RowType>(dataVector->type());
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
index 8ee91ca133..5b3c838633 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
@@ -88,4 +88,38 @@ class GlutenRowBasedChecksumSuite extends
GlutenSQLTestsTrait {
}
}
}
+
+ test("Gluten row-based checksum covers the first data column") {
+ withSQLConf(
+ SQLConf.SHUFFLE_ORDER_INDEPENDENT_CHECKSUM_ENABLED.key -> "true",
+ SQLConf.CLASSIC_SHUFFLE_DEPENDENCY_FILE_CLEANUP_ENABLED.key -> "false") {
+ withTable("t_first1", "t_first2") {
+ // Only the first column differs. Rows are partitioned by the second
column, so each
+ // row goes to the same partition in both writes.
+ spark
+ .range(500)
+ .selectExpr("id % 7 AS a", "id AS b")
+ .repartition(5, col("b"))
+ .write
+ .mode("overwrite")
+ .saveAsTable("t_first1")
+ val checksums1 = getLatestShuffleChecksumValues()
+
+ spark
+ .range(500)
+ .selectExpr("id % 7 + 1 AS a", "id AS b")
+ .repartition(5, col("b"))
+ .write
+ .mode("overwrite")
+ .saveAsTable("t_first2")
+ val checksums2 = getLatestShuffleChecksumValues()
+
+ assert(
+ checksums1.zip(checksums2).exists { case (a, b) => a != b },
+ "Checksums should differ when only the first column changes: " +
+ s"${checksums1.toSeq} vs ${checksums2.toSeq}"
+ )
+ }
+ }
+ }
}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
index 8ee91ca133..5b3c838633 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
@@ -88,4 +88,38 @@ class GlutenRowBasedChecksumSuite extends
GlutenSQLTestsTrait {
}
}
}
+
+ test("Gluten row-based checksum covers the first data column") {
+ withSQLConf(
+ SQLConf.SHUFFLE_ORDER_INDEPENDENT_CHECKSUM_ENABLED.key -> "true",
+ SQLConf.CLASSIC_SHUFFLE_DEPENDENCY_FILE_CLEANUP_ENABLED.key -> "false") {
+ withTable("t_first1", "t_first2") {
+ // Only the first column differs. Rows are partitioned by the second
column, so each
+ // row goes to the same partition in both writes.
+ spark
+ .range(500)
+ .selectExpr("id % 7 AS a", "id AS b")
+ .repartition(5, col("b"))
+ .write
+ .mode("overwrite")
+ .saveAsTable("t_first1")
+ val checksums1 = getLatestShuffleChecksumValues()
+
+ spark
+ .range(500)
+ .selectExpr("id % 7 + 1 AS a", "id AS b")
+ .repartition(5, col("b"))
+ .write
+ .mode("overwrite")
+ .saveAsTable("t_first2")
+ val checksums2 = getLatestShuffleChecksumValues()
+
+ assert(
+ checksums1.zip(checksums2).exists { case (a, b) => a != b },
+ "Checksums should differ when only the first column changes: " +
+ s"${checksums1.toSeq} vs ${checksums2.toSeq}"
+ )
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]