This is an automated email from the ASF dual-hosted git repository.

marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 0447c1aa45 [VL] Include the first data column in the row-based shuffle 
checksum (#13553)
0447c1aa45 is described below

commit 0447c1aa45f3a5c96cf921490bbea8d1f1dd1bfe
Author: YangJie <[email protected]>
AuthorDate: Wed Nov 11 04:22:01 2026 -0500

    [VL] Include the first data column in the row-based shuffle checksum 
(#13553)
---
 cpp/velox/shuffle/VeloxHashShuffleWriter.cc        | 20 ++++---------
 .../sql/gluten/GlutenRowBasedChecksumSuite.scala   | 34 ++++++++++++++++++++++
 .../sql/gluten/GlutenRowBasedChecksumSuite.scala   | 34 ++++++++++++++++++++++
 3 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc 
b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
index 242683450c..877b63995e 100644
--- a/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
+++ b/cpp/velox/shuffle/VeloxHashShuffleWriter.cc
@@ -1655,21 +1655,11 @@ void 
VeloxHashShuffleWriter::computeRowBasedChecksums(const facebook::velox::Row
 
   auto numRows = rv.size();
   VELOX_DCHECK(rv.nulls() == nullptr, "RowVector with top-level nulls not 
supported for checksum");
-  // Get the RowVector to serialize (strip pid column if present).
-  facebook::velox::RowVectorPtr dataVector;
-  if (partitioner_->hasPid()) {
-    // Strip the first column (partition id).
-    auto rowType = std::dynamic_pointer_cast<const 
facebook::velox::RowType>(rv.type());
-    std::vector<std::string> names(rowType->names().begin() + 1, 
rowType->names().end());
-    std::vector<facebook::velox::TypePtr> types(rowType->children().begin() + 
1, rowType->children().end());
-    std::vector<facebook::velox::VectorPtr> children(rv.children().begin() + 
1, rv.children().end());
-    auto dataType = facebook::velox::ROW(std::move(names), std::move(types));
-    dataVector =
-        std::make_shared<facebook::velox::RowVector>(rv.pool(), dataType, 
nullptr, numRows, std::move(children));
-  } else {
-    auto rowType = std::dynamic_pointer_cast<const 
facebook::velox::RowType>(rv.type());
-    dataVector = std::make_shared<facebook::velox::RowVector>(rv.pool(), 
rowType, nullptr, numRows, rv.children());
-  }
+  // Every doSplit caller has already removed the partition id column, so rv 
holds only data
+  // columns.
+  auto rowType = std::dynamic_pointer_cast<const 
facebook::velox::RowType>(rv.type());
+  facebook::velox::RowVectorPtr dataVector =
+      std::make_shared<facebook::velox::RowVector>(rv.pool(), rowType, 
nullptr, numRows, rv.children());
 
   facebook::velox::row::UnsafeRowFast fast(dataVector);
   auto dataType = std::dynamic_pointer_cast<const 
facebook::velox::RowType>(dataVector->type());
diff --git 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
index 8ee91ca133..5b3c838633 100644
--- 
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
+++ 
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
@@ -88,4 +88,38 @@ class GlutenRowBasedChecksumSuite extends 
GlutenSQLTestsTrait {
       }
     }
   }
+
+  test("Gluten row-based checksum covers the first data column") {
+    withSQLConf(
+      SQLConf.SHUFFLE_ORDER_INDEPENDENT_CHECKSUM_ENABLED.key -> "true",
+      SQLConf.CLASSIC_SHUFFLE_DEPENDENCY_FILE_CLEANUP_ENABLED.key -> "false") {
+      withTable("t_first1", "t_first2") {
+        // Only the first column differs. Rows are partitioned by the second 
column, so each
+        // row goes to the same partition in both writes.
+        spark
+          .range(500)
+          .selectExpr("id % 7 AS a", "id AS b")
+          .repartition(5, col("b"))
+          .write
+          .mode("overwrite")
+          .saveAsTable("t_first1")
+        val checksums1 = getLatestShuffleChecksumValues()
+
+        spark
+          .range(500)
+          .selectExpr("id % 7 + 1 AS a", "id AS b")
+          .repartition(5, col("b"))
+          .write
+          .mode("overwrite")
+          .saveAsTable("t_first2")
+        val checksums2 = getLatestShuffleChecksumValues()
+
+        assert(
+          checksums1.zip(checksums2).exists { case (a, b) => a != b },
+          "Checksums should differ when only the first column changes: " +
+            s"${checksums1.toSeq} vs ${checksums2.toSeq}"
+        )
+      }
+    }
+  }
 }
diff --git 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
index 8ee91ca133..5b3c838633 100644
--- 
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
+++ 
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/gluten/GlutenRowBasedChecksumSuite.scala
@@ -88,4 +88,38 @@ class GlutenRowBasedChecksumSuite extends 
GlutenSQLTestsTrait {
       }
     }
   }
+
+  test("Gluten row-based checksum covers the first data column") {
+    withSQLConf(
+      SQLConf.SHUFFLE_ORDER_INDEPENDENT_CHECKSUM_ENABLED.key -> "true",
+      SQLConf.CLASSIC_SHUFFLE_DEPENDENCY_FILE_CLEANUP_ENABLED.key -> "false") {
+      withTable("t_first1", "t_first2") {
+        // Only the first column differs. Rows are partitioned by the second 
column, so each
+        // row goes to the same partition in both writes.
+        spark
+          .range(500)
+          .selectExpr("id % 7 AS a", "id AS b")
+          .repartition(5, col("b"))
+          .write
+          .mode("overwrite")
+          .saveAsTable("t_first1")
+        val checksums1 = getLatestShuffleChecksumValues()
+
+        spark
+          .range(500)
+          .selectExpr("id % 7 + 1 AS a", "id AS b")
+          .repartition(5, col("b"))
+          .write
+          .mode("overwrite")
+          .saveAsTable("t_first2")
+        val checksums2 = getLatestShuffleChecksumValues()
+
+        assert(
+          checksums1.zip(checksums2).exists { case (a, b) => a != b },
+          "Checksums should differ when only the first column changes: " +
+            s"${checksums1.toSeq} vs ${checksums2.toSeq}"
+        )
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to