This is an automated email from the ASF dual-hosted git repository. richox pushed a commit to branch revert-1731-fix-non-partition-size in repository https://gitbox.apache.org/repos/asf/auron.git
commit f9137f0821aa5dfdbf28f1bf992c3000c2a6b179 Author: Zhang Li <[email protected]> AuthorDate: Thu Dec 11 16:43:26 2025 +0800 Revert "[AURON #1730] Fix data size calculation in Celeborn shuffle writer (#…" This reverts commit fbf2a83d73a6a45269d3033c6757da854329d813. --- .../celeborn/AuronCelebornShuffleWriter.scala | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/thirdparty/auron-celeborn-0.6/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/celeborn/AuronCelebornShuffleWriter.scala b/thirdparty/auron-celeborn-0.6/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/celeborn/AuronCelebornShuffleWriter.scala index bdef9f4f..ba5841b1 100644 --- a/thirdparty/auron-celeborn-0.6/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/celeborn/AuronCelebornShuffleWriter.scala +++ b/thirdparty/auron-celeborn-0.6/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/celeborn/AuronCelebornShuffleWriter.scala @@ -76,24 +76,4 @@ class AuronCelebornShuffleWriter[K, V]( Shims.get .getMapStatus(blockManagerId, celebornPartitionWriter.getPartitionLengthMap, mapId)) } - - // Override stop to use partition length map directly instead of rssStop's mapStatus - // because celeborn writer doesn't populate partition sizes correctly when using native writer - override def stop(success: Boolean): Option[MapStatus] = { - if (!success) { - celebornShuffleWriter.stop(success) - return None - } - - celebornShuffleWriter.write(Iterator.empty) - celebornShuffleWriter.stop(success) - - // Always use getPartitionLengthMap for Celeborn to get correct partition sizes - val blockManagerId = SparkEnv.get.blockManager.shuffleServerId - Some( - Shims.get.getMapStatus( - blockManagerId, - celebornPartitionWriter.getPartitionLengthMap, - taskContext.partitionId())) - } }
