jiang13021 commented on code in PR #2924:
URL: https://github.com/apache/celeborn/pull/2924#discussion_r1851254223
##########
worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala:
##########
@@ -1330,21 +1445,28 @@ class PushDataHandler(val workerSource: WorkerSource)
extends BaseMessageHandler
batchOffsets match {
case Some(batchOffsets) =>
var index = 0
+ var indexOfSkipIndexes = 0
+ val skipListLength = sortedSkipIndexes.length
var fileWriter: PartitionDataWriter = null
while (index < fileWriters.length) {
- fileWriter = fileWriters(index)
- if (!writePromise.isCompleted) {
- val offset = body.readerIndex() + batchOffsets(index)
- val length =
- if (index == fileWriters.length - 1) {
- body.readableBytes() - batchOffsets(index)
- } else {
- batchOffsets(index + 1) - batchOffsets(index)
- }
- val batchBody = body.slice(offset, length)
- writeData(fileWriter, batchBody, shuffleKey)
+ if (indexOfSkipIndexes < skipListLength && index ==
sortedSkipIndexes(
+ indexOfSkipIndexes)) {
+ indexOfSkipIndexes += 1
} else {
- fileWriter.decrementPendingWrites()
+ fileWriter = fileWriters(index)
+ if (!writePromise.isCompleted) {
+ val offset = body.readerIndex() + batchOffsets(index)
+ val length =
+ if (index == fileWriters.length - 1) {
+ body.readableBytes() - batchOffsets(index)
+ } else {
+ batchOffsets(index + 1) - batchOffsets(index)
+ }
+ val batchBody = body.slice(offset, length)
+ writeData(fileWriter, batchBody, shuffleKey)
Review Comment:
I have supported some fileWriter failures by changing writePromise to
Promise[Array[StatusCode]]. Thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@celeborn.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
