duanmeng commented on a change in pull request #28525:
URL: https://github.com/apache/spark/pull/28525#discussion_r428427803
##########
File path:
core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
##########
@@ -170,11 +190,38 @@ private[spark] class IndexShuffleBlockResolver(
// There is only one IndexShuffleBlockResolver per executor, this
synchronization make sure
// the following check and rename are atomic.
synchronized {
- val existingLengths = checkIndexAndDataFile(indexFile, dataFile,
lengths.length)
- if (existingLengths != null) {
+ val digests = new Array[Long](lengths.length)
+ val dateIn = if (dataTmp != null && dataTmp.exists()) {
+ new FileInputStream(dataTmp)
+ } else {
+ null
+ }
+ Utils.tryWithSafeFinally {
+ if (digestEnable && dateIn != null) {
+ for (i <- (0 until lengths.length)) {
+ val length = lengths(i)
+ if (length == 0) {
+ digests(i) = -1L
+ } else {
+ digests(i) = DigestUtils.getDigest(new
LimitedInputStream(dateIn, length))
+ }
+ }
+ }
+ } {
+ if (dateIn != null) {
+ dateIn.close()
+ }
+ }
+
+ val existingLengthsDigests =
+ checkIndexAndDataFile(indexFile, dataFile, lengths.length, digests)
+ if (existingLengthsDigests != null) {
+ val existingLengths = existingLengthsDigests._1
+ val existingDigests = existingLengthsDigests._2
// Another attempt for the same task has already written our map
outputs successfully,
// so just use the existing partition lengths and delete our
temporary map outputs.
System.arraycopy(existingLengths, 0, lengths, 0, lengths.length)
+ System.arraycopy(existingDigests, 0, digests, 0, digests.length)
Review comment:
This may be unnecessary
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
