This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch branch-0.6
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.6 by this push:
new abb63b04f [CELEBORN-2181] Modify the shuffleAllocations order in the
disk info log
abb63b04f is described below
commit abb63b04f9d3a41707777d2455c99d9b1c7d574c
Author: sychen <[email protected]>
AuthorDate: Wed Oct 22 10:33:29 2025 +0800
[CELEBORN-2181] Modify the shuffleAllocations order in the disk info log
### What changes were proposed in this pull request?
### Why are the changes needed?
When there is an abnormality in a disk, shuffleAllocations may have a lot
of applications. The log is relatively long, and some statistical information
of each disk cannot be seen intuitively.
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Closes #3513 from cxzl25/CELEBORN-2181.
Authored-by: sychen <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
(cherry picked from commit 5867901a1a791ca9ea9bb6367eca1a51e8a4a8b0)
Signed-off-by: SteNicholas <[email protected]>
---
.../src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala | 6 +++---
.../scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git
a/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
b/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
index 928794fd9..f66df6711 100644
--- a/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
@@ -179,16 +179,16 @@ class DiskInfo(
s" availableSlots: $availableSlots," +
s" committed shuffles ${emptyShuffles.size}," +
s" running applications ${applicationAllocations.size}," +
- s" shuffleAllocations: ${nonEmptyShuffles.toMap}," +
s" mountPoint: $mountPoint," +
s" usableSpace: ${Utils.bytesToString(actualUsableSpace)}," +
s" totalSpace: ${Utils.bytesToString(totalSpace)}," +
s" avgFlushTime: ${Utils.nanoDurationToString(avgFlushTime)}," +
s" avgFetchTime: ${Utils.nanoDurationToString(avgFetchTime)}," +
s" activeSlots: $activeSlots," +
- s" storageType: ${storageType})" +
+ s" storageType: $storageType)" +
s" status: $status" +
- s" dirs ${dirs.mkString("\t")}"
+ s" dirs ${dirs.mkString("\t")}" +
+ s" shuffleAllocations: ${nonEmptyShuffles.toMap}"
}
}
diff --git
a/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
b/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
index 242856500..0595f84b8 100644
---
a/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
+++
b/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
@@ -355,9 +355,9 @@ class WorkerInfoSuite extends CelebornFunSuite {
|SlotsUsed: 60
|LastHeartbeat: 0
|Disks: $placeholder
- | DiskInfo0: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, shuffleAllocations: Map(), mountPoint:
disk3, usableSpace: 2048.0 MiB, totalSpace: 2048.0 MiB, avgFlushTime: 3 ns,
avgFetchTime: 3 ns, activeSlots: 30, storageType: SSD) status: HEALTHY dirs
$placeholder
- | DiskInfo1: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, shuffleAllocations: Map(), mountPoint:
disk1, usableSpace: 2048.0 MiB, totalSpace: 2048.0 MiB, avgFlushTime: 1 ns,
avgFetchTime: 1 ns, activeSlots: 10, storageType: SSD) status: HEALTHY dirs
$placeholder
- | DiskInfo2: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, shuffleAllocations: Map(), mountPoint:
disk2, usableSpace: 2048.0 MiB, totalSpace: 2048.0 MiB, avgFlushTime: 2 ns,
avgFetchTime: 2 ns, activeSlots: 20, storageType: SSD) status: HEALTHY dirs
$placeholder
+ | DiskInfo0: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, mountPoint: disk3, usableSpace: 2048.0 MiB,
totalSpace: 2048.0 MiB, avgFlushTime: 3 ns, avgFetchTime: 3 ns, activeSlots:
30, storageType: SSD) status: HEALTHY dirs $placeholder shuffleAllocations:
Map()
+ | DiskInfo1: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, mountPoint: disk1, usableSpace: 2048.0 MiB,
totalSpace: 2048.0 MiB, avgFlushTime: 1 ns, avgFetchTime: 1 ns, activeSlots:
10, storageType: SSD) status: HEALTHY dirs $placeholder shuffleAllocations:
Map()
+ | DiskInfo2: DiskInfo(maxSlots: 0, availableSlots: 0, committed
shuffles 0, running applications 0, mountPoint: disk2, usableSpace: 2048.0 MiB,
totalSpace: 2048.0 MiB, avgFlushTime: 2 ns, avgFetchTime: 2 ns, activeSlots:
20, storageType: SSD) status: HEALTHY dirs $placeholder shuffleAllocations:
Map()
|UserResourceConsumption: $placeholder
| UserIdentifier: `tenant1`.`name1`, ResourceConsumption:
ResourceConsumption(diskBytesWritten: 20.0 MiB, diskFileCount: 1,
hdfsBytesWritten: 50.0 MiB, hdfsFileCount: 1, subResourceConsumptions:
(application_1697697127390_2171854 -> ResourceConsumption(diskBytesWritten:
20.0 MiB, diskFileCount: 1, hdfsBytesWritten: 50.0 MiB, hdfsFileCount: 1,
subResourceConsumptions: empty)))
|WorkerRef: null
