This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 11569689b [CELEBORN-747] Print time/bytes in human-readable format
11569689b is described below
commit 11569689be5a2d0749c531e34386ae21db244bfb
Author: Cheng Pan <[email protected]>
AuthorDate: Thu Jun 29 16:46:12 2023 +0800
[CELEBORN-747] Print time/bytes in human-readable format
### What changes were proposed in this pull request?
Print time/bytes in human-readable format
### Why are the changes needed?
Make logs readable
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA.
Closes #1659 from pan3793/minor.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala | 8 ++++----
.../scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala | 6 +++---
.../apache/celeborn/service/deploy/worker/PushDataHandler.scala | 3 ++-
.../celeborn/service/deploy/worker/storage/StorageManager.scala | 2 +-
4 files changed, 10 insertions(+), 9 deletions(-)
diff --git
a/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
b/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
index 13d0c3e91..709d03284 100644
--- a/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/meta/DeviceInfo.scala
@@ -28,7 +28,7 @@ import org.slf4j.LoggerFactory
import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.protocol.StorageInfo
-import org.apache.celeborn.common.util.JavaUtils
+import org.apache.celeborn.common.util.{JavaUtils, Utils}
import org.apache.celeborn.common.util.Utils.runCommand
class DiskInfo(
@@ -134,9 +134,9 @@ class DiskInfo(
s" committed shuffles ${emptyShuffles.size}" +
s" shuffleAllocations: $nonEmptyShuffles," +
s" mountPoint: $mountPoint," +
- s" usableSpace: $actualUsableSpace," +
- s" avgFlushTime: $avgFlushTime," +
- s" avgFetchTime: $avgFetchTime," +
+ s" usableSpace: ${Utils.bytesToString(actualUsableSpace)}," +
+ s" avgFlushTime: ${Utils.msDurationToString(avgFlushTime)}," +
+ s" avgFetchTime: ${Utils.msDurationToString(avgFetchTime)}," +
s" activeSlots: $activeSlots)" +
s" status: $status" +
s" dirs ${dirs.mkString("\t")}"
diff --git
a/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
b/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
index 8e92b9cec..6eb9c7d65 100644
---
a/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
+++
b/common/src/test/scala/org/apache/celeborn/common/meta/WorkerInfoSuite.scala
@@ -283,9 +283,9 @@ class WorkerInfoSuite extends CelebornFunSuite {
|SlotsUsed: 60
|LastHeartbeat: 0
|Disks: $placeholder
- | DiskInfo0: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk3, usableSpace: 2147483647,
avgFlushTime: 3, avgFetchTime: 3, activeSlots: 30) status: HEALTHY dirs
$placeholder
- | DiskInfo1: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk1, usableSpace: 2147483647,
avgFlushTime: 1, avgFetchTime: 1, activeSlots: 10) status: HEALTHY dirs
$placeholder
- | DiskInfo2: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk2, usableSpace: 2147483647,
avgFlushTime: 2, avgFetchTime: 2, activeSlots: 20) status: HEALTHY dirs
$placeholder
+ | DiskInfo0: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk3, usableSpace: 2048.0 MiB,
avgFlushTime: 3 ms, avgFetchTime: 3 ms, activeSlots: 30) status: HEALTHY dirs
$placeholder
+ | DiskInfo1: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk1, usableSpace: 2048.0 MiB,
avgFlushTime: 1 ms, avgFetchTime: 1 ms, activeSlots: 10) status: HEALTHY dirs
$placeholder
+ | DiskInfo2: DiskInfo(maxSlots: 0, committed shuffles 0
shuffleAllocations: Map(), mountPoint: disk2, usableSpace: 2048.0 MiB,
avgFlushTime: 2 ms, avgFetchTime: 2 ms, activeSlots: 20) status: HEALTHY dirs
$placeholder
|UserResourceConsumption: $placeholder
| UserIdentifier: `tenant1`.`name1`, ResourceConsumption:
ResourceConsumption(diskBytesWritten: 20.0 MiB, diskFileCount: 1,
hdfsBytesWritten: 50.0 MiB, hdfsFileCount: 1)
|WorkerRef: null
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
index 09d2d2af6..419cafb00 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
@@ -36,6 +36,7 @@ import
org.apache.celeborn.common.network.server.BaseMessageHandler
import org.apache.celeborn.common.protocol.{PartitionLocation,
PartitionSplitMode, PartitionType}
import org.apache.celeborn.common.protocol.message.StatusCode
import org.apache.celeborn.common.unsafe.Platform
+import org.apache.celeborn.common.util.Utils
import
org.apache.celeborn.service.deploy.worker.congestcontrol.CongestionController
import org.apache.celeborn.service.deploy.worker.storage.{FileWriter,
HdfsFlusher, LocalFlusher, MapPartitionFileWriter, StorageManager}
@@ -82,7 +83,7 @@ class PushDataHandler extends BaseMessageHandler with Logging
{
testPushPrimaryDataTimeout = worker.conf.testPushPrimaryDataTimeout
testPushReplicaDataTimeout = worker.conf.testPushReplicaDataTimeout
- logInfo(s"diskReserveSize $diskReserveSize")
+ logInfo(s"diskReserveSize ${Utils.bytesToString(diskReserveSize)}")
}
override def receive(client: TransportClient, msg: RequestMessage): Unit =
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 09a0d832c..d2099df2a 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -707,7 +707,7 @@ final private[worker] class StorageManager(conf:
CelebornConf, workerSource: Abs
diskInfo.updateFlushTime()
diskInfo.updateFetchTime()
}
- logInfo(s"Updated diskInfos: ${disksSnapshot()}")
+ logInfo(s"Updated diskInfos:\n${disksSnapshot().mkString("\n")}")
}
def userResourceConsumptionSnapshot(): Map[UserIdentifier,
ResourceConsumption] = {
