This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new d46b6623b [CELEBORN-1228] Format the timestamp when recording worker
failure
d46b6623b is described below
commit d46b6623b314a46d5b8fa321ea691e2063629e93
Author: Fei Wang <[email protected]>
AuthorDate: Wed Jan 17 14:04:30 2024 +0800
[CELEBORN-1228] Format the timestamp when recording worker failure
### What changes were proposed in this pull request?
Format the timestamp when recoding worker failure inforamtion.
### Why are the changes needed?
Now the long type timestamp is difficult to view and confuse without
reading source code.
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Closes #2230 from turboFei/date_format.
Authored-by: Fei Wang <[email protected]>
Signed-off-by: mingji <[email protected]>
---
.../scala/org/apache/celeborn/client/WorkerStatusTracker.scala | 5 +++--
common/src/main/scala/org/apache/celeborn/common/util/Utils.scala | 5 +++++
.../scala/org/apache/celeborn/service/deploy/master/Master.scala | 4 ++--
.../main/scala/org/apache/celeborn/server/common/HttpService.scala | 7 -------
.../scala/org/apache/celeborn/service/deploy/worker/Worker.scala | 2 +-
5 files changed, 11 insertions(+), 12 deletions(-)
diff --git
a/client/src/main/scala/org/apache/celeborn/client/WorkerStatusTracker.scala
b/client/src/main/scala/org/apache/celeborn/client/WorkerStatusTracker.scala
index 13a8fdade..14a0df092 100644
--- a/client/src/main/scala/org/apache/celeborn/client/WorkerStatusTracker.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/WorkerStatusTracker.scala
@@ -30,6 +30,7 @@ import org.apache.celeborn.common.meta.WorkerInfo
import org.apache.celeborn.common.protocol.PartitionLocation
import
org.apache.celeborn.common.protocol.message.ControlMessages.HeartbeatFromApplicationResponse
import org.apache.celeborn.common.protocol.message.StatusCode
+import org.apache.celeborn.common.util.Utils
class WorkerStatusTracker(
conf: CelebornConf,
@@ -121,10 +122,10 @@ class WorkerStatusTracker(
if (!failures.isEmpty) {
val failedWorker = new ShuffleFailedWorkers(failures)
val failedWorkerMsg = failedWorker.asScala.map { case (worker, (status,
time)) =>
- s"${worker.readableAddress()} ${status.name()} $time"
+ s"${worker.readableAddress()} ${status.name()}
${Utils.formatTimestamp(time)}"
}.mkString("\n")
val excludedWorkerMsg = excludedWorkers.asScala.map { case (worker,
(status, time)) =>
- s"${worker.readableAddress()} ${status.name()} $time"
+ s"${worker.readableAddress()} ${status.name()}
${Utils.formatTimestamp(time)}"
}.mkString("\n")
val shuttingDownMsg =
shuttingWorkers.asScala.map(_.readableAddress()).mkString("\n")
logInfo(
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
index 4100c4ea9..125e1f67e 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -38,6 +38,7 @@ import scala.util.control.{ControlThrowable, NonFatal}
import com.google.protobuf.{ByteString, GeneratedMessageV3}
import io.netty.channel.unix.Errors.NativeIoException
import org.apache.commons.lang3.SystemUtils
+import org.apache.commons.lang3.time.FastDateFormat
import org.roaringbitmap.RoaringBitmap
import org.apache.celeborn.common.CelebornConf
@@ -1092,4 +1093,8 @@ object Utils extends Logging {
}
def getProcessId: String =
ManagementFactory.getRuntimeMXBean.getName.split("@")(0)
+
+ private val dateFmt: FastDateFormat =
+ FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
+ def formatTimestamp(timestamp: Long): String = dateFmt.format(timestamp)
}
diff --git
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 3fcb94b01..1704a95a0 100644
---
a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++
b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -940,7 +940,7 @@ private[celeborn] class Master(
val sb = new StringBuilder
sb.append("======================= Lost Workers in Master
========================\n")
lostWorkersSnapshot.asScala.toSeq.sortBy(_._2).foreach { case (worker,
time) =>
- sb.append(s"${worker.toUniqueId().padTo(50, "
").mkString}${dateFmt.format(time)}\n")
+ sb.append(s"${worker.toUniqueId().padTo(50, "
").mkString}${Utils.formatTimestamp(time)}\n")
}
sb.toString()
}
@@ -983,7 +983,7 @@ private[celeborn] class Master(
val sb = new StringBuilder
sb.append("================= LifecycleManager Application List
======================\n")
statusSystem.appHeartbeatTime.asScala.toSeq.sortBy(_._2).foreach { case
(appId, time) =>
- sb.append(s"${appId.padTo(40, " ").mkString}${dateFmt.format(time)}\n")
+ sb.append(s"${appId.padTo(40, "
").mkString}${Utils.formatTimestamp(time)}\n")
}
sb.toString()
}
diff --git
a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
index b7bdc7d87..9ca1be8f7 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
@@ -17,10 +17,6 @@
package org.apache.celeborn.server.common
-import java.util.Locale
-
-import org.apache.commons.lang3.time.FastDateFormat
-
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.server.common.http.{HttpRequestHandler, HttpServer,
HttpServerInitializer}
@@ -28,9 +24,6 @@ abstract class HttpService extends Service with Logging {
private var httpServer: HttpServer = _
- protected val dateFmt: FastDateFormat =
- FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT)
-
def getConf: String = {
val sb = new StringBuilder
sb.append("=========================== Configuration
============================\n")
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 8ee260e1f..dc4152c86 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -664,7 +664,7 @@ private[celeborn] class Worker(
val sb = new StringBuilder
sb.append("==================== Unavailable Peers of Worker
=====================\n")
unavailablePeers.asScala.foreach { case (peer, time) =>
- sb.append(s"${peer.toUniqueId().padTo(50, "
").mkString}${dateFmt.format(time)}\n")
+ sb.append(s"${peer.toUniqueId().padTo(50, "
").mkString}${Utils.formatTimestamp(time)}\n")
}
sb.toString()
}
