[incubator-celeborn] branch main updated: [CELEBORN-281] Add metrics about buffer stream read buffer. (#1216)

[ethanfeng]

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new 1dcfdb0c [CELEBORN-281] Add metrics about buffer stream read buffer. 
(#1216)
1dcfdb0c is described below

commit 1dcfdb0c8fed1db443efe174088f8cd1d1618ec8
Author: Ethan Feng <ethanf...@apache.org>
AuthorDate: Fri Feb 17 11:20:07 2023 +0800

    [CELEBORN-281] Add metrics about buffer stream read buffer. (#1216)
---
 .../apache/celeborn/common/network/server/memory/MemoryManager.java | 6 +++++-
 .../scala/org/apache/celeborn/service/deploy/worker/Worker.scala    | 3 +++
 .../org/apache/celeborn/service/deploy/worker/WorkerSource.scala    | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git 
a/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
 
b/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
index 2dd8c04c..dd0273dc 100644
--- 
a/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
+++ 
b/common/src/main/java/org/apache/celeborn/common/network/server/memory/MemoryManager.java
@@ -67,7 +67,7 @@ public class MemoryManager {
   private boolean underPressure;
   private final AtomicBoolean trimInProcess = new AtomicBoolean(false);
 
-  // For read buffer
+  // For buffer stream
   private final AtomicLong readBufferCounter = new AtomicLong(0);
   private long readBufferThreshold = 0;
   private final ReadBufferDispatcher readBufferDispatcher;
@@ -342,6 +342,10 @@ public class MemoryManager {
     return diskBufferCounter;
   }
 
+  public AtomicLong getReadBufferCounter() {
+    return readBufferCounter;
+  }
+
   public long getPausePushDataCounter() {
     return pausePushDataCounter.sum();
   }
diff --git 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 800cec31..7617d1c6 100644
--- 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -239,6 +239,9 @@ private[celeborn] class Worker(
   workerSource.addGauge(
     WorkerSource.PausePushDataAndReplicateCount,
     _ => memoryTracker.getPausePushDataAndReplicateCounter)
+  workerSource.addGauge(
+    WorkerSource.BufferStreamReadBuffer,
+    _ => memoryTracker.getReadBufferCounter.get())
 
   private def heartBeatToMaster(): Unit = {
     val activeShuffleKeys = new JHashSet[String]()
diff --git 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
index df662542..ea9b293d 100644
--- 
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
+++ 
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
@@ -91,6 +91,7 @@ object WorkerSource {
   val DiskBuffer = "DiskBuffer"
   val PausePushDataCount = "PausePushData"
   val PausePushDataAndReplicateCount = "PausePushDataAndReplicate"
+  val BufferStreamReadBuffer = "BufferStreamReadBuffer"
 
   // local device
   val DeviceOSFreeCapacity = "DeviceOSFreeCapacity(B)"