(incubator-uniffle) branch master updated: [#1472][part-4] feature(server): Add metrics for Netty's pinnedDirectMemory and usedDirectMemory (#1524)

[roryqi]

This is an automated email from the ASF dual-hosted git repository.

roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new 72d85a778 [#1472][part-4] feature(server): Add metrics for Netty's 
pinnedDirectMemory and usedDirectMemory (#1524)
72d85a778 is described below

commit 72d85a778f559ddf1efe0b6909eb8146e5a1c646
Author: RickyMa <rick...@tencent.com>
AuthorDate: Wed Feb 14 10:23:25 2024 +0800

    [#1472][part-4] feature(server): Add metrics for Netty's pinnedDirectMemory 
and usedDirectMemory (#1524)
    
    ### What changes were proposed in this pull request?
    
    We need to know the exact direct memory usage of `PooledByteBufAllocator` 
in `Netty`.
    So we should introduce metrics for Netty's `pinnedDirectMemory` and 
`usedDirectMemory`.
    
    ### Why are the changes needed?
    
    A sub PR for: https://github.com/apache/incubator-uniffle/pull/1519
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Existing UTs.
---
 .../org/apache/uniffle/server/NettyDirectMemoryTracker.java  | 12 +++++++++++-
 .../java/org/apache/uniffle/server/ShuffleServerMetrics.java |  6 ++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git 
a/server/src/main/java/org/apache/uniffle/server/NettyDirectMemoryTracker.java 
b/server/src/main/java/org/apache/uniffle/server/NettyDirectMemoryTracker.java
index 208e41595..c3a31fd52 100644
--- 
a/server/src/main/java/org/apache/uniffle/server/NettyDirectMemoryTracker.java
+++ 
b/server/src/main/java/org/apache/uniffle/server/NettyDirectMemoryTracker.java
@@ -25,6 +25,7 @@ import io.netty.util.internal.PlatformDependent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.uniffle.common.util.NettyUtils;
 import org.apache.uniffle.common.util.ThreadUtils;
 
 public class NettyDirectMemoryTracker {
@@ -54,10 +55,19 @@ public class NettyDirectMemoryTracker {
         () -> {
           try {
             long usedDirectMemory = PlatformDependent.usedDirectMemory();
+            long allocatedDirectMemory =
+                
NettyUtils.getNettyBufferAllocator().metric().usedDirectMemory();
+            long pinnedDirectMemory = 
NettyUtils.getNettyBufferAllocator().pinnedDirectMemory();
             if (LOG.isDebugEnabled()) {
-              LOG.debug("Current direct memory usage: {}", usedDirectMemory);
+              LOG.debug(
+                  "Current usedDirectMemory:{}, allocatedDirectMemory:{}, 
pinnedDirectMemory:{}",
+                  usedDirectMemory,
+                  allocatedDirectMemory,
+                  pinnedDirectMemory);
             }
             
ShuffleServerMetrics.gaugeUsedDirectMemorySize.set(usedDirectMemory);
+            
ShuffleServerMetrics.gaugeAllocatedDirectMemorySize.set(allocatedDirectMemory);
+            
ShuffleServerMetrics.gaugePinnedDirectMemorySize.set(pinnedDirectMemory);
           } catch (Throwable t) {
             LOG.error("Failed to report direct memory.", t);
           }
diff --git 
a/server/src/main/java/org/apache/uniffle/server/ShuffleServerMetrics.java 
b/server/src/main/java/org/apache/uniffle/server/ShuffleServerMetrics.java
index c5aeac615..649e5044b 100644
--- a/server/src/main/java/org/apache/uniffle/server/ShuffleServerMetrics.java
+++ b/server/src/main/java/org/apache/uniffle/server/ShuffleServerMetrics.java
@@ -79,6 +79,8 @@ public class ShuffleServerMetrics {
   private static final String USED_BUFFER_SIZE = "used_buffer_size";
   private static final String READ_USED_BUFFER_SIZE = "read_used_buffer_size";
   private static final String USED_DIRECT_MEMORY_SIZE = 
"used_direct_memory_size";
+  private static final String ALLOCATED_DIRECT_MEMORY_SIZE = 
"allocated_direct_memory_size";
+  private static final String PINNED_DIRECT_MEMORY_SIZE = 
"pinned_direct_memory_size";
   private static final String TOTAL_FAILED_WRITTEN_EVENT_NUM = 
"total_failed_written_event_num";
   private static final String TOTAL_DROPPED_EVENT_NUM = 
"total_dropped_event_num";
   private static final String TOTAL_HADOOP_WRITE_DATA = 
"total_hadoop_write_data";
@@ -184,6 +186,8 @@ public class ShuffleServerMetrics {
   public static Gauge.Child gaugeUsedBufferSize;
   public static Gauge.Child gaugeReadBufferUsedSize;
   public static Gauge.Child gaugeUsedDirectMemorySize;
+  public static Gauge.Child gaugeAllocatedDirectMemorySize;
+  public static Gauge.Child gaugePinnedDirectMemorySize;
   public static Gauge.Child gaugeWriteHandler;
   public static Gauge.Child gaugeEventQueueSize;
   public static Gauge.Child gaugeHadoopFlushThreadPoolQueueSize;
@@ -380,6 +384,8 @@ public class ShuffleServerMetrics {
     gaugeUsedBufferSize = metricsManager.addLabeledGauge(USED_BUFFER_SIZE);
     gaugeReadBufferUsedSize = 
metricsManager.addLabeledGauge(READ_USED_BUFFER_SIZE);
     gaugeUsedDirectMemorySize = 
metricsManager.addLabeledGauge(USED_DIRECT_MEMORY_SIZE);
+    gaugeAllocatedDirectMemorySize = 
metricsManager.addLabeledGauge(ALLOCATED_DIRECT_MEMORY_SIZE);
+    gaugePinnedDirectMemorySize = 
metricsManager.addLabeledGauge(PINNED_DIRECT_MEMORY_SIZE);
     gaugeWriteHandler = metricsManager.addLabeledGauge(TOTAL_WRITE_HANDLER);
     gaugeEventQueueSize = metricsManager.addLabeledGauge(EVENT_QUEUE_SIZE);
     gaugeHadoopFlushThreadPoolQueueSize =