(iotdb) branch master updated: Pipe: Pin unpin log frequency decrease (#14049)

[rong]

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 32e701eadd5 Pipe: Pin unpin log frequency decrease (#14049)
32e701eadd5 is described below

commit 32e701eadd5055e4f3fa01a1f774fa0c9fba8022
Author: nanxiang xia <162968176+xn...@users.noreply.github.com>
AuthorDate: Wed Nov 13 12:51:50 2024 +0800

    Pipe: Pin unpin log frequency decrease (#14049)
    
    Co-authored-by: nanxiang xia <2250337...@qq.com>
---
 .../db/pipe/resource/tsfile/PipeTsFileResourceManager.java | 14 +++++++-------
 .../iotdb/db/pipe/resource/wal/PipeWALResourceManager.java | 14 ++++++++------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index d3482653f85..01bf9f8ca0e 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -80,7 +80,7 @@ public class PipeTsFileResourceManager {
                 PipeConfig.getInstance().getPipeTsFilePinMaxLogNumPerRound(),
                 
PipeConfig.getInstance().getPipeTsFilePinMaxLogIntervalRounds(),
                 hardlinkOrCopiedFileToPipeTsFileResourceMap.size());
-
+    final StringBuilder logBuilder = new StringBuilder();
     while (iterator.hasNext()) {
       final Map.Entry<String, PipeTsFileResource> entry = iterator.next();
 
@@ -97,12 +97,9 @@ public class PipeTsFileResourceManager {
         if (entry.getValue().closeIfOutOfTimeToLive()) {
           iterator.remove();
         } else {
-          logger.ifPresent(
-              l ->
-                  l.info(
-                      "Pipe file (file name: {}) is still referenced {} times",
-                      entry.getKey(),
-                      entry.getValue().getReferenceCount()));
+          logBuilder.append(
+              String.format(
+                  "<%s , %d times> ", entry.getKey(), 
entry.getValue().getReferenceCount()));
         }
       } catch (final IOException e) {
         LOGGER.warn("failed to close PipeTsFileResource when checking TTL: ", 
e);
@@ -110,6 +107,9 @@ public class PipeTsFileResourceManager {
         segmentLock.unlock(new File(hardlinkOrCopiedFile));
       }
     }
+    if (logBuilder.length() > 0) {
+      logger.ifPresent(l -> l.info("Pipe file {}are still referenced", 
logBuilder));
+    }
   }
 
   /**
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
index 9bc3a4a030a..9c51d79daad 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/wal/PipeWALResourceManager.java
@@ -73,6 +73,7 @@ public abstract class PipeWALResourceManager {
                 PipeConfig.getInstance().getPipeWalPinMaxLogIntervalRounds(),
                 memtableIdToPipeWALResourceMap.size());
 
+    final StringBuilder logBuilder = new StringBuilder();
     try {
       while (iterator.hasNext()) {
         final Map.Entry<Long, PipeWALResource> entry = iterator.next();
@@ -84,12 +85,9 @@ public abstract class PipeWALResourceManager {
           if (entry.getValue().invalidateIfPossible()) {
             iterator.remove();
           } else {
-            logger.ifPresent(
-                l ->
-                    l.info(
-                        "WAL (memtableId {}) is still referenced {} times",
-                        entry.getKey(),
-                        entry.getValue().getReferenceCount()));
+            logBuilder.append(
+                String.format(
+                    "<%d , %d times> ", entry.getKey(), 
entry.getValue().getReferenceCount()));
           }
         } finally {
           lock.unlock();
@@ -99,6 +97,10 @@ public abstract class PipeWALResourceManager {
       LOGGER.error(
           "Concurrent modification issues happened, skipping the WAL in this 
round of ttl check",
           e);
+    } finally {
+      if (logBuilder.length() > 0) {
+        logger.ifPresent(l -> l.info("WAL {}are still referenced", 
logBuilder));
+      }
     }
   }