This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 54bff3d9a42 [Pipe] Allow retry heartbeats past realtime completion 
barriers (#19219)
54bff3d9a42 is described below

commit 54bff3d9a421f9ba148fda3a26626df570422f54
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 12 11:28:18 2026 +0800

    [Pipe] Allow retry heartbeats past realtime completion barriers (#19219)
    
    * Fix pipe retry heartbeats blocked by completion barriers
    
    * Fix retry heartbeats blocked by completion barriers
---
 .../sink/PipeRealtimePriorityBlockingQueue.java    |   4 +-
 .../realtime/PipeRealtimeDataRegionSource.java     |   1 +
 .../PipeRealtimeDataRegionTsFileSource.java        |   3 +-
 .../task/connection/PipeEventCollectorTest.java    |  31 ++++++
 .../db/pipe/source/PipeRealtimeExtractTest.java    | 104 ++++++++++++++++-----
 5 files changed, 120 insertions(+), 23 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
index f224f09425d..83c8666eb3c 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java
@@ -83,9 +83,11 @@ public class PipeRealtimePriorityBlockingQueue extends 
UnboundedBlockingPendingQ
       return true;
     }
 
+    final Event lastEvent = super.peekLast();
     if (event instanceof PipeHeartbeatEvent
         && !((PipeHeartbeatEvent) event).isCompletionBarrier()
-        && super.peekLast() instanceof PipeHeartbeatEvent) {
+        && lastEvent instanceof PipeHeartbeatEvent
+        && !((PipeHeartbeatEvent) lastEvent).isCompletionBarrier()) {
       // We can NOT keep too many PipeHeartbeatEvent in bufferQueue because 
they may cause OOM.
       ((EnrichedEvent) 
event).decreaseReferenceCount(PipeEventCollector.class.getName(), false);
       return false;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
index 801051b991c..2bb6ac9b6a3 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
@@ -466,6 +466,7 @@ public abstract class PipeRealtimeDataRegionSource 
implements PipeExtractor {
     if (!heartbeatEvent.isCompletionBarrier()
         && lastEvent instanceof PipeRealtimeEvent
         && ((PipeRealtimeEvent) lastEvent).getEvent() instanceof 
PipeHeartbeatEvent
+        && !((PipeHeartbeatEvent) ((PipeRealtimeEvent) 
lastEvent).getEvent()).isCompletionBarrier()
         && (((PipeHeartbeatEvent) ((PipeRealtimeEvent) 
lastEvent).getEvent()).isShouldPrintMessage()
             || !heartbeatEvent.isShouldPrintMessage())) {
       // If the last event in the pending queue is a heartbeat event, we 
should not extract any more
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionTsFileSource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionTsFileSource.java
index ec308b65d07..88fbabf81bd 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionTsFileSource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionTsFileSource.java
@@ -210,7 +210,8 @@ public class PipeRealtimeDataRegionTsFileSource extends 
PipeRealtimeDataRegionSo
         final PipeRealtimeEvent event = iterator.next();
         if (event.getEvent() instanceof PipeHeartbeatEvent) {
           if (((PipeHeartbeatEvent) event.getEvent()).isCompletionBarrier()) {
-            return null;
+            // Completion barriers stay ordered, but do not prevent 
retry-triggering heartbeats.
+            continue;
           }
           iterator.remove();
           return event;
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java
index de89e6c02ab..3379876a3e9 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.pipe.agent.task.connection;
 
 import 
org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
 import 
org.apache.iotdb.db.pipe.agent.task.subtask.sink.PipeRealtimePriorityBlockingQueue;
+import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
 import 
org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
 import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
 import org.apache.iotdb.pipe.api.event.Event;
@@ -48,6 +49,36 @@ public class PipeEventCollectorTest {
     verifyCollectorDoesNotOfferEventsOfDroppedPipe(new 
PipeRealtimePriorityBlockingQueue());
   }
 
+  @Test
+  public void testRealtimeQueueRetainsRetryHeartbeatAfterCompletionBarrier() {
+    final PipeRealtimePriorityBlockingQueue pendingQueue = new 
PipeRealtimePriorityBlockingQueue();
+    final PipeHeartbeatEvent completionBarrier = new PipeHeartbeatEvent(1, 
false, true);
+    final PipeHeartbeatEvent retryHeartbeat = new PipeHeartbeatEvent(1, false);
+    final PipeHeartbeatEvent coalescedHeartbeat = new PipeHeartbeatEvent(1, 
false);
+    
Assert.assertTrue(completionBarrier.increaseReferenceCount(PipeEventCollector.class.getName()));
+    
Assert.assertTrue(retryHeartbeat.increaseReferenceCount(PipeEventCollector.class.getName()));
+    Assert.assertTrue(
+        
coalescedHeartbeat.increaseReferenceCount(PipeEventCollector.class.getName()));
+
+    try {
+      Assert.assertTrue(pendingQueue.offer(completionBarrier));
+      Assert.assertTrue(pendingQueue.offer(retryHeartbeat));
+      Assert.assertFalse(pendingQueue.offer(coalescedHeartbeat));
+
+      Assert.assertEquals(2, pendingQueue.size());
+      Assert.assertTrue(coalescedHeartbeat.isReleased());
+      Assert.assertSame(completionBarrier, pendingQueue.directPoll());
+      Assert.assertSame(retryHeartbeat, pendingQueue.directPoll());
+      Assert.assertFalse(completionBarrier.isReleased());
+      Assert.assertFalse(retryHeartbeat.isReleased());
+    } finally {
+      pendingQueue.discardAllEvents();
+      
completionBarrier.clearReferenceCount(PipeEventCollector.class.getName());
+      retryHeartbeat.clearReferenceCount(PipeEventCollector.class.getName());
+      
coalescedHeartbeat.clearReferenceCount(PipeEventCollector.class.getName());
+    }
+  }
+
   private void verifyCollectorDoesNotOfferEventsOfDroppedPipe(
       final UnboundedBlockingPendingQueue<Event> pendingQueue) {
     pendingQueue.discardEventsOfPipe("pipe", 1L, 1);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
index 6c44156cfa5..2bb9501f2e4 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.source;
 import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
+import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
@@ -1336,54 +1337,115 @@ public class PipeRealtimeExtractTest {
   @Test
   public void 
testTsFileSourceRegionLevelDowngradingSuppliesHeartbeatWhileTsFileIsInFlight()
       throws Exception {
+    final PipeEventCommitManager commitManager = 
PipeEventCommitManager.getInstance();
+    commitManager.register(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1, "test");
     final PipeTaskMeta pipeTaskMeta = new 
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
     try (final PipeRealtimeDataRegionTsFileSource extractor =
         createTsFileSourceForTest(
             PipeSourceConstant.SOURCE_REALTIME_REGION_LEVEL_DOWNGRADING_KEY, 
pipeTaskMeta)) {
+      final SimpleProgressIndex firstProgressIndex = new 
SimpleProgressIndex(1, 1);
+      final SimpleProgressIndex secondProgressIndex = new 
SimpleProgressIndex(1, 2);
+      final TsFileResource firstResource = createTsFileResource(dataRegion1, 
"123-1-0-0.tsfile");
+      final TsFileResource secondResource = createTsFileResource(dataRegion1, 
"123-2-0-0.tsfile");
+      firstResource.updateProgressIndex(firstProgressIndex);
+      secondResource.updateProgressIndex(secondProgressIndex);
       final PipeRealtimeEvent firstEvent =
-          extractTsFileForTest(
-              extractor,
-              pipeTaskMeta,
-              createTsFileResource(dataRegion1, "123-1-0-0.tsfile"),
-              false);
+          extractTsFileForTest(extractor, pipeTaskMeta, firstResource, false);
       final PipeRealtimeEvent secondEvent =
-          extractTsFileForTest(
-              extractor,
-              pipeTaskMeta,
-              createTsFileResource(dataRegion1, "123-2-0-0.tsfile"),
-              false);
+          extractTsFileForTest(extractor, pipeTaskMeta, secondResource, false);
+      // Match the assigner binding so successful commits retire the source's 
deduplication entries.
+      ((PipeTsFileInsertionEvent) firstEvent.getEvent())
+          .bindTsFileDedupScopeID(extractor.getTsFileDedupScopeID());
+      ((PipeTsFileInsertionEvent) secondEvent.getEvent())
+          .bindTsFileDedupScopeID(extractor.getTsFileDedupScopeID());
       final PipeRealtimeEvent heartbeat =
-          PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false);
+          bindToTestPipe(
+              PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false),
+              extractor,
+              pipeTaskMeta);
       
Assert.assertTrue(heartbeat.increaseReferenceCount(TEST_REFERENCE_HOLDER));
       extractor.extract(heartbeat);
       final PipeRealtimeEvent completionBarrier =
-          PipeRealtimeEventFactory.createCompletionBarrierEvent(dataRegion1);
+          bindToTestPipe(
+              
PipeRealtimeEventFactory.createCompletionBarrierEvent(dataRegion1),
+              extractor,
+              pipeTaskMeta);
+      completionBarrier.getEvent().bindSource(extractor.getSourceId());
+      final AtomicBoolean completionBarrierCommitted = new 
AtomicBoolean(false);
+      completionBarrier.getEvent().addOnCommittedHook(() -> 
completionBarrierCommitted.set(true));
+      Assert.assertTrue(completionBarrier.getEvent().needToCommit());
       
Assert.assertTrue(completionBarrier.increaseReferenceCount(TEST_REFERENCE_HOLDER));
       extractor.extract(completionBarrier);
 
-      // Drain the pending queue before adding another heartbeat so heartbeat 
coalescing does not
-      // discard it behind the completion barrier.
+      // Keep one retry-triggering heartbeat after the barrier; subsequent 
heartbeats can still be
+      // coalesced behind it to bound the pending queue.
+      final PipeRealtimeEvent coalescedHeartbeat =
+          bindToTestPipe(
+              PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false),
+              extractor,
+              pipeTaskMeta);
+      
Assert.assertTrue(coalescedHeartbeat.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+      extractor.extract(coalescedHeartbeat);
+      Assert.assertFalse(coalescedHeartbeat.getEvent().isReleased());
+
+      // Queue a later heartbeat behind the completion barrier while the first 
TsFile remains in
+      // flight.
       Assert.assertSame(firstEvent.getEvent(), extractor.supply());
       final PipeRealtimeEvent heartbeatAfterCompletionBarrier =
-          PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false);
+          bindToTestPipe(
+              PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false),
+              extractor,
+              pipeTaskMeta);
       Assert.assertTrue(
           
heartbeatAfterCompletionBarrier.increaseReferenceCount(TEST_REFERENCE_HOLDER));
       extractor.extract(heartbeatAfterCompletionBarrier);
 
       Assert.assertSame(heartbeat.getEvent(), extractor.supply());
+      Assert.assertSame(coalescedHeartbeat.getEvent(), extractor.supply());
+      Assert.assertSame(heartbeatAfterCompletionBarrier.getEvent(), 
extractor.supply());
       Assert.assertNull(extractor.supply());
       Assert.assertFalse(secondEvent.getEvent().isReleased());
       Assert.assertFalse(completionBarrier.getEvent().isReleased());
       
Assert.assertFalse(heartbeatAfterCompletionBarrier.getEvent().isReleased());
 
-      releaseSuppliedEvent(heartbeat.getEvent());
-      releaseSuppliedEvent(firstEvent.getEvent());
+      commitSuppliedEvent(heartbeat.getEvent(), commitManager);
+      commitSuppliedEvent(heartbeatAfterCompletionBarrier.getEvent(), 
commitManager);
+      
Assert.assertNull(heartbeatAfterCompletionBarrier.getEvent().getCommitterKey());
+      Assert.assertEquals(MinimumProgressIndex.INSTANCE, 
pipeTaskMeta.getProgressIndex());
+      Assert.assertFalse(completionBarrierCommitted.get());
+
+      // A failed retry can retain the file across more than one heartbeat 
interval.
+      final PipeRealtimeEvent retryHeartbeat =
+          bindToTestPipe(
+              PipeRealtimeEventFactory.createRealtimeEvent(dataRegion1, false),
+              extractor,
+              pipeTaskMeta);
+      
Assert.assertTrue(retryHeartbeat.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+      extractor.extract(retryHeartbeat);
+      Assert.assertSame(retryHeartbeat.getEvent(), extractor.supply());
+      commitSuppliedEvent(retryHeartbeat.getEvent(), commitManager);
+      Assert.assertNull(extractor.supply());
+      Assert.assertFalse(firstEvent.getEvent().isReleased());
+      Assert.assertEquals(MinimumProgressIndex.INSTANCE, 
pipeTaskMeta.getProgressIndex());
+      Assert.assertFalse(completionBarrierCommitted.get());
+
+      // Successful retry must release the ordering gate through commit, 
rather than discard.
+      commitSuppliedEvent(firstEvent.getEvent(), commitManager);
+      Assert.assertEquals(firstProgressIndex, pipeTaskMeta.getProgressIndex());
       Assert.assertSame(secondEvent.getEvent(), extractor.supply());
-      releaseSuppliedEvent(secondEvent.getEvent());
+      Assert.assertNull(extractor.supply());
+      Assert.assertFalse(completionBarrier.getEvent().isReleased());
+      Assert.assertFalse(completionBarrierCommitted.get());
+      commitSuppliedEvent(secondEvent.getEvent(), commitManager);
+      Assert.assertEquals(secondProgressIndex, 
pipeTaskMeta.getProgressIndex());
       Assert.assertSame(completionBarrier.getEvent(), extractor.supply());
-      releaseSuppliedEvent(completionBarrier.getEvent());
-      Assert.assertSame(heartbeatAfterCompletionBarrier.getEvent(), 
extractor.supply());
-      releaseSuppliedEvent(heartbeatAfterCompletionBarrier.getEvent());
+      Assert.assertFalse(completionBarrierCommitted.get());
+      commitSuppliedEvent(completionBarrier.getEvent(), commitManager);
+      Assert.assertTrue(completionBarrierCommitted.get());
+      Assert.assertEquals(secondProgressIndex, 
pipeTaskMeta.getProgressIndex());
+      Assert.assertNull(extractor.supply());
+    } finally {
+      commitManager.deregister(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1);
     }
   }
 

Reply via email to