jt2594838 commented on code in PR #18726:
URL: https://github.com/apache/iotdb/pull/18726#discussion_r4560533981


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java:
##########
@@ -249,71 +262,59 @@ public synchronized void replace(
       eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
     }
 
-    // Handling new events
     final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents = 
new HashSet<>();
-    final AtomicBoolean
-        allSuccess = // To track if all events successfully increased the 
reference count
-        new AtomicBoolean(true);
+    boolean allSuccess = true;
     outerLoop:
-    for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> 
committerKeySetEntry :
-        eventsToBeAddedGroupByCommitterKey.entrySet()) {
-      for (final PipeTsFileInsertionEvent event : 
committerKeySetEntry.getValue()) {
-        if (event != null) {
-          try {
-            if 
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
 {
-              allSuccess.set(false);
-              break outerLoop;
-            } else {
-              successfullyReferenceIncreasedEvents.add(event);
-            }
-          } catch (final Exception e) {
-            allSuccess.set(false);
+    for (final Set<PipeTsFileInsertionEvent> events : 
eventsToBeAddedGroupByCommitterKey.values()) {
+      for (final PipeTsFileInsertionEvent event : events) {
+        try {
+          if 
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
 {
+            allSuccess = false;
             break outerLoop;
           }
+          successfullyReferenceIncreasedEvents.add(event);
+        } catch (final Exception e) {
+          allSuccess = false;
+          break outerLoop;
         }
       }
     }
-    if (!allSuccess.get()) {
-      // If any event failed to increase the reference count,
-      // we need to decrease the reference count for all successfully 
increased events
+    if (!allSuccess) {
       for (final PipeTsFileInsertionEvent event : 
successfullyReferenceIncreasedEvents) {
         try {
           
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
         } catch (final Exception e) {
           
LOGGER.warn(DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT, 
event, e);
         }
       }
-      return; // Exit early if any event failed to increase the reference count
-    } else {
-      // If all events successfully increased reference count,
-      // we can proceed to add them to the deque
-      for (final PipeTsFileInsertionEvent event : 
successfullyReferenceIncreasedEvents) {
-        tsfileInsertEventDeque.add(event);
-        eventCounter.increaseEventCount(event);
-      }
+      return;
     }
 
-    // Handling old events
-    for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
-        eventsToBeRemovedGroupByCommitterKey.entrySet()) {
-      for (final PipeTsFileInsertionEvent event : entry.getValue()) {
-        if (event != null) {
-          try {
-            
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(), 
false);
-          } catch (final Exception e) {
-            LOGGER.warn(
-                
DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT, event, e);
-          }
-          eventCounter.decreaseEventCount(event);
-        }
-      }
-    }
     final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
-    for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+    for (final Set<PipeTsFileInsertionEvent> events :
         eventsToBeRemovedGroupByCommitterKey.values()) {
-      eventsToRemove.addAll(pipeTsFileInsertionEvents);
+      eventsToRemove.addAll(events);
+    }
+
+    // The deque is concurrent, so a source event may be polled while 
replacement is in progress.
+    // Only release events that are still owned by this queue. If a poll wins 
the race, both the
+    // source event and its compacted replacement may be delivered; this 
deliberately favors the
+    // hot-path throughput over strict de-duplication.
+    final Set<PipeTsFileInsertionEvent> removedEvents =
+        
eventsToRemove.stream().filter(tsfileInsertEventDeque::remove).collect(Collectors.toSet());
+    for (final PipeTsFileInsertionEvent event : 
successfullyReferenceIncreasedEvents) {
+      tsfileInsertEventDeque.add(event);
+      eventCounter.increaseEventCount(event);
+    }

Review Comment:
   Is it possible or profitable to replace a sequence in the middle by 
requeuing all events?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to