Caideyipi commented on code in PR #18726:
URL: https://github.com/apache/iotdb/pull/18726#discussion_r4534079786
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java:
##########
@@ -249,71 +262,59 @@ public synchronized void replace(
eventsToBeAddedGroupByCommitterKey.put(committerKey, newEvents);
}
- // Handling new events
final Set<PipeTsFileInsertionEvent> successfullyReferenceIncreasedEvents =
new HashSet<>();
- final AtomicBoolean
- allSuccess = // To track if all events successfully increased the
reference count
- new AtomicBoolean(true);
+ boolean allSuccess = true;
outerLoop:
- for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>>
committerKeySetEntry :
- eventsToBeAddedGroupByCommitterKey.entrySet()) {
- for (final PipeTsFileInsertionEvent event :
committerKeySetEntry.getValue()) {
- if (event != null) {
- try {
- if
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
{
- allSuccess.set(false);
- break outerLoop;
- } else {
- successfullyReferenceIncreasedEvents.add(event);
- }
- } catch (final Exception e) {
- allSuccess.set(false);
+ for (final Set<PipeTsFileInsertionEvent> events :
eventsToBeAddedGroupByCommitterKey.values()) {
+ for (final PipeTsFileInsertionEvent event : events) {
+ try {
+ if
(!event.increaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName()))
{
+ allSuccess = false;
break outerLoop;
}
+ successfullyReferenceIncreasedEvents.add(event);
+ } catch (final Exception e) {
+ allSuccess = false;
+ break outerLoop;
}
}
}
- if (!allSuccess.get()) {
- // If any event failed to increase the reference count,
- // we need to decrease the reference count for all successfully
increased events
+ if (!allSuccess) {
for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
try {
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(),
false);
} catch (final Exception e) {
LOGGER.warn(DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT,
event, e);
}
}
- return; // Exit early if any event failed to increase the reference count
- } else {
- // If all events successfully increased reference count,
- // we can proceed to add them to the deque
- for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
- tsfileInsertEventDeque.add(event);
- eventCounter.increaseEventCount(event);
- }
+ return;
}
- // Handling old events
- for (final Map.Entry<CommitterKey, Set<PipeTsFileInsertionEvent>> entry :
- eventsToBeRemovedGroupByCommitterKey.entrySet()) {
- for (final PipeTsFileInsertionEvent event : entry.getValue()) {
- if (event != null) {
- try {
-
event.decreaseReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName(),
false);
- } catch (final Exception e) {
- LOGGER.warn(
-
DataNodePipeMessages.FAILED_TO_DECREASE_REFERENCE_COUNT_FOR_EVENT, event, e);
- }
- eventCounter.decreaseEventCount(event);
- }
- }
- }
final Set<PipeTsFileInsertionEvent> eventsToRemove = new HashSet<>();
- for (Set<PipeTsFileInsertionEvent> pipeTsFileInsertionEvents :
+ for (final Set<PipeTsFileInsertionEvent> events :
eventsToBeRemovedGroupByCommitterKey.values()) {
- eventsToRemove.addAll(pipeTsFileInsertionEvents);
+ eventsToRemove.addAll(events);
+ }
+
+ // The deque is concurrent, so a source event may be polled while
replacement is in progress.
+ // Only release events that are still owned by this queue. If a poll wins
the race, both the
+ // source event and its compacted replacement may be delivered; this
deliberately favors the
+ // hot-path throughput over strict de-duplication.
+ final Set<PipeTsFileInsertionEvent> removedEvents =
+
eventsToRemove.stream().filter(tsfileInsertEventDeque::remove).collect(Collectors.toSet());
+ for (final PipeTsFileInsertionEvent event :
successfullyReferenceIncreasedEvents) {
+ tsfileInsertEventDeque.add(event);
+ eventCounter.increaseEventCount(event);
+ }
Review Comment:
Yes, appending the replacement changes its position relative to the
remaining files. Fixed in eae842778e3001a0188900d0866bc0a7d467dd7f: replacement
now requires the complete oldest queued source prefix for the same committer,
and inserts the target files at the front in their original order. For example,
`[f0,f1,f2,f3]` with `f0,f1 -> g` becomes `[g,f2,f3]`. If an unrelated older
file is interleaved for that committer, the original events stay queued. Added
regressions for multiple target files, a non-prefix source group, and
independent committers; all 8 queue tests pass. The existing two-ended
real-time scheduling and the documented poll/replacement race tradeoff remain.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]