Caideyipi commented on code in PR #18844:
URL: https://github.com/apache/iotdb/pull/18844#discussion_r4268161477


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java:
##########
@@ -1765,6 +1788,348 @@ private PrefetchRoundResult drivePrefetchOnceInternal() 
{
     }
   }
 
+  private PrefetchRoundResult driveOrderedWriterReplay(
+      final long expectedSeekGeneration,
+      final int maxWalEntries,
+      final int maxTablets,
+      final long maxBatchBytes,
+      final int batchMaxDelayMs)
+      throws IOException {
+    if (orderedReplayStartSearchIndex == Long.MIN_VALUE) {
+      orderedReplayStartSearchIndex = nextExpectedSearchIndex.get();
+      if (!lingerBatch.isEmpty()) {
+        // A runtime transition can add another active writer while a 
single-writer batch lingers.
+        // Put the already reserved batch back into the merge before comparing 
the new writer.
+        bufferRealtimeEntry(
+            new PreparedEntry(
+                new ArrayList<>(lingerBatch.tablets),
+                lingerBatch.estimatedBytes,
+                lingerBatch.physicalTime,
+                lingerBatch.writerNodeId,
+                lingerBatch.lastLocalSeq,
+                lingerBatch.endSearchIndex));
+        lingerBatch.reset();
+      }
+    }
+    final Map<Integer, SafeFrontier> safeFrontiers =
+        serverImpl.getWriterSafeFrontierTracker().snapshotSafeFrontiers();
+    activeWriterNodeIds.forEach(this::getOrCreateWriterReplay);
+    safeFrontiers.keySet().forEach(this::getOrCreateWriterReplay);
+    recoveryWriterProgressByWriter.keySet().forEach(id -> 
getOrCreateWriterReplay(id.getNodeId()));
+    realtimeEntriesByWriter.keySet().forEach(this::getOrCreateWriterReplay);
+    for (final IndexedConsensusRequest request : pendingEntries) {
+      if (hasComparableWriterProgress(request)) {
+        getOrCreateWriterReplay(request.getNodeId());
+      }
+    }
+
+    final int[] readBudget = {Math.max(1, maxWalEntries)};
+    final int roundRobinStart = orderedWriterRoundRobinStart++;
+    while (prefetchingQueue.size() < MAX_PREFETCHING_QUEUE_SIZE) {
+      if (seekGeneration.get() != expectedSeekGeneration || closeRequested) {
+        return PrefetchRoundResult.rescheduleNow();
+      }
+      final int writerCount = orderedWriterReplay.size();
+      final List<Map.Entry<Integer, WriterReplayState>> writers =
+          new ArrayList<>(orderedWriterReplay.entrySet());
+      Collections.rotate(writers, Math.floorMod(roundRobinStart, Math.max(1, 
writers.size())));
+      for (final Map.Entry<Integer, WriterReplayState> writer : writers) {
+        if (!fillOrderedWriterHead(writer.getKey(), writer.getValue(), 
readBudget)) {
+          return flushOrderedBatchAndRetry(expectedSeekGeneration, 
MEMORY_RETRY_SLEEP_MS);
+        }
+      }
+      // A scan can discover a retired writer absent from the runtime set. 
Acquire its head before
+      // releasing any other writer, with the same shared per-round scan 
budget.
+      if (orderedWriterReplay.size() != writerCount && readBudget[0] > 0) {
+        continue;
+      }
+
+      final PriorityQueue<WriterFrontier> frontiers = new PriorityQueue<>();
+      for (final Map.Entry<Integer, WriterReplayState> writer : 
orderedWriterReplay.entrySet()) {
+        final int writerNodeId = writer.getKey();
+        final WriterReplayState state = writer.getValue();
+        final PreparedEntry buffered = peekRealtimeEntry(writerNodeId);
+        if (Objects.nonNull(buffered)) {
+          frontiers.add(WriterFrontier.forHead(writerNodeId, buffered));
+        } else if (Objects.nonNull(state.head)) {
+          final IndexedConsensusRequest head = state.head;
+          frontiers.add(
+              new WriterFrontier(
+                  writerNodeId,
+                  new OrderingKey(head.getPhysicalTime(), writerNodeId, 
head.getProgressLocalSeq()),
+                  false));
+        } else if (isWriterRuntimeActive(writerNodeId) || !state.caughtUp) {
+          final SafeFrontier safe = safeFrontiers.get(writerNodeId);
+          final long safePt =
+              Objects.nonNull(safe) && state.consumedLocalSeq >= 
safe.getLocalSeq()
+                  ? safe.getPhysicalTime()
+                  : Long.MIN_VALUE;
+          frontiers.add(WriterFrontier.forBarrier(writerNodeId, safePt));
+        }
+      }
+      final WriterFrontier next = frontiers.peek();
+      if (Objects.isNull(next) || next.isBarrier) {
+        break;
+      }
+      final PreparedEntry buffered = peekRealtimeEntry(next.writerNodeId);
+      if (Objects.isNull(buffered) && shouldWaitForSubscriptionMemory()) {
+        blockRealtimeAdmission(memoryBlockReason);
+        return flushOrderedBatchAndRetry(expectedSeekGeneration, 
MEMORY_RETRY_SLEEP_MS);
+      }
+      final WriterReplayState state = 
orderedWriterReplay.get(next.writerNodeId);
+      final IndexedConsensusRequest request = state.head;
+      final PreparedEntry entry = Objects.nonNull(buffered) ? buffered : 
prepareEntry(request);
+      if (Objects.isNull(entry) && memoryBlockedEntryBytes > 0L) {
+        blockRealtimeAdmission(memoryBlockReason);
+        return flushOrderedBatchAndRetry(expectedSeekGeneration, 
MEMORY_RETRY_SLEEP_MS);
+      }
+      if (Objects.nonNull(entry)) {
+        if (!canAppendWriterEntry(
+                lingerBatch,
+                entry,
+                entry.estimatedBytes,
+                Integer.MAX_VALUE,
+                maxTablets,
+                maxBatchBytes)
+            && !flushBatch(lingerBatch, expectedSeekGeneration)) {
+          releaseTabletMemory(entry.estimatedBytes);
+          return PrefetchRoundResult.rescheduleNow();
+        }
+        if (entry.tablets.isEmpty()) {
+          if (!commitEmptyWriterEntry(lingerBatch, entry, 
expectedSeekGeneration)) {
+            return PrefetchRoundResult.rescheduleNow();
+          }
+        } else {
+          lingerBatch.append(entry, entry.estimatedBytes, true);
+        }
+      }
+      if (Objects.nonNull(buffered)) {
+        removeRealtimeEntry(buffered.writerNodeId, buffered.localSeq);
+        readBudget[0]--;
+      } else {
+        markMaterializedProgress(request);
+        state.consumedLocalSeq = Math.max(state.consumedLocalSeq, 
request.getProgressLocalSeq());
+        if (state.fromPending) {
+          advanceLocalCursorIfPresent(request);
+          if (pendingEntries.remove(request)) {
+            pendingEntries.release(request);
+          }
+          markAcceptedFromPending();
+        } else {
+          advanceWalReplayCursorIfPresent(request);
+          markAcceptedFromWal();
+        }
+        releaseOrderedWriterHead(state);
+      }
+      unblockRealtimeAdmission();
+      if (readBudget[0] <= 0) {
+        break;
+      }
+    }
+
+    // Pending and WAL can carry the same record. Release obsolete realtime 
references even when
+    // this round used WAL, rather than letting their bounded queue fill with 
already read data.
+    for (final IndexedConsensusRequest request : pendingEntries) {
+      if (shouldSkipForRecoveryProgress(request) || 
shouldSkipForMaterializedProgress(request)) {
+        if (pendingEntries.remove(request)) {
+          pendingEntries.release(request);
+        }
+      }
+    }
+    if (!lingerBatch.isEmpty()
+        && System.currentTimeMillis() - lingerBatch.firstTabletTimeMs >= 
batchMaxDelayMs) {
+      if (!flushBatch(lingerBatch, expectedSeekGeneration)) {
+        return PrefetchRoundResult.rescheduleNow();
+      }
+    }
+    maybeInjectWatermark();
+    if (prefetchingQueue.size() >= MAX_PREFETCHING_QUEUE_SIZE) {
+      return PrefetchRoundResult.dormant();
+    }
+    if (readBudget[0] <= 0
+        || (readBudget[0] < maxWalEntries
+            && orderedWriterReplay.values().stream()
+                .anyMatch(state -> Objects.isNull(state.head) && 
!state.caughtUp))) {
+      return PrefetchRoundResult.rescheduleNow();
+    }
+    long delayMs = WRITER_BARRIER_RETRY_SLEEP_MS;
+    if (!lingerBatch.isEmpty()) {
+      delayMs =
+          Math.min(
+              delayMs,
+              Math.max(
+                  1L,
+                  batchMaxDelayMs - (System.currentTimeMillis() - 
lingerBatch.firstTabletTimeMs)));
+    }
+    return PrefetchRoundResult.rescheduleAfter(delayMs);
+  }
+
+  private PrefetchRoundResult flushOrderedBatchAndRetry(
+      final long expectedSeekGeneration, final long delayMs) {
+    if (!lingerBatch.isEmpty() && !flushBatch(lingerBatch, 
expectedSeekGeneration)) {
+      return PrefetchRoundResult.rescheduleNow();
+    }
+    return PrefetchRoundResult.rescheduleAfter(delayMs);
+  }
+
+  private WriterReplayState getOrCreateWriterReplay(final int writerNodeId) {
+    return orderedWriterReplay.computeIfAbsent(
+        writerNodeId,
+        ignored -> {
+          final WriterProgress recovered =
+              recoveryWriterProgressByWriter.get(
+                  new WriterId(consensusGroupId.toString(), writerNodeId));
+          final WriterProgress materialized = 
materializedProgressByWriter.get(writerNodeId);
+          return new WriterReplayState(
+              createSubscriptionWALIterator(Long.MIN_VALUE),
+              Math.max(
+                  Objects.nonNull(recovered) ? recovered.getLocalSeq() : 0L,
+                  Objects.nonNull(materialized) ? materialized.getLocalSeq() : 
0L));
+        });
+  }
+
+  private boolean fillOrderedWriterHead(
+      final int writerNodeId, final WriterReplayState state, final int[] 
readBudget)
+      throws IOException {
+    if (Objects.nonNull(state.head) || readBudget[0] <= 0) {
+      return true;
+    }
+    for (final IndexedConsensusRequest request : pendingEntries) {
+      if (shouldSkipForRecoveryProgress(request)
+          || shouldSkipForMaterializedProgress(request)
+          || (hasLocalSearchIndex(request)
+              && request.getSearchIndex() < orderedReplayStartSearchIndex)) {
+        if (pendingEntries.remove(request)) {
+          pendingEntries.release(request);
+        }
+        continue;
+      }
+      if (request.getNodeId() == writerNodeId
+          && !shouldSkipForRecoveryProgress(request)
+          && !shouldSkipForMaterializedProgress(request)
+          && (Objects.isNull(state.iterator)
+              || request.getProgressLocalSeq() == state.consumedLocalSeq + 
1L)) {
+        state.head = request;
+        state.fromPending = true;
+        state.caughtUp = false;
+        readBudget[0]--;
+        return true;
+      }
+    }
+    if (Objects.isNull(state.iterator)) {
+      state.caughtUp = true;
+      return true;
+    }
+    
state.iterator.skipReportedBrokenWalFiles(reportedOrderedBrokenWalVersions);
+    final int writerBudget = Math.max(1, readBudget[0] / Math.max(1, 
orderedWriterReplay.size()));
+    for (int scanned = 0; scanned < writerBudget && readBudget[0] > 0; 
scanned++) {
+      if (closeRequested || pendingSeekRequest != null) {
+        return true;
+      }
+      if (!state.iterator.hasNext()) {
+        state.iterator.refresh();
+        if (!state.iterator.hasNext()) {
+          reportOrderedUnreadableWal(state.iterator);
+          state.caughtUp =
+              !state.iterator.hasReadError() && 
!state.iterator.isWaitingForReadableWal();
+          return true;
+        }
+      }
+      reportOrderedUnreadableWal(state.iterator);
+      final IndexedConsensusRequest request = state.iterator.next();
+      readBudget[0]--;
+      state.caughtUp = false;
+      if (hasComparableWriterProgress(request)) {
+        getOrCreateWriterReplay(request.getNodeId());
+      }
+      if (request.getNodeId() != writerNodeId) {
+        continue;
+      }
+      if ((hasLocalSearchIndex(request) && request.getSearchIndex() < 
orderedReplayStartSearchIndex)
+          || shouldSkipForRecoveryProgress(request)
+          || shouldSkipForMaterializedProgress(request)) {
+        state.consumedLocalSeq = Math.max(state.consumedLocalSeq, 
request.getProgressLocalSeq());
+        continue;
+      }
+      // WAL requests wrap ByteBuffers whose default getMemorySize() is zero. 
Build the buffer list
+      // before reserving the retained head so consensus accounting includes 
its actual payload.
+      request.buildSerializedRequests();
+      if (!IoTConsensusMemoryManager.getInstance().reserve(request)) {
+        state.iterator.retry(request);
+        return false;
+      }
+      orderedRequestBytes.addAndGet(request.getRetainedMemorySize());
+      state.head = request;
+      state.fromPending = false;
+      return true;

Review Comment:
   This budget bounds entry-processing work per prefetch round and is shared by 
all writer readers. Each reader traverses the same WAL, so a request skipped 
for another writer or already consumed progress has still incurred a 
read/decode. Counting only retained requests would let a long skipped prefix 
exceed the round's work limit. When the budget runs out, the persistent readers 
keep their positions and unfinished scans reschedule immediately.
   
   Clarified this in f30b5c5aeaaec4740e1387c98515e954b113bbde: renamed 
`readBudget` to `remainingEntryBudget` and added a comment beside the 
decrement. The existing writer-ordering regression verifies at most 32 WAL 
reads per round while finding the older writer beyond a 1024-entry prefix. All 
12 targeted ordering/frontier tests passed; Spotless and `git diff --check` 
also passed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to