jt2594838 commented on code in PR #18844:
URL: https://github.com/apache/iotdb/pull/18844#discussion_r4266686934
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ConsensusPrefetchingQueue.java:
##########
@@ -1765,6 +1788,348 @@ private PrefetchRoundResult drivePrefetchOnceInternal()
{
}
}
+ private PrefetchRoundResult driveOrderedWriterReplay(
+ final long expectedSeekGeneration,
+ final int maxWalEntries,
+ final int maxTablets,
+ final long maxBatchBytes,
+ final int batchMaxDelayMs)
+ throws IOException {
+ if (orderedReplayStartSearchIndex == Long.MIN_VALUE) {
+ orderedReplayStartSearchIndex = nextExpectedSearchIndex.get();
+ if (!lingerBatch.isEmpty()) {
+ // A runtime transition can add another active writer while a
single-writer batch lingers.
+ // Put the already reserved batch back into the merge before comparing
the new writer.
+ bufferRealtimeEntry(
+ new PreparedEntry(
+ new ArrayList<>(lingerBatch.tablets),
+ lingerBatch.estimatedBytes,
+ lingerBatch.physicalTime,
+ lingerBatch.writerNodeId,
+ lingerBatch.lastLocalSeq,
+ lingerBatch.endSearchIndex));
+ lingerBatch.reset();
+ }
+ }
+ final Map<Integer, SafeFrontier> safeFrontiers =
+ serverImpl.getWriterSafeFrontierTracker().snapshotSafeFrontiers();
+ activeWriterNodeIds.forEach(this::getOrCreateWriterReplay);
+ safeFrontiers.keySet().forEach(this::getOrCreateWriterReplay);
+ recoveryWriterProgressByWriter.keySet().forEach(id ->
getOrCreateWriterReplay(id.getNodeId()));
+ realtimeEntriesByWriter.keySet().forEach(this::getOrCreateWriterReplay);
+ for (final IndexedConsensusRequest request : pendingEntries) {
+ if (hasComparableWriterProgress(request)) {
+ getOrCreateWriterReplay(request.getNodeId());
+ }
+ }
+
+ final int[] readBudget = {Math.max(1, maxWalEntries)};
+ final int roundRobinStart = orderedWriterRoundRobinStart++;
+ while (prefetchingQueue.size() < MAX_PREFETCHING_QUEUE_SIZE) {
+ if (seekGeneration.get() != expectedSeekGeneration || closeRequested) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ final int writerCount = orderedWriterReplay.size();
+ final List<Map.Entry<Integer, WriterReplayState>> writers =
+ new ArrayList<>(orderedWriterReplay.entrySet());
+ Collections.rotate(writers, Math.floorMod(roundRobinStart, Math.max(1,
writers.size())));
+ for (final Map.Entry<Integer, WriterReplayState> writer : writers) {
+ if (!fillOrderedWriterHead(writer.getKey(), writer.getValue(),
readBudget)) {
+ return flushOrderedBatchAndRetry(expectedSeekGeneration,
MEMORY_RETRY_SLEEP_MS);
+ }
+ }
+ // A scan can discover a retired writer absent from the runtime set.
Acquire its head before
+ // releasing any other writer, with the same shared per-round scan
budget.
+ if (orderedWriterReplay.size() != writerCount && readBudget[0] > 0) {
+ continue;
+ }
+
+ final PriorityQueue<WriterFrontier> frontiers = new PriorityQueue<>();
+ for (final Map.Entry<Integer, WriterReplayState> writer :
orderedWriterReplay.entrySet()) {
+ final int writerNodeId = writer.getKey();
+ final WriterReplayState state = writer.getValue();
+ final PreparedEntry buffered = peekRealtimeEntry(writerNodeId);
+ if (Objects.nonNull(buffered)) {
+ frontiers.add(WriterFrontier.forHead(writerNodeId, buffered));
+ } else if (Objects.nonNull(state.head)) {
+ final IndexedConsensusRequest head = state.head;
+ frontiers.add(
+ new WriterFrontier(
+ writerNodeId,
+ new OrderingKey(head.getPhysicalTime(), writerNodeId,
head.getProgressLocalSeq()),
+ false));
+ } else if (isWriterRuntimeActive(writerNodeId) || !state.caughtUp) {
+ final SafeFrontier safe = safeFrontiers.get(writerNodeId);
+ final long safePt =
+ Objects.nonNull(safe) && state.consumedLocalSeq >=
safe.getLocalSeq()
+ ? safe.getPhysicalTime()
+ : Long.MIN_VALUE;
+ frontiers.add(WriterFrontier.forBarrier(writerNodeId, safePt));
+ }
+ }
+ final WriterFrontier next = frontiers.peek();
+ if (Objects.isNull(next) || next.isBarrier) {
+ break;
+ }
+ final PreparedEntry buffered = peekRealtimeEntry(next.writerNodeId);
+ if (Objects.isNull(buffered) && shouldWaitForSubscriptionMemory()) {
+ blockRealtimeAdmission(memoryBlockReason);
+ return flushOrderedBatchAndRetry(expectedSeekGeneration,
MEMORY_RETRY_SLEEP_MS);
+ }
+ final WriterReplayState state =
orderedWriterReplay.get(next.writerNodeId);
+ final IndexedConsensusRequest request = state.head;
+ final PreparedEntry entry = Objects.nonNull(buffered) ? buffered :
prepareEntry(request);
+ if (Objects.isNull(entry) && memoryBlockedEntryBytes > 0L) {
+ blockRealtimeAdmission(memoryBlockReason);
+ return flushOrderedBatchAndRetry(expectedSeekGeneration,
MEMORY_RETRY_SLEEP_MS);
+ }
+ if (Objects.nonNull(entry)) {
+ if (!canAppendWriterEntry(
+ lingerBatch,
+ entry,
+ entry.estimatedBytes,
+ Integer.MAX_VALUE,
+ maxTablets,
+ maxBatchBytes)
+ && !flushBatch(lingerBatch, expectedSeekGeneration)) {
+ releaseTabletMemory(entry.estimatedBytes);
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ if (entry.tablets.isEmpty()) {
+ if (!commitEmptyWriterEntry(lingerBatch, entry,
expectedSeekGeneration)) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ } else {
+ lingerBatch.append(entry, entry.estimatedBytes, true);
+ }
+ }
+ if (Objects.nonNull(buffered)) {
+ removeRealtimeEntry(buffered.writerNodeId, buffered.localSeq);
+ readBudget[0]--;
+ } else {
+ markMaterializedProgress(request);
+ state.consumedLocalSeq = Math.max(state.consumedLocalSeq,
request.getProgressLocalSeq());
+ if (state.fromPending) {
+ advanceLocalCursorIfPresent(request);
+ if (pendingEntries.remove(request)) {
+ pendingEntries.release(request);
+ }
+ markAcceptedFromPending();
+ } else {
+ advanceWalReplayCursorIfPresent(request);
+ markAcceptedFromWal();
+ }
+ releaseOrderedWriterHead(state);
+ }
+ unblockRealtimeAdmission();
+ if (readBudget[0] <= 0) {
+ break;
+ }
+ }
+
+ // Pending and WAL can carry the same record. Release obsolete realtime
references even when
+ // this round used WAL, rather than letting their bounded queue fill with
already read data.
+ for (final IndexedConsensusRequest request : pendingEntries) {
+ if (shouldSkipForRecoveryProgress(request) ||
shouldSkipForMaterializedProgress(request)) {
+ if (pendingEntries.remove(request)) {
+ pendingEntries.release(request);
+ }
+ }
+ }
+ if (!lingerBatch.isEmpty()
+ && System.currentTimeMillis() - lingerBatch.firstTabletTimeMs >=
batchMaxDelayMs) {
+ if (!flushBatch(lingerBatch, expectedSeekGeneration)) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ }
+ maybeInjectWatermark();
+ if (prefetchingQueue.size() >= MAX_PREFETCHING_QUEUE_SIZE) {
+ return PrefetchRoundResult.dormant();
+ }
+ if (readBudget[0] <= 0
+ || (readBudget[0] < maxWalEntries
+ && orderedWriterReplay.values().stream()
+ .anyMatch(state -> Objects.isNull(state.head) &&
!state.caughtUp))) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ long delayMs = WRITER_BARRIER_RETRY_SLEEP_MS;
+ if (!lingerBatch.isEmpty()) {
+ delayMs =
+ Math.min(
+ delayMs,
+ Math.max(
+ 1L,
+ batchMaxDelayMs - (System.currentTimeMillis() -
lingerBatch.firstTabletTimeMs)));
+ }
+ return PrefetchRoundResult.rescheduleAfter(delayMs);
+ }
+
+ private PrefetchRoundResult flushOrderedBatchAndRetry(
+ final long expectedSeekGeneration, final long delayMs) {
+ if (!lingerBatch.isEmpty() && !flushBatch(lingerBatch,
expectedSeekGeneration)) {
+ return PrefetchRoundResult.rescheduleNow();
+ }
+ return PrefetchRoundResult.rescheduleAfter(delayMs);
+ }
+
+ private WriterReplayState getOrCreateWriterReplay(final int writerNodeId) {
+ return orderedWriterReplay.computeIfAbsent(
+ writerNodeId,
+ ignored -> {
+ final WriterProgress recovered =
+ recoveryWriterProgressByWriter.get(
+ new WriterId(consensusGroupId.toString(), writerNodeId));
+ final WriterProgress materialized =
materializedProgressByWriter.get(writerNodeId);
+ return new WriterReplayState(
+ createSubscriptionWALIterator(Long.MIN_VALUE),
+ Math.max(
+ Objects.nonNull(recovered) ? recovered.getLocalSeq() : 0L,
+ Objects.nonNull(materialized) ? materialized.getLocalSeq() :
0L));
+ });
+ }
+
+ private boolean fillOrderedWriterHead(
+ final int writerNodeId, final WriterReplayState state, final int[]
readBudget)
+ throws IOException {
+ if (Objects.nonNull(state.head) || readBudget[0] <= 0) {
+ return true;
+ }
+ for (final IndexedConsensusRequest request : pendingEntries) {
+ if (shouldSkipForRecoveryProgress(request)
+ || shouldSkipForMaterializedProgress(request)
+ || (hasLocalSearchIndex(request)
+ && request.getSearchIndex() < orderedReplayStartSearchIndex)) {
+ if (pendingEntries.remove(request)) {
+ pendingEntries.release(request);
+ }
+ continue;
+ }
+ if (request.getNodeId() == writerNodeId
+ && !shouldSkipForRecoveryProgress(request)
+ && !shouldSkipForMaterializedProgress(request)
+ && (Objects.isNull(state.iterator)
+ || request.getProgressLocalSeq() == state.consumedLocalSeq +
1L)) {
+ state.head = request;
+ state.fromPending = true;
+ state.caughtUp = false;
+ readBudget[0]--;
+ return true;
+ }
+ }
+ if (Objects.isNull(state.iterator)) {
+ state.caughtUp = true;
+ return true;
+ }
+
state.iterator.skipReportedBrokenWalFiles(reportedOrderedBrokenWalVersions);
+ final int writerBudget = Math.max(1, readBudget[0] / Math.max(1,
orderedWriterReplay.size()));
+ for (int scanned = 0; scanned < writerBudget && readBudget[0] > 0;
scanned++) {
+ if (closeRequested || pendingSeekRequest != null) {
+ return true;
+ }
+ if (!state.iterator.hasNext()) {
+ state.iterator.refresh();
+ if (!state.iterator.hasNext()) {
+ reportOrderedUnreadableWal(state.iterator);
+ state.caughtUp =
+ !state.iterator.hasReadError() &&
!state.iterator.isWaitingForReadableWal();
+ return true;
+ }
+ }
+ reportOrderedUnreadableWal(state.iterator);
+ final IndexedConsensusRequest request = state.iterator.next();
+ readBudget[0]--;
+ state.caughtUp = false;
+ if (hasComparableWriterProgress(request)) {
+ getOrCreateWriterReplay(request.getNodeId());
+ }
+ if (request.getNodeId() != writerNodeId) {
+ continue;
+ }
+ if ((hasLocalSearchIndex(request) && request.getSearchIndex() <
orderedReplayStartSearchIndex)
+ || shouldSkipForRecoveryProgress(request)
+ || shouldSkipForMaterializedProgress(request)) {
+ state.consumedLocalSeq = Math.max(state.consumedLocalSeq,
request.getProgressLocalSeq());
+ continue;
+ }
+ // WAL requests wrap ByteBuffers whose default getMemorySize() is zero.
Build the buffer list
+ // before reserving the retained head so consensus accounting includes
its actual payload.
+ request.buildSerializedRequests();
+ if (!IoTConsensusMemoryManager.getInstance().reserve(request)) {
+ state.iterator.retry(request);
+ return false;
+ }
+ orderedRequestBytes.addAndGet(request.getRetainedMemorySize());
+ state.head = request;
+ state.fromPending = false;
+ return true;
Review Comment:
Why is the budget reduced even for skipped entry?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]