This is an automated email from the ASF dual-hosted git repository. Caideyipi pushed a commit to branch codex/subscription-wal-refresh-rescan-pr in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ea4639bf11ab5e104838ff3deaef1c4beba218bb Author: Caideyipi <[email protected]> AuthorDate: Thu Nov 5 12:20:53 2026 +0800 Reduce WAL directory scans during subscription replay refresh --- .../broker/consensus/ProgressWALIterator.java | 98 ++++++- .../broker/consensus/ProgressWALIteratorTest.java | 325 ++++++++++++++++++++- 2 files changed, 409 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIterator.java index 5122ec86eeb..e1246720838 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIterator.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.subscription.broker.consensus; import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest; import org.apache.iotdb.consensus.common.request.IoTConsensusRequest; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -48,6 +49,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -56,6 +58,7 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.LongSupplier; /** @@ -66,6 +69,8 @@ import java.util.function.LongSupplier; */ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensusRequest> { + private record VersionedWalFile(File file, long versionId) {} + @FunctionalInterface public interface WriterProgressCoverage { @@ -115,6 +120,7 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus private static final Logger LOGGER = LoggerFactory.getLogger(ProgressWALIterator.class); private static final int MAX_SKIPPED_FILE_DETAILS = 8; + private static final long WAL_FILE_LIST_REFRESH_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(1L); private static final int SEARCH_INDEX_OFFSET = WALInfoEntry.FIXED_SERIALIZED_SIZE + PlanNodeType.BYTES; @@ -125,8 +131,14 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus private final File logDirectory; private long minimumSearchIndex; private final WALNode liveWalNode; + private final LongSupplier nanoTime; private File[] walFiles; private long[] walFileVersionIds; + private long walDirectoryLastModified = Long.MIN_VALUE; + private long liveWalFileVersion = Long.MIN_VALUE; + private long walFileListRefreshNanos; + private boolean walFileListDirty = true; + private boolean walFileListRefreshFailed; private int currentFileIndex = -1; private ProgressWALReader currentReader; private File currentReaderFile; @@ -177,27 +189,64 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus private ProgressWALIterator( final File logDirectory, final long startSearchIndex, final WALNode liveWalNode) { + this(logDirectory, startSearchIndex, liveWalNode, System::nanoTime); + } + + @TestOnly + ProgressWALIterator( + final File logDirectory, + final long startSearchIndex, + final WALNode liveWalNode, + final LongSupplier nanoTime) { this.logDirectory = logDirectory; this.minimumSearchIndex = startSearchIndex; this.liveWalNode = liveWalNode; + this.nanoTime = nanoTime; refreshFileList(); } private void refreshFileList() { + final long observedDirectoryLastModified = logDirectory.lastModified(); + final long observedLiveWalFileVersion = + liveWalNode == null ? -1L : liveWalNode.getCurrentWALFileVersion(); + final long nowNanos = nanoTime.getAsLong(); + if (!walFileListDirty + && walFiles != null + && observedDirectoryLastModified > 0 + && observedDirectoryLastModified == walDirectoryLastModified + && observedLiveWalFileVersion == liveWalFileVersion + && nowNanos - walFileListRefreshNanos < WAL_FILE_LIST_REFRESH_INTERVAL_NANOS) { + return; + } + final File[] discoveredWalFiles = WALFileUtils.listAllWALFiles(logDirectory); if (discoveredWalFiles == null) { - walFiles = new File[0]; - walFileVersionIds = new long[0]; + // A failed listing says nothing about the known files or the replay cursor. + if (walFiles == null) { + walFiles = new File[0]; + walFileVersionIds = new long[0]; + } + walFileListDirty = true; + walFileListRefreshFailed = true; return; } - WALFileUtils.ascSortByVersionId(discoveredWalFiles); + + final VersionedWalFile[] sortedWalFiles = new VersionedWalFile[discoveredWalFiles.length]; + for (int i = 0; i < discoveredWalFiles.length; i++) { + final File walFile = discoveredWalFiles[i]; + sortedWalFiles[i] = + new VersionedWalFile(walFile, WALFileUtils.parseVersionId(walFile.getName())); + } + Arrays.sort(sortedWalFiles, Comparator.comparingLong(VersionedWalFile::versionId)); + final File[] filteredWalFiles = new File[discoveredWalFiles.length]; final long[] filteredWalFileVersionIds = new long[discoveredWalFiles.length]; int filteredWalFileCount = 0; - for (int i = 0; i < discoveredWalFiles.length; i++) { - final File walFile = discoveredWalFiles[i]; - final long versionId = WALFileUtils.parseVersionId(walFile.getName()); - final boolean isLastWalFile = i == discoveredWalFiles.length - 1; + for (int i = 0; i < sortedWalFiles.length; i++) { + final VersionedWalFile versionedWalFile = sortedWalFiles[i]; + final File walFile = versionedWalFile.file(); + final long versionId = versionedWalFile.versionId(); + final boolean isLastWalFile = i == sortedWalFiles.length - 1; if (!isLastWalFile && shouldSkipWalFile(walFile, versionId)) { continue; } @@ -207,6 +256,16 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus } walFiles = Arrays.copyOf(filteredWalFiles, filteredWalFileCount); walFileVersionIds = Arrays.copyOf(filteredWalFileVersionIds, filteredWalFileCount); + updateWalFileListRefreshState(observedDirectoryLastModified, observedLiveWalFileVersion); + } + + private void updateWalFileListRefreshState( + final long directoryLastModified, final long currentWalFileVersion) { + walDirectoryLastModified = directoryLastModified; + liveWalFileVersion = currentWalFileVersion; + walFileListRefreshNanos = nanoTime.getAsLong(); + walFileListDirty = false; + walFileListRefreshFailed = false; } private boolean shouldSkipWalFile(final File walFile, final long versionId) { @@ -498,6 +557,7 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus pendingLocalSeq = Long.MIN_VALUE; } if (skippedBrokenWalVersionIds.addAll(versionIds)) { + walFileListDirty = true; refresh(); } if (skipsCurrentFile) { @@ -506,7 +566,7 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus } boolean isWaitingForReadableWal() { - return retryCurrentReader; + return retryCurrentReader || walFileListRefreshFailed; } @Override @@ -566,7 +626,10 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus } public boolean hasIncompleteScan() { - return incompleteScan || hasReadError() || hasSkippedBrokenWalFiles(); + return walFileListRefreshFailed + || incompleteScan + || hasReadError() + || hasSkippedBrokenWalFiles(); } public String getIncompleteScanDetail() { @@ -594,6 +657,7 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus incompleteScan = false; incompleteScanDetail = null; scanYielded = false; + walFileListRefreshFailed = false; resetCurrentFileTracking(); } @@ -895,10 +959,16 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus final File walFile = walFiles[fileIndex]; final long versionId = walFileVersionIds[fileIndex]; currentFileIndex = fileIndex; + if (skippedBrokenWalVersionIds.contains(versionId)) { + // A failed directory refresh may have retained this version in the cached file table. + resetCurrentFileTracking(); + return false; + } currentReaderFile = walFile; readerOpenAttempts++; - final boolean useLiveSnapshot = - liveWalNode != null && versionId == liveWalNode.getCurrentWALFileVersion(); + final long currentLiveVersionId = + liveWalNode == null ? -1L : liveWalNode.getCurrentWALFileVersion(); + final boolean useLiveSnapshot = liveWalNode != null && versionId == currentLiveVersionId; ProgressWALReader reader = null; try { @@ -908,7 +978,10 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus : liveMetaDataSnapshot != null ? liveMetaDataSnapshot : liveWalNode.getWALMetaDataSnapshot(versionId); - if (useLiveSnapshot && snapshot == null) { + // Rotation creates the successor header before publishing its version. Reading it as a + // sealed empty WAL would move the cursor past entries appended after publication. + if ((liveWalNode != null && versionId > currentLiveVersionId) + || (useLiveSnapshot && snapshot == null)) { markIncompleteScan( String.format( DataNodePipeMessages @@ -1004,6 +1077,7 @@ public class ProgressWALIterator implements Closeable, Iterator<IndexedConsensus if (!skippedBrokenWalVersionIds.add(versionId)) { return; } + walFileListDirty = true; final int entryCount = metadata == null ? -1 : metadata.getBuffersSize().size(); final long skippedEntries = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIteratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIteratorTest.java index e2d807614e8..033d1fa9888 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIteratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/broker/consensus/ProgressWALIteratorTest.java @@ -43,11 +43,14 @@ import org.junit.Test; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FilenameFilter; import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.junit.Assert.assertEquals; @@ -502,6 +505,7 @@ public class ProgressWALIteratorTest { @Test public void testIteratorAggregatesUnreadableRetainedWalFiles() throws Exception { final Path dir = Files.createTempDirectory("progress-wal-iterator-unreadable-files"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); final File firstBrokenWal = dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) .toFile(); @@ -527,7 +531,8 @@ public class ProgressWALIteratorTest { writer.write(searchableEntry(2L), singleEntryMeta(19, 2L, 1L, 200L, 7, 2L)); } - try (ProgressWALIterator iterator = new ProgressWALIterator(dir.toFile(), Long.MIN_VALUE)) { + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, Long.MIN_VALUE, null, () -> 0L)) { assertTrue(iterator.hasNext()); // A continuously readable iterator must log every skip before reaching exhaustion. assertEquals(2, appender.list.size()); @@ -548,6 +553,9 @@ public class ProgressWALIteratorTest { assertEquals(2, iterator.getSkippedBrokenWalFileCount()); assertTrue(iterator.hasIncompleteScan()); assertFalse(iterator.hasReadError()); + iterator.refresh(); + assertEquals(2, countingDirectory.getListCalls()); + assertFalse(iterator.hasNext()); } } finally { logger.detachAppender(appender); @@ -1375,6 +1383,7 @@ public class ProgressWALIteratorTest { @Test public void testRefreshAfterExhaustionDiscoversNextWalFile() throws Exception { final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-after-exhaustion"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); final File firstWal = dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) .toFile(); @@ -1387,15 +1396,21 @@ public class ProgressWALIteratorTest { writer.write(searchableEntry(1L), singleEntryMeta(19, 1L, 1L, 100L, 7, 1L)); } - try (ProgressWALIterator iterator = new ProgressWALIterator(dir.toFile(), 1L)) { + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, null, () -> 0L)) { + assertEquals(1, countingDirectory.getListCalls()); assertTrue(iterator.hasNext()); assertEquals(1L, iterator.next().getSearchIndex()); assertFalse(iterator.hasNext()); + iterator.refresh(); + assertEquals(1, countingDirectory.getListCalls()); try (WALWriter writer = new WALWriter(secondWal, WALFileVersion.V3)) { writer.write(searchableEntry(2L), singleEntryMeta(19, 2L, 1L, 200L, 7, 2L)); } + countingDirectory.markModified(); iterator.refresh(); + assertEquals(2, countingDirectory.getListCalls()); assertTrue(iterator.hasNext()); assertEquals(2L, iterator.next().getSearchIndex()); @@ -1407,6 +1422,272 @@ public class ProgressWALIteratorTest { } } + @Test + public void testRefreshRetriesAfterWalDirectoryListingFailure() throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-list-failure"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final File walFile = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter writer = new WALWriter(walFile, WALFileVersion.V3)) { + writer.write(searchableEntry(1L), singleEntryMeta(19, 1L, 1L, 100L, 7, 1L)); + } + + countingDirectory.failNextList(); + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, null, () -> 0L)) { + assertEquals(1, countingDirectory.getListCalls()); + assertFalse(iterator.hasNext()); + assertTrue(iterator.hasIncompleteScan()); + assertTrue(iterator.isWaitingForReadableWal()); + iterator.refresh(); + assertEquals(2, countingDirectory.getListCalls()); + assertFalse(iterator.hasIncompleteScan()); + assertFalse(iterator.isWaitingForReadableWal()); + assertTrue(iterator.hasNext()); + assertEquals(1L, iterator.next().getSearchIndex()); + } + } finally { + Files.deleteIfExists(walFile.toPath()); + Files.deleteIfExists(dir); + } + } + + @Test + public void testRefreshFailurePreservesCurrentReaderPosition() throws Exception { + assertRefreshFailurePreservesPosition(false); + } + + @Test + public void testRefreshFailurePreservesExhaustedCursor() throws Exception { + assertRefreshFailurePreservesPosition(true); + } + + private void assertRefreshFailurePreservesPosition(final boolean exhausted) throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-failure-cursor"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final File firstWal = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File secondWal = + dir.resolve(WALFileUtils.getLogFileName(1, 2, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter writer = new WALWriter(firstWal, WALFileVersion.V3)) { + writer.write(searchableEntry(1L), singleEntryMeta(19, 1L, 1L, 100L, 7, 1L)); + writer.write(searchableEntry(2L), singleEntryMeta(19, 2L, 1L, 200L, 7, 2L)); + } + try (WALWriter writer = new WALWriter(secondWal, WALFileVersion.V3)) { + writer.write(searchableEntry(3L), singleEntryMeta(19, 3L, 1L, 300L, 7, 3L)); + } + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, null, () -> 0L)) { + assertEquals(1L, iterator.next().getSearchIndex()); + if (exhausted) { + assertEquals(2L, iterator.next().getSearchIndex()); + assertEquals(3L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + } + + countingDirectory.markModified(); + countingDirectory.failNextList(); + iterator.refresh(); + assertEquals(2, countingDirectory.getListCalls()); + assertTrue(iterator.hasIncompleteScan()); + assertTrue(iterator.isWaitingForReadableWal()); + iterator.refresh(); + assertEquals(3, countingDirectory.getListCalls()); + assertFalse(iterator.isWaitingForReadableWal()); + if (!exhausted) { + assertEquals(2L, iterator.next().getSearchIndex()); + assertEquals(3L, iterator.next().getSearchIndex()); + } + assertFalse(iterator.hasNext()); + assertFalse(iterator.hasIncompleteScan()); + } + } finally { + Files.deleteIfExists(firstWal.toPath()); + Files.deleteIfExists(secondWal.toPath()); + Files.deleteIfExists(dir); + } + } + + @Test + public void testRefreshDiscoversWalAfterCacheExpiryWithUnchangedDirectoryTimestamp() + throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-expiry"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final AtomicLong clock = new AtomicLong(); + final File firstWal = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File secondWal = + dir.resolve(WALFileUtils.getLogFileName(1, 1, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter writer = new WALWriter(firstWal, WALFileVersion.V3)) { + writer.write(searchableEntry(1L), singleEntryMeta(19, 1L, 1L, 100L, 7, 1L)); + } + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, null, clock::get)) { + assertEquals(1L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + try (WALWriter writer = new WALWriter(secondWal, WALFileVersion.V3)) { + writer.write(searchableEntry(2L), singleEntryMeta(19, 2L, 1L, 200L, 7, 2L)); + } + clock.set(TimeUnit.SECONDS.toNanos(1L) - 1L); + iterator.refresh(); + assertEquals(1, countingDirectory.getListCalls()); + assertFalse(iterator.hasNext()); + clock.incrementAndGet(); + iterator.refresh(); + assertEquals(2, countingDirectory.getListCalls()); + assertEquals(2L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + } + } finally { + Files.deleteIfExists(firstWal.toPath()); + Files.deleteIfExists(secondWal.toPath()); + Files.deleteIfExists(dir); + } + } + + @Test + public void testLiveRotationRefreshesHeaderOnlyWalWithUnchangedDirectoryTimestamp() + throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-rotation"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final File emptyWal = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File successorWal = + dir.resolve(WALFileUtils.getLogFileName(1, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter ignored = new WALWriter(emptyWal, WALFileVersion.V3)) { + // An empty active WAL must remain in the cache until rotation seals it. + } + final WALNode walNode = mock(WALNode.class); + when(walNode.getCurrentWALFileVersion()).thenReturn(0L); + when(walNode.getWALMetaDataSnapshot(0L)).thenReturn(new WALMetaData()); + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, walNode, () -> 0L)) { + assertFalse(iterator.hasNext()); + iterator.refresh(); + assertEquals(1, countingDirectory.getListCalls()); + + final WALMetaData metadata = singleEntryMeta(19, 1L, 1L, 100L, 7, 1L); + try (WALWriter writer = new WALWriter(successorWal, WALFileVersion.V3)) { + writer.write(searchableEntry(1L), metadata); + } + when(walNode.getCurrentWALFileVersion()).thenReturn(1L); + when(walNode.getWALMetaDataSnapshot(1L)).thenReturn(metadata); + assertTrue(iterator.hasNext()); + assertEquals(2, countingDirectory.getListCalls()); + assertEquals(1L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + assertFalse(iterator.hasIncompleteScan()); + } + } finally { + Files.deleteIfExists(emptyWal.toPath()); + Files.deleteIfExists(successorWal.toPath()); + Files.deleteIfExists(dir); + } + } + + @Test + public void testRefreshDoesNotSkipUnpublishedSuccessorWal() throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-unpublished-successor"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final File sealedWal = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File successorWal = + dir.resolve(WALFileUtils.getLogFileName(1, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter ignored = new WALWriter(sealedWal, WALFileVersion.V3)) { + // The old writer is sealed before its successor is constructed. + } + final WALNode walNode = mock(WALNode.class); + when(walNode.getCurrentWALFileVersion()).thenReturn(0L); + try (WALWriter writer = new WALWriter(successorWal, WALFileVersion.V3); + ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, walNode, () -> 0L)) { + // Rotation has created the successor header but has not published its version yet. + assertFalse(iterator.hasNext()); + assertFalse(iterator.hasNext()); + assertEquals(1, countingDirectory.getListCalls()); + assertTrue(iterator.isWaitingForReadableWal()); + assertTrue(iterator.hasIncompleteScan()); + assertFalse(iterator.hasReadError()); + verify(walNode, times(0)).getWALMetaDataSnapshot(anyLong()); + + final WALMetaData metadata = singleEntryMeta(19, 1L, 1L, 100L, 7, 1L); + when(walNode.getCurrentWALFileVersion()).thenReturn(1L); + when(walNode.getWALMetaDataSnapshot(1L)).thenReturn(metadata); + writer.write(searchableEntry(1L), metadata); + iterator.refresh(); + + assertTrue(iterator.hasNext()); + assertEquals(2, countingDirectory.getListCalls()); + assertEquals(1L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + assertFalse(iterator.isWaitingForReadableWal()); + assertFalse(iterator.hasSkippedBrokenWalFiles()); + } + } finally { + Files.deleteIfExists(sealedWal.toPath()); + Files.deleteIfExists(successorWal.toPath()); + Files.deleteIfExists(dir); + } + } + + @Test + public void testReportedBrokenWalInvalidatesCacheAfterListingFailure() throws Exception { + final Path dir = Files.createTempDirectory("progress-wal-iterator-refresh-broken-files"); + final CountingDirectory countingDirectory = new CountingDirectory(dir.toFile()); + final File firstWal = + dir.resolve(WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File secondWal = + dir.resolve(WALFileUtils.getLogFileName(1, 1, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + final File thirdWal = + dir.resolve(WALFileUtils.getLogFileName(2, 2, WALFileStatus.CONTAINS_SEARCH_INDEX)) + .toFile(); + try { + try (WALWriter writer = new WALWriter(firstWal, WALFileVersion.V3)) { + writer.write(searchableEntry(1L), singleEntryMeta(19, 1L, 1L, 100L, 7, 1L)); + } + try (WALWriter writer = new WALWriter(secondWal, WALFileVersion.V3)) { + writer.write(searchableEntry(2L), singleEntryMeta(19, 2L, 1L, 200L, 7, 2L)); + } + try (WALWriter writer = new WALWriter(thirdWal, WALFileVersion.V3)) { + writer.write(searchableEntry(3L), singleEntryMeta(19, 3L, 1L, 300L, 7, 3L)); + } + try (ProgressWALIterator iterator = + new ProgressWALIterator(countingDirectory, 1L, null, () -> 0L)) { + countingDirectory.failNextList(); + iterator.skipReportedBrokenWalFiles(Collections.singleton(1L)); + assertEquals(2, countingDirectory.getListCalls()); + // The previous table is still usable, but reported broken versions must not be opened. + assertEquals(1L, iterator.next().getSearchIndex()); + assertEquals(3L, iterator.next().getSearchIndex()); + assertFalse(iterator.hasNext()); + iterator.refresh(); + assertEquals(3, countingDirectory.getListCalls()); + assertFalse(iterator.hasNext()); + } + } finally { + Files.deleteIfExists(firstWal.toPath()); + Files.deleteIfExists(secondWal.toPath()); + Files.deleteIfExists(thirdWal.toPath()); + Files.deleteIfExists(dir); + } + } + @Test public void testIteratorReusePerformance() throws Exception { Assume.assumeTrue( @@ -1556,4 +1837,44 @@ public class ProgressWALIteratorTest { metaData.add(size, searchIndex, memTableId, physicalTime, nodeId, localSeq); return metaData; } + + private static class CountingDirectory extends File { + + private static final long serialVersionUID = 1L; + + private int listCalls; + private boolean failNextList; + private long lastModified = 1L; + + private CountingDirectory(final File directory) { + super(directory.getAbsolutePath()); + } + + @Override + public long lastModified() { + return lastModified; + } + + private void markModified() { + lastModified++; + } + + @Override + public File[] listFiles(final FilenameFilter filter) { + listCalls++; + if (failNextList) { + failNextList = false; + return null; + } + return super.listFiles(filter); + } + + private void failNextList() { + failNextList = true; + } + + private int getListCalls() { + return listCalls; + } + } }
