jt2594838 commented on code in PR #18693:
URL: https://github.com/apache/iotdb/pull/18693#discussion_r4078428547
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java:
##########
@@ -223,6 +255,28 @@ private void loadNextSegmentV1() throws IOException {
private void loadNextSegmentV2() throws IOException {
long position = channel.position();
SegmentInfo segmentInfo = getNextSegmentInfo();
+ long remainingBytes = fileSize - channel.position();
+ if (recoveringEntries && segmentInfo.compressionType ==
CompressionType.UNCOMPRESSED) {
+ // Complete entries in a partially written uncompressed segment remain
readable. A compressed
+ // segment needs its full payload before any of its entries can be
recovered.
+ segmentInfo.dataInDiskSize = (int) Math.min(segmentInfo.dataInDiskSize,
remainingBytes);
+ segmentInfo.uncompressedSize = segmentInfo.dataInDiskSize;
+ }
+ if (segmentInfo.dataInDiskSize <= 0
+ || segmentInfo.uncompressedSize <= 0
+ || segmentInfo.dataInDiskSize > remainingBytes) {
+ throw new EOFException(StorageEngineMessages.UNEXPECTED_END_OF_FILE);
+ }
+ // Recovery inspects untrusted headers. Bound allocations by the writer's
configured segment
+ // capacity so a tiny corrupt payload cannot request a huge decompression
buffer.
+ if (recoveringEntries
Review Comment:
This guard follows the WAL writer segment capacity and runs before either
direct buffer allocation. It prevents a corrupt recovery header from turning a
tiny on-disk payload into an unbounded decompression allocation; the oversized
LZ4 regression passed as EOFException before decompression.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java:
##########
@@ -250,27 +257,224 @@ public void testReadBrokenFile() throws IOException,
IllegalPathException {
}
@Test
- public void testReadMetadataFromBrokenFile() throws IOException {
- ILogWriter walWriter = new WALWriter(walFile);
- final FileChannel fileChannel1 = FileChannel.open(walFile.toPath());
- assertThrows(IOException.class, () -> WALMetaData.readFromWALFile(walFile,
fileChannel1));
- walWriter.close();
-
- if (!walFile.exists()) {
- Files.createFile(walFile.toPath());
- Files.write(walFile.toPath(),
ByteBuffer.wrap(WALFileVersion.V2.getVersionBytes()).array());
+ public void testReadMetadataFromEmptyFile() throws IOException {
+ try (WALWriter walWriter = new WALWriter(walFile);
+ FileChannel fileChannel = FileChannel.open(walFile.toPath())) {
+ WALMetaData walMetaData = WALMetaData.readFromWALFile(walFile,
fileChannel);
+ assertTrue(walMetaData.getBuffersSize().isEmpty());
+ assertTrue(walMetaData.getMemTablesId().isEmpty());
}
+ assertEquals(WALFileVersion.V3.getVersionBytes().length, walFile.length());
+ try (WALByteBufReader reader = new WALByteBufReader(walFile)) {
+ assertFalse(reader.hasNext());
+ }
+ }
+
+ @Test
+ public void testReadMetadataFromTruncatedFile() throws IOException {
+ Files.write(walFile.toPath(), WALFileVersion.V3.getVersionBytes());
+ Files.write(walFile.toPath(), new byte[] {1}, StandardOpenOption.APPEND);
+
+ try (FileChannel fileChannel = FileChannel.open(walFile.toPath())) {
+ assertThrows(IOException.class, () ->
WALMetaData.readFromWALFile(walFile, fileChannel));
+ }
+ assertFalse(walFile.exists());
+ assertTrue(new File(walFile + ".broken").exists());
+ }
+
+ @Test
+ public void testRecoverEntriesWithCorruptedMetadataLength() throws Exception
{
+ for (WALFileVersion version : new WALFileVersion[] {WALFileVersion.V2,
WALFileVersion.V3}) {
+ for (int length : new int[] {-1, Integer.MAX_VALUE, 0, 12}) {
+ Files.deleteIfExists(walFile.toPath());
+ WALEntry entry = new WALInfoEntry(42, getInsertRowNode(devicePath));
+ WALMetaData metadata = new WALMetaData();
+ metadata.add(entry.serializedSize(), -1, 42);
+ WALByteBufferForTest buffer =
+ new
WALByteBufferForTest(ByteBuffer.allocate(entry.serializedSize()));
+ entry.serialize(buffer);
+ try (WALWriter writer = new WALWriter(walFile, version)) {
+ writer.write(buffer.getBuffer(), metadata);
+ }
+ byte[] bytes = Files.readAllBytes(walFile.toPath());
+ ByteBuffer.wrap(bytes).putInt(bytes.length -
version.getVersionBytes().length - 4, length);
+ Files.write(walFile.toPath(), bytes);
+ try (WALByteBufReader reader = new WALByteBufReader(walFile)) {
+ assertTrue(reader.getMetaData().isRecoveredFromEntries());
+ assertEquals(Collections.singleton(42L),
reader.getMetaData().getMemTablesId());
+ assertTrue(reader.hasNext());
+ assertEquals(
+ entry,
+ WALEntry.deserialize(
+ new DataInputStream(new
ByteArrayInputStream(reader.next().array()))));
+ assertFalse(reader.hasNext());
+ }
+ // Readers reconstruct an in-memory prefix without replacing bytes
under other readers.
+ assertArrayEquals(bytes, Files.readAllBytes(walFile.toPath()));
+ }
+ }
+ }
+
+ @Test
+ public void testRecoveryRejectsOversizedCompressedSegment() throws Exception
{
+ // Include the declared payload so rejection is caused by its logical
size, not a short read.
+ ByteBuffer bytes =
+ ByteBuffer.allocate(WALFileVersion.V3.getVersionBytes().length + 2 + 2
* Integer.BYTES);
+ bytes.put(WALFileVersion.V3.getVersionBytes());
+ bytes.put(CompressionType.LZ4.serialize());
+ bytes.putInt(1);
+ bytes.putInt(64 * 1024 * 1024);
+ bytes.put((byte) 0);
+ Files.write(walFile.toPath(), bytes.array());
+
+ try (WALInputStream input = new WALInputStream(walFile, true)) {
+ // A decompressor failure is wrapped in IOException; this must fail
before reaching it.
+ assertThrows(EOFException.class, input::read);
Review Comment:
The test includes the complete one-byte compressed payload, so a truncated
physical read cannot make it pass. EOFException specifically verifies rejection
by the size guard rather than the IOException wrapper used for decompressor
failures. All 14 WALFileTest tests passed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]