cheeeee commented on code in PR #23445:
URL: https://github.com/apache/kafka/pull/23445#discussion_r4286760893
##########
clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java:
##########
@@ -519,6 +527,181 @@ public void testBytesLengthOfWriteTo() throws IOException
{
verify(channel).transferFrom(any(), anyLong(), eq((long) size -
firstWritten));
}
+ /**
+ * A failed send whose file is still readable must be reported exactly as
it arrived: the socket was at
+ * fault, and turning that into a storage failure would take a log
directory offline because a client
+ * went away.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenFileIsStillReadable() throws
IOException {
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = fileRecords.withReadFailureHandler((message, e)
-> reported.add(e));
+ IOException socketFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(socketFailure), 0,
records.sizeInBytes()));
+ assertSame(socketFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A failed send whose file cannot be read is a storage failure: the
handler is told once, and the send
+ * fails with a KafkaStorageException naming the file.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureWhenRegionIsUnreadable()
throws IOException {
+ FileChannel fileChannel = fileChannelOfSize(1024);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class),
anyLong())).thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, 1024,
reported);
+ IOException transferFailure = new IOException("Input/output error");
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, 1024));
+ assertSame(transferFailure, thrown.getCause());
+ assertSame(readFailure, thrown.getSuppressed()[0]);
+ assertTrue(thrown.getMessage().contains(LOG_FILE_PATH));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * Depending on the platform, the transfer may fail anywhere within the
region without reporting
+ * progress, so a failure past the first bytes of the region must be found
as well.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureInsideRegion() throws
IOException {
+ int size = 64 * 1024;
Review Comment:
Done.
##########
clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java:
##########
@@ -519,6 +527,181 @@ public void testBytesLengthOfWriteTo() throws IOException
{
verify(channel).transferFrom(any(), anyLong(), eq((long) size -
firstWritten));
}
+ /**
+ * A failed send whose file is still readable must be reported exactly as
it arrived: the socket was at
+ * fault, and turning that into a storage failure would take a log
directory offline because a client
+ * went away.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenFileIsStillReadable() throws
IOException {
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = fileRecords.withReadFailureHandler((message, e)
-> reported.add(e));
+ IOException socketFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(socketFailure), 0,
records.sizeInBytes()));
+ assertSame(socketFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A failed send whose file cannot be read is a storage failure: the
handler is told once, and the send
+ * fails with a KafkaStorageException naming the file.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureWhenRegionIsUnreadable()
throws IOException {
+ FileChannel fileChannel = fileChannelOfSize(1024);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class),
anyLong())).thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, 1024,
reported);
+ IOException transferFailure = new IOException("Input/output error");
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, 1024));
+ assertSame(transferFailure, thrown.getCause());
+ assertSame(readFailure, thrown.getSuppressed()[0]);
+ assertTrue(thrown.getMessage().contains(LOG_FILE_PATH));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * Depending on the platform, the transfer may fail anywhere within the
region without reporting
+ * progress, so a failure past the first bytes of the region must be found
as well.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureInsideRegion() throws
IOException {
+ int size = 64 * 1024;
+ FileChannel fileChannel = fileChannelOfSize(size);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class), anyLong()))
+ .thenAnswer(FileRecordsTest::fillBuffer)
+ .thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, size,
reported);
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(new IOException("Input/output
error")), 0, size));
+ assertTrue(thrown.getMessage().contains("at position " + size / 2));
+ verify(fileChannel).read(any(ByteBuffer.class), eq(0L));
+ verify(fileChannel).read(any(ByteBuffer.class), eq((long) size / 2));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * The log can be truncated while the region is re-read, for instance when
the replica becomes a
+ * follower while a fetch is being served. The end of the file then says
nothing about the medium, so
+ * the original exception must survive.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenTruncatedDuringReRead()
throws IOException {
+ int size = 64 * 1024;
+ FileChannel fileChannel = fileChannelOfSize(size);
+ when(fileChannel.read(any(ByteBuffer.class), anyLong()))
+ .thenAnswer(FileRecordsTest::fillBuffer)
+ .thenReturn(-1);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, size,
reported);
+ IOException transferFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, size));
+ assertSame(transferFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A channel closed while the region is re-read, by a deletion of the
segment, a shutdown or an
+ * interrupt, says nothing about the medium, so the original exception
must survive.
Review Comment:
Dropped the interrupt case.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
+ reReadRegion(position, count, e);
Review Comment:
Done: `writeTo` passes the region to the handler and rethrows; `LocalLog`
re-reads it on its scheduler, at most once at a time per segment file, and
offlines the directory only if the read fails.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
Review Comment:
Done. As in KAFKA-19571, a closed channel comes from a concurrent close, not
the medium.
##########
storage/src/test/java/org/apache/kafka/storage/internals/log/LocalLogTest.java:
##########
@@ -256,6 +259,20 @@ public void testLogAppend() throws IOException {
assertEquals(keyValues, recordsToKvs(fetchDataInfo.records.records()));
}
+ @Test
+ public void testReadFailureWhileWritingFetchedRecordsMarksLogDirOffline()
throws IOException {
Review Comment:
Done.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
Review Comment:
Done.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
+ reReadRegion(position, count, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Re-read the given region of the backing file, used by {@link #writeTo}
to tell a storage failure
+ * from a socket failure. The whole region is read rather than its first
byte because, depending on
+ * the platform, the transfer may fail anywhere within it without
reporting progress. Only a failed
+ * read of an open channel counts as a storage failure: the end of the
file may be reached because the
+ * log was truncated concurrently (e.g. when the replica becomes a
follower), and the channel may have
+ * been closed concurrently, by a deletion or a shutdown; neither says
anything about the medium.
+ *
+ * @throws KafkaStorageException If a read failed, after notifying the
read failure handler
+ */
+ private void reReadRegion(long position, int count, IOException
transferFailure) {
+ ByteBuffer buffer = ByteBuffer.allocate(Math.min(count,
REREAD_BUFFER_SIZE));
+ long end = position + count;
Review Comment:
Done.
##########
storage/src/main/java/org/apache/kafka/storage/internals/log/LocalLog.java:
##########
@@ -526,6 +529,21 @@ else if (segment.baseOffset() ==
maxOffsetMetadata.segmentBaseOffset && !maxOffs
);
}
+ /**
+ * The records of a fetch are read from the segment only when they are
written to the network, after this log
+ * has returned them, so an IOException from that read cannot be caught
here. Have the records report it instead,
+ * so that the log directory is marked offline as for any other
IOException in this log.
+ */
+ private FetchDataInfo withReadFailureHandler(FetchDataInfo fetchDataInfo) {
+ if (!(fetchDataInfo.records instanceof FileRecords fileRecords))
+ return fetchDataInfo;
+ return new FetchDataInfo(fetchDataInfo.fetchOffsetMetadata,
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]