junrao commented on code in PR #23445:
URL: https://github.com/apache/kafka/pull/23445#discussion_r4277124066
##########
clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java:
##########
@@ -519,6 +527,181 @@ public void testBytesLengthOfWriteTo() throws IOException
{
verify(channel).transferFrom(any(), anyLong(), eq((long) size -
firstWritten));
}
+ /**
+ * A failed send whose file is still readable must be reported exactly as
it arrived: the socket was at
+ * fault, and turning that into a storage failure would take a log
directory offline because a client
+ * went away.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenFileIsStillReadable() throws
IOException {
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = fileRecords.withReadFailureHandler((message, e)
-> reported.add(e));
+ IOException socketFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(socketFailure), 0,
records.sizeInBytes()));
+ assertSame(socketFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A failed send whose file cannot be read is a storage failure: the
handler is told once, and the send
+ * fails with a KafkaStorageException naming the file.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureWhenRegionIsUnreadable()
throws IOException {
+ FileChannel fileChannel = fileChannelOfSize(1024);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class),
anyLong())).thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, 1024,
reported);
+ IOException transferFailure = new IOException("Input/output error");
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, 1024));
+ assertSame(transferFailure, thrown.getCause());
+ assertSame(readFailure, thrown.getSuppressed()[0]);
+ assertTrue(thrown.getMessage().contains(LOG_FILE_PATH));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * Depending on the platform, the transfer may fail anywhere within the
region without reporting
+ * progress, so a failure past the first bytes of the region must be found
as well.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureInsideRegion() throws
IOException {
+ int size = 64 * 1024;
+ FileChannel fileChannel = fileChannelOfSize(size);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class), anyLong()))
+ .thenAnswer(FileRecordsTest::fillBuffer)
+ .thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, size,
reported);
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(new IOException("Input/output
error")), 0, size));
+ assertTrue(thrown.getMessage().contains("at position " + size / 2));
+ verify(fileChannel).read(any(ByteBuffer.class), eq(0L));
+ verify(fileChannel).read(any(ByteBuffer.class), eq((long) size / 2));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * The log can be truncated while the region is re-read, for instance when
the replica becomes a
+ * follower while a fetch is being served. The end of the file then says
nothing about the medium, so
+ * the original exception must survive.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenTruncatedDuringReRead()
throws IOException {
+ int size = 64 * 1024;
+ FileChannel fileChannel = fileChannelOfSize(size);
+ when(fileChannel.read(any(ByteBuffer.class), anyLong()))
+ .thenAnswer(FileRecordsTest::fillBuffer)
+ .thenReturn(-1);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, size,
reported);
+ IOException transferFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, size));
+ assertSame(transferFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A channel closed while the region is re-read, by a deletion of the
segment, a shutdown or an
+ * interrupt, says nothing about the medium, so the original exception
must survive.
Review Comment:
We don't interrupt the IO threads to avoid failing a read on the file
channel.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
+ reReadRegion(position, count, e);
Review Comment:
Claude found the following issue, which seems reasonable.
```
1. The re-read runs on the network thread (FileRecords.java:325-367).
writeTo is called from the SocketServer processor thread, through
Selector.attemptWrite. So every socket error on such a slice makes that thread
read up to count bytes from disk, in 32 KB chunks. Socket errors include a
consumer closing, a rebalance, or a timeout ("Connection reset", "Broken
pipe"). count is the rest of the partition's slice, typically up to 1 MB. If
the failure happened right at position, the rest of the region may not be in
the page cache. Meanwhile every other connection on that processor waits, and a
burst of disconnects multiplies the stall.
The re-read's result doesn't change what happens to the connection, which
closes either way. It only decides whether to mark the log directory offline,
and that's already handled asynchronously by the log-dir failure handler. So
the re-read can move off the network thread:
In writeTo: on an IOException, pass (position, count, e) to the handler and
rethrow e unchanged.
In LocalLog's handler: schedule the re-read on scheduler (LocalLog.java:83),
and call maybeAddOfflineLogDir only if it fails. Run at most one re-read per
segment file at a time, so a burst of disconnects doesn't queue up hundreds.
This also removes the duplicate logging. Today each affected connection logs
a WARN "Unexpected error … closing connection" from Selector in addition to the
ERROR from LogDirFailureChannel, because KafkaStorageException isn't an
IOException.
```
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
Review Comment:
Claude found the following issue.
`2. Skip the re-read when the transfer failed with ClosedChannelException
(FileRecords.java:327). That means the socket (or the file) was already closed,
so the file was never read and the re-read can't tell you anything. Add if (e
instanceof ClosedChannelException) throw e; before the re-read. This is the
most common way a send fails on a connection that's already closing.`
##########
clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java:
##########
@@ -519,6 +527,181 @@ public void testBytesLengthOfWriteTo() throws IOException
{
verify(channel).transferFrom(any(), anyLong(), eq((long) size -
firstWritten));
}
+ /**
+ * A failed send whose file is still readable must be reported exactly as
it arrived: the socket was at
+ * fault, and turning that into a storage failure would take a log
directory offline because a client
+ * went away.
+ */
+ @Test
+ public void testWriteToPreservesExceptionWhenFileIsStillReadable() throws
IOException {
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = fileRecords.withReadFailureHandler((message, e)
-> reported.add(e));
+ IOException socketFailure = new IOException("Broken pipe");
+
+ IOException thrown = assertThrows(IOException.class,
+ () -> records.writeTo(failingChannel(socketFailure), 0,
records.sizeInBytes()));
+ assertSame(socketFailure, thrown);
+ assertTrue(reported.isEmpty());
+ }
+
+ /**
+ * A failed send whose file cannot be read is a storage failure: the
handler is told once, and the send
+ * fails with a KafkaStorageException naming the file.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureWhenRegionIsUnreadable()
throws IOException {
+ FileChannel fileChannel = fileChannelOfSize(1024);
+ IOException readFailure = new IOException("Input/output error");
+ when(fileChannel.read(any(ByteBuffer.class),
anyLong())).thenThrow(readFailure);
+ List<IOException> reported = new ArrayList<>();
+ FileRecords records = recordsWithReadFailureHandler(fileChannel, 1024,
reported);
+ IOException transferFailure = new IOException("Input/output error");
+
+ KafkaStorageException thrown =
assertThrows(KafkaStorageException.class,
+ () -> records.writeTo(failingChannel(transferFailure), 0, 1024));
+ assertSame(transferFailure, thrown.getCause());
+ assertSame(readFailure, thrown.getSuppressed()[0]);
+ assertTrue(thrown.getMessage().contains(LOG_FILE_PATH));
+ assertEquals(List.of(readFailure), reported);
+ }
+
+ /**
+ * Depending on the platform, the transfer may fail anywhere within the
region without reporting
+ * progress, so a failure past the first bytes of the region must be found
as well.
+ */
+ @Test
+ public void testWriteToReportsStorageFailureInsideRegion() throws
IOException {
+ int size = 64 * 1024;
Review Comment:
Could we use 2 * FileRecords.REREAD_BUFFER_SIZE? Otherwise, it's not clear
later why the position is half the size.
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
Review Comment:
Claude reported the following.
`channel.size() is outside the new try (FileRecords.java:310). An
IOException from it skips the handler. That's rare, but moving it inside the
try costs nothing.`
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -146,7 +157,25 @@ public FileRecords slice(int position, int size) {
int availableBytes = availableBytes(position, size);
int startPosition = this.start + position;
- return new FileRecords(file, channel, startPosition, startPosition +
availableBytes);
+ return new FileRecords(file, channel, startPosition, startPosition +
availableBytes, readFailureHandler);
+ }
+
+ /**
+ * Return a slice covering the current records of this instance whose
{@link #writeTo} reports a failed read of
Review Comment:
This doesn't prevent a non-sliced FileRecords calling this method. In that
case, a non-slice is returned.
##########
storage/src/main/java/org/apache/kafka/storage/internals/log/LocalLog.java:
##########
@@ -526,6 +529,21 @@ else if (segment.baseOffset() ==
maxOffsetMetadata.segmentBaseOffset && !maxOffs
);
}
+ /**
+ * The records of a fetch are read from the segment only when they are
written to the network, after this log
+ * has returned them, so an IOException from that read cannot be caught
here. Have the records report it instead,
+ * so that the log directory is marked offline as for any other
IOException in this log.
+ */
+ private FetchDataInfo withReadFailureHandler(FetchDataInfo fetchDataInfo) {
+ if (!(fetchDataInfo.records instanceof FileRecords fileRecords))
+ return fetchDataInfo;
+ return new FetchDataInfo(fetchDataInfo.fetchOffsetMetadata,
Review Comment:
Claude recommended the following.
`The FetchDataInfo rebuild copies every field by hand
(LocalLog.java:537-545), so a field added later would be silently dropped. A
FetchDataInfo.withRecords(Records) method would keep the copy in one place. The
extra allocations per read are negligible.`
##########
storage/src/test/java/org/apache/kafka/storage/internals/log/LocalLogTest.java:
##########
@@ -256,6 +259,20 @@ public void testLogAppend() throws IOException {
assertEquals(keyValues, recordsToKvs(fetchDataInfo.records.records()));
}
+ @Test
+ public void testReadFailureWhileWritingFetchedRecordsMarksLogDirOffline()
throws IOException {
Review Comment:
Claude reported the following issue.
`3. The new LocalLogTest leaks the segment files
(LocalLogTest.java:262-274). The test leaves the log directory offline, so
tearDown's log.close() hits maybeHandleIOException and throws before closing
the segments. The open FileChannel and memory-mapped indexes then stay open for
the rest of the test JVM. End the test with log.closeQuietly(), as the other
offline tests do (:284, :300).
`
##########
clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java:
##########
@@ -287,8 +316,53 @@ public int writeTo(TransferableChannel destChannel, int
offset, int length) thro
long position = start + offset;
int count = Math.min(length, oldSize - offset);
- // safe to cast to int since `count` is an int
- return (int) destChannel.transferFrom(channel, position, count);
+ try {
+ // safe to cast to int since `count` is an int
+ return (int) destChannel.transferFrom(channel, position, count);
+ } catch (IOException e) {
+ // The transfer reads the file and writes the socket in one call,
so a storage failure and a
+ // socket failure both arrive here as a bare IOException. Tell
them apart by re-reading the
+ // region: only on the error path, and only when someone handles
storage failures. The message
+ // text is not inspected, it is platform specific.
+ if (readFailureHandler != null)
+ reReadRegion(position, count, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Re-read the given region of the backing file, used by {@link #writeTo}
to tell a storage failure
+ * from a socket failure. The whole region is read rather than its first
byte because, depending on
+ * the platform, the transfer may fail anywhere within it without
reporting progress. Only a failed
+ * read of an open channel counts as a storage failure: the end of the
file may be reached because the
+ * log was truncated concurrently (e.g. when the replica becomes a
follower), and the channel may have
+ * been closed concurrently, by a deletion or a shutdown; neither says
anything about the medium.
+ *
+ * @throws KafkaStorageException If a read failed, after notifying the
read failure handler
+ */
+ private void reReadRegion(long position, int count, IOException
transferFailure) {
+ ByteBuffer buffer = ByteBuffer.allocate(Math.min(count,
REREAD_BUFFER_SIZE));
+ long end = position + count;
Review Comment:
This hides the instance level field of the same name. Could we rename it to
sth like regionEnd?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]