zhijiangW commented on a change in pull request #6705: [FLINK-10356][network]
add sanity checks to SpillingAdaptiveSpanningRecordDeserializer
URL: https://github.com/apache/flink/pull/6705#discussion_r227256749
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
##########
@@ -549,21 +584,53 @@ private void addNextChunkFromMemorySegment(MemorySegment
segment, int offset, in
}
else {
spillingChannel.close();
+ spillingChannel = null;
- BufferedInputStream inStream = new
BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024);
+ BufferedInputStream inStream =
+ new BufferedInputStream(
+ new
FileInputStream(checkNotNull(spillFile)),
+ 2 * 1024 * 1024);
this.spillFileReader = new
DataInputViewStreamWrapper(inStream);
}
}
}
- private void
moveRemainderToNonSpanningDeserializer(NonSpanningWrapper deserializer) {
+ private void
moveRemainderToNonSpanningDeserializer(NonSpanningWrapper deserializer) throws
IOException {
+ Optional<String> deserializationError =
getDeserializationError(0);
+ if (deserializationError.isPresent()) {
+ throw new
IOException(deserializationError.get());
+ }
+
deserializer.clear();
if (leftOverData != null) {
deserializer.initializeFromMemorySegment(leftOverData, leftOverStart,
leftOverLimit);
}
}
+ private Optional<String> getDeserializationError(int
addToReadBytes) {
Review comment:
👍
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
