This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 08833b990787e3b5ac34748484546c63cc5766a7 Author: Mingyu Chen <[email protected]> AuthorDate: Wed Sep 20 10:43:16 2023 +0800 [fix](broker) fix broker read issue (#24635) The given "length" of broker's pread() method is the buffer length, not the length required from file. So it may larger than the file length. So we should return all read data, instead of return EOF when `read()` method return -1 I will add regression test case later when the framework support broker process. --- .../org/apache/doris/broker/hdfs/FileSystemManager.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java index 997cf0cb2c..55ba457404 100644 --- a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java +++ b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java @@ -1212,8 +1212,14 @@ public class FileSystemManager { try { int readLength = fsDataInputStream.read(buf, hasRead, bufSize); if (readLength < 0) { - throw new BrokerException(TBrokerOperationStatusCode.END_OF_FILE, - "end of file reached"); + // If no data is read, just return EOF + // Otherwise, return the read data. + // Because the "length" may be longer than the rest length of the file. + if (hasRead == 0) { + throw new BrokerException(TBrokerOperationStatusCode.END_OF_FILE, + "end of file reached"); + } + break; } if (logger.isDebugEnabled()) { logger.debug("read buffer from input stream, buffer size:" + buf.length + ", read length:" @@ -1227,8 +1233,7 @@ public class FileSystemManager { } } if (hasRead != length) { - throw new BrokerException(TBrokerOperationStatusCode.TARGET_STORAGE_SERVICE_ERROR, - String.format("errors while read data from stream: hasRead(%d) != length(%d)", hasRead, length)); + logger.debug("broker pread return diff length. read bytes: " + hasRead + ", request bytes: " + length); } return ByteBuffer.wrap(buf, 0, hasRead); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
