This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 62b8a34 [HUDI-1792] flink-client query error when processing files
larger than 128mb (#2814)
62b8a34 is described below
commit 62b8a341ddae0ab80195c41c7a44b84c1fe23d31
Author: hj2016 <[email protected]>
AuthorDate: Fri Apr 16 13:59:19 2021 +0800
[HUDI-1792] flink-client query error when processing files larger than
128mb (#2814)
Co-authored-by: huangjing <[email protected]>
---
.../org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git
a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
index 77f3e2a..477f54b 100644
---
a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
+++
b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -209,7 +210,13 @@ public class CopyOnWriteInputFormat extends
FileInputFormat<RowData> {
// get the block locations and make sure they are in order with
respect to their offset
final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
- Arrays.sort(blocks);
+ Arrays.sort(blocks, new Comparator<BlockLocation>() {
+ @Override
+ public int compare(BlockLocation o1, BlockLocation o2) {
+ long diff = o1.getLength() - o2.getOffset();
+ return diff < 0L ? -1 : (diff > 0L ? 1 : 0);
+ }
+ });
long bytesUnassigned = len;
long position = 0;
