(incubator-paimon) branch master updated: [core] AppendOnlySplitGenerator skips the check of sequence number overlap in UNAWARE mode to reduce invalid logs. (#2398)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 60d7ca193 [core] AppendOnlySplitGenerator skips the check of sequence 
number overlap in UNAWARE mode to reduce invalid logs. (#2398)
60d7ca193 is described below

commit 60d7ca193ba0663a8d9387f7f0f5eff37c9794a1
Author: liming.1018 <69444450+limin...@users.noreply.github.com>
AuthorDate: Mon Nov 27 17:07:01 2023 +0800

    [core] AppendOnlySplitGenerator skips the check of sequence number overlap 
in UNAWARE mode to reduce invalid logs. (#2398)
---
 .../java/org/apache/paimon/append/AppendOnlyCompactManager.java     | 6 +++---
 .../org/apache/paimon/table/source/AppendOnlySplitGenerator.java    | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git 
a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java
 
b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java
index a9edf0f42..58106e070 100644
--- 
a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java
+++ 
b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyCompactManager.java
@@ -70,7 +70,7 @@ public class AppendOnlyCompactManager extends 
CompactFutureManager {
             CompactRewriter rewriter,
             @Nullable CompactionMetrics metrics) {
         this.executor = executor;
-        this.toCompact = new TreeSet<>(fileComparator());
+        this.toCompact = new TreeSet<>(fileComparator(false));
         this.toCompact.addAll(restored);
         this.minFileNum = minFileNum;
         this.maxFileNum = maxFileNum;
@@ -304,13 +304,13 @@ public class AppendOnlyCompactManager extends 
CompactFutureManager {
      * may be put after the new files, and this order will be disrupted. We 
need to ensure this
      * order, so we force the order by sequence.
      */
-    public static Comparator<DataFileMeta> fileComparator() {
+    public static Comparator<DataFileMeta> fileComparator(boolean 
ignoreOverlap) {
         return (o1, o2) -> {
             if (o1 == o2) {
                 return 0;
             }
 
-            if (isOverlap(o1, o2)) {
+            if (!ignoreOverlap && isOverlap(o1, o2)) {
                 LOG.warn(
                         String.format(
                                 "There should no overlap in append files, but 
Range1(%s, %s), Range2(%s, %s),"
diff --git 
a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java
 
b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java
index e0ee8388d..002072a38 100644
--- 
a/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java
+++ 
b/paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java
@@ -46,7 +46,7 @@ public class AppendOnlySplitGenerator implements 
SplitGenerator {
     @Override
     public List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> input) {
         List<DataFileMeta> files = new ArrayList<>(input);
-        files.sort(fileComparator());
+        files.sort(fileComparator(bucketMode == BucketMode.UNAWARE));
         Function<DataFileMeta, Long> weightFunc = file -> 
Math.max(file.fileSize(), openFileCost);
         return BinPacking.packForOrdered(files, weightFunc, targetSplitSize);
     }