[incubator-paimon] branch master updated: [core] Do not create new Index file if there is no change (#1522)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new e2c265be3 [core] Do not create new Index file if there is no change 
(#1522)
e2c265be3 is described below

commit e2c265be3abc7b468c23ac728cb2e9f95be817a2
Author: Jingsong Lee <jingsongl...@gmail.com>
AuthorDate: Mon Jul 10 14:12:17 2023 +0800

    [core] Do not create new Index file if there is no change (#1522)
---
 .../src/main/java/org/apache/paimon/utils/IntHashSet.java |  4 ++--
 .../test/java/org/apache/paimon/utils/IntHashSetTest.java | 10 +++++++++-
 .../java/org/apache/paimon/index/HashIndexMaintainer.java |  6 ++++--
 .../org/apache/paimon/index/HashIndexMaintainerTest.java  | 15 +++++++++++++++
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git 
a/paimon-common/src/main/java/org/apache/paimon/utils/IntHashSet.java 
b/paimon-common/src/main/java/org/apache/paimon/utils/IntHashSet.java
index 2d73bc6d6..472b84b8f 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/IntHashSet.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/IntHashSet.java
@@ -35,8 +35,8 @@ public class IntHashSet {
         this.set = new IntOpenHashSet(expected);
     }
 
-    public void add(int value) {
-        set.add(value);
+    public boolean add(int value) {
+        return set.add(value);
     }
 
     public int size() {
diff --git 
a/paimon-common/src/test/java/org/apache/paimon/utils/IntHashSetTest.java 
b/paimon-common/src/test/java/org/apache/paimon/utils/IntHashSetTest.java
index 25e3e033f..051598f33 100644
--- a/paimon-common/src/test/java/org/apache/paimon/utils/IntHashSetTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/IntHashSetTest.java
@@ -40,10 +40,18 @@ public class IntHashSetTest {
             values.add(0);
             values.add(-1);
             values.add(1);
+            values.add(Integer.MAX_VALUE);
         }
 
         IntHashSet set = new IntHashSet();
-        values.forEach(set::add);
+        for (Integer i : values) {
+            boolean add = set.add(i);
+            assertThat(add).isTrue();
+        }
+        for (Integer i : values) {
+            boolean add = set.add(i);
+            assertThat(add).isFalse();
+        }
 
         int[] expected = 
values.stream().mapToInt(Integer::intValue).sorted().toArray();
         assertThat(set.toInts()).containsExactlyInAnyOrder(expected);
diff --git 
a/paimon-core/src/main/java/org/apache/paimon/index/HashIndexMaintainer.java 
b/paimon-core/src/main/java/org/apache/paimon/index/HashIndexMaintainer.java
index 644be637f..998c72399 100644
--- a/paimon-core/src/main/java/org/apache/paimon/index/HashIndexMaintainer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/index/HashIndexMaintainer.java
@@ -76,8 +76,10 @@ public class HashIndexMaintainer implements 
IndexMaintainer<KeyValue> {
         if (!(key instanceof BinaryRow)) {
             throw new IllegalArgumentException("Unsupported key type: " + 
key.getClass());
         }
-        hashcode.add(key.hashCode());
-        modified = true;
+        boolean changed = hashcode.add(key.hashCode());
+        if (changed) {
+            modified = true;
+        }
     }
 
     @Override
diff --git 
a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexMaintainerTest.java
 
b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexMaintainerTest.java
index 8c646457e..92915b458 100644
--- 
a/paimon-core/src/test/java/org/apache/paimon/index/HashIndexMaintainerTest.java
+++ 
b/paimon-core/src/test/java/org/apache/paimon/index/HashIndexMaintainerTest.java
@@ -123,4 +123,19 @@ public class HashIndexMaintainerTest extends 
PrimaryKeyTableTestBase {
 
         write.close();
     }
+
+    @Test
+    public void testNotCreateNewFile() throws Exception {
+        // commit two partitions
+        write.write(createRow(1, 1, 1, 1));
+        write.write(createRow(2, 2, 2, 2));
+        commit.commit(0, write.prepareCommit(true, 0));
+
+        // same record
+        write.write(createRow(1, 1, 1, 1));
+        List<CommitMessage> commitMessages = write.prepareCommit(true, 1);
+        assertThat(readIndex(commitMessages)).isEmpty();
+
+        write.close();
+    }
 }