(paimon) branch master updated: [core] Pass null value to file index (#3766)

[junhao]

This is an automated email from the ASF dual-hosted git repository.

junhao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new b7e11d7bd [core] Pass null value to file index (#3766)
b7e11d7bd is described below

commit b7e11d7bd3f0e39e6a769d8cd62c4076b4d8e3e5
Author: YeJunHao <41894543+leaves12...@users.noreply.github.com>
AuthorDate: Fri Jul 19 10:49:24 2024 +0800

    [core] Pass null value to file index (#3766)
---
 .../java/org/apache/paimon/fileindex/FileIndexWriter.java  |  6 ++----
 .../paimon/fileindex/bloomfilter/BloomFilterFileIndex.java |  4 +++-
 .../java/org/apache/paimon/io/DataFileIndexWriter.java     | 14 ++++++++++++++
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git 
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java 
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java
index 57b583f9b..cfb05d20d 100644
--- 
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java
+++ 
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexWriter.java
@@ -24,10 +24,8 @@ public abstract class FileIndexWriter {
     private boolean empty = true;
 
     public void writeRecord(Object key) {
-        if (key != null) {
-            empty = false;
-            write(key);
-        }
+        empty = false;
+        write(key);
     }
 
     public abstract void write(Object key);
diff --git 
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java
 
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java
index 690ba7b6f..ce7827a98 100644
--- 
a/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java
+++ 
b/paimon-common/src/main/java/org/apache/paimon/fileindex/bloomfilter/BloomFilterFileIndex.java
@@ -92,7 +92,9 @@ public class BloomFilterFileIndex implements FileIndexer {
 
         @Override
         public void write(Object key) {
-            filter.addHash(hashFunction.hash(key));
+            if (key != null) {
+                filter.addHash(hashFunction.hash(key));
+            }
         }
 
         @Override
diff --git 
a/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java 
b/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java
index 83d0cf078..71ff6b7ba 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileIndexWriter.java
@@ -43,8 +43,10 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /** Index file writer for a data file. */
 public final class DataFileIndexWriter implements Closeable {
@@ -303,18 +305,30 @@ public final class DataFileIndexWriter implements 
Closeable {
         }
 
         public void write(InternalRow row) {
+            if (row.isNullAt(position)) {
+                indexWritersMap.values().forEach(write -> 
write.writeRecord(null));
+                return;
+            }
             InternalMap internalMap = row.getMap(position);
             InternalArray keyArray = internalMap.keyArray();
             InternalArray valueArray = internalMap.valueArray();
 
+            Set<String> writedKeys = new HashSet<>();
             for (int i = 0; i < keyArray.size(); i++) {
                 String key = keyArray.getString(i).toString();
                 org.apache.paimon.fileindex.FileIndexWriter writer =
                         indexWritersMap.getOrDefault(key, null);
                 if (writer != null) {
+                    writedKeys.add(key);
                     
writer.writeRecord(valueElementGetter.getElementOrNull(valueArray, i));
                 }
             }
+
+            for (Map.Entry<String, FileIndexWriter> writerEntry : 
indexWritersMap.entrySet()) {
+                if (!writedKeys.contains(writerEntry.getKey())) {
+                    writerEntry.getValue().writeRecord(null);
+                }
+            }
         }
 
         public void add(String nestedKey, Options nestedOptions) {