(incubator-paimon) branch master updated: [core] Use var length encoding for row position (#3031)

[lzljs3620320]

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 0d4cbf0a6 [core] Use var length encoding for row position (#3031)
0d4cbf0a6 is described below

commit 0d4cbf0a626834d4b1b005efb3c1801264b080c7
Author: Jingsong Lee <jingsongl...@gmail.com>
AuthorDate: Mon Mar 18 11:22:50 2024 +0800

    [core] Use var length encoding for row position (#3031)
---
 .../paimon/data/serializer/RowCompactedSerializer.java    |  3 ++-
 .../java/org/apache/paimon/utils/VarLengthIntUtils.java   |  3 +++
 .../java/org/apache/paimon/mergetree/LookupLevels.java    | 15 +++++++++------
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git 
a/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java
 
b/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java
index 3faebdabe..72678c364 100644
--- 
a/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java
+++ 
b/paimon-common/src/main/java/org/apache/paimon/data/serializer/RowCompactedSerializer.java
@@ -50,6 +50,7 @@ import static 
org.apache.paimon.memory.MemorySegmentUtils.bitSet;
 import static org.apache.paimon.types.DataTypeChecks.getPrecision;
 import static org.apache.paimon.types.DataTypeChecks.getScale;
 import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.VarLengthIntUtils.MAX_VAR_INT_SIZE;
 
 /** A {@link Serializer} for {@link InternalRow} using compacted binary. */
 public class RowCompactedSerializer implements Serializer<InternalRow> {
@@ -415,7 +416,7 @@ public class RowCompactedSerializer implements 
Serializer<InternalRow> {
 
         private void writeUnsignedInt(int value) {
             checkArgument(value >= 0);
-            ensureCapacity(5);
+            ensureCapacity(MAX_VAR_INT_SIZE);
             int len = VarLengthIntUtils.encodeInt(buffer, position, value);
             position += len;
         }
diff --git 
a/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java 
b/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
index ea556c307..1b2baaa56 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/VarLengthIntUtils.java
@@ -29,6 +29,9 @@ import java.io.IOException;
 /** Utils for encoding int/long to var length bytes. */
 public final class VarLengthIntUtils {
 
+    public static final int MAX_VAR_LONG_SIZE = 9;
+    public static final int MAX_VAR_INT_SIZE = 5;
+
     /** @return bytes length. */
     public static int encodeLong(DataOutput os, long value) throws IOException 
{
 
diff --git 
a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java 
b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
index 3e7e12702..dd45e7fc1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
@@ -48,6 +48,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Comparator;
 import java.util.TreeSet;
 import java.util.function.Function;
@@ -55,6 +56,9 @@ import java.util.function.Supplier;
 
 import static org.apache.paimon.mergetree.LookupUtils.fileKibiBytes;
 import static org.apache.paimon.utils.Preconditions.checkArgument;
+import static org.apache.paimon.utils.VarLengthIntUtils.MAX_VAR_LONG_SIZE;
+import static org.apache.paimon.utils.VarLengthIntUtils.decodeLong;
+import static org.apache.paimon.utils.VarLengthIntUtils.encodeLong;
 
 /** Provide lookup by key. */
 public class LookupLevels<T> implements Levels.DropFileCallback, Closeable {
@@ -342,10 +346,9 @@ public class LookupLevels<T> implements 
Levels.DropFileCallback, Closeable {
                 segment.put(bytes.length - 1, kv.valueKind().toByteValue());
                 return bytes;
             } else {
-                byte[] bytes = new byte[8];
-                MemorySegment segment = MemorySegment.wrap(bytes);
-                segment.putLong(0, rowPosition);
-                return bytes;
+                byte[] bytes = new byte[MAX_VAR_LONG_SIZE];
+                int len = encodeLong(bytes, rowPosition);
+                return Arrays.copyOf(bytes, len);
             }
         }
 
@@ -363,8 +366,8 @@ public class LookupLevels<T> implements 
Levels.DropFileCallback, Closeable {
                         fileName,
                         rowPosition);
             } else {
-                MemorySegment segment = MemorySegment.wrap(bytes);
-                return new PositionedKeyValue(null, fileName, 
segment.getLong(0));
+                long rowPosition = decodeLong(bytes, 0);
+                return new PositionedKeyValue(null, fileName, rowPosition);
             }
         }
     }