This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new b6f2ea857 [core] Fix deserialization of sequenceNumber in lookup
(#2831)
b6f2ea857 is described below
commit b6f2ea8572df08de8f83331df70270cd93112a2b
Author: Zouxxyy <[email protected]>
AuthorDate: Thu Feb 1 11:31:34 2024 +0800
[core] Fix deserialization of sequenceNumber in lookup (#2831)
---
.../java/org/apache/paimon/mergetree/LookupLevels.java | 6 +++---
.../java/org/apache/paimon/mergetree/LookupUtils.java | 2 +-
.../main/java/org/apache/paimon/mergetree/SortedRun.java | 6 +++++-
.../org/apache/paimon/mergetree/LookupLevelsTest.java | 16 ++++++++++------
4 files changed, 19 insertions(+), 11 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
index 5420d42a9..c35bda970 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupLevels.java
@@ -133,19 +133,19 @@ public class LookupLevels implements
Levels.DropFileCallback, Closeable {
private KeyValue lookup(InternalRow key, DataFileMeta file) throws
IOException {
LookupFile lookupFile = lookupFiles.getIfPresent(file.fileName());
- byte[] keyBytes = keySerializer.serializeToBytes(key);
-
while (lookupFile == null || lookupFile.isClosed) {
lookupFile = createLookupFile(file);
lookupFiles.put(file.fileName(), lookupFile);
}
+ byte[] keyBytes = keySerializer.serializeToBytes(key);
byte[] valueBytes = lookupFile.get(keyBytes);
if (valueBytes == null) {
return null;
}
InternalRow value = valueSerializer.deserialize(valueBytes);
- long sequenceNumber =
MemorySegment.wrap(valueBytes).getLong(valueBytes.length - 9);
+ long sequenceNumber =
+
MemorySegment.wrap(valueBytes).getLongBigEndian(valueBytes.length - 9);
RowKind rowKind = RowKind.fromByteValue(valueBytes[valueBytes.length -
1]);
return new KeyValue()
.replace(key, sequenceNumber, rowKind, value)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupUtils.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupUtils.java
index a71e0b8fa..d048bfe0f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/LookupUtils.java
@@ -81,7 +81,7 @@ public class LookupUtils {
SortedRun level,
BiFunctionWithIOE<InternalRow, DataFileMeta, T> lookup)
throws IOException {
- if (!level.nonEmpty()) {
+ if (level.isEmpty()) {
return null;
}
List<DataFileMeta> files = level.files();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/SortedRun.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/SortedRun.java
index 831499aec..54b2893a0 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/SortedRun.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/SortedRun.java
@@ -72,8 +72,12 @@ public class SortedRun {
return files;
}
+ public boolean isEmpty() {
+ return files.isEmpty();
+ }
+
public boolean nonEmpty() {
- return !files.isEmpty();
+ return !isEmpty();
}
public long totalSize() {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
index 836f8f874..69dcf5062 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/LookupLevelsTest.java
@@ -86,29 +86,29 @@ public class LookupLevelsTest {
new Levels(
comparator,
Arrays.asList(
- newFile(1, kv(1, 11), kv(3, 33), kv(5, 5)),
- newFile(2, kv(2, 22), kv(5, 55))),
+ newFile(1, kv(1, 11, 1), kv(3, 33, 2), kv(5,
5, 3)),
+ newFile(2, kv(2, 22, 4), kv(5, 55, 5))),
3);
LookupLevels lookupLevels = createLookupLevels(levels,
MemorySize.ofMebiBytes(10));
// only in level 1
KeyValue kv = lookupLevels.lookup(row(1), 1);
assertThat(kv).isNotNull();
- assertThat(kv.sequenceNumber()).isEqualTo(UNKNOWN_SEQUENCE);
+ assertThat(kv.sequenceNumber()).isEqualTo(1);
assertThat(kv.level()).isEqualTo(1);
assertThat(kv.value().getInt(1)).isEqualTo(11);
// only in level 2
kv = lookupLevels.lookup(row(2), 1);
assertThat(kv).isNotNull();
- assertThat(kv.sequenceNumber()).isEqualTo(UNKNOWN_SEQUENCE);
+ assertThat(kv.sequenceNumber()).isEqualTo(4);
assertThat(kv.level()).isEqualTo(2);
assertThat(kv.value().getInt(1)).isEqualTo(22);
// both in level 1 and level 2
kv = lookupLevels.lookup(row(5), 1);
assertThat(kv).isNotNull();
- assertThat(kv.sequenceNumber()).isEqualTo(UNKNOWN_SEQUENCE);
+ assertThat(kv.sequenceNumber()).isEqualTo(3);
assertThat(kv.level()).isEqualTo(1);
assertThat(kv.value().getInt(1)).isEqualTo(5);
@@ -233,8 +233,12 @@ public class LookupLevelsTest {
}
private KeyValue kv(int key, int value) {
+ return kv(key, value, UNKNOWN_SEQUENCE);
+ }
+
+ private KeyValue kv(int key, int value, long seqNumber) {
return new KeyValue()
- .replace(GenericRow.of(key), RowKind.INSERT,
GenericRow.of(key, value));
+ .replace(GenericRow.of(key), seqNumber, RowKind.INSERT,
GenericRow.of(key, value));
}
private DataFileMeta newFile(int level, KeyValue... records) throws
IOException {
