This is an automated email from the ASF dual-hosted git repository.
zouxxyy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ac27b66f9 [test] Fix test in KeyValueFileStoreScanTest, make it more
accurate (#3111)
ac27b66f9 is described below
commit ac27b66f995875e8ae715eff78719af6988db191
Author: YeJunHao <[email protected]>
AuthorDate: Thu Mar 28 15:23:24 2024 +0800
[test] Fix test in KeyValueFileStoreScanTest, make it more accurate (#3111)
---
.../operation/KeyValueFileStoreScanTest.java | 58 ++++++++++------------
1 file changed, 25 insertions(+), 33 deletions(-)
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java
index 8ea35082a..c58136339 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java
@@ -23,8 +23,6 @@ import org.apache.paimon.Snapshot;
import org.apache.paimon.TestFileStore;
import org.apache.paimon.TestKeyValueGenerator;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryRowWriter;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.manifest.ManifestEntry;
@@ -148,19 +146,19 @@ public class KeyValueFileStoreScanTest {
}
@Test
- public void testWithValueFilter() throws Exception {
+ public void testWithValueFilterBucket() throws Exception {
ThreadLocalRandom random = ThreadLocalRandom.current();
- // 0 <= hr <= 999
- List<KeyValue> data = generateData(100, random.nextInt(1000));
+ // 0 <= item <= 999
+ List<KeyValue> data = generateData(100, 0, (long)
random.nextInt(1000));
writeData(data, 0);
- // 1000 <= hr <= 1999
- data = generateData(100, random.nextInt(1000) + 1000);
+ // 1000 <= item <= 1999
+ data = generateData(100, 0, (long) random.nextInt(1000) + 1000);
writeData(data, 1);
- // 2000 <= hr <= 2999
- data = generateData(100, random.nextInt(1000) + 2000);
+ // 2000 <= item <= 2999
+ data = generateData(100, 0, (long) random.nextInt(1000) + 2000);
writeData(data, 2);
- // 3000 <= hr <= 3999
- data = generateData(100, random.nextInt(1000) + 3000);
+ // 3000 <= item <= 3999
+ data = generateData(100, 0, (long) random.nextInt(1000) + 3000);
Snapshot snapshot = writeData(data, 3);
KeyValueFileStoreScan scan = store.newScan();
@@ -171,7 +169,7 @@ public class KeyValueFileStoreScanTest {
scan.withSnapshot(snapshot.id());
scan.withValueFilter(
new PredicateBuilder(TestKeyValueGenerator.DEFAULT_ROW_TYPE)
- .between(1, 1000, 1999));
+ .between(4, 1000L, 1999L));
List<ManifestEntry> filesFiltered = scan.plan().files();
@@ -180,16 +178,16 @@ public class KeyValueFileStoreScanTest {
}
@Test
- public void testWithValuePartitionFilter() throws Exception {
+ public void testWithValueFilterPartition() throws Exception {
ThreadLocalRandom random = ThreadLocalRandom.current();
- List<KeyValue> data = generateData(100,
Math.abs(random.nextInt(1000)));
- writeData(data, "0", 0);
- data = generateData(100, Math.abs(random.nextInt(1000)) + 1000);
- writeData(data, "1", 0);
- data = generateData(100, Math.abs(random.nextInt(1000)) + 2000);
- writeData(data, "2", 0);
- generateData(100, Math.abs(random.nextInt(1000)) + 3000);
- Snapshot snapshot = writeData(data, "3", 0);
+ List<KeyValue> data = generateData(100, 0, (long)
Math.abs(random.nextInt(1000)));
+ writeData(data, 0);
+ data = generateData(100, 1, (long) Math.abs(random.nextInt(1000)) +
1000);
+ writeData(data, 0);
+ data = generateData(100, 2, (long) Math.abs(random.nextInt(1000)) +
2000);
+ writeData(data, 0);
+ data = generateData(100, 3, (long) Math.abs(random.nextInt(1000)) +
3000);
+ Snapshot snapshot = writeData(data, 0);
KeyValueFileStoreScan scan = store.newScan();
scan.withSnapshot(snapshot.id());
@@ -199,7 +197,7 @@ public class KeyValueFileStoreScanTest {
scan.withSnapshot(snapshot.id());
scan.withValueFilter(
new PredicateBuilder(TestKeyValueGenerator.DEFAULT_ROW_TYPE)
- .between(1, 1000, 2000));
+ .between(4, 1000L, 1999L));
List<ManifestEntry> filesFiltered = scan.plan().files();
@@ -312,9 +310,13 @@ public class KeyValueFileStoreScanTest {
}
private List<KeyValue> generateData(int numRecords, int hr) {
+ return generateData(numRecords, hr, null);
+ }
+
+ private List<KeyValue> generateData(int numRecords, int hr, Long itemId) {
List<KeyValue> data = new ArrayList<>();
for (int i = 0; i < numRecords; i++) {
- data.add(gen.nextInsert("", hr, null, null, null));
+ data.add(gen.nextInsert("", hr, itemId, null, null));
}
return data;
}
@@ -329,16 +331,6 @@ public class KeyValueFileStoreScanTest {
return snapshots.get(snapshots.size() - 1);
}
- private Snapshot writeData(List<KeyValue> kvs, String partition, int
bucket) throws Exception {
- BinaryRow binaryRow = new BinaryRow(2);
- BinaryRowWriter binaryRowWriter = new BinaryRowWriter(binaryRow);
- binaryRowWriter.writeString(0, BinaryString.fromString(partition));
- binaryRowWriter.writeInt(1, 0);
- binaryRowWriter.complete();
- List<Snapshot> snapshots = store.commitData(kvs, p -> binaryRow, b ->
bucket);
- return snapshots.get(snapshots.size() - 1);
- }
-
private int getBucket(KeyValue kv) {
return (kv.key().hashCode() % NUM_BUCKETS + NUM_BUCKETS) % NUM_BUCKETS;
}
