This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a079fc3c28 [test][core] Fix test in DataEvolutionSplitGeneratorTest,
make it more accurate (#6495)
a079fc3c28 is described below
commit a079fc3c285071231d1cd8a45fa6e6e2c6e11a86
Author: YeJunHao <[email protected]>
AuthorDate: Wed Oct 29 19:12:47 2025 +0800
[test][core] Fix test in DataEvolutionSplitGeneratorTest, make it more
accurate (#6495)
---
.../source/DataEvolutionSplitGeneratorTest.java | 147 ++++++++-------------
1 file changed, 58 insertions(+), 89 deletions(-)
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/source/DataEvolutionSplitGeneratorTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/source/DataEvolutionSplitGeneratorTest.java
index 27e7841f9d..553d1731f9 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/source/DataEvolutionSplitGeneratorTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/source/DataEvolutionSplitGeneratorTest.java
@@ -23,8 +23,6 @@ import org.apache.paimon.manifest.FileSource;
import org.junit.jupiter.api.Test;
-import javax.annotation.Nullable;
-
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -35,49 +33,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
/** Test for {@link DataEvolutionSplitGenerator}. */
public class DataEvolutionSplitGeneratorTest {
- private static DataFileMeta createFile(
- String name, @Nullable Long firstRowId, long maxSequence) {
- return DataFileMeta.create(
- name,
- 10000L,
- 1,
- EMPTY_ROW,
- EMPTY_ROW,
- null,
- null,
- 0L,
- maxSequence,
- 0,
- 0,
- 0L,
- null,
- FileSource.APPEND,
- null,
- firstRowId,
- null);
- }
-
- @Test
- public void testSplitWithNullFirstRowId() {
- DataFileMeta file1 = createFile("file1", null, 10);
- DataFileMeta file2 = createFile("file2", 1L, 20);
- DataFileMeta file3 = createFile("file3", 1L, 30);
- DataFileMeta file4 = createFile("file4", null, 40);
-
- List<DataFileMeta> files = Arrays.asList(file1, file2, file3, file4);
- List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
-
- assertEquals(3, result.size());
- assertEquals(Collections.singletonList(file4), result.get(0));
- assertEquals(Collections.singletonList(file1), result.get(1));
- assertEquals(Arrays.asList(file3, file2), result.get(2));
- }
-
@Test
public void testSplitWithSameFirstRowId() {
- DataFileMeta file1 = createFile("file1", 1L, 10);
- DataFileMeta file2 = createFile("file2", 1L, 20);
- DataFileMeta file3 = createFile("file3", 1L, 30);
+ DataFileMeta file1 = createFile("file1.parquet", 1L, 1, 10);
+ DataFileMeta file2 = createFile("file2.parquet", 1L, 1, 20);
+ DataFileMeta file3 = createFile("file3.parquet", 1L, 1, 30);
List<DataFileMeta> files = Arrays.asList(file1, file2, file3);
List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
@@ -86,28 +46,13 @@ public class DataEvolutionSplitGeneratorTest {
assertEquals(Arrays.asList(file3, file2, file1), result.get(0));
}
- @Test
- public void testSplitWithDifferentFirstRowId() {
- DataFileMeta file1 = createFile("file1", 1L, 10);
- DataFileMeta file2 = createFile("file2", 2L, 20);
- DataFileMeta file3 = createFile("file3", 3L, 30);
-
- List<DataFileMeta> files = Arrays.asList(file1, file2, file3);
- List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
-
- assertEquals(3, result.size());
- assertEquals(Collections.singletonList(file1), result.get(0));
- assertEquals(Collections.singletonList(file2), result.get(1));
- assertEquals(Collections.singletonList(file3), result.get(2));
- }
-
@Test
public void testSplitWithMixedFirstRowId() {
- DataFileMeta file1 = createFile("file1", 1L, 10);
- DataFileMeta file2 = createFile("file2", 2L, 20);
- DataFileMeta file3 = createFile("file3", 1L, 30);
- DataFileMeta file4 = createFile("file4", 2L, 40);
- DataFileMeta file5 = createFile("file5", 3L, 50);
+ DataFileMeta file1 = createFile("file1.parquet", 1L, 1, 1);
+ DataFileMeta file2 = createFile("file2.parquet", 2L, 1, 2);
+ DataFileMeta file3 = createFile("file3.parquet", 1L, 1, 3);
+ DataFileMeta file4 = createFile("file4.parquet", 2L, 1, 4);
+ DataFileMeta file5 = createFile("file5.parquet", 3L, 1, 5);
List<DataFileMeta> files = Arrays.asList(file1, file2, file3, file4,
file5);
List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
@@ -120,41 +65,65 @@ public class DataEvolutionSplitGeneratorTest {
@Test
public void testSplitWithComplexScenario() {
- DataFileMeta file1 = createFile("file1", 1L, 30);
- DataFileMeta file2 = createFile("file2", 2L, 40);
- DataFileMeta file3 = createFile("file3", null, 10);
- DataFileMeta file4 = createFile("file4", 1L, 20);
- DataFileMeta file5 = createFile("file5", null, 50);
- DataFileMeta file6 = createFile("file6", 2L, 60);
- DataFileMeta file7 = createFile("file7", 3L, 70);
- DataFileMeta file8 = createFile("file8", 3L, 80);
- DataFileMeta file9 = createFile("file9", null, 90);
+ DataFileMeta file1 = createFile("file1.parquet", 1L, 1, 1);
+ DataFileMeta file2 = createFile("file2.parquet", 2L, 1, 3);
+ DataFileMeta file3 = createFile("file3.parquet", 3L, 1, 5);
+ DataFileMeta file4 = createFile("file4.parquet", 1L, 1, 2);
+ DataFileMeta file5 = createFile("file5.parquet", 4L, 1, 8);
+ DataFileMeta file6 = createFile("file6.parquet", 2L, 1, 4);
+ DataFileMeta file7 = createFile("file7.parquet", 3L, 1, 6);
+ DataFileMeta file8 = createFile("file8.parquet", 3L, 1, 7);
+ DataFileMeta file9 = createFile("file9.parquet", 5L, 1, 9);
List<DataFileMeta> files =
Arrays.asList(file1, file2, file3, file4, file5, file6, file7,
file8, file9);
List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
- assertEquals(6, result.size());
- assertEquals(Collections.singletonList(file9), result.get(0));
- assertEquals(Collections.singletonList(file5), result.get(1));
- assertEquals(Collections.singletonList(file3), result.get(2));
- assertEquals(Arrays.asList(file1, file4), result.get(3));
- assertEquals(Arrays.asList(file6, file2), result.get(4));
- assertEquals(Arrays.asList(file8, file7), result.get(5));
+ assertEquals(5, result.size());
+ assertEquals(Arrays.asList(file4, file1), result.get(0));
+ assertEquals(Arrays.asList(file6, file2), result.get(1));
+ assertEquals(Arrays.asList(file8, file7, file3), result.get(2));
+ assertEquals(Collections.singletonList(file5), result.get(3));
+ assertEquals(Collections.singletonList(file9), result.get(4));
}
@Test
- public void testOnlyNullFirstRowId() {
- DataFileMeta file1 = createFile("file1", null, 10);
- DataFileMeta file2 = createFile("file2", null, 20);
- DataFileMeta file3 = createFile("file3", null, 30);
-
- List<DataFileMeta> files = Arrays.asList(file2, file1, file3);
+ public void testSplitWithMultipleBlobFilesPerGroup() {
+ DataFileMeta file1 = createFile("file1.parquet", 1L, 10, 1);
+ DataFileMeta file2 = createFile("file2.blob", 1L, 1, 1);
+ DataFileMeta file3 = createFile("file3.blob", 2L, 9, 1);
+ DataFileMeta file4 = createFile("file4.parquet", 20L, 10, 2);
+ DataFileMeta file5 = createFile("file5.blob", 20L, 5, 2);
+ DataFileMeta file6 = createFile("file6.blob", 25L, 5, 2);
+ DataFileMeta file7 = createFile("file7.parquet", 1L, 10, 3);
+
+ List<DataFileMeta> files = Arrays.asList(file1, file2, file3, file4,
file5, file6, file7);
List<List<DataFileMeta>> result =
DataEvolutionSplitGenerator.split(files);
- assertEquals(3, result.size());
- assertEquals(Collections.singletonList(file3), result.get(0));
- assertEquals(Collections.singletonList(file2), result.get(1));
- assertEquals(Collections.singletonList(file1), result.get(2));
+ assertEquals(2, result.size());
+ assertEquals(Arrays.asList(file7, file1, file2, file3), result.get(0));
+ assertEquals(Arrays.asList(file4, file5, file6), result.get(1));
+ }
+
+ private static DataFileMeta createFile(
+ String name, long firstRowId, long rowCount, long maxSequence) {
+ return DataFileMeta.create(
+ name,
+ 10000L,
+ (int) rowCount,
+ EMPTY_ROW,
+ EMPTY_ROW,
+ null,
+ null,
+ 0L,
+ maxSequence,
+ 0,
+ 0,
+ 0L,
+ null,
+ FileSource.APPEND,
+ null,
+ firstRowId,
+ null);
}
}
