This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6c7f32cab [core] remove
PrimaryKeyFileStoreTableTest.testStreamingChangelogCompatibility02 test method
(#4170)
6c7f32cab is described below
commit 6c7f32cabf7733156eebe9e3184f8314794a002c
Author: Kerwin <[email protected]>
AuthorDate: Wed Sep 11 18:40:33 2024 +0800
[core] remove
PrimaryKeyFileStoreTableTest.testStreamingChangelogCompatibility02 test method
(#4170)
---
.../shortcodes/generated/core_configuration.html | 14 +--
.../main/java/org/apache/paimon/CoreOptions.java | 4 +-
.../org/apache/paimon/io/DataFilePathFactory.java | 15 +---
.../mergetree/ChangelogMergeTreeRewriterTest.java | 21 +++--
.../paimon/operation/OrphanFilesCleanTest.java | 6 +-
.../paimon/table/PrimaryKeyFileStoreTableTest.java | 97 ---------------------
.../compatibility/table-changelog-0.2.zip | Bin 11903 -> 0 bytes
7 files changed, 24 insertions(+), 133 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 65fa26a51..de6c683aa 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -64,7 +64,7 @@ under the License.
</tr>
<tr>
<td><h5>changelog-file.prefix</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
+ <td style="word-wrap: break-word;">"changelog-"</td>
<td>String</td>
<td>Specify the file name prefix of changelog files.</td>
</tr>
@@ -212,6 +212,12 @@ under the License.
<td>Duration</td>
<td>The TTL in rocksdb index for cross partition upsert (primary
keys not contain all partition fields), this can avoid maintaining too many
indexes and lead to worse and worse performance, but please note that this may
also cause data duplication.</td>
</tr>
+ <tr>
+ <td><h5>data-file.prefix</h5></td>
+ <td style="word-wrap: break-word;">"data-"</td>
+ <td>String</td>
+ <td>Specify the file name prefix of data files.</td>
+ </tr>
<tr>
<td><h5>delete-file.thread-num</h5></td>
<td style="word-wrap: break-word;">(none)</td>
@@ -326,12 +332,6 @@ under the License.
<td>Map</td>
<td>Define different file format for different level, you can add
the conf like this: 'file.format.per.level' = '0:avro,3:parquet', if the file
format for level is not provided, the default format which set by `file.format`
will be used.</td>
</tr>
- <tr>
- <td><h5>data-file.prefix</h5></td>
- <td style="word-wrap: break-word;">(none)</td>
- <td>String</td>
- <td>Specify the file name prefix of data files.</td>
- </tr>
<tr>
<td><h5>force-lookup</h5></td>
<td style="word-wrap: break-word;">false</td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index a7a744197..0f024c9f9 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -174,13 +174,13 @@ public class CoreOptions implements Serializable {
public static final ConfigOption<String> DATA_FILE_PREFIX =
key("data-file.prefix")
.stringType()
- .noDefaultValue()
+ .defaultValue("data-")
.withDescription("Specify the file name prefix of data
files.");
public static final ConfigOption<String> CHANGELOG_FILE_PREFIX =
key("changelog-file.prefix")
.stringType()
- .noDefaultValue()
+ .defaultValue("changelog-")
.withDescription("Specify the file name prefix of
changelog files.");
public static final ConfigOption<MemorySize> FILE_BLOCK_SIZE =
diff --git
a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
index 163efb0e7..742888b36 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFilePathFactory.java
@@ -20,7 +20,6 @@ package org.apache.paimon.io;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.fs.Path;
-import org.apache.paimon.utils.StringUtils;
import javax.annotation.concurrent.ThreadSafe;
@@ -31,10 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
@ThreadSafe
public class DataFilePathFactory {
- public static final String DEFAULT_DATA_FILE_PREFIX = "data-";
-
- public static final String DEFAULT_CHANGELOG_FILE_PREFIX = "changelog-";
-
public static final String INDEX_PATH_SUFFIX = ".index";
private final Path parent;
@@ -59,17 +54,11 @@ public class DataFilePathFactory {
}
public Path newPath() {
- if (!StringUtils.isNullOrWhitespaceOnly(dataFilePrefix)) {
- return newPath(dataFilePrefix);
- }
- return newPath(DEFAULT_DATA_FILE_PREFIX);
+ return newPath(dataFilePrefix);
}
public Path newChangelogPath() {
- if (!StringUtils.isNullOrWhitespaceOnly(changelogFilePrefix)) {
- return newPath(changelogFilePrefix);
- }
- return newPath(DEFAULT_CHANGELOG_FILE_PREFIX);
+ return newPath(changelogFilePrefix);
}
private Path newPath(String prefix) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
index 59106491d..bbfa4d365 100644
---
a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java
@@ -28,7 +28,6 @@ import org.apache.paimon.format.FlushingFileFormat;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.io.FileReaderFactory;
import org.apache.paimon.io.KeyValueFileReaderFactory;
import org.apache.paimon.io.KeyValueFileWriterFactory;
@@ -109,13 +108,14 @@ public class ChangelogMergeTreeRewriterTest {
public void testRewriteFailAndCleanupFiles(boolean rewriteChangelog)
throws Exception {
List<List<SortedRun>> sections = createTestSections(2);
Path testPath = new Path(path, UUID.randomUUID().toString());
+ CoreOptions coreOptions = new CoreOptions(new Options());
try (ChangelogMergeTreeRewriter rewriter =
new TestRewriter(
createReaderFactory(schemaManager, tableSchema,
keyType, valueType),
createWriterFactory(testPath, keyType, valueType),
comparator,
new MergeSorter(
- new CoreOptions(new Options()),
+ coreOptions,
tableSchema.logicalPrimaryKeysType(),
tableSchema.logicalRowType(),
null),
@@ -136,13 +136,12 @@ public class ChangelogMergeTreeRewriterTest {
p.getFileName()
.toString()
.startsWith(
-
DataFilePathFactory
-
.DEFAULT_DATA_FILE_PREFIX)
+
coreOptions.dataFilePrefix())
|| p.getFileName()
.toString()
.startsWith(
-
DataFilePathFactory
-
.DEFAULT_CHANGELOG_FILE_PREFIX))
+ coreOptions
+
.changelogFilePrefix()))
.collect(Collectors.toList());
Assertions.assertEquals(0, files.size());
}
@@ -153,13 +152,14 @@ public class ChangelogMergeTreeRewriterTest {
public void testRewriteSuccess(boolean rewriteChangelog) throws Exception {
List<List<SortedRun>> sections = createTestSections(2);
Path testPath = new Path(path, UUID.randomUUID().toString());
+ CoreOptions coreOptions = new CoreOptions(new Options());
try (ChangelogMergeTreeRewriter rewriter =
new TestRewriter(
createReaderFactory(schemaManager, tableSchema,
keyType, valueType),
createWriterFactory(testPath, keyType, valueType),
comparator,
new MergeSorter(
- new CoreOptions(new Options()),
+ coreOptions,
tableSchema.logicalPrimaryKeysType(),
tableSchema.logicalRowType(),
null),
@@ -175,13 +175,12 @@ public class ChangelogMergeTreeRewriterTest {
p.getFileName()
.toString()
.startsWith(
-
DataFilePathFactory
-
.DEFAULT_DATA_FILE_PREFIX)
+
coreOptions.dataFilePrefix())
|| p.getFileName()
.toString()
.startsWith(
-
DataFilePathFactory
-
.DEFAULT_CHANGELOG_FILE_PREFIX))
+ coreOptions
+
.changelogFilePrefix()))
.collect(Collectors.toList());
if (rewriteChangelog) {
Assertions.assertEquals(2, files.size()); // changelog + data
file
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java
index 4260cd0e6..9135015d7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java
@@ -80,8 +80,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
-import static
org.apache.paimon.io.DataFilePathFactory.DEFAULT_CHANGELOG_FILE_PREFIX;
-import static
org.apache.paimon.io.DataFilePathFactory.DEFAULT_DATA_FILE_PREFIX;
import static org.apache.paimon.utils.BranchManager.branchPath;
import static org.apache.paimon.utils.FileStorePathFactory.BUCKET_PATH_PREFIX;
import static org.assertj.core.api.Assertions.assertThat;
@@ -587,7 +585,9 @@ public class OrphanFilesCleanTest {
path,
1,
Arrays.asList(
- DEFAULT_DATA_FILE_PREFIX,
DEFAULT_CHANGELOG_FILE_PREFIX, "UNKNOWN-"));
+ table.coreOptions().dataFilePrefix(),
+ table.coreOptions().changelogFilePrefix(),
+ "UNKNOWN-"));
}
addedFiles += corruptedBuckets.size();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
index 2723e61f4..47e46fba1 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
@@ -58,7 +58,6 @@ import org.apache.paimon.table.source.ScanMode;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.StreamTableScan;
import org.apache.paimon.table.source.TableRead;
-import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.table.system.AuditLogTable;
import org.apache.paimon.table.system.FileMonitorTable;
@@ -67,7 +66,6 @@ import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowKind;
import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.CompatibilityTestUtils;
import org.apache.paimon.utils.Pair;
import org.assertj.core.api.Assertions;
@@ -605,101 +603,6 @@ public class PrimaryKeyFileStoreTableTest extends
FileStoreTableTestBase {
"+U 2|40|242|binary|varbinary|mapKey:mapVal|multiset");
}
- @Test
- public void testStreamingChangelogCompatibility02() throws Exception {
- // already contains 2 commits
- CompatibilityTestUtils.unzip(
- "compatibility/table-changelog-0.2.zip",
tablePath.toUri().getPath());
- FileStoreTable table =
- createFileStoreTable(
- conf -> conf.set(CHANGELOG_PRODUCER,
ChangelogProducer.INPUT),
- COMPATIBILITY_ROW_TYPE);
-
- List<List<List<String>>> expected =
- Arrays.asList(
- // first changelog snapshot
- Arrays.asList(
- // partition 1
- Arrays.asList(
- "+I 1|10|100|binary|varbinary",
- "+I 1|20|200|binary|varbinary",
- "-D 1|10|100|binary|varbinary",
- "+I 1|10|101|binary|varbinary",
- "-U 1|10|101|binary|varbinary",
- "+U 1|10|102|binary|varbinary"),
- // partition 2
- Collections.singletonList("+I
2|10|300|binary|varbinary")),
- // second changelog snapshot
- Arrays.asList(
- // partition 1
- Collections.singletonList("-D
1|20|200|binary|varbinary"),
- // partition 2
- Arrays.asList(
- "-U 2|10|300|binary|varbinary",
- "+U 2|10|301|binary|varbinary",
- "+I 2|20|400|binary|varbinary")),
- // third changelog snapshot
- Arrays.asList(
- // partition 1
- Arrays.asList(
- "-U 1|10|102|binary|varbinary",
- "+U 1|10|103|binary|varbinary",
- "+I 1|20|201|binary|varbinary"),
- // partition 2
- Collections.singletonList("-D
2|10|301|binary|varbinary")));
-
- StreamTableScan scan = table.newStreamScan();
- scan.restore(1L);
-
- Function<Integer, Void> assertNextSnapshot =
- i -> {
- TableScan.Plan plan = scan.plan();
- assertThat(plan).isNotNull();
-
- List<Split> splits = plan.splits();
- TableRead read = table.newRead();
- for (int j = 0; j < 2; j++) {
- try {
- assertThat(
- getResult(
- read,
- splits,
- binaryRow(j + 1),
- 0,
-
COMPATIBILITY_CHANGELOG_ROW_TO_STRING))
- .isEqualTo(expected.get(i).get(j));
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- return null;
- };
-
- for (int i = 0; i < 2; i++) {
- assertNextSnapshot.apply(i);
- }
-
- // no more changelog
- assertThat(scan.plan().splits()).isEmpty();
-
- // write another commit
- StreamTableWrite write = table.newWrite(commitUser);
- StreamTableCommit commit = table.newCommit(commitUser);
- write.write(rowDataWithKind(RowKind.UPDATE_BEFORE, 1, 10, 102L));
- write.write(rowDataWithKind(RowKind.UPDATE_AFTER, 1, 10, 103L));
- write.write(rowDataWithKind(RowKind.INSERT, 1, 20, 201L));
- write.write(rowDataWithKind(RowKind.DELETE, 2, 10, 301L));
- commit.commit(2, write.prepareCommit(true, 2));
- write.close();
- commit.close();
-
- assertNextSnapshot.apply(2);
-
- // no more changelog
- assertThat(scan.plan().splits()).isEmpty();
- }
-
private void writeData() throws Exception {
FileStoreTable table = createFileStoreTable();
StreamTableWrite write = table.newWrite(commitUser);
diff --git
a/paimon-core/src/test/resources/compatibility/table-changelog-0.2.zip
b/paimon-core/src/test/resources/compatibility/table-changelog-0.2.zip
deleted file mode 100644
index bb202764a..000000000
Binary files
a/paimon-core/src/test/resources/compatibility/table-changelog-0.2.zip and
/dev/null differ
