This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e6f10e3a7 [core] Disable snapshot.clean-empty-directories by default
(#3749)
e6f10e3a7 is described below
commit e6f10e3a7278b8ab2da0dda72382b74125672794
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Jul 15 15:17:52 2024 +0800
[core] Disable snapshot.clean-empty-directories by default (#3749)
---
.../shortcodes/generated/core_configuration.html | 6 ++---
.../main/java/org/apache/paimon/CoreOptions.java | 21 ++++++++++-------
.../java/org/apache/paimon/AbstractFileStore.java | 9 +++++---
.../apache/paimon/operation/ChangelogDeletion.java | 13 +++++++++--
.../apache/paimon/operation/FileDeletionBase.java | 9 +++++---
.../apache/paimon/operation/SnapshotDeletion.java | 12 ++++++++--
.../org/apache/paimon/operation/TagDeletion.java | 12 ++++++++--
.../paimon/table/AbstractFileStoreTable.java | 10 ++-------
.../apache/paimon/table/ExpireChangelogImpl.java | 9 ++------
.../apache/paimon/table/ExpireSnapshotsImpl.java | 9 ++------
.../org/apache/paimon/table/RollbackHelper.java | 6 ++---
.../java/org/apache/paimon/utils/TagManager.java | 2 +-
.../test/java/org/apache/paimon/TestFileStore.java | 26 ++++++----------------
.../paimon/operation/ExpireSnapshotsTest.java | 6 ++---
.../apache/paimon/operation/FileDeletionTest.java | 11 +++++----
.../paimon/table/FileStoreTableTestBase.java | 7 ++++--
.../paimon/table/PrimaryKeyFileStoreTableTest.java | 2 +-
17 files changed, 92 insertions(+), 78 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 81f80c0fd..9faeb0865 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -630,10 +630,10 @@ This config option does not affect the default filesystem
metastore.</td>
<td>The time zone to parse the long watermark value to TIMESTAMP
value. The default value is 'UTC', which means the watermark is defined on
TIMESTAMP column or not defined. If the watermark is defined on TIMESTAMP_LTZ
column, the time zone of watermark is user configured time zone, the value
should be the user configured local time zone. The option value is either a
full name such as 'America/Los_Angeles', or a custom timezone id such as
'GMT-08:00'.</td>
</tr>
<tr>
- <td><h5>snapshot.expire.clean-empty-directories</h5></td>
- <td style="word-wrap: break-word;">true</td>
+ <td><h5>snapshot.clean-empty-directories</h5></td>
+ <td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
- <td>Whether to try to clean empty directories when expiring
snapshots. Note that trying to clean directories might throw exceptions in
filesystem, but in most cases it won't cause problems.</td>
+ <td>Whether to try to clean empty directories when expiring
snapshots, if enabled, please note:<ul><li>hdfs: may print exceptions in
NameNode.</li><li>oss/s3: may cause performance issue.</li></ul></td>
</tr>
<tr>
<td><h5>snapshot.expire.execution-mode</h5></td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 69c236b9d..0ec6d59a0 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -282,14 +282,19 @@ public class CoreOptions implements Serializable {
.withDescription(
"The maximum number of snapshots allowed to expire
at a time.");
- public static final ConfigOption<Boolean>
SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES =
- key("snapshot.expire.clean-empty-directories")
+ public static final ConfigOption<Boolean> SNAPSHOT_CLEAN_EMPTY_DIRECTORIES
=
+ key("snapshot.clean-empty-directories")
.booleanType()
- .defaultValue(true)
+ .defaultValue(false)
+
.withDeprecatedKeys("snapshot.expire.clean-empty-directories")
.withDescription(
- "Whether to try to clean empty directories when
expiring snapshots. "
- + "Note that trying to clean directories
might throw exceptions in filesystem, "
- + "but in most cases it won't cause
problems.");
+ Description.builder()
+ .text(
+ "Whether to try to clean empty
directories when expiring snapshots, if enabled, please note:")
+ .list(
+ text("hdfs: may print exceptions
in NameNode."),
+ text("oss/s3: may cause
performance issue."))
+ .build());
public static final ConfigOption<Duration> CONTINUOUS_DISCOVERY_INTERVAL =
key("continuous.discovery-interval")
@@ -1480,8 +1485,8 @@ public class CoreOptions implements Serializable {
return options.get(SNAPSHOT_EXPIRE_LIMIT);
}
- public boolean snapshotExpireCleanEmptyDirectories() {
- return options.get(SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES);
+ public boolean cleanEmptyDirectories() {
+ return options.get(SNAPSHOT_CLEAN_EMPTY_DIRECTORIES);
}
public ExpireConfig expireConfig() {
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index 90dfbc157..bdbeedd19 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -217,7 +217,8 @@ abstract class AbstractFileStore<T> implements FileStore<T>
{
manifestListFactory().create(),
newIndexFileHandler(),
newStatsFileHandler(),
- options.changelogProducer() !=
CoreOptions.ChangelogProducer.NONE);
+ options.changelogProducer() !=
CoreOptions.ChangelogProducer.NONE,
+ options.cleanEmptyDirectories());
}
@Override
@@ -228,7 +229,8 @@ abstract class AbstractFileStore<T> implements FileStore<T>
{
manifestFileFactory().create(),
manifestListFactory().create(),
newIndexFileHandler(),
- newStatsFileHandler());
+ newStatsFileHandler(),
+ options.cleanEmptyDirectories());
}
@Override
@@ -244,7 +246,8 @@ abstract class AbstractFileStore<T> implements FileStore<T>
{
manifestFileFactory().create(),
manifestListFactory().create(),
newIndexFileHandler(),
- newStatsFileHandler());
+ newStatsFileHandler(),
+ options.cleanEmptyDirectories());
}
public abstract Comparator<InternalRow> newKeyComparator();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
index 198c66a1c..ea67f7e45 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/ChangelogDeletion.java
@@ -38,14 +38,23 @@ import java.util.function.Predicate;
/** Delete changelog files. */
public class ChangelogDeletion extends FileDeletionBase<Changelog> {
+
public ChangelogDeletion(
FileIO fileIO,
FileStorePathFactory pathFactory,
ManifestFile manifestFile,
ManifestList manifestList,
IndexFileHandler indexFileHandler,
- StatsFileHandler statsFileHandler) {
- super(fileIO, pathFactory, manifestFile, manifestList,
indexFileHandler, statsFileHandler);
+ StatsFileHandler statsFileHandler,
+ boolean cleanEmptyDirectories) {
+ super(
+ fileIO,
+ pathFactory,
+ manifestFile,
+ manifestList,
+ indexFileHandler,
+ statsFileHandler,
+ cleanEmptyDirectories);
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index f8d777ab7..fb09acdc2 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -70,6 +70,7 @@ public abstract class FileDeletionBase<T extends Snapshot> {
protected final ManifestList manifestList;
protected final IndexFileHandler indexFileHandler;
protected final StatsFileHandler statsFileHandler;
+ private final boolean cleanEmptyDirectories;
protected final Map<BinaryRow, Set<Integer>> deletionBuckets;
protected final Executor ioExecutor;
@@ -87,13 +88,15 @@ public abstract class FileDeletionBase<T extends Snapshot> {
ManifestFile manifestFile,
ManifestList manifestList,
IndexFileHandler indexFileHandler,
- StatsFileHandler statsFileHandler) {
+ StatsFileHandler statsFileHandler,
+ boolean cleanEmptyDirectories) {
this.fileIO = fileIO;
this.pathFactory = pathFactory;
this.manifestFile = manifestFile;
this.manifestList = manifestList;
this.indexFileHandler = indexFileHandler;
this.statsFileHandler = statsFileHandler;
+ this.cleanEmptyDirectories = cleanEmptyDirectories;
this.deletionBuckets = new HashMap<>();
this.ioExecutor = FileUtils.COMMON_IO_FORK_JOIN_POOL;
}
@@ -121,8 +124,8 @@ public abstract class FileDeletionBase<T extends Snapshot> {
}
/** Try to delete data directories that may be empty after data file
deletion. */
- public void cleanDataDirectories() {
- if (deletionBuckets.isEmpty()) {
+ public void cleanEmptyDirectories() {
+ if (!cleanEmptyDirectories || deletionBuckets.isEmpty()) {
return;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
index e83056638..91400ae50 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/SnapshotDeletion.java
@@ -49,8 +49,16 @@ public class SnapshotDeletion extends
FileDeletionBase<Snapshot> {
ManifestList manifestList,
IndexFileHandler indexFileHandler,
StatsFileHandler statsFileHandler,
- boolean produceChangelog) {
- super(fileIO, pathFactory, manifestFile, manifestList,
indexFileHandler, statsFileHandler);
+ boolean produceChangelog,
+ boolean cleanEmptyDirectories) {
+ super(
+ fileIO,
+ pathFactory,
+ manifestFile,
+ manifestList,
+ indexFileHandler,
+ statsFileHandler,
+ cleanEmptyDirectories);
this.produceChangelog = produceChangelog;
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java
b/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java
index 3b1174223..74eeaee90 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/TagDeletion.java
@@ -53,8 +53,16 @@ public class TagDeletion extends FileDeletionBase<Snapshot> {
ManifestFile manifestFile,
ManifestList manifestList,
IndexFileHandler indexFileHandler,
- StatsFileHandler statsFileHandler) {
- super(fileIO, pathFactory, manifestFile, manifestList,
indexFileHandler, statsFileHandler);
+ StatsFileHandler statsFileHandler,
+ boolean cleanEmptyDirectories) {
+ super(
+ fileIO,
+ pathFactory,
+ manifestFile,
+ manifestList,
+ indexFileHandler,
+ statsFileHandler,
+ cleanEmptyDirectories);
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index 3839896eb..a73f6150a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -317,19 +317,13 @@ abstract class AbstractFileStoreTable implements
FileStoreTable {
@Override
public ExpireSnapshots newExpireSnapshots() {
return new ExpireSnapshotsImpl(
- snapshotManager(),
- store().newSnapshotDeletion(),
- store().newTagManager(),
- coreOptions().snapshotExpireCleanEmptyDirectories());
+ snapshotManager(), store().newSnapshotDeletion(),
store().newTagManager());
}
@Override
public ExpireSnapshots newExpireChangelog() {
return new ExpireChangelogImpl(
- snapshotManager(),
- tagManager(),
- store().newChangelogDeletion(),
- coreOptions().snapshotExpireCleanEmptyDirectories());
+ snapshotManager(), tagManager(),
store().newChangelogDeletion());
}
@Override
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
index 33891997c..6d5e76d5b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireChangelogImpl.java
@@ -45,7 +45,6 @@ public class ExpireChangelogImpl implements ExpireSnapshots {
private final SnapshotManager snapshotManager;
private final ConsumerManager consumerManager;
private final ChangelogDeletion changelogDeletion;
- private final boolean cleanEmptyDirectories;
private final TagManager tagManager;
private ExpireConfig expireConfig;
@@ -53,13 +52,11 @@ public class ExpireChangelogImpl implements ExpireSnapshots
{
public ExpireChangelogImpl(
SnapshotManager snapshotManager,
TagManager tagManager,
- ChangelogDeletion changelogDeletion,
- boolean cleanEmptyDirectories) {
+ ChangelogDeletion changelogDeletion) {
this.snapshotManager = snapshotManager;
this.tagManager = tagManager;
this.consumerManager =
new ConsumerManager(snapshotManager.fileIO(),
snapshotManager.tablePath());
- this.cleanEmptyDirectories = cleanEmptyDirectories;
this.changelogDeletion = changelogDeletion;
this.expireConfig = ExpireConfig.builder().build();
}
@@ -162,9 +159,7 @@ public class ExpireChangelogImpl implements ExpireSnapshots
{
snapshotManager.fileIO().deleteQuietly(snapshotManager.longLivedChangelogPath(id));
}
- if (cleanEmptyDirectories) {
- changelogDeletion.cleanDataDirectories();
- }
+ changelogDeletion.cleanEmptyDirectories();
writeEarliestHintFile(endExclusiveId);
return (int) (endExclusiveId - earliestId);
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
index 48a2a1259..73fac37e6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ExpireSnapshotsImpl.java
@@ -47,21 +47,18 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots
{
private final ConsumerManager consumerManager;
private final SnapshotDeletion snapshotDeletion;
private final TagManager tagManager;
- private final boolean cleanEmptyDirectories;
private ExpireConfig expireConfig;
public ExpireSnapshotsImpl(
SnapshotManager snapshotManager,
SnapshotDeletion snapshotDeletion,
- TagManager tagManager,
- boolean cleanEmptyDirectories) {
+ TagManager tagManager) {
this.snapshotManager = snapshotManager;
this.consumerManager =
new ConsumerManager(snapshotManager.fileIO(),
snapshotManager.tablePath());
this.snapshotDeletion = snapshotDeletion;
this.tagManager = tagManager;
- this.cleanEmptyDirectories = cleanEmptyDirectories;
this.expireConfig = ExpireConfig.builder().build();
}
@@ -194,9 +191,7 @@ public class ExpireSnapshotsImpl implements ExpireSnapshots
{
// data files and changelog files in bucket directories has been
deleted
// then delete changed bucket directories if they are empty
- if (cleanEmptyDirectories) {
- snapshotDeletion.cleanDataDirectories();
- }
+ snapshotDeletion.cleanEmptyDirectories();
// delete manifests and indexFiles
List<Snapshot> skippingSnapshots =
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java
b/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java
index 499631836..374d71e9e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/RollbackHelper.java
@@ -137,7 +137,7 @@ public class RollbackHelper {
}
// delete directories
- snapshotDeletion.cleanDataDirectories();
+ snapshotDeletion.cleanEmptyDirectories();
return toBeCleaned;
}
@@ -165,7 +165,7 @@ public class RollbackHelper {
}
// delete directories
- snapshotDeletion.cleanDataDirectories();
+ snapshotDeletion.cleanEmptyDirectories();
// modify the latest hint
try {
@@ -221,7 +221,7 @@ public class RollbackHelper {
tagDeletion.cleanUnusedDataFiles(s, dataFileSkipper);
}
// delete directories
- tagDeletion.cleanDataDirectories();
+ tagDeletion.cleanEmptyDirectories();
}
return toBeCleaned;
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
index 3cfa22934..65c6c232d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
@@ -230,7 +230,7 @@ public class TagManager {
}
if (success) {
tagDeletion.cleanUnusedDataFiles(taggedSnapshot, dataFileSkipper);
- tagDeletion.cleanDataDirectories();
+ tagDeletion.cleanEmptyDirectories();
}
// delete manifests
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index 6a73a120e..db822d7be 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -155,19 +155,10 @@ public class TestFileStore extends KeyValueFileStore {
}
public ExpireSnapshots newExpire(int numRetainedMin, int numRetainedMax,
long millisRetained) {
- return newExpire(numRetainedMin, numRetainedMax, millisRetained, true);
- }
-
- public ExpireSnapshots newExpire(
- int numRetainedMin,
- int numRetainedMax,
- long millisRetained,
- boolean snapshotExpireCleanEmptyDirectories) {
return new ExpireSnapshotsImpl(
snapshotManager(),
newSnapshotDeletion(),
- new TagManager(fileIO, options.path()),
- snapshotExpireCleanEmptyDirectories)
+ new TagManager(fileIO, options.path()))
.config(
ExpireConfig.builder()
.snapshotRetainMax(numRetainedMax)
@@ -176,24 +167,20 @@ public class TestFileStore extends KeyValueFileStore {
.build());
}
- public ExpireSnapshots newExpire(
- ExpireConfig expireConfig, boolean
snapshotExpireCleanEmptyDirectories) {
+ public ExpireSnapshots newExpire(ExpireConfig expireConfig) {
return new ExpireSnapshotsImpl(
snapshotManager(),
newSnapshotDeletion(),
- new TagManager(fileIO, options.path()),
- snapshotExpireCleanEmptyDirectories)
+ new TagManager(fileIO, options.path()))
.config(expireConfig);
}
- public ExpireSnapshots newChangelogExpire(
- ExpireConfig config, boolean snapshotExpireCleanEmptyDirectories) {
+ public ExpireSnapshots newChangelogExpire(ExpireConfig config) {
ExpireChangelogImpl impl =
new ExpireChangelogImpl(
snapshotManager(),
new TagManager(fileIO, options.path()),
- newChangelogDeletion(),
- snapshotExpireCleanEmptyDirectories);
+ newChangelogDeletion());
impl.config(config);
return impl;
}
@@ -777,7 +764,8 @@ public class TestFileStore extends KeyValueFileStore {
}
public TestFileStore build() {
- Options conf = new Options();
+ Options conf =
+ tableSchema == null ? new Options() :
Options.fromMap(tableSchema.options());
conf.set(CoreOptions.WRITE_BUFFER_SIZE, WRITE_BUFFER_SIZE);
conf.set(CoreOptions.PAGE_SIZE, PAGE_SIZE);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
index 9b32842fa..bd32e48e5 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java
@@ -336,7 +336,7 @@ public class ExpireSnapshotsTest {
builder.snapshotRetainMin(1)
.snapshotRetainMax(Integer.MAX_VALUE)
.snapshotTimeRetain(Duration.ofMillis(1000));
- ExpireSnapshots expire = store.newExpire(builder.build(), true);
+ ExpireSnapshots expire = store.newExpire(builder.build());
List<KeyValue> allData = new ArrayList<>();
List<Integer> snapshotPositions = new ArrayList<>();
@@ -417,8 +417,8 @@ public class ExpireSnapshotsTest {
.changelogRetainMax(3)
.build();
- ExpireSnapshots snapshot = store.newExpire(config, true);
- ExpireSnapshots changelog = store.newChangelogExpire(config, true);
+ ExpireSnapshots snapshot = store.newExpire(config);
+ ExpireSnapshots changelog = store.newChangelogExpire(config);
// expire twice to check for idempotence
snapshot.expire();
snapshot.expire();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
index a648da19b..b6c80ad74 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
@@ -64,6 +64,7 @@ import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
+import static org.apache.paimon.CoreOptions.SNAPSHOT_CLEAN_EMPTY_DIRECTORIES;
import static
org.apache.paimon.operation.FileStoreCommitImpl.mustConflictCheck;
import static org.apache.paimon.operation.FileStoreTestUtils.assertPathExists;
import static
org.apache.paimon.operation.FileStoreTestUtils.assertPathNotExists;
@@ -160,7 +161,8 @@ public class FileDeletionTest {
cleanBucket(store, partition, 0);
// step 5: expire and check file paths
- store.newExpire(1, 1, Long.MAX_VALUE, cleanEmptyDirs).expire();
+
store.options().toConfiguration().set(SNAPSHOT_CLEAN_EMPTY_DIRECTORIES, true);
+ store.newExpire(1, 1, Long.MAX_VALUE).expire();
// check dt=0401
if (cleanEmptyDirs) {
@@ -671,8 +673,7 @@ public class FileDeletionTest {
// action: expire snapshot 1 -> delete tag1 -> expire snapshot 2
// result: exist A & B (because of tag2)
ExpireSnapshots expireSnapshots =
- new ExpireSnapshotsImpl(
- snapshotManager, store.newSnapshotDeletion(),
tagManager, true);
+ new ExpireSnapshotsImpl(snapshotManager,
store.newSnapshotDeletion(), tagManager);
expireSnapshots
.config(
ExpireConfig.builder()
@@ -732,13 +733,15 @@ public class FileDeletionTest {
}
SchemaManager schemaManager = new SchemaManager(fileIO, new
Path(root));
+
TableSchema tableSchema =
schemaManager.createTable(
new Schema(
rowType.getFields(),
partitionType.getFieldNames(),
TestKeyValueGenerator.getPrimaryKeys(mode),
- Collections.emptyMap(),
+ Collections.singletonMap(
+
SNAPSHOT_CLEAN_EMPTY_DIRECTORIES.key(), "true"),
null));
return new TestFileStore.Builder(
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
index fce09392c..daf5db716 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/FileStoreTableTestBase.java
@@ -108,6 +108,7 @@ import static
org.apache.paimon.CoreOptions.COMPACTION_MAX_FILE_NUM;
import static org.apache.paimon.CoreOptions.CONSUMER_IGNORE_PROGRESS;
import static org.apache.paimon.CoreOptions.ExpireExecutionMode;
import static org.apache.paimon.CoreOptions.FILE_FORMAT;
+import static org.apache.paimon.CoreOptions.SNAPSHOT_CLEAN_EMPTY_DIRECTORIES;
import static org.apache.paimon.CoreOptions.SNAPSHOT_EXPIRE_EXECUTION_MODE;
import static org.apache.paimon.CoreOptions.SNAPSHOT_EXPIRE_LIMIT;
import static org.apache.paimon.CoreOptions.SNAPSHOT_NUM_RETAINED_MAX;
@@ -965,12 +966,14 @@ public abstract class FileStoreTableTestBase {
private FileStoreTable prepareRollbackTable(int commitTimes) throws
Exception {
FileStoreTable table = createFileStoreTable();
- prepareRollbackTable(commitTimes, table);
- return table;
+ return prepareRollbackTable(commitTimes, table);
}
protected FileStoreTable prepareRollbackTable(int commitTimes,
FileStoreTable table)
throws Exception {
+ table =
+ table.copy(
+
Collections.singletonMap(SNAPSHOT_CLEAN_EMPTY_DIRECTORIES.key(), "true"));
StreamTableWrite write = table.newWrite(commitUser);
StreamTableCommit commit = table.newCommit(commitUser);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
index de4b14c33..c4867bcac 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeyFileStoreTableTest.java
@@ -1516,7 +1516,7 @@ public class PrimaryKeyFileStoreTableTest extends
FileStoreTableTestBase {
options ->
options.setString(
CoreOptions.CHANGELOG_PRODUCER.key(),
changelogProducer));
- prepareRollbackTable(commitTimes, table);
+ table = prepareRollbackTable(commitTimes, table);
int t1 = 1;
int t2 = commitTimes - 3;
