This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 2a5a256046 [core] Avoid strict-mode conflicts for index-only compact
(#8996)
2a5a256046 is described below
commit 2a5a2560460357d3dba64d69496f412d5090a5da
Author: LsomeYeah <[email protected]>
AuthorDate: Mon Aug 3 18:01:37 2026 +0800
[core] Avoid strict-mode conflicts for index-only compact (#8996)
---
.../paimon/operation/commit/StrictModeChecker.java | 19 ++++--
.../apache/paimon/table/sink/TableCommitTest.java | 77 ++++++++++++++++++++++
2 files changed, 91 insertions(+), 5 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
index fa2a972051..52944ea713 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
@@ -68,7 +68,13 @@ public class StrictModeChecker {
}
if (snapshot.commitKind() == CommitKind.COMPACT
|| snapshot.commitKind() == CommitKind.OVERWRITE) {
- if (hasOverlappedPartition(snapshot, newPartitions)) {
+ boolean hasOverlap = hasOverlappedDataPartition(snapshot,
newPartitions);
+ // OVERWRITE may contain logical changes represented only by
deletion vectors,
+ // while index-only COMPACT does not change table data.
+ if (!hasOverlap && snapshot.commitKind() ==
CommitKind.OVERWRITE) {
+ hasOverlap = hasOverlappedIndexPartition(snapshot,
newPartitions);
+ }
+ if (hasOverlap) {
throw new RuntimeException(
String.format(
"When trying to commit snapshot %d, "
@@ -109,7 +115,7 @@ public class StrictModeChecker {
}
}
- private boolean hasOverlappedPartition(Snapshot snapshot, Set<BinaryRow>
newPartitions) {
+ private boolean hasOverlappedDataPartition(Snapshot snapshot,
Set<BinaryRow> newPartitions) {
if (newPartitions.isEmpty()) {
return false;
}
@@ -120,10 +126,13 @@ public class StrictModeChecker {
.withKind(ScanMode.DELTA)
.dropStats()
.readFileIterator();
- if (hasOverlappedPartition(entries, newPartitions)) {
- return true;
- }
+ return hasOverlappedPartition(entries, newPartitions);
+ }
+ private boolean hasOverlappedIndexPartition(Snapshot snapshot,
Set<BinaryRow> newPartitions) {
+ if (newPartitions.isEmpty()) {
+ return false;
+ }
String indexManifest = snapshot.indexManifest();
if (indexManifest == null) {
return false;
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
index 25abbaf12b..ee11348b31 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
@@ -25,6 +25,7 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.deletionvectors.BucketedDvMaintainer;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
@@ -541,6 +542,82 @@ public class TableCommitTest {
commit2.close();
}
+ @Test
+ public void testStrictModeIgnoresIndexOnlyCompact() throws Exception {
+ String path = tempDir.toString();
+ RowType rowType =
+ RowType.of(
+ new DataType[] {DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()},
+ new String[] {"pt", "k", "v"});
+
+ Options options = new Options();
+ options.set(CoreOptions.PATH, path);
+ options.set(CoreOptions.BUCKET, 1);
+ options.set(CoreOptions.BUCKET_KEY, "k");
+ options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10);
+ TableSchema tableSchema =
+ SchemaUtils.forceCommit(
+ new SchemaManager(LocalFileIO.create(), new
Path(path)),
+ new Schema(
+ rowType.getFields(),
+ Collections.singletonList("pt"),
+ Collections.emptyList(),
+ options.toMap(),
+ ""));
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ LocalFileIO.create(),
+ new Path(path),
+ tableSchema,
+ CatalogEnvironment.empty());
+ BinaryRow pt1 = partitionRow(1);
+
+ String user1 = UUID.randomUUID().toString();
+ TableWriteImpl<?> write1 = table.newWrite(user1);
+ TableCommitImpl commit1 = table.newCommit(user1);
+ write1.write(GenericRow.of(1, 0, 0L));
+ commit1.commit(1, write1.prepareCommit(false, 1));
+
+ String user2 = UUID.randomUUID().toString();
+ FileStoreTable tableWithStrict =
+
table.copy(singletonMap(COMMIT_STRICT_MODE_LAST_SAFE_SNAPSHOT.key(), "1"));
+ TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
+ TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
+ write2.write(GenericRow.of(1, 1, 1L));
+ write2.compact(pt1, 0, true);
+
+ IndexFileMeta btreeIndex =
+ new IndexFileMeta(
+ "btree",
+ "index-only-compact",
+ 1,
+ 1,
+ new GlobalIndexMeta(0, 0, 1, null, null),
+ null);
+ commit1.commit(
+ 2,
+ Collections.singletonList(
+ new CommitMessageImpl(
+ pt1,
+ 0,
+ 1,
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(btreeIndex),
+ Collections.emptyList()))));
+
+ assertThatCode(() -> commit2.commit(1, write2.prepareCommit(true, 1)))
+ .doesNotThrowAnyException();
+
+ write1.close();
+ commit1.close();
+ write2.close();
+ commit2.close();
+ }
+
@Test
public void testStrictModeForDvOnlyOverwrite() throws Exception {
// Regression test for the partition-overlap check on DV-only OVERWRITE
