This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 70d114db4d [core] Refactor compact file identifiers
70d114db4d is described below
commit 70d114db4d5f6c839d2c751f6e7638f760e2e9d5
Author: JingsongLi <[email protected]>
AuthorDate: Tue Aug 11 13:44:16 2026 +0800
[core] Refactor compact file identifiers
---
.../DataEvolutionRowIdAssignmentPlanner.java | 6 +-
.../DataEvolutionGlobalIndexRefreshPlanner.java | 8 +-
.../paimon/manifest/BinaryManifestEntry.java | 102 -------
...ifierSet.java => CompactFileIdentifierSet.java} | 12 +-
.../java/org/apache/paimon/manifest/FileEntry.java | 109 ++++++++
.../operation/ManifestEntryExternalSort.java | 14 +-
.../paimon/operation/ManifestFileSorter.java | 20 +-
.../BinaryManifestEntryReusableIdentifierTest.java | 137 ----------
.../paimon/manifest/BinaryManifestEntryTest.java | 3 +-
...Test.java => CompactFileIdentifierSetTest.java} | 14 +-
.../manifest/FileEntryReusableIdentifierTest.java | 297 +++++++++++++++++++++
11 files changed, 444 insertions(+), 278 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdAssignmentPlanner.java
b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdAssignmentPlanner.java
index 0b241afde5..1dd3ae4508 100644
---
a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdAssignmentPlanner.java
+++
b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdAssignmentPlanner.java
@@ -26,8 +26,8 @@ import org.apache.paimon.io.BinaryDataFileMeta;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.BinaryManifestEntry;
import org.apache.paimon.manifest.BinaryManifestEntry.Projection;
-import org.apache.paimon.manifest.BinaryManifestEntry.ReusableIdentifier;
-import org.apache.paimon.manifest.DeletedIdentifierSet;
+import org.apache.paimon.manifest.CompactFileIdentifierSet;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
@@ -556,7 +556,7 @@ final class DataEvolutionRowIdAssignmentPlanner {
private static final class GroupState {
private final GroupPartitionDictionary partitions;
- private final DeletedIdentifierSet deletedIdentifiers = new
DeletedIdentifierSet();
+ private final CompactFileIdentifierSet deletedIdentifiers = new
CompactFileIdentifierSet();
private final LiveFileRowIdRangeCollector liveFileRanges;
private GroupState(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexRefreshPlanner.java
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexRefreshPlanner.java
index 36e11b38f4..53d110f827 100644
---
a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexRefreshPlanner.java
+++
b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexRefreshPlanner.java
@@ -25,7 +25,7 @@ import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.io.BinaryDataFileMeta;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.BinaryManifestEntry;
-import org.apache.paimon.manifest.DeletedIdentifierSet;
+import org.apache.paimon.manifest.CompactFileIdentifierSet;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.manifest.ManifestEntry;
@@ -127,7 +127,7 @@ public final class DataEvolutionGlobalIndexRefreshPlanner {
groupPartitions.add(key.getLeft());
}
- DeletedIdentifierSet deleted = new DeletedIdentifierSet();
+ CompactFileIdentifierSet deleted = new CompactFileIdentifierSet();
try {
collectDeletedIdentifiers(manifestFile, manifests,
groupPartitions, deleted);
collectUpdatedRanges(
@@ -178,7 +178,7 @@ public final class DataEvolutionGlobalIndexRefreshPlanner {
ManifestFile manifestFile,
List<ManifestFileMeta> manifests,
Set<BinaryRow> groupPartitions,
- DeletedIdentifierSet deleted) {
+ CompactFileIdentifierSet deleted) {
for (ManifestFileMeta manifest : manifests) {
if (manifest.numDeletedFiles() <= 0) {
continue;
@@ -204,7 +204,7 @@ public final class DataEvolutionGlobalIndexRefreshPlanner {
SchemaManager schemaManager,
ManifestFile manifestFile,
List<ManifestFileMeta> manifests,
- DeletedIdentifierSet deleted,
+ CompactFileIdentifierSet deleted,
Map<Pair<BinaryRow, Integer>, RefreshGroup> groups,
Set<Integer> indexedFieldIds) {
Map<Pair<Long, List<String>>, Set<Integer>> fileFieldIdsCache = new
HashMap<>();
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/BinaryManifestEntry.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/BinaryManifestEntry.java
index c66f165c96..646f19ae44 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/BinaryManifestEntry.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/BinaryManifestEntry.java
@@ -19,11 +19,9 @@
package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.io.BinaryDataFileMeta;
import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.memory.MemorySegmentUtils;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
@@ -284,106 +282,6 @@ public final class BinaryManifestEntry implements
ManifestEntry {
String.format("Binary manifest entry does not support %s.",
operation));
}
- /**
- * Reusable byte encoding of a binary manifest entry's identity fields.
- *
- * <p>The encoded identifier is the prefix of {@link #bytes()} ending at
{@link #length()}. It
- * is valid until the next call to {@link #replace(BinaryManifestEntry)}
or {@link #release()}
- * and must not be modified by callers.
- */
- public static final class ReusableIdentifier {
-
- private byte[] bytes = new byte[256];
- private int length;
-
- public ReusableIdentifier replace(BinaryManifestEntry entry) {
- checkArgument(entry != null, "Binary manifest entry cannot be
null.");
- length = 0;
- return appendEntryFields(entry);
- }
-
- /** Replaces this encoding with the entry's partition and identity
fields. */
- public ReusableIdentifier replaceWithPartition(BinaryManifestEntry
entry) {
- checkArgument(entry != null, "Binary manifest entry cannot be
null.");
- length = 0;
- putBytes(entry.partitionBytes());
- return appendEntryFields(entry);
- }
-
- private ReusableIdentifier appendEntryFields(BinaryManifestEntry
entry) {
- putInt(entry.bucket());
- BinaryDataFileMeta file = entry.file();
- putInt(file.level());
- putString(file.fileNameBinary());
-
- int extraFileCount = file.extraFileCount();
- putInt(extraFileCount);
- for (int i = 0; i < extraFileCount; i++) {
- putString(file.extraFile(i));
- }
-
- if (!file.hasEmbeddedIndex()) {
- putInt(-1);
- } else {
- putBytes(file.embeddedIndex());
- }
- if (!file.hasExternalPath()) {
- putInt(-1);
- } else {
- putString(file.externalPathBinary());
- }
- return this;
- }
-
- public byte[] bytes() {
- return bytes;
- }
-
- public int length() {
- return length;
- }
-
- public void release() {
- bytes = new byte[0];
- length = 0;
- }
-
- private void putString(BinaryString value) {
- checkState(value != null, "Manifest string field cannot be null.");
- int valueLength = value.getSizeInBytes();
- putInt(valueLength);
- ensureCapacity(valueLength);
- MemorySegmentUtils.copyToBytes(
- value.getSegments(), value.getOffset(), bytes, length,
valueLength);
- length += valueLength;
- }
-
- private void putBytes(byte[] value) {
- checkState(value != null, "Manifest binary field cannot be null.");
- putInt(value.length);
- ensureCapacity(value.length);
- System.arraycopy(value, 0, bytes, length, value.length);
- length += value.length;
- }
-
- private void putInt(int value) {
- ensureCapacity(Integer.BYTES);
- bytes[length++] = (byte) (value >>> 24);
- bytes[length++] = (byte) (value >>> 16);
- bytes[length++] = (byte) (value >>> 8);
- bytes[length++] = (byte) value;
- }
-
- private void ensureCapacity(int additional) {
- int required = Math.addExact(length, additional);
- if (required <= bytes.length) {
- return;
- }
- int grown = Math.max(required, bytes.length + (bytes.length >>>
1));
- bytes = Arrays.copyOf(bytes, grown);
- }
- }
-
/**
* Projected manifest schema together with its bound binary field layout.
*
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/DeletedIdentifierSet.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java
similarity index 95%
rename from
paimon-core/src/main/java/org/apache/paimon/manifest/DeletedIdentifierSet.java
rename to
paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java
index 8b97cb533c..5106edab1a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/DeletedIdentifierSet.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/CompactFileIdentifierSet.java
@@ -18,7 +18,7 @@
package org.apache.paimon.manifest;
-import org.apache.paimon.manifest.BinaryManifestEntry.ReusableIdentifier;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
import javax.annotation.Nullable;
@@ -28,12 +28,13 @@ import static
org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkState;
/**
- * Compact, collision-safe set backed by primitive arrays and one identifier
byte arena.
+ * Compact, collision-safe set for {@link FileEntry.Identifier} values, backed
by primitive arrays
+ * and one identifier byte arena.
*
* <p>The reusable identifier is only used as lookup scratch. Added identifier
bytes are copied into
* the arena.
*/
-public final class DeletedIdentifierSet {
+public final class CompactFileIdentifierSet {
private static final float LOAD_FACTOR = 0.75f;
@@ -145,8 +146,7 @@ public final class DeletedIdentifierSet {
}
private void growBuckets() {
- checkState(
- buckets.length < (1 << 30), "Too many deleted identifiers in
one manifest group.");
+ checkState(buckets.length < (1 << 30), "Too many file identifiers in
one manifest group.");
int[] grown = filledWithMinusOne(buckets.length << 1);
for (int entry = 0; entry < size; entry++) {
int bucket = bucket(hashes[entry], grown.length);
@@ -174,7 +174,7 @@ public final class DeletedIdentifierSet {
required = Math.addExact(arenaSize, additional);
} catch (ArithmeticException e) {
throw new IllegalStateException(
- "Deleted identifier arena exceeds the Java array limit.",
e);
+ "File identifier arena exceeds the Java array limit.", e);
}
if (required <= arena.length) {
return;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java
index f619f02d60..e33f2e6f85 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/FileEntry.java
@@ -19,7 +19,10 @@
package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.io.BinaryDataFileMeta;
+import org.apache.paimon.memory.MemorySegmentUtils;
import org.apache.paimon.utils.CloseableIterator;
import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.Filter;
@@ -41,6 +44,7 @@ import java.util.stream.Collectors;
import static
org.apache.paimon.utils.ManifestReadThreadPool.randomlyExecuteSequentialReturn;
import static
org.apache.paimon.utils.ManifestReadThreadPool.sequentialBatchedExecute;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkState;
/** Entry representing a file. */
@@ -178,6 +182,111 @@ public interface FileEntry {
}
}
+ /**
+ * Reusable byte encoding of a binary manifest entry's {@link Identifier}
fields.
+ *
+ * <p>The encoded identifier is the prefix of {@link #bytes()} ending at
{@link #length()}. It
+ * is valid until the next call to {@link #replace(BinaryManifestEntry)},
{@link
+ * #replaceWithPartition(BinaryManifestEntry)}, or {@link #release()} and
must not be modified
+ * by callers.
+ *
+ * <p>{@link #replace(BinaryManifestEntry)} omits the partition so callers
can represent it with
+ * a compact dictionary id. {@link
#replaceWithPartition(BinaryManifestEntry)} includes the
+ * serialized partition and represents the complete base {@link
Identifier}.
+ */
+ final class ReusableIdentifier {
+
+ private byte[] bytes = new byte[256];
+ private int length;
+
+ public ReusableIdentifier replace(BinaryManifestEntry entry) {
+ checkArgument(entry != null, "Binary manifest entry cannot be
null.");
+ length = 0;
+ return appendEntryFields(entry);
+ }
+
+ /** Replaces this encoding with the entry's partition and identity
fields. */
+ public ReusableIdentifier replaceWithPartition(BinaryManifestEntry
entry) {
+ checkArgument(entry != null, "Binary manifest entry cannot be
null.");
+ length = 0;
+ putBytes(entry.partitionBytes());
+ return appendEntryFields(entry);
+ }
+
+ private ReusableIdentifier appendEntryFields(BinaryManifestEntry
entry) {
+ putInt(entry.bucket());
+ BinaryDataFileMeta file = entry.file();
+ putInt(file.level());
+ putString(file.fileNameBinary());
+
+ int extraFileCount = file.extraFileCount();
+ putInt(extraFileCount);
+ for (int i = 0; i < extraFileCount; i++) {
+ putString(file.extraFile(i));
+ }
+
+ if (!file.hasEmbeddedIndex()) {
+ putInt(-1);
+ } else {
+ putBytes(file.embeddedIndex());
+ }
+ if (!file.hasExternalPath()) {
+ putInt(-1);
+ } else {
+ putString(file.externalPathBinary());
+ }
+ return this;
+ }
+
+ public byte[] bytes() {
+ return bytes;
+ }
+
+ public int length() {
+ return length;
+ }
+
+ public void release() {
+ bytes = new byte[0];
+ length = 0;
+ }
+
+ private void putString(BinaryString value) {
+ checkState(value != null, "Manifest string field cannot be null.");
+ int valueLength = value.getSizeInBytes();
+ putInt(valueLength);
+ ensureCapacity(valueLength);
+ MemorySegmentUtils.copyToBytes(
+ value.getSegments(), value.getOffset(), bytes, length,
valueLength);
+ length += valueLength;
+ }
+
+ private void putBytes(byte[] value) {
+ checkState(value != null, "Manifest binary field cannot be null.");
+ putInt(value.length);
+ ensureCapacity(value.length);
+ System.arraycopy(value, 0, bytes, length, value.length);
+ length += value.length;
+ }
+
+ private void putInt(int value) {
+ ensureCapacity(Integer.BYTES);
+ bytes[length++] = (byte) (value >>> 24);
+ bytes[length++] = (byte) (value >>> 16);
+ bytes[length++] = (byte) (value >>> 8);
+ bytes[length++] = (byte) value;
+ }
+
+ private void ensureCapacity(int additional) {
+ int required = Math.addExact(length, additional);
+ if (required <= bytes.length) {
+ return;
+ }
+ int grown = Math.max(required, bytes.length + (bytes.length >>>
1));
+ bytes = Arrays.copyOf(bytes, grown);
+ }
+ }
+
static <T extends FileEntry> Collection<T> mergeEntries(Iterable<T>
entries) {
LinkedHashMap<Identifier, T> map = new LinkedHashMap<>();
mergeEntries(entries, map);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryExternalSort.java
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryExternalSort.java
index 94cc2caf75..dd8bcd9d53 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryExternalSort.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestEntryExternalSort.java
@@ -26,8 +26,8 @@ import
org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.io.RollingFileWriter;
import org.apache.paimon.manifest.BinaryManifestEntry;
-import org.apache.paimon.manifest.BinaryManifestEntry.ReusableIdentifier;
-import org.apache.paimon.manifest.DeletedIdentifierSet;
+import org.apache.paimon.manifest.CompactFileIdentifierSet;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
@@ -58,7 +58,7 @@ public class ManifestEntryExternalSort {
@Nullable Integer manifestReadParallelism)
throws Exception {
try (EntrySorter sorter = new EntrySorter(sortKey, config)) {
- DeletedIdentifierSet deleteEntries = new DeletedIdentifierSet();
+ CompactFileIdentifierSet deleteEntries = new
CompactFileIdentifierSet();
try {
scanEntries(
section,
@@ -84,7 +84,7 @@ public class ManifestEntryExternalSort {
ExternalSortConfig config,
ManifestFile manifestFile,
List<ManifestFileMeta> newFilesForAbort,
- DeletedIdentifierSet deleteEntries,
+ CompactFileIdentifierSet deleteEntries,
@Nullable Integer manifestReadParallelism)
throws Exception {
try (EntrySorter sorter = new EntrySorter(sortKey, config)) {
@@ -262,7 +262,7 @@ public class ManifestEntryExternalSort {
private Pair<List<ManifestFileMeta>, List<ManifestFileMeta>>
writeMinorToManifest(
ManifestFile manifestFile,
- DeletedIdentifierSet deleteEntries,
+ CompactFileIdentifierSet deleteEntries,
List<ManifestFileMeta> newFilesForAbort)
throws Exception {
if (isEmpty()) {
@@ -273,8 +273,8 @@ public class ManifestEntryExternalSort {
manifestFile.createRollingWriter();
RollingFileWriter<ManifestEntry, ManifestFileMeta> deleteWriter =
manifestFile.createRollingWriter();
- DeletedIdentifierSet matchedEntries = new DeletedIdentifierSet();
- DeletedIdentifierSet emittedDeletes = new DeletedIdentifierSet();
+ CompactFileIdentifierSet matchedEntries = new
CompactFileIdentifierSet();
+ CompactFileIdentifierSet emittedDeletes = new
CompactFileIdentifierSet();
ReusableIdentifier identifier = new ReusableIdentifier();
Exception exception = null;
try {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
index 40a76a5914..bee9ddc187 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/ManifestFileSorter.java
@@ -27,7 +27,7 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.manifest.BinaryManifestEntry;
-import org.apache.paimon.manifest.DeletedIdentifierSet;
+import org.apache.paimon.manifest.CompactFileIdentifierSet;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
@@ -72,7 +72,7 @@ public class ManifestFileSorter {
final boolean fullCompaction;
final ManifestSortKey sortKey;
final ManifestEntryExternalSort.ExternalSortConfig externalSortConfig;
- final DeletedIdentifierSet deleteEntries;
+ final CompactFileIdentifierSet deleteEntries;
/**
* Manifest files that need unsorted compaction.
*
@@ -90,7 +90,7 @@ public class ManifestFileSorter {
boolean fullCompaction,
ManifestSortKey sortKey,
ManifestEntryExternalSort.ExternalSortConfig
externalSortConfig,
- DeletedIdentifierSet deleteEntries,
+ CompactFileIdentifierSet deleteEntries,
Map<ManifestFileMeta, Boolean> compactWithoutSort,
List<ManifestAdjacentSortedRun> levelRuns,
List<ManifestAdjacentSortedRun> pickedRuns) {
@@ -112,7 +112,7 @@ public class ManifestFileSorter {
/** Result of classifying manifest files. */
private static class ClassifyResult {
final List<ManifestFileMeta> lsmFiles;
- final DeletedIdentifierSet deleteEntries;
+ final CompactFileIdentifierSet deleteEntries;
/**
* Manifest files that need unsorted compaction.
*
@@ -125,7 +125,7 @@ public class ManifestFileSorter {
ClassifyResult(
List<ManifestFileMeta> lsmFiles,
- DeletedIdentifierSet deleteEntries,
+ CompactFileIdentifierSet deleteEntries,
Map<ManifestFileMeta, Boolean> compactWithoutSort) {
this.lsmFiles = lsmFiles;
this.deleteEntries = deleteEntries;
@@ -135,10 +135,10 @@ public class ManifestFileSorter {
/** Binary identifiers and partition values collected from DELETE entries.
*/
private static class DeletedEntryInfo {
- final DeletedIdentifierSet identifiers;
+ final CompactFileIdentifierSet identifiers;
final Set<BinaryRow> partitions;
- private DeletedEntryInfo(DeletedIdentifierSet identifiers,
Set<BinaryRow> partitions) {
+ private DeletedEntryInfo(CompactFileIdentifierSet identifiers,
Set<BinaryRow> partitions) {
this.identifiers = identifiers;
this.partitions = partitions;
}
@@ -511,7 +511,7 @@ public class ManifestFileSorter {
// Initialize classification containers and read delete entries
Map<ManifestFileMeta, Boolean> compactWithoutSort = new
LinkedHashMap<>();
List<ManifestFileMeta> lsmFiles = new LinkedList<>(input);
- DeletedIdentifierSet classifiedDeleteEntries = new
DeletedIdentifierSet();
+ CompactFileIdentifierSet classifiedDeleteEntries = new
CompactFileIdentifierSet();
Set<BinaryRow> deletePartitions = Collections.emptySet();
PartitionPredicate predicate = null;
if (fullCompaction) {
@@ -557,7 +557,7 @@ public class ManifestFileSorter {
ManifestFile manifestFile,
List<ManifestFileMeta> manifestFiles,
@Nullable Integer manifestReadParallelism) {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
Set<BinaryRow> partitions = new HashSet<>();
List<ManifestFileMeta> filesWithDeletes = new ArrayList<>();
for (ManifestFileMeta meta : manifestFiles) {
@@ -588,7 +588,7 @@ public class ManifestFileSorter {
private static void collectDeletedEntries(
ManifestFileMeta meta,
ManifestFile manifestFile,
- DeletedIdentifierSet identifiers,
+ CompactFileIdentifierSet identifiers,
Set<BinaryRow> partitions,
boolean synchronize) {
try (CloseableIterator<BinaryManifestEntry> entries =
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryReusableIdentifierTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryReusableIdentifierTest.java
deleted file mode 100644
index 294418cfa3..0000000000
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryReusableIdentifierTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.manifest;
-
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.GenericArray;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.manifest.BinaryManifestEntry.ReusableIdentifier;
-import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.RowType;
-
-import org.junit.jupiter.api.Test;
-
-import javax.annotation.Nullable;
-
-import java.util.Arrays;
-import java.util.List;
-
-import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests for {@link ReusableIdentifier}. */
-class BinaryManifestEntryReusableIdentifierTest {
-
- @Test
- void testEncodesIdentifierFields() {
- ReusableIdentifier identifier = new ReusableIdentifier();
-
- assertThat(
- identifier.replace(
- entry(3, 2, "f", new String[] {"a", "bc"}, new
byte[] {7, 8}, "x")))
- .isSameAs(identifier);
-
- assertThat(Arrays.copyOf(identifier.bytes(), identifier.length()))
- .containsExactly(
- 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 1, 102, 0, 0, 0, 2,
0, 0, 0, 1, 97, 0, 0,
- 0, 2, 98, 99, 0, 0, 0, 2, 7, 8, 0, 0, 0, 1, 120);
- }
-
- @Test
- void testReusesAndReleasesBuffer() {
- ReusableIdentifier identifier = new ReusableIdentifier();
- BinaryManifestEntry entry = entry(1, 0, "file", new String[0], null,
null);
-
- identifier.replace(entry);
- byte[] expected = Arrays.copyOf(identifier.bytes(),
identifier.length());
- assertThat(expected).endsWith((byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff);
-
- identifier.release();
- assertThat(identifier.length()).isZero();
- assertThat(identifier.bytes()).isEmpty();
-
- identifier.replace(entry);
- assertThat(Arrays.copyOf(identifier.bytes(),
identifier.length())).isEqualTo(expected);
- }
-
- @Test
- void testRejectsNullEntry() {
- assertThatThrownBy(() -> new ReusableIdentifier().replace(null))
- .isInstanceOf(IllegalArgumentException.class);
- }
-
- @Test
- void testDeletedIdentifierSetReusesIdentifierForEntryLookup() {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
- BinaryManifestEntry first = entry(1, 0, "first", new String[0], null,
null);
- BinaryManifestEntry second = entry(2, 0, "second", new String[0],
null, null);
-
- identifiers.add(first);
- assertThat(identifiers.contains(first)).isTrue();
- assertThat(identifiers.contains(second)).isFalse();
- assertThat(identifiers.contains(first)).isTrue();
-
- identifiers.add(second);
- assertThat(identifiers.contains(first)).isTrue();
- assertThat(identifiers.contains(second)).isTrue();
- }
-
- private static BinaryManifestEntry entry(
- int bucket,
- int level,
- String fileName,
- String[] extraFiles,
- @Nullable byte[] embeddedIndex,
- @Nullable String externalPath) {
- RowType manifestType = ManifestEntry.MANIFEST_ROW_TYPE;
- RowType fileType =
- DataFileMeta.SCHEMA.project(
- DataFileMeta.LEVEL,
- DataFileMeta.FILE_NAME,
- DataFileMeta.EXTRA_FILES,
- DataFileMeta.EMBEDDED_FILE_INDEX,
- DataFileMeta.EXTERNAL_PATH);
- List<DataField> fields =
- Arrays.asList(
- manifestType.getField(ManifestEntry.PARTITION),
- manifestType.getField(ManifestEntry.BUCKET),
-
manifestType.getField(ManifestEntry.FILE).newType(fileType));
- Object[] extraFileValues = new Object[extraFiles.length];
- for (int i = 0; i < extraFiles.length; i++) {
- extraFileValues[i] = BinaryString.fromString(extraFiles[i]);
- }
- return BinaryManifestEntry.Projection.create(new RowType(false,
fields))
- .createEntry()
- .replace(
- GenericRow.of(
- serializeBinaryRow(BinaryRow.EMPTY_ROW),
- bucket,
- GenericRow.of(
- level,
- BinaryString.fromString(fileName),
- new GenericArray(extraFileValues),
- embeddedIndex,
- externalPath == null
- ? null
- :
BinaryString.fromString(externalPath))));
- }
-}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryTest.java
index aceb117264..e60615a1ca 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/BinaryManifestEntryTest.java
@@ -215,8 +215,7 @@ public class BinaryManifestEntryTest {
DataFileMeta.EMBEDDED_FILE_INDEX,
DataFileMeta.EXTERNAL_PATH)
.createEntry();
- BinaryManifestEntry.ReusableIdentifier identifier =
- new BinaryManifestEntry.ReusableIdentifier();
+ FileEntry.ReusableIdentifier identifier = new
FileEntry.ReusableIdentifier();
entry.replace(identityRow(partition(1)));
BinaryRow firstPartition = entry.partition();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/DeletedIdentifierSetTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java
similarity index 87%
rename from
paimon-core/src/test/java/org/apache/paimon/manifest/DeletedIdentifierSetTest.java
rename to
paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java
index 0de63608e8..95d071eabe 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/DeletedIdentifierSetTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/CompactFileIdentifierSetTest.java
@@ -23,12 +23,12 @@ import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-/** Tests for {@link DeletedIdentifierSet}. */
-class DeletedIdentifierSetTest {
+/** Tests for {@link CompactFileIdentifierSet}. */
+class CompactFileIdentifierSetTest {
@Test
void testDeduplicatesAndIncludesPartition() {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
byte[] identifier = {1, 2, 3, 4};
identifiers.add(3, identifier, identifier.length);
@@ -46,7 +46,7 @@ class DeletedIdentifierSetTest {
@Test
void testCopiesOnlyIdentifierPrefix() {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
byte[] identifier = {1, 2, 3};
identifiers.add(0, identifier, 2);
@@ -59,7 +59,7 @@ class DeletedIdentifierSetTest {
@Test
void testGrowsAndReleases() {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
for (int i = 0; i < 1_000; i++) {
byte[] identifier = {(byte) i, (byte) (i >>> 8)};
identifiers.add(i % 7, identifier, identifier.length);
@@ -81,7 +81,7 @@ class DeletedIdentifierSetTest {
@Test
void testRejectsInvalidIdentifier() {
- DeletedIdentifierSet identifiers = new DeletedIdentifierSet();
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
assertThatThrownBy(() -> identifiers.add(0, (byte[]) null, 0))
.isInstanceOf(IllegalArgumentException.class);
@@ -89,7 +89,7 @@ class DeletedIdentifierSetTest {
.isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> identifiers.contains(0, new byte[1], 2))
.isInstanceOf(IllegalArgumentException.class);
- assertThatThrownBy(() -> identifiers.add(0,
(BinaryManifestEntry.ReusableIdentifier) null))
+ assertThatThrownBy(() -> identifiers.add(0,
(FileEntry.ReusableIdentifier) null))
.isInstanceOf(IllegalArgumentException.class);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/FileEntryReusableIdentifierTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/FileEntryReusableIdentifierTest.java
new file mode 100644
index 0000000000..82a4c70b8c
--- /dev/null
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/FileEntryReusableIdentifierTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.manifest;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.manifest.FileEntry.ReusableIdentifier;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nullable;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link ReusableIdentifier}. */
+class FileEntryReusableIdentifierTest {
+
+ @Test
+ void testEncodesIdentifierFields() {
+ ReusableIdentifier identifier = new ReusableIdentifier();
+
+ assertThat(
+ identifier.replace(
+ entry(3, 2, "f", new String[] {"a", "bc"}, new
byte[] {7, 8}, "x")))
+ .isSameAs(identifier);
+
+ assertThat(Arrays.copyOf(identifier.bytes(), identifier.length()))
+ .containsExactly(
+ 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 1, 102, 0, 0, 0, 2,
0, 0, 0, 1, 97, 0, 0,
+ 0, 2, 98, 99, 0, 0, 0, 2, 7, 8, 0, 0, 0, 1, 120);
+ }
+
+ @Test
+ void testReusesAndReleasesBuffer() {
+ ReusableIdentifier identifier = new ReusableIdentifier();
+ BinaryManifestEntry entry = entry(1, 0, "file", new String[0], null,
null);
+
+ identifier.replace(entry);
+ byte[] expected = Arrays.copyOf(identifier.bytes(),
identifier.length());
+ assertThat(expected).endsWith((byte) 0xff, (byte) 0xff, (byte) 0xff,
(byte) 0xff);
+
+ identifier.release();
+ assertThat(identifier.length()).isZero();
+ assertThat(identifier.bytes()).isEmpty();
+
+ identifier.replace(entry);
+ assertThat(Arrays.copyOf(identifier.bytes(),
identifier.length())).isEqualTo(expected);
+ }
+
+ @Test
+ void testRejectsNullEntry() {
+ assertThatThrownBy(() -> new ReusableIdentifier().replace(null))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ @Test
+ void testCompactFileIdentifierSetReusesIdentifierForEntryLookup() {
+ CompactFileIdentifierSet identifiers = new CompactFileIdentifierSet();
+ BinaryManifestEntry first = entry(1, 0, "first", new String[0], null,
null);
+ BinaryManifestEntry second = entry(2, 0, "second", new String[0],
null, null);
+
+ identifiers.add(first);
+ assertThat(identifiers.contains(first)).isTrue();
+ assertThat(identifiers.contains(second)).isFalse();
+ assertThat(identifiers.contains(first)).isTrue();
+
+ identifiers.add(second);
+ assertThat(identifiers.contains(first)).isTrue();
+ assertThat(identifiers.contains(second)).isTrue();
+ }
+
+ @Test
+ void testCompactAndMaterializedIdentifierSemanticsMatch() {
+ BinaryManifestEntry base =
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x");
+ List<BinaryManifestEntry> candidates =
+ Arrays.asList(
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ // totalBuckets is deliberately not part of
FileEntry.Identifier.
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 32,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.singleColumn(1),
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 4,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 1,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "g",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"bc", "a"},
+ new byte[] {7, 8},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 9},
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ null,
+ "x"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ "y"),
+ entry(
+ BinaryRow.EMPTY_ROW,
+ 3,
+ 8,
+ 2,
+ "f",
+ new String[] {"a", "bc"},
+ new byte[] {7, 8},
+ null));
+
+ for (BinaryManifestEntry candidate : candidates) {
+ assertIdentifierSemanticsMatch(base, candidate);
+ }
+ }
+
+ private static void assertIdentifierSemanticsMatch(
+ BinaryManifestEntry left, BinaryManifestEntry right) {
+ boolean expected = left.identifier().equals(right.identifier());
+
+ CompactFileIdentifierSet completeIdentifiers = new
CompactFileIdentifierSet();
+ completeIdentifiers.add(left);
+ assertThat(completeIdentifiers.contains(right)).isEqualTo(expected);
+
+ ReusableIdentifier reusable = new ReusableIdentifier();
+ CompactFileIdentifierSet dictionaryPartitionIdentifiers = new
CompactFileIdentifierSet();
+ dictionaryPartitionIdentifiers.add(1, reusable.replace(left));
+ int rightPartitionId = left.partition().equals(right.partition()) ? 1
: 2;
+ assertThat(
+ dictionaryPartitionIdentifiers.contains(
+ rightPartitionId, reusable.replace(right)))
+ .isEqualTo(expected);
+ }
+
+ private static BinaryManifestEntry entry(
+ int bucket,
+ int level,
+ String fileName,
+ String[] extraFiles,
+ @Nullable byte[] embeddedIndex,
+ @Nullable String externalPath) {
+ return entry(
+ BinaryRow.EMPTY_ROW,
+ bucket,
+ 1,
+ level,
+ fileName,
+ extraFiles,
+ embeddedIndex,
+ externalPath);
+ }
+
+ private static BinaryManifestEntry entry(
+ BinaryRow partition,
+ int bucket,
+ int totalBuckets,
+ int level,
+ String fileName,
+ String[] extraFiles,
+ @Nullable byte[] embeddedIndex,
+ @Nullable String externalPath) {
+ RowType manifestType = ManifestEntry.MANIFEST_ROW_TYPE;
+ RowType fileType =
+ DataFileMeta.SCHEMA.project(
+ DataFileMeta.LEVEL,
+ DataFileMeta.FILE_NAME,
+ DataFileMeta.EXTRA_FILES,
+ DataFileMeta.EMBEDDED_FILE_INDEX,
+ DataFileMeta.EXTERNAL_PATH);
+ List<DataField> fields =
+ Arrays.asList(
+ manifestType.getField(ManifestEntry.PARTITION),
+ manifestType.getField(ManifestEntry.BUCKET),
+ manifestType.getField(ManifestEntry.TOTAL_BUCKETS),
+
manifestType.getField(ManifestEntry.FILE).newType(fileType));
+ Object[] extraFileValues = new Object[extraFiles.length];
+ for (int i = 0; i < extraFiles.length; i++) {
+ extraFileValues[i] = BinaryString.fromString(extraFiles[i]);
+ }
+ return BinaryManifestEntry.Projection.create(new RowType(false,
fields))
+ .createEntry()
+ .replace(
+ GenericRow.of(
+ serializeBinaryRow(partition),
+ bucket,
+ totalBuckets,
+ GenericRow.of(
+ level,
+ BinaryString.fromString(fileName),
+ new GenericArray(extraFileValues),
+ embeddedIndex,
+ externalPath == null
+ ? null
+ :
BinaryString.fromString(externalPath))));
+ }
+}
