This is an automated email from the ASF dual-hosted git repository. JingsongLi pushed a commit to branch release-2.0 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e31008d6dd4bba8b80940e9accdce7113c3443db Author: Jingsong Lee <[email protected]> AuthorDate: Mon Aug 3 21:22:39 2026 +0800 [core] Preserve index manifest compatibility (#9004) --- .../manifest/IndexManifestEntrySerializer.java | 9 ++-- .../manifest/IndexManifestEntrySerializerTest.java | 33 ++++---------- .../manifest/IndexManifestFileHandlerTest.java | 51 ++++++++++++++++++++++ 3 files changed, 63 insertions(+), 30 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java index d8649dab2f..d6979de802 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java @@ -43,7 +43,7 @@ public class IndexManifestEntrySerializer extends VersionedObjectSerializer<Inde @Override public int getVersion() { - return 2; + return 1; } @Override @@ -77,21 +77,20 @@ public class IndexManifestEntrySerializer extends VersionedObjectSerializer<Inde @Override public IndexManifestEntry convertFrom(int version, InternalRow row) { - if (version < 1 || version > 2) { + if (version != 1) { throw new UnsupportedOperationException("Unsupported version: " + version); } GlobalIndexMeta globalIndexMeta = null; if (!row.isNullAt(9)) { - InternalRow globalIndexRow = row.getRow(9, version == 1 ? 5 : 6); + InternalRow globalIndexRow = row.getRow(9, GlobalIndexMeta.SCHEMA.getFieldCount()); long rowRangeStart = globalIndexRow.getLong(0); long rowRangeEnd = globalIndexRow.getLong(1); int indexFieldId = globalIndexRow.getInt(2); int[] extralFields = globalIndexRow.isNullAt(3) ? null : globalIndexRow.getArray(3).toIntArray(); byte[] indexMeta = globalIndexRow.isNullAt(4) ? null : globalIndexRow.getBinary(4); - byte[] sourceMeta = - version == 1 || globalIndexRow.isNullAt(5) ? null : globalIndexRow.getBinary(5); + byte[] sourceMeta = globalIndexRow.isNullAt(5) ? null : globalIndexRow.getBinary(5); globalIndexMeta = new GlobalIndexMeta( rowRangeStart, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java index d7cccdd880..28be88b809 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java @@ -20,20 +20,13 @@ package org.apache.paimon.manifest; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.data.InternalRow; -import org.apache.paimon.data.JoinedRow; -import org.apache.paimon.data.serializer.InternalRowSerializer; -import org.apache.paimon.data.serializer.InternalSerializers; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; -import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.utils.ObjectSerializer; import org.apache.paimon.utils.ObjectSerializerTestBase; -import org.apache.paimon.utils.VersionedObjectSerializer; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Random; @@ -45,7 +38,7 @@ import static org.assertj.core.api.Assertions.assertThat; public class IndexManifestEntrySerializerTest extends ObjectSerializerTestBase<IndexManifestEntry> { @Test - void testReadsGlobalIndexWithoutSourceMeta() throws IOException { + void testReadsGlobalIndexWithoutSourceMeta() { IndexManifestEntrySerializer serializer = new IndexManifestEntrySerializer(); IndexManifestEntry entry = new IndexManifestEntry( @@ -60,25 +53,14 @@ public class IndexManifestEntrySerializerTest extends ObjectSerializerTestBase<I new GlobalIndexMeta(0, 9, 7, null, new byte[] {1}), null)); GenericRow serialized = (GenericRow) serializer.convertTo(entry); - InternalRowSerializer legacyGlobalIndexSerializer = - InternalSerializers.create( - GlobalIndexMeta.SCHEMA.copy( - GlobalIndexMeta.SCHEMA.getFields().subList(0, 5))); - BinaryRow legacyGlobalIndexRow = - legacyGlobalIndexSerializer - .toBinaryRow(GenericRow.of(0L, 9L, 7, null, new byte[] {1})) - .copy(); - serialized.setField(9, legacyGlobalIndexRow); - - InternalRow version1Row = new JoinedRow().replace(GenericRow.of(1), serialized); - InternalRowSerializer versionedRowSerializer = - InternalSerializers.create( - VersionedObjectSerializer.versionType(IndexManifestEntry.SCHEMA)); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - versionedRowSerializer.serialize(version1Row, new DataOutputViewStreamWrapper(out)); + assertThat(serialized.getRow(9, GlobalIndexMeta.SCHEMA.getFieldCount()).getFieldCount()) + .isEqualTo(6); GlobalIndexMeta restored = - serializer.deserializeFromBytes(out.toByteArray()).indexFile().globalIndexMeta(); + serializer + .convertFrom(serializer.getVersion(), serialized) + .indexFile() + .globalIndexMeta(); assertThat(restored.indexMeta()).containsExactly(1); assertThat(restored.sourceMeta()).isNull(); @@ -87,6 +69,7 @@ public class IndexManifestEntrySerializerTest extends ObjectSerializerTestBase<I @Test void testGlobalIndexSourceMetaRoundTrip() throws IOException { IndexManifestEntrySerializer serializer = new IndexManifestEntrySerializer(); + assertThat(serializer.getVersion()).isEqualTo(1); IndexManifestEntry entry = new IndexManifestEntry( FileKind.ADD, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java index cfcc896525..c022d05b72 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java @@ -20,19 +20,28 @@ package org.apache.paimon.manifest; import org.apache.paimon.TestAppendFileStore; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; import org.apache.paimon.format.FileFormat; +import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.fs.Path; import org.apache.paimon.index.GlobalIndexMeta; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.table.BucketMode; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.VersionedObjectSerializer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import static org.apache.paimon.index.IndexFileMetaSerializerTest.randomDeletionVectorIndexFile; +import static org.apache.paimon.utils.FileUtils.createFormatReader; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -118,6 +127,48 @@ public class IndexManifestFileHandlerTest { assertThat(entries.contains(entry4)).isTrue(); } + @Test + public void testNewIndexManifestReadableWithLegacySchema() throws Exception { + TestAppendFileStore fileStore = + TestAppendFileStore.createAppendStore(tempDir, new HashMap<>()); + FileFormat fileFormat = FileFormat.manifestFormat(fileStore.options()); + IndexManifestFile indexManifestFile = + new IndexManifestFile.Factory( + fileStore.fileIO(), + fileFormat, + "zstd", + fileStore.pathFactory(), + null) + .create(); + IndexManifestFileHandler handler = + new IndexManifestFileHandler(indexManifestFile, BucketMode.HASH_FIXED); + + String manifestFile = handler.write(null, Arrays.asList(pkVectorEntry("btree", "index"))); + + RowType legacyGlobalIndexSchema = + GlobalIndexMeta.SCHEMA.copy(GlobalIndexMeta.SCHEMA.getFields().subList(0, 5)); + List<DataField> legacyEntryFields = new ArrayList<>(IndexManifestEntry.SCHEMA.getFields()); + legacyEntryFields.set(9, legacyEntryFields.get(9).newType(legacyGlobalIndexSchema)); + RowType legacySchema = + VersionedObjectSerializer.versionType(new RowType(false, legacyEntryFields)); + FormatReaderFactory legacyReaderFactory = + fileFormat.createReaderFactory(legacySchema, legacySchema, new ArrayList<>()); + Path path = fileStore.pathFactory().indexManifestFileFactory().toPath(manifestFile); + + try (CloseableIterator<InternalRow> iterator = + createFormatReader(fileStore.fileIO(), legacyReaderFactory, path, null) + .toCloseableIterator()) { + InternalRow row = iterator.next(); + assertThat(row.getInt(0)).isEqualTo(1); + InternalRow globalIndex = row.getRow(10, 5); + assertThat(globalIndex.getLong(0)).isEqualTo(0); + assertThat(globalIndex.getLong(1)).isEqualTo(1); + assertThat(globalIndex.getInt(2)).isEqualTo(1); + assertThat(globalIndex.isNullAt(4)).isTrue(); + assertThat(iterator.hasNext()).isFalse(); + } + } + @Test public void testGlobalIndexOverlappingRangeRejectedWhenPreviousFileKept() throws Exception { TestAppendFileStore fileStore =
