This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new a686d5440d [core] Remove versioned serializer abstraction (#9039)
a686d5440d is described below
commit a686d5440d0b12f14ef08066473cee5ebdeeedb1
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Aug 5 15:05:17 2026 +0800
[core] Remove versioned serializer abstraction (#9039)
---
.../paimon/index/IndexFileMetaSerializer.java | 3 +-
.../paimon/index/IndexFileMetaV3Deserializer.java | 3 +-
.../manifest/IndexManifestEntrySerializer.java | 37 ++++++++++----
.../apache/paimon/manifest/IndexManifestFile.java | 3 +-
.../org/apache/paimon/manifest/ManifestEntry.java | 3 +-
.../paimon/manifest/ManifestEntrySerializer.java | 40 ++++++++++-----
.../manifest/ManifestFileMetaSerializer.java | 39 +++++++++-----
.../org/apache/paimon/manifest/ManifestList.java | 3 +-
.../ManifestSchemaUtils.java} | 40 +++------------
.../org/apache/paimon/utils/ObjectSerializer.java | 6 +++
.../manifest/IndexManifestEntrySerializerTest.java | 15 +++---
.../manifest/IndexManifestFileHandlerTest.java | 3 +-
.../LegacyManifestFileMetaSerializerPaimon10.java | 59 ++++++++++++----------
.../manifest/ManifestEntrySerializerTest.java | 11 +++-
.../manifest/ManifestFileMetaSerializerTest.java | 9 ++++
.../apache/paimon/manifest/ManifestListTest.java | 3 +-
16 files changed, 157 insertions(+), 120 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
index ffaff2d0ad..6e71c5f74a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaSerializer.java
@@ -23,14 +23,13 @@ import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.utils.ObjectSerializer;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import java.util.Collection;
import java.util.LinkedHashMap;
import static org.apache.paimon.data.BinaryString.fromString;
-/** A {@link VersionedObjectSerializer} for {@link IndexFileMeta}. */
+/** Serializer for {@link IndexFileMeta}. */
public class IndexFileMetaSerializer extends ObjectSerializer<IndexFileMeta> {
public IndexFileMetaSerializer() {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV3Deserializer.java
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV3Deserializer.java
index ac38b7c11b..917916dd4a 100644
---
a/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV3Deserializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/index/IndexFileMetaV3Deserializer.java
@@ -26,7 +26,6 @@ import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import java.io.IOException;
import java.io.Serializable;
@@ -37,7 +36,7 @@ import java.util.List;
import static
org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvMetas;
import static org.apache.paimon.utils.SerializationUtils.newStringType;
-/** A {@link VersionedObjectSerializer} for {@link IndexFileMeta}. */
+/** Deserializer for version 3 {@link IndexFileMeta}. */
public class IndexFileMetaV3Deserializer implements Serializable {
private static final long serialVersionUID = 1L;
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
index d6979de802..ca691a55a1 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestEntrySerializer.java
@@ -22,9 +22,11 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.utils.VersionedObjectSerializer;
+import org.apache.paimon.utils.ObjectSerializer;
+import org.apache.paimon.utils.OffsetRow;
import java.util.function.Function;
@@ -34,20 +36,26 @@ import static
org.apache.paimon.index.IndexFileMetaSerializer.rowArrayDataToDvMe
import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
-/** A {@link VersionedObjectSerializer} for {@link IndexManifestEntry}. */
-public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<IndexManifestEntry> {
+/** Serializer for {@link IndexManifestEntry}. */
+public class IndexManifestEntrySerializer extends
ObjectSerializer<IndexManifestEntry> {
+
+ /**
+ * Permanent on-disk format identifier, not a schema version.
+ *
+ * <p>Do not change when adding nullable fields. Old manifest readers skip
unknown fields.
+ */
+ private static final int FORMAT_IDENTIFIER = 1;
public IndexManifestEntrySerializer() {
- super(IndexManifestEntry.SCHEMA);
+
super(ManifestSchemaUtils.withFormatIdentifier(IndexManifestEntry.SCHEMA));
}
@Override
- public int getVersion() {
- return 1;
+ public InternalRow toRow(IndexManifestEntry record) {
+ return new JoinedRow().replace(GenericRow.of(FORMAT_IDENTIFIER),
toDataRow(record));
}
- @Override
- public InternalRow convertTo(IndexManifestEntry record) {
+ private InternalRow toDataRow(IndexManifestEntry record) {
IndexFileMeta indexFile = record.indexFile();
GlobalIndexMeta globalIndexMeta = indexFile.globalIndexMeta();
InternalRow globalIndexRow =
@@ -76,11 +84,18 @@ public class IndexManifestEntrySerializer extends
VersionedObjectSerializer<Inde
}
@Override
- public IndexManifestEntry convertFrom(int version, InternalRow row) {
- if (version != 1) {
- throw new UnsupportedOperationException("Unsupported version: " +
version);
+ public IndexManifestEntry fromRow(InternalRow row) {
+ checkFormatIdentifier(row.getInt(0));
+ return fromDataRow(new OffsetRow(row.getFieldCount() - 1,
1).replace(row));
+ }
+
+ private void checkFormatIdentifier(int formatIdentifier) {
+ if (formatIdentifier != FORMAT_IDENTIFIER) {
+ throw new UnsupportedOperationException("Unsupported version: " +
formatIdentifier);
}
+ }
+ private IndexManifestEntry fromDataRow(InternalRow row) {
GlobalIndexMeta globalIndexMeta = null;
if (!row.isNullAt(9)) {
InternalRow globalIndexRow = row.getRow(9,
GlobalIndexMeta.SCHEMA.getFieldCount());
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFile.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFile.java
index a46762c1af..b4e7240aea 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFile.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFile.java
@@ -29,7 +29,6 @@ import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.ObjectsFile;
import org.apache.paimon.utils.PathFactory;
import org.apache.paimon.utils.SegmentsCache;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import javax.annotation.Nullable;
@@ -98,7 +97,7 @@ public class IndexManifestFile extends
ObjectsFile<IndexManifestEntry> {
}
public IndexManifestFile create() {
- RowType schema =
VersionedObjectSerializer.versionType(IndexManifestEntry.SCHEMA);
+ RowType schema =
ManifestSchemaUtils.withFormatIdentifier(IndexManifestEntry.SCHEMA);
return new IndexManifestFile(
fileIO,
schema,
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java
index 788a86794b..f9c6b6a03b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntry.java
@@ -25,7 +25,6 @@ import org.apache.paimon.types.DataField;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.TinyIntType;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import javax.annotation.Nullable;
@@ -58,7 +57,7 @@ public interface ManifestEntry extends FileEntry {
new DataField(3, TOTAL_BUCKETS, new
IntType(false)),
new DataField(4, FILE, DataFileMeta.SCHEMA)));
- RowType MANIFEST_ROW_TYPE = VersionedObjectSerializer.versionType(SCHEMA);
+ RowType MANIFEST_ROW_TYPE =
ManifestSchemaUtils.withFormatIdentifier(SCHEMA);
static ManifestEntry create(
FileKind kind, BinaryRow partition, int bucket, int totalBuckets,
DataFileMeta file) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntrySerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntrySerializer.java
index c31d79713e..6ab15761f7 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntrySerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestEntrySerializer.java
@@ -21,9 +21,11 @@ package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFileMetaSerializer;
-import org.apache.paimon.utils.VersionedObjectSerializer;
+import org.apache.paimon.utils.ObjectSerializer;
+import org.apache.paimon.utils.OffsetRow;
import java.util.function.Function;
@@ -31,24 +33,30 @@ import static
org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
/** Serializer for {@link ManifestEntry}. */
-public class ManifestEntrySerializer extends
VersionedObjectSerializer<ManifestEntry> {
+public class ManifestEntrySerializer extends ObjectSerializer<ManifestEntry> {
private static final long serialVersionUID = 1L;
+ /**
+ * Permanent on-disk format identifier, not a schema version.
+ *
+ * <p>Do not change when adding nullable fields. Old manifest readers skip
unknown fields.
+ */
+ private static final int FORMAT_IDENTIFIER = 2;
+
private final DataFileMetaSerializer dataFileMetaSerializer;
public ManifestEntrySerializer() {
- super(ManifestEntry.SCHEMA);
+ super(ManifestSchemaUtils.withFormatIdentifier(ManifestEntry.SCHEMA));
this.dataFileMetaSerializer = new DataFileMetaSerializer();
}
@Override
- public int getVersion() {
- return 2;
+ public InternalRow toRow(ManifestEntry entry) {
+ return new JoinedRow().replace(GenericRow.of(FORMAT_IDENTIFIER),
toDataRow(entry));
}
- @Override
- public InternalRow convertTo(ManifestEntry entry) {
+ private InternalRow toDataRow(ManifestEntry entry) {
GenericRow row = new GenericRow(5);
row.setField(0, entry.kind().toByteValue());
row.setField(1, serializeBinaryRow(entry.partition()));
@@ -59,16 +67,24 @@ public class ManifestEntrySerializer extends
VersionedObjectSerializer<ManifestE
}
@Override
- public ManifestEntry convertFrom(int version, InternalRow row) {
- if (version != 2) {
- if (version == 1) {
+ public ManifestEntry fromRow(InternalRow row) {
+ checkFormatIdentifier(row.getInt(0));
+ return fromDataRow(new OffsetRow(row.getFieldCount() - 1,
1).replace(row));
+ }
+
+ private void checkFormatIdentifier(int formatIdentifier) {
+ if (formatIdentifier != FORMAT_IDENTIFIER) {
+ if (formatIdentifier == 1) {
throw new IllegalArgumentException(
String.format(
"The current version %s is not compatible with
the version %s, please recreate the table.",
- getVersion(), version));
+ FORMAT_IDENTIFIER, formatIdentifier));
}
- throw new IllegalArgumentException("Unsupported version: " +
version);
+ throw new IllegalArgumentException("Unsupported version: " +
formatIdentifier);
}
+ }
+
+ private ManifestEntry fromDataRow(InternalRow row) {
return ManifestEntry.create(
FileKind.fromByteValue(row.getByte(0)),
deserializeBinaryRow(row.getBinary(1)),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
index 95e30a8856..c71362f92e 100644
---
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFileMetaSerializer.java
@@ -21,25 +21,33 @@ package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.stats.SimpleStats;
-import org.apache.paimon.utils.VersionedObjectSerializer;
+import org.apache.paimon.utils.ObjectSerializer;
+import org.apache.paimon.utils.OffsetRow;
/** Serializer for {@link ManifestFileMeta}. */
-public class ManifestFileMetaSerializer extends
VersionedObjectSerializer<ManifestFileMeta> {
+public class ManifestFileMetaSerializer extends
ObjectSerializer<ManifestFileMeta> {
private static final long serialVersionUID = 1L;
+ /**
+ * Permanent on-disk format identifier, not a schema version.
+ *
+ * <p>Do not change when adding nullable fields. Old manifest readers skip
unknown fields.
+ */
+ private static final int FORMAT_IDENTIFIER = 2;
+
public ManifestFileMetaSerializer() {
- super(ManifestFileMeta.SCHEMA);
+
super(ManifestSchemaUtils.withFormatIdentifier(ManifestFileMeta.SCHEMA));
}
@Override
- public int getVersion() {
- return 2;
+ public InternalRow toRow(ManifestFileMeta meta) {
+ return new JoinedRow().replace(GenericRow.of(FORMAT_IDENTIFIER),
toDataRow(meta));
}
- @Override
- public InternalRow convertTo(ManifestFileMeta meta) {
+ private InternalRow toDataRow(ManifestFileMeta meta) {
return GenericRow.of(
BinaryString.fromString(meta.fileName()),
meta.fileSize(),
@@ -56,17 +64,24 @@ public class ManifestFileMetaSerializer extends
VersionedObjectSerializer<Manife
}
@Override
- public ManifestFileMeta convertFrom(int version, InternalRow row) {
- if (version != 2) {
- if (version == 1) {
+ public ManifestFileMeta fromRow(InternalRow row) {
+ checkFormatIdentifier(row.getInt(0));
+ return fromDataRow(new OffsetRow(row.getFieldCount() - 1,
1).replace(row));
+ }
+
+ private void checkFormatIdentifier(int formatIdentifier) {
+ if (formatIdentifier != FORMAT_IDENTIFIER) {
+ if (formatIdentifier == 1) {
throw new IllegalArgumentException(
String.format(
"The current version %s is not compatible with
the version %s, please recreate the table.",
- getVersion(), version));
+ FORMAT_IDENTIFIER, formatIdentifier));
}
- throw new IllegalArgumentException("Unsupported version: " +
version);
+ throw new IllegalArgumentException("Unsupported version: " +
formatIdentifier);
}
+ }
+ private ManifestFileMeta fromDataRow(InternalRow row) {
return new ManifestFileMeta(
row.getString(0).toString(),
row.getLong(1),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestList.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestList.java
index 33cf4cfa3f..11213417c2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestList.java
+++ b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestList.java
@@ -32,7 +32,6 @@ import org.apache.paimon.utils.ObjectsFile;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.PathFactory;
import org.apache.paimon.utils.SegmentsCache;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import javax.annotation.Nullable;
@@ -144,7 +143,7 @@ public class ManifestList extends
ObjectsFile<ManifestFileMeta> {
}
public ManifestList create() {
- RowType metaType =
VersionedObjectSerializer.versionType(ManifestFileMeta.SCHEMA);
+ RowType metaType =
ManifestSchemaUtils.withFormatIdentifier(ManifestFileMeta.SCHEMA);
return new ManifestList(
fileIO,
new ManifestFileMetaSerializer(),
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/VersionedObjectSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSchemaUtils.java
similarity index 51%
rename from
paimon-core/src/main/java/org/apache/paimon/utils/VersionedObjectSerializer.java
rename to
paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSchemaUtils.java
index b4da0a5ac7..e58efc0c13 100644
---
a/paimon-core/src/main/java/org/apache/paimon/utils/VersionedObjectSerializer.java
+++
b/paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSchemaUtils.java
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-package org.apache.paimon.utils;
+package org.apache.paimon.manifest;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
@@ -28,40 +25,17 @@ import org.apache.paimon.types.RowType;
import java.util.ArrayList;
import java.util.List;
-/** A {@link ObjectSerializer} for versioned serialization. */
-public abstract class VersionedObjectSerializer<T> extends ObjectSerializer<T>
{
+/** Utilities for manifest schemas. */
+final class ManifestSchemaUtils {
- private static final long serialVersionUID = 1L;
+ private ManifestSchemaUtils() {}
- public VersionedObjectSerializer(RowType rowType) {
- super(versionType(rowType));
- }
-
- public static RowType versionType(RowType rowType) {
+ /** Adds the permanent on-disk format identifier field to a manifest row
type. */
+ static RowType withFormatIdentifier(RowType rowType) {
List<DataField> fields = new ArrayList<>();
+ // Keep the historical field name for compatibility with existing
manifest files.
fields.add(new DataField(-1, "_VERSION", new IntType(false)));
fields.addAll(rowType.getFields());
return new RowType(false, fields);
}
-
- /**
- * Gets the version with which this serializer serializes.
- *
- * @return The version of the serialization schema.
- */
- public abstract int getVersion();
-
- public abstract InternalRow convertTo(T record);
-
- public abstract T convertFrom(int version, InternalRow row);
-
- @Override
- public final InternalRow toRow(T record) {
- return new JoinedRow().replace(GenericRow.of(getVersion()),
convertTo(record));
- }
-
- @Override
- public final T fromRow(InternalRow row) {
- return convertFrom(row.getInt(0), new OffsetRow(row.getFieldCount() -
1, 1).replace(row));
- }
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectSerializer.java
b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectSerializer.java
index a83d805491..a60f56c82f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/ObjectSerializer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/ObjectSerializer.java
@@ -112,6 +112,11 @@ public abstract class ObjectSerializer<T> implements
Serializable {
return deserializeList(view);
}
+ /**
+ * Serializes a record to schema-dependent bytes.
+ *
+ * <p>The bytes must be deserialized with the same row schema.
+ */
public byte[] serializeToBytes(T record) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
DataOutputViewStreamWrapper view = new
DataOutputViewStreamWrapper(out);
@@ -119,6 +124,7 @@ public abstract class ObjectSerializer<T> implements
Serializable {
return out.toByteArray();
}
+ /** Deserializes bytes produced with this serializer's current row schema.
*/
public T deserializeFromBytes(byte[] bytes) throws IOException {
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
DataInputViewStreamWrapper view = new DataInputViewStreamWrapper(in);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
index 28be88b809..945f54fdf4 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestEntrySerializerTest.java
@@ -19,7 +19,7 @@
package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.utils.ObjectSerializer;
@@ -52,15 +52,12 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
10,
new GlobalIndexMeta(0, 9, 7, null, new byte[]
{1}),
null));
- GenericRow serialized = (GenericRow) serializer.convertTo(entry);
- assertThat(serialized.getRow(9,
GlobalIndexMeta.SCHEMA.getFieldCount()).getFieldCount())
+ InternalRow serialized = serializer.toRow(entry);
+ assertThat(serialized.getInt(0)).isEqualTo(1);
+ assertThat(serialized.getRow(10,
GlobalIndexMeta.SCHEMA.getFieldCount()).getFieldCount())
.isEqualTo(6);
- GlobalIndexMeta restored =
- serializer
- .convertFrom(serializer.getVersion(), serialized)
- .indexFile()
- .globalIndexMeta();
+ GlobalIndexMeta restored =
serializer.fromRow(serialized).indexFile().globalIndexMeta();
assertThat(restored.indexMeta()).containsExactly(1);
assertThat(restored.sourceMeta()).isNull();
@@ -69,7 +66,6 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
@Test
void testGlobalIndexSourceMetaRoundTrip() throws IOException {
IndexManifestEntrySerializer serializer = new
IndexManifestEntrySerializer();
- assertThat(serializer.getVersion()).isEqualTo(1);
IndexManifestEntry entry =
new IndexManifestEntry(
FileKind.ADD,
@@ -83,6 +79,7 @@ public class IndexManifestEntrySerializerTest extends
ObjectSerializerTestBase<I
new GlobalIndexMeta(
0, 9, 7, null, new byte[] {3, 4}, new
byte[] {1, 2}),
null));
+ assertThat(serializer.toRow(entry).getInt(0)).isEqualTo(1);
GlobalIndexMeta restored =
serializer
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
index c022d05b72..f706972b6d 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java
@@ -30,7 +30,6 @@ import org.apache.paimon.table.BucketMode;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.CloseableIterator;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -150,7 +149,7 @@ public class IndexManifestFileHandlerTest {
List<DataField> legacyEntryFields = new
ArrayList<>(IndexManifestEntry.SCHEMA.getFields());
legacyEntryFields.set(9,
legacyEntryFields.get(9).newType(legacyGlobalIndexSchema));
RowType legacySchema =
- VersionedObjectSerializer.versionType(new RowType(false,
legacyEntryFields));
+ ManifestSchemaUtils.withFormatIdentifier(new RowType(false,
legacyEntryFields));
FormatReaderFactory legacyReaderFactory =
fileFormat.createReaderFactory(legacySchema, legacySchema, new
ArrayList<>());
Path path =
fileStore.pathFactory().indexManifestFileFactory().toPath(manifestFile);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/LegacyManifestFileMetaSerializerPaimon10.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/LegacyManifestFileMetaSerializerPaimon10.java
index a371f15526..1b755752e7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/LegacyManifestFileMetaSerializerPaimon10.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/LegacyManifestFileMetaSerializerPaimon10.java
@@ -21,21 +21,24 @@ package org.apache.paimon.manifest;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VarCharType;
-import org.apache.paimon.utils.VersionedObjectSerializer;
+import org.apache.paimon.utils.ObjectSerializer;
+import org.apache.paimon.utils.OffsetRow;
import java.util.Arrays;
/** Legacy serializer for {@link ManifestFileMeta} in Paimon 1.0. */
-public class LegacyManifestFileMetaSerializerPaimon10
- extends VersionedObjectSerializer<ManifestFileMeta> {
+public class LegacyManifestFileMetaSerializerPaimon10 extends
ObjectSerializer<ManifestFileMeta> {
private static final long serialVersionUID = 1L;
+ private static final int FORMAT_IDENTIFIER = 2;
+
public static final RowType SCHEMA =
new RowType(
false,
@@ -49,44 +52,44 @@ public class LegacyManifestFileMetaSerializerPaimon10
new DataField(5, "_SCHEMA_ID", new
BigIntType(false))));
public LegacyManifestFileMetaSerializerPaimon10() {
- super(SCHEMA);
- }
-
- @Override
- public int getVersion() {
- return 2;
+ super(ManifestSchemaUtils.withFormatIdentifier(SCHEMA));
}
@Override
- public InternalRow convertTo(ManifestFileMeta meta) {
- return GenericRow.of(
- BinaryString.fromString(meta.fileName()),
- meta.fileSize(),
- meta.numAddedFiles(),
- meta.numDeletedFiles(),
- meta.partitionStats().toRow(),
- meta.schemaId());
+ public InternalRow toRow(ManifestFileMeta meta) {
+ return new JoinedRow()
+ .replace(
+ GenericRow.of(FORMAT_IDENTIFIER),
+ GenericRow.of(
+ BinaryString.fromString(meta.fileName()),
+ meta.fileSize(),
+ meta.numAddedFiles(),
+ meta.numDeletedFiles(),
+ meta.partitionStats().toRow(),
+ meta.schemaId()));
}
@Override
- public ManifestFileMeta convertFrom(int version, InternalRow row) {
- if (version != 2) {
- if (version == 1) {
+ public ManifestFileMeta fromRow(InternalRow row) {
+ int formatIdentifier = row.getInt(0);
+ if (formatIdentifier != FORMAT_IDENTIFIER) {
+ if (formatIdentifier == 1) {
throw new IllegalArgumentException(
String.format(
"The current version %s is not compatible with
the version %s, please recreate the table.",
- getVersion(), version));
+ FORMAT_IDENTIFIER, formatIdentifier));
}
- throw new IllegalArgumentException("Unsupported version: " +
version);
+ throw new IllegalArgumentException("Unsupported version: " +
formatIdentifier);
}
+ InternalRow data = new OffsetRow(row.getFieldCount() - 1,
1).replace(row);
return new ManifestFileMeta(
- row.getString(0).toString(),
- row.getLong(1),
- row.getLong(2),
- row.getLong(3),
- SimpleStats.fromRow(row.getRow(4, 3)),
- row.getLong(5),
+ data.getString(0).toString(),
+ data.getLong(1),
+ data.getLong(2),
+ data.getLong(3),
+ SimpleStats.fromRow(data.getRow(4, 3)),
+ data.getLong(5),
null,
null,
null,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestEntrySerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestEntrySerializerTest.java
index 5609b9e214..0547c9acce 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestEntrySerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestEntrySerializerTest.java
@@ -21,11 +21,20 @@ package org.apache.paimon.manifest;
import org.apache.paimon.utils.ObjectSerializer;
import org.apache.paimon.utils.ObjectSerializerTestBase;
-/** Tests for {@link ManifestEntrySerializerTest}. */
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link ManifestEntrySerializer}. */
public class ManifestEntrySerializerTest extends
ObjectSerializerTestBase<ManifestEntry> {
private final ManifestTestDataGenerator gen =
ManifestTestDataGenerator.builder().build();
+ @Test
+ void testFormatIdentifier() {
+ assertThat(new
ManifestEntrySerializer().toRow(gen.next()).getInt(0)).isEqualTo(2);
+ }
+
@Override
protected ObjectSerializer<ManifestEntry> serializer() {
return new ManifestEntrySerializer();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
index 900846afff..57b5a08ed0 100644
---
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaSerializerTest.java
@@ -21,9 +21,13 @@ package org.apache.paimon.manifest;
import org.apache.paimon.utils.ObjectSerializer;
import org.apache.paimon.utils.ObjectSerializerTestBase;
+import org.junit.jupiter.api.Test;
+
import java.util.ArrayList;
import java.util.List;
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Tests for {@link ManifestFileMetaSerializer}. */
public class ManifestFileMetaSerializerTest extends
ObjectSerializerTestBase<ManifestFileMeta> {
@@ -31,6 +35,11 @@ public class ManifestFileMetaSerializerTest extends
ObjectSerializerTestBase<Man
private final ManifestTestDataGenerator gen =
ManifestTestDataGenerator.builder().build();
+ @Test
+ void testFormatIdentifier() {
+ assertThat(new
ManifestFileMetaSerializer().toRow(object()).getInt(0)).isEqualTo(2);
+ }
+
@Override
protected ObjectSerializer<ManifestFileMeta> serializer() {
return new ManifestFileMetaSerializer();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
index 2b945f87b4..8442be28c6 100644
--- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestListTest.java
@@ -28,7 +28,6 @@ import org.apache.paimon.options.Options;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FailingFileIO;
import org.apache.paimon.utils.FileStorePathFactory;
-import org.apache.paimon.utils.VersionedObjectSerializer;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
@@ -113,7 +112,7 @@ public class ManifestListTest {
private ManifestList createLegacyManifestListPaimon10() {
FileStorePathFactory pathFactory =
createPathFactory(tempDir.toString());
RowType legacyMetaType =
- VersionedObjectSerializer.versionType(
+ ManifestSchemaUtils.withFormatIdentifier(
LegacyManifestFileMetaSerializerPaimon10.SCHEMA);
return new ManifestList(
LocalFileIO.create(),
