gaborkaszab commented on code in PR #16936: URL: https://github.com/apache/iceberg/pull/16936#discussion_r3460690800
########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( Review Comment: I don't see how the user will use these functions, but would it make sense to merge the `fromDataFile` and `fromDeleteFile` into a common `fromManifestEntry`? Then the user doesn't have to decide if it's data or delete, the adapter can do it instead. ########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { + throw new IllegalArgumentException( + ContentFileUtil.isDV(file) + ? String.format( + Locale.ROOT, + "v3 delete vectors must be colocated on the data file's content_entry, not " + + "written as a delete manifest entry: %s referencing %s", + file.location(), + file.referencedDataFile()) + : String.format( + Locale.ROOT, + "v2 position delete files have no v4 representation; carry them over via a " + + "legacy v3 manifest with writer_format_version=0: %s", + file.location())); + } + + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES, + "Invalid content for delete file: %s", + file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + /** + * Builds a manifest reference content_entry for the v4 root manifest. + * + * @param manifest the leaf manifest being referenced + * @param writerFormatVersion {@link #V4_WRITER_FORMAT_VERSION} (4) for a v4 leaf manifest, or + * {@link #LEGACY_WRITER_FORMAT_VERSION} (0) for a pre-v4 (v1, v2, or v3) leaf manifest + * carried over during a v3-to-v4 upgrade. Callers are responsible for resolving the value; + * this avoids adding a v4-only accessor to the public ManifestFile interface for a value with + * no production consumer at this layer. + * @param status entry status (typically ADDED for a newly written leaf, EXISTING for a + * carried-over reference) + * @param firstRowId the resolved first-row-id to write for this reference, or null for delete + * manifests. Callers are responsible for resolving the value (either carrying over {@link + * ManifestFile#firstRowId()} or assigning from a writer-side counter); the adapter does not + * decide between the two. + */ + static TrackedFile fromManifestFile( + ManifestFile manifest, int writerFormatVersion, EntryStatus status, Long firstRowId) { + Preconditions.checkArgument(manifest != null, "Invalid manifest file: null"); + Preconditions.checkArgument( Review Comment: Should this check go into the TrackedFileBuilder instead? ########## core/src/main/java/org/apache/iceberg/TrackingBuilder.java: ########## @@ -111,6 +144,11 @@ TrackingBuilder dvUpdated() { return this; } + TrackingBuilder firstRowId(Long newFirstRowId) { Review Comment: I have [a PR](https://github.com/apache/iceberg/pull/16849) to set `firstRowID`. I had the impression that we don't want to do that through the builder and rather use a setter, but I can change the approach. ########## core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java: ########## @@ -179,6 +183,46 @@ TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) { return this; } + TrackedFileBuilder firstRowId(Long newFirstRowId) { + this.firstRowId = newFirstRowId; + return this; + } + + /** + * Sets an explicit {@link EntryStatus} for the tracking row, bypassing the {@link + * TrackingBuilder#added(long)} / {@link TrackingBuilder#from(Tracking, long)} status-derivation + * path. Required when paired with {@link #dataSequenceNumber(Long)} / {@link + * #fileSequenceNumber(Long)} for manifest references and non-ADDED transitions whose tracking + * values can't be derived from a source. + * + * <p>When this setter is used, {@link #build()} constructs the {@link Tracking} directly from the + * accumulated field values and bypasses {@link TrackingBuilder}. Cannot be combined with the + * source-tracking path ({@link #from(TrackedFile, long)}). + */ + TrackedFileBuilder status(EntryStatus newStatus) { Review Comment: As written in an earlier comment, I think this and the seq num setters are too much freedom given to the user. Can't we avoid exposing these? 1) `status()` is needed for EXISTING state. I think this can be achieved with the existing `from()` builder method 2) can't we avoid these using the `inheritFrom()` method? ########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { + throw new IllegalArgumentException( + ContentFileUtil.isDV(file) + ? String.format( + Locale.ROOT, + "v3 delete vectors must be colocated on the data file's content_entry, not " + + "written as a delete manifest entry: %s referencing %s", + file.location(), + file.referencedDataFile()) + : String.format( + Locale.ROOT, + "v2 position delete files have no v4 representation; carry them over via a " + + "legacy v3 manifest with writer_format_version=0: %s", + file.location())); + } + + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES, + "Invalid content for delete file: %s", + file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + /** + * Builds a manifest reference content_entry for the v4 root manifest. + * + * @param manifest the leaf manifest being referenced + * @param writerFormatVersion {@link #V4_WRITER_FORMAT_VERSION} (4) for a v4 leaf manifest, or + * {@link #LEGACY_WRITER_FORMAT_VERSION} (0) for a pre-v4 (v1, v2, or v3) leaf manifest + * carried over during a v3-to-v4 upgrade. Callers are responsible for resolving the value; + * this avoids adding a v4-only accessor to the public ManifestFile interface for a value with + * no production consumer at this layer. + * @param status entry status (typically ADDED for a newly written leaf, EXISTING for a + * carried-over reference) + * @param firstRowId the resolved first-row-id to write for this reference, or null for delete + * manifests. Callers are responsible for resolving the value (either carrying over {@link + * ManifestFile#firstRowId()} or assigning from a writer-side counter); the adapter does not + * decide between the two. + */ + static TrackedFile fromManifestFile( + ManifestFile manifest, int writerFormatVersion, EntryStatus status, Long firstRowId) { + Preconditions.checkArgument(manifest != null, "Invalid manifest file: null"); + Preconditions.checkArgument( + writerFormatVersion == LEGACY_WRITER_FORMAT_VERSION + || writerFormatVersion >= V4_WRITER_FORMAT_VERSION, + "Invalid writer_format_version: %s (must be %s for legacy v1-v3 or >= %s for v4+)", + writerFormatVersion, + LEGACY_WRITER_FORMAT_VERSION, + V4_WRITER_FORMAT_VERSION); + Preconditions.checkArgument(status != null, "Invalid status: null"); + Long manifestSnapshotId = manifest.snapshotId(); + Preconditions.checkArgument(manifestSnapshotId != null, "Invalid manifest snapshot id: null"); + Preconditions.checkArgument( + firstRowId == null || manifest.content() == ManifestContent.DATA, + "firstRowId is only valid for DATA manifests, but content is %s", + manifest.content()); + + long manifestSeq = Math.max(0L, manifest.sequenceNumber()); + PartitionData emptyPartition = new PartitionData(Types.StructType.of()); + ManifestInfo info = manifestInfo(manifest); + + TrackedFileBuilder builder = + manifest.content() == ManifestContent.DATA + ? TrackedFileBuilder.dataManifest(manifestSnapshotId) + : TrackedFileBuilder.deleteManifest(manifestSnapshotId); + builder + .status(status) + .dataSequenceNumber(manifestSeq) + .fileSequenceNumber(manifestSeq) + .writerFormatVersion(writerFormatVersion) + .location(manifest.path()) + .fileFormat(FileFormat.fromFileName(manifest.path())) + .partition(emptyPartition) + .recordCount(totalRecordCount(manifest)) + .fileSizeInBytes(manifest.length()) + .specId(manifest.partitionSpecId()) + .manifestInfo(info); + + if (firstRowId != null) { Review Comment: I recall there was an argument not to put the `firstRowId` setter to the builder and have a separate setter method. Seeing the usage here, could be part of the builder unless there is any other usage that I don't know of. [PR](https://github.com/apache/iceberg/pull/16849) ########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { + throw new IllegalArgumentException( + ContentFileUtil.isDV(file) + ? String.format( + Locale.ROOT, + "v3 delete vectors must be colocated on the data file's content_entry, not " + + "written as a delete manifest entry: %s referencing %s", + file.location(), + file.referencedDataFile()) + : String.format( + Locale.ROOT, + "v2 position delete files have no v4 representation; carry them over via a " + + "legacy v3 manifest with writer_format_version=0: %s", + file.location())); + } + + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES, + "Invalid content for delete file: %s", + file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + /** + * Builds a manifest reference content_entry for the v4 root manifest. + * + * @param manifest the leaf manifest being referenced + * @param writerFormatVersion {@link #V4_WRITER_FORMAT_VERSION} (4) for a v4 leaf manifest, or + * {@link #LEGACY_WRITER_FORMAT_VERSION} (0) for a pre-v4 (v1, v2, or v3) leaf manifest + * carried over during a v3-to-v4 upgrade. Callers are responsible for resolving the value; + * this avoids adding a v4-only accessor to the public ManifestFile interface for a value with + * no production consumer at this layer. + * @param status entry status (typically ADDED for a newly written leaf, EXISTING for a + * carried-over reference) + * @param firstRowId the resolved first-row-id to write for this reference, or null for delete + * manifests. Callers are responsible for resolving the value (either carrying over {@link + * ManifestFile#firstRowId()} or assigning from a writer-side counter); the adapter does not + * decide between the two. + */ + static TrackedFile fromManifestFile( + ManifestFile manifest, int writerFormatVersion, EntryStatus status, Long firstRowId) { + Preconditions.checkArgument(manifest != null, "Invalid manifest file: null"); + Preconditions.checkArgument( + writerFormatVersion == LEGACY_WRITER_FORMAT_VERSION + || writerFormatVersion >= V4_WRITER_FORMAT_VERSION, + "Invalid writer_format_version: %s (must be %s for legacy v1-v3 or >= %s for v4+)", + writerFormatVersion, + LEGACY_WRITER_FORMAT_VERSION, + V4_WRITER_FORMAT_VERSION); + Preconditions.checkArgument(status != null, "Invalid status: null"); + Long manifestSnapshotId = manifest.snapshotId(); + Preconditions.checkArgument(manifestSnapshotId != null, "Invalid manifest snapshot id: null"); + Preconditions.checkArgument( + firstRowId == null || manifest.content() == ManifestContent.DATA, + "firstRowId is only valid for DATA manifests, but content is %s", + manifest.content()); + + long manifestSeq = Math.max(0L, manifest.sequenceNumber()); + PartitionData emptyPartition = new PartitionData(Types.StructType.of()); + ManifestInfo info = manifestInfo(manifest); + + TrackedFileBuilder builder = + manifest.content() == ManifestContent.DATA + ? TrackedFileBuilder.dataManifest(manifestSnapshotId) + : TrackedFileBuilder.deleteManifest(manifestSnapshotId); + builder + .status(status) + .dataSequenceNumber(manifestSeq) + .fileSequenceNumber(manifestSeq) + .writerFormatVersion(writerFormatVersion) + .location(manifest.path()) + .fileFormat(FileFormat.fromFileName(manifest.path())) + .partition(emptyPartition) + .recordCount(totalRecordCount(manifest)) + .fileSizeInBytes(manifest.length()) + .specId(manifest.partitionSpecId()) + .manifestInfo(info); + + if (firstRowId != null) { + builder.firstRowId(firstRowId); + } + + if (manifest.keyMetadata() != null) { + builder.keyMetadata(manifest.keyMetadata()); + } + + return builder.build(); + } + + private static TrackedFile buildContentFileEntry( + ContentFile<?> file, + EntryStatus status, + long snapshotId, + Long dataSequenceNumber, + Long fileSequenceNumber, + Schema tableSchema) { + Preconditions.checkArgument(status != null, "Invalid status: null"); + // fromDataFile / fromDeleteFile project legacy ManifestEntry rows, whose status is ADDED, + // EXISTING, or DELETED. MODIFIED and REPLACED have no legacy representation — they're written + // directly by V4Writer.prepareWithStatus via TrackedFileBuilder.from(source, sid). + // deletionVector(dv).build() (MODIFIED) and TrackedFileBuilder.replaced(source, sid) + // (REPLACED). + Preconditions.checkArgument( + status == EntryStatus.ADDED + || status == EntryStatus.EXISTING + || status == EntryStatus.DELETED, + "Unsupported status for content file entry: %s (use V4Writer.prepareWithStatus for " + + "MODIFIED/REPLACED transitions)", + status); + PartitionData partition = toPartitionData(file); + FileFormat format = file.format(); + Preconditions.checkArgument( + format != null, "Invalid file format: null for %s", file.location()); + ContentStats stats = MetricsUtil.fromMetrics(tableSchema, toMetrics(file, tableSchema)); + boolean isDataFile = file.content() == FileContent.DATA; + + TrackedFileBuilder builder = + isDataFile + ? TrackedFileBuilder.data(snapshotId) + : TrackedFileBuilder.equalityDelete(snapshotId); + + if (status == EntryStatus.ADDED) { + Long firstRowId = isDataFile ? ((DataFile) file).firstRowId() : null; + if (firstRowId != null) { + builder.firstRowId(firstRowId); + } + } else { + Preconditions.checkArgument( + dataSequenceNumber != null, "Invalid data sequence number: null for non-ADDED entry"); + Preconditions.checkArgument( + fileSequenceNumber != null, "Invalid file sequence number: null for non-ADDED entry"); + builder + .status(status) + .dataSequenceNumber(dataSequenceNumber) + .fileSequenceNumber(fileSequenceNumber); + Long firstRowId = isDataFile ? ((DataFile) file).firstRowId() : null; Review Comment: This seems identical to the check in the `status == EntryStatus.ADDED` branch. Could this be moved out of this if after line 237? ########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { Review Comment: nit: this if is only necessary to produce specific error messages based on how pos-deletes are represented. Isn't the check needed on line 109 that only eq-deletes are allowed? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
