github-actions[bot] commented on code in PR #41461: URL: https://github.com/apache/beam/pull/41461#discussion_r4630466616
########## sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/maintenance/ReadManifests.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.io.iceberg.maintenance; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.apache.beam.sdk.annotations.Internal; +import org.apache.beam.sdk.io.iceberg.IcebergCatalogConfig; +import org.apache.beam.sdk.io.iceberg.TableCache; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.transforms.Redistribute; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList; +import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileIO; + +/** + * Reads Iceberg manifests in parallel and emits a {@link FileInfo} for every live file they + * reference. + * + * <p>This is an internal building block of the table maintenance operations, and is not intended to + * be applied directly by users. Its input is produced by the maintenance pipeline's metadata + * planning step. That step runs once per table: it loads current table metadata, selects the + * snapshots relevant to the operation (for example, the expired or the retained snapshots), and + * reads their manifest lists. Each manifest becomes one {@link ManifestDescriptor}, created with + * {@link ManifestDescriptor#from}, which records whether the files it references are reachable. + * When a manifest's partition spec may no longer be in table metadata by the time it is read, the + * planning step attaches the spec with {@link ManifestDescriptor#withPartitionSpec}. The output + * {@link FileInfo}s are then compared against other file sets to find files that can be deleted. + * + * <p>Input manifests are redistributed before they are read. Planning produces all of a table's + * manifests from a single element, and redistributing spreads their reads across workers. + * + * <p>Only live entries ({@code ADDED} and {@code EXISTING}) are emitted; entries recording a file + * deletion are skipped. Every emitted {@link FileInfo} takes its {@link FileInfo#getReachable() + * reachable} flag from the manifest's descriptor. + * + * <p>The output may contain the same path more than once: a file can be referenced by several + * manifests, and in format v3 several deletion vectors can be stored in one Puffin file. Consumers + * that need distinct paths must deduplicate. + * + * <p>Tables with format version 4 or later are rejected, because their manifests can reference + * other manifests, which this transform does not follow. The check uses the worker's cached table + * metadata, so the planning step must also reject such tables using current metadata. + */ +@Internal +public class ReadManifests + extends PTransform<PCollection<ManifestDescriptor>, PCollection<FileInfo>> { + + static final String MANIFESTS_READ = "manifestsRead"; + static final String MANIFEST_ENTRIES_READ = "manifestEntriesRead"; + + /** The highest table format version whose manifests this transform can read. */ + @VisibleForTesting static final int MAX_SUPPORTED_FORMAT_VERSION = 3; + + /** The only manifest entry columns needed to build a {@link FileInfo}. */ + private static final List<String> COLUMNS = + ImmutableList.of(DataFile.FILE_PATH.name(), DataFile.CONTENT.name()); + + private final IcebergCatalogConfig catalogConfig; + + private ReadManifests(IcebergCatalogConfig catalogConfig) { + this.catalogConfig = catalogConfig; + } + + /** Creates a transform that loads tables from {@code catalogConfig}. */ + public static ReadManifests of(IcebergCatalogConfig catalogConfig) { + return new ReadManifests(catalogConfig); + } + + @Override + public PCollection<FileInfo> expand(PCollection<ManifestDescriptor> input) { + return input + .apply("Redistribute", Redistribute.arbitrarily()) + .apply("Read", ParDo.of(new ReadManifestDoFn(catalogConfig))); + } + + static class ReadManifestDoFn extends DoFn<ManifestDescriptor, FileInfo> { + private final IcebergCatalogConfig catalogConfig; + private final Counter manifestsRead = Metrics.counter(ReadManifests.class, MANIFESTS_READ); + private final Counter manifestEntriesRead = + Metrics.counter(ReadManifests.class, MANIFEST_ENTRIES_READ); + + ReadManifestDoFn(IcebergCatalogConfig catalogConfig) { + this.catalogConfig = catalogConfig; + } + + @ProcessElement + public void processElement( + @Element ManifestDescriptor descriptor, OutputReceiver<FileInfo> output) + throws IOException { + String tableIdentifier = descriptor.getTableIdentifier(); + Table table = TableCache.get(catalogConfig, tableIdentifier); + checkFormatVersion(table, tableIdentifier); + Map<Integer, PartitionSpec> specs = resolveSpecs(table, descriptor); + + long entries = 0; + try (CloseableIterable<? extends ContentFile<?>> files = + open(new ManifestFileAdapter(descriptor), table.io(), specs)) { + for (ContentFile<?> file : files) { + output.output( + FileInfo.of( + file.location(), + category(file.content(), descriptor), + descriptor.getReachable(), + tableIdentifier)); + entries++; + } + } + manifestsRead.inc(); + manifestEntriesRead.inc(entries); + } + + /** + * Returns partition specs that include the manifest's spec. + * + * <p>A spec carried by the descriptor is preferred: it was captured when the manifest was + * planned, while a spec ID in current table metadata may have been reassigned to a different + * spec after the original was removed. Otherwise the cached table's specs are used if they + * include the spec ID, and failing that, the table is refreshed in case the cached copy + * predates the spec. + * + * @throws IllegalStateException if the spec cannot be found + */ + private Map<Integer, PartitionSpec> resolveSpecs(Table table, ManifestDescriptor descriptor) { + int specId = descriptor.getPartitionSpecId(); + PartitionSpec carried = descriptor.partitionSpec(); + if (carried != null) { + return ImmutableMap.of(specId, carried); + } + + if (table.specs().containsKey(specId)) { + return table.specs(); + } + + Map<Integer, PartitionSpec> refreshed = + TableCache.getRefreshed(catalogConfig, descriptor.getTableIdentifier()).specs(); Review Comment: **🟢 Low**: `checkFormatVersion` is called on line 124 against the initially cached `Table` before `resolveSpecs` potentially refreshes it here. If a cached v2/v3 table was concurrently upgraded to format v4 alongside a partition spec update, the refreshed table metadata would bypass `checkFormatVersion`. Consider calling `checkFormatVersion(refreshedTable, descriptor.getTableIdentifier())` after `TableCache.getRefreshed(...)` (or moving `checkFormatVersion` after `resolveSpecs`). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
