manuzhang commented on code in PR #14264:
URL: https://github.com/apache/iceberg/pull/14264#discussion_r3426826670
##########
core/src/main/java/org/apache/iceberg/BaseIncrementalChangelogScan.java:
##########
@@ -133,13 +205,701 @@ private static Map<Long, Integer>
computeSnapshotOrdinals(Deque<Snapshot> snapsh
return snapshotOrdinals;
}
+ /**
+ * Builds a delete file index for existing deletes that were present before
the start snapshot.
+ * These deletes should be applied to data files but should not generate
DELETE changelog rows.
+ * Uses manifest pruning and caching to optimize performance.
+ */
+ private DeleteFileIndex buildExistingDeleteIndex(Long
fromSnapshotIdExclusive) {
+ if (fromSnapshotIdExclusive == null) {
+ return DeleteFileIndex.emptyIndex();
+ }
+ Snapshot fromSnapshot = table().snapshot(fromSnapshotIdExclusive);
+ Preconditions.checkState(
+ fromSnapshot != null, "Cannot find starting snapshot: %s",
fromSnapshotIdExclusive);
+
+ List<ManifestFile> existingDeleteManifests =
fromSnapshot.deleteManifests(table().io());
+ if (existingDeleteManifests.isEmpty()) {
+ return DeleteFileIndex.emptyIndex();
+ }
+
+ // Prune manifests based on partition filter to avoid processing
irrelevant manifests
+ List<ManifestFile> prunedManifests =
pruneManifestsByPartition(existingDeleteManifests);
+ if (prunedManifests.isEmpty()) {
+ return DeleteFileIndex.emptyIndex();
+ }
+
+ // Load delete files from manifests
+ Iterable<DeleteFile> deleteFiles = loadDeleteFiles(prunedManifests, null);
+
+ return DeleteFileIndex.builderFor(deleteFiles)
+ .specsById(table().specs())
+ .caseSensitive(isCaseSensitive())
+ .build();
+ }
+
+ /**
+ * Wrapper method that tracks build calls and caches the result for reuse.
This ensures we only
+ * build the index once even if called from multiple places.
+ */
+ private DeleteFileIndex buildExistingDeleteIndexTracked(Long
fromSnapshotIdExclusive) {
+ if (cachedExistingDeleteIndex != null) {
+ return cachedExistingDeleteIndex;
+ }
+ existingDeleteIndexBuildCallCount++;
+ cachedExistingDeleteIndex =
buildExistingDeleteIndex(fromSnapshotIdExclusive);
+ return cachedExistingDeleteIndex;
+ }
+
+ // Visible for testing
+ int getExistingDeleteIndexBuildCallCount() {
+ return existingDeleteIndexBuildCallCount;
+ }
+
+ // Visible for testing
+ boolean wasExistingDeleteIndexBuilt() {
+ return existingDeleteIndexBuildCallCount > 0;
+ }
+
+ /**
+ * Builds per-snapshot delete file indexes for newly added delete files in
each changelog
+ * snapshot. These deletes should generate DELETE changelog rows. Uses
caching to avoid re-parsing
+ * manifests.
+ */
+ private Map<Long, DeleteFileIndex> buildAddedDeleteIndexes(Deque<Snapshot>
changelogSnapshots) {
+ Map<Long, DeleteFileIndex> addedDeletesBySnapshot =
Maps.newConcurrentMap();
+ Tasks.foreach(changelogSnapshots)
+ .retry(3)
+ .stopOnFailure()
+ .throwFailureWhenFinished()
+ .executeWith(planExecutor())
+ .onFailure(
+ (snapshot, exc) ->
+ LOG.warn(
+ "Failed to build delete index for snapshot {}",
snapshot.snapshotId(), exc))
+ .run(
+ snapshot -> {
+ List<ManifestFile> snapshotDeleteManifests =
snapshot.deleteManifests(table().io());
+ if (snapshotDeleteManifests.isEmpty()) {
+ addedDeletesBySnapshot.put(snapshot.snapshotId(),
DeleteFileIndex.emptyIndex());
+ return;
+ }
+
+ // Filter to only include delete files added in this snapshot
+ List<ManifestFile> addedDeleteManifests =
+ snapshotDeleteManifests.stream()
+ .filter(manifest ->
manifest.snapshotId().equals(snapshot.snapshotId()))
+ .collect(Collectors.toUnmodifiableList());
+
+ if (addedDeleteManifests.isEmpty()) {
+ addedDeletesBySnapshot.put(snapshot.snapshotId(),
DeleteFileIndex.emptyIndex());
+ } else {
+ // Load delete files from manifests
+ Iterable<DeleteFile> deleteFiles =
+ loadDeleteFiles(addedDeleteManifests,
snapshot.snapshotId());
+
+ DeleteFileIndex index =
+ DeleteFileIndex.builderFor(deleteFiles)
+ .specsById(table().specs())
+ .caseSensitive(isCaseSensitive())
+ .build();
+ addedDeletesBySnapshot.put(snapshot.snapshotId(), index);
+ }
+ });
+ return addedDeletesBySnapshot;
+ }
+
+ /**
+ * Plans tasks for EXISTING data files that are affected by newly added
delete files. These files
+ * were not added or deleted in the changelog snapshot range, but have new
delete files applied to
+ * them.
+ */
+ private CloseableIterable<ChangelogScanTask> planDeletedRowsTasks(
+ Deque<Snapshot> changelogSnapshots,
+ DeleteFileIndex existingDeleteIndex,
+ Map<Long, DeleteFileIndex> addedDeletesBySnapshot,
+ Set<Long> changelogSnapshotIds) {
+
+ Map<Long, Integer> snapshotOrdinals =
computeSnapshotOrdinals(changelogSnapshots);
+ List<ChangelogScanTask> tasks = Lists.newArrayList();
+
+ // Build a map of file statuses and collect affected partitions for each
snapshot
+ Pair<Map<Long, Map<String, ManifestEntry.Status>>, PartitionSet>
fileStatusAndPartitions =
+ buildFileStatusBySnapshot(changelogSnapshots, changelogSnapshotIds);
+ Map<Long, Map<String, ManifestEntry.Status>> fileStatusBySnapshot =
+ fileStatusAndPartitions.first();
+ PartitionSet affectedPartitions = fileStatusAndPartitions.second();
+
+ // Accumulate actual DeleteFile entries chronologically
+ List<DeleteFile> accumulatedDeletes = Lists.newArrayList();
+
+ // Start with deletes from before the changelog range
+ if (!existingDeleteIndex.isEmpty()) {
+ for (DeleteFile df : existingDeleteIndex.referencedDeleteFiles()) {
+ accumulatedDeletes.add(df);
+ }
+ }
+
+ for (Snapshot snapshot : changelogSnapshots) {
+ DeleteFileIndex addedDeleteIndex =
addedDeletesBySnapshot.get(snapshot.snapshotId());
+ if (addedDeleteIndex.isEmpty()) {
+ continue;
+ }
+
+ // Collect partitions of newly added delete files for pruning (important
for the current
+ // snapshot)
+ for (DeleteFile df : addedDeleteIndex.referencedDeleteFiles()) {
+ affectedPartitions.add(df.specId(), df.partition());
+ }
+
+ DeleteFileIndex cumulativeDeleteIndex =
+ buildDeleteIndex(accumulatedDeletes, affectedPartitions);
+
+ // Process data files for this snapshot
+ // Use a local set per snapshot to track processed files
+ Set<String> alreadyProcessedPaths = Sets.newHashSet();
+ processSnapshotForDeletedRowsTasks(
+ snapshot,
+ addedDeleteIndex,
+ cumulativeDeleteIndex,
+ fileStatusBySnapshot.get(snapshot.snapshotId()),
+ alreadyProcessedPaths,
+ snapshotOrdinals,
+ affectedPartitions,
+ tasks);
+
+ // Accumulate this snapshot's added deletes for subsequent snapshots
+ for (DeleteFile df : addedDeleteIndex.referencedDeleteFiles()) {
+ accumulatedDeletes.add(df);
+ }
+ }
+
+ return CloseableIterable.withNoopClose(tasks);
+ }
+
+ /**
+ * Builds a map of file statuses for each snapshot, tracking which files
were added or deleted in
+ * each snapshot.
+ */
+ private Pair<Map<Long, Map<String, ManifestEntry.Status>>, PartitionSet>
+ buildFileStatusBySnapshot(
+ Deque<Snapshot> changelogSnapshots, Set<Long> changelogSnapshotIds) {
+
+ Map<Long, Map<String, ManifestEntry.Status>> fileStatusBySnapshot =
Maps.newConcurrentMap();
+ java.util.Queue<PartitionSet> localPartitionsQueue =
+ new java.util.concurrent.ConcurrentLinkedQueue<>();
+
+ Tasks.foreach(changelogSnapshots)
+ .stopOnFailure()
+ .throwFailureWhenFinished()
+ .executeWith(planExecutor())
+ .run(
+ snapshot -> {
+ Map<String, ManifestEntry.Status> fileStatuses =
Maps.newHashMap();
+ PartitionSet localAffected =
PartitionSet.create(table().specs());
+
+ List<ManifestFile> changedDataManifests =
+ FluentIterable.from(snapshot.dataManifests(table().io()))
+ .filter(manifest ->
manifest.snapshotId().equals(snapshot.snapshotId()))
+ .toList();
+
+ if (!changedDataManifests.isEmpty()) {
+ ManifestGroup changedGroup =
+ new ManifestGroup(table().io(), changedDataManifests,
ImmutableList.of())
+ .specsById(table().specs())
+ .caseSensitive(isCaseSensitive())
+ .select(scanColumns())
+ .filterData(filter())
+ .ignoreExisting()
+ .columnsToKeepStats(columnsToKeepStats());
+
+ try (CloseableIterable<ManifestEntry<DataFile>> entries =
changedGroup.entries()) {
+ for (ManifestEntry<DataFile> entry : entries) {
+ if (changelogSnapshotIds.contains(entry.snapshotId())) {
+ fileStatuses.put(entry.file().location(),
entry.status());
+ localAffected.add(entry.file().specId(),
entry.file().partition());
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Failed to collect file statuses for snapshot " +
snapshot.snapshotId(), e);
+ }
+ }
+
+ fileStatusBySnapshot.put(snapshot.snapshotId(), fileStatuses);
+ localPartitionsQueue.add(localAffected);
+ });
+
+ PartitionSet globalAffected = PartitionSet.create(table().specs());
+ for (PartitionSet local : localPartitionsQueue) {
+ globalAffected.addAll(local);
+ }
+
+ return Pair.of(fileStatusBySnapshot, globalAffected);
+ }
+
+ private List<ManifestFile> pruneManifestsByAffectedPartitions(
+ List<ManifestFile> manifests, PartitionSet affectedPartitions) {
+ if (affectedPartitions.isEmpty()) {
+ return manifests;
+ }
+
+ Expression affectedExpr =
buildAffectedPartitionExpression(affectedPartitions);
+ if (affectedExpr == Expressions.alwaysFalse()) {
+ return manifests;
+ }
+
+ List<ManifestFile> pruned = Lists.newArrayList();
+ for (ManifestFile manifest : manifests) {
+ PartitionSpec spec = table().specs().get(manifest.partitionSpecId());
+ if (spec == null || spec.isUnpartitioned()) {
+ pruned.add(manifest);
+ } else if (manifestOverlapsFilter(manifest, spec, affectedExpr)) {
+ pruned.add(manifest);
+ }
+ }
+ return pruned;
+ }
+
+ private Expression buildAffectedPartitionExpression(PartitionSet
affectedPartitions) {
+ Expression combined = null;
+
+ for (Pair<Integer, StructLike> pair : affectedPartitions) {
+ int specId = pair.first();
+ StructLike partition = pair.second();
+ PartitionSpec spec = table().specs().get(specId);
+ if (spec == null) {
+ continue;
+ } else if (spec.isUnpartitioned()) {
+ return Expressions.alwaysTrue(); // FALLBACK: Global delete exists,
include ALL manifests!
+ }
+
+ Expression specExpr = null;
+ for (int i = 0; i < spec.fields().size(); i++) {
+ org.apache.iceberg.PartitionField field = spec.fields().get(i);
+ Object value = partition.get(i, Object.class);
+ if (value != null) {
+ String columnName =
table().schema().findColumnName(field.sourceId());
+ if (columnName != null) {
+ Expression equalExpr = Expressions.equal(columnName, value);
Review Comment:
this is not correct for bucket transform
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
