wombatu-kun commented on code in PR #16910:
URL: https://github.com/apache/iceberg/pull/16910#discussion_r3791832997
##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java:
##########
@@ -302,21 +302,31 @@ private Result rebuildMetadata() {
Set<Snapshot> validSnapshots =
Sets.difference(snapshotSet(endMetadata), snapshotSet(startMetadata));
- // rebuild manifest-list files
- Set<RewriteResult<ManifestFile>> manifestListResults =
Sets.newConcurrentHashSet();
+ // Read every valid snapshot's manifest list once, before writing
anything. The rewritten
+ // manifest list must record each referenced manifest's rewritten length
(manifest_length),
+ // which is only known after the manifests are rewritten below.
+ Map<Snapshot, List<ManifestFile>> manifestsBySnapshot =
Maps.newConcurrentMap();
Tasks.foreach(validSnapshots)
.noRetry()
.throwFailureWhenFinished()
.executeWith(executorService)
.run(
snapshot ->
- manifestListResults.add(
- rewriteManifestList(snapshot, endMetadata,
manifestsToRewrite)));
-
- RewriteResult<ManifestFile> rewriteManifestListResult = new
RewriteResult<>();
- manifestListResults.forEach(rewriteManifestListResult::append);
-
- Set<ManifestFile> manifestFiles = rewriteManifestListResult.toRewrite();
+ manifestsBySnapshot.put(
+ snapshot,
+ RewriteTablePathUtil.manifestsInSnapshot(snapshot,
table.io(), sourcePrefix)));
+
+ // Manifests selected for rewrite. In an incremental run this is only the
manifests added by the
+ // delta snapshots, so a manifest carried over from an earlier run is not
rewritten here and
+ // keeps its source length in the manifest list. See the note on
rewriteManifestList.
+ Set<ManifestFile> manifestFiles = Sets.newHashSet();
Review Comment:
Done 90d25bf7d
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
