JingsongLi commented on code in PR #10980:
URL: https://github.com/apache/paimon/pull/10980#discussion_r4624963829
##########
paimon-python/pypaimon/branch/filesystem_branch_manager.py:
##########
@@ -278,24 +280,83 @@ def fast_forward(self, branch_name: str) -> None:
self.snapshot_manager, branch_name
)
- earliest_snapshot_id =
branch_snapshot_manager.try_get_earliest_snapshot()
- if earliest_snapshot_id is None:
+ earliest_snapshot =
branch_snapshot_manager.try_get_earliest_snapshot()
+ if earliest_snapshot is None:
raise RuntimeError(
f"Cannot fast forward branch {branch_name}, "
"because it does not have snapshot."
)
- # Delete snapshot, schema, and tag from the main branch which
occurs after
- # earliestSnapshotId
- # Note: This is a simplified version - full implementation would
need
- # to properly handle these deletions
-
- # Copy files from branch to main
+ branch_schema_manager =
self.schema_manager.copy_with_branch(branch_name)
+ branch_tag_manager = self._copy_with_branch(self.tag_manager,
branch_name)
+ branch_snapshots = self._snapshot_paths(branch_snapshot_manager)
+ main_snapshots = self._snapshot_paths(self.snapshot_manager)
+ branch_schema_ids =
set(branch_schema_manager._list_versioned_files())
+ if (earliest_snapshot.id not in branch_snapshots
+ or earliest_snapshot.schema_id not in branch_schema_ids):
+ raise RuntimeError(f"Incomplete snapshot or schema metadata in
branch '{branch_name}'.")
+ branch_tag_directory = branch_tag_manager.tag_directory()
+ branch_tags = (set(branch_tag_manager.list_tags())
+ if self.file_io.exists(branch_tag_directory) else
set())
+
+ # Replace metadata from the branch's earliest retained snapshot
onward.
+ # Files with the same name are overwritten; only main-only files
need deletion.
+ # Inspect snapshot names, not their contents, so a partially
copied snapshot
+ # from a failed attempt can be overwritten on retry.
+ delete_paths = [
+ path for snapshot_id, path in main_snapshots.items()
+ if snapshot_id >= earliest_snapshot.id and snapshot_id not in
branch_snapshots
+ ]
+ delete_paths.extend(
+ self.schema_manager._to_schema_path(schema_id)
+ for schema_id in self.schema_manager._list_versioned_files()
+ if schema_id >= earliest_snapshot.schema_id and schema_id not
in branch_schema_ids
+ )
+ if self.file_io.exists(self.tag_manager.tag_directory()):
+ delete_paths.extend(
+ self.tag_manager.tag_path(tag_name)
+ for tag_name in self.tag_manager.list_tags()
+ if tag_name not in branch_tags
+ and self.tag_manager.get_or_throw(tag_name).id >=
earliest_snapshot.id
+ )
+
+ if isinstance(self.file_io, CachingFileIO):
+ # Clear replaced and removed paths, including IDs a later
commit may reuse.
+ # Fail before changing metadata if persistent cache entries
cannot be removed.
+ invalidate_paths = set(delete_paths)
+
invalidate_paths.update(self.schema_manager._to_schema_path(schema_id)
+ for schema_id in branch_schema_ids)
+
invalidate_paths.update(self.snapshot_manager.get_snapshot_path(snapshot_id)
+ for snapshot_id in branch_snapshots)
+ self.file_io.invalidate_cache(invalidate_paths)
+
+ # Install schemas before snapshots which may reference them.
+ # This operation is not a multi-file transaction.
+ self.schema_manager.schema_cache.clear()
self.file_io.copy_files(
Review Comment:
**[P1] Keep schemas referenced by the retained main snapshots**
I wrote real Parquet data after adding `main_col STRING` on main and
`branch_col STRING` independently on a branch. Both additions use schema ID 1.
With the branch retaining snapshots from 3 onward, `fast_forward` leaves main
snapshot 2 in place but overwrites its schema file. Time travel to snapshot 2
read the original `main_col='new-value'` before the operation. After
`fast_forward` the same snapshot returns `branch_col=NULL` and no `main_col`.
Check schema dependencies of retained main snapshots and tags before
installing branch schemas. If both meanings of an ID cannot coexist, reject
before changing metadata or remap one side and its references.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]