JingsongLi commented on code in PR #10980:
URL: https://github.com/apache/paimon/pull/10980#discussion_r4624963829


##########
paimon-python/pypaimon/branch/filesystem_branch_manager.py:
##########
@@ -278,24 +280,83 @@ def fast_forward(self, branch_name: str) -> None:
                 self.snapshot_manager, branch_name
             )
 
-            earliest_snapshot_id = 
branch_snapshot_manager.try_get_earliest_snapshot()
-            if earliest_snapshot_id is None:
+            earliest_snapshot = 
branch_snapshot_manager.try_get_earliest_snapshot()
+            if earliest_snapshot is None:
                 raise RuntimeError(
                     f"Cannot fast forward branch {branch_name}, "
                     "because it does not have snapshot."
                 )
             
-            # Delete snapshot, schema, and tag from the main branch which 
occurs after
-            # earliestSnapshotId
-            # Note: This is a simplified version - full implementation would 
need
-            # to properly handle these deletions
-            
-            # Copy files from branch to main
+            branch_schema_manager = 
self.schema_manager.copy_with_branch(branch_name)
+            branch_tag_manager = self._copy_with_branch(self.tag_manager, 
branch_name)
+            branch_snapshots = self._snapshot_paths(branch_snapshot_manager)
+            main_snapshots = self._snapshot_paths(self.snapshot_manager)
+            branch_schema_ids = 
set(branch_schema_manager._list_versioned_files())
+            if (earliest_snapshot.id not in branch_snapshots
+                    or earliest_snapshot.schema_id not in branch_schema_ids):
+                raise RuntimeError(f"Incomplete snapshot or schema metadata in 
branch '{branch_name}'.")
+            branch_tag_directory = branch_tag_manager.tag_directory()
+            branch_tags = (set(branch_tag_manager.list_tags())
+                           if self.file_io.exists(branch_tag_directory) else 
set())
+
+            # Replace metadata from the branch's earliest retained snapshot 
onward.
+            # Files with the same name are overwritten; only main-only files 
need deletion.
+            # Inspect snapshot names, not their contents, so a partially 
copied snapshot
+            # from a failed attempt can be overwritten on retry.
+            delete_paths = [
+                path for snapshot_id, path in main_snapshots.items()
+                if snapshot_id >= earliest_snapshot.id and snapshot_id not in 
branch_snapshots
+            ]
+            delete_paths.extend(
+                self.schema_manager._to_schema_path(schema_id)
+                for schema_id in self.schema_manager._list_versioned_files()
+                if schema_id >= earliest_snapshot.schema_id and schema_id not 
in branch_schema_ids
+            )
+            if self.file_io.exists(self.tag_manager.tag_directory()):
+                delete_paths.extend(
+                    self.tag_manager.tag_path(tag_name)
+                    for tag_name in self.tag_manager.list_tags()
+                    if tag_name not in branch_tags
+                    and self.tag_manager.get_or_throw(tag_name).id >= 
earliest_snapshot.id
+                )
+
+            if isinstance(self.file_io, CachingFileIO):
+                # Clear replaced and removed paths, including IDs a later 
commit may reuse.
+                # Fail before changing metadata if persistent cache entries 
cannot be removed.
+                invalidate_paths = set(delete_paths)
+                
invalidate_paths.update(self.schema_manager._to_schema_path(schema_id)
+                                        for schema_id in branch_schema_ids)
+                
invalidate_paths.update(self.snapshot_manager.get_snapshot_path(snapshot_id)
+                                        for snapshot_id in branch_snapshots)
+                self.file_io.invalidate_cache(invalidate_paths)
+
+            # Install schemas before snapshots which may reference them.
+            # This operation is not a multi-file transaction.
+            self.schema_manager.schema_cache.clear()
             self.file_io.copy_files(

Review Comment:
   **[P1] Keep schemas referenced by the retained main snapshots**
   
   I wrote real Parquet data after adding `main_col STRING` on main and 
`branch_col STRING` independently on a branch. Both additions use schema ID 1. 
With the branch retaining snapshots from 3 onward, `fast_forward` leaves main 
snapshot 2 in place but overwrites its schema file. Time travel to snapshot 2 
read the original `main_col='new-value'` before the operation. After 
`fast_forward` the same snapshot returns `branch_col=NULL` and no `main_col`.
   
   Check schema dependencies of retained main snapshots and tags before 
installing branch schemas. If both meanings of an ID cannot coexist, reject 
before changing metadata or remap one side and its references.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to