skyyyy955 commented on code in PR #10980:
URL: https://github.com/apache/paimon/pull/10980#discussion_r4625556796


##########
paimon-python/pypaimon/branch/filesystem_branch_manager.py:
##########
@@ -278,24 +280,83 @@ def fast_forward(self, branch_name: str) -> None:
                 self.snapshot_manager, branch_name
             )
 
-            earliest_snapshot_id = 
branch_snapshot_manager.try_get_earliest_snapshot()
-            if earliest_snapshot_id is None:
+            earliest_snapshot = 
branch_snapshot_manager.try_get_earliest_snapshot()
+            if earliest_snapshot is None:
                 raise RuntimeError(
                     f"Cannot fast forward branch {branch_name}, "
                     "because it does not have snapshot."
                 )
             
-            # Delete snapshot, schema, and tag from the main branch which 
occurs after
-            # earliestSnapshotId
-            # Note: This is a simplified version - full implementation would 
need
-            # to properly handle these deletions
-            
-            # Copy files from branch to main
+            branch_schema_manager = 
self.schema_manager.copy_with_branch(branch_name)
+            branch_tag_manager = self._copy_with_branch(self.tag_manager, 
branch_name)
+            branch_snapshots = self._snapshot_paths(branch_snapshot_manager)
+            main_snapshots = self._snapshot_paths(self.snapshot_manager)
+            branch_schema_ids = 
set(branch_schema_manager._list_versioned_files())
+            if (earliest_snapshot.id not in branch_snapshots
+                    or earliest_snapshot.schema_id not in branch_schema_ids):
+                raise RuntimeError(f"Incomplete snapshot or schema metadata in 
branch '{branch_name}'.")
+            branch_tag_directory = branch_tag_manager.tag_directory()
+            branch_tags = (set(branch_tag_manager.list_tags())
+                           if self.file_io.exists(branch_tag_directory) else 
set())
+
+            # Replace metadata from the branch's earliest retained snapshot 
onward.
+            # Files with the same name are overwritten; only main-only files 
need deletion.
+            # Inspect snapshot names, not their contents, so a partially 
copied snapshot
+            # from a failed attempt can be overwritten on retry.
+            delete_paths = [
+                path for snapshot_id, path in main_snapshots.items()
+                if snapshot_id >= earliest_snapshot.id and snapshot_id not in 
branch_snapshots
+            ]
+            delete_paths.extend(
+                self.schema_manager._to_schema_path(schema_id)
+                for schema_id in self.schema_manager._list_versioned_files()
+                if schema_id >= earliest_snapshot.schema_id and schema_id not 
in branch_schema_ids
+            )
+            if self.file_io.exists(self.tag_manager.tag_directory()):
+                delete_paths.extend(
+                    self.tag_manager.tag_path(tag_name)
+                    for tag_name in self.tag_manager.list_tags()
+                    if tag_name not in branch_tags
+                    and self.tag_manager.get_or_throw(tag_name).id >= 
earliest_snapshot.id
+                )
+
+            if isinstance(self.file_io, CachingFileIO):
+                # Clear replaced and removed paths, including IDs a later 
commit may reuse.
+                # Fail before changing metadata if persistent cache entries 
cannot be removed.
+                invalidate_paths = set(delete_paths)
+                
invalidate_paths.update(self.schema_manager._to_schema_path(schema_id)
+                                        for schema_id in branch_schema_ids)
+                
invalidate_paths.update(self.snapshot_manager.get_snapshot_path(snapshot_id)
+                                        for snapshot_id in branch_snapshots)
+                self.file_io.invalidate_cache(invalidate_paths)
+
+            # Install schemas before snapshots which may reference them.
+            # This operation is not a multi-file transaction.
+            self.schema_manager.schema_cache.clear()
             self.file_io.copy_files(

Review Comment:
   Thanks for catching this, @JingsongLi. I reproduced it with Parquet. After 
fast-forward, reading main snapshot 2 returned `branch_col=NULL` and lost the 
value in `main_col`. I saw the same result when only a tag retained that 
snapshot.
   
   Fixed in 333bd1a0dc. Before changing metadata, fast-forward now checks the 
snapshots and tags it will keep. It conservatively protects schema IDs up to 
the highest one used by that history, since a snapshot can still read files 
written under an older schema. Matching schema files stay in place. If a 
protected schema would be replaced with a different definition or deleted, 
fast-forward fails with the schema ID.
   
   All four new Parquet cases reproduced on the previous head and pass with the 
fix. The Windows branch, cache and time-travel selection passed (114 tests).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to