AynSiddiqui commented on code in PR #2669:
URL: https://github.com/apache/iceberg-go/pull/2669#discussion_r4648675016


##########
table/transaction_delete.go:
##########
@@ -1030,10 +1043,72 @@ func (t *Transaction) writePositionDeletesForFiles(ctx 
context.Context, fs io.IO
        for _, dvFile := range existingDVs {
                updater.removeDeletionVector(dvFile)
        }
+       for _, f := range folded.drop {
+               updater.removeDeleteFile(f)
+       }
 
        return referenced, nil
 }
 
+// foldedPositionDeletes is what a v3 delete takes over from v2 position 
delete files.
+type foldedPositionDeletes struct {
+       // seeds maps a data file without a deletion vector to the positions 
its position delete files removed.
+       seeds map[string]*dv.RoaringPositionBitmap
+       // drop lists the file-scoped position delete files whose positions 
moved into a seed.
+       drop []iceberg.DataFile
+}
+
+// foldPositionDeletes collects the v2 position deletes of every file in 
partial that has no deletion vector yet.
+//
+// Like Java's BaseDVFileWriter, only file-scoped delete files (one target 
data file) are dropped. A
+// partition-scoped file can cover other data files, so it stays: readers 
ignore it for files with a vector
+// and still apply it to the rest. Dropping is safe because every seeded file 
gets a vector written.
+func foldPositionDeletes(ctx context.Context, fs io.IO, partial 
[]iceberg.ManifestEntry, positional []iceberg.ManifestEntry, dvs 
liveDeletionVectors) (foldedPositionDeletes, error) {
+       var folded foldedPositionDeletes
+       if len(partial) == 0 || len(positional) == 0 {
+               return folded, nil
+       }
+       idx, err := buildPositionalDeleteIndex(positional)
+       if err != nil {
+               return folded, err
+       }
+
+       folded.seeds = make(map[string]*dv.RoaringPositionBitmap)
+       read := make(map[string]map[string]*dv.RoaringPositionBitmap) // delete 
file path -> target path -> positions
+       for _, entry := range partial {
+               path := entry.DataFile().FilePath()
+               if _, ok := dvs[path]; ok {
+                       continue
+               }
+               matched, err := idx.forDataFile(entry)
+               if err != nil {
+                       return folded, err
+               }
+               for _, del := range matched {
+                       targets, ok := read[del.FilePath()]
+                       if !ok {
+                               if targets, err = 
readPositionDeleteBitmapsForPaths(ctx, fs, del, nil); err != nil {

Review Comment:
   Done in 6e8e23f. foldPositionDeletes now collects, per delete file, the 
paths of the partial files without a DV that it matches, and passes that set to 
readDeletesForPaths instead of nil. The delete file is still scanned, but rows 
for other data files are dropped as they are read, so a large partition-wide v2 
file keeps only the folded positions in memory (row groups can also be skipped 
by statistics). The description's memory note is updated.
   



##########
table/transaction_delete.go:
##########
@@ -1030,10 +1043,72 @@ func (t *Transaction) writePositionDeletesForFiles(ctx 
context.Context, fs io.IO
        for _, dvFile := range existingDVs {
                updater.removeDeletionVector(dvFile)
        }
+       for _, f := range folded.drop {
+               updater.removeDeleteFile(f)
+       }

Review Comment:
   Updated both Delete docs (transaction_delete.go and table.go) in 6e8e23f. 
They now say that a commit that removes a DV or drops v2 position delete files 
is not replayed: if another writer commits first it fails with ErrCommitFailed 
instead of retrying, and on an upgraded table the first delete can hit this, so 
reload and rerun.
   



##########
table/transaction_delete.go:
##########
@@ -995,6 +1006,8 @@ func (t *Transaction) writePositionDeletesForFiles(ctx 
context.Context, fs io.IO
                existingDVs[df.FilePath()] = entry.DataFile()
                existingDVBitmaps[df.FilePath()] = bitmap
        }
+       // A file without a DV starts from the positions its v2 position 
deletes removed.
+       maps.Copy(existingDVBitmaps, folded.seeds)

Review Comment:
   Added TestV3MergeOnReadSharedPositionDeleteFileDoesNotResurrectDVRows and a 
comment above maps.Copy in 6e8e23f. One shared v2 delete file covers two data 
files; after the upgrade, a first delete gives file A a DV and a second delete 
on A runs while the shared file still matches it. With the dvs skip removed in 
foldPositionDeletes the test fails (id 2 comes back); with it, it passes.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to