zeroshade commented on code in PR #2669:
URL: https://github.com/apache/iceberg-go/pull/2669#discussion_r4645701709


##########
table/transaction_delete.go:
##########
@@ -1030,10 +1043,72 @@ func (t *Transaction) writePositionDeletesForFiles(ctx 
context.Context, fs io.IO
        for _, dvFile := range existingDVs {
                updater.removeDeletionVector(dvFile)
        }
+       for _, f := range folded.drop {
+               updater.removeDeleteFile(f)
+       }
 
        return referenced, nil
 }
 
+// foldedPositionDeletes is what a v3 delete takes over from v2 position 
delete files.
+type foldedPositionDeletes struct {
+       // seeds maps a data file without a deletion vector to the positions 
its position delete files removed.
+       seeds map[string]*dv.RoaringPositionBitmap
+       // drop lists the file-scoped position delete files whose positions 
moved into a seed.
+       drop []iceberg.DataFile
+}
+
+// foldPositionDeletes collects the v2 position deletes of every file in 
partial that has no deletion vector yet.
+//
+// Like Java's BaseDVFileWriter, only file-scoped delete files (one target 
data file) are dropped. A
+// partition-scoped file can cover other data files, so it stays: readers 
ignore it for files with a vector
+// and still apply it to the rest. Dropping is safe because every seeded file 
gets a vector written.
+func foldPositionDeletes(ctx context.Context, fs io.IO, partial 
[]iceberg.ManifestEntry, positional []iceberg.ManifestEntry, dvs 
liveDeletionVectors) (foldedPositionDeletes, error) {
+       var folded foldedPositionDeletes
+       if len(partial) == 0 || len(positional) == 0 {
+               return folded, nil
+       }
+       idx, err := buildPositionalDeleteIndex(positional)
+       if err != nil {
+               return folded, err
+       }
+
+       folded.seeds = make(map[string]*dv.RoaringPositionBitmap)
+       read := make(map[string]map[string]*dv.RoaringPositionBitmap) // delete 
file path -> target path -> positions
+       for _, entry := range partial {
+               path := entry.DataFile().FilePath()
+               if _, ok := dvs[path]; ok {
+                       continue
+               }
+               matched, err := idx.forDataFile(entry)
+               if err != nil {
+                       return folded, err
+               }
+               for _, del := range matched {
+                       targets, ok := read[del.FilePath()]
+                       if !ok {
+                               if targets, err = 
readPositionDeleteBitmapsForPaths(ctx, fs, del, nil); err != nil {

Review Comment:
   `readDeletesForPaths` already takes a target set. Passing the paths of the 
partial files that have no DV, instead of `nil`, would keep only the positions 
that actually get folded. Then a large partition-wide v2 file no longer loads 
every target into memory, which covers the memory note in the description.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to