laskoviymishka commented on code in PR #2624:
URL: https://github.com/apache/iceberg-go/pull/2624#discussion_r4640125000


##########
table/arrow_utils.go:
##########
@@ -2519,7 +2504,25 @@ func positionDeleteRecordsToDataFilesDV(ctx 
context.Context, rootLocation string
 
                                        return
                                }
-                               if err := writer.AddPosition(filePath, 
positions.Value(int(i)), pCtx.specID, pCtx.partitionData); err != nil {
+                               position := positions.Value(int(i))
+                               if position < 0 {
+                                       yield(nil, fmt.Errorf("%w: invalid 
deletion position %d for %q: positions must be >= 0",
+                                               iceberg.ErrInvalidArgument, 
position, filePath))
+
+                                       return
+                               }
+                               if bitmap := args.existingDeletes[filePath]; 
bitmap != nil && bitmap.Contains(uint64(position)) {
+                                       continue
+                               }
+                               if _, ok := seededPaths[filePath]; !ok {

Review Comment:
   Could we add an upgrade test with two stats-selected files, where A gets a 
new position and B's matching row is already deleted by a legacy file? This 
gate is what keeps B's legacy file live with no DV for B, but in the current 
cases stats prune the sibling 
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/mor_delete_pruning_test.go#L574-L683),
 so a regression here would bring B's rows back silently.



##########
table/transaction_delete.go:
##########
@@ -1018,30 +1019,54 @@ func (t *Transaction) writePositionDeletesForFiles(ctx 
context.Context, fs io.IO
                        }
                }
        }
-       for _, ref := range streamedPathOrder {
-               if _, ok := seen[ref]; !ok {
-                       seen[ref] = struct{}{}
-                       referenced = append(referenced, ref)
+       if meta.formatVersion < 3 {
+               for _, ref := range streamedPathOrder {
+                       if _, ok := seen[ref]; !ok {
+                               seen[ref] = struct{}{}
+                               referenced = append(referenced, ref)
+                       }
                }
        }
 
-       // Supersede the old DVs only after the merged replacements are staged, 
so a
-       // write failure above leaves the existing DVs untouched.
-       for _, dvFile := range existingDVs {
-               updater.removeDeletionVector(dvFile)
-       }
+       // Only replace old deletes for data files that actually received a DV.
+       supersedeDeletesForWrittenDVs(updater, seen, oldPositionDeletes, dvs)
 
        return referenced, nil
 }
 
-func (t *Transaction) makePositionDeleteRecordsForFilter(ctx context.Context, 
fs io.IO, files []iceberg.DataFile, filter iceberg.BooleanExpression, 
caseSensitive bool, concurrency int) (seq2 iter.Seq2[arrow.RecordBatch, error], 
err error) {
+// supersedeDeletesForWrittenDVs removes old deletes only for data files that
+// received a replacement DV. File-scoped legacy deletes are folded into the
+// new bitmap; shared position-delete files must remain live because another
+// data file can still need them.
+func supersedeDeletesForWrittenDVs(updater *snapshotProducer, replacedPaths 
map[string]struct{}, oldPositionDeletes *legacyPositionDeletes, oldDVs 
liveDeletionVectors) {
+       for dataFilePath := range replacedPaths {
+               for _, deleteFile := range 
oldPositionDeletes.fileScoped[dataFilePath] {
+                       updater.removeDeleteFile(deleteFile)

Review Comment:
   Can we pin this new non-replayable trigger with a test? On `main` this 
delete removed nothing on an upgraded table and stayed replayable; now 
`removeDeleteFile` sets `noReplay` 
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/snapshot_producers.go#L1730-L1732),
 and `no_replay_removals_test.go` only covers DV supersede and `ReplaceFiles`. 
Something like the DV case: land a peer commit, assert `ErrCommitFailed`, then 
that a retry succeeds.



##########
table/transaction_delete.go:
##########
@@ -1127,7 +1152,41 @@ func (t *Transaction) 
makePositionDeleteRecordsForFilter(ctx context.Context, fs
                // the error if some goroutines completed before the failure.
                releasePerFilePosDeletes(deletesPerFile)
 
-               return nil, err
+               return nil, nil, err
+       }
+
+       oldPositionDeletes = &legacyPositionDeletes{}
+       // Reuse the planner-matched position deletes when upgrading a v2 table 
to
+       // v3. They have already been read for filtering out previously deleted
+       // rows, so turning them into DV seeds does not require another file 
read.
+       if meta.formatVersion >= 3 {
+               oldPositionDeletes.bitmaps = 
make(map[string]*dv.RoaringPositionBitmap, len(deletesPerFile))
+               for path, positions := range deletesPerFile {
+                       bitmap, err := collectPosDeleteBitmap(positions)
+                       if err != nil {
+                               releasePerFilePosDeletes(deletesPerFile)
+
+                               return nil, nil, fmt.Errorf("invalid existing 
position deletes for %s: %w", path, err)
+                       }
+                       if !bitmap.IsEmpty() {
+                               oldPositionDeletes.bitmaps[path] = bitmap
+                       }
+               }
+
+               // Java considers a position-delete file file-scoped when it 
has an
+               // explicit referenced_data_file or equal file_path bounds. 
Only such
+               // files can be removed once their target receives a 
replacement DV.
+               oldPositionDeletes.fileScoped = 
make(map[string][]iceberg.DataFile)
+               for _, task := range tasks {
+                       path := task.File.FilePath()
+                       for _, deleteFile := range task.DeleteFiles {
+                               if deleteFile.ContentType() != 
iceberg.EntryContentPosDeletes ||
+                                       IsDeletionVector(deleteFile) || 
referencedDataFilePath(deleteFile) != path {

Review Comment:
   Could one upgrade test use a legacy delete file with no 
`referenced_data_file` and no `file_path` bounds? `referencedDataFilePath` 
returns `""` for it 
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/conflict_validation.go#L1300-L1322),
 so it counts as shared and stays live after the DV is written. That's the safe 
direction and what Java's `isFileScoped` does; a test would lock it in.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to