laskoviymishka commented on code in PR #2624:
URL: https://github.com/apache/iceberg-go/pull/2624#discussion_r4640125000
##########
table/arrow_utils.go:
##########
@@ -2519,7 +2504,25 @@ func positionDeleteRecordsToDataFilesDV(ctx
context.Context, rootLocation string
return
}
- if err := writer.AddPosition(filePath,
positions.Value(int(i)), pCtx.specID, pCtx.partitionData); err != nil {
+ position := positions.Value(int(i))
+ if position < 0 {
+ yield(nil, fmt.Errorf("%w: invalid
deletion position %d for %q: positions must be >= 0",
+ iceberg.ErrInvalidArgument,
position, filePath))
+
+ return
+ }
+ if bitmap := args.existingDeletes[filePath];
bitmap != nil && bitmap.Contains(uint64(position)) {
+ continue
+ }
+ if _, ok := seededPaths[filePath]; !ok {
Review Comment:
Could we add an upgrade test with two stats-selected files, where A gets a
new position and B's matching row is already deleted by a legacy file? This
gate is what keeps B's legacy file live with no DV for B, but in the current
cases stats prune the sibling
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/mor_delete_pruning_test.go#L574-L683),
so a regression here would bring B's rows back silently.
##########
table/transaction_delete.go:
##########
@@ -1018,30 +1019,54 @@ func (t *Transaction) writePositionDeletesForFiles(ctx
context.Context, fs io.IO
}
}
}
- for _, ref := range streamedPathOrder {
- if _, ok := seen[ref]; !ok {
- seen[ref] = struct{}{}
- referenced = append(referenced, ref)
+ if meta.formatVersion < 3 {
+ for _, ref := range streamedPathOrder {
+ if _, ok := seen[ref]; !ok {
+ seen[ref] = struct{}{}
+ referenced = append(referenced, ref)
+ }
}
}
- // Supersede the old DVs only after the merged replacements are staged,
so a
- // write failure above leaves the existing DVs untouched.
- for _, dvFile := range existingDVs {
- updater.removeDeletionVector(dvFile)
- }
+ // Only replace old deletes for data files that actually received a DV.
+ supersedeDeletesForWrittenDVs(updater, seen, oldPositionDeletes, dvs)
return referenced, nil
}
-func (t *Transaction) makePositionDeleteRecordsForFilter(ctx context.Context,
fs io.IO, files []iceberg.DataFile, filter iceberg.BooleanExpression,
caseSensitive bool, concurrency int) (seq2 iter.Seq2[arrow.RecordBatch, error],
err error) {
+// supersedeDeletesForWrittenDVs removes old deletes only for data files that
+// received a replacement DV. File-scoped legacy deletes are folded into the
+// new bitmap; shared position-delete files must remain live because another
+// data file can still need them.
+func supersedeDeletesForWrittenDVs(updater *snapshotProducer, replacedPaths
map[string]struct{}, oldPositionDeletes *legacyPositionDeletes, oldDVs
liveDeletionVectors) {
+ for dataFilePath := range replacedPaths {
+ for _, deleteFile := range
oldPositionDeletes.fileScoped[dataFilePath] {
+ updater.removeDeleteFile(deleteFile)
Review Comment:
Can we pin this new non-replayable trigger with a test? On `main` this
delete removed nothing on an upgraded table and stayed replayable; now
`removeDeleteFile` sets `noReplay`
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/snapshot_producers.go#L1730-L1732),
and `no_replay_removals_test.go` only covers DV supersede and `ReplaceFiles`.
Something like the DV case: land a peer commit, assert `ErrCommitFailed`, then
that a retry succeeds.
##########
table/transaction_delete.go:
##########
@@ -1127,7 +1152,41 @@ func (t *Transaction)
makePositionDeleteRecordsForFilter(ctx context.Context, fs
// the error if some goroutines completed before the failure.
releasePerFilePosDeletes(deletesPerFile)
- return nil, err
+ return nil, nil, err
+ }
+
+ oldPositionDeletes = &legacyPositionDeletes{}
+ // Reuse the planner-matched position deletes when upgrading a v2 table
to
+ // v3. They have already been read for filtering out previously deleted
+ // rows, so turning them into DV seeds does not require another file
read.
+ if meta.formatVersion >= 3 {
+ oldPositionDeletes.bitmaps =
make(map[string]*dv.RoaringPositionBitmap, len(deletesPerFile))
+ for path, positions := range deletesPerFile {
+ bitmap, err := collectPosDeleteBitmap(positions)
+ if err != nil {
+ releasePerFilePosDeletes(deletesPerFile)
+
+ return nil, nil, fmt.Errorf("invalid existing
position deletes for %s: %w", path, err)
+ }
+ if !bitmap.IsEmpty() {
+ oldPositionDeletes.bitmaps[path] = bitmap
+ }
+ }
+
+ // Java considers a position-delete file file-scoped when it
has an
+ // explicit referenced_data_file or equal file_path bounds.
Only such
+ // files can be removed once their target receives a
replacement DV.
+ oldPositionDeletes.fileScoped =
make(map[string][]iceberg.DataFile)
+ for _, task := range tasks {
+ path := task.File.FilePath()
+ for _, deleteFile := range task.DeleteFiles {
+ if deleteFile.ContentType() !=
iceberg.EntryContentPosDeletes ||
+ IsDeletionVector(deleteFile) ||
referencedDataFilePath(deleteFile) != path {
Review Comment:
Could one upgrade test use a legacy delete file with no
`referenced_data_file` and no `file_path` bounds? `referencedDataFilePath`
returns `""` for it
(https://github.com/apache/iceberg-go/blob/6655a714e05e9a00bb24ab9cf575aa6e9a3e4d85/table/conflict_validation.go#L1300-L1322),
so it counts as shared and stays live after the DV is written. That's the safe
direction and what Java's `isFileScoped` does; a test would lock it in.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]