nssalian commented on code in PR #2624:
URL: https://github.com/apache/iceberg-go/pull/2624#discussion_r4606162011


##########
website/src/api.md:
##########
@@ -421,6 +421,26 @@ newTbl, err := tbl.Delete(ctx,
 )
 ```
 
+With `write.delete.mode=merge-on-read`, format v2 writes position-delete
+files and format v3 writes deletion vectors (DVs). After a table is upgraded
+from v2 to v3, its older position-delete files remain valid. The **first**
+DV written for a data file includes all positions previously deleted from

Review Comment:
   nit: "includes all positions previously deleted from that file" could be 
read as including rows removed by equality deletes. Only position deletes are 
folded into the DV, and equality deletes keep applying on their own. The 
release note's wording, "applicable legacy position-delete files", is more 
precise and could be used here too.



##########
table/mor_delete_pruning_test.go:
##########
@@ -365,3 +404,373 @@ func TestMergeOnReadDeleteSkipsRowsAlreadyDeleted(t 
*testing.T) {
                assert.Equal(t, []int64{3, 4}, ids, "main's delete of id=1 
doesn't count on the branch")
        })
 }
+
+// TestV3MergeOnReadUpgradePreservesPositionDeletes covers the first DV written
+// after a v2 to v3 upgrade. The DV must include older position deletes, even
+// when one position-delete file targets more than one data file.
+func TestV3MergeOnReadUpgradePreservesPositionDeletes(t *testing.T) {
+       tests := []struct {
+               name        string
+               batches     []string
+               firstFilter iceberg.BooleanExpression
+               wantV2      []int64
+               wantV3      []int64
+               shared      bool
+       }{
+               {
+                       name: "single file",
+                       batches: []string{
+                               
`[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"}]`,
+                       },
+                       firstFilter: iceberg.EqualTo(iceberg.Reference("id"), 
int64(1)),
+                       wantV2:      []int64{2, 3},
+                       wantV3:      []int64{3},
+               },
+               {
+                       name: "shared position delete file",
+                       batches: []string{
+                               `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`,
+                               `[{"id":3,"data":"c"},{"id":4,"data":"d"}]`,
+                       },
+                       firstFilter: iceberg.IsIn(iceberg.Reference("id"), 
int64(1), int64(3)),
+                       wantV2:      []int64{2, 4},
+                       wantV3:      []int64{4},
+                       shared:      true,
+               },
+       }
+
+       for _, tt := range tests {
+               t.Run(tt.name, func(t *testing.T) {
+                       ctx := t.Context()
+                       tbl := newMergeOnReadTestTable(t)
+                       for _, batch := range tt.batches {
+                               tbl = appendRows(t, tbl, batch)
+                       }
+
+                       var err error
+                       tbl, err = tbl.Delete(ctx, tt.firstFilter, nil)
+                       require.NoError(t, err)
+                       require.Equal(t, tt.wantV2, idsInTable(t, tbl))
+
+                       if tt.shared {
+                               tasks, err := tbl.Scan().PlanFiles(ctx)
+                               require.NoError(t, err)
+                               require.Len(t, tasks, 2)
+                               require.Len(t, tasks[0].DeleteFiles, 1)
+                               require.Len(t, tasks[1].DeleteFiles, 1)
+                               require.Equal(t, 
tasks[0].DeleteFiles[0].FilePath(), tasks[1].DeleteFiles[0].FilePath())
+                       }
+
+                       tx := tbl.NewTransaction()
+                       require.NoError(t, tx.UpgradeFormatVersion(3))
+                       tbl, err = tx.Commit(ctx)
+                       require.NoError(t, err)
+                       require.Equal(t, tt.wantV2, idsInTable(t, tbl))
+
+                       tbl, err = tbl.Delete(ctx, 
iceberg.EqualTo(iceberg.Reference("id"), int64(2)), nil)
+                       require.NoError(t, err)
+                       require.Equal(t, tt.wantV3, idsInTable(t, tbl),
+                               "the first v3 DV must preserve previously 
deleted rows")
+                       require.Equal(t, 1, liveDVCount(t, tbl))
+                       if tt.shared {
+                               require.Equal(t, 1, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes),
+                                       "a shared legacy delete file must 
remain live")
+                       } else {
+                               require.Zero(t, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes),
+                                       "file-scoped legacy deletes must be 
removed after their DV replacement")
+                       }
+
+                       tasks, err := tbl.Scan().PlanFiles(ctx)
+                       require.NoError(t, err)
+                       var withDV, withOldPositionDeletes int
+                       for _, task := range tasks {
+                               if len(task.DeletionVectorFiles) > 0 {
+                                       withDV++
+                                       require.Empty(t, task.DeleteFiles)
+                               }
+                               if len(task.DeleteFiles) > 0 {
+                                       withOldPositionDeletes++
+                                       require.Empty(t, 
task.DeletionVectorFiles)
+                               }
+                       }
+                       require.Equal(t, 1, withDV)
+                       if tt.shared {
+                               require.Equal(t, 1, withOldPositionDeletes,
+                                       "the sibling file must still use the 
shared old position-delete file")
+                               tbl, err = tbl.Delete(ctx, 
iceberg.EqualTo(iceberg.Reference("id"), int64(4)), nil)
+                               require.NoError(t, err)
+                               require.Empty(t, idsInTable(t, tbl))
+                               require.Equal(t, 2, liveDVCount(t, tbl))
+                               require.Equal(t, 1, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes),
+                                       "the shared legacy file is still live 
after both data files have DVs")
+                       } else {
+                               require.Zero(t, withOldPositionDeletes)
+                       }
+               })
+       }
+}
+
+// TestV3MergeOnReadUpgradeKeepsUntouchedFileScopedDeletes ensures a DV
+// supersedes only legacy files for its own target. An old position-delete file
+// for another data file must stay live until that file is rewritten as a DV.
+func TestV3MergeOnReadUpgradeKeepsUntouchedFileScopedDeletes(t *testing.T) {
+       ctx := t.Context()
+       tbl := newMergeOnReadTestTable(t)
+       tbl = appendRows(t, tbl,
+               `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"}]`)
+       tbl = appendRows(t, tbl,
+               `[{"id":4,"data":"d"},{"id":5,"data":"e"},{"id":6,"data":"f"}]`)
+
+       var err error
+       tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), 
int64(1)), nil)
+       require.NoError(t, err)
+       tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), 
int64(4)), nil)
+       require.NoError(t, err)
+       require.Equal(t, []int64{2, 3, 5, 6}, idsInTable(t, tbl))
+       require.Equal(t, 2, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes))
+
+       tx := tbl.NewTransaction()
+       require.NoError(t, tx.UpgradeFormatVersion(3))
+       tbl, err = tx.Commit(ctx)
+       require.NoError(t, err)
+
+       // Only the first data file gets a new DV. The sibling's old delete file
+       // must still hide id=4, and must not be removed from the manifest.
+       tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), 
int64(2)), nil)
+       require.NoError(t, err)
+       require.Equal(t, []int64{3, 5, 6}, idsInTable(t, tbl))
+       require.Equal(t, 1, liveDVCount(t, tbl))
+       require.Equal(t, 1, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes))
+
+       tasks, err := tbl.Scan().PlanFiles(ctx)
+       require.NoError(t, err)
+       require.Len(t, tasks, 2)
+       var dvTasks, legacyTasks int
+       for _, task := range tasks {
+               switch {
+               case len(task.DeletionVectorFiles) == 1:
+                       dvTasks++
+                       require.Empty(t, task.DeleteFiles)
+               case len(task.DeleteFiles) == 1:
+                       legacyTasks++
+                       require.Empty(t, task.DeletionVectorFiles)
+               default:
+                       t.Fatalf("expected one DV or legacy position delete per 
data file: %+v", task)
+               }
+       }
+       require.Equal(t, 1, dvTasks)
+       require.Equal(t, 1, legacyTasks)
+
+       // Replacing the other file's legacy delete should remove only that 
file.
+       tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), 
int64(5)), nil)
+       require.NoError(t, err)
+       require.Equal(t, []int64{3, 6}, idsInTable(t, tbl))
+       require.Equal(t, 2, liveDVCount(t, tbl))
+       require.Zero(t, liveDeleteFileCount(t, tbl, 
iceberg.EntryContentPosDeletes))
+}
+
+// TestV3MergeOnReadMixedExistingDVsKeepsUnchangedFile guards the case
+// where one candidate has no new matching rows but another gets new deletes.
+// The first file's DV must not be removed or unnecessarily rewritten.
+func TestV3MergeOnReadMixedExistingDVsKeepsUnchangedFile(t *testing.T) {
+       ctx := t.Context()
+       tbl := newPartitionedMergeOnReadTestTable(t)
+       arrowSchema := arrow.NewSchema([]arrow.Field{

Review Comment:
   This test builds its six rows by a bit by hand: an Arrow schema for `id` and 
`category`, then `array.TableFromJSON`, then `tbl.Append`. 
`TestV3MergeOnReadUpgradePreservesPartitionedPositionDeletes` (line 712) 
repeats the same ~12 lines, and so does the existing 
`TestV3MergeOnReadDoubleDeletePartitionedMergesDeletionVectors` (line 232). The 
`appendRows(t, tbl, json)` helper in `cow_rewrite_deletes_test.go:408` gets the 
Arrow schema from the table and does the append, so each block can be a single 
`tbl = appendRows(t, tbl, ...)` call.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to