nssalian commented on code in PR #2601:
URL: https://github.com/apache/iceberg-go/pull/2601#discussion_r4604697678
##########
website/src/api.md:
##########
@@ -853,6 +853,21 @@ if err != nil { /* ... */ }
newTbl, err := txn.Commit(ctx)
```
+Snapshot expiration needs complete ancestry to honor branch retention and
snapshot
+references. A table loaded by REST with `snapshot-loading-mode=refs` may
contain
+only directly referenced snapshots. Calling `Transaction.ExpireSnapshots` on
+such a table fails before staging any updates; `errors.Is(err,
+table.ErrInvalidOperation)` returns true. The rejected operation does not
+prevent other operations, such as appends, from using the transaction.
+
+If a concurrent writer forces a commit retry or a second `Commit` attempt
+to reload refs-only history, the expiration cannot be replayed safely.
+`Transaction.Commit` then returns `table.ErrTransactionUnusable` and
+does not resubmit the expiration; it leaves the catalog's existing snapshots
Review Comment:
For library users `snapshot-loading-mode` is a client catalog property that
wins over server defaults (catalog/rest/rest.go:1301-1306), and reloading
through the same catalog stays in refs mode (rest.go:2027). The note at
api.md:684 already says "Load with snapshot-loading-mode=all instead". Suggest:
"Create the REST catalog with `snapshot-loading-mode=all` (or omit it), reload,
and start a new transaction. If the server forces `refs` through config
overrides, expiration is not available through that catalog." Same wording as
suggested on https://github.com/apache/iceberg-go/pull/2599.
##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,130 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t
*testing.T) {
})
}
+// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a
+// history-dependent operation does not poison an otherwise valid transaction.
+// Even a no-op expiration is unsafe to plan from refs-only metadata.
+func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) {
+ fs, cat, tbl := newExpireReplayTable(t, nil)
+ tbl.snapshotHistoryRefsOnly = true
+ tx := tbl.NewTransaction()
+
+ err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false))
+ require.ErrorIs(t, err, ErrInvalidOperation)
+ require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration)
+ staged, err := tx.StagedTable()
+ require.NoError(t, err)
+ require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration
must not stage updates")
Review Comment:
nit: the fixture starts with no snapshots, so this can't fail. Comparing the
transaction's staged updates and requirements before and after the call would
check what the message says.
##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,130 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t
*testing.T) {
})
}
+// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a
+// history-dependent operation does not poison an otherwise valid transaction.
+// Even a no-op expiration is unsafe to plan from refs-only metadata.
+func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) {
+ fs, cat, tbl := newExpireReplayTable(t, nil)
+ tbl.snapshotHistoryRefsOnly = true
+ tx := tbl.NewTransaction()
+
+ err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false))
+ require.ErrorIs(t, err, ErrInvalidOperation)
+ require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration)
+ staged, err := tx.StagedTable()
+ require.NoError(t, err)
+ require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration
must not stage updates")
+
+ stageFiles(t, tx, "after-rejected-expiration")
+ committed, err := tx.Commit(t.Context())
+ require.NoError(t, err)
+ require.Len(t, committed.Metadata().Snapshots(), 1)
+ require.Equal(t, []string{"after-rejected-expiration"},
+ branchFiles(t, fs, committed.Metadata(), MainBranch))
+ require.Equal(t, int32(1), cat.attempts.Load(), "append should commit
once")
+}
+
+func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ fs, inner, tbl := newExpireReplayTable(t, nil)
+ var ids [4]int64
+ for i, name := range []string{"s1", "s2", "s3", "s4"} {
+ tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name)
+ }
+ peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag",
ids[0], TagRef, time.Hour.Milliseconds(), 0, 0))
+ time.Sleep(2 * time.Hour)
+
+ cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner}
+ tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction()
+ require.NoError(t, tx.ExpireSnapshots(
+ WithOlderThan(0), WithRetainLast(5),
WithPostCommit(false),
+ ))
+ staged, err := tx.StagedTable()
+ require.NoError(t, err)
+ assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag"))
+ assert.Equal(t, sortedIDs(ids[:]...),
survivors(staged.Metadata()))
+
+ peer := peerAppend(t, fs, inner, MainBranch, "s5")
+ partial := refsOnlySnapshotMetadata(t, inner.metadata, ids[0],
peer)
+ assert.Equal(t, sortedIDs(ids[0], peer), survivors(partial))
+ cat.partial = partial
+ attemptsBefore := inner.attempts.Load()
+
+ _, err = tx.Commit(t.Context())
+ require.ErrorIs(t, err, ErrTransactionUnusable)
+ require.ErrorIs(t, err,
errSnapshotHistoryIncompleteForExpiration)
+ assert.Equal(t, 1, cat.loads)
+ assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "retry
must stop before resubmitting expiration")
+ assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3],
peer), survivors(inner.metadata))
+ assert.NotNil(t, inner.metadata.SnapshotByID(ids[0]), "retained
history must remain in catalog metadata")
+ assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag"),
"failed expiry must not remove the tag")
+ assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch))
+ })
+}
+
+func TestExpireReplayRejectsRefsOnlyHistoryOnResume(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ fs, inner, tbl := newExpireReplayTable(t,
iceberg.Properties{CommitNumRetriesKey: "0"})
Review Comment:
minor: the setup through the expired-tag assertion is the same 14 lines as
...AfterRetry, and the tail assertions match too. A `stage := func(t
*testing.T, numRetries string)` closure like
TestExpireReplayKeepsExpiryThatReplannedToNothing (:580) would let both run as
subtests.
##########
table/transaction.go:
##########
@@ -827,6 +835,9 @@ func (t *Transaction) ExpireSnapshots(opts
...ExpireSnapshotsOpt) error {
if err := validateGCEnabledForSnapshotExpiration(meta.props); err !=
nil {
return err
}
+ if t.tbl.snapshotHistoryRefsOnly {
Review Comment:
not blocking: a table returned by Commit keeps the refs-only flag
(table.go:1029 copies it) even though REST commit responses are parsed with the
full snapshot list (rest.go:1892-1894), so ExpireSnapshots on the committed
table is rejected too. Failing closed is fine for 0.8.0 or do we want to clean
it up here. CC: @zeroshade @laskoviymishka
##########
table/staged_replay.go:
##########
@@ -52,6 +52,13 @@ func newStagedReplay(branch string, updates []Update,
records []expirationRecord
}
}
+// A staged expiration may have yielded no metadata updates but can produce
+// removals after a concurrent commit. Check the recorded operations, not the
+// length of staged updates, before replaying against incomplete history.
+func (r *stagedReplay) hasExpirationRecords() bool {
+ return r != nil && len(r.records) > 0
Review Comment:
The comment above is the reason for checking records, but nothing pins it:
with this returning `len(r.walkedRefs) > 0` (only expiries that produced
updates) all of ./table/ still passes. A test that stages an ExpireSnapshots
that plans nothing plus an append, then retries into refs-only history and
expects ErrTransactionUnusable, would help cover that gap.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]