badalprasadsingh commented on code in PR #2601:
URL: https://github.com/apache/iceberg-go/pull/2601#discussion_r4596384928
##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,65 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t
*testing.T) {
})
}
+func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ fs, inner, tbl := newExpireReplayTable(t, nil)
+ var ids [4]int64
+ for i, name := range []string{"s1", "s2", "s3", "s4"} {
+ tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name)
+ }
+ peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag",
ids[0], TagRef, time.Hour.Milliseconds(), 0, 0))
+ time.Sleep(2 * time.Hour)
+
+ cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner}
+ tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction()
+ require.NoError(t, tx.ExpireSnapshots(
+ WithOlderThan(0), WithRetainLast(5),
WithPostCommit(false),
+ ))
+ staged, err := tx.StagedTable()
+ require.NoError(t, err)
+ assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag"))
+ assert.Equal(t, sortedIDs(ids[:]...),
survivors(staged.Metadata()))
+
+ peer := peerAppend(t, fs, inner, MainBranch, "s5")
+ partial := refsOnlySnapshotMetadata(t, inner.metadata, ids[0],
peer)
+ assert.Equal(t, sortedIDs(ids[0], peer), survivors(partial))
+ cat.partial = partial
+ attemptsBefore := inner.attempts.Load()
+
+ _, err = tx.Commit(t.Context())
+ require.ErrorIs(t, err, ErrTransactionUnusable)
+ require.ErrorIs(t, err,
errSnapshotHistoryIncompleteForExpiration)
+ assert.Equal(t, 1, cat.loads)
+ assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "retry
must stop before resubmitting expiration")
+ assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3],
peer), survivors(inner.metadata))
+ assert.NotNil(t, inner.metadata.SnapshotByID(ids[0]), "retained
history must remain in catalog metadata")
+ assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag"),
"failed expiry must not remove the tag")
+ assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch))
+ })
+}
+
+func TestRefsOnlyReloadDoesNotBlockAppendReplay(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ fs, inner, tbl := newExpireReplayTable(t, nil)
+ tbl, first := appendOnBranch(t, tbl, MainBranch, "first")
+ tbl, _ = appendOnBranch(t, tbl, MainBranch, "head")
Review Comment:
this is where the lint is failing as tbl is never read after this line
##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,65 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t
*testing.T) {
})
}
+func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) {
Review Comment:
a second Commit after a failed one also refreshes, but no test covers that
path
can you please add this test case: with CommitNumRetriesKey: "0": the first
Commit fails with "ErrCommitFailed", the second returns
"ErrTransactionUnusable", and the catalog still has the tag and ids[0]
##########
table/transaction.go:
##########
@@ -827,6 +827,9 @@ func (t *Transaction) ExpireSnapshots(opts
...ExpireSnapshotsOpt) error {
if err := validateGCEnabledForSnapshotExpiration(meta.props); err !=
nil {
return err
}
+ if t.tbl.snapshotHistoryRefsOnly {
+ return errSnapshotHistoryIncompleteForExpiration
Review Comment:
callers cant' match it with error.Is
better: do this and check it with "require.ErrorIs" in the test
```go
fmt.Errorf("%w: %w", ErrInvalidOperation,
errSnapshotHistoryIncompleteForExpiration)
```
##########
website/src/releases.md:
##########
@@ -139,6 +139,8 @@ Per-release notes (highlights, breaking changes,
contributors) are published on
- **Commit rejections now match `table.ErrCommitRejected`.** SQL, Glue, Hive
and Hadoop failures while preparing metadata, plus REST commit responses 400,
401, 403 and 404, leave transactions committable (after a REST 404, the next
`Commit` fails when it reloads the table). Existing `errors.Is` matches keep
working, but SQL, Glue, Hive and Hadoop rejection messages now start with
`commit rejected: `, so code that matches error text may need updating.
- **Float comparisons against literals beyond the float32 range follow IEEE
754.** On a float32 column, `x < 1e100` no longer returns NaN, `+Inf` or NULL
rows, and `NOT(x > 1e100)` no longer returns NULL rows. An infinite literal now
binds to a float32 infinity, so `x = +Inf` matches `+Inf` rows. Java still
folds these comparisons to always true or false, see
[apache/iceberg#18700](https://github.com/apache/iceberg/issues/18700).
+- **Snapshot expiration requires complete snapshot history.** Staging
`Transaction.ExpireSnapshots` fails when REST loaded the table with
`snapshot-loading-mode=refs`. A commit retry can also return
`table.ErrTransactionUnusable` if its refresh omits history before expiration
replay. Reload with `snapshot-loading-mode=all` and start a new transaction.
Review Comment:
there is a "ExpireSnapshots" doc and the website that don't mention this
limit yet
better: add a line to the doc, to api.md "Expiring snapshots" and to
"cli.md" expire-snapshots, like the existing refs-mode notes for fast-forward
and the branch commands. this note could also name the iceberg expire-snapshots
command.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]