ebyhr commented on code in PR #4437:
URL: https://github.com/apache/iceberg-python/pull/4437#discussion_r4637734551
##########
pyiceberg/table/update/snapshot.py:
##########
@@ -1460,5 +1478,24 @@ def older_than(self, dt: datetime) -> ExpireSnapshots:
expire_from = datetime_to_millis(dt)
for snapshot in self._transaction.table_metadata.snapshots:
if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id
not in protected_ids:
- self._snapshot_ids_to_expire.add(snapshot.snapshot_id)
+ self._snapshot_ids_older_than.add(snapshot.snapshot_id)
+ return self
+
+ def retain_last(self, num_snapshots: int) -> ExpireSnapshots:
Review Comment:
In Java, `retainLast(n)` alone still expires snapshots older than
`history.expire.max-snapshot-age-ms`, keeping `n` per branch
([`RemoveSnapshots`](https://github.com/apache/iceberg/blob/95aac2f4b100db7f8f37563d58d09daf01196d47/core/src/main/java/org/apache/iceberg/RemoveSnapshots.java#L91-L99),
test: https://github.com/apache/iceberg/pull/19049). Here it does nothing
without `older_than`.
Could we note that in the docstring? Honoring
`history.expire.max-snapshot-age-ms` and `history.expire.min-snapshots-to-keep`
(and the ref-level overrides) looks like a separate follow-up.
##########
tests/table/test_expire_snapshots.py:
##########
@@ -343,3 +347,77 @@ def
test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> N
assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] ==
[KEEP_SNAPSHOT]
assert [stat.snapshot_id for stat in new_metadata.partition_statistics] ==
[KEEP_SNAPSHOT]
+
+
+def _table_with_branches(example_table_metadata_v2: dict[str, Any]) -> Table:
+ """Create a table where main is 1 <- 2 <- 3 <- 4 and branch "b" is 1 <- 2
<- 5."""
+ parents = {1: None, 2: 1, 3: 2, 4: 3, 5: 2}
+ snapshots = [
+ {
+ "snapshot-id": snapshot_id,
+ **({"parent-snapshot-id": parent_id} if parent_id is not None else
{}),
+ "timestamp-ms": snapshot_id * 1000,
+ "sequence-number": snapshot_id,
+ "summary": {"operation": "append"},
+ "manifest-list": f"s3://a/b/{snapshot_id}.avro",
+ }
+ for snapshot_id, parent_id in parents.items()
+ ]
+ metadata = TableMetadataV2(
+ **{
+ **example_table_metadata_v2,
+ "current-snapshot-id": 4,
+ "last-sequence-number": 5,
+ "snapshots": snapshots,
+ "snapshot-log": [],
+ "refs": {"main": {"snapshot-id": 4, "type": "branch"}, "b":
{"snapshot-id": 5, "type": "branch"}},
+ }
+ )
+ return Table(
+ identifier=("database", "table"),
+ metadata=metadata,
+ metadata_location=f"{metadata.location}/uuid.metadata.json",
+ io=load_file_io(),
+ catalog=NoopCatalog("NoopCatalog"),
+ )
+
+
[email protected](
+ "num_snapshots, expected_remaining",
+ [(1, {4, 5}), (2, {2, 3, 4, 5}), (10, {1, 2, 3, 4, 5})],
+)
+def test_expire_snapshots_retain_last(
+ example_table_metadata_v2: dict[str, Any], num_snapshots: int,
expected_remaining: set[int]
+) -> None:
+ table = _table_with_branches(example_table_metadata_v2)
+ future_datetime = datetime.now() + timedelta(days=1)
+
+ txn = table.transaction()
+
ExpireSnapshots(txn).older_than(future_datetime).retain_last(num_snapshots).commit()
+
+ assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots}
== expected_remaining
+
+
+def
test_expire_snapshots_retain_last_before_older_than(example_table_metadata_v2:
dict[str, Any]) -> None:
+ table = _table_with_branches(example_table_metadata_v2)
+ future_datetime = datetime.now() + timedelta(days=1)
+
+ txn = table.transaction()
+ ExpireSnapshots(txn).retain_last(2).older_than(future_datetime).commit()
+
+ assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots}
== {2, 3, 4, 5}
+
+
+def
test_expire_snapshots_retain_last_does_not_retain_by_id(example_table_metadata_v2:
dict[str, Any]) -> None:
Review Comment:
Could we add a test that `retain_last` alone expires nothing? It pins the
current behavior, so it's a visible change if the `history.expire.*` defaults
get wired in later.
```python
def
test_expire_snapshots_retain_last_without_older_than(example_table_metadata_v2:
dict[str, Any]) -> None:
table = _table_with_branches(example_table_metadata_v2)
txn = table.transaction()
ExpireSnapshots(txn).retain_last(1).commit()
assert {snapshot.snapshot_id for snapshot in
txn.table_metadata.snapshots} == {1, 2, 3, 4, 5}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]