AndriiSushko commented on issue #18758: URL: https://github.com/apache/iceberg/issues/18758#issuecomment-6694781539
A smaller repro for the case where `main` already has a snapshot, using only the Python stdlib. Both commits require `main` to still be at the same existing snapshot, so at most one should succeed. It doesn't need readable manifest lists: `manifest-list` is only a path, and the change is dropped without the catalog reading it. Against `docker run -p 8181:8181 apache/iceberg-rest-fixture:latest` (built from `b0df3ca`), 2 runs of 10 rounds per mode: ``` transactions at once: statuses [(204, 204)], lost in 10 of 10 rounds table endpoint: statuses [(200, 409)], lost in 0 of 10 rounds one after the other: statuses [(204, 409)], lost in 0 of 10 rounds ``` The catalog logged `Nothing to commit.` exactly once per lost change, 20 times across the 20 transaction rounds. <details><summary>repro.py</summary> ```python #!/usr/bin/env python3 """Two commits guarded on the same existing snapshot. Python stdlib only: repro.py [catalog_url] [rounds]""" import json, random, sys, threading, urllib.error, urllib.request URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8181" ROUNDS = int(sys.argv[2]) if len(sys.argv) > 2 else 10 NS, TABLE = "repro_18758", "t" TABLES, TABLE_PATH, COMMIT = f"/v1/namespaces/{NS}/tables", f"/v1/namespaces/{NS}/tables/{TABLE}", "/v1/transactions/commit" def call(method, path, body=None): data = json.dumps(body).encode() if body is not None else None req = urllib.request.Request(URL + path, data=data, method=method, headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req) as r: return r.status, json.loads(r.read() or b"null") except urllib.error.HTTPError as e: return e.code, json.loads(e.read() or b"null") def add_snapshot(meta, parent): """Adds a snapshot to main, requiring main to still be at parent. The manifest list is only a path.""" sid = random.randrange(1, 2**62) snapshot = {"snapshot-id": sid, "timestamp-ms": meta["last-updated-ms"] + 1, "schema-id": 0, "sequence-number": meta["last-sequence-number"] + 1, "summary": {"operation": "append"}, "manifest-list": f"{meta['location']}/metadata/snap-{sid}.avro"} if parent: snapshot["parent-snapshot-id"] = parent return sid, {"requirements": [{"type": "assert-ref-snapshot-id", "ref": "main", "snapshot-id": parent}], "updates": [{"action": "add-snapshot", "snapshot": snapshot}, {"action": "set-snapshot-ref", "ref-name": "main", "type": "branch", "snapshot-id": sid}]} def transaction(change): return {"table-changes": [{"identifier": {"namespace": [NS], "name": TABLE}, **change}]} def run_round(mode): call("DELETE", TABLE_PATH) meta = call("POST", TABLES, {"name": TABLE, "properties": {"format-version": "2"}, "schema": { "type": "struct", "fields": [{"id": 1, "name": "id", "type": "string", "required": False}]}})[1]["metadata"] seed, change = add_snapshot(meta, None) assert call("POST", COMMIT, transaction(change))[0] == 204 meta = call("GET", TABLE_PATH)[1]["metadata"] ids, changes = zip(add_snapshot(meta, seed), add_snapshot(meta, seed)) path, bodies = (TABLE_PATH, changes) if mode == "table endpoint" else (COMMIT, [transaction(c) for c in changes]) statuses = [None, None] gate = threading.Barrier(2) def send(i): if mode != "one after the other": gate.wait() statuses[i] = call("POST", path, bodies[i])[0] if mode == "one after the other": send(0), send(1) else: threads = [threading.Thread(target=send, args=(i,)) for i in range(2)] [t.start() for t in threads], [t.join() for t in threads] present = {s["snapshot-id"] for s in call("GET", TABLE_PATH)[1]["metadata"]["snapshots"]} lost = any(st < 300 and sid not in present for sid, st in zip(ids, statuses)) return tuple(sorted(statuses)), lost call("POST", "/v1/namespaces", {"namespace": [NS]}) for mode in ("transactions at once", "table endpoint", "one after the other"): results = [run_round(mode) for _ in range(ROUNDS)] print(f"{mode}: statuses {sorted({r[0] for r in results})}, lost in {sum(r[1] for r in results)} of {ROUNDS} rounds") call("DELETE", TABLE_PATH) call("DELETE", f"/v1/namespaces/{NS}") ``` </details> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
