AndriiSushko commented on issue #18758:
URL: https://github.com/apache/iceberg/issues/18758#issuecomment-6694781539

   A smaller repro for the case where `main` already has a snapshot, using only 
the Python stdlib. Both commits require `main` to still be at the same existing 
snapshot, so at most one should succeed. It doesn't need readable manifest 
lists: `manifest-list` is only a path, and the change is dropped without the 
catalog reading it.
   
   Against `docker run -p 8181:8181 apache/iceberg-rest-fixture:latest` (built 
from `b0df3ca`), 2 runs of 10 rounds per mode:
   
   ```
   transactions at once: statuses [(204, 204)], lost in 10 of 10 rounds
   table endpoint: statuses [(200, 409)], lost in 0 of 10 rounds
   one after the other: statuses [(204, 409)], lost in 0 of 10 rounds
   ```
   
   The catalog logged `Nothing to commit.` exactly once per lost change, 20 
times across the 20 transaction rounds.
   
   <details><summary>repro.py</summary>
   
   ```python
   #!/usr/bin/env python3
   """Two commits guarded on the same existing snapshot. Python stdlib only: 
repro.py [catalog_url] [rounds]"""
   import json, random, sys, threading, urllib.error, urllib.request
   
   URL = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8181";
   ROUNDS = int(sys.argv[2]) if len(sys.argv) > 2 else 10
   NS, TABLE = "repro_18758", "t"
   TABLES, TABLE_PATH, COMMIT = f"/v1/namespaces/{NS}/tables", 
f"/v1/namespaces/{NS}/tables/{TABLE}", "/v1/transactions/commit"
   
   
   def call(method, path, body=None):
       data = json.dumps(body).encode() if body is not None else None
       req = urllib.request.Request(URL + path, data=data, method=method, 
headers={"Content-Type": "application/json"})
       try:
           with urllib.request.urlopen(req) as r:
               return r.status, json.loads(r.read() or b"null")
       except urllib.error.HTTPError as e:
           return e.code, json.loads(e.read() or b"null")
   
   
   def add_snapshot(meta, parent):
       """Adds a snapshot to main, requiring main to still be at parent. The 
manifest list is only a path."""
       sid = random.randrange(1, 2**62)
       snapshot = {"snapshot-id": sid, "timestamp-ms": meta["last-updated-ms"] 
+ 1, "schema-id": 0,
                   "sequence-number": meta["last-sequence-number"] + 1, 
"summary": {"operation": "append"},
                   "manifest-list": 
f"{meta['location']}/metadata/snap-{sid}.avro"}
       if parent:
           snapshot["parent-snapshot-id"] = parent
       return sid, {"requirements": [{"type": "assert-ref-snapshot-id", "ref": 
"main", "snapshot-id": parent}],
                    "updates": [{"action": "add-snapshot", "snapshot": 
snapshot},
                                {"action": "set-snapshot-ref", "ref-name": 
"main", "type": "branch", "snapshot-id": sid}]}
   
   
   def transaction(change):
       return {"table-changes": [{"identifier": {"namespace": [NS], "name": 
TABLE}, **change}]}
   
   
   def run_round(mode):
       call("DELETE", TABLE_PATH)
       meta = call("POST", TABLES, {"name": TABLE, "properties": 
{"format-version": "2"}, "schema": {
           "type": "struct", "fields": [{"id": 1, "name": "id", "type": 
"string", "required": False}]}})[1]["metadata"]
       seed, change = add_snapshot(meta, None)
       assert call("POST", COMMIT, transaction(change))[0] == 204
       meta = call("GET", TABLE_PATH)[1]["metadata"]
   
       ids, changes = zip(add_snapshot(meta, seed), add_snapshot(meta, seed))
       path, bodies = (TABLE_PATH, changes) if mode == "table endpoint" else 
(COMMIT, [transaction(c) for c in changes])
       statuses = [None, None]
       gate = threading.Barrier(2)
   
       def send(i):
           if mode != "one after the other":
               gate.wait()
           statuses[i] = call("POST", path, bodies[i])[0]
   
       if mode == "one after the other":
           send(0), send(1)
       else:
           threads = [threading.Thread(target=send, args=(i,)) for i in 
range(2)]
           [t.start() for t in threads], [t.join() for t in threads]
   
       present = {s["snapshot-id"] for s in call("GET", 
TABLE_PATH)[1]["metadata"]["snapshots"]}
       lost = any(st < 300 and sid not in present for sid, st in zip(ids, 
statuses))
       return tuple(sorted(statuses)), lost
   
   
   call("POST", "/v1/namespaces", {"namespace": [NS]})
   for mode in ("transactions at once", "table endpoint", "one after the 
other"):
       results = [run_round(mode) for _ in range(ROUNDS)]
       print(f"{mode}: statuses {sorted({r[0] for r in results})}, lost in 
{sum(r[1] for r in results)} of {ROUNDS} rounds")
   call("DELETE", TABLE_PATH)
   call("DELETE", f"/v1/namespaces/{NS}")
   ```
   
   </details>


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to