imbajin commented on code in PR #391:
URL: https://github.com/apache/hugegraph-ai/pull/391#discussion_r4505264340


##########
ontogeny/server/ontogeny/action/outbox.py:
##########
@@ -155,31 +157,82 @@ async def _deliver_sse(self, event: OutboxRow) -> bool:
         )
         return True
 
-    async def _deliver_derivation(self, event: OutboxRow) -> bool:
+    async def _deliver_derivation(self, event: OutboxRow, session) -> bool:
         if self.derivation_hook is None:
             return True
-        return bool(await self.derivation_hook(event))
+        return bool(await self.derivation_hook(event, session))
 
     # --------------------------------------------------------------- driving
 
     async def dispatch_pending(self, *, limit: int = 100) -> dict[str, int]:
-        consumers: dict[str, Consumer] = {
-            "webhook": self._deliver_webhook,
-            "projection": self._deliver_projection,
-            "derivation": self._deliver_derivation,
-            "sse": self._deliver_sse,
-        }
+        # The background loop and the admin endpoint share cursors and must
+        # never dispatch the same batch concurrently in this process.
+        async with self._dispatch_lock:
+            return await self._dispatch_pending(limit=limit)
+
+    async def replay(self, consumer: str, *, from_id: int, through_id: int) -> 
int:
+        """Explicitly retry acknowledged events in a bounded inclusive range.
+
+        Replaying a webhook may deliver it again. This also permits operators
+        to recover old pre-migration gaps that legacy cursors cannot 
distinguish
+        from completed deliveries.
+        """
+        if consumer not in {"webhook", "projection", "derivation", "sse"} or 
from_id < 1 or through_id < from_id:
+            raise ValueError("invalid outbox consumer or replay range")
+        async with self._dispatch_lock, self.session_factory() as session:
+            cursor = await session.get(OutboxConsumerRow, consumer)
+            if cursor is None or not cursor.receipts_initialized:
+                raise ValueError("consumer receipts have not been initialized; 
dispatch first")
+            result = await session.execute(
+                delete(OutboxReceiptRow).where(
+                    OutboxReceiptRow.consumer == consumer,
+                    OutboxReceiptRow.event_id >= from_id,
+                    OutboxReceiptRow.event_id <= through_id,
+                )
+            )
+            await session.commit()
+            return int(result.rowcount)
+
+    async def _dispatch_pending(self, *, limit: int = 100) -> dict[str, int]:
         stats: dict[str, int] = {}
         async with self.session_factory() as session:
+            consumers: dict[str, Consumer] = {
+                "webhook": self._deliver_webhook,
+                "projection": self._deliver_projection,
+                "derivation": lambda event: self._deliver_derivation(event, 
session),
+                "sse": self._deliver_sse,
+            }
             for name, deliver in consumers.items():
                 cursor = (
                     await 
session.execute(select(OutboxConsumerRow).where(OutboxConsumerRow.consumer == 
name))
                 ).scalar_one_or_none()
-                last_id = cursor.last_id if cursor else 0
+                if cursor is None:
+                    cursor = OutboxConsumerRow(consumer=name, last_id=0, 
receipts_initialized=True)
+                    session.add(cursor)
+                    await session.flush()
+                elif not cursor.receipts_initialized:
+                    # Snapshot the *visible* legacy history once. Do not replay
+                    # old webhooks implicitly; a transaction that commits after
+                    # this statement has no receipt even if its ID is lower.
+                    await session.execute(
+                        insert(OutboxReceiptRow).from_select(
+                            ["consumer", "event_id"],
+                            select(literal(name), 
OutboxRow.id).where(OutboxRow.id <= cursor.last_id),
+                        )
+                    )
+                    cursor.receipts_initialized = True
+                    await session.flush()
                 rows = (
                     (
                         await session.execute(
-                            select(OutboxRow).where(OutboxRow.id > 
last_id).order_by(OutboxRow.id).limit(limit)
+                            select(OutboxRow)
+                            .where(
+                                ~exists().where(

Review Comment:
   Fixed in 066a796247c5fd8afaa31f3178ad64a42d829ec2 with indexed per-consumer 
pending flags rather than an ID floor. Receipt and flag updates, including 
replay, are transactional; startup reconciles receipts in both directions to 
support an upgrade after rollback. Tests against SQLite and real PostgreSQL 16 
cover late lower-ID commits, upgrade/rollback, replay, failures, and index 
plans (12 passed). Upgrade requires stopping old workers first and running only 
one service process per database; initial reconciliation scans history once 
after startup.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to