imbajin commented on code in PR #391:
URL: https://github.com/apache/hugegraph-ai/pull/391#discussion_r4505264340
##########
ontogeny/server/ontogeny/action/outbox.py:
##########
@@ -155,31 +157,82 @@ async def _deliver_sse(self, event: OutboxRow) -> bool:
)
return True
- async def _deliver_derivation(self, event: OutboxRow) -> bool:
+ async def _deliver_derivation(self, event: OutboxRow, session) -> bool:
if self.derivation_hook is None:
return True
- return bool(await self.derivation_hook(event))
+ return bool(await self.derivation_hook(event, session))
# --------------------------------------------------------------- driving
async def dispatch_pending(self, *, limit: int = 100) -> dict[str, int]:
- consumers: dict[str, Consumer] = {
- "webhook": self._deliver_webhook,
- "projection": self._deliver_projection,
- "derivation": self._deliver_derivation,
- "sse": self._deliver_sse,
- }
+ # The background loop and the admin endpoint share cursors and must
+ # never dispatch the same batch concurrently in this process.
+ async with self._dispatch_lock:
+ return await self._dispatch_pending(limit=limit)
+
+ async def replay(self, consumer: str, *, from_id: int, through_id: int) ->
int:
+ """Explicitly retry acknowledged events in a bounded inclusive range.
+
+ Replaying a webhook may deliver it again. This also permits operators
+ to recover old pre-migration gaps that legacy cursors cannot
distinguish
+ from completed deliveries.
+ """
+ if consumer not in {"webhook", "projection", "derivation", "sse"} or
from_id < 1 or through_id < from_id:
+ raise ValueError("invalid outbox consumer or replay range")
+ async with self._dispatch_lock, self.session_factory() as session:
+ cursor = await session.get(OutboxConsumerRow, consumer)
+ if cursor is None or not cursor.receipts_initialized:
+ raise ValueError("consumer receipts have not been initialized;
dispatch first")
+ result = await session.execute(
+ delete(OutboxReceiptRow).where(
+ OutboxReceiptRow.consumer == consumer,
+ OutboxReceiptRow.event_id >= from_id,
+ OutboxReceiptRow.event_id <= through_id,
+ )
+ )
+ await session.commit()
+ return int(result.rowcount)
+
+ async def _dispatch_pending(self, *, limit: int = 100) -> dict[str, int]:
stats: dict[str, int] = {}
async with self.session_factory() as session:
+ consumers: dict[str, Consumer] = {
+ "webhook": self._deliver_webhook,
+ "projection": self._deliver_projection,
+ "derivation": lambda event: self._deliver_derivation(event,
session),
+ "sse": self._deliver_sse,
+ }
for name, deliver in consumers.items():
cursor = (
await
session.execute(select(OutboxConsumerRow).where(OutboxConsumerRow.consumer ==
name))
).scalar_one_or_none()
- last_id = cursor.last_id if cursor else 0
+ if cursor is None:
+ cursor = OutboxConsumerRow(consumer=name, last_id=0,
receipts_initialized=True)
+ session.add(cursor)
+ await session.flush()
+ elif not cursor.receipts_initialized:
+ # Snapshot the *visible* legacy history once. Do not replay
+ # old webhooks implicitly; a transaction that commits after
+ # this statement has no receipt even if its ID is lower.
+ await session.execute(
+ insert(OutboxReceiptRow).from_select(
+ ["consumer", "event_id"],
+ select(literal(name),
OutboxRow.id).where(OutboxRow.id <= cursor.last_id),
+ )
+ )
+ cursor.receipts_initialized = True
+ await session.flush()
rows = (
(
await session.execute(
- select(OutboxRow).where(OutboxRow.id >
last_id).order_by(OutboxRow.id).limit(limit)
+ select(OutboxRow)
+ .where(
+ ~exists().where(
Review Comment:
Fixed in 066a796247c5fd8afaa31f3178ad64a42d829ec2 with indexed per-consumer
pending flags rather than an ID floor. Receipt and flag updates, including
replay, are transactional; startup reconciles receipts in both directions to
support an upgrade after rollback. Tests against SQLite and real PostgreSQL 16
cover late lower-ID commits, upgrade/rollback, replay, failures, and index
plans (12 passed). Upgrade requires stopping old workers first and running only
one service process per database; initial reconciliation scans history once
after startup.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]