This is an automated email from the ASF dual-hosted git repository. wu-sheng pushed a commit to branch fix/wait-for-oap-readiness-before-bootseed in repository https://gitbox.apache.org/repos/asf/skywalking-horizon-ui.git
commit dead36e37f6e84338afab682a546548f670bed3f Author: Wu Sheng <[email protected]> AuthorDate: Tue May 26 22:58:45 2026 +0800 fix(bff): wait for OAP admin readiness before bootSeed; warn-per-fail; no periodic polling Previously `bootSeed()` was fire-and-forget on the very first listen callback — a single `client.list()` attempt, and if OAP's admin port wasn't bound yet (compose / k8s rollout where OAP starts after the BFF binds, or the admin module is wired lazily) the function logged one warn and returned `unreachable: true` with no retry. Every bundled template stayed un-pushed for that BFF lifetime; the only recovery was an operator restart of the BFF after OAP came up. Adds `waitForOapAdminReady(deps, signal)` in apps/bff/src/logic/templates/sync.ts that uses `client.list()` as a readiness check, backs off 1s → 2s → 4s → … capped at 60s, and logs one warn per failed attempt. Each warn carries `attempt` and `nextRetryInMs` so an operator grepping logs sees the wait progress without ambiguity. On the first successful list it returns and the loop terminates — there's no steady-state polling: the admin port is only touched again when an operator triggers a UI sync or an admin-edit action. Wired into server.ts as a two-step boot: 1. `await waitForOapAdminReady(deps, bootSeedAbort.signal)` 2. `await bootSeed(deps)` (unchanged — still absent-only via `seedMissing`, so a fully-seeded OAP from a prior boot is a no-op write-wise). The wait is cancellable via an AbortController bound to shutdown so the BFF process can exit cleanly when signal-killed mid-backoff (k8s rolling restart, ctrl-c) instead of blocking the close handler. If `client.list()` flaps between the readiness probe and the subsequent `bootSeed` (very narrow window) we still log a clear warn on the partial seed; the absent-only semantics protect operator edits in that case. Tested: 80 / 80 BFF unit tests pass. --- apps/bff/src/logic/templates/sync.ts | 68 ++++++++++++++++++++++++++++++++++++ apps/bff/src/server.ts | 52 ++++++++++++++++++--------- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/apps/bff/src/logic/templates/sync.ts b/apps/bff/src/logic/templates/sync.ts index 9002a05..752a5e5 100644 --- a/apps/bff/src/logic/templates/sync.ts +++ b/apps/bff/src/logic/templates/sync.ts @@ -178,6 +178,74 @@ export async function bootSeed(deps: SyncDeps): Promise<SyncStatus> { return status; } +/** + * Block until OAP admin is reachable, then return. Uses `client.list()` + * as the readiness check (same call bootSeed itself runs first, so a + * success here proves the seed can proceed). Backs off 1s → 2s → 4s + * → … capped at 60s so a slow OAP startup doesn't pin the loop on the + * fast end; each failed attempt emits one warn line so an operator + * grepping logs sees the wait progress. + * + * Why we wait here instead of letting `bootSeed` fall through on the + * first failure: when OAP and Horizon start in the same compose / k8s + * rollout, OAP's admin module often binds after the BFF process is + * already up. The old behaviour was a single attempt → warn → no + * retry → no templates ever pushed for that BFF lifetime. This + * function fixes the race without introducing any steady-state + * polling: once `list()` succeeds we return, the seed runs once, + * we never touch the admin port from here again until the operator + * triggers an admin action. + * + * `signal` lets the caller (server boot) cancel the wait on shutdown + * so the BFF process can exit cleanly even mid-backoff. + */ +const READINESS_INITIAL_DELAY_MS = 1000; +const READINESS_MAX_DELAY_MS = 60_000; + +export async function waitForOapAdminReady( + deps: SyncDeps, + signal?: AbortSignal, +): Promise<void> { + let delay = READINESS_INITIAL_DELAY_MS; + let attempt = 0; + for (;;) { + if (signal?.aborted) return; + attempt++; + try { + await deps.client.list(); + if (attempt > 1) { + deps.logger.info({ attempt }, 'OAP admin reachable — proceeding with boot seed'); + } + return; + } catch (err) { + deps.logger.warn( + { err: errMsg(err), attempt, nextRetryInMs: delay }, + 'OAP admin unreachable — retrying readiness check', + ); + } + await sleepCancelable(delay, signal); + delay = Math.min(delay * 2, READINESS_MAX_DELAY_MS); + } +} + +function sleepCancelable(ms: number, signal?: AbortSignal): Promise<void> { + return new Promise((resolve) => { + if (signal?.aborted) { + resolve(); + return; + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + const onAbort = (): void => { + clearTimeout(timer); + resolve(); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + /** Force the next caller of `getSyncStatus` to re-list OAP. No I/O here. */ export function resync(): void { invalidateSyncCache(); diff --git a/apps/bff/src/server.ts b/apps/bff/src/server.ts index 78c20a3..8c0fed4 100644 --- a/apps/bff/src/server.ts +++ b/apps/bff/src/server.ts @@ -58,7 +58,7 @@ import { registerOverviewRoutes } from './http/config/overview.js'; import { registerConfigBundleRoute } from './http/config/bundle.js'; import { registerTemplateSyncAdminRoutes } from './http/admin/template-sync.js'; import { buildOapClients } from './client/index.js'; -import { bootSeed } from './logic/templates/sync.js'; +import { bootSeed, waitForOapAdminReady } from './logic/templates/sync.js'; import { iterateBundledTemplates, iterateBundledOverlays } from './logic/templates/aggregator.js'; // Admin (operational tools) import { registerDslCatalogRoutes } from './http/admin/dsl/catalog.js'; @@ -251,34 +251,49 @@ app.get('/api/health', async () => ({ })); const { host, port } = source.current.server; +// AbortController bound to shutdown so the readiness wait can exit +// cleanly when the BFF is signal-killed mid-backoff (k8s rolling +// restart, ctrl-c, etc.) instead of holding the process open. +const bootSeedAbort = new AbortController(); app.listen({ host, port }).then( () => { logger.info(`BFF listening on http://${host}:${port}`); - // Fire-and-forget the boot-time OAP template seed: list OAP, POST any - // bundled template that's missing on the OAP side. This is the ONLY - // path that writes implicitly to OAP — runtime sync is read-only. - // Failures are non-fatal: the BFF stays up, the UI falls back to - // bundled templates and shows the read-only banner. - void bootSeed({ - client: buildOapClients(source.current).uiTemplate(), - bundled: () => iterateBundledTemplates(), - bundledOverlays: () => iterateBundledOverlays(), - logger, - }) - .then((status) => { + // Wait for OAP admin readiness, then run the boot-time template + // seed ONCE. Two-phase so we don't lose the seed when OAP is still + // starting up alongside the BFF (compose / k8s rollout, slow OAP + // module wiring). The wait is a backoff loop that warn-logs each + // failed ping; once `client.list()` succeeds we run bootSeed and + // never touch the admin port from here again until an operator + // admin action triggers a fresh sync. The seed itself is + // absent-only (`seedMissing` skips templates already present), so + // a successful previous boot leaves nothing to re-push. + void (async (): Promise<void> => { + const deps = { + client: buildOapClients(source.current).uiTemplate(), + bundled: () => iterateBundledTemplates(), + bundledOverlays: () => iterateBundledOverlays(), + logger, + }; + try { + await waitForOapAdminReady(deps, bootSeedAbort.signal); + if (bootSeedAbort.signal.aborted) return; + const status = await bootSeed(deps); if (status.unreachable) { + // Admin port flapped between readiness probe and the actual + // `bootSeed.list()` — very narrow window, but log it the + // same way so the operator notices. logger.warn( { lastSuccessfulSyncAt: status.lastSuccessfulSyncAt }, - 'OAP UI-template boot seed: admin unreachable, rendering bundled (admin pages will be read-only until OAP comes back)', + 'OAP UI-template boot seed: admin became unreachable between readiness and seed', ); } else { const counts = countByStatus(status.rows); logger.info(counts, 'OAP UI-template boot seed: complete'); } - }) - .catch((err) => { + } catch (err) { logger.error({ err }, 'OAP UI-template boot seed: unexpected error'); - }); + } + })(); }, (err) => { logger.fatal({ err }, 'failed to start BFF'); @@ -294,6 +309,9 @@ function countByStatus(rows: Array<{ status: string }>): Record<string, number> async function shutdown(signal: string) { logger.info({ signal }, 'shutting down'); + // Cancel an in-flight OAP-admin readiness wait so the boot-seed + // promise resolves quickly instead of blocking shutdown. + bootSeedAbort.abort(); await app.close(); await sessions.close(); await audit.close();
