diff --git a/runner/docs/adr/0041-observability-stack.md b/runner/docs/adr/0041-observability-stack.md index 4972ca4121..ba42c0f825 100644 --- a/runner/docs/adr/0041-observability-stack.md +++ b/runner/docs/adr/0041-observability-stack.md @@ -592,20 +592,20 @@ state would not survive a sleep. |---|---|---| | Uncaught error, new issue, regression | Sentry | seconds | | Spend thresholds (200/500/800) | `reconcile.ts` `captureMessage` to Sentry, as today | nightly | -| `at_capacity` rate, 5xx rate from `api.request`, preview-ready rate per tier, session start p95, embed error rate per demo id, compile-error rate per `ht_major` day over day, snapshot-build failed rate per framework, a wake or reload that ran without the `ae.internal` interception (`o11y.ae_degraded`), LiteLLM error rate (`chat.answer` + `theme.ai`), inbox backlog age, a `rejected` inbox key, the o11y spend cap | o11y worker `*/10` cron over Analytics Engine and `InboxWriter` → Slack | minutes | +| `at_capacity` rate, 5xx rate from `api.request`, preview-ready rate per tier, Tier-2 liveness (no `ready` preview in 6 h), Tier-2 slow previews (ready p95), session start p95, embed error rate per demo id, compile-error rate per `ht_major` day over day, snapshot-build failed rate per framework, a wake or reload that ran without the `ae.internal` interception (`o11y.ae_degraded`), LiteLLM error rate (`chat.answer` + `theme.ai`), inbox backlog age, a `rejected` inbox key, the o11y spend cap | o11y worker `*/10` cron over Analytics Engine and `InboxWriter` → Slack | minutes | | New handled-error fingerprint | the exact first-seen registry in `InboxWriter` (not sampled data), excluding `surface = demo-runtime`, whose keystroke ladders are authored-code output. **Not a page**: the `*/10` cron writes one `o11y.new_fingerprint` point per new fingerprint and the Observability self dashboard lists them in a table; nothing goes to Slack | minutes, dashboard only | | The o11y stack itself stale (no cron tick or ingest for 30 min) | the API worker's `*/5` cron reads the o11y heartbeat over a service binding and sends `captureMessage` to Sentry | minutes | Alert state (firing, resolved, last notified) lives in Durable Object storage: a rule notifies once when it fires and once when it resolves, never on every tick. Thresholds are starting values, tuned after launch: preview-ready below 97 % (Tier-1) or 95 % (Tier-2) -over 1 h, evaluated per tier only with at least 10 non-abandoned previews in that hour; +over 6 h, evaluated per tier only with at least 10 non-abandoned previews and at least 3 failed (error or timeout) in that window (`preview.ready_ms` carries no demo or session id, so the failures cannot be required to span distinct sources); Tier-2 liveness fires when no Tier-2 preview reached `ready` in 6 h with at least 5 non-abandoned attempts; Tier-2 slow previews fire when the `ready` time-to-ready p95 is above 180 s over 6 h with at least 10 `ready` previews (a starting value above today's worst per-framework p95 of 167 s); session start p95 above 20 s, evaluated only with at least 20 `ready` starts in the hour (below either floor the rule is not firing, so a firing alert resolves through the normal path); `at_capacity` above 5/h; 5xx above 1 % over -15 min, evaluated only with at least 100 requests after the exclusions (at 1 % one error exceeds the threshold only below 100 requests, so a single 500 cannot page); +1 h, evaluated only with at least 100 requests after the exclusions (at 1 % one error exceeds the threshold only below 100 requests, so a single 500 cannot page); LiteLLM errors above 5 % over 1 h, evaluated only with at least 20 non-denied calls (same reasoning at 5 %); compile errors on one `ht_major` doubling day over -day; any `o11y.ae_degraded` point in 2 h (written only at a wake or reload, so a short window would resolve before anyone read the page); snapshot builds failing above 50 % per framework over 30 min with at least 10 -failed; an embed above 20 % errors with more than 50 views in 24 h; backlog older +day, with at least 20 today; any `o11y.ae_degraded` point in 2 h (written only at a wake or reload, so a short window would resolve before anyone read the page); snapshot builds failing above 50 % per framework over 30 min with at least 10 +failed; an embed above 20 % errors with at least 50 views in 7 days; backlog older than 2 h. ### G. Cost is its own number, with its own cap diff --git a/runner/pipeline/o11y-alerts.test.mjs b/runner/pipeline/o11y-alerts.test.mjs index 89f3dd7119..291c9c6d4b 100644 --- a/runner/pipeline/o11y-alerts.test.mjs +++ b/runner/pipeline/o11y-alerts.test.mjs @@ -22,6 +22,13 @@ const { FIVE_XX_MIN_REQUESTS, previewReadyRateRule, PREVIEW_READY_MIN_SAMPLES, + PREVIEW_READY_MIN_FAILURES, + tier2LivenessRule, + TIER2_LIVENESS_MIN_ATTEMPTS, + previewSlowRule, + PREVIEW_SLOW_MIN_SAMPLES, + EMBED_ERROR_MIN_VIEWS, + COMPILE_ERROR_DOUBLING_FLOOR, sessionStartP95Rule, SESSION_START_P95_MIN_SAMPLES, embedErrorRateRule, @@ -938,7 +945,7 @@ test("fiveXxRateRule: excludes deliberate refusals via exact counts, and still-b const fake = makeFakeAeQuery(rows); const result = await fiveXxRateRule({}, fake.queryFn); assert.equal(result.firing, false, "fully-matched deliberate degradations must not push the rate over threshold"); - assert.match(result.detail, /^0\.10% 5xx over the last 15 min \(1\/1000/); + assert.match(result.detail, /^0\.10% 5xx over the last hour \(1\/1000/); assert.ok(fake.calls.some((sql) => sql.includes(AE_COLUMNS.route_class)), "SQL must filter on the route_class column"); }); @@ -954,7 +961,7 @@ test("fiveXxRateRule: a genuine api/session 500 with NO matching session.start r const fake = makeFakeAeQuery(rows); const result = await fiveXxRateRule({}, fake.queryFn); assert.equal(result.firing, true, "a genuine api/session fault must still be able to fire the rule"); - assert.match(result.detail, /^1\.48% 5xx over the last 15 min \(15\/1014/); + assert.match(result.detail, /^1\.48% 5xx over the last hour \(15\/1014/); }); test("fiveXxRateRule: a real problem on a normal route still fires even alongside a full load of excluded degradation noise", async () => { @@ -972,7 +979,7 @@ test("fiveXxRateRule: a real problem on a normal route still fires even alongsid const fake = makeFakeAeQuery(rows); const result = await fiveXxRateRule({}, fake.queryFn); assert.equal(result.firing, true, "the excluded noise must not mask a real problem on a normal route"); - assert.match(result.detail, /^5\.00% 5xx over the last 15 min \(5\/100/); + assert.match(result.detail, /^5\.00% 5xx over the last hour \(5\/100/); }); test("fiveXxRateRule: a still-building 503 on d/:id and embed/:id is excluded, but a build-failed 500 there counts (DEV-3143)", async () => { @@ -994,7 +1001,7 @@ test("fiveXxRateRule: a still-building 503 on d/:id and embed/:id is excluded, b ]); const fired = await fiveXxRateRule({}, broken.queryFn); assert.equal(fired.firing, true, "a build-failed 500 on d/:id / embed/:id must not be hidden"); - assert.match(fired.detail, /^2\.91% 5xx over the last 15 min \(30\/1030/); + assert.match(fired.detail, /^2\.91% 5xx over the last hour \(30\/1030/); assert.ok(broken.calls.some((sql) => sql.includes(AE_COLUMNS.reason)), "SQL must filter on the reason column carrying the status"); }); @@ -1032,12 +1039,29 @@ test("fiveXxRateRule sample floor: below 100 requests even 100% 5xx does not fir const atFloor = await fiveXxRateRule({}, makeFakeAeQuery(apiRows(98, 2)).queryFn); assert.equal(atFloor.firing, true, atFloor.detail); - assert.match(atFloor.detail, /^2\.00% 5xx over the last 15 min \(2\/100,/); + assert.match(atFloor.detail, /^2\.00% 5xx over the last hour \(2\/100,/); const healthy = await fiveXxRateRule({}, makeFakeAeQuery(apiRows(500, 0)).queryFn); assert.equal(healthy.firing, false); }); +test("fiveXxRateRule: reads a 1 h window, so traffic spread over 45 min still counts and traffic older than 1 h does not", async () => { + const spread = makeFakeAeQuery([ + { metric: "api.request", route_class: "api/demos", outcome: "2xx", count: 98, ageMs: 45 * 60_000 }, + { metric: "api.request", route_class: "api/demos", outcome: "5xx", count: 2, ageMs: 45 * 60_000 }, + ]); + const result = await fiveXxRateRule({}, spread.queryFn); + assert.equal(result.firing, true, result.detail); + assert.ok(spread.calls.every((sql) => sql.includes("INTERVAL '3600' SECOND")), "1-hour window"); + + const stale = makeFakeAeQuery([ + { metric: "api.request", route_class: "api/demos", outcome: "2xx", count: 98, ageMs: 70 * 60_000 }, + { metric: "api.request", route_class: "api/demos", outcome: "5xx", count: 2, ageMs: 70 * 60_000 }, + ]); + const staleResult = await fiveXxRateRule({}, stale.queryFn); + assert.equal(staleResult.firing, false, "70 minutes ago is outside the window"); +}); + test("fiveXxRateRule sample floor: the floor counts the ADJUSTED total, not the raw one", async () => { const rows = [ ...apiRows(0, 5), @@ -1115,7 +1139,7 @@ test("previewReadyRateRule: a real below-threshold tier still fires correctly al ]); const result = await previewReadyRateRule({}, fake.queryFn); assert.equal(result.firing, true, result.detail); - assert.match(result.detail, /tier 1: 80\.0% ready \(80\/100,/, "abandoned must not appear in the reported denominator"); + assert.match(result.detail, /tier 1: 80\.0% ready over the last 6 h \(80\/100,/, "abandoned must not appear in the reported denominator"); }); function previewRows(tier, ready, error, abandoned = 0) { @@ -1136,12 +1160,40 @@ test("previewReadyRateRule sample floor: 9 non-abandoned previews at 44% ready d const atFloor = await previewReadyRateRule({}, makeFakeAeQuery(previewRows("1", 5, 5)).queryFn); assert.equal(atFloor.firing, true, atFloor.detail); - assert.match(atFloor.detail, /tier 1: 50\.0% ready \(5\/10,/); + assert.match(atFloor.detail, /tier 1: 50\.0% ready over the last 6 h \(5\/10,/); const healthyAtFloor = await previewReadyRateRule({}, makeFakeAeQuery(previewRows("1", 10, 0)).queryFn); assert.equal(healthyAtFloor.firing, false); }); +test("previewReadyRateRule failure floor: 97 ready + 2 failed does not fire even when the rate is under threshold, 3 failed does", async () => { + assert.equal(PREVIEW_READY_MIN_FAILURES, 3); + const twoFailed = await previewReadyRateRule({}, makeFakeAeQuery(previewRows("2", 8, 2)).queryFn); + assert.equal(twoFailed.firing, false, `8/10 = 80% ready but only 2 failures: ${twoFailed.detail}`); + + const threeFailed = await previewReadyRateRule({}, makeFakeAeQuery(previewRows("2", 7, 3)).queryFn); + assert.equal(threeFailed.firing, true, threeFailed.detail); + assert.match(threeFailed.detail, /tier 2: 70\.0% ready over the last 6 h \(7\/10, 3 failed,/); + + const timeouts = await previewReadyRateRule({}, makeFakeAeQuery([ + ...previewRows("2", 7, 1), + { metric: "preview.ready_ms", tier: "2", outcome: "timeout", count: 2 }, + ]).queryFn); + assert.equal(timeouts.firing, true, `timeouts count as failures: ${timeouts.detail}`); +}); + +test("previewReadyRateRule: reads a 6 h window (a failure burst 5 h ago still counts, one 7 h ago does not)", async () => { + const HOUR = 3_600_000; + const recent = makeFakeAeQuery(previewRows("2", 7, 3).map((r) => ({ ...r, ageMs: 5 * HOUR }))); + const recentResult = await previewReadyRateRule({}, recent.queryFn); + assert.equal(recentResult.firing, true, recentResult.detail); + assert.ok(recent.calls.every((sql) => sql.includes("INTERVAL '21600' SECOND")), "6-hour window"); + + const stale = makeFakeAeQuery(previewRows("2", 7, 3).map((r) => ({ ...r, ageMs: 7 * HOUR }))); + const staleResult = await previewReadyRateRule({}, stale.queryFn); + assert.equal(staleResult.firing, false, "7 h ago is outside the window"); +}); + test("previewReadyRateRule sample floor: abandoned previews never count toward the floor", async () => { const result = await previewReadyRateRule({}, makeFakeAeQuery(previewRows("1", 2, 3, 1000)).queryFn); assert.equal(result.firing, false, "5 real previews plus 1000 abandoned is still below the floor"); @@ -1162,6 +1214,81 @@ test("previewReadyRateRule sample floor: a firing alert resolves once the tier d assert.match(slackCalls[1], /resolved/); }); +function tier2Rows(ready, error, abandoned = 0, tier = "2") { + return previewRows(tier, ready, error, abandoned); +} + +test("tier2LivenessRule: zero ready with at least 5 attempts fires; below 5 attempts, any ready, or Tier-1 only does not", async () => { + assert.equal(TIER2_LIVENESS_MIN_ATTEMPTS, 5); + const dead = makeFakeAeQuery(tier2Rows(0, 5)); + const deadResult = await tier2LivenessRule({}, dead.queryFn); + assert.equal(deadResult.firing, true, deadResult.detail); + assert.equal(deadResult.rule, "tier2-liveness"); + assert.match(deadResult.detail, /5 attempted/); + assert.ok(dead.calls.every((sql) => sql.includes("INTERVAL '21600' SECOND")), "6-hour window"); + assert.ok(dead.calls.every((sql) => sql.includes(`${AE_COLUMNS.tier} = '2'`)), "Tier 2 only"); + + const quiet = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(0, 4)).queryFn); + assert.equal(quiet.firing, false, `4 attempts is a quiet period: ${quiet.detail}`); + + const oneReady = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(1, 20)).queryFn); + assert.equal(oneReady.firing, false, "a single ready proves the tier is alive"); + + const abandonedOnly = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(0, 0, 500)).queryFn); + assert.equal(abandonedOnly.firing, false, "abandoned previews are not attempts"); + + const tier1Dead = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(0, 50, 0, "1")).queryFn); + assert.equal(tier1Dead.firing, false, "a dead Tier 1 is not a Tier-2 outage"); + + const outOfWindow = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(0, 9).map((r) => ({ ...r, ageMs: 7 * 3_600_000 }))).queryFn); + assert.equal(outOfWindow.firing, false, "failures older than 6 h are outside the window"); +}); + +test("tier2LivenessRule: a firing alert resolves once a preview reaches ready", async () => { + const writer = fakeInboxWriter(); + const slackCalls = []; + const deps = { inboxWriter: writer, postSlack: async (t) => slackCalls.push(t), aeSink: fakeAeSink(), commonAttrs: COMMON_ATTRS, nowMs: 1000 }; + const dead = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(0, 6)).queryFn); + assert.equal(await evaluateAndNotify(dead, deps), "fired"); + const alive = await tier2LivenessRule({}, makeFakeAeQuery(tier2Rows(1, 6)).queryFn); + assert.equal(await evaluateAndNotify(alive, { ...deps, nowMs: 2000 }), "resolved"); +}); + +function readyDurations(n, ms, tier = "2") { + return Array.from({ length: n }, () => ({ metric: "preview.ready_ms", tier, outcome: "ready", duration_ms: ms })); +} + +test("previewSlowRule: Tier-2 ready p95 above 180 s fires with 10 ready previews; at 180 s, below the floor, or other tiers/outcomes do not", async () => { + assert.equal(PREVIEW_SLOW_MIN_SAMPLES, 10); + const slow = makeFakeAeQuery([...readyDurations(8, 40_000), ...readyDurations(2, 200_000)]); + const slowResult = await previewSlowRule({}, slow.queryFn); + assert.equal(slowResult.firing, true, slowResult.detail); + assert.equal(slowResult.rule, "preview-slow"); + assert.match(slowResult.detail, /p95 200\.0s over the last 6 h \(threshold 180s/); + assert.ok(slow.calls.every((sql) => sql.includes("INTERVAL '21600' SECOND")), "6-hour window"); + assert.ok(slow.calls.some((sql) => sql.includes(AE_COLUMNS.duration_ms)), "reads duration_ms"); + assert.ok(slow.calls.every((sql) => sql.includes(`${AE_COLUMNS.outcome} = 'ready'`)), "ready only"); + + const atThreshold = await previewSlowRule({}, makeFakeAeQuery([...readyDurations(8, 40_000), ...readyDurations(2, 180_000)]).queryFn); + assert.equal(atThreshold.firing, false, "exactly 180 s is not above the threshold"); + + const belowFloor = await previewSlowRule({}, makeFakeAeQuery(readyDurations(9, 500_000)).queryFn); + assert.equal(belowFloor.firing, false, `9 ready previews are below the floor: ${belowFloor.detail}`); + assert.match(belowFloor.detail, /9 ready Tier-2 preview\(s\)/); + + const healthy = await previewSlowRule({}, makeFakeAeQuery(readyDurations(30, 60_000)).queryFn); + assert.equal(healthy.firing, false); + + const wrongTier = await previewSlowRule({}, makeFakeAeQuery(readyDurations(30, 500_000, "1")).queryFn); + assert.equal(wrongTier.firing, false, "Tier 1 durations must not feed the Tier-2 p95"); + + const errorsExcluded = await previewSlowRule({}, makeFakeAeQuery([ + ...readyDurations(20, 60_000), + ...Array.from({ length: 10 }, () => ({ metric: "preview.ready_ms", tier: "2", outcome: "timeout", duration_ms: 600_000 })), + ]).queryFn); + assert.equal(errorsExcluded.firing, false, "timeouts carry long durations but are not ready times"); +}); + test("sessionStartP95Rule: over 20s fires, under does not; outcome='ready' filter is real, not decorative", async () => { const over = makeFakeAeQuery([ ...Array.from({ length: 19 }, () => ({ metric: "session.start", outcome: "ready", duration_ms: 5000 })), @@ -1243,7 +1370,7 @@ test("sessionStartP95Rule sample floor: a firing alert resolves once ready start assert.match(slackCalls[1], /resolved/); }); -test("embedErrorRateRule: a demo over 20% error rate with >50 views fires; under threshold and the views floor do not", async () => { +test("embedErrorRateRule: a demo over 20% error rate with at least 50 views in 7 days fires; under threshold and the views floor do not", async () => { const over = makeFakeAeQuery([ { metric: "error.uncaught", surface: "embed", demo_id: "r-react", count: 30 }, { metric: "serve.embed", outcome: "2xx", demo_id: "r-react", count: 100 }, @@ -1258,21 +1385,46 @@ test("embedErrorRateRule: a demo over 20% error rate with >50 views fires; under // Under threshold: 5% error rate on a demo with plenty of views. { metric: "error.uncaught", surface: "embed", demo_id: "r-vue", count: 5 }, { metric: "serve.embed", outcome: "2xx", demo_id: "r-vue", count: 100 }, - // Views floor: 100% error rate but only 50 views (not > 50) — excluded. - { metric: "error.uncaught", surface: "embed", demo_id: "r-tiny", count: 50 }, - { metric: "serve.embed", outcome: "2xx", demo_id: "r-tiny", count: 50 }, + // Views floor: 100% error rate but only 49 views (< 50) — excluded. + { metric: "error.uncaught", surface: "embed", demo_id: "r-tiny", count: 49 }, + { metric: "serve.embed", outcome: "2xx", demo_id: "r-tiny", count: 49 }, ]); const underResult = await embedErrorRateRule({}, under.queryFn); assert.equal(underResult.firing, false, underResult.detail); }); +test("embedErrorRateRule: reads a 7-day window, and a demo at exactly 50 views is evaluated", async () => { + assert.equal(EMBED_ERROR_MIN_VIEWS, 50); + const DAY = 24 * 3_600_000; + const week = makeFakeAeQuery([ + { metric: "error.uncaught", surface: "embed", demo_id: "r-slow", count: 20, ageMs: 5 * DAY }, + { metric: "serve.embed", outcome: "2xx", demo_id: "r-slow", count: 60, ageMs: 5 * DAY }, + ]); + const weekResult = await embedErrorRateRule({}, week.queryFn); + assert.equal(weekResult.firing, true, `views 5 days ago fall inside the window: ${weekResult.detail}`); + assert.match(weekResult.detail, /r-slow: 33\.3% \(20\/60 views\)/); + assert.ok(week.calls.every((sql) => sql.includes("INTERVAL '604800' SECOND")), "7-day window"); + + const stale = makeFakeAeQuery([ + { metric: "error.uncaught", surface: "embed", demo_id: "r-old", count: 20, ageMs: 8 * DAY }, + { metric: "serve.embed", outcome: "2xx", demo_id: "r-old", count: 60, ageMs: 8 * DAY }, + ]); + assert.equal((await embedErrorRateRule({}, stale.queryFn)).firing, false, "8 days ago is outside the window"); + + const atFloor = makeFakeAeQuery([ + { metric: "error.uncaught", surface: "embed", demo_id: "r-edge", count: 11 }, + { metric: "serve.embed", outcome: "2xx", demo_id: "r-edge", count: 50 }, + ]); + assert.equal((await embedErrorRateRule({}, atFloor.queryFn)).firing, true, "11/50 = 22% at the 50-view floor"); +}); + test("compileErrorDoublingRule: today >= 2x yesterday (floor met) fires; under the doubling ratio and under the floor do not", async () => { const TODAY_AGE_MS = 60_000; // within the last 24h const YESTERDAY_AGE_MS = 90_000_000; // ~25h ago, within the 24-48h-ago window const over = makeFakeAeQuery([ - { metric: "sandpack.compile_error", ht_major: "18", count: 10, ageMs: TODAY_AGE_MS }, - { metric: "sandpack.compile_error", ht_major: "18", count: 4, ageMs: YESTERDAY_AGE_MS }, + { metric: "sandpack.compile_error", ht_major: "18", count: 40, ageMs: TODAY_AGE_MS }, + { metric: "sandpack.compile_error", ht_major: "18", count: 15, ageMs: YESTERDAY_AGE_MS }, ]); const overResult = await compileErrorDoublingRule({}, over.queryFn); assert.equal(overResult.firing, true, overResult.detail); @@ -1280,16 +1432,29 @@ test("compileErrorDoublingRule: today >= 2x yesterday (floor met) fires; under t assert.ok(over.calls.some((s) => s.includes(AE_COLUMNS.ht_major)), "SQL must group by ht_major (AE_COLUMNS.ht_major)"); const under = makeFakeAeQuery([ - // Under the doubling ratio: 10 today vs 6 yesterday (< 2x). - { metric: "sandpack.compile_error", ht_major: "19", count: 10, ageMs: TODAY_AGE_MS }, - { metric: "sandpack.compile_error", ht_major: "19", count: 6, ageMs: YESTERDAY_AGE_MS }, - // Under the floor: 3 today (< 5) even against 0 yesterday — never "doubling". - { metric: "sandpack.compile_error", ht_major: "20", count: 3, ageMs: TODAY_AGE_MS }, + // Under the doubling ratio: 30 today vs 20 yesterday (< 2x). + { metric: "sandpack.compile_error", ht_major: "19", count: 30, ageMs: TODAY_AGE_MS }, + { metric: "sandpack.compile_error", ht_major: "19", count: 20, ageMs: YESTERDAY_AGE_MS }, + // Under the floor: 19 today (< 20) even against 0 yesterday — never "doubling". + { metric: "sandpack.compile_error", ht_major: "20", count: 19, ageMs: TODAY_AGE_MS }, ]); const underResult = await compileErrorDoublingRule({}, under.queryFn); assert.equal(underResult.firing, false, underResult.detail); }); +test("compileErrorDoublingRule floor: 6 today vs 0 yesterday does not fire, 20 vs 0 does", async () => { + assert.equal(COMPILE_ERROR_DOUBLING_FLOOR, 20); + const small = await compileErrorDoublingRule({}, makeFakeAeQuery([ + { metric: "sandpack.compile_error", ht_major: "18", count: 6, ageMs: 60_000 }, + ]).queryFn); + assert.equal(small.firing, false, small.detail); + + const atFloor = await compileErrorDoublingRule({}, makeFakeAeQuery([ + { metric: "sandpack.compile_error", ht_major: "18", count: 20, ageMs: 60_000 }, + ]).queryFn); + assert.equal(atFloor.firing, true, atFloor.detail); +}); + test("snapshotBuildFailedRateRule: a framework-wide build break fires; one author's failing saves do not", async () => { const systemic = makeFakeAeQuery([ ...Array.from({ length: 6 }, (_, i) => ({ metric: "snapshot.build", framework: "next.js", outcome: "failed", demo_id: `d-${i}`, count: 2 })), diff --git a/runner/workers/o11y/src/alerts/index.ts b/runner/workers/o11y/src/alerts/index.ts index 2f3e86a0ae..a73be0fe53 100644 --- a/runner/workers/o11y/src/alerts/index.ts +++ b/runner/workers/o11y/src/alerts/index.ts @@ -20,9 +20,11 @@ import { newFingerprintRule, o11yCapRule, previewReadyRateRule, + previewSlowRule, rejectedKeyRule, sessionStartP95Rule, snapshotBuildFailedRateRule, + tier2LivenessRule, type RuleResult, } from "./rules.js"; @@ -68,6 +70,8 @@ const QUERY_RULES: { id: string; fn: RuleFn }[] = [ { id: "ae-outbound-degraded", fn: aeOutboundDegradedRule }, { id: "api-5xx-rate", fn: fiveXxRateRule }, { id: "preview-ready-rate", fn: previewReadyRateRule }, + { id: "tier2-liveness", fn: tier2LivenessRule }, + { id: "preview-slow", fn: previewSlowRule }, { id: "session-start-p95", fn: sessionStartP95Rule }, { id: "embed-error-rate", fn: embedErrorRateRule }, { id: "compile-error-doubling", fn: compileErrorDoublingRule }, diff --git a/runner/workers/o11y/src/alerts/rules.ts b/runner/workers/o11y/src/alerts/rules.ts index ea489e245f..781ab4570f 100644 --- a/runner/workers/o11y/src/alerts/rules.ts +++ b/runner/workers/o11y/src/alerts/rules.ts @@ -191,7 +191,7 @@ export async function aeOutboundDegradedRule(env: Env, queryFn: AeQueryFn = runA }; } -// ---- api.request 5xx rate: above 1% over 15 min --------------------------- +// ---- api.request 5xx rate: above 1% over 1 h ------------------------------ /** The API's deliberate 503s (at-capacity, container-starting, chat/theme * refusals) are excluded from the 5xx ratio by their own exact counts. The @@ -200,11 +200,11 @@ export async function aeOutboundDegradedRule(env: Env, queryFn: AeQueryFn = runA * `reason`, and only the 503 is dropped. A build-failed 500 there still counts. */ const FIVE_XX_STILL_BUILDING_ROUTE_CLASSES = ["d/:id", "embed/:id"]; -/** Requests (after the exclusions below) in the 15-minute window under which the rule is not evaluated: at a 1 % threshold one error only exceeds it when the total is under 100, so a floor of 100 means a single 500 can never page. */ +/** Requests (after the exclusions below) in the 1-hour window under which the rule is not evaluated: at a 1 % threshold one error only exceeds it when the total is under 100, so a floor of 100 means a single 500 can never page. The hour is long enough for typical traffic to clear the floor. */ export const FIVE_XX_MIN_REQUESTS = 100; export async function fiveXxRateRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { - const windowMs = 15 * 60 * 1000; + const windowMs = HOUR_MS; const routeClassCol = col("route_class"); const outcomeCol = col("outcome"); const reasonCol = col("reason"); @@ -240,25 +240,31 @@ export async function fiveXxRateRule(env: Env, queryFn: AeQueryFn = runAeQuery): rule: "api-5xx-rate", firing: enoughSamples && pct > 1, detail: - `${pct.toFixed(2)}% 5xx over the last 15 min (${adjustedFiveXx}/${adjustedTotal}, threshold 1%, ` + + `${pct.toFixed(2)}% 5xx over the last hour (${adjustedFiveXx}/${adjustedTotal}, threshold 1%, ` + `excludes at-capacity/container-starting/chat-theme-gateway refusals and still-building 503s)` + (enoughSamples ? "" : `; only ${adjustedTotal} request(s), below the ${FIVE_XX_MIN_REQUESTS} needed to evaluate the rate`), }; } -// ---- preview-ready rate: below 97% (tier 1) or 95% (tier 2) over 1h ------- +// ---- preview-ready rate: below 97% (tier 1) or 95% (tier 2) over 6 h ------ const PREVIEW_READY_THRESHOLD_PCT: Record = { "1": 97, "2": 95 }; -/** Non-abandoned previews per tier per hour below which the tier is not evaluated, because one failure in two previews (50 %) says nothing about reliability. */ +/** The window every preview rule reads: Tier-2 volume is a few previews an hour, so an hour rarely reaches the floors and only a failure burst did. */ +const PREVIEW_WINDOW_MS = 6 * HOUR_MS; + +/** Non-abandoned previews per tier in the window below which the tier is not evaluated, because one failure in two previews (50 %) says nothing about reliability. */ export const PREVIEW_READY_MIN_SAMPLES = 10; +/** Failed (error + timeout) previews a tier needs in the window to fire. `preview.ready_ms` carries no demo or session id (its blobs are surface, tier, framework, ht_major, outcome, bucket), so distinct sources cannot be counted; a count at least keeps one stray failure from paging. */ +export const PREVIEW_READY_MIN_FAILURES = 3; + export async function previewReadyRateRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { const tierCol = col("tier"); const results: string[] = []; let anyFiring = false; for (const [tier, thresholdPct] of Object.entries(PREVIEW_READY_THRESHOLD_PCT)) { - const counts = await countByOutcome(env, "preview.ready_ms", HOUR_MS, `AND ${tierCol} = '${tier}'`, queryFn); + const counts = await countByOutcome(env, "preview.ready_ms", PREVIEW_WINDOW_MS, `AND ${tierCol} = '${tier}'`, queryFn); // `abandoned` (navigated away before the preview finished) is excluded // from both numerator and denominator — otherwise a burst of ordinary // navigation-aways would fire a false preview-reliability alert. @@ -267,10 +273,12 @@ export async function previewReadyRateRule(env: Env, queryFn: AeQueryFn = runAeQ .reduce((sum, [, c]) => sum + c, 0); const ready = counts.get("ready") ?? 0; if (total < PREVIEW_READY_MIN_SAMPLES) continue; + const failed = (counts.get("error") ?? 0) + (counts.get("timeout") ?? 0); + if (failed < PREVIEW_READY_MIN_FAILURES) continue; const pct = ratio(ready, total) * 100; if (pct < thresholdPct) { anyFiring = true; - results.push(`tier ${tier}: ${pct.toFixed(1)}% ready (${ready}/${total}, threshold ${thresholdPct}%)`); + results.push(`tier ${tier}: ${pct.toFixed(1)}% ready over the last 6 h (${ready}/${total}, ${failed} failed, threshold ${thresholdPct}%)`); } } return { @@ -280,6 +288,59 @@ export async function previewReadyRateRule(env: Env, queryFn: AeQueryFn = runAeQ }; } +// ---- tier-2 liveness: no ready preview in 6 h despite attempts ------------- + +/** Non-abandoned Tier-2 previews attempted in the window below which a lack of `ready` is just a quiet period. */ +export const TIER2_LIVENESS_MIN_ATTEMPTS = 5; + +export async function tier2LivenessRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { + const tierCol = col("tier"); + const counts = await countByOutcome(env, "preview.ready_ms", PREVIEW_WINDOW_MS, `AND ${tierCol} = '2'`, queryFn); + const attempts = [...counts.entries()] + .filter(([outcome]) => outcome !== "abandoned") + .reduce((sum, [, c]) => sum + c, 0); + const ready = counts.get("ready") ?? 0; + const firing = attempts >= TIER2_LIVENESS_MIN_ATTEMPTS && ready === 0; + return { + rule: "tier2-liveness", + firing, + detail: firing + ? `no Tier-2 preview reached ready in the last 6 h (${attempts} attempted)` + : `${ready} ready of ${attempts} non-abandoned Tier-2 preview(s) in the last 6 h (fires at 0 ready with at least ${TIER2_LIVENESS_MIN_ATTEMPTS} attempted)`, + }; +} + +// ---- tier-2 slow previews: ready p95 above 180 s over 6 h ------------------ + +/** Ready Tier-2 previews in the window below which the p95 is not evaluated, because a p95 over a handful of previews is one slow preview. */ +export const PREVIEW_SLOW_MIN_SAMPLES = 10; + +/** Starting value, above today's worst per-framework p95 (167 s); tune from real traffic. */ +export const PREVIEW_SLOW_P95_MS = 180_000; + +export async function previewSlowRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { + const tierCol = col("tier"); + const outcomeCol = col("outcome"); + // `ready` only: an error or timeout carries its own duration and would be counted by the reliability rules. + const where = `AND ${tierCol} = '2' AND ${outcomeCol} = 'ready'`; + const [p95, byOutcome] = await Promise.all([ + weightedQuantile(env, "preview.ready_ms", PREVIEW_WINDOW_MS, 0.95, where, queryFn), + countByOutcome(env, "preview.ready_ms", PREVIEW_WINDOW_MS, where, queryFn), + ]); + const readyPreviews = byOutcome.get("ready") ?? 0; + const enoughSamples = readyPreviews >= PREVIEW_SLOW_MIN_SAMPLES; + const firing = enoughSamples && p95 !== null && p95 > PREVIEW_SLOW_P95_MS; + return { + rule: "preview-slow", + firing, + detail: p95 === null + ? "no ready Tier-2 preview samples in the last 6 h" + : !enoughSamples + ? `${readyPreviews} ready Tier-2 preview(s) in the last 6 h, below the ${PREVIEW_SLOW_MIN_SAMPLES} needed to evaluate the p95` + : `Tier-2 time-to-ready p95 ${(p95 / 1000).toFixed(1)}s over the last 6 h (threshold ${PREVIEW_SLOW_P95_MS / 1000}s, outcome=ready only)`, + }; +} + // ---- session-start p95: above 20s (window: 1h — the ADR text names the -- // threshold but not an evaluation window) ----------------------------------- @@ -311,10 +372,16 @@ export async function sessionStartP95Rule(env: Env, queryFn: AeQueryFn = runAeQu }; } -// ---- embed error rate: above 20% with more than 50 views in 24h, per demo - +// ---- embed error rate: above 20% with at least 50 views in 7 days, per demo - + +/** Embed views are a trickle (about 114 for the busiest demo in 14 days), so a day never reaches the floor. */ +const EMBED_ERROR_WINDOW_MS = 7 * DAY_MS; + +/** Views per demo in the window below which the demo is not evaluated. */ +export const EMBED_ERROR_MIN_VIEWS = 50; export async function embedErrorRateRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { - const windowMs = DAY_MS; + const windowMs = EMBED_ERROR_WINDOW_MS; const surfaceCol = col("surface"); const outcomeCol = col("outcome"); const [errorsByDemo, viewsByDemo] = await Promise.all([ @@ -323,7 +390,7 @@ export async function embedErrorRateRule(env: Env, queryFn: AeQueryFn = runAeQue ]); const offenders: string[] = []; for (const [demoId, views] of viewsByDemo) { - if (!demoId || views <= 50) continue; + if (!demoId || views < EMBED_ERROR_MIN_VIEWS) continue; const errors = errorsByDemo.get(demoId) ?? 0; const pct = ratio(errors, views) * 100; if (pct > 20) offenders.push(`${demoId}: ${pct.toFixed(1)}% (${errors}/${views} views)`); @@ -331,14 +398,14 @@ export async function embedErrorRateRule(env: Env, queryFn: AeQueryFn = runAeQue return { rule: "embed-error-rate", firing: offenders.length > 0, - detail: offenders.join("; ") || "no demo over threshold", + detail: offenders.length > 0 ? `${offenders.join("; ")} over the last 7 days (threshold 20%)` : "no demo over threshold", }; } // ---- compile-error rate per ht_major: doubling day over day (a floor of -- -// 5 today-count avoids "doubling" noise on tiny counts like 0->1) ----------- +// 20 today-count avoids "doubling" noise on small counts like 0->6) --------- -const COMPILE_ERROR_DOUBLING_FLOOR = 5; +export const COMPILE_ERROR_DOUBLING_FLOOR = 20; export async function compileErrorDoublingRule(env: Env, queryFn: AeQueryFn = runAeQuery): Promise { const [today, yesterday] = await Promise.all([