{ "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "recipe": { "slug": "background-job-retry-rate", "title": "Measure Background Job Retry and Failure Rate", "testedWith": "Apache DataFusion 45.2.0", "lastReviewed": "2026-07-27" }, "contract": { "tableName": "job_runs", "schema": [ { "name": "timestamp_utc", "type": "Timestamp", "description": "When the run completed or failed." }, { "name": "job_name", "type": "Utf8", "description": "Stable logical job name." }, { "name": "status", "type": "Utf8", "description": "success or failed." }, { "name": "attempt", "type": "Int64", "description": "One-based execution attempt." }, { "name": "duration_ms", "type": "Float64", "description": "Duration of the attempt." } ] }, "query": "SELECT\n job_name,\n COUNT(*) AS attempts,\n SUM(CASE WHEN attempt > 1 THEN 1 ELSE 0 END) AS retries,\n SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) AS failures,\n 100.0 * SUM(CASE WHEN attempt > 1 THEN 1 ELSE 0 END)\n / NULLIF(COUNT(*), 0) AS retry_rate_pct,\n approx_percentile_cont(duration_ms, 0.95) AS p95_duration_ms\nFROM job_runs\nWHERE timestamp_utc >= now() - INTERVAL '7 days'\nGROUP BY job_name\nORDER BY retry_rate_pct DESC, failures DESC;", "fixtureKind": "reproducible", "inputRows": [ { "timestamp_utc": "2026-07-28T18:00:00.000Z", "job_name": "sync_subscription", "status": "failed", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:59:00.000Z", "job_name": "sync_subscription", "status": "failed", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:58:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:57:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:56:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:55:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:54:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:53:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:52:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:51:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:30:00.000Z", "job_name": "generate_report", "status": "failed", "attempt": 2, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:29:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:28:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:27:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:26:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:25:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:24:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:23:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:00:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:59:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:58:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:57:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:56:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:55:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:54:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:53:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:52:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:51:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 } ], "illustrativeInputRows": [ { "timestamp_utc": "2026-07-28T18:00:00.000Z", "job_name": "sync_subscription", "status": "failed", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:59:00.000Z", "job_name": "sync_subscription", "status": "failed", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:58:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 2, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:57:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:56:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:55:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:54:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:53:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:52:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:51:00.000Z", "job_name": "sync_subscription", "status": "success", "attempt": 1, "duration_ms": 8000 }, { "timestamp_utc": "2026-07-28T17:30:00.000Z", "job_name": "generate_report", "status": "failed", "attempt": 2, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:29:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:28:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:27:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:26:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:25:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:24:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:23:00.000Z", "job_name": "generate_report", "status": "success", "attempt": 1, "duration_ms": 12000 }, { "timestamp_utc": "2026-07-28T17:00:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:59:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:58:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:57:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:56:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:55:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:54:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:53:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:52:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 }, { "timestamp_utc": "2026-07-28T16:51:00.000Z", "job_name": "send_email", "status": "success", "attempt": 1, "duration_ms": 900 } ], "expectedOutput": { "columns": [ "job_name", "attempts", "retries", "failures", "retry_rate_pct", "p95_duration_ms" ], "rows": [ { "job_name": "sync_subscription", "attempts": 10, "retries": 3, "failures": 2, "retry_rate_pct": 30, "p95_duration_ms": 8000 }, { "job_name": "generate_report", "attempts": 8, "retries": 1, "failures": 1, "retry_rate_pct": 12.5, "p95_duration_ms": 12000 }, { "job_name": "send_email", "attempts": 10, "retries": 0, "failures": 0, "retry_rate_pct": 0, "p95_duration_ms": 900 } ] }, "notes": [ "The included synthetic input rows generate the expected output when timestamps are shifted into the documented query window.", "Run this fixture in the public browser SQL playground; the selected query executes locally with DuckDB-Wasm.", "Adapt time windows, thresholds, identities, and business definitions before operational use." ] }