{
  "license": "CC BY 4.0",
  "licenseUrl": "https://creativecommons.org/licenses/by/4.0/",
  "recipe": {
    "slug": "background-job-retry-storm",
    "title": "Detect Background-Job Retry Storms",
    "testedWith": "Apache DataFusion 45.2.0",
    "lastReviewed": "2026-07-28"
  },
  "contract": {
    "tableName": "job_attempts",
    "schema": [
      {
        "name": "timestamp_utc",
        "type": "Timestamp",
        "description": "Attempt completion time."
      },
      {
        "name": "job_id",
        "type": "Utf8",
        "description": "Stable logical job identifier."
      },
      {
        "name": "job_name",
        "type": "Utf8",
        "description": "Stable job type."
      },
      {
        "name": "attempt",
        "type": "Int64",
        "description": "One-based attempt number."
      },
      {
        "name": "status",
        "type": "Utf8",
        "description": "completed, retrying, or failed."
      }
    ]
  },
  "query": "SELECT\n  date_trunc('hour', timestamp_utc) AS hour,\n  job_name,\n  COUNT(*) AS attempts,\n  COUNT(DISTINCT job_id) AS logical_jobs,\n  SUM(CASE WHEN attempt > 1 THEN 1 ELSE 0 END) AS retry_attempts,\n  100.0 * SUM(CASE WHEN attempt > 1 THEN 1 ELSE 0 END)\n    / NULLIF(COUNT(*), 0) AS retry_attempt_rate_pct,\n  100.0 * SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END)\n    / NULLIF(COUNT(*), 0) AS failed_attempt_rate_pct\nFROM job_attempts\nWHERE timestamp_utc >= now() - INTERVAL '24 hours'\nGROUP BY date_trunc('hour', timestamp_utc), job_name\nHAVING COUNT(*) >= 20\nORDER BY retry_attempt_rate_pct DESC, attempts DESC;",
  "fixtureKind": "schema-example",
  "inputRows": [
    {
      "timestamp_utc": "2026-07-28T10:00:00Z",
      "job_id": "job_id_example_1",
      "job_name": "job_name_example_1",
      "attempt": 1,
      "status": "success"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00Z",
      "job_id": "job_id_example_2",
      "job_name": "job_name_example_2",
      "attempt": 2,
      "status": "failed"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00Z",
      "job_id": "job_id_example_3",
      "job_name": "job_name_example_3",
      "attempt": 3,
      "status": "success"
    }
  ],
  "illustrativeInputRows": [
    {
      "timestamp_utc": "2026-07-28T10:00:00Z",
      "job_id": "job_id_example_1",
      "job_name": "job_name_example_1",
      "attempt": 1,
      "status": "success"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00Z",
      "job_id": "job_id_example_2",
      "job_name": "job_name_example_2",
      "attempt": 2,
      "status": "failed"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00Z",
      "job_id": "job_id_example_3",
      "job_name": "job_name_example_3",
      "attempt": 3,
      "status": "success"
    }
  ],
  "expectedOutput": {
    "columns": [
      "hour",
      "job_name",
      "attempts",
      "logical_jobs",
      "retry_attempts",
      "retry_attempt_rate_pct",
      "failed_attempt_rate_pct"
    ],
    "rows": [
      {
        "hour": "2026-07-27 14:00",
        "job_name": "sync_crm_accounts",
        "attempts": 1840,
        "logical_jobs": 492,
        "retry_attempts": 1118,
        "retry_attempt_rate_pct": 60.76,
        "failed_attempt_rate_pct": 22.34
      },
      {
        "hour": "2026-07-27 14:00",
        "job_name": "send_receipt_email",
        "attempts": 3180,
        "logical_jobs": 3061,
        "retry_attempts": 119,
        "retry_attempt_rate_pct": 3.74,
        "failed_attempt_rate_pct": 0.44
      }
    ]
  },
  "notes": [
    "The compact input rows demonstrate field names and JSON types; they are not claimed to generate the displayed aggregate by themselves.",
    "The expected output is deterministic synthetic review data for the documented output shape.",
    "Adapt time windows, thresholds, identities, and business definitions before operational use."
  ]
}