{
  "license": "CC BY 4.0",
  "recipe": {
    "slug": "ai-agent-task-success-and-handoff",
    "title": "Measure AI Agent Task Success and Human Handoff",
    "testedWith": "Apache DataFusion 45.2.0",
    "lastReviewed": "2026-07-28"
  },
  "contract": {
    "tableName": "agent_run_outcomes",
    "schema": [
      {
        "name": "timestamp_utc",
        "type": "Timestamp",
        "description": "Terminal agent-run time in UTC."
      },
      {
        "name": "workflow",
        "type": "Utf8",
        "description": "Stable product workflow or task category."
      },
      {
        "name": "model",
        "type": "Utf8",
        "description": "Model identifier recorded from the provider response."
      },
      {
        "name": "status",
        "type": "Utf8",
        "description": "success or failed for the technical run."
      },
      {
        "name": "reviewer_outcome",
        "type": "Utf8",
        "description": "accepted, revised, rejected, or not_reviewed."
      },
      {
        "name": "human_handoff",
        "type": "Boolean",
        "description": "Whether a human took over the task."
      },
      {
        "name": "duration_ms",
        "type": "Float64",
        "description": "End-to-end workflow duration."
      },
      {
        "name": "estimated_cost_usd",
        "type": "Float64",
        "description": "Normalized estimated model cost for the run."
      }
    ]
  },
  "query": "SELECT\n  workflow,\n  model,\n  COUNT(*) AS runs,\n  100.0 * SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END)\n    / NULLIF(COUNT(*), 0) AS technical_success_rate_pct,\n  100.0 * SUM(CASE WHEN reviewer_outcome = 'accepted' THEN 1 ELSE 0 END)\n    / NULLIF(SUM(CASE\n      WHEN reviewer_outcome <> 'not_reviewed' THEN 1 ELSE 0\n    END), 0) AS reviewed_acceptance_rate_pct,\n  100.0 * SUM(CASE WHEN human_handoff THEN 1 ELSE 0 END)\n    / NULLIF(COUNT(*), 0) AS human_handoff_rate_pct,\n  AVG(duration_ms) AS average_duration_ms,\n  SUM(estimated_cost_usd) / NULLIF(COUNT(*), 0) AS cost_per_run_usd\nFROM agent_run_outcomes\nWHERE timestamp_utc >= now() - INTERVAL '30 days'\nGROUP BY workflow, model\nHAVING COUNT(*) >= 20\nORDER BY reviewed_acceptance_rate_pct, runs DESC;",
  "fixtureKind": "schema-example",
  "inputRows": [
    {
      "timestamp_utc": "2026-07-28T10:00:00Z",
      "workflow": "workflow_example_1",
      "model": "configured-model",
      "status": "success",
      "reviewer_outcome": "success",
      "human_handoff": true,
      "duration_ms": 12,
      "estimated_cost_usd": 0.014
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00Z",
      "workflow": "workflow_example_2",
      "model": "configured-model",
      "status": "failed",
      "reviewer_outcome": "failed",
      "human_handoff": false,
      "duration_ms": 37,
      "estimated_cost_usd": 0.052
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00Z",
      "workflow": "workflow_example_3",
      "model": "configured-model",
      "status": "success",
      "reviewer_outcome": "success",
      "human_handoff": true,
      "duration_ms": 24,
      "estimated_cost_usd": 0.021
    }
  ],
  "illustrativeInputRows": [
    {
      "timestamp_utc": "2026-07-28T10:00:00Z",
      "workflow": "workflow_example_1",
      "model": "configured-model",
      "status": "success",
      "reviewer_outcome": "success",
      "human_handoff": true,
      "duration_ms": 12,
      "estimated_cost_usd": 0.014
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00Z",
      "workflow": "workflow_example_2",
      "model": "configured-model",
      "status": "failed",
      "reviewer_outcome": "failed",
      "human_handoff": false,
      "duration_ms": 37,
      "estimated_cost_usd": 0.052
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00Z",
      "workflow": "workflow_example_3",
      "model": "configured-model",
      "status": "success",
      "reviewer_outcome": "success",
      "human_handoff": true,
      "duration_ms": 24,
      "estimated_cost_usd": 0.021
    }
  ],
  "expectedOutput": {
    "columns": [
      "workflow",
      "model",
      "runs",
      "technical_success_rate_pct",
      "reviewed_acceptance_rate_pct",
      "human_handoff_rate_pct",
      "average_duration_ms",
      "cost_per_run_usd"
    ],
    "rows": [
      {
        "workflow": "support_triage",
        "model": "configured-model-a",
        "runs": 1840,
        "technical_success_rate_pct": 98.1,
        "reviewed_acceptance_rate_pct": 86.4,
        "human_handoff_rate_pct": 18.2,
        "average_duration_ms": 4820,
        "cost_per_run_usd": 0.041
      },
      {
        "workflow": "refund_review",
        "model": "configured-model-b",
        "runs": 620,
        "technical_success_rate_pct": 96.8,
        "reviewed_acceptance_rate_pct": 68.7,
        "human_handoff_rate_pct": 42.9,
        "average_duration_ms": 8110,
        "cost_per_run_usd": 0.073
      },
      {
        "workflow": "knowledge_draft",
        "model": "configured-model-a",
        "runs": 940,
        "technical_success_rate_pct": 99.2,
        "reviewed_acceptance_rate_pct": 91.6,
        "human_handoff_rate_pct": 8.4,
        "average_duration_ms": 3910,
        "cost_per_run_usd": 0.036
      }
    ]
  },
  "notes": [
    "The compact input rows demonstrate field names and JSON types; they are not claimed to generate the displayed aggregate by themselves.",
    "The expected output is deterministic synthetic review data for the documented output shape.",
    "Adapt time windows, thresholds, identities, and business definitions before operational use."
  ]
}