{ "license": "CC BY 4.0", "recipe": { "slug": "ai-quality-regression-by-prompt-version", "title": "Find AI Quality Regressions by Prompt Version", "testedWith": "Apache DataFusion 45.2.0", "lastReviewed": "2026-07-28" }, "contract": { "tableName": "ai_quality_events", "schema": [ { "name": "timestamp_utc", "type": "Timestamp", "description": "Evaluation completion time in UTC." }, { "name": "workflow_name", "type": "Utf8", "description": "Bounded AI workflow name." }, { "name": "prompt_version", "type": "Utf8", "description": "Reviewed prompt or policy version." }, { "name": "model_name", "type": "Utf8", "description": "Bounded model name." }, { "name": "quality_score", "type": "Float64", "description": "Normalized score from a documented evaluator." }, { "name": "quality_passed", "type": "Boolean", "description": "Whether the run passed the reviewed quality threshold." }, { "name": "accepted_without_edit", "type": "Boolean", "description": "Whether a user accepted the output without editing." }, { "name": "human_handoff", "type": "Boolean", "description": "Whether the workflow escalated to a person." }, { "name": "cost_usd", "type": "Float64", "description": "Normalized model cost for the run." }, { "name": "environment", "type": "Utf8", "description": "Deployment environment." } ] }, "query": "SELECT\n workflow_name,\n prompt_version,\n COUNT(*) AS evaluated_runs,\n 100.0 * SUM(CASE WHEN quality_passed THEN 1 ELSE 0 END)\n / NULLIF(COUNT(*), 0) AS quality_pass_rate_pct,\n 100.0 * SUM(CASE WHEN accepted_without_edit THEN 1 ELSE 0 END)\n / NULLIF(COUNT(*), 0) AS acceptance_rate_pct,\n 100.0 * SUM(CASE WHEN human_handoff THEN 1 ELSE 0 END)\n / NULLIF(COUNT(*), 0) AS handoff_rate_pct,\n AVG(quality_score) AS average_quality_score,\n SUM(cost_usd) / NULLIF(\n SUM(CASE WHEN quality_passed THEN 1 ELSE 0 END),\n 0\n ) AS cost_per_quality_pass_usd\nFROM ai_quality_events\nWHERE timestamp_utc >= now() - INTERVAL '30 days'\n AND environment = 'production'\nGROUP BY workflow_name, prompt_version\nORDER BY workflow_name, prompt_version;", "fixtureKind": "reproducible", "inputRows": [ { "timestamp_utc": "2026-07-28T18:00:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.92, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:59:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.87, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:58:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.81, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:57:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.48, "quality_passed": false, "accepted_without_edit": false, "human_handoff": true, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:56:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.39, "quality_passed": false, "accepted_without_edit": false, "human_handoff": true, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:30:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.94, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:29:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.91, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:28:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.88, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:27:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.84, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:26:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.58, "quality_passed": false, "accepted_without_edit": false, "human_handoff": false, "cost_usd": 0.02, "environment": "production" } ], "illustrativeInputRows": [ { "timestamp_utc": "2026-07-28T18:00:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.92, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:59:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.87, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:58:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.81, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:57:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.48, "quality_passed": false, "accepted_without_edit": false, "human_handoff": true, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:56:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v3", "model_name": "gpt-5-mini", "quality_score": 0.39, "quality_passed": false, "accepted_without_edit": false, "human_handoff": true, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:30:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.94, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:29:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.91, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:28:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.88, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:27:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.84, "quality_passed": true, "accepted_without_edit": true, "human_handoff": false, "cost_usd": 0.02, "environment": "production" }, { "timestamp_utc": "2026-07-28T17:26:00.000Z", "workflow_name": "support_reply", "prompt_version": "support-v4", "model_name": "gpt-5-mini", "quality_score": 0.58, "quality_passed": false, "accepted_without_edit": false, "human_handoff": false, "cost_usd": 0.02, "environment": "production" } ], "expectedOutput": { "columns": [ "workflow_name", "prompt_version", "evaluated_runs", "quality_pass_rate_pct", "acceptance_rate_pct", "handoff_rate_pct", "average_quality_score", "cost_per_quality_pass_usd" ], "rows": [ { "workflow_name": "support_reply", "prompt_version": "support-v3", "evaluated_runs": 5, "quality_pass_rate_pct": 60, "acceptance_rate_pct": 60, "handoff_rate_pct": 40, "average_quality_score": 0.694, "cost_per_quality_pass_usd": 0.0333333333 }, { "workflow_name": "support_reply", "prompt_version": "support-v4", "evaluated_runs": 5, "quality_pass_rate_pct": 80, "acceptance_rate_pct": 80, "handoff_rate_pct": 0, "average_quality_score": 0.83, "cost_per_quality_pass_usd": 0.025 } ] }, "notes": [ "The included synthetic input rows generate the expected output when timestamps are shifted into the documented query window.", "Run this fixture in the public browser SQL playground; the selected query executes locally with DuckDB-Wasm.", "Adapt time windows, thresholds, identities, and business definitions before operational use." ] }