構造化イベントと SQL を使用して AI エージェントを評価する

import telemetry from "telemetry-sh";

telemetry.init(process.env.TELEMETRY_API_KEY);

export async function recordAgentEvaluation({
  operationId,
  evaluationId,
  evaluatorType,
  evaluatorVersion,
  metricName,
  score,
  threshold,
  reviewOutcome,
  isFinal,
  datasetVersion,
  promptVersion,
  release,
}) {
  await telemetry.log("ai_output_reviewed", {
    operation_id: operationId,
    evaluation_id: evaluationId,
    evaluator_type: evaluatorType,
    evaluator_version: evaluatorVersion,
    metric_name: metricName,
    score,
    threshold,
    passed: score >= threshold,
    review_outcome: reviewOutcome,
    is_final: isFinal,
    dataset_version: datasetVersion,
    prompt_version: promptVersion,
    release,
  });
}

WITH completed AS (
  SELECT release, operation_id
  FROM agent_run_completed
  WHERE timestamp_utc >= now() - INTERVAL '30 days'
    AND status = 'success'
  GROUP BY release, operation_id
),
final_reviews AS (
  SELECT release, operation_id, MAX(score) AS score,
    MAX(CASE WHEN passed THEN 1 ELSE 0 END) AS passed
  FROM ai_output_reviewed
  WHERE timestamp_utc >= now() - INTERVAL '30 days'
    AND metric_name = 'task_quality'
    AND evaluator_version = 'quality-rubric-v3'
    AND is_final = true
  GROUP BY release, operation_id
  HAVING COUNT(*) = 1
)
SELECT
  c.release,
  COUNT(*) AS completed_operations,
  COUNT(f.operation_id) AS evaluated_operations,
  ROUND(
    100.0 * COUNT(f.operation_id)
    / NULLIF(COUNT(*), 0),
    1
  ) AS evaluation_coverage_pct,
  ROUND(
    100.0 * SUM(CASE WHEN f.passed = 1 THEN 1 ELSE 0 END)
    / NULLIF(COUNT(f.operation_id), 0),
    1
  ) AS evaluated_pass_rate_pct,
  ROUND(AVG(f.score), 3) AS average_final_score
FROM completed c
LEFT JOIN final_reviews f
  ON f.release = c.release AND f.operation_id = c.operation_id
GROUP BY c.release
ORDER BY c.release;

SELECT release, operation_id, COUNT(*) AS conflicting_final_reviews
FROM ai_output_reviewed
WHERE timestamp_utc >= now() - INTERVAL '30 days'
  AND metric_name = 'task_quality'
  AND evaluator_version = 'quality-rubric-v3'
  AND is_final = true
GROUP BY release, operation_id
HAVING COUNT(*) > 1
ORDER BY release, operation_id;

WITH operation_cost AS (
  SELECT
    operation_id,
    SUM(estimated_cost_usd) AS total_cost_usd
  FROM llm_request_completed
  WHERE timestamp_utc >= now() - INTERVAL '30 days'
  GROUP BY operation_id
),
terminal_outcome AS (
  SELECT
    operation_id,
    MAX(CASE WHEN review_outcome = 'accepted' THEN 1 ELSE 0 END) AS accepted
  FROM ai_output_reviewed
  WHERE timestamp_utc >= now() - INTERVAL '30 days'
    AND metric_name = 'task_quality'
    AND evaluator_version = 'quality-rubric-v3'
    AND is_final = true
  GROUP BY operation_id
  HAVING COUNT(*) = 1
)
SELECT
  COUNT(*) AS evaluated_operations,
  SUM(accepted) AS accepted_operations,
  ROUND(SUM(total_cost_usd), 4) AS evaluated_cost_usd,
  ROUND(
    SUM(total_cost_usd) / NULLIF(SUM(accepted), 0),
    4
  ) AS cost_per_accepted_operation_usd
FROM operation_cost c
JOIN terminal_outcome o ON o.operation_id = c.operation_id;