{
  "license": "CC BY 4.0",
  "recipe": {
    "slug": "database-replication-and-cdc-lag",
    "title": "Measure Database Replication and CDC Lag",
    "testedWith": "Apache DataFusion 45.2.0",
    "lastReviewed": "2026-07-28"
  },
  "contract": {
    "tableName": "database_replication_samples",
    "schema": [
      {
        "name": "timestamp_utc",
        "type": "Timestamp",
        "description": "Replication sample time in UTC."
      },
      {
        "name": "consumer_name",
        "type": "Utf8",
        "description": "Stable replica, connector, or downstream consumer name."
      },
      {
        "name": "source_type",
        "type": "Utf8",
        "description": "postgres_replica, cdc_pipeline, or another controlled source type."
      },
      {
        "name": "region",
        "type": "Utf8",
        "description": "Approved deployment region."
      },
      {
        "name": "lag_seconds",
        "type": "Float64",
        "description": "Observed time lag behind the source."
      },
      {
        "name": "replay_bytes_behind",
        "type": "Int64",
        "description": "Approximate bytes waiting to be replayed or consumed."
      },
      {
        "name": "status",
        "type": "Utf8",
        "description": "healthy, lagging, stalled, or unavailable."
      },
      {
        "name": "environment",
        "type": "Utf8",
        "description": "Deployment environment."
      }
    ]
  },
  "query": "SELECT\n  consumer_name,\n  source_type,\n  region,\n  COUNT(*) AS samples,\n  AVG(lag_seconds) AS average_lag_seconds,\n  MAX(lag_seconds) AS maximum_lag_seconds,\n  MAX(replay_bytes_behind) AS maximum_bytes_behind,\n  SUM(CASE WHEN lag_seconds > 30 THEN 1 ELSE 0 END) AS stale_samples,\n  100.0 * SUM(CASE WHEN lag_seconds > 30 THEN 1 ELSE 0 END)\n    / NULLIF(COUNT(*), 0) AS stale_sample_rate_pct\nFROM database_replication_samples\nWHERE timestamp_utc >= now() - INTERVAL '24 hours'\n  AND environment = 'production'\nGROUP BY consumer_name, source_type, region\nHAVING COUNT(*) >= 10\nORDER BY maximum_lag_seconds DESC;",
  "fixtureKind": "reproducible",
  "inputRows": [
    {
      "timestamp_utc": "2026-07-28T18:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T17:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T16:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T15:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 5,
      "replay_bytes_behind": 5242880,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T14:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 7,
      "replay_bytes_behind": 7340032,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T13:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 9,
      "replay_bytes_behind": 9437184,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 12,
      "replay_bytes_behind": 12582912,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 18,
      "replay_bytes_behind": 18874368,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T10:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 34,
      "replay_bytes_behind": 35651584,
      "status": "lagging",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T09:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 42,
      "replay_bytes_behind": 44040192,
      "status": "lagging",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T08:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 28,
      "replay_bytes_behind": 29360128,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T07:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 11,
      "replay_bytes_behind": 11534336,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T16:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 1,
      "replay_bytes_behind": 1048576,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T15:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T14:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 1,
      "replay_bytes_behind": 1048576,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T13:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T10:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T09:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 5,
      "replay_bytes_behind": 5242880,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T08:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 6,
      "replay_bytes_behind": 6291456,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T07:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T06:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T05:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    }
  ],
  "illustrativeInputRows": [
    {
      "timestamp_utc": "2026-07-28T18:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T17:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T16:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T15:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 5,
      "replay_bytes_behind": 5242880,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T14:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 7,
      "replay_bytes_behind": 7340032,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T13:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 9,
      "replay_bytes_behind": 9437184,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 12,
      "replay_bytes_behind": 12582912,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 18,
      "replay_bytes_behind": 18874368,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T10:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 34,
      "replay_bytes_behind": 35651584,
      "status": "lagging",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T09:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 42,
      "replay_bytes_behind": 44040192,
      "status": "lagging",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T08:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 28,
      "replay_bytes_behind": 29360128,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T07:00:00.000Z",
      "consumer_name": "replica-us-west",
      "source_type": "postgres_replica",
      "region": "us-west",
      "lag_seconds": 11,
      "replay_bytes_behind": 11534336,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T16:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 1,
      "replay_bytes_behind": 1048576,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T15:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T14:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 1,
      "replay_bytes_behind": 1048576,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T13:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T12:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T11:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T10:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T09:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 5,
      "replay_bytes_behind": 5242880,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T08:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 6,
      "replay_bytes_behind": 6291456,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T07:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 4,
      "replay_bytes_behind": 4194304,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T06:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 3,
      "replay_bytes_behind": 3145728,
      "status": "healthy",
      "environment": "production"
    },
    {
      "timestamp_utc": "2026-07-28T05:00:00.000Z",
      "consumer_name": "billing-cdc",
      "source_type": "cdc_pipeline",
      "region": "us-east",
      "lag_seconds": 2,
      "replay_bytes_behind": 2097152,
      "status": "healthy",
      "environment": "production"
    }
  ],
  "expectedOutput": {
    "columns": [
      "consumer_name",
      "source_type",
      "region",
      "samples",
      "average_lag_seconds",
      "maximum_lag_seconds",
      "maximum_bytes_behind",
      "stale_samples",
      "stale_sample_rate_pct"
    ],
    "rows": [
      {
        "consumer_name": "replica-us-west",
        "source_type": "postgres_replica",
        "region": "us-west",
        "samples": 12,
        "average_lag_seconds": 14.58,
        "maximum_lag_seconds": 42,
        "maximum_bytes_behind": 44040192,
        "stale_samples": 2,
        "stale_sample_rate_pct": 16.67
      },
      {
        "consumer_name": "billing-cdc",
        "source_type": "cdc_pipeline",
        "region": "us-east",
        "samples": 12,
        "average_lag_seconds": 3,
        "maximum_lag_seconds": 6,
        "maximum_bytes_behind": 6291456,
        "stale_samples": 0,
        "stale_sample_rate_pct": 0
      }
    ]
  },
  "notes": [
    "The included synthetic input rows generate the expected output when timestamps are shifted into the documented query window.",
    "Run this fixture in the public browser SQL playground; the selected query executes locally with DuckDB-Wasm.",
    "Adapt time windows, thresholds, identities, and business definitions before operational use."
  ]
}