{
  "classes": [
    "interested",
    "not_now",
    "not_interested",
    "referral",
    "out_of_office",
    "unsubscribe",
    "auto_bounce"
  ],
  "routing": {
    "interested": {
      "destination": "Notify (Slack) and flag for review. Nothing auto-sends.",
      "miss_cost": "A missed one is a lost deal. This is the expensive direction."
    },
    "not_now": {
      "destination": "CRM: set a follow-up date from the reply, drop from the live sequence.",
      "miss_cost": "Cheap if it lands in review. Expensive if it is filed as a hard no."
    },
    "not_interested": {
      "destination": "CRM: closed, suppressed from future sends.",
      "miss_cost": "Cheap. Worst case someone reads a dead reply."
    },
    "referral": {
      "destination": "Notify, with the named person pulled out for a new contact record.",
      "miss_cost": "High. A referral is a warm intro that expires quietly."
    },
    "out_of_office": {
      "destination": "Pause the sequence until the stated return date. No notification.",
      "miss_cost": "Low, unless it is filed as a hard no and the person never gets a follow-up."
    },
    "unsubscribe": {
      "destination": "Suppress immediately, write to the compliance log. No notification.",
      "miss_cost": "The only class with a legal edge. A miss here is worse than a miss anywhere else."
    },
    "auto_bounce": {
      "destination": "Suppress, increment the domain-health counter for that domain.",
      "miss_cost": "Low individually. Ignored in bulk, it is how a sending domain dies."
    }
  },
  "corpus_counts": {
    "clean": {
      "interested": 7,
      "not_now": 6,
      "not_interested": 7,
      "referral": 6,
      "out_of_office": 6,
      "unsubscribe": 6,
      "auto_bounce": 6
    },
    "realistic": {
      "interested": 9,
      "not_now": 8,
      "not_interested": 8,
      "referral": 8,
      "out_of_office": 7,
      "unsubscribe": 7,
      "auto_bounce": 7
    },
    "realistic_contested": 10
  },
  "arms": {
    "clean_haiku": {
      "corpus": null,
      "model": "claude-haiku-4-5",
      "strip_quotes": false,
      "temperature": 0.0,
      "temperature_note": "sent",
      "n": 44,
      "correct": 44,
      "accuracy": 1.0,
      "misses": [],
      "input_tokens": 31540,
      "output_tokens": 1683,
      "mean_seconds": 1.59,
      "cost": {
        "usd_per_reply": 0.000908,
        "usd_per_1000_replies": 0.908,
        "price_in_per_mtok": 1.0,
        "price_out_per_mtok": 5.0
      },
      "confusion": {
        "interested": {
          "interested": 7,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 0,
          "not_now": 6,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 7,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 6,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 6,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 6,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 6
        }
      },
      "per_class": {
        "interested": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_now": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_interested": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "referral": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "out_of_office": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "unsubscribe": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "auto_bounce": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        }
      }
    },
    "clean_baseline": {
      "corpus": null,
      "model": "keyword rules (control arm, no model call)",
      "strip_quotes": false,
      "temperature": null,
      "temperature_note": "n/a",
      "n": 44,
      "correct": 42,
      "accuracy": 0.9545,
      "misses": [
        {
          "id": "R-23",
          "gold": "referral",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: patient scheduling",
          "contested": false,
          "note": "Referral plus a genuine endorsement."
        },
        {
          "id": "R-25",
          "gold": "referral",
          "predicted": "out_of_office",
          "confidence": null,
          "evidence": "",
          "subject": "Automatic reply: Bertrand Ashgrove is no longer with Hensleigh Group",
          "contested": false,
          "note": "Left-the-company auto-reply that NAMES a successor. Convention here is referral, not bounce, because there is a live person to route to."
        }
      ],
      "input_tokens": 0,
      "output_tokens": 0,
      "mean_seconds": null,
      "cost": null,
      "confusion": {
        "interested": {
          "interested": 7,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 0,
          "not_now": 6,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 7,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 1,
          "referral": 4,
          "out_of_office": 1,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 6,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 6,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 6
        }
      },
      "per_class": {
        "interested": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_now": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_interested": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.875,
          "recall": 1.0,
          "f1": 0.933
        },
        "referral": {
          "support": 6,
          "true_positives": 4,
          "false_positives": 0,
          "false_negatives": 2,
          "precision": 1.0,
          "recall": 0.667,
          "f1": 0.8
        },
        "out_of_office": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.857,
          "recall": 1.0,
          "f1": 0.923
        },
        "unsubscribe": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "auto_bounce": {
          "support": 6,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        }
      }
    },
    "real_haiku": {
      "corpus": "replies-realistic.json",
      "model": "claude-haiku-4-5",
      "strip_quotes": false,
      "temperature": 0.0,
      "temperature_note": "sent",
      "n": 54,
      "correct": 53,
      "accuracy": 0.9815,
      "misses": [
        {
          "id": "H-46",
          "gold": "referral",
          "predicted": "not_now",
          "confidence": 0.72,
          "evidence": "I have forwarded this internally. If it is of interest to anyone here they will get in touch.",
          "subject": "Re: inbound routing",
          "contested": true,
          "note": "Forwarded to an unnamed person with no commitment. Arguably not_interested with a courtesy, arguably a referral with no name attached."
        }
      ],
      "input_tokens": 47098,
      "output_tokens": 2170,
      "mean_seconds": 1.46,
      "cost": {
        "usd_per_reply": 0.001073,
        "usd_per_1000_replies": 1.073,
        "price_in_per_mtok": 1.0,
        "price_out_per_mtok": 5.0
      },
      "confusion": {
        "interested": {
          "interested": 9,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 0,
          "not_now": 8,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 8,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 1,
          "not_interested": 0,
          "referral": 7,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 7,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 7,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 7
        }
      },
      "per_class": {
        "interested": {
          "support": 9,
          "true_positives": 9,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_now": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.889,
          "recall": 1.0,
          "f1": 0.941
        },
        "not_interested": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "referral": {
          "support": 8,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.875,
          "f1": 0.933
        },
        "out_of_office": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "unsubscribe": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "auto_bounce": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        }
      }
    },
    "real_haiku_stripped": {
      "corpus": "replies-realistic.json",
      "model": "claude-haiku-4-5",
      "strip_quotes": true,
      "temperature": 0.0,
      "temperature_note": "sent",
      "n": 54,
      "correct": 53,
      "accuracy": 0.9815,
      "misses": [
        {
          "id": "H-46",
          "gold": "referral",
          "predicted": "not_now",
          "confidence": 0.72,
          "evidence": "I have forwarded this internally. If it is of interest to anyone here they will get in touch.",
          "subject": "Re: inbound routing",
          "contested": true,
          "note": "Forwarded to an unnamed person with no commitment. Arguably not_interested with a courtesy, arguably a referral with no name attached."
        }
      ],
      "input_tokens": 38793,
      "output_tokens": 2092,
      "mean_seconds": 1.52,
      "cost": {
        "usd_per_reply": 0.000912,
        "usd_per_1000_replies": 0.912,
        "price_in_per_mtok": 1.0,
        "price_out_per_mtok": 5.0
      },
      "confusion": {
        "interested": {
          "interested": 9,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 0,
          "not_now": 8,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 8,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 1,
          "not_interested": 0,
          "referral": 7,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 7,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 7,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 7
        }
      },
      "per_class": {
        "interested": {
          "support": 9,
          "true_positives": 9,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "not_now": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.889,
          "recall": 1.0,
          "f1": 0.941
        },
        "not_interested": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "referral": {
          "support": 8,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.875,
          "f1": 0.933
        },
        "out_of_office": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "unsubscribe": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "auto_bounce": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        }
      }
    },
    "real_opus": {
      "corpus": "replies-realistic.json",
      "model": "claude-opus-5",
      "strip_quotes": false,
      "temperature": null,
      "temperature_note": "not sent: this model rejects sampling parameters with HTTP 400",
      "n": 54,
      "correct": 52,
      "accuracy": 0.963,
      "misses": [
        {
          "id": "H-45",
          "gold": "not_now",
          "predicted": "interested",
          "confidence": 0.45,
          "evidence": "Leave it with me.",
          "subject": "Re: worth fifteen minutes?",
          "contested": true,
          "note": "Reads as a soft defer. Could equally be a polite brush off, which would make it not_interested. Nothing in the text settles it."
        },
        {
          "id": "H-46",
          "gold": "referral",
          "predicted": "not_interested",
          "confidence": 0.58,
          "evidence": "If it is of interest to anyone here they will get in touch.",
          "subject": "Re: inbound routing",
          "contested": true,
          "note": "Forwarded to an unnamed person with no commitment. Arguably not_interested with a courtesy, arguably a referral with no name attached."
        }
      ],
      "input_tokens": 61457,
      "output_tokens": 3463,
      "mean_seconds": 3.43,
      "cost": {
        "usd_per_reply": 0.007294,
        "usd_per_1000_replies": 7.294,
        "price_in_per_mtok": 5.0,
        "price_out_per_mtok": 25.0
      },
      "confusion": {
        "interested": {
          "interested": 9,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 1,
          "not_now": 7,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 8,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 1,
          "referral": 7,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 7,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 7,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 7
        }
      },
      "per_class": {
        "interested": {
          "support": 9,
          "true_positives": 9,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.9,
          "recall": 1.0,
          "f1": 0.947
        },
        "not_now": {
          "support": 8,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.875,
          "f1": 0.933
        },
        "not_interested": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.889,
          "recall": 1.0,
          "f1": 0.941
        },
        "referral": {
          "support": 8,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.875,
          "f1": 0.933
        },
        "out_of_office": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "unsubscribe": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        },
        "auto_bounce": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 0,
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0
        }
      }
    },
    "real_baseline": {
      "corpus": "replies-realistic.json",
      "model": "keyword rules (control arm, no model call)",
      "strip_quotes": false,
      "temperature": null,
      "temperature_note": "n/a",
      "n": 54,
      "correct": 44,
      "accuracy": 0.8148,
      "misses": [
        {
          "id": "H-23",
          "gold": "referral",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: patient scheduling",
          "contested": false,
          "note": "Referral plus a genuine endorsement."
        },
        {
          "id": "H-25",
          "gold": "referral",
          "predicted": "out_of_office",
          "confidence": null,
          "evidence": "",
          "subject": "Automatic reply: Bertrand Ashgrove is no longer with Hensleigh Group",
          "contested": false,
          "note": "Left-the-company auto-reply that NAMES a successor. Convention here is referral, not bounce, because there is a live person to route to."
        },
        {
          "id": "H-45",
          "gold": "not_now",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: worth fifteen minutes?",
          "contested": true,
          "note": "Reads as a soft defer. Could equally be a polite brush off, which would make it not_interested. Nothing in the text settles it."
        },
        {
          "id": "H-46",
          "gold": "referral",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: inbound routing",
          "contested": true,
          "note": "Forwarded to an unnamed person with no commitment. Arguably not_interested with a courtesy, arguably a referral with no name attached."
        },
        {
          "id": "H-47",
          "gold": "interested",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: quick question",
          "contested": true,
          "note": "Two buying questions with an explicit disclaimer. The disclaimer is not a decline, but a reasonable labeller could call it not_now."
        },
        {
          "id": "H-50",
          "gold": "unsubscribe",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: quick question",
          "contested": true,
          "note": "No removal vocabulary at all. It is a removal request by reference to a promise in the original mail, which is the kind of thing a rules pass cannot see."
        },
        {
          "id": "H-51",
          "gold": "not_now",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: patient intake",
          "contested": true,
          "note": "A conditional deferral with no date. The condition is real, so it is not a decline, but there is nothing to schedule a follow up against."
        },
        {
          "id": "H-52",
          "gold": "interested",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: worth fifteen minutes?",
          "contested": true,
          "note": "Accepting one branch of the offer and declining the other. Engagement, but a labeller keying on 'declining a call' could file it as not_interested."
        },
        {
          "id": "H-53",
          "gold": "referral",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Re: inbound routing",
          "contested": true,
          "note": "Redirects without a name, and the redirect target is on the sender's side rather than theirs. Genuinely awkward to route."
        },
        {
          "id": "H-54",
          "gold": "auto_bounce",
          "predicted": "not_interested",
          "confidence": null,
          "evidence": "",
          "subject": "Warning: message delayed",
          "contested": true,
          "note": "A delay warning, not a failure. Suppressing on this would be wrong, and the corpus still files it under auto_bounce because it is a delivery notice with no human in it. This is the clearest case in the set where the label set itself is the limitation."
        }
      ],
      "input_tokens": 0,
      "output_tokens": 0,
      "mean_seconds": null,
      "cost": null,
      "confusion": {
        "interested": {
          "interested": 7,
          "not_now": 0,
          "not_interested": 2,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_now": {
          "interested": 0,
          "not_now": 6,
          "not_interested": 2,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "not_interested": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 8,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "referral": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 3,
          "referral": 4,
          "out_of_office": 1,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "out_of_office": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 0,
          "referral": 0,
          "out_of_office": 7,
          "unsubscribe": 0,
          "auto_bounce": 0
        },
        "unsubscribe": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 1,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 6,
          "auto_bounce": 0
        },
        "auto_bounce": {
          "interested": 0,
          "not_now": 0,
          "not_interested": 1,
          "referral": 0,
          "out_of_office": 0,
          "unsubscribe": 0,
          "auto_bounce": 6
        }
      },
      "per_class": {
        "interested": {
          "support": 9,
          "true_positives": 7,
          "false_positives": 0,
          "false_negatives": 2,
          "precision": 1.0,
          "recall": 0.778,
          "f1": 0.875
        },
        "not_now": {
          "support": 8,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 2,
          "precision": 1.0,
          "recall": 0.75,
          "f1": 0.857
        },
        "not_interested": {
          "support": 8,
          "true_positives": 8,
          "false_positives": 9,
          "false_negatives": 0,
          "precision": 0.471,
          "recall": 1.0,
          "f1": 0.64
        },
        "referral": {
          "support": 8,
          "true_positives": 4,
          "false_positives": 0,
          "false_negatives": 4,
          "precision": 1.0,
          "recall": 0.5,
          "f1": 0.667
        },
        "out_of_office": {
          "support": 7,
          "true_positives": 7,
          "false_positives": 1,
          "false_negatives": 0,
          "precision": 0.875,
          "recall": 1.0,
          "f1": 0.933
        },
        "unsubscribe": {
          "support": 7,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.857,
          "f1": 0.923
        },
        "auto_bounce": {
          "support": 7,
          "true_positives": 6,
          "false_positives": 0,
          "false_negatives": 1,
          "precision": 1.0,
          "recall": 0.857,
          "f1": 0.923
        }
      }
    }
  },
  "stability": {
    "repeats": 5,
    "items": 54,
    "identical_across_runs": 54,
    "unstable": {},
    "accuracy_per_run": [
      0.9815,
      0.9815,
      0.9815,
      0.9815,
      0.9815
    ]
  },
  "threshold_sweep": [
    {
      "threshold": 0.6,
      "sent_to_review": 0,
      "share_of_volume": 0.0,
      "errors_total": 1,
      "errors_caught": 0,
      "errors_reaching_a_destination_unreviewed": 1
    },
    {
      "threshold": 0.7,
      "sent_to_review": 0,
      "share_of_volume": 0.0,
      "errors_total": 1,
      "errors_caught": 0,
      "errors_reaching_a_destination_unreviewed": 1
    },
    {
      "threshold": 0.75,
      "sent_to_review": 1,
      "share_of_volume": 0.0185,
      "errors_total": 1,
      "errors_caught": 1,
      "errors_reaching_a_destination_unreviewed": 0
    },
    {
      "threshold": 0.8,
      "sent_to_review": 1,
      "share_of_volume": 0.0185,
      "errors_total": 1,
      "errors_caught": 1,
      "errors_reaching_a_destination_unreviewed": 0
    },
    {
      "threshold": 0.9,
      "sent_to_review": 5,
      "share_of_volume": 0.0926,
      "errors_total": 1,
      "errors_caught": 1,
      "errors_reaching_a_destination_unreviewed": 0
    },
    {
      "threshold": 0.95,
      "sent_to_review": 9,
      "share_of_volume": 0.1667,
      "errors_total": 1,
      "errors_caught": 1,
      "errors_reaching_a_destination_unreviewed": 0
    }
  ],
  "model_disagreement": {
    "compared": 54,
    "disagreements": [
      {
        "id": "H-45",
        "gold": "not_now",
        "a": "not_now",
        "b": "interested",
        "a_right": true,
        "b_right": false
      },
      {
        "id": "H-46",
        "gold": "referral",
        "a": "not_now",
        "b": "not_interested",
        "a_right": false,
        "b_right": false
      }
    ],
    "share_of_volume": 0.037,
    "items_where_either_arm_is_wrong": [
      "H-45",
      "H-46"
    ],
    "errors_flagged_by_disagreement": [
      "H-45",
      "H-46"
    ]
  },
  "quote_stripping": {
    "accuracy_raw": 0.9815,
    "accuracy_stripped": 0.9815,
    "accuracy_delta": 0.0,
    "input_tokens_raw": 47098,
    "input_tokens_stripped": 38793,
    "input_token_reduction": 0.1763,
    "cost_raw_per_1000": 1.073,
    "cost_stripped_per_1000": 0.912
  },
  "two_stage_design": {
    "gate_confidence_below": 0.9,
    "second_pass_share": 0.0926,
    "second_pass_items": [
      "H-18",
      "H-45",
      "H-46",
      "H-47",
      "H-52"
    ],
    "escalated_to_human": [
      "H-45",
      "H-46"
    ],
    "cheap_model_errors": [
      "H-46"
    ],
    "cheap_errors_inside_second_pass": [
      "H-46"
    ],
    "cheap_errors_reaching_a_destination": [],
    "usd_per_1000_replies": 1.748,
    "caveat": "One error in the cheap arm and two in the strong arm is far too few to state a rate. This shows the mechanism works on this set, not how often it will fire on a real inbox."
  }
}