{
 "datasets": {
  "banking77": {
   "title": "Support triage",
   "detail": "Banking77: 3,080 held-out bank-support messages, 77 categories",
   "source": "PolyAI Banking77, CC BY 4.0",
   "models": [
    {
     "key": "ours",
     "label": "Qwen3-4B + LoRA (ours)",
     "kind": "owned",
     "prompt": "fine-tuned, one L4",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.939935,
     "ci": [
      0.931494,
      0.948052
     ],
     "macro_f1": 0.94,
     "invalid": 0,
     "failed": 0
    },
    {
     "key": "emb-bge-base",
     "label": "bge-base embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.93539,
     "ci": [
      0.926623,
      0.944156
     ],
     "macro_f1": 0.9353,
     "invalid": 0,
     "failed": 0,
     "ours_minus": {
      "diff": 0.004545,
      "ci": [
       -0.002922,
       0.012013
      ]
     }
    },
    {
     "key": "emb-bge-small",
     "label": "bge-small embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.934091,
     "ci": [
      0.925,
      0.942857
     ],
     "macro_f1": 0.934,
     "invalid": 0,
     "failed": 0,
     "ours_minus": {
      "diff": 0.005844,
      "ci": [
       -0.001948,
       0.013312
      ]
     }
    },
    {
     "key": "emb-minilm",
     "label": "MiniLM embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.927922,
     "ci": [
      0.918506,
      0.937013
     ],
     "macro_f1": 0.9278,
     "invalid": 0,
     "failed": 0,
     "ours_minus": {
      "diff": 0.012013,
      "ci": [
       0.003896,
       0.019805
      ]
     }
    },
    {
     "key": "tfidf",
     "label": "TF-IDF + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.892532,
     "ci": [
      0.881494,
      0.903247
     ],
     "macro_f1": 0.893,
     "invalid": 0,
     "failed": 0,
     "ours_minus": {
      "diff": 0.047403,
      "ci": [
       0.037005,
       0.057792
      ]
     }
    },
    {
     "key": "luna56-shot",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.872078,
     "ci": [
      0.86039,
      0.883766
     ],
     "macro_f1": 0.8688,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 978,
     "latency_ms_p95": 1368,
     "ours_minus": {
      "diff": 0.067857,
      "ci": [
       0.056818,
       0.079221
      ]
     }
    },
    {
     "key": "luna56-list",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.853571,
     "ci": [
      0.841234,
      0.866234
     ],
     "macro_f1": 0.8463,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 863,
     "latency_ms_p95": 1194,
     "ours_minus": {
      "diff": 0.086364,
      "ci": [
       0.074675,
       0.098377
      ]
     }
    },
    {
     "key": "luna6-shot",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.821753,
     "ci": [
      0.808117,
      0.83539
     ],
     "macro_f1": 0.8187,
     "invalid": 4,
     "failed": 0,
     "latency_ms_p50": 1117,
     "latency_ms_p95": 2391,
     "ours_minus": {
      "diff": 0.118182,
      "ci": [
       0.105195,
       0.131177
      ]
     }
    },
    {
     "key": "luna6-list",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.797403,
     "ci": [
      0.782792,
      0.811372
     ],
     "macro_f1": 0.7889,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 989,
     "latency_ms_p95": 2414,
     "ours_minus": {
      "diff": 0.142532,
      "ci": [
       0.128571,
       0.156169
      ]
     }
    },
    {
     "key": "gpt41mini-shot",
     "label": "GPT-4.1-mini",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.786688,
     "ci": [
      0.772078,
      0.800974
     ],
     "macro_f1": 0.779,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 735,
     "latency_ms_p95": 1912,
     "ours_minus": {
      "diff": 0.153247,
      "ci": [
       0.138961,
       0.167541
      ]
     }
    },
    {
     "key": "gpt41mini-list",
     "label": "GPT-4.1-mini",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.748701,
     "ci": [
      0.733442,
      0.764286
     ],
     "macro_f1": 0.7378,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 552,
     "latency_ms_p95": 1718,
     "ours_minus": {
      "diff": 0.191234,
      "ci": [
       0.175649,
       0.206494
      ]
     }
    },
    {
     "key": "oss120b-shot",
     "label": "gpt-oss-120b",
     "kind": "open-big",
     "prompt": "an example of every category, one H100",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.855519,
     "ci": [
      0.842857,
      0.868182
     ],
     "macro_f1": 0.8499,
     "invalid": 1,
     "failed": 1,
     "ours_minus": {
      "diff": 0.084416,
      "ci": [
       0.072078,
       0.096429
      ]
     }
    },
    {
     "key": "oss120b-list",
     "label": "gpt-oss-120b",
     "kind": "open-big",
     "prompt": "category list only, one H100",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.820455,
     "ci": [
      0.806818,
      0.834091
     ],
     "macro_f1": 0.8132,
     "invalid": 0,
     "failed": 0,
     "ours_minus": {
      "diff": 0.119481,
      "ci": [
       0.106169,
       0.133117
      ]
     }
    },
    {
     "key": "luna56-schema",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "category list, Structured Outputs",
     "status": "not_run",
     "note": "not run: it didn't fit the test's $20 budget, set before any results"
    },
    {
     "key": "luna56-near20",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.937987,
     "ci": [
      0.929221,
      0.946429
     ],
     "macro_f1": 0.938,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 812,
     "latency_ms_p95": 1158,
     "ours_minus": {
      "diff": 0.001948,
      "ci": [
       -0.005519,
       0.009091
      ]
     }
    },
    {
     "key": "luna56-near20-low",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs, reasoning effort low",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.935065,
     "ci": [
      0.926299,
      0.943506
     ],
     "macro_f1": 0.935,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 847,
     "latency_ms_p95": 1424,
     "ours_minus": {
      "diff": 0.00487,
      "ci": [
       -0.002922,
       0.012662
      ]
     }
    },
    {
     "key": "luna6-schema",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "category list, Structured Outputs",
     "status": "not_run",
     "note": "not run: it didn't fit the test's $20 budget, set before any results"
    },
    {
     "key": "luna6-near20",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.930844,
     "ci": [
      0.921429,
      0.93961
     ],
     "macro_f1": 0.9309,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 764,
     "latency_ms_p95": 1554,
     "ours_minus": {
      "diff": 0.009091,
      "ci": [
       0.000649,
       0.017208
      ]
     }
    },
    {
     "key": "luna6-near20-low",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs, reasoning effort low",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.911364,
     "ci": [
      0.901299,
      0.921104
     ],
     "macro_f1": 0.9093,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 983,
     "latency_ms_p95": 2021,
     "ours_minus": {
      "diff": 0.028571,
      "ci": [
       0.018831,
       0.037987
      ]
     }
    },
    {
     "key": "decisions-luna6",
     "label": "OpenAI Decisions API (GPT-6 Luna)",
     "kind": "api",
     "prompt": "Decisions API, the 77 categories as choices",
     "status": "ok",
     "n": 3080,
     "accuracy": 0.767532,
     "ci": [
      0.752597,
      0.782468
     ],
     "macro_f1": 0.7631,
     "invalid": 10,
     "failed": 0,
     "latency_ms_p50": 120,
     "latency_ms_p95": 246,
     "ours_minus": {
      "diff": 0.172403,
      "ci": [
       0.158117,
       0.187013
      ]
     }
    }
   ]
  },
  "clinc150": {
   "title": "Intents + off-topic",
   "detail": "CLINC150: 5,500 held-out assistant requests, 150 intents plus out-of-scope",
   "source": "Clinc CLINC150, CC BY 3.0",
   "models": [
    {
     "key": "ours",
     "label": "Qwen3-4B + LoRA (ours)",
     "kind": "owned",
     "prompt": "fine-tuned, one L4",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.951455,
     "ci": [
      0.945818,
      0.957091
     ],
     "macro_f1": 0.9615,
     "invalid": 0,
     "failed": 0,
     "oos_recall": 0.839,
     "oos_recall_ci": [
      0.816,
      0.862
     ],
     "false_oos": 0.004,
     "false_oos_ci": [
      0.0022,
      0.006
     ]
    },
    {
     "key": "emb-bge-base",
     "label": "bge-base embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.906,
     "ci": [
      0.898182,
      0.913636
     ],
     "macro_f1": 0.932,
     "invalid": 0,
     "failed": 0,
     "oos_recall": 0.629,
     "oos_recall_ci": [
      0.599,
      0.659
     ],
     "false_oos": 0.0036,
     "false_oos_ci": [
      0.002,
      0.0053
     ],
     "ours_minus": {
      "diff": 0.045455,
      "ci": [
       0.038364,
       0.052545
      ]
     }
    },
    {
     "key": "emb-bge-small",
     "label": "bge-small embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.893455,
     "ci": [
      0.885273,
      0.901818
     ],
     "macro_f1": 0.922,
     "invalid": 0,
     "failed": 0,
     "oos_recall": 0.586,
     "oos_recall_ci": [
      0.556,
      0.616
     ],
     "false_oos": 0.0033,
     "false_oos_ci": [
      0.0018,
      0.0051
     ],
     "ours_minus": {
      "diff": 0.058,
      "ci": [
       0.050364,
       0.065636
      ]
     }
    },
    {
     "key": "emb-minilm",
     "label": "MiniLM embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.888909,
     "ci": [
      0.880545,
      0.897091
     ],
     "macro_f1": 0.9157,
     "invalid": 0,
     "failed": 0,
     "oos_recall": 0.605,
     "oos_recall_ci": [
      0.575,
      0.635
     ],
     "false_oos": 0.0049,
     "false_oos_ci": [
      0.0029,
      0.0069
     ],
     "ours_minus": {
      "diff": 0.062545,
      "ci": [
       0.054727,
       0.070364
      ]
     }
    },
    {
     "key": "tfidf",
     "label": "TF-IDF + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.815636,
     "ci": [
      0.805273,
      0.825818
     ],
     "macro_f1": 0.8574,
     "invalid": 0,
     "failed": 0,
     "oos_recall": 0.409,
     "oos_recall_ci": [
      0.379,
      0.44
     ],
     "false_oos": 0.0124,
     "false_oos_ci": [
      0.0093,
      0.0158
     ],
     "ours_minus": {
      "diff": 0.135818,
      "ci": [
       0.126,
       0.146
      ]
     }
    },
    {
     "key": "luna56-shot",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "incomplete",
     "answered": 339,
     "n": 5500,
     "note": "stopped part-way when our prepaid OpenAI credit ran out on 8 October; not resumed"
    },
    {
     "key": "luna56-list",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.883636,
     "ci": [
      0.875091,
      0.892
     ],
     "macro_f1": 0.8927,
     "invalid": 15,
     "failed": 0,
     "latency_ms_p50": 825,
     "latency_ms_p95": 1135,
     "oos_recall": 0.778,
     "oos_recall_ci": [
      0.752,
      0.804
     ],
     "false_oos": 0.0231,
     "false_oos_ci": [
      0.0189,
      0.0276
     ],
     "ours_minus": {
      "diff": 0.067818,
      "ci": [
       0.059455,
       0.076182
      ]
     }
    },
    {
     "key": "luna6-shot",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "incomplete",
     "answered": 92,
     "n": 5500,
     "note": "stopped part-way when our prepaid OpenAI credit ran out on 8 October; not resumed"
    },
    {
     "key": "luna6-list",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.883091,
     "ci": [
      0.874364,
      0.891636
     ],
     "macro_f1": 0.8853,
     "invalid": 6,
     "failed": 0,
     "latency_ms_p50": 907,
     "latency_ms_p95": 2079,
     "oos_recall": 0.9,
     "oos_recall_ci": [
      0.881,
      0.918
     ],
     "false_oos": 0.0549,
     "false_oos_ci": [
      0.0484,
      0.0618
     ],
     "ours_minus": {
      "diff": 0.068364,
      "ci": [
       0.059636,
       0.077273
      ]
     }
    },
    {
     "key": "luna56-near20",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.946364,
     "ci": [
      0.940182,
      0.952364
     ],
     "macro_f1": 0.9604,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 830,
     "latency_ms_p95": 1190,
     "oos_recall": 0.791,
     "oos_recall_ci": [
      0.766,
      0.816
     ],
     "false_oos": 0.0018,
     "false_oos_ci": [
      0.0007,
      0.0031
     ],
     "ours_minus": {
      "diff": 0.005091,
      "ci": [
       -0.000364,
       0.010364
      ]
     }
    },
    {
     "key": "luna6-near20",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "ok",
     "n": 5500,
     "accuracy": 0.944182,
     "ci": [
      0.938,
      0.950364
     ],
     "macro_f1": 0.9579,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 840,
     "latency_ms_p95": 1396,
     "oos_recall": 0.799,
     "oos_recall_ci": [
      0.774,
      0.823
     ],
     "false_oos": 0.0042,
     "false_oos_ci": [
      0.0024,
      0.0062
     ],
     "ours_minus": {
      "diff": 0.007273,
      "ci": [
       0.001636,
       0.012727
      ]
     }
    }
   ]
  },
  "cfpb-product": {
   "title": "Long real complaints",
   "detail": "CFPB: 5,117 held-out consumer complaints, 10 products (renamed products merged)",
   "source": "US Consumer Financial Protection Bureau, public domain",
   "models": [
    {
     "key": "ours",
     "label": "Qwen3-4B + LoRA (ours)",
     "kind": "owned",
     "prompt": "fine-tuned, one H100",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.895251,
     "ci": [
      0.886848,
      0.903459
     ],
     "macro_f1": 0.7194,
     "invalid": 0,
     "failed": 0,
     "accuracy_as_filed": 0.670119
    },
    {
     "key": "emb-bge-base",
     "label": "bge-base embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.861833,
     "ci": [
      0.852453,
      0.871214
     ],
     "macro_f1": 0.647,
     "invalid": 0,
     "failed": 0,
     "accuracy_as_filed": 0.642955,
     "ours_minus": {
      "diff": 0.033418,
      "ci": [
       0.025601,
       0.041235
      ]
     }
    },
    {
     "key": "emb-bge-small",
     "label": "bge-small embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.863787,
     "ci": [
      0.854597,
      0.872972
     ],
     "macro_f1": 0.6265,
     "invalid": 0,
     "failed": 0,
     "accuracy_as_filed": 0.636506,
     "ours_minus": {
      "diff": 0.031464,
      "ci": [
       0.023256,
       0.039476
      ]
     }
    },
    {
     "key": "emb-minilm",
     "label": "MiniLM embeddings + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.856557,
     "ci": [
      0.846981,
      0.866132
     ],
     "macro_f1": 0.6334,
     "invalid": 0,
     "failed": 0,
     "accuracy_as_filed": 0.629666,
     "ours_minus": {
      "diff": 0.038695,
      "ci": [
       0.030487,
       0.046902
      ]
     }
    },
    {
     "key": "tfidf",
     "label": "TF-IDF + classifier",
     "kind": "owned",
     "prompt": "trained, runs on a CPU",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.865742,
     "ci": [
      0.856557,
      0.874927
     ],
     "macro_f1": 0.6145,
     "invalid": 0,
     "failed": 0,
     "accuracy_as_filed": 0.677155,
     "ours_minus": {
      "diff": 0.029509,
      "ci": [
       0.021497,
       0.037522
      ]
     }
    },
    {
     "key": "luna56-shot",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "incomplete",
     "answered": 484,
     "n": 5117,
     "note": "stopped part-way when our prepaid OpenAI credit ran out on 8 October; not resumed"
    },
    {
     "key": "luna56-list",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.866328,
     "ci": [
      0.857143,
      0.875708
     ],
     "macro_f1": 0.6976,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 851,
     "latency_ms_p95": 1121,
     "accuracy_as_filed": 0.512214,
     "ours_minus": {
      "diff": 0.028923,
      "ci": [
       0.020715,
       0.037327
      ]
     }
    },
    {
     "key": "luna6-shot",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "an example of every category",
     "status": "incomplete",
     "answered": 203,
     "n": 5117,
     "note": "stopped part-way when our prepaid OpenAI credit ran out on 8 October; not resumed"
    },
    {
     "key": "luna6-list",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "category list only",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.86027,
     "ci": [
      0.850694,
      0.869846
     ],
     "macro_f1": 0.6823,
     "invalid": 2,
     "failed": 0,
     "latency_ms_p50": 944,
     "latency_ms_p95": 1867,
     "accuracy_as_filed": 0.550322,
     "ours_minus": {
      "diff": 0.034981,
      "ci": [
       0.026383,
       0.04358
      ]
     }
    },
    {
     "key": "luna56-near20",
     "label": "GPT-5.6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "ok",
     "n": 5117,
     "accuracy": 0.874927,
     "ci": [
      0.865937,
      0.883916
     ],
     "macro_f1": 0.7016,
     "invalid": 0,
     "failed": 0,
     "latency_ms_p50": 902,
     "latency_ms_p95": 1225,
     "accuracy_as_filed": 0.623217,
     "ours_minus": {
      "diff": 0.020324,
      "ci": [
       0.011921,
       0.028532
      ]
     }
    },
    {
     "key": "luna6-near20",
     "label": "GPT-6 Luna",
     "kind": "api",
     "prompt": "20 most similar training examples, Structured Outputs",
     "status": "not_run",
     "note": "not run: it didn't fit the test's $50 budget, set before any results"
    }
   ]
  }
 },
 "generated": "2026-10-09",
 "method": "Accuracy with 95% bootstrap CI (10,000 resamples). 'Ours minus' is a paired bootstrap on the same test items. Prompted models: reasoning effort none unless the row says otherwise, temperature default, answers matched to the label list (held to it by Structured Outputs or the Decisions API where the row says so)."
}