{
  "a4_pass_agreement_detail": [
    {
      "agree": true,
      "pass1_rank": 208,
      "pass2_rank": 208,
      "qid": "01_cloudflare_outage_2025-11-18.md"
    },
    {
      "agree": false,
      "pass1_rank": 4,
      "pass2_rank": 5,
      "qid": "02_gitlab_db_outage_postmortem_2017-01-31.md"
    },
    {
      "agree": true,
      "pass1_rank": 364,
      "pass2_rank": 364,
      "qid": "03_gitlab_issue_17057.md"
    },
    {
      "agree": true,
      "pass1_rank": 8,
      "pass2_rank": 8,
      "qid": "05_gitlab_issue_18596.md"
    },
    {
      "agree": true,
      "pass1_rank": 6,
      "pass2_rank": 6,
      "qid": "06_gitlab_issue_15999.md"
    },
    {
      "agree": true,
      "pass1_rank": 21,
      "pass2_rank": 21,
      "qid": "07_aws_s3_disruption_2017-02-28.md"
    },
    {
      "agree": false,
      "pass1_rank": 5,
      "pass2_rank": 6,
      "qid": "10_backstage_adr005_catalog_core_entities.md"
    },
    {
      "agree": true,
      "pass1_rank": 15,
      "pass2_rank": 15,
      "qid": "13_twilio_billing_postmortem_2013.md"
    }
  ],
  "apparatus": {
    "a1_a3_double_pass_bit_identical": true,
    "a4_endpoint": "https://api.tokenfactory.uk-south1.nebius.com/v1/chat/completions",
    "a4_gold_rank_agreement": "6/8",
    "a4_model": "deepseek-ai/DeepSeek-V4-Pro",
    "a4_total_calls_pass1": 400,
    "a4_total_calls_pass2": 400,
    "datadir_copy": "/Users/jgray/projects/ai/ngramdb/scratch/cqr_datadir_baseline",
    "datadir_copy_hash": "0a4796edd7f07ae8b2d84d22d77b1202c7ff01d09cd6becd11fbf3e6a57f4afa",
    "datadir_copy_hash_unchanged_final": true,
    "datadir_source": "/Users/jgray/projects/ai/ngramdb/scratch/cqr_datadir_primary",
    "dials": {
      "d1_cross_encoder": {
        "device": "cpu",
        "dtype": "fp32",
        "model": "BAAI/bge-reranker-v2-m3",
        "revision": "953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e"
      },
      "d2_rerank_depth": {
        "a3_topk": 100,
        "a4_topk": 50
      },
      "d3_a4_prompt": {
        "system": "You score how well a passage answers a question. Reply with only an integer 0-100.",
        "user_template": "Question: {question}\n\nPassage: {passage}\n\nScore 0-100 how directly this passage states the answer to the question. 100 = this passage is the answer. 0 = unrelated. Reply with only the integer."
      },
      "d4_bm25": {
        "b": 0.75,
        "k1": 1.2,
        "tokenizer": "lowercase + \\w+, no stemming, no stopwords"
      },
      "rrf_k": 60
    },
    "exclusion_clause": "No causal-augmented-embedding arm run here, per spec S4 frozen exclusion clause.",
    "git_sha": "5f849f9d220199f337421b0289a0f213dc60a6b8",
    "n_engrams": 657,
    "reference_rows_source": "cqr_scoring_v1.json:r7_per_question / r7_summary_pooled (no rerun, per spec S2/S6)"
  },
  "metrics_table": {
    "a1_bm25": {
      "decoy_cleared": "0/8",
      "mean_latency_s": 0.0030840933322906494,
      "mean_rank": 113.625,
      "median_rank": 17.0,
      "mrr": 0.07231700134660661,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.25,
      "recall@20": 0.625,
      "recall@5": 0.125
    },
    "a2_hybrid_rrf": {
      "decoy_cleared": "0/8",
      "mean_latency_s": 0.0004260241985321045,
      "mean_rank": 86.125,
      "median_rank": 26.5,
      "mrr": 0.08459742171588101,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.25,
      "recall@20": 0.375,
      "recall@5": 0.125
    },
    "a3_cross_encoder": {
      "decoy_cleared": "0/8",
      "mean_latency_s": 10.676091372966766,
      "mean_rank": 95,
      "median_rank": 51.5,
      "mrr": 0.06208986326173826,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.375,
      "recall@20": 0.375,
      "recall@5": 0.0
    },
    "a4_llm_ceiling": {
      "decoy_cleared": "0/8",
      "mean_latency_s": 0.6239713823795319,
      "mean_rank": 78.875,
      "median_rank": 11.5,
      "mrr": 0.10793841575091576,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.5,
      "recall@20": 0.625,
      "recall@5": 0.25
    },
    "reference_ablated": {
      "decoy_cleared": "n/a (no decoy data)",
      "mean_rank": 97.25,
      "median_rank": 74.0,
      "mrr": 0.013146141047303952,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.0,
      "recall@20": 0.0,
      "recall@5": 0.0
    },
    "reference_graph": {
      "decoy_cleared": "0/8",
      "mean_rank": 87.25,
      "median_rank": 77.0,
      "mrr": 0.013451714985370183,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.0,
      "recall@20": 0.0,
      "recall@5": 0.0
    },
    "reference_similarity": {
      "decoy_cleared": "0/8",
      "mean_rank": 58.375,
      "median_rank": 28.0,
      "mrr": 0.11355433927583895,
      "n": 8,
      "recall@1": 0.0,
      "recall@10": 0.25,
      "recall@20": 0.375,
      "recall@5": 0.25
    }
  },
  "questions": [
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 119,
      "gold_engram_ids": [
        "3eb8f056-59d6-4c74-add5-1aa58195d41e",
        "911ef837-6af7-425e-9f8e-10cb3a1ff6c3"
      ],
      "qid": "01_cloudflare_outage_2025-11-18.md",
      "rank_a1_bm25": 266,
      "rank_a2_hybrid": 208,
      "rank_a3_cross_encoder": 208,
      "rank_a4_llm_pass1": 208,
      "rank_a4_llm_pass2": 208
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 151,
      "gold_engram_ids": [
        "47d94f7e-f3ef-4c90-a366-1316c10ef489",
        "599bc853-14a6-47e6-8a92-277426dd04b6"
      ],
      "qid": "02_gitlab_db_outage_postmortem_2017-01-31.md",
      "rank_a1_bm25": 16,
      "rank_a2_hybrid": 7,
      "rank_a3_cross_encoder": 8,
      "rank_a4_llm_pass1": 4,
      "rank_a4_llm_pass2": 5
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 47,
      "gold_engram_ids": [
        "4a9386f8-778d-4cc8-aed2-6a317d7034d2",
        "ce0d0527-dcea-45f9-9a61-02d2e5d73557"
      ],
      "qid": "03_gitlab_issue_17057.md",
      "rank_a1_bm25": 495,
      "rank_a2_hybrid": 364,
      "rank_a3_cross_encoder": 364,
      "rank_a4_llm_pass1": 364,
      "rank_a4_llm_pass2": 364
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 11,
      "gold_engram_ids": [
        "373ffc76-8877-4803-b836-2f026f928be9"
      ],
      "qid": "05_gitlab_issue_18596.md",
      "rank_a1_bm25": 18,
      "rank_a2_hybrid": 32,
      "rank_a3_cross_encoder": 55,
      "rank_a4_llm_pass1": 8,
      "rank_a4_llm_pass2": 8
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 53,
      "gold_engram_ids": [
        "dc9f2dd5-e8f3-4b68-8623-d03f67dfe989"
      ],
      "qid": "06_gitlab_issue_15999.md",
      "rank_a1_bm25": 90,
      "rank_a2_hybrid": 43,
      "rank_a3_cross_encoder": 48,
      "rank_a4_llm_pass1": 6,
      "rank_a4_llm_pass2": 6
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 41,
      "gold_engram_ids": [
        "780c2067-ca32-4ea0-8fe2-382556793379"
      ],
      "qid": "07_aws_s3_disruption_2017-02-28.md",
      "rank_a1_bm25": 13,
      "rank_a2_hybrid": 21,
      "rank_a3_cross_encoder": 64,
      "rank_a4_llm_pass1": 21,
      "rank_a4_llm_pass2": 21
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 23,
      "gold_engram_ids": [
        "ccf6595d-2304-4dd6-beba-966ba0aefa98"
      ],
      "qid": "10_backstage_adr005_catalog_core_entities.md",
      "rank_a1_bm25": 5,
      "rank_a2_hybrid": 3,
      "rank_a3_cross_encoder": 7,
      "rank_a4_llm_pass1": 5,
      "rank_a4_llm_pass2": 6
    },
    {
      "decoy_best_rank_a1": 1,
      "decoy_best_rank_a2": 1,
      "decoy_best_rank_a3": 1,
      "decoy_best_rank_a4": 1,
      "decoy_count": 51,
      "gold_engram_ids": [
        "0a031d7d-7063-4c47-ba6d-0b9e4a5594ef",
        "d47de9e7-f6c2-454b-bc86-8f40351b1609"
      ],
      "qid": "13_twilio_billing_postmortem_2013.md",
      "rank_a1_bm25": 6,
      "rank_a2_hybrid": 11,
      "rank_a3_cross_encoder": 6,
      "rank_a4_llm_pass1": 15,
      "rank_a4_llm_pass2": 15
    }
  ]
}