feat(rag): close eval pipeline with live snapshots

eval/rag-retrieval/reports/baseline.json

@@ -1,11 +1,15 @@
 {
-  "generatedAt": "2026-07-04T17:59:52.172759+00:00",
+  "generatedAt": "2026-07-06T13:37:59.726351+00:00",
   "caseFile": "eval/rag-retrieval/cases/golden-cases.json",
   "fixtureDir": "eval/rag-retrieval/fixtures",
   "aggregate": {
-    "caseCount": 6,
+    "caseCount": 7,
     "topK": 5,
-    "strongHitCount": 6,
+    "passedCount": 7,
+    "failedCount": 0,
+    "passRate": 1.0,
+    "lookupResultCaseCount": 7,
+    "strongHitCount": 7,
     "mediumHitCount": 0,
     "weakHitCount": 0,
     "missCount": 0,
@@ -18,6 +22,7 @@
       "caseId": "chat-mysql-connection-pool",
       "scenario": "chat",
       "query": "MySQL connection pool is exhausted. How should I diagnose it?",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -31,12 +36,22 @@
         "hikaricp"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "mysql-connection-pool",
+        "incident-diagnosis-flow"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "mysql-connection-pool",
       "failedChecks": []
     },
     {
       "caseId": "chat-diagnosis-flow",
       "scenario": "chat",
       "query": "What is the standard troubleshooting flow for an application incident?",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -50,12 +65,22 @@
         "remediation"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "incident-diagnosis-flow",
+        "rag-chunk-context-reconstruction"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "incident-diagnosis-flow",
       "failedChecks": []
     },
     {
       "caseId": "aiops-payment-latency-alert",
       "scenario": "aiops",
       "query": "Alert HighLatency on payment-service with p95 latency above threshold",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -69,12 +94,22 @@
         "downstream dependency"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "payment-service-latency",
+        "mysql-connection-pool"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "payment-service-latency",
       "failedChecks": []
     },
     {
       "caseId": "aiops-prometheus-alert-scope",
       "scenario": "aiops",
       "query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -87,12 +122,21 @@
         "scope"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "aiops-alert-scope-control"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "aiops-alert-scope-control",
       "failedChecks": []
     },
     {
       "caseId": "chat-rag-chunk-context",
       "scenario": "chat",
       "query": "If a long section is split into multiple chunks, how do we keep retrieval context?",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -106,12 +150,22 @@
         "breadcrumb"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "rag-chunk-context-reconstruction",
+        "rag-breadcrumb-embedding-gap"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "rag-chunk-context-reconstruction",
       "failedChecks": []
     },
     {
       "caseId": "chat-l0-domain-hint",
       "scenario": "chat",
       "query": "Should L0 keyword matching decide the final retrieval result?",
+      "dataShape": "lookupResult",
       "hitLevel": "strong",
       "passed": true,
       "firstExpectedRank": 1,
@@ -125,6 +179,44 @@
         "metadata filter"
       ],
       "breadcrumbMatched": true,
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "rag-l0-domain-entity-hint",
+        "rag-l0-l1-fusion-ranking"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "rag-l0-domain-entity-hint",
+      "failedChecks": []
+    },
+    {
+      "caseId": "chat-l0-filter-fallback",
+      "scenario": "chat",
+      "query": "RAG query was over-filtered by L0 and filtered vector search returned low quality evidence. What should happen?",
+      "dataShape": "lookupResult",
+      "hitLevel": "strong",
+      "passed": true,
+      "firstExpectedRank": 1,
+      "topCandidates": [
+        "1:rag-l0-filter-fallback",
+        "2:rag-l0-domain-entity-hint"
+      ],
+      "matchedKeywords": [
+        "skip the l0 filter",
+        "unfiltered vector retry",
+        "low quality"
+      ],
+      "breadcrumbMatched": true,
+      "selectedAttempt": "UNFILTERED_VECTOR_RETRY",
+      "fallbackReason": "filtered_vector_low_quality",
+      "evidenceStatus": "supported",
+      "includedSources": [
+        "rag-l0-filter-fallback",
+        "rag-l0-domain-entity-hint"
+      ],
+      "omittedSources": [],
+      "rerankTopSource": "rag-l0-filter-fallback",
       "failedChecks": []
     }
   ]