feat(rag): close eval pipeline with live snapshots

2026-07-06 21:39:27 +08:00
parent cf3333d607
commit ed7efc58b7
47 changed files with 2613 additions and 177 deletions
@@ -1,16 +1,65 @@
 {
  "caseId": "aiops-prometheus-alert-scope",
  "query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
-  "retrievedAt": "2026-07-05T00:00:00Z",
-  "candidates": [
-    {
-      "rank": 1,
-      "docId": "aiops-alert-scope-control",
-      "title": "AIOps Alert Scope Control",
-      "breadcrumb": "AIOps > Alert Scope Control",
-      "content": "When payload mode is active, queryPrometheusAlerts can verify the supplied alert, but unrelated active alerts must remain scoped context and should not become full diagnoses.",
-      "score": 0.9,
-      "retrievalLayer": "L0+L1"
+  "retrievedAt": "2026-07-06T00:00:00Z",
+  "lookupResult": {
+    "found": true,
+    "evidenceBlocks": [
+      {
+        "source": "aiops-alert-scope-control",
+        "title": "AIOps Alert Scope Control",
+        "breadcrumb": "AIOps > Alert Scope Control",
+        "retrievalLayer": "L1",
+        "content": "When payload mode is used, diagnose the input alert payload and do not expand unrelated active alerts into the main diagnosis scope.",
+        "score": 0.88,
+        "hitReasons": ["domain_match:+0.15", "keyword_match:+0.10"]
+      }
+    ],
+    "contextPack": {
+      "packedText": "[1] AIOps Alert Scope Control\nAIOps > Alert Scope Control\nWhen payload mode is used, diagnose the input alert payload and do not expand unrelated active alerts into the main diagnosis scope.",
+      "strategy": "top_evidence_blocks",
+      "charBudget": 3500,
+      "usedChars": 188,
+      "includedSources": ["aiops-alert-scope-control"],
+      "omittedSources": []
+    },
+    "retrievalTrace": {
+      "originalQuery": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
+      "rewrittenQuery": "AIOps alert payload scope unrelated active alerts diagnosis",
+      "categoryFilter": "AIOps",
+      "selectedAttempt": "FILTERED_VECTOR",
+      "fallbackReason": null,
+      "evidenceStatus": "supported",
+      "queryHints": {
+        "domains": ["AIOps"],
+        "matched_keywords": ["payload", "unrelated active alerts", "scope"],
+        "entities": ["alert payload"],
+        "l0_titles": ["AIOps Alert Scope Control"],
+        "l0_match_count": 1
+      },
+      "attempts": [
+        {
+          "name": "FILTERED_VECTOR",
+          "query": "AIOps alert payload scope unrelated active alerts diagnosis",
+          "categoryFilter": "AIOps",
+          "candidateCount": 1,
+          "usable": true,
+          "durationMs": 8,
+          "topScore": 0.88,
+          "topSimilarity": 0.88
+        }
+      ]
+    },
+    "rerankTrace": {
+      "items": [
+        {
+          "finalRank": 1,
+          "source": "aiops-alert-scope-control",
+          "baseScore": 0.88,
+          "finalScore": 1.13,
+          "boostReasons": ["domain_match:+0.15", "keyword_match:+0.10"]
+        }
+      ]
    }
-  ]
+  }
 }