test: add rag retrieval baseline

eval/rag-retrieval/fixtures/aiops-payment-latency-alert.json

@@ -0,0 +1,25 @@
+{
+  "caseId": "aiops-payment-latency-alert",
+  "query": "Alert HighLatency on payment-service with p95 latency above threshold",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "payment-service-latency",
+      "title": "Payment Service Latency Alert Playbook",
+      "breadcrumb": "AIOps > Service Alerts > Payment Latency",
+      "content": "For payment-service p95 latency alerts, check downstream dependency latency, thread pool saturation, gateway retries, and recent deployment changes.",
+      "score": 0.84,
+      "retrievalLayer": "L1"
+    },
+    {
+      "rank": 2,
+      "docId": "mysql-connection-pool",
+      "title": "MySQL Connection Pool Troubleshooting",
+      "breadcrumb": "Database > MySQL > Connection Pool",
+      "content": "Database connection pool saturation can increase payment latency when checkout paths wait for connections.",
+      "score": 0.68,
+      "retrievalLayer": "L1"
+    }
+  ]
+}

eval/rag-retrieval/fixtures/aiops-prometheus-alert-scope.json

@@ -0,0 +1,16 @@
+{
+  "caseId": "aiops-prometheus-alert-scope",
+  "query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "aiops-alert-scope-control",
+      "title": "AIOps Alert Scope Control",
+      "breadcrumb": "AIOps > Alert Scope Control",
+      "content": "When payload mode is active, queryPrometheusAlerts can verify the supplied alert, but unrelated active alerts must remain scoped context and should not become full diagnoses.",
+      "score": 0.9,
+      "retrievalLayer": "L0+L1"
+    }
+  ]
+}

eval/rag-retrieval/fixtures/chat-diagnosis-flow.json

@@ -0,0 +1,25 @@
+{
+  "caseId": "chat-diagnosis-flow",
+  "query": "What is the standard troubleshooting flow for an application incident?",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "incident-diagnosis-flow",
+      "title": "Incident Diagnosis Flow",
+      "breadcrumb": "AIOps > Diagnosis Flow",
+      "content": "The standard flow is to collect evidence, identify the suspected fault domain, verify the hypothesis, apply remediation, and confirm recovery.",
+      "score": 0.82,
+      "retrievalLayer": "L1"
+    },
+    {
+      "rank": 2,
+      "docId": "rag-chunk-context-reconstruction",
+      "title": "RAG Chunk Context Reconstruction",
+      "breadcrumb": "RAG > Chunking > Context Reconstruction",
+      "content": "Long sections may require neighbor chunk expansion and breadcrumb-aware packing.",
+      "score": 0.55,
+      "retrievalLayer": "L1"
+    }
+  ]
+}

eval/rag-retrieval/fixtures/chat-l0-domain-hint.json

@@ -0,0 +1,25 @@
+{
+  "caseId": "chat-l0-domain-hint",
+  "query": "Should L0 keyword matching decide the final retrieval result?",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "rag-l0-domain-entity-hint",
+      "title": "RAG L0 Domain Entity Hint",
+      "breadcrumb": "RAG > L0 > Domain Entity Hint",
+      "content": "L0 should be retained as a domain detector, entity extractor, metadata filter generator, and explainability signal, not as the final retrieval decision.",
+      "score": 0.88,
+      "retrievalLayer": "L0"
+    },
+    {
+      "rank": 2,
+      "docId": "rag-l0-l1-fusion-ranking",
+      "title": "RAG L0 L1 Fusion Ranking",
+      "breadcrumb": "RAG > Ranking > Fusion",
+      "content": "L0 and L1 candidates should eventually be fused rather than handled as an early-return branch.",
+      "score": 0.75,
+      "retrievalLayer": "L1"
+    }
+  ]
+}

eval/rag-retrieval/fixtures/chat-mysql-connection-pool.json

@@ -0,0 +1,25 @@
+{
+  "caseId": "chat-mysql-connection-pool",
+  "query": "MySQL connection pool is exhausted. How should I diagnose it?",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "mysql-connection-pool",
+      "title": "MySQL Connection Pool Troubleshooting",
+      "breadcrumb": "Database > MySQL > Connection Pool",
+      "content": "When the connection pool is exhausted, inspect HikariCP active connections, max_connections, slow SQL, leak detection, and database wait events.",
+      "score": 0.86,
+      "retrievalLayer": "L0+L1"
+    },
+    {
+      "rank": 2,
+      "docId": "incident-diagnosis-flow",
+      "title": "Incident Diagnosis Flow",
+      "breadcrumb": "AIOps > Diagnosis Flow",
+      "content": "Collect evidence, compare metrics and logs, then verify remediation before closing the incident.",
+      "score": 0.61,
+      "retrievalLayer": "L1"
+    }
+  ]
+}

eval/rag-retrieval/fixtures/chat-rag-chunk-context.json

@@ -0,0 +1,25 @@
+{
+  "caseId": "chat-rag-chunk-context",
+  "query": "If a long section is split into multiple chunks, how do we keep retrieval context?",
+  "retrievedAt": "2026-07-05T00:00:00Z",
+  "candidates": [
+    {
+      "rank": 1,
+      "docId": "rag-chunk-context-reconstruction",
+      "title": "RAG Chunk Context Reconstruction",
+      "breadcrumb": "RAG > Chunking > Context Reconstruction",
+      "content": "After a chunk hit, expand to neighbor chunk candidates from the same section and preserve breadcrumb metadata in the evidence pack.",
+      "score": 0.79,
+      "retrievalLayer": "L1"
+    },
+    {
+      "rank": 2,
+      "docId": "rag-breadcrumb-embedding-gap",
+      "title": "RAG Breadcrumb Embedding Gap",
+      "breadcrumb": "RAG > Embedding > Breadcrumb",
+      "content": "Embedding title and breadcrumb with content helps recover section semantics.",
+      "score": 0.72,
+      "retrievalLayer": "L1"
+    }
+  ]
+}