feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
This commit is contained in:
zhuyongxin
@@ -1,81 +1,122 @@
|
||||
{
|
||||
"caseId": "aiops-payment-latency-alert",
|
||||
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"retrievedAt": "2026-07-06T00:00:00Z",
|
||||
"lookupResult": {
|
||||
"found": true,
|
||||
"evidenceBlocks": [
|
||||
{
|
||||
"source": "payment-service-latency",
|
||||
"title": "Payment Service Latency Alert Playbook",
|
||||
"breadcrumb": "AIOps > Service Alerts > Payment Latency",
|
||||
"retrievalLayer": "L1",
|
||||
"content": "For payment-service p95 latency alerts, check downstream dependency latency, thread pool saturation, gateway retries, and recent deployment changes.",
|
||||
"score": 0.84,
|
||||
"hitReasons": ["domain_match:+0.15", "entity_match:+0.20", "keyword_match:+0.10"]
|
||||
},
|
||||
{
|
||||
"source": "mysql-connection-pool",
|
||||
"title": "MySQL Connection Pool Troubleshooting",
|
||||
"breadcrumb": "Database > MySQL > Connection Pool",
|
||||
"retrievalLayer": "L1",
|
||||
"content": "Database connection pool saturation can increase payment latency when checkout paths wait for connections.",
|
||||
"score": 0.68,
|
||||
"hitReasons": ["keyword_match:+0.10"]
|
||||
}
|
||||
],
|
||||
"contextPack": {
|
||||
"packedText": "[1] Payment Service Latency Alert Playbook\nAIOps > Service Alerts > Payment Latency\nFor payment-service p95 latency alerts, check downstream dependency latency, thread pool saturation, gateway retries, and recent deployment changes.",
|
||||
"strategy": "top_evidence_blocks",
|
||||
"charBudget": 3500,
|
||||
"usedChars": 236,
|
||||
"includedSources": ["payment-service-latency", "mysql-connection-pool"],
|
||||
"omittedSources": []
|
||||
"caseId" : "aiops-payment-latency-alert",
|
||||
"query" : "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"retrievedAt" : "2026-07-28T06:54:51.843450400Z",
|
||||
"searchMode" : "hybrid",
|
||||
"kbScope" : "rag-eval",
|
||||
"lookupResult" : {
|
||||
"found" : true,
|
||||
"evidenceBlocks" : [ {
|
||||
"docId" : "payment-service-latency",
|
||||
"chunkIndex" : 2,
|
||||
"evidenceKey" : "payment-service-latency#chunk-2",
|
||||
"source" : "payment-service-latency",
|
||||
"title" : "Payment Latency",
|
||||
"breadcrumb" : "AIOps > Service Alerts > Payment Latency",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "### Payment Latency\n\nFor `HighLatency` alerts on `payment-service`, treat p95 latency as the primary symptom.\n\nDiagnosis steps:\n\n1. Confirm whether p95 latency is isolated to payment-service or shared across upstream callers.\n2. Compare payment-service latency with downstream dependency latency for gateway, risk, and order services.\n3. Check connection pool wait time, retry spikes, and timeout rates.\n4. If downstream dependency latency increased first, classify payment-service as affected rather than root cause.\n\nThe expected evidence terms are p95 latency, payment-service, and downstream dependency.",
|
||||
"score" : 0.032786883413791656,
|
||||
"hitReasons" : [ "semantic_rank:1", "attempt:FILTERED_VECTOR", "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"docId" : "aiops-alert-scope-control",
|
||||
"chunkIndex" : 1,
|
||||
"evidenceKey" : "aiops-alert-scope-control#chunk-1",
|
||||
"source" : "aiops-alert-scope-control",
|
||||
"title" : "Alert Scope Control",
|
||||
"breadcrumb" : "AIOps > Alert Scope Control",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "## Alert Scope Control\n\nWhen an AIOps request already includes an alert payload, the agent should diagnose that payload first.\nIt must not expand the task into unrelated active alerts unless the user asks for broad alert triage.\n\nScope rules:\n\n1. Treat the provided payload as the primary incident boundary.\n2. Use unrelated active alerts only as correlation evidence when they share service, dependency, time window, or trace context.\n3. Do not replace the requested alert with a louder but unrelated alert.\n\nThis runbook anchors payload, unrelated active alerts, and scope behavior.",
|
||||
"score" : 0.0320020467042923,
|
||||
"hitReasons" : [ "semantic_rank:2", "attempt:FILTERED_VECTOR", "l0_domain_overlap" ]
|
||||
}, {
|
||||
"docId" : "payment-service-latency",
|
||||
"chunkIndex" : 1,
|
||||
"evidenceKey" : "payment-service-latency#chunk-1",
|
||||
"source" : "payment-service-latency",
|
||||
"title" : "Service Alerts",
|
||||
"breadcrumb" : "AIOps > Service Alerts > Payment Latency",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "## Service Alerts",
|
||||
"score" : 0.0320020467042923,
|
||||
"hitReasons" : [ "semantic_rank:3", "attempt:FILTERED_VECTOR", "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"docId" : "aiops-alert-scope-control",
|
||||
"chunkIndex" : 0,
|
||||
"evidenceKey" : "aiops-alert-scope-control#chunk-0",
|
||||
"source" : "aiops-alert-scope-control",
|
||||
"title" : "AIOps",
|
||||
"breadcrumb" : "AIOps > Alert Scope Control",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "# AIOps",
|
||||
"score" : 0.015384615398943424,
|
||||
"hitReasons" : [ "semantic_rank:5", "attempt:FILTERED_VECTOR", "l0_domain_overlap" ]
|
||||
} ],
|
||||
"contextPack" : {
|
||||
"packedText" : "[Evidence 1]\nsource: payment-service-latency\ntitle: Payment Latency\nbreadcrumb: AIOps > Service Alerts > Payment Latency\nlayer: L1\nreasons: semantic_rank:1, attempt:FILTERED_VECTOR, l0_domain_overlap, l0_entity_overlap, l0_keyword_overlap\ncontent:\n### Payment Latency\n\nFor `HighLatency` alerts on `payment-service`, treat p95 latency as the primary symptom.\n\nDiagnosis steps:\n\n1. Confirm whether p95 latency is isolated to payment-service or shared across upstream callers.\n2. Compare payment-service latency with downstream dependency latency for gateway, risk, and order services.\n3. Check connection pool wait time, retry spikes, and timeout rates.\n4. If downstream dependency latency increased first, classify payment-service as affected rather than root cause.\n\nThe expected evidence terms are p95 latency, payment-service, and downstream dependency.\n\n[Evidence 2]\nsource: aiops-alert-scope-control\ntitle: Alert Scope Control\nbreadcrumb: AIOps > Alert Scope Control\nlayer: L1\nreasons: semantic_rank:2, attempt:FILTERED_VECTOR, l0_domain_overlap\ncontent:\n## Alert Scope Control\n\nWhen an AIOps request already includes an alert payload, the agent should diagnose that payload first.\nIt must not expand the task into unrelated active alerts unless the user asks for broad alert triage.\n\nScope rules:\n\n1. Treat the provided payload as the primary incident boundary.\n2. Use unrelated active alerts only as correlation evidence when they share service, dependency, time window, or trace context.\n3. Do not replace the requested alert with a louder but unrelated alert.\n\nThis runbook anchors payload, unrelated active alerts, and scope behavior.\n\n[Evidence 3]\nsource: payment-service-latency\ntitle: Service Alerts\nbreadcrumb: AIOps > Service Alerts > Payment Latency\nlayer: L1\nreasons: semantic_rank:3, attempt:FILTERED_VECTOR, l0_domain_overlap, l0_entity_overlap, l0_keyword_overlap\ncontent:\n## Service Alerts\n\n[Evidence 4]\nsource: aiops-alert-scope-control\ntitle: AIOps\nbreadcrumb: AIOps > Alert Scope Control\nlayer: L1\nreasons: semantic_rank:5, attempt:FILTERED_VECTOR, l0_domain_overlap\ncontent:\n# AIOps",
|
||||
"strategy" : "ranked_evidence_char_budget",
|
||||
"charBudget" : 4000,
|
||||
"usedChars" : 2108,
|
||||
"includedSources" : [ "payment-service-latency", "aiops-alert-scope-control", "payment-service-latency", "aiops-alert-scope-control" ],
|
||||
"omittedSources" : [ ]
|
||||
},
|
||||
"retrievalTrace": {
|
||||
"originalQuery": "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"rewrittenQuery": "HighLatency payment-service p95 latency alert downstream dependency diagnosis",
|
||||
"categoryFilter": "AIOps",
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"queryHints": {
|
||||
"domains": ["AIOps"],
|
||||
"matched_keywords": ["p95 latency", "payment-service", "downstream dependency"],
|
||||
"entities": ["payment-service", "HighLatency"],
|
||||
"l0_titles": ["Payment Service Latency Alert Playbook"],
|
||||
"l0_match_count": 1
|
||||
"retrievalTrace" : {
|
||||
"originalQuery" : "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"rewrittenQuery" : "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"categoryFilter" : "aiops",
|
||||
"selectedAttempt" : "FILTERED_VECTOR",
|
||||
"fallbackReason" : null,
|
||||
"evidenceStatus" : "supported",
|
||||
"queryHints" : {
|
||||
"domains" : [ "aiops" ],
|
||||
"matched_keywords" : [ "HighLatency", "payment-service", "p95 latency" ],
|
||||
"entities" : [ "HighLatency", "payment-service", "p95 latency" ],
|
||||
"l0_titles" : [ "Payment Service Latency Alert" ],
|
||||
"l0_match_count" : 1
|
||||
},
|
||||
"attempts": [
|
||||
{
|
||||
"name": "FILTERED_VECTOR",
|
||||
"query": "HighLatency payment-service p95 latency alert downstream dependency diagnosis",
|
||||
"categoryFilter": "AIOps",
|
||||
"candidateCount": 2,
|
||||
"usable": true,
|
||||
"durationMs": 11,
|
||||
"topScore": 0.84,
|
||||
"topSimilarity": 0.84
|
||||
}
|
||||
]
|
||||
"attempts" : [ {
|
||||
"name" : "FILTERED_VECTOR",
|
||||
"query" : "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"categoryFilter" : "aiops",
|
||||
"candidateCount" : 5,
|
||||
"usable" : true,
|
||||
"errorMessage" : null,
|
||||
"durationMs" : 969,
|
||||
"topScore" : 0.032786883413791656,
|
||||
"topSimilarity" : 0.7736010700464249
|
||||
} ]
|
||||
},
|
||||
"rerankTrace": {
|
||||
"items": [
|
||||
{
|
||||
"finalRank": 1,
|
||||
"source": "payment-service-latency",
|
||||
"baseScore": 0.84,
|
||||
"finalScore": 1.29,
|
||||
"boostReasons": ["domain_match:+0.15", "entity_match:+0.20", "keyword_match:+0.10"]
|
||||
},
|
||||
{
|
||||
"finalRank": 2,
|
||||
"source": "mysql-connection-pool",
|
||||
"baseScore": 0.68,
|
||||
"finalScore": 0.78,
|
||||
"boostReasons": ["keyword_match:+0.10"]
|
||||
}
|
||||
]
|
||||
}
|
||||
"rerankTrace" : {
|
||||
"items" : [ {
|
||||
"finalRank" : 1,
|
||||
"source" : "payment-service-latency",
|
||||
"baseScore" : 0.7736010700464249,
|
||||
"finalScore" : 0.7736010700464249,
|
||||
"boostReasons" : [ "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"finalRank" : 2,
|
||||
"source" : "aiops-alert-scope-control",
|
||||
"baseScore" : 0.4683566689491272,
|
||||
"finalScore" : 0.4683566689491272,
|
||||
"boostReasons" : [ "l0_domain_overlap" ]
|
||||
}, {
|
||||
"finalRank" : 3,
|
||||
"source" : "payment-service-latency",
|
||||
"baseScore" : 0.4981400966644287,
|
||||
"finalScore" : 0.4981400966644287,
|
||||
"boostReasons" : [ "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"finalRank" : 4,
|
||||
"source" : "aiops-alert-scope-control",
|
||||
"baseScore" : 0.3814886808395386,
|
||||
"finalScore" : 0.3814886808395386,
|
||||
"boostReasons" : [ "l0_domain_overlap" ]
|
||||
} ]
|
||||
},
|
||||
"evidenceCandidateCount" : 5,
|
||||
"evidenceBlockCount" : 4,
|
||||
"relevanceLevel" : "PRECISE",
|
||||
"completenessHint" : "知识库中不存在比上述结果更精准的文档",
|
||||
"retrievedDomainsThisSession" : null,
|
||||
"message" : null
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user