feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
This commit is contained in:
zhuyongxin
@@ -1,81 +1,88 @@
|
||||
{
|
||||
"caseId": "chat-mysql-connection-pool",
|
||||
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"retrievedAt": "2026-07-06T00:00:00Z",
|
||||
"lookupResult": {
|
||||
"found": true,
|
||||
"evidenceBlocks": [
|
||||
{
|
||||
"source": "mysql-connection-pool",
|
||||
"title": "MySQL Connection Pool Troubleshooting",
|
||||
"breadcrumb": "Database > MySQL > Connection Pool",
|
||||
"retrievalLayer": "L1",
|
||||
"content": "When the connection pool is exhausted, inspect HikariCP active connections, max_connections, slow SQL, leak detection, and database wait events.",
|
||||
"score": 0.86,
|
||||
"hitReasons": ["domain_match:+0.15", "keyword_match:+0.10"]
|
||||
},
|
||||
{
|
||||
"source": "incident-diagnosis-flow",
|
||||
"title": "Incident Diagnosis Flow",
|
||||
"breadcrumb": "AIOps > Diagnosis Flow",
|
||||
"retrievalLayer": "L1",
|
||||
"content": "Collect evidence, compare metrics and logs, then verify remediation before closing the incident.",
|
||||
"score": 0.61,
|
||||
"hitReasons": []
|
||||
}
|
||||
],
|
||||
"contextPack": {
|
||||
"packedText": "[1] MySQL Connection Pool Troubleshooting\nDatabase > MySQL > Connection Pool\nWhen the connection pool is exhausted, inspect HikariCP active connections, max_connections, slow SQL, leak detection, and database wait events.",
|
||||
"strategy": "top_evidence_blocks",
|
||||
"charBudget": 3500,
|
||||
"usedChars": 216,
|
||||
"includedSources": ["mysql-connection-pool", "incident-diagnosis-flow"],
|
||||
"omittedSources": []
|
||||
"caseId" : "chat-mysql-connection-pool",
|
||||
"query" : "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"retrievedAt" : "2026-07-28T06:54:51.843450400Z",
|
||||
"searchMode" : "hybrid",
|
||||
"kbScope" : "rag-eval",
|
||||
"lookupResult" : {
|
||||
"found" : true,
|
||||
"evidenceBlocks" : [ {
|
||||
"docId" : "mysql-connection-pool",
|
||||
"chunkIndex" : 2,
|
||||
"evidenceKey" : "mysql-connection-pool#chunk-2",
|
||||
"source" : "mysql-connection-pool",
|
||||
"title" : "Connection Pool",
|
||||
"breadcrumb" : "Database > MySQL > Connection Pool",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "### Connection Pool\n\nWhen MySQL connection pool is exhausted, first compare application pool usage with database `max_connections`.\nFor HikariCP, check `active`, `idle`, `pending`, and connection acquisition timeout metrics.\n\nRecommended diagnosis:\n\n1. Verify whether HikariCP active connections stay near maximum while pending threads grow.\n2. Check MySQL `Threads_connected`, `Threads_running`, and `max_connections`.\n3. Inspect slow SQL and long transactions that keep connections checked out.\n4. If the database is healthy, look for application connection leaks or missing transaction boundaries.\n\nUse this runbook as evidence for connection pool, max_connections, and HikariCP incidents.",
|
||||
"score" : 0.032786883413791656,
|
||||
"hitReasons" : [ "semantic_rank:1", "attempt:FILTERED_VECTOR", "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"docId" : "mysql-connection-pool",
|
||||
"chunkIndex" : 1,
|
||||
"evidenceKey" : "mysql-connection-pool#chunk-1",
|
||||
"source" : "mysql-connection-pool",
|
||||
"title" : "MySQL",
|
||||
"breadcrumb" : "Database > MySQL > Connection Pool",
|
||||
"retrievalLayer" : "L1",
|
||||
"content" : "## MySQL",
|
||||
"score" : 0.032258063554763794,
|
||||
"hitReasons" : [ "semantic_rank:2", "attempt:FILTERED_VECTOR", "l0_domain_overlap" ]
|
||||
} ],
|
||||
"contextPack" : {
|
||||
"packedText" : "[Evidence 1]\nsource: mysql-connection-pool\ntitle: Connection Pool\nbreadcrumb: Database > MySQL > Connection Pool\nlayer: L1\nreasons: semantic_rank:1, attempt:FILTERED_VECTOR, l0_domain_overlap, l0_entity_overlap, l0_keyword_overlap\ncontent:\n### Connection Pool\n\nWhen MySQL connection pool is exhausted, first compare application pool usage with database `max_connections`.\nFor HikariCP, check `active`, `idle`, `pending`, and connection acquisition timeout metrics.\n\nRecommended diagnosis:\n\n1. Verify whether HikariCP active connections stay near maximum while pending threads grow.\n2. Check MySQL `Threads_connected`, `Threads_running`, and `max_connections`.\n3. Inspect slow SQL and long transactions that keep connections checked out.\n4. If the database is healthy, look for application connection leaks or missing transaction boundaries.\n\nUse this runbook as evidence for connection pool, max_connections, and HikariCP incidents.\n\n[Evidence 2]\nsource: mysql-connection-pool\ntitle: MySQL\nbreadcrumb: Database > MySQL > Connection Pool\nlayer: L1\nreasons: semantic_rank:2, attempt:FILTERED_VECTOR, l0_domain_overlap\ncontent:\n## MySQL",
|
||||
"strategy" : "ranked_evidence_char_budget",
|
||||
"charBudget" : 4000,
|
||||
"usedChars" : 1135,
|
||||
"includedSources" : [ "mysql-connection-pool", "mysql-connection-pool" ],
|
||||
"omittedSources" : [ ]
|
||||
},
|
||||
"retrievalTrace": {
|
||||
"originalQuery": "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"rewrittenQuery": "MySQL connection pool exhausted HikariCP max_connections diagnosis",
|
||||
"categoryFilter": "Database",
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"queryHints": {
|
||||
"domains": ["Database", "MySQL"],
|
||||
"matched_keywords": ["connection pool", "HikariCP", "max_connections"],
|
||||
"entities": ["MySQL", "HikariCP"],
|
||||
"l0_titles": ["MySQL Connection Pool Troubleshooting"],
|
||||
"l0_match_count": 1
|
||||
"retrievalTrace" : {
|
||||
"originalQuery" : "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"rewrittenQuery" : "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"categoryFilter" : "database",
|
||||
"selectedAttempt" : "FILTERED_VECTOR",
|
||||
"fallbackReason" : null,
|
||||
"evidenceStatus" : "supported",
|
||||
"queryHints" : {
|
||||
"domains" : [ "database" ],
|
||||
"matched_keywords" : [ "MySQL connection pool" ],
|
||||
"entities" : [ "MySQL connection pool" ],
|
||||
"l0_titles" : [ "MySQL Connection Pool Runbook" ],
|
||||
"l0_match_count" : 1
|
||||
},
|
||||
"attempts": [
|
||||
{
|
||||
"name": "FILTERED_VECTOR",
|
||||
"query": "MySQL connection pool exhausted HikariCP max_connections diagnosis",
|
||||
"categoryFilter": "Database",
|
||||
"candidateCount": 2,
|
||||
"usable": true,
|
||||
"durationMs": 12,
|
||||
"topScore": 0.86,
|
||||
"topSimilarity": 0.86
|
||||
}
|
||||
]
|
||||
"attempts" : [ {
|
||||
"name" : "FILTERED_VECTOR",
|
||||
"query" : "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"categoryFilter" : "database",
|
||||
"candidateCount" : 3,
|
||||
"usable" : true,
|
||||
"errorMessage" : null,
|
||||
"durationMs" : 5267,
|
||||
"topScore" : 0.032786883413791656,
|
||||
"topSimilarity" : 0.8114794194698334
|
||||
} ]
|
||||
},
|
||||
"rerankTrace": {
|
||||
"items": [
|
||||
{
|
||||
"finalRank": 1,
|
||||
"source": "mysql-connection-pool",
|
||||
"baseScore": 0.86,
|
||||
"finalScore": 1.11,
|
||||
"boostReasons": ["domain_match:+0.15", "keyword_match:+0.10"]
|
||||
},
|
||||
{
|
||||
"finalRank": 2,
|
||||
"source": "incident-diagnosis-flow",
|
||||
"baseScore": 0.61,
|
||||
"finalScore": 0.61,
|
||||
"boostReasons": []
|
||||
}
|
||||
]
|
||||
}
|
||||
"rerankTrace" : {
|
||||
"items" : [ {
|
||||
"finalRank" : 1,
|
||||
"source" : "mysql-connection-pool",
|
||||
"baseScore" : 0.8114794194698334,
|
||||
"finalScore" : 0.8114794194698334,
|
||||
"boostReasons" : [ "l0_domain_overlap", "l0_entity_overlap", "l0_keyword_overlap" ]
|
||||
}, {
|
||||
"finalRank" : 2,
|
||||
"source" : "mysql-connection-pool",
|
||||
"baseScore" : 0.49219560623168945,
|
||||
"finalScore" : 0.49219560623168945,
|
||||
"boostReasons" : [ "l0_domain_overlap" ]
|
||||
} ]
|
||||
},
|
||||
"evidenceCandidateCount" : 3,
|
||||
"evidenceBlockCount" : 2,
|
||||
"relevanceLevel" : "PRECISE",
|
||||
"completenessHint" : "知识库中不存在比上述结果更精准的文档",
|
||||
"retrievedDomainsThisSession" : null,
|
||||
"message" : null
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user