feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
This commit is contained in:
zhuyongxin
@@ -0,0 +1,245 @@
|
||||
{
|
||||
"generatedAt": "2026-07-28T06:48:56.421877+00:00",
|
||||
"caseFile": "eval/rag-retrieval/cases/golden-cases.json",
|
||||
"fixtureDir": "eval/rag-retrieval/fixtures",
|
||||
"aggregate": {
|
||||
"caseCount": 7,
|
||||
"topK": 5,
|
||||
"passedCount": 6,
|
||||
"failedCount": 1,
|
||||
"passRate": 0.8571,
|
||||
"lookupResultCaseCount": 7,
|
||||
"strongHitCount": 6,
|
||||
"mediumHitCount": 0,
|
||||
"weakHitCount": 1,
|
||||
"missCount": 0,
|
||||
"recallAtK": 0.8571,
|
||||
"strongHitRate": 0.8571,
|
||||
"averageFirstHitRank": 1.0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"caseId": "chat-mysql-connection-pool",
|
||||
"scenario": "chat",
|
||||
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:mysql-connection-pool",
|
||||
"2:mysql-connection-pool"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"connection pool",
|
||||
"max_connections",
|
||||
"hikaricp"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"mysql-connection-pool",
|
||||
"mysql-connection-pool"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "mysql-connection-pool",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "chat-diagnosis-flow",
|
||||
"scenario": "chat",
|
||||
"query": "What is the standard troubleshooting flow for an application incident?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:incident-diagnosis-flow",
|
||||
"2:incident-diagnosis-flow"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"collect evidence",
|
||||
"verify",
|
||||
"remediation"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"incident-diagnosis-flow",
|
||||
"incident-diagnosis-flow"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "incident-diagnosis-flow",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "aiops-payment-latency-alert",
|
||||
"scenario": "aiops",
|
||||
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:payment-service-latency",
|
||||
"2:aiops-alert-scope-control",
|
||||
"3:payment-service-latency",
|
||||
"4:aiops-alert-scope-control"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"p95 latency",
|
||||
"payment-service",
|
||||
"downstream dependency"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"payment-service-latency",
|
||||
"aiops-alert-scope-control",
|
||||
"payment-service-latency",
|
||||
"aiops-alert-scope-control"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "payment-service-latency",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "aiops-prometheus-alert-scope",
|
||||
"scenario": "aiops",
|
||||
"query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:aiops-alert-scope-control",
|
||||
"2:payment-service-latency",
|
||||
"3:payment-service-latency",
|
||||
"4:aiops-alert-scope-control"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"payload",
|
||||
"unrelated active alerts",
|
||||
"scope"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"aiops-alert-scope-control",
|
||||
"payment-service-latency",
|
||||
"payment-service-latency",
|
||||
"aiops-alert-scope-control"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "aiops-alert-scope-control",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "chat-rag-chunk-context",
|
||||
"scenario": "chat",
|
||||
"query": "If a long section is split into multiple chunks, how do we keep retrieval context?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:rag-chunk-context-reconstruction",
|
||||
"2:rag-l0-domain-entity-hint",
|
||||
"3:rag-chunk-context-reconstruction",
|
||||
"4:rag-l0-domain-entity-hint"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"neighbor chunk",
|
||||
"same section",
|
||||
"breadcrumb"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"rag-chunk-context-reconstruction",
|
||||
"rag-l0-domain-entity-hint",
|
||||
"rag-chunk-context-reconstruction",
|
||||
"rag-l0-domain-entity-hint"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "rag-chunk-context-reconstruction",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "chat-l0-domain-hint",
|
||||
"scenario": "chat",
|
||||
"query": "Should L0 keyword matching decide the final retrieval result?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "strong",
|
||||
"passed": true,
|
||||
"firstExpectedRank": 1,
|
||||
"topCandidates": [
|
||||
"1:rag-l0-domain-entity-hint",
|
||||
"2:rag-chunk-context-reconstruction",
|
||||
"3:rag-l0-domain-entity-hint",
|
||||
"4:rag-chunk-context-reconstruction"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"domain detector",
|
||||
"entity extractor",
|
||||
"metadata filter"
|
||||
],
|
||||
"breadcrumbMatched": true,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"rag-l0-domain-entity-hint",
|
||||
"rag-chunk-context-reconstruction",
|
||||
"rag-l0-domain-entity-hint",
|
||||
"rag-chunk-context-reconstruction"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "rag-l0-domain-entity-hint",
|
||||
"failedChecks": []
|
||||
},
|
||||
{
|
||||
"caseId": "chat-l0-filter-fallback",
|
||||
"scenario": "chat",
|
||||
"query": "RAG query was over-filtered by L0 and filtered vector search returned low quality evidence. What should happen?",
|
||||
"dataShape": "lookupResult",
|
||||
"hitLevel": "weak",
|
||||
"passed": false,
|
||||
"firstExpectedRank": null,
|
||||
"topCandidates": [
|
||||
"1:rag-l0-filter-decoy",
|
||||
"2:rag-l0-filter-decoy"
|
||||
],
|
||||
"matchedKeywords": [
|
||||
"low quality"
|
||||
],
|
||||
"breadcrumbMatched": false,
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"evidenceStatus": "supported",
|
||||
"includedSources": [
|
||||
"rag-l0-filter-decoy",
|
||||
"rag-l0-filter-decoy"
|
||||
],
|
||||
"omittedSources": [],
|
||||
"rerankTopSource": "rag-l0-filter-decoy",
|
||||
"failedChecks": [
|
||||
"expected document not found",
|
||||
"selected attempt mismatch: expected UNFILTERED_VECTOR_RETRY, got FILTERED_VECTOR",
|
||||
"fallback reason mismatch: expected one of [filtered_vector_low_quality, filtered_vector_no_evidence], got <none>",
|
||||
"rerank top source mismatch: expected rag-l0-filter-fallback, got rag-l0-filter-decoy",
|
||||
"expected context sources missing: rag-l0-filter-fallback"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user