zhuyongxin
7ae9707a3b
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
108 lines
3.0 KiB
JSON
108 lines
3.0 KiB
JSON
{
|
|
"generatedAt": "2026-07-28T06:48:56.439554+00:00",
|
|
"baselineReport": "eval/rag-retrieval/cases/golden-cases.json",
|
|
"currentReport": "eval/rag-retrieval/cases/golden-cases.json",
|
|
"baselineCaseCount": 7,
|
|
"currentCaseCount": 7,
|
|
"baselinePassRate": 1.0,
|
|
"currentPassRate": 0.8571,
|
|
"baselineRecallAtK": 1.0,
|
|
"currentRecallAtK": 0.8571,
|
|
"regressionCount": 6,
|
|
"improvementCount": 0,
|
|
"changedCount": 3,
|
|
"hasRegression": true,
|
|
"items": [
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "aggregate",
|
|
"caseId": null,
|
|
"metric": "passRate",
|
|
"baselineValue": "1.0",
|
|
"currentValue": "0.8571",
|
|
"delta": -0.14290000000000003,
|
|
"message": "aggregate passRate changed"
|
|
},
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "aggregate",
|
|
"caseId": null,
|
|
"metric": "recallAtK",
|
|
"baselineValue": "1.0",
|
|
"currentValue": "0.8571",
|
|
"delta": -0.14290000000000003,
|
|
"message": "aggregate recallAtK changed"
|
|
},
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "aggregate",
|
|
"caseId": null,
|
|
"metric": "strongHitRate",
|
|
"baselineValue": "1.0",
|
|
"currentValue": "0.8571",
|
|
"delta": -0.14290000000000003,
|
|
"message": "aggregate strongHitRate changed"
|
|
},
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "passed",
|
|
"baselineValue": "True",
|
|
"currentValue": "False",
|
|
"delta": -1.0,
|
|
"message": "chat-l0-filter-fallback passed changed"
|
|
},
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "hitLevel",
|
|
"baselineValue": "strong",
|
|
"currentValue": "weak",
|
|
"delta": -2.0,
|
|
"message": "chat-l0-filter-fallback hitLevel changed"
|
|
},
|
|
{
|
|
"type": "REGRESSION",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "firstExpectedRank",
|
|
"baselineValue": "1",
|
|
"currentValue": "-",
|
|
"delta": null,
|
|
"message": "chat-l0-filter-fallback firstExpectedRank changed"
|
|
},
|
|
{
|
|
"type": "CHANGED",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "selectedAttempt",
|
|
"baselineValue": "UNFILTERED_VECTOR_RETRY",
|
|
"currentValue": "FILTERED_VECTOR",
|
|
"delta": null,
|
|
"message": "chat-l0-filter-fallback selectedAttempt changed"
|
|
},
|
|
{
|
|
"type": "CHANGED",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "fallbackReason",
|
|
"baselineValue": "filtered_vector_low_quality",
|
|
"currentValue": "-",
|
|
"delta": null,
|
|
"message": "chat-l0-filter-fallback fallbackReason changed"
|
|
},
|
|
{
|
|
"type": "CHANGED",
|
|
"scope": "case",
|
|
"caseId": "chat-l0-filter-fallback",
|
|
"metric": "rerankTopSource",
|
|
"baselineValue": "rag-l0-filter-fallback",
|
|
"currentValue": "rag-l0-filter-decoy",
|
|
"delta": null,
|
|
"message": "chat-l0-filter-fallback rerankTopSource changed"
|
|
}
|
|
]
|
|
}
|