Expand diagnosis eval fixtures
This commit is contained in:
aruo
@@ -7,6 +7,7 @@ This folder contains the first fixed-case evaluation set for the MVP diagnosis A
|
||||
- Case definitions: `cases/diagnosis-cases.json`
|
||||
- Offline trace fixtures: `fixtures/*.json`
|
||||
- Field definitions: `schema.md`
|
||||
- Baseline reports: `reports/baseline-report.json` and `reports/baseline-report.md`
|
||||
- Evaluator implementation: `DiagnosisTraceEvaluator`
|
||||
- Report writer: `DiagnosisEvalReportWriter`
|
||||
|
||||
@@ -22,6 +23,17 @@ Run the focused evaluator test:
|
||||
mvn -q "-Dtest=DiagnosisTraceEvaluatorTest" test
|
||||
```
|
||||
|
||||
The committed baseline report represents the current fixed fixture set:
|
||||
|
||||
```text
|
||||
5 fixed cases
|
||||
5 passing fixture evaluations
|
||||
2 PASS verdicts
|
||||
3 LOW_CONFID verdicts
|
||||
```
|
||||
|
||||
When fixtures or evaluator rules change, regenerate the report from the same case file and fixture directory, then update both JSON and Markdown outputs together.
|
||||
|
||||
## Interview Story
|
||||
|
||||
The harness gives the MVP a repeatable baseline:
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
"id": "redis-timeout",
|
||||
"title": "Redis timeout",
|
||||
"question": "支付服务出现 Redis 连接超时，请定位可能原因。",
|
||||
"traceFixture": "redis-timeout-missing.json",
|
||||
"traceFixture": "redis-timeout-low-confid.json",
|
||||
"expectedRootCauseKeywords": ["redis", "超时"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_logs"],
|
||||
@@ -36,7 +36,7 @@
|
||||
"id": "slow-response",
|
||||
"title": "Slow response",
|
||||
"question": "用户服务 P99 响应时间升高，请结合指标和日志分析。",
|
||||
"traceFixture": "slow-response-missing.json",
|
||||
"traceFixture": "slow-response-pass.json",
|
||||
"expectedRootCauseKeywords": ["p99", "慢响应"],
|
||||
"minKeywordMatches": 1,
|
||||
"requiredEvidenceTools": ["query_metrics", "query_logs"],
|
||||
@@ -47,7 +47,7 @@
|
||||
"id": "jvm-memory-risk",
|
||||
"title": "JVM memory risk",
|
||||
"question": "订单服务内存使用率过高，请判断是否存在 OOM 风险。",
|
||||
"traceFixture": "jvm-memory-risk-missing.json",
|
||||
"traceFixture": "jvm-memory-risk-low-confid.json",
|
||||
"expectedRootCauseKeywords": ["jvm", "内存", "oom"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_metrics", "query_logs"],
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"session": {
|
||||
"sessionId": "eval-jvm-memory-risk",
|
||||
"query": "订单服务内存使用率过高，请判断是否存在 OOM 风险。",
|
||||
"status": "SUCCESS",
|
||||
"agentFlow": "CHAT",
|
||||
"totalDurationMs": 53000,
|
||||
"toolCallCount": 2,
|
||||
"answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n订单服务存在 JVM 内存风险，但还不能完全确认会发生 OOM。指标显示 heap 使用率持续高于 88%，日志出现多次 Full GC 和 allocation pressure，需要继续观察对象增长来源并检查最近发布。",
|
||||
"selfEvaluation": {
|
||||
"verifier_evaluation": {
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.52,
|
||||
"tool_trace_summary": [
|
||||
{
|
||||
"tool_name": "query_metrics",
|
||||
"success": true,
|
||||
"evidence_level": "direct"
|
||||
},
|
||||
{
|
||||
"tool_name": "query_logs",
|
||||
"success": true,
|
||||
"evidence_level": "indirect"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"steps": [],
|
||||
"toolInvocations": [
|
||||
{
|
||||
"id": 1,
|
||||
"sessionId": "eval-jvm-memory-risk",
|
||||
"toolName": "query_metrics",
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"sessionId": "eval-jvm-memory-risk",
|
||||
"toolName": "query_logs",
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"persistedStepCount": 3,
|
||||
"returnedStepCount": 3,
|
||||
"persistedToolCallCount": 2,
|
||||
"returnedToolCallCount": 2,
|
||||
"hasVerifierEvaluation": true,
|
||||
"hasFeedback": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"session": {
|
||||
"sessionId": "eval-redis-timeout",
|
||||
"query": "支付服务出现 Redis 连接超时，请定位可能原因。",
|
||||
"status": "SUCCESS",
|
||||
"agentFlow": "CHAT",
|
||||
"totalDurationMs": 36000,
|
||||
"toolCallCount": 1,
|
||||
"answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\nRedis 连接超时可能和支付服务到 Redis 的网络抖动或连接池等待有关。日志中出现 redis timeout 和 command timeout 记录，但当前缺少指标侧证据，因此只能作为低置信结论处理。",
|
||||
"selfEvaluation": {
|
||||
"verifier_evaluation": {
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.46,
|
||||
"tool_trace_summary": [
|
||||
{
|
||||
"tool_name": "query_logs",
|
||||
"success": true,
|
||||
"evidence_level": "direct"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"steps": [],
|
||||
"toolInvocations": [
|
||||
{
|
||||
"id": 1,
|
||||
"sessionId": "eval-redis-timeout",
|
||||
"toolName": "query_logs",
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"persistedStepCount": 2,
|
||||
"returnedStepCount": 2,
|
||||
"persistedToolCallCount": 1,
|
||||
"returnedToolCallCount": 1,
|
||||
"hasVerifierEvaluation": true,
|
||||
"hasFeedback": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"session": {
|
||||
"sessionId": "eval-slow-response",
|
||||
"query": "用户服务 P99 响应时间升高，请结合指标和日志分析。",
|
||||
"status": "SUCCESS",
|
||||
"agentFlow": "CHAT",
|
||||
"totalDurationMs": 47000,
|
||||
"toolCallCount": 2,
|
||||
"answer": "用户服务 P99 升高主要表现为慢响应。指标显示 P99 latency 从 280ms 上升到 1800ms，日志中同时出现 slow request 和 downstream timeout，因此优先排查下游依赖耗时和线程池排队。",
|
||||
"selfEvaluation": {
|
||||
"verifier_evaluation": {
|
||||
"verdict": "PASS",
|
||||
"groundedness_score": 0.78,
|
||||
"tool_trace_summary": [
|
||||
{
|
||||
"tool_name": "query_metrics",
|
||||
"success": true,
|
||||
"evidence_level": "direct"
|
||||
},
|
||||
{
|
||||
"tool_name": "query_logs",
|
||||
"success": true,
|
||||
"evidence_level": "direct"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"steps": [],
|
||||
"toolInvocations": [
|
||||
{
|
||||
"id": 1,
|
||||
"sessionId": "eval-slow-response",
|
||||
"toolName": "query_metrics",
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"sessionId": "eval-slow-response",
|
||||
"toolName": "query_logs",
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"persistedStepCount": 3,
|
||||
"returnedStepCount": 3,
|
||||
"persistedToolCallCount": 2,
|
||||
"returnedToolCallCount": 2,
|
||||
"hasVerifierEvaluation": true,
|
||||
"hasFeedback": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"totalCases" : 5,
|
||||
"passedCases" : 5,
|
||||
"passRate" : 1.0,
|
||||
"verdictDistribution" : {
|
||||
"PASS" : 2,
|
||||
"LOW_CONFID" : 3
|
||||
},
|
||||
"averageToolCallCount" : 2.0,
|
||||
"averageDurationMs" : 45800.0,
|
||||
"results" : [ {
|
||||
"caseId" : "payment-timeout",
|
||||
"title" : "Payment API timeout",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "PASS",
|
||||
"matchedKeywordCount" : 3,
|
||||
"requiredKeywordCount" : 3,
|
||||
"evidenceCoverage" : {
|
||||
"lookup_knowledge" : true,
|
||||
"query_logs" : true,
|
||||
"query_metrics" : true
|
||||
},
|
||||
"toolCallCount" : 3,
|
||||
"durationMs" : 42000
|
||||
}, {
|
||||
"caseId" : "mysql-pool-exhausted",
|
||||
"title" : "MySQL connection pool exhausted",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "LOW_CONFID",
|
||||
"matchedKeywordCount" : 3,
|
||||
"requiredKeywordCount" : 3,
|
||||
"evidenceCoverage" : {
|
||||
"lookup_knowledge" : true,
|
||||
"query_logs" : true
|
||||
},
|
||||
"toolCallCount" : 2,
|
||||
"durationMs" : 51000
|
||||
}, {
|
||||
"caseId" : "redis-timeout",
|
||||
"title" : "Redis timeout",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "LOW_CONFID",
|
||||
"matchedKeywordCount" : 2,
|
||||
"requiredKeywordCount" : 2,
|
||||
"evidenceCoverage" : {
|
||||
"query_logs" : true
|
||||
},
|
||||
"toolCallCount" : 1,
|
||||
"durationMs" : 36000
|
||||
}, {
|
||||
"caseId" : "slow-response",
|
||||
"title" : "Slow response",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "PASS",
|
||||
"matchedKeywordCount" : 2,
|
||||
"requiredKeywordCount" : 2,
|
||||
"evidenceCoverage" : {
|
||||
"query_metrics" : true,
|
||||
"query_logs" : true
|
||||
},
|
||||
"toolCallCount" : 2,
|
||||
"durationMs" : 47000
|
||||
}, {
|
||||
"caseId" : "jvm-memory-risk",
|
||||
"title" : "JVM memory risk",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "LOW_CONFID",
|
||||
"matchedKeywordCount" : 3,
|
||||
"requiredKeywordCount" : 3,
|
||||
"evidenceCoverage" : {
|
||||
"query_metrics" : true,
|
||||
"query_logs" : true
|
||||
},
|
||||
"toolCallCount" : 2,
|
||||
"durationMs" : 53000
|
||||
} ]
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# Diagnosis Eval Report
|
||||
|
||||
- Total cases: 5
|
||||
- Passed cases: 5
|
||||
- Pass rate: 100.00%
|
||||
- Average tool calls: 2.00
|
||||
- Average duration ms: 45800.00
|
||||
|
||||
## Verdict Distribution
|
||||
|
||||
- PASS: 2
|
||||
- LOW_CONFID: 3
|
||||
|
||||
## Cases
|
||||
|
||||
| Case | Result | Verdict | Keywords | Tool Calls | Duration ms | Failed Checks |
|
||||
| --- | --- | --- | --- | ---: | ---: | --- |
|
||||
| payment-timeout | PASS | PASS | 3/3 | 3 | 42000 | - |
|
||||
| mysql-pool-exhausted | PASS | LOW_CONFID | 3/3 | 2 | 51000 | - |
|
||||
| redis-timeout | PASS | LOW_CONFID | 2/2 | 1 | 36000 | - |
|
||||
| slow-response | PASS | PASS | 2/2 | 2 | 47000 | - |
|
||||
| jvm-memory-risk | PASS | LOW_CONFID | 3/3 | 2 | 53000 | - |
|
||||
@@ -8,3 +8,4 @@
|
||||
| ISS-004 | Executor 域级检索水位控制（Phase 2） | 低 | 待规划 | [ISS-004-executor-domain-hard-limit.md](ISS-004-executor-domain-hard-limit.md) |
|
||||
| ISS-005 | 证据链补齐与降级契约收敛 | 高 | 已归档 | [ISS-005-evidence-trace-hardening.md](ISS-005-evidence-trace-hardening.md) |
|
||||
| ISS-006 | 固定诊断评测集与回归 Harness | 高 | 已归档 | [ISS-006-diagnosis-eval-harness.md](ISS-006-diagnosis-eval-harness.md) |
|
||||
| expand-diagnosis-eval-fixtures | 补齐固定诊断评测 fixture 与 baseline | 中 | 已归档 | [expand-diagnosis-eval-fixtures.md](expand-diagnosis-eval-fixtures.md) |
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# Expand Diagnosis Eval Fixtures
|
||||
|
||||
**状态**：已归档
|
||||
**严重程度**：中
|
||||
**发现时间**：2026-07-04
|
||||
**来源**：P1-B follow-up
|
||||
**依赖**：`diagnosis-eval-harness`
|
||||
|
||||
---
|
||||
|
||||
## 背景
|
||||
|
||||
`diagnosis-eval-harness` 已经把固定 case、trace evaluator、JSON / Markdown report 和字段文档搭起来了。
|
||||
|
||||
现在还差一步：5 条固定诊断 case 里，只有 2 条有 fixture，另外 3 条还是 missing 状态。这个状态可以验证 evaluator 的错误报告能力，但还不能作为完整 baseline 展示。
|
||||
|
||||
---
|
||||
|
||||
## 问题
|
||||
|
||||
当前 baseline 还不够完整：
|
||||
|
||||
- `redis-timeout` 没有对应 trace fixture。
|
||||
- `slow-response` 没有对应 trace fixture。
|
||||
- `jvm-memory-risk` 没有对应 trace fixture。
|
||||
- 仓库里还没有一份固定的 baseline JSON / Markdown 报告可供对比。
|
||||
|
||||
---
|
||||
|
||||
## 目标
|
||||
|
||||
补齐固定诊断评测集，让它从“框架可跑”变成“基准可用”。
|
||||
|
||||
完成后应该做到：
|
||||
|
||||
- 5 条固定 case 都能加载到对应 fixture。
|
||||
- evaluator 能输出完整 baseline report。
|
||||
- baseline report 被保存到仓库，后续 Agent 改动可以拿它做对比。
|
||||
- 文档说明怎么重新生成和怎么看报告。
|
||||
|
||||
---
|
||||
|
||||
## 范围
|
||||
|
||||
### In scope
|
||||
|
||||
- 补齐 3 个缺失 fixture。
|
||||
- 保存 baseline JSON / Markdown 报告。
|
||||
- 更新 eval 文档。
|
||||
- 补充测试，确保 case 文件引用的 fixture 都存在。
|
||||
|
||||
### Out of scope
|
||||
|
||||
- 不新增 case 数量。
|
||||
- 不改生产 Agent 主链路。
|
||||
- 不引入 LLM-as-judge。
|
||||
- 不启动真实 MySQL、Redis、Milvus 或 LLM。
|
||||
|
||||
---
|
||||
|
||||
## 面试表达
|
||||
|
||||
可以这样讲：
|
||||
|
||||
```text
|
||||
我先搭了评测 harness，然后把固定 case 的 trace fixture 补齐，
|
||||
生成一份可复现的 baseline report。
|
||||
这样以后每次改 prompt、tool 或 verifier，
|
||||
都能看固定诊断集有没有行为回退，而不是只靠人工感觉。
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 相关文件
|
||||
|
||||
- `mvp/eval/cases/diagnosis-cases.json`
|
||||
- `mvp/eval/fixtures/`
|
||||
- `mvp/eval/reports/`
|
||||
- `mvp/eval/README.md`
|
||||
- `mvp/eval/schema.md`
|
||||
- `src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java`
|
||||
- `src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java`
|
||||
- `openspec/specs/diagnosis-eval-harness/spec.md`
|
||||
Reference in New Issue
Block a user