From ca5c61fabfede711cda2fdb66b0a252652c2d1bb Mon Sep 17 00:00:00 2001 From: aruo <40362743+zyongxin@users.noreply.github.com> Date: Sat, 4 Jul 2026 23:51:43 +0800 Subject: [PATCH] Add diagnosis eval harness --- devflow/index.md | 3 +- .../acceptance.md | 36 +++ .../brief.md | 31 +++ .../decisions.md | 28 +++ .../evidence.md | 10 + mvp/eval/README.md | 35 +++ mvp/eval/cases/diagnosis-cases.json | 57 +++++ mvp/eval/fixtures/mysql-pool-low-confid.json | 53 +++++ mvp/eval/fixtures/payment-timeout-pass.json | 64 ++++++ mvp/eval/schema.md | 137 +++++++++++ mvp/issues/ISS-006-diagnosis-eval-harness.md | 99 ++++++++ mvp/issues/README.md | 3 +- .../.openspec.yaml | 2 + .../design.md | 54 +++++ .../proposal.md | 27 +++ .../specs/diagnosis-eval-harness/spec.md | 59 +++++ .../tasks.md | 25 ++ openspec/specs/diagnosis-eval-harness/spec.md | 64 ++++++ .../agent/eval/DiagnosisEvalCase.java | 25 ++ .../agent/eval/DiagnosisEvalReport.java | 24 ++ .../agent/eval/DiagnosisEvalReportWriter.java | 76 ++++++ .../agent/eval/DiagnosisEvalResult.java | 27 +++ .../agent/eval/DiagnosisTraceEvaluator.java | 217 ++++++++++++++++++ .../eval/DiagnosisTraceEvaluatorTest.java | 97 ++++++++ 24 files changed, 1251 insertions(+), 2 deletions(-) create mode 100644 devflow/projects/2026-07-04-diagnosis-eval-harness/acceptance.md create mode 100644 devflow/projects/2026-07-04-diagnosis-eval-harness/brief.md create mode 100644 devflow/projects/2026-07-04-diagnosis-eval-harness/decisions.md create mode 100644 devflow/projects/2026-07-04-diagnosis-eval-harness/evidence.md create mode 100644 mvp/eval/README.md create mode 100644 mvp/eval/cases/diagnosis-cases.json create mode 100644 mvp/eval/fixtures/mysql-pool-low-confid.json create mode 100644 mvp/eval/fixtures/payment-timeout-pass.json create mode 100644 mvp/eval/schema.md create mode 100644 mvp/issues/ISS-006-diagnosis-eval-harness.md create mode 100644 openspec/changes/archive/2026-07-04-diagnosis-eval-harness/.openspec.yaml create mode 100644 openspec/changes/archive/2026-07-04-diagnosis-eval-harness/design.md create mode 100644 openspec/changes/archive/2026-07-04-diagnosis-eval-harness/proposal.md create mode 100644 openspec/changes/archive/2026-07-04-diagnosis-eval-harness/specs/diagnosis-eval-harness/spec.md create mode 100644 openspec/changes/archive/2026-07-04-diagnosis-eval-harness/tasks.md create mode 100644 openspec/specs/diagnosis-eval-harness/spec.md create mode 100644 src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java create mode 100644 src/main/java/com/superbiz/agent/eval/DiagnosisEvalReport.java create mode 100644 src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java create mode 100644 src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java create mode 100644 src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java create mode 100644 src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java diff --git a/devflow/index.md b/devflow/index.md index a008b25..430ff27 100644 --- a/devflow/index.md +++ b/devflow/index.md @@ -4,7 +4,8 @@ | 日期 | slug | 领域 | 关键词 | 状态 | |---|---|---|---|---| -| 2026-07-04 | evidence-trace-hardening | 证据链/降级契约/离线验证 | ToolInvocationRecorder, ToolTraceSummaryService, lookup_knowledge, query_logs, query_metrics, LOW_CONFID, REJECT | openspec/changes/evidence-trace-hardening | active | +| 2026-07-04 | diagnosis-eval-harness | Agent 评测/回归 Harness | fixed cases, trace validation, evidence coverage, verdict distribution, markdown report | openspec/changes/archive/2026-07-04-diagnosis-eval-harness | archived | +| 2026-07-04 | evidence-trace-hardening | 证据链/降级契约/离线验证 | ToolInvocationRecorder, ToolTraceSummaryService, lookup_knowledge, query_logs, query_metrics, LOW_CONFID, REJECT | openspec/changes/archive/2026-07-04-evidence-trace-hardening | archived | | 2026-07-03 | mvp-demo-trace-acceptance | MVP Demo/trace/acceptance | mvp-demo, trace API, diagnosis_session, agent_step, tool_invocation, feedback | openspec/changes/archive/2026-07-03-mvp-demo-trace-acceptance | archived | | 2026-05-29 | chatmodel-abstraction | 解耦/多模型路由 | ChatModel, EmbeddingModel, DeepSeek, BGE-M3, SiliconFlow, Spring AI | archived | | 2026-06-23 | phase1-infrastructure | 基础设施/文档管理 | MySQL, Redis, Milvus, Flyway, JPA, 向量检索, 类别过滤 | archived | diff --git a/devflow/projects/2026-07-04-diagnosis-eval-harness/acceptance.md b/devflow/projects/2026-07-04-diagnosis-eval-harness/acceptance.md new file mode 100644 index 0000000..8afc48d --- /dev/null +++ b/devflow/projects/2026-07-04-diagnosis-eval-harness/acceptance.md @@ -0,0 +1,36 @@ +# Acceptance: diagnosis-eval-harness + +## Classification + +standard-light + +## Task Status + +| Task | Status | Notes | +| --- | --- | --- | +| Issue and OpenSpec setup | Done | `ISS-006` and initial OpenSpec artifacts were created. | +| Implementation | Done | Added fixed cases, fixture-mode trace evaluation, aggregate metrics, and JSON / Markdown report writer. | +| Verification | Done | Targeted evaluator tests, compile verification, and OpenSpec validation passed. | + +## Current State + +- First implementation uses fixture-mode evaluation. +- Live trace API polling remains a follow-up option. + +## Verification + +### Script Verification + +- Command: `mvn -q "-Dtest=DiagnosisTraceEvaluatorTest" test` +- Result: passed +- Notes: Covers fixed case loading, fixture evaluation, missing fixture reporting, reject degraded-output validation, and report writing. + +### Static Verification + +- Command: `mvn -q -DskipTests compile` +- Result: passed + +### OpenSpec Verification + +- Command: `openspec validate diagnosis-eval-harness --strict` +- Result: passed diff --git a/devflow/projects/2026-07-04-diagnosis-eval-harness/brief.md b/devflow/projects/2026-07-04-diagnosis-eval-harness/brief.md new file mode 100644 index 0000000..0ab1a8c --- /dev/null +++ b/devflow/projects/2026-07-04-diagnosis-eval-harness/brief.md @@ -0,0 +1,31 @@ +# Brief: diagnosis-eval-harness + +## Background + +The MVP has a runnable demo and hardened evidence trace semantics, but it still lacks a fixed regression baseline for Agent diagnosis quality. P1-B creates a small evaluation harness that can validate diagnosis traces against fixed cases and produce repeatable reports. + +## Goals + +1. Define fixed diagnosis cases for the MVP demo domain. +2. Validate trace evidence, verifier verdicts, answer keywords, and degraded-output behavior. +3. Produce JSON and Markdown reports for interview and regression use. +4. Keep the first version offline by supporting trace fixtures. + +## Scope + +- Evaluation case definitions +- Trace fixture shape +- Rule-based evaluator +- JSON / Markdown report output +- Focused offline tests and docs + +## Non-Goals + +- No LLM-as-judge +- No live end-to-end runtime requirement +- No production API +- No chat or verifier runtime change + +## Related OpenSpec + +`openspec/changes/diagnosis-eval-harness/` diff --git a/devflow/projects/2026-07-04-diagnosis-eval-harness/decisions.md b/devflow/projects/2026-07-04-diagnosis-eval-harness/decisions.md new file mode 100644 index 0000000..1ce0b25 --- /dev/null +++ b/devflow/projects/2026-07-04-diagnosis-eval-harness/decisions.md @@ -0,0 +1,28 @@ +# Diagnosis Eval Harness Decisions + +## Clarify + +- Entry summary: build P1-B fixed case evaluation after evidence trace hardening. +- Slug: `diagnosis-eval-harness` +- Devflow scale: standard-light + +## Context + +- P1-A `evidence-trace-hardening` created stable evidence semantics for supported, no-evidence, deduped, and failed tool calls. +- The MVP demo trace API already provides an aggregate trace shape suitable for evaluation. +- The first evaluator should avoid depending on external infrastructure so it can run in regular development. + +## Key Decisions + +- Decision: Start with rule-based trace validation instead of LLM-as-judge. + - Reason: The first regression signal should be deterministic and tied to trace contracts. + +- Decision: Support offline fixture traces first. + - Reason: This makes the harness usable without MySQL, Redis, Milvus, or a real LLM. + +- Decision: Output both JSON and Markdown. + - Reason: JSON supports automation; Markdown is easier to discuss in interviews. + +## Open Questions + +- Whether live trace API polling belongs in this change or a follow-up after fixture mode lands. diff --git a/devflow/projects/2026-07-04-diagnosis-eval-harness/evidence.md b/devflow/projects/2026-07-04-diagnosis-eval-harness/evidence.md new file mode 100644 index 0000000..1653291 --- /dev/null +++ b/devflow/projects/2026-07-04-diagnosis-eval-harness/evidence.md @@ -0,0 +1,10 @@ +# Diagnosis Eval Harness Evidence + +## Evidence + +| Source | Evidence | Conclusion | Reported | +|---|---|---|---| +| `openspec/specs/evidence-trace-hardening/spec.md` | Defines stable evidence states and summary behavior | Evaluation can rely on trace semantics rather than ad hoc log parsing | Yes | +| `mvp/demo/README.md` | Documents an end-to-end demo flow with chat, trace, and feedback | Existing demo flow provides the runtime story, but not a reusable evaluation baseline | Yes | +| `DiagnosisTraceService` | Aggregates session, steps, tools, and self-evaluation | Trace response shape can be reused as evaluation input | Yes | +| `ToolTraceSummaryService` | Builds verifier-facing evidence summaries from persisted tool rows | Evaluator can check evidence coverage through persisted trace artifacts | Yes | diff --git a/mvp/eval/README.md b/mvp/eval/README.md new file mode 100644 index 0000000..0153363 --- /dev/null +++ b/mvp/eval/README.md @@ -0,0 +1,35 @@ +# Diagnosis Eval Harness + +This folder contains the first fixed-case evaluation set for the MVP diagnosis Agent. + +## Scope + +- Case definitions: `cases/diagnosis-cases.json` +- Offline trace fixtures: `fixtures/*.json` +- Field definitions: `schema.md` +- Evaluator implementation: `DiagnosisTraceEvaluator` +- Report writer: `DiagnosisEvalReportWriter` + +## Current Mode + +The first version evaluates saved trace fixtures. It does not start the application and does not require MySQL, Redis, Milvus, or a real LLM. + +## Verification + +Run the focused evaluator test: + +```powershell +mvn -q "-Dtest=DiagnosisTraceEvaluatorTest" test +``` + +## Interview Story + +The harness gives the MVP a repeatable baseline: + +```text +fixed diagnosis case +-> saved or runtime trace +-> rule-based trace validation +-> JSON / Markdown report +-> regression signal for prompts, tools, retrieval, and verifier behavior +``` diff --git a/mvp/eval/cases/diagnosis-cases.json b/mvp/eval/cases/diagnosis-cases.json new file mode 100644 index 0000000..8e75fcf --- /dev/null +++ b/mvp/eval/cases/diagnosis-cases.json @@ -0,0 +1,57 @@ +[ + { + "id": "payment-timeout", + "title": "Payment API timeout", + "question": "支付接口最近出现超时，请结合知识库、日志和指标判断可能原因，并给出修复建议。", + "traceFixture": "payment-timeout-pass.json", + "expectedRootCauseKeywords": ["支付", "超时", "连接池"], + "minKeywordMatches": 2, + "requiredEvidenceTools": ["lookup_knowledge", "query_logs", "query_metrics"], + "allowedVerdicts": ["PASS", "LOW_CONFID"], + "forbiddenAnswerKeywords": ["无证据确定"] + }, + { + "id": "mysql-pool-exhausted", + "title": "MySQL connection pool exhausted", + "question": "订单服务大量请求超时，请判断是否和 MySQL 连接池有关。", + "traceFixture": "mysql-pool-low-confid.json", + "expectedRootCauseKeywords": ["mysql", "连接池", "超时"], + "minKeywordMatches": 2, + "requiredEvidenceTools": ["lookup_knowledge", "query_logs"], + "allowedVerdicts": ["LOW_CONFID", "PASS"], + "forbiddenAnswerKeywords": ["已经完全确认"] + }, + { + "id": "redis-timeout", + "title": "Redis timeout", + "question": "支付服务出现 Redis 连接超时，请定位可能原因。", + "traceFixture": "redis-timeout-missing.json", + "expectedRootCauseKeywords": ["redis", "超时"], + "minKeywordMatches": 2, + "requiredEvidenceTools": ["query_logs"], + "allowedVerdicts": ["LOW_CONFID", "PASS"], + "forbiddenAnswerKeywords": ["无需进一步排查"] + }, + { + "id": "slow-response", + "title": "Slow response", + "question": "用户服务 P99 响应时间升高，请结合指标和日志分析。", + "traceFixture": "slow-response-missing.json", + "expectedRootCauseKeywords": ["p99", "慢响应"], + "minKeywordMatches": 1, + "requiredEvidenceTools": ["query_metrics", "query_logs"], + "allowedVerdicts": ["LOW_CONFID", "PASS"], + "forbiddenAnswerKeywords": ["没有风险"] + }, + { + "id": "jvm-memory-risk", + "title": "JVM memory risk", + "question": "订单服务内存使用率过高，请判断是否存在 OOM 风险。", + "traceFixture": "jvm-memory-risk-missing.json", + "expectedRootCauseKeywords": ["jvm", "内存", "oom"], + "minKeywordMatches": 2, + "requiredEvidenceTools": ["query_metrics", "query_logs"], + "allowedVerdicts": ["LOW_CONFID", "PASS"], + "forbiddenAnswerKeywords": ["可以忽略"] + } +] diff --git a/mvp/eval/fixtures/mysql-pool-low-confid.json b/mvp/eval/fixtures/mysql-pool-low-confid.json new file mode 100644 index 0000000..93913f7 --- /dev/null +++ b/mvp/eval/fixtures/mysql-pool-low-confid.json @@ -0,0 +1,53 @@ +{ + "session": { + "sessionId": "eval-mysql-pool", + "query": "订单服务大量请求超时，请判断是否和 MySQL 连接池有关。", + "status": "SUCCESS", + "agentFlow": "CHAT", + "totalDurationMs": 51000, + "toolCallCount": 2, + "answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\nMySQL 连接池可能参与了本次超时问题。日志中出现 connection pool exhausted，但当前缺少完整指标证据，因此只能作为低置信结论处理。", + "selfEvaluation": { + "verifier_evaluation": { + "verdict": "LOW_CONFID", + "groundedness_score": 0.48, + "tool_trace_summary": [ + { + "tool_name": "lookup_knowledge", + "success": true, + "evidence_level": "direct" + }, + { + "tool_name": "query_logs", + "success": true, + "evidence_level": "direct" + } + ] + } + } + }, + "steps": [], + "toolInvocations": [ + { + "id": 1, + "sessionId": "eval-mysql-pool", + "toolName": "lookup_knowledge", + "success": true, + "relevanceLevel": "PRECISE" + }, + { + "id": 2, + "sessionId": "eval-mysql-pool", + "toolName": "query_logs", + "success": true + } + ], + "summary": { + "persistedStepCount": 3, + "returnedStepCount": 3, + "persistedToolCallCount": 2, + "returnedToolCallCount": 2, + "hasVerifierEvaluation": true, + "hasFeedback": false + } +} diff --git a/mvp/eval/fixtures/payment-timeout-pass.json b/mvp/eval/fixtures/payment-timeout-pass.json new file mode 100644 index 0000000..fe2bc97 --- /dev/null +++ b/mvp/eval/fixtures/payment-timeout-pass.json @@ -0,0 +1,64 @@ +{ + "session": { + "sessionId": "eval-payment-timeout", + "query": "支付接口最近出现超时，请结合知识库、日志和指标判断可能原因，并给出修复建议。", + "status": "SUCCESS", + "agentFlow": "CHAT", + "totalDurationMs": 42000, + "toolCallCount": 3, + "answer": "支付接口超时与连接池等待有关。知识库说明支付超时需要同时检查连接池、日志和指标；日志出现 connection pool exhausted；指标显示支付服务延迟升高。", + "selfEvaluation": { + "verifier_evaluation": { + "verdict": "PASS", + "groundedness_score": 0.86, + "tool_trace_summary": [ + { + "tool_name": "lookup_knowledge", + "success": true, + "evidence_level": "direct" + }, + { + "tool_name": "query_logs", + "success": true, + "evidence_level": "direct" + }, + { + "tool_name": "query_metrics", + "success": true, + "evidence_level": "direct" + } + ] + } + } + }, + "steps": [], + "toolInvocations": [ + { + "id": 1, + "sessionId": "eval-payment-timeout", + "toolName": "lookup_knowledge", + "success": true, + "relevanceLevel": "PRECISE" + }, + { + "id": 2, + "sessionId": "eval-payment-timeout", + "toolName": "query_logs", + "success": true + }, + { + "id": 3, + "sessionId": "eval-payment-timeout", + "toolName": "query_metrics", + "success": true + } + ], + "summary": { + "persistedStepCount": 3, + "returnedStepCount": 3, + "persistedToolCallCount": 3, + "returnedToolCallCount": 3, + "hasVerifierEvaluation": true, + "hasFeedback": false + } +} diff --git a/mvp/eval/schema.md b/mvp/eval/schema.md new file mode 100644 index 0000000..e58b2ac --- /dev/null +++ b/mvp/eval/schema.md @@ -0,0 +1,137 @@ +# Diagnosis Eval Data Schema + +这份文档记录评测基准里的数据结构。口语化理解就是： + +```text +用例文件说“我要考什么” +trace 文件说“Agent 实际做了什么” +评测结果说“这次有没有跑偏” +汇总报告说“整体稳定性怎么样” +``` + +当前这套评测是代码规则判断，不是再调用一个 LLM 来打分。 + +## 1. 用例定义 + +文件：`mvp/eval/cases/diagnosis-cases.json` + +每一条 case 是一个固定考题，告诉评测器“这个问题应该看哪些点、需要哪些证据、哪些结论可以接受”。 + +```json +{ + "id": "payment-timeout", + "title": "Payment API timeout", + "question": "支付接口最近出现超时，请结合知识库、日志和指标判断可能原因，并给出修复建议。", + "traceFixture": "payment-timeout-pass.json", + "expectedRootCauseKeywords": ["支付", "超时", "连接池"], + "minKeywordMatches": 2, + "requiredEvidenceTools": ["lookup_knowledge", "query_logs", "query_metrics"], + "allowedVerdicts": ["PASS", "LOW_CONFID"], + "forbiddenAnswerKeywords": ["无证据确定"] +} +``` + +字段说明： + +| 字段 | 意思 | 评测器怎么用 | +| --- | --- | --- | +| `id` | 这条用例的唯一名字 | 出现在报告里，方便定位是哪条 case 挂了 | +| `title` | 给人看的标题 | 出现在结果里，方便快速理解场景 | +| `question` | 要问 Agent 的问题 | fixture 模式下不会真的发送给 Agent，但它记录了这条 case 的原始输入 | +| `traceFixture` | 对应的 trace 文件名 | 评测器会去 `fixtures/` 目录加载这个文件 | +| `expectedRootCauseKeywords` | 最终回答里希望看到的关键点 | 评测器会在 `session.answer` 里做关键词命中检查 | +| `minKeywordMatches` | 至少要命中几个关键词 | 命中数低于这个值，就认为根因覆盖不够 | +| `requiredEvidenceTools` | 这条 case 至少应该用到哪些证据工具 | 评测器会检查 trace 里是否出现这些工具 | +| `allowedVerdicts` | Verifier 允许给出的结论 | 比如 `PASS` 或 `LOW_CONFID`，不在列表里就失败 | +| `forbiddenAnswerKeywords` | 回答里不应该出现的危险说法 | 命中这些词，说明回答可能过度自信或不符合降级策略 | + +## 2. Trace Fixture + +目录：`mvp/eval/fixtures/*.json` + +trace fixture 是一次 Agent 运行后的“留痕快照”。评测器不会关心整个 trace 的所有字段，只读取当前能支撑基准判断的字段。 + +当前会读取这些字段： + +| Trace 字段 | 意思 | 评测器怎么用 | +| --- | --- | --- | +| `session.answer` | Agent 最终给用户的回答 | 用来检查根因关键词和禁用词 | +| `session.totalDurationMs` | 这次运行耗时 | 进入报告，帮助观察性能是否明显变差 | +| `session.selfEvaluation.verifier_evaluation.verdict` | Verifier 对最终回答的判断 | 必须存在，并且要落在 case 的 `allowedVerdicts` 里 | +| `session.selfEvaluation.verifier_evaluation.tool_trace_summary[*].tool_name` | Verifier 总结里看到的工具证据 | 用来补充判断证据工具是否出现 | +| `toolInvocations[*].toolName` | Agent 实际调用过的工具名 | 用来检查 `requiredEvidenceTools` 是否满足 | +| `toolInvocations[*].success` | 工具调用是否成功 | 当前主要保留在 trace 里，后续可以升级成更严格的成功率检查 | + +简单说，trace 里最重要的是三类信息： + +```text +最终回答：它说了什么 +工具证据：它查了什么 +Verifier：它自己有没有承认这个结论可靠 +``` + +## 3. 单条评测结果 + +Java 类型：`DiagnosisEvalResult` + +这是每条 case 跑完之后的判断结果。 + +| 字段 | 意思 | +| --- | --- | +| `caseId` | 对应的 case id | +| `title` | case 标题 | +| `passed` | 这条 case 是否通过 | +| `failedChecks` | 没通过的具体原因，比如缺工具、关键词不够、verdict 不允许 | +| `verdict` | 从 trace 里读出来的 Verifier verdict | +| `matchedKeywordCount` | 最终回答命中的关键词数量 | +| `requiredKeywordCount` | case 定义里一共有多少个关键词 | +| `evidenceCoverage` | 每个必需工具是否出现，例如 `{ "query_logs": true }` | +| `toolCallCount` | 本次 trace 里工具调用总数 | +| `durationMs` | 本次 trace 的耗时 | + +判断通过的口语化规则： + +```text +回答要说到关键点 +该查的证据工具要查到 +Verifier 的结论要在可接受范围内 +回答不能出现危险的过度自信表达 +如果是 REJECT，就必须走降级模板 +``` + +## 4. 汇总报告 + +Java 类型：`DiagnosisEvalReport` + +这是整个基准集跑完之后的总结果。 + +| 字段 | 意思 | +| --- | --- | +| `totalCases` | 总共评测了多少条 case | +| `passedCases` | 通过了多少条 | +| `passRate` | 通过率，范围是 `0.0` 到 `1.0` | +| `verdictDistribution` | Verifier verdict 的分布，比如有几个 `PASS`、几个 `LOW_CONFID` | +| `averageToolCallCount` | 平均每条 case 调用了多少次工具 | +| `averageDurationMs` | 平均耗时 | +| `results` | 每条 case 的详细结果列表 | + +## 5. 怎么看这个基准 + +这套结构不是为了证明 Agent 永远正确，而是为了在每次改 prompt、工具、检索、Verifier 之后，有一个固定尺子能回答： + +```text +以前能过的诊断题，现在还过不过？ +它是不是少查了某些证据？ +它是不是变得更自信但证据不足？ +它是不是开始输出不该说的话？ +它是不是明显变慢了？ +``` + +所以面试里可以这样讲： + +```text +我没有只看一次 demo 效果，而是把典型诊断场景固化成 case。 +每条 case 都定义预期关键点、必需证据工具和可接受的 verifier 结论。 +Agent 每次运行会留下 trace，评测器用代码规则读取 trace，输出结构化报告。 +这样我改 Agent 的时候，可以用同一套基准判断有没有行为回退。 +``` diff --git a/mvp/issues/ISS-006-diagnosis-eval-harness.md b/mvp/issues/ISS-006-diagnosis-eval-harness.md new file mode 100644 index 0000000..e4c13d9 --- /dev/null +++ b/mvp/issues/ISS-006-diagnosis-eval-harness.md @@ -0,0 +1,99 @@ +# ISS-006 固定诊断评测集与回归 Harness + +**状态**：进行中（sm-flow） +**严重程度**：高 +**发现时间**：2026-07-04 +**来源**：P1-B 面试打磨项 +**依赖**：ISS-005 / `evidence-trace-hardening` + +--- + +## 背景 + +MVP 已经具备可追溯证据链、Verifier 质量门禁、trace API 和固定 demo 流程。上一阶段 `evidence-trace-hardening` 进一步统一了 evidence tool 的状态语义，让系统能稳定区分： + +- `supported` +- `no_evidence` +- `deduped` +- `failed` + +下一步需要证明 Agent 在一组固定诊断场景下的表现，而不是只依赖单次 demo。 + +--- + +## 问题 + +当前项目能演示一次支付超时诊断，但还缺少稳定的评测基线： + +- 每次改 prompt、工具、Verifier 或检索逻辑后，无法快速判断是否退化。 +- 只能人工看 trace，缺少结构化通过 / 失败结果。 +- 缺少面试时能展示的指标，如 evidence coverage、verdict 分布、工具调用数量和耗时。 + +--- + +## 目标 + +建立一个轻量的固定 case 评测 harness，用于验证 MVP Agent 的诊断质量和证据链完整性。 + +第一版不做 LLM-as-judge，优先做规则化校验： + +- 固定 5 个 MVP 诊断 case +- 每个 case 定义 expected root-cause keywords、required evidence tools、allowed verdicts +- 基于 trace 结果校验 evidence coverage、verifier evaluation、tool invocation、final answer shape +- 输出 JSON 和 Markdown 报告 + +--- + +## 范围 + +### In scope + +- 评测 case 定义文件 +- trace 规则校验器 +- eval runner 或测试入口 +- JSON / Markdown 报告输出 +- demo 文档和 devflow 记录 + +### Out of scope + +- 不引入 LLM-as-judge +- 不要求完整离线 LLM runtime +- 不新增生产 API +- 不修改 Chat 主链路 +- 不修改 evidence trace 运行时语义 + +--- + +## 预期面试表达 + +完成后可以这样描述： + +```text +我不仅有一个可演示的 Agent，还给它建立了固定 case 的回归评测。 +每次修改 prompt、工具或 verifier 后，都可以跑同一批诊断 case， +检查证据覆盖、verdict 分布、工具调用成本和关键结论是否退化。 +``` + +--- + +## 初始候选 case + +| Case | 目标 | +| --- | --- | +| payment-timeout | 支付接口超时，验证知识库 + 日志 + 指标证据 | +| mysql-pool-exhausted | 数据库连接池耗尽，验证日志和知识库证据 | +| redis-timeout | Redis 连接超时，验证日志依赖证据 | +| slow-response | P99 响应时间过高，验证指标 + 慢请求日志 | +| jvm-memory-risk | JVM 内存 / OOM 风险，验证指标 + 系统事件日志 | + +--- + +## 相关文件 + +- `mvp/demo/README.md` +- `mvp/demo/payment-timeout-acceptance.md` +- `src/main/java/com/superbiz/agent/service/DiagnosisTraceService.java` +- `src/main/java/com/superbiz/agent/service/ToolTraceSummaryService.java` +- `src/main/java/com/superbiz/agent/domain/entity/DiagnosisSession.java` +- `src/main/java/com/superbiz/agent/domain/entity/ToolInvocation.java` +- `openspec/specs/evidence-trace-hardening/spec.md` diff --git a/mvp/issues/README.md b/mvp/issues/README.md index 4836085..1b66cb0 100644 --- a/mvp/issues/README.md +++ b/mvp/issues/README.md @@ -6,4 +6,5 @@ | ISS-002 | Executor 无约束重复调用 lookup_knowledge | 中 | 已修复 | [ISS-002-executor-unconstrained-lookup.md](ISS-002-executor-unconstrained-lookup.md) | | ISS-003 | MVP 设计与实现 Review 收敛 | 高 | 待规划 | [ISS-003-mvp-design-implementation-review.md](ISS-003-mvp-design-implementation-review.md) | | ISS-004 | Executor 域级检索水位控制（Phase 2） | 低 | 待规划 | [ISS-004-executor-domain-hard-limit.md](ISS-004-executor-domain-hard-limit.md) | -| ISS-005 | 证据链补齐与降级契约收敛 | 高 | 进行中（sm-flow） | [ISS-005-evidence-trace-hardening.md](ISS-005-evidence-trace-hardening.md) | +| ISS-005 | 证据链补齐与降级契约收敛 | 高 | 已归档 | [ISS-005-evidence-trace-hardening.md](ISS-005-evidence-trace-hardening.md) | +| ISS-006 | 固定诊断评测集与回归 Harness | 高 | 已归档 | [ISS-006-diagnosis-eval-harness.md](ISS-006-diagnosis-eval-harness.md) | diff --git a/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/.openspec.yaml b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/.openspec.yaml new file mode 100644 index 0000000..d86f152 --- /dev/null +++ b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-04 diff --git a/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/design.md b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/design.md new file mode 100644 index 0000000..1179378 --- /dev/null +++ b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/design.md @@ -0,0 +1,54 @@ +## Context + +The project now has the pieces needed for trace-based evaluation: + +- `diagnosis_session` stores final answer, status, duration, counts, feedback, and `self_evaluation` +- `agent_step` stores ordered agent execution records +- `tool_invocation` stores evidence tool calls with normalized evidence semantics +- `GET /api/diagnosis/{sessionId}/trace` can aggregate one diagnosis trace for demo review +- `evidence-trace-hardening` defined stable `supported`, `no_evidence`, `deduped`, and `failed` semantics + +P1-B should not add another runtime agent. It should create a repeatable evaluation surface that can be used after changing prompts, retrieval behavior, tools, or verifier logic. + +## Goals / Non-Goals + +**Goals:** +- Define fixed MVP diagnosis cases with expected evidence and verdict rules. +- Build a deterministic evaluator that can validate a diagnosis trace against a case definition. +- Produce JSON and Markdown reports with pass/fail status and key metrics. +- Keep the first version usable without a real LLM by allowing fixture trace inputs. +- Leave room for a later runtime mode that queries the trace API after a demo run. + +**Non-Goals:** +- No LLM-as-judge in this slice. +- No automatic prompt optimization. +- No new production API. +- No change to chat, verifier, retrieval, upload, or feedback behavior. +- No requirement to start MySQL/Redis/Milvus/LLM for the first offline evaluator. + +## Decisions + +| Decision | Choice | Alternative Considered | Rationale | +|---|---|---|---| +| Evaluation source | Start with fixture / persisted trace JSON input | Always run live `/api/chat` first | Keeps the first harness deterministic and avoids mixing quality checks with external infrastructure availability. | +| Judging strategy | Rule-based trace validation | LLM-as-judge | The immediate goal is regression signal for evidence coverage and degraded behavior, not subjective answer scoring. | +| Case format | Static JSON/YAML case definitions | Hard-coded Java tests only | Case files are easier to inspect and explain in interviews. | +| Report format | JSON plus Markdown | Console-only output | JSON supports automation; Markdown supports quick human review. | +| Metrics | Evidence coverage, verdict distribution, tool-call count, duration, answer keyword coverage | Full semantic correctness | These metrics are available from existing trace data and align with the MVP's observable contract. | + +## Risks / Trade-offs + +- [Risk] Rule-based keyword checks can be brittle. -> Mitigation: keep checks focused on required evidence, verdicts, and high-signal root-cause terms rather than exact answer text. +- [Risk] Fixture-only evaluation may drift from runtime behavior. -> Mitigation: design the evaluator around the same trace response shape so runtime traces can be fed in later. +- [Risk] Metrics may encourage gaming tool counts. -> Mitigation: report tool counts as cost/efficiency signals, not the sole pass/fail criterion. +- [Risk] Too many cases can slow iteration. -> Mitigation: start with 5 MVP cases and keep each case small. + +## Migration Plan + +- No deployment migration is required. +- The harness is additive and can be run locally as a test or script. +- Rollback is deleting the eval case files, runner, and report docs. + +## Open Questions + +- Should runtime trace API polling be included in the first implementation, or left as a follow-up after the fixture validator lands? diff --git a/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/proposal.md b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/proposal.md new file mode 100644 index 0000000..216e728 --- /dev/null +++ b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/proposal.md @@ -0,0 +1,27 @@ +## Why + +The MVP can now run a traceable diagnosis flow, but it still lacks a repeatable way to evaluate whether changes to prompts, tools, retrieval, or verifier behavior improve or regress agent quality. A fixed diagnosis evaluation harness gives the project an interview-ready quality baseline instead of relying on a single manual demo. + +## What Changes + +- Add a small fixed evaluation set for representative MVP diagnosis scenarios. +- Define expected assertions per case: root-cause keywords, required evidence tools, allowed verifier verdicts, and forbidden behavior. +- Add a trace-based evaluator that checks persisted diagnosis traces for evidence coverage, verifier output, final answer shape, tool-call count, and duration. +- Add JSON and Markdown report output for quick review after a run. +- Add documentation that explains how this evaluation harness should be used during prompt/tool/verifier iteration. + +## Capabilities + +### New Capabilities +- `diagnosis-eval-harness`: Defines fixed diagnosis cases, trace-based validation rules, and evaluation report output for MVP Agent regression checks. + +### Modified Capabilities +- None. + +## Impact + +- Affected areas: evaluation resources/scripts/tests, MVP demo documentation, and devflow records. +- Affected runtime behavior: none. This change reads persisted trace data or fixture trace data and does not modify the chat execution path. +- Affected APIs: none. +- Dependencies: relies on the evidence semantics from `evidence-trace-hardening`, especially `tool_invocation`, `tool_trace_summary`, `verifier_evaluation`, and evidence status conventions. +- Non-goals: no LLM-as-judge, no full offline LLM runtime, no new production endpoint, no schema migration. diff --git a/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/specs/diagnosis-eval-harness/spec.md b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/specs/diagnosis-eval-harness/spec.md new file mode 100644 index 0000000..0200255 --- /dev/null +++ b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/specs/diagnosis-eval-harness/spec.md @@ -0,0 +1,59 @@ +## ADDED Requirements + +### Requirement: Evaluation harness SHALL define fixed diagnosis cases +The system SHALL provide a small fixed set of MVP diagnosis evaluation cases with explicit expected trace and answer criteria. + +#### Scenario: Case definition includes expected evidence +- **WHEN** an evaluation case is defined +- **THEN** it SHALL include a case id, user question, expected root-cause keywords, required evidence tools, and allowed verifier verdicts + +#### Scenario: Case definition can express forbidden behavior +- **WHEN** a case has known unsafe behavior +- **THEN** the case definition SHALL be able to declare forbidden answer keywords or forbidden verdicts + +### Requirement: Evaluation harness SHALL validate diagnosis traces +The system SHALL validate a diagnosis trace against the corresponding case definition using deterministic rules. + +#### Scenario: Evidence coverage validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that required evidence tools appear in `toolInvocations` or verifier trace summaries + +#### Scenario: Verifier evaluation validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that `selfEvaluation.verifier_evaluation.verdict` exists +- **AND** the verdict SHALL be one of the case's allowed verdicts + +#### Scenario: Answer keyword validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that the final answer includes at least the configured minimum root-cause keyword coverage + +#### Scenario: Degraded output validation +- **WHEN** a trace verdict is `REJECT` +- **THEN** the evaluator SHALL verify that the final answer follows the degraded-output contract rather than leaking an unverified executor answer + +### Requirement: Evaluation harness SHALL report quality and cost signals +The system SHALL produce a report that summarizes pass/fail results and key trace metrics. + +#### Scenario: JSON report output +- **WHEN** an evaluation run completes +- **THEN** the evaluator SHALL output a JSON report with per-case result, failed checks, verdict, evidence coverage, tool-call count, and duration + +#### Scenario: Markdown report output +- **WHEN** an evaluation run completes +- **THEN** the evaluator SHALL output a Markdown report suitable for review in the repository + +#### Scenario: Aggregate metrics +- **WHEN** multiple cases are evaluated +- **THEN** the report SHALL include aggregate pass rate, verdict distribution, average tool-call count, and average duration when available + +### Requirement: Evaluation harness SHALL support offline fixture mode +The first evaluator version SHALL be runnable without live MySQL, Redis, Milvus, or LLM services by evaluating saved trace fixtures. + +#### Scenario: Fixture trace evaluation +- **WHEN** the evaluator is run against a directory of trace fixture files +- **THEN** it SHALL evaluate each trace file against its matching case definition +- **AND** it SHALL not require a running application service + +#### Scenario: Missing fixture is reported clearly +- **WHEN** a case has no matching trace fixture +- **THEN** the evaluator SHALL mark the case as not run or failed with a clear reason diff --git a/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/tasks.md b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/tasks.md new file mode 100644 index 0000000..b20dfce --- /dev/null +++ b/openspec/changes/archive/2026-07-04-diagnosis-eval-harness/tasks.md @@ -0,0 +1,25 @@ +## 1. Case Definitions + +- [x] 1.1 Add evaluation case definition format for fixed MVP diagnosis scenarios. +- [x] 1.2 Add the first 5 case definitions: payment timeout, MySQL pool exhausted, Redis timeout, slow response, and JVM memory risk. +- [x] 1.3 Document the meaning of expected keywords, required evidence tools, allowed verdicts, and forbidden behavior. + +## 2. Trace Fixtures + +- [x] 2.1 Add fixture trace schema or DTOs that match `DiagnosisTraceResponse` enough for offline evaluation. +- [x] 2.2 Add at least one representative trace fixture for a passing case. +- [x] 2.3 Add at least one fixture covering low-confidence or degraded behavior. + +## 3. Evaluator + +- [x] 3.1 Implement trace validation rules for evidence coverage, verifier verdict, answer keyword coverage, and degraded-output contract. +- [x] 3.2 Implement aggregate metrics: pass rate, verdict distribution, average tool-call count, and average duration. +- [x] 3.3 Implement JSON report output. +- [x] 3.4 Implement Markdown report output. + +## 4. Tests And Documentation + +- [x] 4.1 Add focused offline tests for the evaluator. +- [x] 4.2 Add run instructions under `mvp/demo` or `mvp/notes`. +- [x] 4.3 Run targeted tests for the evaluator. +- [x] 4.4 Run compile verification. diff --git a/openspec/specs/diagnosis-eval-harness/spec.md b/openspec/specs/diagnosis-eval-harness/spec.md new file mode 100644 index 0000000..306bac9 --- /dev/null +++ b/openspec/specs/diagnosis-eval-harness/spec.md @@ -0,0 +1,64 @@ +# diagnosis-eval-harness Specification + +## Purpose +Provide a repeatable offline evaluation harness for MVP diagnosis Agent behavior, so prompt, tool, retrieval, and verifier changes can be checked against fixed trace-based regression cases. + +## Requirements + +### Requirement: Evaluation harness SHALL define fixed diagnosis cases +The system SHALL provide a small fixed set of MVP diagnosis evaluation cases with explicit expected trace and answer criteria. + +#### Scenario: Case definition includes expected evidence +- **WHEN** an evaluation case is defined +- **THEN** it SHALL include a case id, user question, expected root-cause keywords, required evidence tools, and allowed verifier verdicts + +#### Scenario: Case definition can express forbidden behavior +- **WHEN** a case has known unsafe behavior +- **THEN** the case definition SHALL be able to declare forbidden answer keywords or forbidden verdicts + +### Requirement: Evaluation harness SHALL validate diagnosis traces +The system SHALL validate a diagnosis trace against the corresponding case definition using deterministic rules. + +#### Scenario: Evidence coverage validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that required evidence tools appear in `toolInvocations` or verifier trace summaries + +#### Scenario: Verifier evaluation validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that `selfEvaluation.verifier_evaluation.verdict` exists +- **AND** the verdict SHALL be one of the case's allowed verdicts + +#### Scenario: Answer keyword validation +- **WHEN** a trace is evaluated +- **THEN** the evaluator SHALL verify that the final answer includes at least the configured minimum root-cause keyword coverage + +#### Scenario: Degraded output validation +- **WHEN** a trace verdict is `REJECT` +- **THEN** the evaluator SHALL verify that the final answer follows the degraded-output contract rather than leaking an unverified executor answer + +### Requirement: Evaluation harness SHALL report quality and cost signals +The system SHALL produce a report that summarizes pass/fail results and key trace metrics. + +#### Scenario: JSON report output +- **WHEN** an evaluation run completes +- **THEN** the evaluator SHALL output a JSON report with per-case result, failed checks, verdict, evidence coverage, tool-call count, and duration + +#### Scenario: Markdown report output +- **WHEN** an evaluation run completes +- **THEN** the evaluator SHALL output a Markdown report suitable for review in the repository + +#### Scenario: Aggregate metrics +- **WHEN** multiple cases are evaluated +- **THEN** the report SHALL include aggregate pass rate, verdict distribution, average tool-call count, and average duration when available + +### Requirement: Evaluation harness SHALL support offline fixture mode +The first evaluator version SHALL be runnable without live MySQL, Redis, Milvus, or LLM services by evaluating saved trace fixtures. + +#### Scenario: Fixture trace evaluation +- **WHEN** the evaluator is run against a directory of trace fixture files +- **THEN** it SHALL evaluate each trace file against its matching case definition +- **AND** it SHALL not require a running application service + +#### Scenario: Missing fixture is reported clearly +- **WHEN** a case has no matching trace fixture +- **THEN** the evaluator SHALL mark the case as not run or failed with a clear reason diff --git a/src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java new file mode 100644 index 0000000..8c5de19 --- /dev/null +++ b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java @@ -0,0 +1,25 @@ +package com.superbiz.agent.eval; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DiagnosisEvalCase { + + private String id; + private String title; + private String question; + private String traceFixture; + private List expectedRootCauseKeywords; + private Integer minKeywordMatches; + private List requiredEvidenceTools; + private List allowedVerdicts; + private List forbiddenAnswerKeywords; +} diff --git a/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReport.java b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReport.java new file mode 100644 index 0000000..e7f0f05 --- /dev/null +++ b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReport.java @@ -0,0 +1,24 @@ +package com.superbiz.agent.eval; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DiagnosisEvalReport { + + private int totalCases; + private int passedCases; + private double passRate; + private Map verdictDistribution; + private double averageToolCallCount; + private double averageDurationMs; + private List results; +} diff --git a/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java new file mode 100644 index 0000000..35fc10f --- /dev/null +++ b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java @@ -0,0 +1,76 @@ +package com.superbiz.agent.eval; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +public class DiagnosisEvalReportWriter { + + private final ObjectMapper objectMapper; + + public DiagnosisEvalReportWriter(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public void writeJson(DiagnosisEvalReport report, Path outputFile) throws IOException { + Files.createDirectories(outputFile.getParent()); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report); + } + + public void writeMarkdown(DiagnosisEvalReport report, Path outputFile) throws IOException { + Files.createDirectories(outputFile.getParent()); + Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8); + } + + public String toMarkdown(DiagnosisEvalReport report) { + StringBuilder builder = new StringBuilder(); + builder.append("# Diagnosis Eval Report\n\n"); + builder.append("- Total cases: ").append(report.getTotalCases()).append("\n"); + builder.append("- Passed cases: ").append(report.getPassedCases()).append("\n"); + builder.append("- Pass rate: ").append(String.format("%.2f%%", report.getPassRate() * 100)).append("\n"); + builder.append("- Average tool calls: ").append(String.format("%.2f", report.getAverageToolCallCount())).append("\n"); + builder.append("- Average duration ms: ").append(String.format("%.2f", report.getAverageDurationMs())).append("\n\n"); + + builder.append("## Verdict Distribution\n\n"); + if (report.getVerdictDistribution() == null || report.getVerdictDistribution().isEmpty()) { + builder.append("- None\n\n"); + } else { + for (Map.Entry entry : report.getVerdictDistribution().entrySet()) { + builder.append("- ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n"); + } + builder.append("\n"); + } + + builder.append("## Cases\n\n"); + builder.append("| Case | Result | Verdict | Keywords | Tool Calls | Duration ms | Failed Checks |\n"); + builder.append("| --- | --- | --- | --- | ---: | ---: | --- |\n"); + for (DiagnosisEvalResult result : report.getResults()) { + builder.append("| ") + .append(result.getCaseId()) + .append(" | ") + .append(result.isPassed() ? "PASS" : "FAIL") + .append(" | ") + .append(valueOrDash(result.getVerdict())) + .append(" | ") + .append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount()) + .append(" | ") + .append(result.getToolCallCount() == null ? "-" : result.getToolCallCount()) + .append(" | ") + .append(result.getDurationMs() == null ? "-" : result.getDurationMs()) + .append(" | ") + .append(result.getFailedChecks() == null || result.getFailedChecks().isEmpty() + ? "-" + : String.join("; ", result.getFailedChecks())) + .append(" |\n"); + } + return builder.toString(); + } + + private String valueOrDash(String value) { + return value == null || value.isBlank() ? "-" : value; + } +} diff --git a/src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java new file mode 100644 index 0000000..a18b2c8 --- /dev/null +++ b/src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java @@ -0,0 +1,27 @@ +package com.superbiz.agent.eval; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DiagnosisEvalResult { + + private String caseId; + private String title; + private boolean passed; + private List failedChecks; + private String verdict; + private int matchedKeywordCount; + private int requiredKeywordCount; + private Map evidenceCoverage; + private Integer toolCallCount; + private Integer durationMs; +} diff --git a/src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java b/src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java new file mode 100644 index 0000000..77588f5 --- /dev/null +++ b/src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java @@ -0,0 +1,217 @@ +package com.superbiz.agent.eval; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.dto.DiagnosisTraceResponse; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class DiagnosisTraceEvaluator { + + private static final TypeReference> CASE_LIST_TYPE = new TypeReference<>() {}; + private static final String REJECT_DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论"; + + private final ObjectMapper objectMapper; + + public DiagnosisTraceEvaluator(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public List loadCases(Path casesFile) throws IOException { + return objectMapper.readValue(casesFile.toFile(), CASE_LIST_TYPE); + } + + public DiagnosisTraceResponse loadTrace(Path traceFile) throws IOException { + return objectMapper.readValue(traceFile.toFile(), DiagnosisTraceResponse.class); + } + + public DiagnosisEvalReport evaluate(List cases, Path fixtureDir) { + List results = new ArrayList<>(); + for (DiagnosisEvalCase evalCase : cases) { + try { + DiagnosisTraceResponse trace = loadTrace(fixtureDir.resolve(evalCase.getTraceFixture())); + results.add(evaluate(evalCase, trace)); + } catch (Exception e) { + results.add(DiagnosisEvalResult.builder() + .caseId(evalCase.getId()) + .title(evalCase.getTitle()) + .passed(false) + .failedChecks(List.of("trace fixture unavailable: " + e.getMessage())) + .verdict(null) + .matchedKeywordCount(0) + .requiredKeywordCount(size(evalCase.getExpectedRootCauseKeywords())) + .evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools())) + .toolCallCount(null) + .durationMs(null) + .build()); + } + } + return toReport(results); + } + + public DiagnosisEvalResult evaluate(DiagnosisEvalCase evalCase, DiagnosisTraceResponse trace) { + List failedChecks = new ArrayList<>(); + String answer = trace.getSession() == null ? "" : nullToEmpty(trace.getSession().getAnswer()); + String normalizedAnswer = answer.toLowerCase(Locale.ROOT); + + int requiredKeywordCount = size(evalCase.getExpectedRootCauseKeywords()); + int matchedKeywordCount = countMatches(normalizedAnswer, evalCase.getExpectedRootCauseKeywords()); + int minKeywordMatches = evalCase.getMinKeywordMatches() == null + ? requiredKeywordCount + : evalCase.getMinKeywordMatches(); + if (matchedKeywordCount < minKeywordMatches) { + failedChecks.add("answer keyword coverage too low: " + matchedKeywordCount + "/" + minKeywordMatches); + } + + for (String forbidden : safeList(evalCase.getForbiddenAnswerKeywords())) { + if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) { + failedChecks.add("answer contains forbidden keyword: " + forbidden); + } + } + + Set evidenceTools = collectEvidenceTools(trace); + Map evidenceCoverage = new LinkedHashMap<>(); + for (String requiredTool : safeList(evalCase.getRequiredEvidenceTools())) { + boolean present = evidenceTools.contains(requiredTool); + evidenceCoverage.put(requiredTool, present); + if (!present) { + failedChecks.add("missing required evidence tool: " + requiredTool); + } + } + + String verdict = extractVerifierVerdict(trace); + if (verdict == null || verdict.isBlank()) { + failedChecks.add("missing verifier verdict"); + } else if (!safeList(evalCase.getAllowedVerdicts()).isEmpty() + && !safeList(evalCase.getAllowedVerdicts()).contains(verdict)) { + failedChecks.add("verdict not allowed: " + verdict); + } + + if ("REJECT".equals(verdict) && !answer.startsWith(REJECT_DEGRADED_PREFIX)) { + failedChecks.add("reject output does not use degraded template"); + } + + Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size(); + Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs(); + + return DiagnosisEvalResult.builder() + .caseId(evalCase.getId()) + .title(evalCase.getTitle()) + .passed(failedChecks.isEmpty()) + .failedChecks(failedChecks) + .verdict(verdict) + .matchedKeywordCount(matchedKeywordCount) + .requiredKeywordCount(requiredKeywordCount) + .evidenceCoverage(evidenceCoverage) + .toolCallCount(toolCallCount) + .durationMs(durationMs) + .build(); + } + + private DiagnosisEvalReport toReport(List results) { + int total = results.size(); + int passed = (int) results.stream().filter(DiagnosisEvalResult::isPassed).count(); + Map verdictDistribution = results.stream() + .map(DiagnosisEvalResult::getVerdict) + .filter(Objects::nonNull) + .collect(Collectors.groupingBy(value -> value, LinkedHashMap::new, Collectors.counting())); + double averageToolCallCount = results.stream() + .map(DiagnosisEvalResult::getToolCallCount) + .filter(Objects::nonNull) + .mapToInt(Integer::intValue) + .average() + .orElse(0.0); + double averageDurationMs = results.stream() + .map(DiagnosisEvalResult::getDurationMs) + .filter(Objects::nonNull) + .mapToInt(Integer::intValue) + .average() + .orElse(0.0); + + return DiagnosisEvalReport.builder() + .totalCases(total) + .passedCases(passed) + .passRate(total == 0 ? 0.0 : (double) passed / total) + .verdictDistribution(verdictDistribution) + .averageToolCallCount(averageToolCallCount) + .averageDurationMs(averageDurationMs) + .results(results) + .build(); + } + + private Set collectEvidenceTools(DiagnosisTraceResponse trace) { + Set tools = new LinkedHashSet<>(); + if (trace.getToolInvocations() != null) { + for (DiagnosisTraceResponse.ToolInvocationTrace invocation : trace.getToolInvocations()) { + if (invocation.getToolName() != null) { + tools.add(invocation.getToolName()); + } + } + } + Object summaries = nestedValue(trace, "verifier_evaluation", "tool_trace_summary"); + if (summaries instanceof List list) { + for (Object item : list) { + if (item instanceof Map map && map.get("tool_name") != null) { + tools.add(String.valueOf(map.get("tool_name"))); + } + } + } + return tools; + } + + private String extractVerifierVerdict(DiagnosisTraceResponse trace) { + Object value = nestedValue(trace, "verifier_evaluation", "verdict"); + return value == null ? null : String.valueOf(value); + } + + private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) { + if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) { + return null; + } + Object first = trace.getSession().getSelfEvaluation().get(firstKey); + if (!(first instanceof Map map)) { + return null; + } + return map.get(secondKey); + } + + private int countMatches(String normalizedAnswer, List keywords) { + int count = 0; + for (String keyword : safeList(keywords)) { + if (normalizedAnswer.contains(keyword.toLowerCase(Locale.ROOT))) { + count++; + } + } + return count; + } + + private Map emptyCoverage(List tools) { + Map coverage = new LinkedHashMap<>(); + for (String tool : safeList(tools)) { + coverage.put(tool, false); + } + return coverage; + } + + private List safeList(List values) { + return values == null ? List.of() : values; + } + + private int size(List values) { + return values == null ? 0 : values.size(); + } + + private String nullToEmpty(String value) { + return value == null ? "" : value; + } +} diff --git a/src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java b/src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java new file mode 100644 index 0000000..a59c4a1 --- /dev/null +++ b/src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java @@ -0,0 +1,97 @@ +package com.superbiz.agent.eval; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.dto.DiagnosisTraceResponse; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DiagnosisTraceEvaluatorTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper); + + @Test + void evaluateFixtureReportsPassingAndMissingCases() { + List cases = readCases(); + + DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures")); + + assertEquals(5, report.getTotalCases()); + assertEquals(2, report.getPassedCases()); + assertEquals(0.4, report.getPassRate(), 0.001); + assertEquals(1L, report.getVerdictDistribution().get("PASS")); + assertEquals(1L, report.getVerdictDistribution().get("LOW_CONFID")); + + DiagnosisEvalResult payment = result(report, "payment-timeout"); + assertTrue(payment.isPassed()); + assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge")); + assertTrue(payment.getEvidenceCoverage().get("query_logs")); + assertTrue(payment.getEvidenceCoverage().get("query_metrics")); + + DiagnosisEvalResult missing = result(report, "redis-timeout"); + assertFalse(missing.isPassed()); + assertTrue(missing.getFailedChecks().get(0).contains("trace fixture unavailable")); + } + + @Test + void evaluateRejectRequiresDegradedOutput() { + DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() + .id("reject-case") + .title("Reject case") + .expectedRootCauseKeywords(List.of()) + .requiredEvidenceTools(List.of()) + .allowedVerdicts(List.of("REJECT")) + .build(); + DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() + .session(DiagnosisTraceResponse.SessionTrace.builder() + .answer("EXECUTOR_FINAL_ANSWER") + .selfEvaluation(java.util.Map.of( + "verifier_evaluation", java.util.Map.of("verdict", "REJECT"))) + .build()) + .toolInvocations(List.of()) + .build(); + + DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); + + assertFalse(result.isPassed()); + assertTrue(result.getFailedChecks().contains("reject output does not use degraded template")); + } + + @Test + void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception { + DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures")); + DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper); + + Path json = tempDir.resolve("eval-report.json"); + Path markdown = tempDir.resolve("eval-report.md"); + writer.writeJson(report, json); + writer.writeMarkdown(report, markdown); + + assertTrue(Files.exists(json)); + assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report")); + assertTrue(Files.readString(markdown).contains("payment-timeout")); + } + + private List readCases() { + try { + return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json")); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) { + return report.getResults().stream() + .filter(item -> caseId.equals(item.getCaseId())) + .findFirst() + .orElseThrow(); + } +}