osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(eval): add evidence pipeline acceptance closure

Browse Source
This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mvp/eval/README.md
+8 -4
View File
@@ -29,18 +29,21 @@ The baseline evaluates saved trace fixtures. It does not start the application a
The committed baseline currently contains:
```text
8 fixed cases
8 passing fixture evaluations
2 PASS verdicts
10 fixed cases
10 passing fixture evaluations
4 PASS verdicts
5 LOW_CONFID verdicts
1 REJECT verdict
```
The three V2 audit-closure cases cover:
The V2 evidence-pipeline matrix covers:
- Positive supported evidence for a narrow HighCPU observation.
- No-evidence `negative_observation` using `$.no_evidence`.
- Gatekeeper failure for a fabricated tool invocation reference.
- Unsupported claim filtering before the final answer.
- Composer fallback rendering without raw Executor JSON leakage.
- Gatekeeper rule set version audit for new matrix fixtures.
## Verification
@@ -76,6 +79,7 @@ Stage 5 adds these V2 checks:
- Required V2 fixtures must include `gatekeeper_result`, `claim_checks`, and `composer_output`.
- `claim_checks` must be structurally auditable.
- Composer output must record whether normal parsing or fallback rendering was used.
- Gatekeeper rule set version can be asserted per fixture.
- Final answers must not leak raw Executor protocol markers such as `executor_evidence_v2`, `answer_version`, `evidence_bindings`, or `claim_id`.
- Configured unsupported claim keywords must not appear as confirmed final-answer content.
mvp/eval/cases/diagnosis-cases.json
+36
View File
@@ -1,4 +1,40 @@
[
{
"id": "narrow-highcpu-observation",
"title": "Narrow HighCPU observation",
"question": "确认 payment-service 当前是否存在 HighCPUUsage 告警。",
"traceFixture": "narrow-highcpu-observation-pass.json",
"expectedRootCauseKeywords": ["payment-service", "HighCPUUsage", "92%"],
"minKeywordMatches": 2,
"requiredEvidenceTools": ["query_metrics"],
"allowedVerdicts": ["PASS"],
"forbiddenAnswerKeywords": ["根因", "修复建议", "通常情况下"],
"requireV2AuditClosure": true,
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["pass"],
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
"expectedComposerStatuses": ["valid"],
"forbiddenConfirmedClaimKeywords": ["数据库连接池"]
},
{
"id": "hikari-no-evidence-negative-observation",
"title": "Hikari no-evidence negative observation",
"question": "确认 inventory-service 当前是否有 HikariCP 连接池耗尽日志。",
"traceFixture": "hikari-no-evidence-negative-observation-pass.json",
"expectedRootCauseKeywords": ["未检索到", "HikariCP", "匹配证据"],
"minKeywordMatches": 2,
"requiredEvidenceTools": ["query_logs"],
"allowedVerdicts": ["PASS"],
"forbiddenAnswerKeywords": ["已排除", "确认没有", "日志层面已排除"],
"requireV2AuditClosure": true,
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["pass"],
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
"expectedComposerStatuses": ["valid"],
"forbiddenConfirmedClaimKeywords": ["已排除 HikariCP"]
},
{
"id": "payment-timeout",
"title": "Payment API timeout",
mvp/eval/fixtures/hikari-no-evidence-negative-observation-pass.json
+126
View File
@@ -0,0 +1,126 @@
{
"session": {
"sessionId": "eval-hikari-no-evidence-negative-observation",
"query": "确认 inventory-service 当前是否有 HikariCP 连接池耗尽日志。",
"status": "SUCCESS",
"agentFlow": "CHAT",
"totalDurationMs": 21000,
"toolCallCount": 1,
"answer": "本次查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志；这只表示当前查询没有匹配证据，仍不能据此判断系统一定健康。",
"selfEvaluation": {
"verifier_evaluation": {
"verdict": "PASS",
"groundedness_score": 1.0,
"critical_fact_count": 1,
"gatekeeper_result": {
"status": "pass",
"severity": "none",
"rule_set_version": "gatekeeper-rules-v1",
"rules": [
{
"id": "evidence.raw_path",
"description": "raw_path must exist in retrieval_details.evidence_refs",
"enabled": true,
"default_severity": "reject"
}
],
"checked_bindings": [
{
"claim_id": "claim-1",
"tool_name": "query_logs",
"source_invocation_id": 12,
"raw_path": "$.no_evidence",
"matched_text": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志",
"status": "pass"
}
],
"failed_rules": [],
"warnings": [],
"errors": []
},
"executor_structured_output": {
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "negative_observation",
"claim_text": "当前查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志。",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"tool_name": "query_logs",
"source_invocation_id": 12,
"raw_path": "$.no_evidence",
"evidence_excerpt": "query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": [
"仅查询了 application-logs 中 inventory-service HikariCP 相关日志"
]
},
"claim_checks": [
{
"claim_id": "claim-1",
"claim_text": "当前查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志。",
"claim_type": "negative_observation",
"verification": "direct_observation",
"detail": "$.no_evidence 只支持本次查询未检索到匹配证据。",
"evidence_refs": [
{
"source_invocation_id": 12,
"raw_path": "$.no_evidence"
}
]
}
],
"facts_checked": [],
"composer_output": {
"status": "valid",
"answer_summary": "本次查询未检索到匹配日志。",
"recommended_actions": [],
"user_facing_answer": "本次查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志；这只表示当前查询没有匹配证据，仍不能据此判断系统一定健康。"
},
"tool_trace_summary": [
{
"tool_name": "query_logs",
"success": true,
"source_invocation_ids": [12],
"evidence_level": "no_evidence"
}
]
}
}
},
"steps": [],
"toolInvocations": [
{
"id": 12,
"sessionId": "eval-hikari-no-evidence-negative-observation",
"toolName": "query_logs",
"outputPreview": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志",
"retrievalDetails": {
"evidence_status": "no_evidence",
"evidence_refs": [
{
"raw_path": "$.no_evidence",
"text": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志"
}
]
},
"success": true
}
],
"summary": {
"persistedStepCount": 3,
"returnedStepCount": 3,
"persistedToolCallCount": 1,
"returnedToolCallCount": 1,
"hasVerifierEvaluation": true,
"hasFeedback": false
}
}
mvp/eval/fixtures/narrow-highcpu-observation-pass.json
+124
View File
@@ -0,0 +1,124 @@
{
"session": {
"sessionId": "eval-narrow-highcpu-observation",
"query": "确认 payment-service 当前是否存在 HighCPUUsage 告警。",
"status": "SUCCESS",
"agentFlow": "CHAT",
"totalDurationMs": 18000,
"toolCallCount": 1,
"answer": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
"selfEvaluation": {
"verifier_evaluation": {
"verdict": "PASS",
"groundedness_score": 1.0,
"critical_fact_count": 1,
"gatekeeper_result": {
"status": "pass",
"severity": "none",
"rule_set_version": "gatekeeper-rules-v1",
"rules": [
{
"id": "evidence.raw_path",
"description": "raw_path must exist in retrieval_details.evidence_refs",
"enabled": true,
"default_severity": "reject"
}
],
"checked_bindings": [
{
"claim_id": "claim-1",
"tool_name": "query_metrics",
"source_invocation_id": 11,
"raw_path": "$.alerts[0]",
"matched_text": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m",
"status": "pass"
}
],
"failed_rules": [],
"warnings": [],
"errors": []
},
"executor_structured_output": {
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "observation",
"claim_text": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_id": 11,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
},
"claim_checks": [
{
"claim_id": "claim-1",
"claim_text": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
"claim_type": "observation",
"verification": "direct_observation",
"detail": "已核验的指标证据直接包含服务名、告警名和 CPU 当前值。",
"evidence_refs": [
{
"source_invocation_id": 11,
"raw_path": "$.alerts[0]"
}
]
}
],
"facts_checked": [],
"composer_output": {
"status": "valid",
"answer_summary": "payment-service 当前存在 HighCPUUsage 告警。",
"recommended_actions": [],
"user_facing_answer": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。"
},
"tool_trace_summary": [
{
"tool_name": "query_metrics",
"success": true,
"source_invocation_ids": [11],
"evidence_level": "direct"
}
]
}
}
},
"steps": [],
"toolInvocations": [
{
"id": 11,
"sessionId": "eval-narrow-highcpu-observation",
"toolName": "query_metrics",
"outputPreview": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m",
"retrievalDetails": {
"evidence_status": "supported",
"evidence_refs": [
{
"raw_path": "$.alerts[0]",
"text": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m"
}
]
},
"success": true
}
],
"summary": {
"persistedStepCount": 3,
"returnedStepCount": 3,
"persistedToolCallCount": 1,
"returnedToolCallCount": 1,
"hasVerifierEvaluation": true,
"hasFeedback": false
}
}
mvp/eval/reports/baseline-report.json
+48 -6
View File
@@ -1,15 +1,49 @@
{
"totalCases" : 8,
"passedCases" : 8,
"totalCases" : 10,
"passedCases" : 10,
"passRate" : 1.0,
"verdictDistribution" : {
"PASS" : 2,
"PASS" : 4,
"LOW_CONFID" : 5,
"REJECT" : 1
},
"averageToolCallCount" : 1.625,
"averageDurationMs" : 44875.0,
"averageToolCallCount" : 1.5,
"averageDurationMs" : 39800.0,
"results" : [ {
"caseId" : "narrow-highcpu-observation",
"title" : "Narrow HighCPU observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 18000
}, {
"caseId" : "hikari-no-evidence-negative-observation",
"title" : "Hikari no-evidence negative observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 21000
}, {
"caseId" : "payment-timeout",
"title" : "Payment API timeout",
"passed" : true,
@@ -23,6 +57,7 @@
"query_metrics" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 3,
@@ -40,6 +75,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -56,6 +92,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 1,
@@ -73,6 +110,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -90,6 +128,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -106,6 +145,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "fail",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
@@ -122,6 +162,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 2,
"toolCallCount" : 1,
@@ -138,9 +179,10 @@
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "composer_malformed",
"claimCheckCount" : 2,
"toolCallCount" : 1,
"durationMs" : 47000
} ]
}
}
mvp/eval/reports/baseline-report.md
+17 -15
View File
@@ -1,26 +1,28 @@
# Diagnosis Eval Report
- Total cases: 8
- Passed cases: 8
- Total cases: 10
- Passed cases: 10
- Pass rate: 100.00%
- Average tool calls: 1.63
- Average duration ms: 44875.00
- Average tool calls: 1.50
- Average duration ms: 39800.00
## Verdict Distribution
- PASS: 2
- PASS: 4
- LOW_CONFID: 5
- REJECT: 1
## Cases
| Case | Result | Verdict | Gatekeeper | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |
| payment-timeout | PASS | PASS | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | valid | 1 | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | valid | 2 | 2/2 | 1 | 44000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | composer_malformed | 2 | 2/2 | 1 | 47000 | - |
| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |
| narrow-highcpu-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 18000 | - |
| hikari-no-evidence-negative-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 21000 | - |
| payment-timeout | PASS | PASS | - | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | - | valid | 1 | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | - | valid | 2 | 2/2 | 1 | 44000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | - | composer_malformed | 2 | 2/2 | 1 | 47000 | - |
mvp/eval/schema.md
+5
View File
@@ -32,6 +32,7 @@ baseline report：整套固定集当前认可的结果
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["pass"],
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
"expectedComposerStatuses": ["valid"],
"forbiddenConfirmedClaimKeywords": ["主库故障"]
}
@@ -54,6 +55,7 @@ baseline report：整套固定集当前认可的结果
| `requireClaimChecks` | 是否要求 `claim_checks` | 要求 claim check 数组存在且非空 |
| `requireComposerOutput` | 是否要求 `composer_output` | 要求 Composer 审计存在并带 `status` |
| `expectedGatekeeperStatuses` | 允许的 Gatekeeper 状态 | 实际 `gatekeeper_result.status` 不在列表中则失败 |
| `expectedGatekeeperRuleSetVersion` | 期望的 Gatekeeper 规则集版本 | 配置后校验 `gatekeeper_result.rule_set_version` |
| `expectedComposerStatuses` | 允许的 Composer 状态 | 实际 `composer_output.status` 不在列表中则失败 |
| `forbiddenConfirmedClaimKeywords` | 不得进入最终答案的未支持结论关键词 | 用于证明 unsupported/external_unknown claim 被过滤 |
@@ -69,6 +71,7 @@ fixture 是一次 Agent 运行后的 trace 快照。评测器只读取当前规
| `session.totalDurationMs` | 运行耗时 | 进入报告 |
| `session.selfEvaluation.verifier_evaluation.verdict` | Verifier 判定 | 必须存在并符合 case 的 `allowedVerdicts` |
| `session.selfEvaluation.verifier_evaluation.gatekeeper_result.status` | Gatekeeper 结果 | V2 case 必须存在；`fail` 不允许搭配 `PASS` |
| `session.selfEvaluation.verifier_evaluation.gatekeeper_result.rule_set_version` | Gatekeeper 规则集版本 | 新矩阵 case 可显式断言该版本 |
| `session.selfEvaluation.verifier_evaluation.claim_checks` | Verifier V2 claim 级校验 | V2 case 必须存在；每项需要 `claim_id`、`verification`、`detail` |
| `session.selfEvaluation.verifier_evaluation.composer_output.status` | Composer 渲染状态 | V2 case 必须存在；记录 `valid`、`composer_malformed` 等 |
| `session.selfEvaluation.verifier_evaluation.executor_structured_output.claims[*].evidence_bindings` | Executor claim 证据绑定 | 如果结构化输出存在，每条 claim 需要证据绑定 |
@@ -91,6 +94,7 @@ Java 类型：`DiagnosisEvalResult`
| `requiredKeywordCount` | case 配置的关键词数量 |
| `evidenceCoverage` | 每个必需工具是否出现 |
| `gatekeeperStatus` | 读到的 `gatekeeper_result.status` |
| `gatekeeperRuleSetVersion` | 读到的 `gatekeeper_result.rule_set_version` |
| `composerStatus` | 读到的 `composer_output.status` |
| `claimCheckCount` | `claim_checks` 数量 |
| `toolCallCount` | trace 中工具调用总数 |
@@ -125,6 +129,7 @@ Executor structured output
- V2 case 必须有 `gatekeeper_result`、`claim_checks`、`composer_output`。
- `gatekeeper_result.status = fail` 时，Verifier verdict 不能是 `PASS`。
- 配置 `expectedGatekeeperRuleSetVersion` 的 case 必须匹配 `gatekeeper_result.rule_set_version`。
- `claim_checks[*].verification` 只能是 `direct_observation`、`reasonable_inference`、`overstated`、`unsupported`、`external_unknown`、`contradicted`。
- Composer 输出必须记录 `status`。
- 最终答案不能泄漏 `executor_evidence_v2`、`answer_version`、`evidence_bindings`、`claim_id`。