feat(eval): add executor audit closure checks

This commit is contained in:
aruo
2026-07-08 10:21:39 +08:00
parent 6015bcbf6f
commit a08672b31e
20 changed files with 1142 additions and 188 deletions
+51
View File
@@ -53,5 +53,56 @@
"requiredEvidenceTools": ["query_metrics", "query_logs"],
"allowedVerdicts": ["LOW_CONFID", "PASS"],
"forbiddenAnswerKeywords": ["可以忽略"]
},
{
"id": "gatekeeper-fabricated-invocation",
"title": "Gatekeeper fabricated invocation",
"question": "支付失败是否能确认由日志中的连接池耗尽导致?",
"traceFixture": "gatekeeper-fabricated-invocation-reject.json",
"expectedRootCauseKeywords": ["证据", "引用", "失败"],
"minKeywordMatches": 2,
"requiredEvidenceTools": ["query_logs"],
"allowedVerdicts": ["REJECT", "LOW_CONFID"],
"forbiddenAnswerKeywords": ["已经完全确认"],
"requireV2AuditClosure": true,
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["fail"],
"expectedComposerStatuses": ["valid"],
"forbiddenConfirmedClaimKeywords": ["连接池耗尽导致支付失败"]
},
{
"id": "unsupported-claim-filtering",
"title": "Unsupported claim filtering",
"question": "订单超时是否可以确认由数据库主库故障导致?",
"traceFixture": "unsupported-claim-filtering-low-confid.json",
"expectedRootCauseKeywords": ["超时", "证据"],
"minKeywordMatches": 2,
"requiredEvidenceTools": ["query_logs"],
"allowedVerdicts": ["LOW_CONFID"],
"forbiddenAnswerKeywords": ["已经确认"],
"requireV2AuditClosure": true,
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["pass"],
"expectedComposerStatuses": ["valid"],
"forbiddenConfirmedClaimKeywords": ["主库故障"]
},
{
"id": "composer-fallback-no-raw-json",
"title": "Composer fallback no raw JSON",
"question": "库存服务慢响应是否可以直接输出 Executor JSON?",
"traceFixture": "composer-fallback-no-raw-json-low-confid.json",
"expectedRootCauseKeywords": ["慢响应", "证据"],
"minKeywordMatches": 2,
"requiredEvidenceTools": ["query_metrics"],
"allowedVerdicts": ["LOW_CONFID"],
"forbiddenAnswerKeywords": ["executor_evidence_v2", "answer_version", "claim_id"],
"requireV2AuditClosure": true,
"requireClaimChecks": true,
"requireComposerOutput": true,
"expectedGatekeeperStatuses": ["pass"],
"expectedComposerStatuses": ["composer_malformed"],
"forbiddenConfirmedClaimKeywords": ["线程池已经耗尽"]
}
]