feat(eval): add executor audit closure checks
This commit is contained in:
@@ -53,5 +53,56 @@
|
||||
"requiredEvidenceTools": ["query_metrics", "query_logs"],
|
||||
"allowedVerdicts": ["LOW_CONFID", "PASS"],
|
||||
"forbiddenAnswerKeywords": ["可以忽略"]
|
||||
},
|
||||
{
|
||||
"id": "gatekeeper-fabricated-invocation",
|
||||
"title": "Gatekeeper fabricated invocation",
|
||||
"question": "支付失败是否能确认由日志中的连接池耗尽导致?",
|
||||
"traceFixture": "gatekeeper-fabricated-invocation-reject.json",
|
||||
"expectedRootCauseKeywords": ["证据", "引用", "失败"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_logs"],
|
||||
"allowedVerdicts": ["REJECT", "LOW_CONFID"],
|
||||
"forbiddenAnswerKeywords": ["已经完全确认"],
|
||||
"requireV2AuditClosure": true,
|
||||
"requireClaimChecks": true,
|
||||
"requireComposerOutput": true,
|
||||
"expectedGatekeeperStatuses": ["fail"],
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["连接池耗尽导致支付失败"]
|
||||
},
|
||||
{
|
||||
"id": "unsupported-claim-filtering",
|
||||
"title": "Unsupported claim filtering",
|
||||
"question": "订单超时是否可以确认由数据库主库故障导致?",
|
||||
"traceFixture": "unsupported-claim-filtering-low-confid.json",
|
||||
"expectedRootCauseKeywords": ["超时", "证据"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_logs"],
|
||||
"allowedVerdicts": ["LOW_CONFID"],
|
||||
"forbiddenAnswerKeywords": ["已经确认"],
|
||||
"requireV2AuditClosure": true,
|
||||
"requireClaimChecks": true,
|
||||
"requireComposerOutput": true,
|
||||
"expectedGatekeeperStatuses": ["pass"],
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["主库故障"]
|
||||
},
|
||||
{
|
||||
"id": "composer-fallback-no-raw-json",
|
||||
"title": "Composer fallback no raw JSON",
|
||||
"question": "库存服务慢响应是否可以直接输出 Executor JSON?",
|
||||
"traceFixture": "composer-fallback-no-raw-json-low-confid.json",
|
||||
"expectedRootCauseKeywords": ["慢响应", "证据"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_metrics"],
|
||||
"allowedVerdicts": ["LOW_CONFID"],
|
||||
"forbiddenAnswerKeywords": ["executor_evidence_v2", "answer_version", "claim_id"],
|
||||
"requireV2AuditClosure": true,
|
||||
"requireClaimChecks": true,
|
||||
"requireComposerOutput": true,
|
||||
"expectedGatekeeperStatuses": ["pass"],
|
||||
"expectedComposerStatuses": ["composer_malformed"],
|
||||
"forbiddenConfirmedClaimKeywords": ["线程池已经耗尽"]
|
||||
}
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user