feat(demo): add interview quality audit
This commit is contained in:
@@ -17,6 +17,33 @@
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["数据库连接池"]
|
||||
},
|
||||
{
|
||||
"id": "prompt-gatekeeper-audit-closure",
|
||||
"title": "Prompt and Gatekeeper audit closure",
|
||||
"question": "确认 payment-service 当前是否存在 HighCPUUsage 告警,并检查审计元数据是否完整。",
|
||||
"traceFixture": "prompt-gatekeeper-audit-closure-pass.json",
|
||||
"expectedRootCauseKeywords": ["payment-service", "HighCPUUsage", "92%"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_metrics"],
|
||||
"allowedVerdicts": ["PASS"],
|
||||
"forbiddenAnswerKeywords": ["根因", "修复建议", "通常情况下"],
|
||||
"requireV2AuditClosure": true,
|
||||
"requireClaimChecks": true,
|
||||
"requireComposerOutput": true,
|
||||
"expectedGatekeeperStatuses": ["pass"],
|
||||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["数据库连接池"],
|
||||
"requirePromptAudit": true,
|
||||
"expectedPromptAuditVersion": "chat-prompts-v1",
|
||||
"expectedPromptVersions": {
|
||||
"chat_planner": "chat-planner-v1",
|
||||
"chat_executor": "chat-executor-v2",
|
||||
"chat_verifier": "chat-verifier-v2",
|
||||
"chat_composer": "chat-composer-v1"
|
||||
},
|
||||
"requireGatekeeperRules": true
|
||||
},
|
||||
{
|
||||
"id": "hikari-no-evidence-negative-observation",
|
||||
"title": "Hikari no-evidence negative observation",
|
||||
@@ -124,6 +151,33 @@
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["主库故障"]
|
||||
},
|
||||
{
|
||||
"id": "audit-metadata-low-confid",
|
||||
"title": "Audit metadata low confidence",
|
||||
"question": "订单超时是否可以确认由数据库主库故障导致,并检查审计元数据是否完整?",
|
||||
"traceFixture": "audit-metadata-low-confid.json",
|
||||
"expectedRootCauseKeywords": ["超时", "证据"],
|
||||
"minKeywordMatches": 2,
|
||||
"requiredEvidenceTools": ["query_logs"],
|
||||
"allowedVerdicts": ["LOW_CONFID"],
|
||||
"forbiddenAnswerKeywords": ["已经确认"],
|
||||
"requireV2AuditClosure": true,
|
||||
"requireClaimChecks": true,
|
||||
"requireComposerOutput": true,
|
||||
"expectedGatekeeperStatuses": ["pass"],
|
||||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||||
"expectedComposerStatuses": ["valid"],
|
||||
"forbiddenConfirmedClaimKeywords": ["主库故障"],
|
||||
"requirePromptAudit": true,
|
||||
"expectedPromptAuditVersion": "chat-prompts-v1",
|
||||
"expectedPromptVersions": {
|
||||
"chat_planner": "chat-planner-v1",
|
||||
"chat_executor": "chat-executor-v2",
|
||||
"chat_verifier": "chat-verifier-v2",
|
||||
"chat_composer": "chat-composer-v1"
|
||||
},
|
||||
"requireGatekeeperRules": true
|
||||
},
|
||||
{
|
||||
"id": "composer-fallback-no-raw-json",
|
||||
"title": "Composer fallback no raw JSON",
|
||||
|
||||
Reference in New Issue
Block a user