199 lines
8.4 KiB
JSON
199 lines
8.4 KiB
JSON
[
|
||
{
|
||
"id": "narrow-highcpu-observation",
|
||
"title": "Narrow HighCPU observation",
|
||
"question": "确认 payment-service 当前是否存在 HighCPUUsage 告警。",
|
||
"traceFixture": "narrow-highcpu-observation-pass.json",
|
||
"expectedRootCauseKeywords": ["payment-service", "HighCPUUsage", "92%"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_metrics"],
|
||
"allowedVerdicts": ["PASS"],
|
||
"forbiddenAnswerKeywords": ["根因", "修复建议", "通常情况下"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["数据库连接池"]
|
||
},
|
||
{
|
||
"id": "prompt-gatekeeper-audit-closure",
|
||
"title": "Prompt and Gatekeeper audit closure",
|
||
"question": "确认 payment-service 当前是否存在 HighCPUUsage 告警，并检查审计元数据是否完整。",
|
||
"traceFixture": "prompt-gatekeeper-audit-closure-pass.json",
|
||
"expectedRootCauseKeywords": ["payment-service", "HighCPUUsage", "92%"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_metrics"],
|
||
"allowedVerdicts": ["PASS"],
|
||
"forbiddenAnswerKeywords": ["根因", "修复建议", "通常情况下"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["数据库连接池"],
|
||
"requirePromptAudit": true,
|
||
"expectedPromptAuditVersion": "chat-prompts-v1",
|
||
"expectedPromptVersions": {
|
||
"chat_planner": "chat-planner-v1",
|
||
"chat_executor": "chat-executor-v2",
|
||
"chat_verifier": "chat-verifier-v2",
|
||
"chat_composer": "chat-composer-v1"
|
||
},
|
||
"requireGatekeeperRules": true
|
||
},
|
||
{
|
||
"id": "hikari-no-evidence-negative-observation",
|
||
"title": "Hikari no-evidence negative observation",
|
||
"question": "确认 inventory-service 当前是否有 HikariCP 连接池耗尽日志。",
|
||
"traceFixture": "hikari-no-evidence-negative-observation-pass.json",
|
||
"expectedRootCauseKeywords": ["未检索到", "HikariCP", "匹配证据"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_logs"],
|
||
"allowedVerdicts": ["PASS"],
|
||
"forbiddenAnswerKeywords": ["已排除", "确认没有", "日志层面已排除"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["已排除 HikariCP"]
|
||
},
|
||
{
|
||
"id": "payment-timeout",
|
||
"title": "Payment API timeout",
|
||
"question": "支付接口最近出现超时，请结合知识库、日志和指标判断可能原因，并给出修复建议。",
|
||
"traceFixture": "payment-timeout-pass.json",
|
||
"expectedRootCauseKeywords": ["支付", "超时", "连接池"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["lookup_knowledge", "query_logs", "query_metrics"],
|
||
"allowedVerdicts": ["PASS", "LOW_CONFID"],
|
||
"forbiddenAnswerKeywords": ["无证据确定"]
|
||
},
|
||
{
|
||
"id": "mysql-pool-exhausted",
|
||
"title": "MySQL connection pool exhausted",
|
||
"question": "订单服务大量请求超时，请判断是否和 MySQL 连接池有关。",
|
||
"traceFixture": "mysql-pool-low-confid.json",
|
||
"expectedRootCauseKeywords": ["mysql", "连接池", "超时"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["lookup_knowledge", "query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID", "PASS"],
|
||
"forbiddenAnswerKeywords": ["已经完全确认"]
|
||
},
|
||
{
|
||
"id": "redis-timeout",
|
||
"title": "Redis timeout",
|
||
"question": "支付服务出现 Redis 连接超时，请定位可能原因。",
|
||
"traceFixture": "redis-timeout-low-confid.json",
|
||
"expectedRootCauseKeywords": ["redis", "超时"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID", "PASS"],
|
||
"forbiddenAnswerKeywords": ["无需进一步排查"]
|
||
},
|
||
{
|
||
"id": "slow-response",
|
||
"title": "Slow response",
|
||
"question": "用户服务 P99 响应时间升高，请结合指标和日志分析。",
|
||
"traceFixture": "slow-response-pass.json",
|
||
"expectedRootCauseKeywords": ["p99", "慢响应"],
|
||
"minKeywordMatches": 1,
|
||
"requiredEvidenceTools": ["query_metrics", "query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID", "PASS"],
|
||
"forbiddenAnswerKeywords": ["没有风险"]
|
||
},
|
||
{
|
||
"id": "jvm-memory-risk",
|
||
"title": "JVM memory risk",
|
||
"question": "订单服务内存使用率过高，请判断是否存在 OOM 风险。",
|
||
"traceFixture": "jvm-memory-risk-low-confid.json",
|
||
"expectedRootCauseKeywords": ["jvm", "内存", "oom"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_metrics", "query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID", "PASS"],
|
||
"forbiddenAnswerKeywords": ["可以忽略"]
|
||
},
|
||
{
|
||
"id": "gatekeeper-fabricated-invocation",
|
||
"title": "Gatekeeper fabricated invocation",
|
||
"question": "支付失败是否能确认由日志中的连接池耗尽导致？",
|
||
"traceFixture": "gatekeeper-fabricated-invocation-reject.json",
|
||
"expectedRootCauseKeywords": ["证据", "引用", "失败"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_logs"],
|
||
"allowedVerdicts": ["REJECT", "LOW_CONFID"],
|
||
"forbiddenAnswerKeywords": ["已经完全确认"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["fail"],
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["连接池耗尽导致支付失败"]
|
||
},
|
||
{
|
||
"id": "unsupported-claim-filtering",
|
||
"title": "Unsupported claim filtering",
|
||
"question": "订单超时是否可以确认由数据库主库故障导致？",
|
||
"traceFixture": "unsupported-claim-filtering-low-confid.json",
|
||
"expectedRootCauseKeywords": ["超时", "证据"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID"],
|
||
"forbiddenAnswerKeywords": ["已经确认"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["主库故障"]
|
||
},
|
||
{
|
||
"id": "audit-metadata-low-confid",
|
||
"title": "Audit metadata low confidence",
|
||
"question": "订单超时是否可以确认由数据库主库故障导致，并检查审计元数据是否完整？",
|
||
"traceFixture": "audit-metadata-low-confid.json",
|
||
"expectedRootCauseKeywords": ["超时", "证据"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_logs"],
|
||
"allowedVerdicts": ["LOW_CONFID"],
|
||
"forbiddenAnswerKeywords": ["已经确认"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedGatekeeperRuleSetVersion": "gatekeeper-rules-v1",
|
||
"expectedComposerStatuses": ["valid"],
|
||
"forbiddenConfirmedClaimKeywords": ["主库故障"],
|
||
"requirePromptAudit": true,
|
||
"expectedPromptAuditVersion": "chat-prompts-v1",
|
||
"expectedPromptVersions": {
|
||
"chat_planner": "chat-planner-v1",
|
||
"chat_executor": "chat-executor-v2",
|
||
"chat_verifier": "chat-verifier-v2",
|
||
"chat_composer": "chat-composer-v1"
|
||
},
|
||
"requireGatekeeperRules": true
|
||
},
|
||
{
|
||
"id": "composer-fallback-no-raw-json",
|
||
"title": "Composer fallback no raw JSON",
|
||
"question": "库存服务慢响应是否可以直接输出 Executor JSON？",
|
||
"traceFixture": "composer-fallback-no-raw-json-low-confid.json",
|
||
"expectedRootCauseKeywords": ["慢响应", "证据"],
|
||
"minKeywordMatches": 2,
|
||
"requiredEvidenceTools": ["query_metrics"],
|
||
"allowedVerdicts": ["LOW_CONFID"],
|
||
"forbiddenAnswerKeywords": ["executor_evidence_v2", "answer_version", "claim_id"],
|
||
"requireV2AuditClosure": true,
|
||
"requireClaimChecks": true,
|
||
"requireComposerOutput": true,
|
||
"expectedGatekeeperStatuses": ["pass"],
|
||
"expectedComposerStatuses": ["composer_malformed"],
|
||
"forbiddenConfirmedClaimKeywords": ["线程池已经耗尽"]
|
||
}
|
||
]
|