193 lines
6.5 KiB
JSON
193 lines
6.5 KiB
JSON
{
|
|
"run": {
|
|
"sessionId": "eval-audit-metadata-low-confid",
|
|
"query": "订单超时是否可以确认由数据库主库故障导致，并检查审计元数据是否完整？",
|
|
"status": "SUCCESS",
|
|
"agentFlow": "CHAT",
|
|
"totalDurationMs": 45000,
|
|
"toolCallCount": 1,
|
|
"answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：日志显示订单接口出现超时。\n\n仍需补充信息：目前没有数据库故障日志或主库状态证据，不能把该方向写成确认根因。",
|
|
"selfEvaluation": {
|
|
"verifier_evaluation": {
|
|
"verdict": "LOW_CONFID",
|
|
"groundedness_score": 0.42,
|
|
"critical_fact_count": 2,
|
|
"prompt_audit": {
|
|
"version": "chat-prompts-v1",
|
|
"prompts": [
|
|
{
|
|
"name": "chat_planner",
|
|
"version": "chat-planner-v1",
|
|
"resource": "prompts/chat-planner-prompt.md"
|
|
},
|
|
{
|
|
"name": "chat_executor",
|
|
"version": "chat-executor-v2",
|
|
"resource": "prompts/chat-executor-prompt.md"
|
|
},
|
|
{
|
|
"name": "chat_verifier",
|
|
"version": "chat-verifier-v2",
|
|
"resource": "prompts/chat-verifier-prompt.md"
|
|
},
|
|
{
|
|
"name": "chat_composer",
|
|
"version": "chat-composer-v1",
|
|
"resource": "prompts/chat-composer-prompt.md"
|
|
}
|
|
]
|
|
},
|
|
"gatekeeper_result": {
|
|
"status": "pass",
|
|
"severity": "none",
|
|
"rule_set_version": "gatekeeper-rules-v1",
|
|
"rules": [
|
|
{
|
|
"id": "evidence.invocation",
|
|
"description": "source_invocation_id must reference an existing tool invocation",
|
|
"enabled": true,
|
|
"default_severity": "reject"
|
|
},
|
|
{
|
|
"id": "evidence.excerpt",
|
|
"description": "evidence_excerpt must be supported by recorded evidence text",
|
|
"enabled": true,
|
|
"default_severity": "reject"
|
|
}
|
|
],
|
|
"checked_bindings": [
|
|
{
|
|
"claim_id": "claim-timeout",
|
|
"tool_name": "query_logs",
|
|
"source_invocation_id": 22,
|
|
"raw_path": "$.logs[0]",
|
|
"matched_text": "order api timeout",
|
|
"status": "pass"
|
|
}
|
|
],
|
|
"failed_rules": [],
|
|
"warnings": [],
|
|
"errors": []
|
|
},
|
|
"executor_structured_output": {
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-timeout",
|
|
"claim_type": "symptom",
|
|
"claim_text": "订单接口出现超时",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"tool_name": "query_logs",
|
|
"source_invocation_id": 22,
|
|
"raw_path": "$.logs[0]",
|
|
"evidence_excerpt": "order api timeout"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"claim_id": "claim-db-primary",
|
|
"claim_type": "root_cause",
|
|
"claim_text": "数据库主库故障导致订单超时",
|
|
"support_level": "weak",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"tool_name": "query_logs",
|
|
"source_invocation_id": 22,
|
|
"raw_path": "$.logs[0]",
|
|
"evidence_excerpt": "order api timeout"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [
|
|
{
|
|
"action_text": "补充查询数据库主库状态和错误日志",
|
|
"reason": "当前只有订单接口超时日志"
|
|
}
|
|
],
|
|
"missing_info": ["数据库主库状态", "数据库错误日志"]
|
|
},
|
|
"claim_checks": [
|
|
{
|
|
"claim_id": "claim-timeout",
|
|
"claim_text": "订单接口出现超时",
|
|
"claim_type": "symptom",
|
|
"verification": "direct_observation",
|
|
"detail": "日志直接记录 order api timeout",
|
|
"evidence_refs": [
|
|
{
|
|
"source_invocation_id": 22,
|
|
"raw_path": "$.logs[0]"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"claim_id": "claim-db-primary",
|
|
"claim_text": "数据库主库故障导致订单超时",
|
|
"claim_type": "root_cause",
|
|
"verification": "unsupported",
|
|
"detail": "日志只能证明订单接口超时，不能证明数据库主库故障",
|
|
"evidence_refs": [
|
|
{
|
|
"source_invocation_id": 22,
|
|
"raw_path": "$.logs[0]"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"facts_checked": [],
|
|
"composer_output": {
|
|
"status": "valid",
|
|
"answer_summary": "日志显示订单接口超时，但数据库方向证据不足。",
|
|
"recommended_actions": [
|
|
{
|
|
"action_text": "补充查询数据库主库状态和错误日志",
|
|
"reason": "当前只有订单接口超时日志"
|
|
}
|
|
],
|
|
"user_facing_answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：日志显示订单接口出现超时。\n\n仍需补充信息：目前没有数据库故障日志或主库状态证据，不能把该方向写成确认根因。"
|
|
},
|
|
"tool_trace_summary": [
|
|
{
|
|
"tool_name": "query_logs",
|
|
"success": true,
|
|
"evidence_level": "direct"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"steps": [],
|
|
"toolInvocations": [
|
|
{
|
|
"id": 22,
|
|
"sessionId": "eval-audit-metadata-low-confid",
|
|
"toolName": "query_logs",
|
|
"outputPreview": "order api timeout",
|
|
"retrievalDetails": {
|
|
"evidence_status": "supported",
|
|
"evidence_refs": [
|
|
{
|
|
"raw_path": "$.logs[0]",
|
|
"text": "order api timeout"
|
|
}
|
|
]
|
|
},
|
|
"success": true
|
|
}
|
|
],
|
|
"summary": {
|
|
"persistedStepCount": 3,
|
|
"returnedStepCount": 3,
|
|
"persistedToolCallCount": 1,
|
|
"returnedToolCallCount": 1,
|
|
"hasVerifierEvaluation": true,
|
|
"hasFeedback": false
|
|
}
|
|
}
|