feat(demo): add interview quality audit
This commit is contained in:
zhuyongxin
@@ -0,0 +1,192 @@
|
||||
{
|
||||
"session": {
|
||||
"sessionId": "eval-audit-metadata-low-confid",
|
||||
"query": "订单超时是否可以确认由数据库主库故障导致，并检查审计元数据是否完整？",
|
||||
"status": "SUCCESS",
|
||||
"agentFlow": "CHAT",
|
||||
"totalDurationMs": 45000,
|
||||
"toolCallCount": 1,
|
||||
"answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：日志显示订单接口出现超时。\n\n仍需补充信息：目前没有数据库故障日志或主库状态证据，不能把该方向写成确认根因。",
|
||||
"selfEvaluation": {
|
||||
"verifier_evaluation": {
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.42,
|
||||
"critical_fact_count": 2,
|
||||
"prompt_audit": {
|
||||
"version": "chat-prompts-v1",
|
||||
"prompts": [
|
||||
{
|
||||
"name": "chat_planner",
|
||||
"version": "chat-planner-v1",
|
||||
"resource": "prompts/chat-planner-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_executor",
|
||||
"version": "chat-executor-v2",
|
||||
"resource": "prompts/chat-executor-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_verifier",
|
||||
"version": "chat-verifier-v2",
|
||||
"resource": "prompts/chat-verifier-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_composer",
|
||||
"version": "chat-composer-v1",
|
||||
"resource": "prompts/chat-composer-prompt.md"
|
||||
}
|
||||
]
|
||||
},
|
||||
"gatekeeper_result": {
|
||||
"status": "pass",
|
||||
"severity": "none",
|
||||
"rule_set_version": "gatekeeper-rules-v1",
|
||||
"rules": [
|
||||
{
|
||||
"id": "evidence.invocation",
|
||||
"description": "source_invocation_id must reference an existing tool invocation",
|
||||
"enabled": true,
|
||||
"default_severity": "reject"
|
||||
},
|
||||
{
|
||||
"id": "evidence.excerpt",
|
||||
"description": "evidence_excerpt must be supported by recorded evidence text",
|
||||
"enabled": true,
|
||||
"default_severity": "reject"
|
||||
}
|
||||
],
|
||||
"checked_bindings": [
|
||||
{
|
||||
"claim_id": "claim-timeout",
|
||||
"tool_name": "query_logs",
|
||||
"source_invocation_id": 22,
|
||||
"raw_path": "$.logs[0]",
|
||||
"matched_text": "order api timeout",
|
||||
"status": "pass"
|
||||
}
|
||||
],
|
||||
"failed_rules": [],
|
||||
"warnings": [],
|
||||
"errors": []
|
||||
},
|
||||
"executor_structured_output": {
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-timeout",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "订单接口出现超时",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"tool_name": "query_logs",
|
||||
"source_invocation_id": 22,
|
||||
"raw_path": "$.logs[0]",
|
||||
"evidence_excerpt": "order api timeout"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "claim-db-primary",
|
||||
"claim_type": "root_cause",
|
||||
"claim_text": "数据库主库故障导致订单超时",
|
||||
"support_level": "weak",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"tool_name": "query_logs",
|
||||
"source_invocation_id": 22,
|
||||
"raw_path": "$.logs[0]",
|
||||
"evidence_excerpt": "order api timeout"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [
|
||||
{
|
||||
"action_text": "补充查询数据库主库状态和错误日志",
|
||||
"reason": "当前只有订单接口超时日志"
|
||||
}
|
||||
],
|
||||
"missing_info": ["数据库主库状态", "数据库错误日志"]
|
||||
},
|
||||
"claim_checks": [
|
||||
{
|
||||
"claim_id": "claim-timeout",
|
||||
"claim_text": "订单接口出现超时",
|
||||
"claim_type": "symptom",
|
||||
"verification": "direct_observation",
|
||||
"detail": "日志直接记录 order api timeout",
|
||||
"evidence_refs": [
|
||||
{
|
||||
"source_invocation_id": 22,
|
||||
"raw_path": "$.logs[0]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "claim-db-primary",
|
||||
"claim_text": "数据库主库故障导致订单超时",
|
||||
"claim_type": "root_cause",
|
||||
"verification": "unsupported",
|
||||
"detail": "日志只能证明订单接口超时，不能证明数据库主库故障",
|
||||
"evidence_refs": [
|
||||
{
|
||||
"source_invocation_id": 22,
|
||||
"raw_path": "$.logs[0]"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"facts_checked": [],
|
||||
"composer_output": {
|
||||
"status": "valid",
|
||||
"answer_summary": "日志显示订单接口超时，但数据库方向证据不足。",
|
||||
"recommended_actions": [
|
||||
{
|
||||
"action_text": "补充查询数据库主库状态和错误日志",
|
||||
"reason": "当前只有订单接口超时日志"
|
||||
}
|
||||
],
|
||||
"user_facing_answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：日志显示订单接口出现超时。\n\n仍需补充信息：目前没有数据库故障日志或主库状态证据，不能把该方向写成确认根因。"
|
||||
},
|
||||
"tool_trace_summary": [
|
||||
{
|
||||
"tool_name": "query_logs",
|
||||
"success": true,
|
||||
"evidence_level": "direct"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"steps": [],
|
||||
"toolInvocations": [
|
||||
{
|
||||
"id": 22,
|
||||
"sessionId": "eval-audit-metadata-low-confid",
|
||||
"toolName": "query_logs",
|
||||
"outputPreview": "order api timeout",
|
||||
"retrievalDetails": {
|
||||
"evidence_status": "supported",
|
||||
"evidence_refs": [
|
||||
{
|
||||
"raw_path": "$.logs[0]",
|
||||
"text": "order api timeout"
|
||||
}
|
||||
]
|
||||
},
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"persistedStepCount": 3,
|
||||
"returnedStepCount": 3,
|
||||
"persistedToolCallCount": 1,
|
||||
"returnedToolCallCount": 1,
|
||||
"hasVerifierEvaluation": true,
|
||||
"hasFeedback": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
{
|
||||
"session": {
|
||||
"sessionId": "eval-prompt-gatekeeper-audit-closure",
|
||||
"query": "确认 payment-service 当前是否存在 HighCPUUsage 告警，并检查审计元数据是否完整。",
|
||||
"status": "SUCCESS",
|
||||
"agentFlow": "CHAT",
|
||||
"totalDurationMs": 19000,
|
||||
"toolCallCount": 1,
|
||||
"answer": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
|
||||
"selfEvaluation": {
|
||||
"verifier_evaluation": {
|
||||
"verdict": "PASS",
|
||||
"groundedness_score": 1.0,
|
||||
"critical_fact_count": 1,
|
||||
"prompt_audit": {
|
||||
"version": "chat-prompts-v1",
|
||||
"prompts": [
|
||||
{
|
||||
"name": "chat_planner",
|
||||
"version": "chat-planner-v1",
|
||||
"resource": "prompts/chat-planner-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_executor",
|
||||
"version": "chat-executor-v2",
|
||||
"resource": "prompts/chat-executor-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_verifier",
|
||||
"version": "chat-verifier-v2",
|
||||
"resource": "prompts/chat-verifier-prompt.md"
|
||||
},
|
||||
{
|
||||
"name": "chat_composer",
|
||||
"version": "chat-composer-v1",
|
||||
"resource": "prompts/chat-composer-prompt.md"
|
||||
}
|
||||
]
|
||||
},
|
||||
"gatekeeper_result": {
|
||||
"status": "pass",
|
||||
"severity": "none",
|
||||
"rule_set_version": "gatekeeper-rules-v1",
|
||||
"rules": [
|
||||
{
|
||||
"id": "evidence.invocation",
|
||||
"description": "source_invocation_id must reference an existing tool invocation",
|
||||
"enabled": true,
|
||||
"default_severity": "reject"
|
||||
},
|
||||
{
|
||||
"id": "evidence.raw_path",
|
||||
"description": "raw_path must exist in retrieval_details.evidence_refs",
|
||||
"enabled": true,
|
||||
"default_severity": "reject"
|
||||
}
|
||||
],
|
||||
"checked_bindings": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 21,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"matched_text": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m",
|
||||
"status": "pass"
|
||||
}
|
||||
],
|
||||
"failed_rules": [],
|
||||
"warnings": [],
|
||||
"errors": []
|
||||
},
|
||||
"executor_structured_output": {
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "observation",
|
||||
"claim_text": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 21,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
},
|
||||
"claim_checks": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_text": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。",
|
||||
"claim_type": "observation",
|
||||
"verification": "direct_observation",
|
||||
"detail": "已核验的指标证据直接包含服务名、告警名和 CPU 当前值。",
|
||||
"evidence_refs": [
|
||||
{
|
||||
"source_invocation_id": 21,
|
||||
"raw_path": "$.alerts[0]"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"facts_checked": [],
|
||||
"composer_output": {
|
||||
"status": "valid",
|
||||
"answer_summary": "payment-service 当前存在 HighCPUUsage 告警。",
|
||||
"recommended_actions": [],
|
||||
"user_facing_answer": "payment-service 当前存在 HighCPUUsage 告警，CPU 使用率为 92%。"
|
||||
},
|
||||
"tool_trace_summary": [
|
||||
{
|
||||
"tool_name": "query_metrics",
|
||||
"success": true,
|
||||
"source_invocation_ids": [21],
|
||||
"evidence_level": "direct"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"steps": [],
|
||||
"toolInvocations": [
|
||||
{
|
||||
"id": 21,
|
||||
"sessionId": "eval-prompt-gatekeeper-audit-closure",
|
||||
"toolName": "query_metrics",
|
||||
"outputPreview": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m",
|
||||
"retrievalDetails": {
|
||||
"evidence_status": "supported",
|
||||
"evidence_refs": [
|
||||
{
|
||||
"raw_path": "$.alerts[0]",
|
||||
"text": "HighCPUUsage firing, service=payment-service, current=92%, duration=25m"
|
||||
}
|
||||
]
|
||||
},
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"persistedStepCount": 3,
|
||||
"returnedStepCount": 3,
|
||||
"persistedToolCallCount": 1,
|
||||
"returnedToolCallCount": 1,
|
||||
"hasVerifierEvaluation": true,
|
||||
"hasFeedback": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user