feat(eval): add executor audit closure checks

2026-07-08 10:21:39 +08:00
parent 6015bcbf6f
commit a08672b31e
20 changed files with 1142 additions and 188 deletions
@@ -0,0 +1,110 @@
+{
+  "session": {
+    "sessionId": "eval-gatekeeper-fabricated-invocation",
+    "query": "支付失败是否能确认由日志中的连接池耗尽导致？",
+    "status": "SUCCESS",
+    "agentFlow": "CHAT",
+    "totalDurationMs": 39000,
+    "toolCallCount": 1,
+    "answer": "当前无法基于已获取证据生成可靠结论。\n\n证据引用校验失败：Executor 引用了不存在的工具调用记录，因此不能把连接池问题作为确认结论。建议重新收集日志证据后再判断。",
+    "selfEvaluation": {
+      "verifier_evaluation": {
+        "verdict": "REJECT",
+        "groundedness_score": 0.1,
+        "critical_fact_count": 1,
+        "gatekeeper_result": {
+          "status": "fail",
+          "failed_rules": ["evidence.invocation_ref"],
+          "warnings": [],
+          "errors": [
+            {
+              "rule_id": "evidence.invocation_ref",
+              "field": "claims[0].evidence_bindings[0].tool_invocation_id",
+              "message": "tool_invocation_id does not exist"
+            }
+          ]
+        },
+        "executor_structured_output": {
+          "answer_version": "executor_evidence_v2",
+          "claims": [
+            {
+              "claim_id": "claim-1",
+              "claim_type": "root_cause",
+              "claim_text": "连接池耗尽导致支付失败",
+              "support_level": "direct",
+              "evidence_bindings": [
+                {
+                  "tool_invocation_id": 99,
+                  "tool_name": "query_logs",
+                  "evidence_excerpt": "connection pool exhausted"
+                }
+              ]
+            }
+          ],
+          "hypotheses": [],
+          "recommended_actions": [
+            {
+              "action_text": "重新查询支付服务错误日志",
+              "reason": "当前 Executor 证据引用无法回溯"
+            }
+          ],
+          "missing_info": ["需要有效的日志工具调用记录"]
+        },
+        "claim_checks": [
+          {
+            "claim_id": "claim-1",
+            "claim_text": "连接池耗尽导致支付失败",
+            "claim_type": "root_cause",
+            "verification": "unsupported",
+            "detail": "Gatekeeper 已判定证据引用不存在，不能确认该结论",
+            "evidence_refs": []
+          }
+        ],
+        "facts_checked": [
+          {
+            "fact": "连接池耗尽导致支付失败",
+            "verification": "unsupported",
+            "detail": "Gatekeeper 已判定证据引用不存在，不能确认该结论",
+            "evidence_refs": []
+          }
+        ],
+        "composer_output": {
+          "status": "valid",
+          "answer_summary": "证据引用校验失败，不能确认根因。",
+          "recommended_actions": [
+            {
+              "action_text": "重新查询支付服务错误日志",
+              "reason": "当前 Executor 证据引用无法回溯"
+            }
+          ],
+          "user_facing_answer": "当前无法基于已获取证据生成可靠结论。\n\n证据引用校验失败：Executor 引用了不存在的工具调用记录，因此不能把连接池问题作为确认结论。建议重新收集日志证据后再判断。"
+        },
+        "tool_trace_summary": [
+          {
+            "tool_name": "query_logs",
+            "success": true,
+            "evidence_level": "direct"
+          }
+        ]
+      }
+    }
+  },
+  "steps": [],
+  "toolInvocations": [
+    {
+      "id": 1,
+      "sessionId": "eval-gatekeeper-fabricated-invocation",
+      "toolName": "query_logs",
+      "outputPreview": "payment failed without matching connection pool exhaustion entry",
+      "success": true
+    }
+  ],
+  "summary": {
+    "persistedStepCount": 3,
+    "returnedStepCount": 3,
+    "persistedToolCallCount": 1,
+    "returnedToolCallCount": 1,
+    "hasVerifierEvaluation": true,
+    "hasFeedback": false
+  }
+}