feat(eval): add executor audit closure checks

2026-07-08 10:21:39 +08:00
parent 6015bcbf6f
commit a08672b31e
20 changed files with 1142 additions and 188 deletions
@@ -0,0 +1,142 @@
+{
+  "session": {
+    "sessionId": "eval-composer-fallback-no-raw-json",
+    "query": "库存服务慢响应是否可以直接输出 Executor JSON？",
+    "status": "SUCCESS",
+    "agentFlow": "CHAT",
+    "totalDurationMs": 47000,
+    "toolCallCount": 1,
+    "answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：指标显示库存服务出现慢响应。\n\n仍需补充信息：当前没有线程池队列或线程耗尽证据，不能确认线程池方向。\n\n建议动作：补充查询库存服务线程池指标。",
+    "selfEvaluation": {
+      "verifier_evaluation": {
+        "verdict": "LOW_CONFID",
+        "groundedness_score": 0.45,
+        "critical_fact_count": 2,
+        "gatekeeper_result": {
+          "status": "pass",
+          "failed_rules": [],
+          "warnings": [],
+          "errors": []
+        },
+        "executor_structured_output": {
+          "answer_version": "executor_evidence_v2",
+          "claims": [
+            {
+              "claim_id": "claim-slow-response",
+              "claim_type": "symptom",
+              "claim_text": "库存服务出现慢响应",
+              "support_level": "direct",
+              "evidence_bindings": [
+                {
+                  "tool_invocation_id": 1,
+                  "tool_name": "query_metrics",
+                  "evidence_excerpt": "inventory p99 latency increased"
+                }
+              ]
+            },
+            {
+              "claim_id": "claim-thread-pool",
+              "claim_type": "root_cause",
+              "claim_text": "线程池已经耗尽",
+              "support_level": "weak",
+              "evidence_bindings": [
+                {
+                  "tool_invocation_id": 1,
+                  "tool_name": "query_metrics",
+                  "evidence_excerpt": "inventory p99 latency increased"
+                }
+              ]
+            }
+          ],
+          "hypotheses": [],
+          "recommended_actions": [
+            {
+              "action_text": "补充查询库存服务线程池指标",
+              "reason": "当前只有慢响应指标"
+            }
+          ],
+          "missing_info": ["线程池队列长度", "活跃线程数"]
+        },
+        "claim_checks": [
+          {
+            "claim_id": "claim-slow-response",
+            "claim_text": "库存服务出现慢响应",
+            "claim_type": "symptom",
+            "verification": "direct_observation",
+            "detail": "指标显示 inventory p99 latency increased",
+            "evidence_refs": [
+              {
+                "tool_invocation_id": 1,
+                "tool_name": "query_metrics"
+              }
+            ]
+          },
+          {
+            "claim_id": "claim-thread-pool",
+            "claim_text": "线程池已经耗尽",
+            "claim_type": "root_cause",
+            "verification": "external_unknown",
+            "detail": "没有线程池队列或活跃线程指标，不能确认该结论",
+            "evidence_refs": []
+          }
+        ],
+        "facts_checked": [
+          {
+            "fact": "库存服务出现慢响应",
+            "verification": "direct_evidence",
+            "detail": "指标显示 inventory p99 latency increased",
+            "evidence_refs": [
+              {
+                "tool_invocation_id": 1,
+                "tool_name": "query_metrics"
+              }
+            ]
+          },
+          {
+            "fact": "线程池已经耗尽",
+            "verification": "external_unknown",
+            "detail": "没有线程池队列或活跃线程指标，不能确认该结论",
+            "evidence_refs": []
+          }
+        ],
+        "composer_output": {
+          "status": "composer_malformed",
+          "detail": "used safe fallback rendering",
+          "answer_summary": "指标显示库存服务出现慢响应。",
+          "recommended_actions": [
+            {
+              "action_text": "补充查询库存服务线程池指标",
+              "reason": "当前只有慢响应指标"
+            }
+          ],
+          "user_facing_answer": "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。\n\n已确认信息：指标显示库存服务出现慢响应。\n\n仍需补充信息：当前没有线程池队列或线程耗尽证据，不能确认线程池方向。\n\n建议动作：补充查询库存服务线程池指标。"
+        },
+        "tool_trace_summary": [
+          {
+            "tool_name": "query_metrics",
+            "success": true,
+            "evidence_level": "direct"
+          }
+        ]
+      }
+    }
+  },
+  "steps": [],
+  "toolInvocations": [
+    {
+      "id": 1,
+      "sessionId": "eval-composer-fallback-no-raw-json",
+      "toolName": "query_metrics",
+      "outputPreview": "inventory p99 latency increased",
+      "success": true
+    }
+  ],
+  "summary": {
+    "persistedStepCount": 3,
+    "returnedStepCount": 3,
+    "persistedToolCallCount": 1,
+    "returnedToolCallCount": 1,
+    "hasVerifierEvaluation": true,
+    "hasFeedback": false
+  }
+}