feat(eval): add evidence pipeline acceptance closure

2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
@@ -0,0 +1,126 @@
+{
+  "session": {
+    "sessionId": "eval-hikari-no-evidence-negative-observation",
+    "query": "确认 inventory-service 当前是否有 HikariCP 连接池耗尽日志。",
+    "status": "SUCCESS",
+    "agentFlow": "CHAT",
+    "totalDurationMs": 21000,
+    "toolCallCount": 1,
+    "answer": "本次查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志；这只表示当前查询没有匹配证据，仍不能据此判断系统一定健康。",
+    "selfEvaluation": {
+      "verifier_evaluation": {
+        "verdict": "PASS",
+        "groundedness_score": 1.0,
+        "critical_fact_count": 1,
+        "gatekeeper_result": {
+          "status": "pass",
+          "severity": "none",
+          "rule_set_version": "gatekeeper-rules-v1",
+          "rules": [
+            {
+              "id": "evidence.raw_path",
+              "description": "raw_path must exist in retrieval_details.evidence_refs",
+              "enabled": true,
+              "default_severity": "reject"
+            }
+          ],
+          "checked_bindings": [
+            {
+              "claim_id": "claim-1",
+              "tool_name": "query_logs",
+              "source_invocation_id": 12,
+              "raw_path": "$.no_evidence",
+              "matched_text": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志",
+              "status": "pass"
+            }
+          ],
+          "failed_rules": [],
+          "warnings": [],
+          "errors": []
+        },
+        "executor_structured_output": {
+          "answer_version": "executor_evidence_v2",
+          "claims": [
+            {
+              "claim_id": "claim-1",
+              "claim_type": "negative_observation",
+              "claim_text": "当前查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志。",
+              "support_level": "direct",
+              "evidence_bindings": [
+                {
+                  "source_type": "tool_trace",
+                  "tool_name": "query_logs",
+                  "source_invocation_id": 12,
+                  "raw_path": "$.no_evidence",
+                  "evidence_excerpt": "query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"
+                }
+              ]
+            }
+          ],
+          "hypotheses": [],
+          "recommended_actions": [],
+          "missing_info": [
+            "仅查询了 application-logs 中 inventory-service HikariCP 相关日志"
+          ]
+        },
+        "claim_checks": [
+          {
+            "claim_id": "claim-1",
+            "claim_text": "当前查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志。",
+            "claim_type": "negative_observation",
+            "verification": "direct_observation",
+            "detail": "$.no_evidence 只支持本次查询未检索到匹配证据。",
+            "evidence_refs": [
+              {
+                "source_invocation_id": 12,
+                "raw_path": "$.no_evidence"
+              }
+            ]
+          }
+        ],
+        "facts_checked": [],
+        "composer_output": {
+          "status": "valid",
+          "answer_summary": "本次查询未检索到匹配日志。",
+          "recommended_actions": [],
+          "user_facing_answer": "本次查询未检索到 inventory-service 的 HikariCP 连接池耗尽日志；这只表示当前查询没有匹配证据，仍不能据此判断系统一定健康。"
+        },
+        "tool_trace_summary": [
+          {
+            "tool_name": "query_logs",
+            "success": true,
+            "source_invocation_ids": [12],
+            "evidence_level": "no_evidence"
+          }
+        ]
+      }
+    }
+  },
+  "steps": [],
+  "toolInvocations": [
+    {
+      "id": 12,
+      "sessionId": "eval-hikari-no-evidence-negative-observation",
+      "toolName": "query_logs",
+      "outputPreview": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志",
+      "retrievalDetails": {
+        "evidence_status": "no_evidence",
+        "evidence_refs": [
+          {
+            "raw_path": "$.no_evidence",
+            "text": "query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志"
+          }
+        ]
+      },
+      "success": true
+    }
+  ],
+  "summary": {
+    "persistedStepCount": 3,
+    "returnedStepCount": 3,
+    "persistedToolCallCount": 1,
+    "returnedToolCallCount": 1,
+    "hasVerifierEvaluation": true,
+    "hasFeedback": false
+  }
+}