Add diagnosis eval harness

2026-07-04 23:51:43 +08:00
parent dc6cd32a67
commit ca5c61fabf
24 changed files with 1251 additions and 2 deletions
@@ -0,0 +1,64 @@
+{
+  "session": {
+    "sessionId": "eval-payment-timeout",
+    "query": "支付接口最近出现超时，请结合知识库、日志和指标判断可能原因，并给出修复建议。",
+    "status": "SUCCESS",
+    "agentFlow": "CHAT",
+    "totalDurationMs": 42000,
+    "toolCallCount": 3,
+    "answer": "支付接口超时与连接池等待有关。知识库说明支付超时需要同时检查连接池、日志和指标；日志出现 connection pool exhausted；指标显示支付服务延迟升高。",
+    "selfEvaluation": {
+      "verifier_evaluation": {
+        "verdict": "PASS",
+        "groundedness_score": 0.86,
+        "tool_trace_summary": [
+          {
+            "tool_name": "lookup_knowledge",
+            "success": true,
+            "evidence_level": "direct"
+          },
+          {
+            "tool_name": "query_logs",
+            "success": true,
+            "evidence_level": "direct"
+          },
+          {
+            "tool_name": "query_metrics",
+            "success": true,
+            "evidence_level": "direct"
+          }
+        ]
+      }
+    }
+  },
+  "steps": [],
+  "toolInvocations": [
+    {
+      "id": 1,
+      "sessionId": "eval-payment-timeout",
+      "toolName": "lookup_knowledge",
+      "success": true,
+      "relevanceLevel": "PRECISE"
+    },
+    {
+      "id": 2,
+      "sessionId": "eval-payment-timeout",
+      "toolName": "query_logs",
+      "success": true
+    },
+    {
+      "id": 3,
+      "sessionId": "eval-payment-timeout",
+      "toolName": "query_metrics",
+      "success": true
+    }
+  ],
+  "summary": {
+    "persistedStepCount": 3,
+    "returnedStepCount": 3,
+    "persistedToolCallCount": 3,
+    "returnedToolCallCount": 3,
+    "hasVerifierEvaluation": true,
+    "hasFeedback": false
+  }
+}