Expand diagnosis eval fixtures

2026-07-05 00:27:57 +08:00
parent ca5c61fabf
commit 4c7c53b024
21 changed files with 622 additions and 11 deletions
@@ -0,0 +1,82 @@
+{
+  "totalCases" : 5,
+  "passedCases" : 5,
+  "passRate" : 1.0,
+  "verdictDistribution" : {
+    "PASS" : 2,
+    "LOW_CONFID" : 3
+  },
+  "averageToolCallCount" : 2.0,
+  "averageDurationMs" : 45800.0,
+  "results" : [ {
+    "caseId" : "payment-timeout",
+    "title" : "Payment API timeout",
+    "passed" : true,
+    "failedChecks" : [ ],
+    "verdict" : "PASS",
+    "matchedKeywordCount" : 3,
+    "requiredKeywordCount" : 3,
+    "evidenceCoverage" : {
+      "lookup_knowledge" : true,
+      "query_logs" : true,
+      "query_metrics" : true
+    },
+    "toolCallCount" : 3,
+    "durationMs" : 42000
+  }, {
+    "caseId" : "mysql-pool-exhausted",
+    "title" : "MySQL connection pool exhausted",
+    "passed" : true,
+    "failedChecks" : [ ],
+    "verdict" : "LOW_CONFID",
+    "matchedKeywordCount" : 3,
+    "requiredKeywordCount" : 3,
+    "evidenceCoverage" : {
+      "lookup_knowledge" : true,
+      "query_logs" : true
+    },
+    "toolCallCount" : 2,
+    "durationMs" : 51000
+  }, {
+    "caseId" : "redis-timeout",
+    "title" : "Redis timeout",
+    "passed" : true,
+    "failedChecks" : [ ],
+    "verdict" : "LOW_CONFID",
+    "matchedKeywordCount" : 2,
+    "requiredKeywordCount" : 2,
+    "evidenceCoverage" : {
+      "query_logs" : true
+    },
+    "toolCallCount" : 1,
+    "durationMs" : 36000
+  }, {
+    "caseId" : "slow-response",
+    "title" : "Slow response",
+    "passed" : true,
+    "failedChecks" : [ ],
+    "verdict" : "PASS",
+    "matchedKeywordCount" : 2,
+    "requiredKeywordCount" : 2,
+    "evidenceCoverage" : {
+      "query_metrics" : true,
+      "query_logs" : true
+    },
+    "toolCallCount" : 2,
+    "durationMs" : 47000
+  }, {
+    "caseId" : "jvm-memory-risk",
+    "title" : "JVM memory risk",
+    "passed" : true,
+    "failedChecks" : [ ],
+    "verdict" : "LOW_CONFID",
+    "matchedKeywordCount" : 3,
+    "requiredKeywordCount" : 3,
+    "evidenceCoverage" : {
+      "query_metrics" : true,
+      "query_logs" : true
+    },
+    "toolCallCount" : 2,
+    "durationMs" : 53000
+  } ]
+}
@@ -0,0 +1,22 @@
+# Diagnosis Eval Report
+
+- Total cases: 5
+- Passed cases: 5
+- Pass rate: 100.00%
+- Average tool calls: 2.00
+- Average duration ms: 45800.00
+
+## Verdict Distribution
+
+- PASS: 2
+- LOW_CONFID: 3
+
+## Cases
+
+| Case | Result | Verdict | Keywords | Tool Calls | Duration ms | Failed Checks |
+| --- | --- | --- | --- | ---: | ---: | --- |
+| payment-timeout | PASS | PASS | 3/3 | 3 | 42000 | - |
+| mysql-pool-exhausted | PASS | LOW_CONFID | 3/3 | 2 | 51000 | - |
+| redis-timeout | PASS | LOW_CONFID | 2/2 | 1 | 36000 | - |
+| slow-response | PASS | PASS | 2/2 | 2 | 47000 | - |
+| jvm-memory-risk | PASS | LOW_CONFID | 3/3 | 2 | 53000 | - |