feat(eval): add evidence pipeline acceptance closure

This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
+48 -6
View File
@@ -1,15 +1,49 @@
{
"totalCases" : 8,
"passedCases" : 8,
"totalCases" : 10,
"passedCases" : 10,
"passRate" : 1.0,
"verdictDistribution" : {
"PASS" : 2,
"PASS" : 4,
"LOW_CONFID" : 5,
"REJECT" : 1
},
"averageToolCallCount" : 1.625,
"averageDurationMs" : 44875.0,
"averageToolCallCount" : 1.5,
"averageDurationMs" : 39800.0,
"results" : [ {
"caseId" : "narrow-highcpu-observation",
"title" : "Narrow HighCPU observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 18000
}, {
"caseId" : "hikari-no-evidence-negative-observation",
"title" : "Hikari no-evidence negative observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 21000
}, {
"caseId" : "payment-timeout",
"title" : "Payment API timeout",
"passed" : true,
@@ -23,6 +57,7 @@
"query_metrics" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 3,
@@ -40,6 +75,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -56,6 +92,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 1,
@@ -73,6 +110,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -90,6 +128,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -106,6 +145,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "fail",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
@@ -122,6 +162,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 2,
"toolCallCount" : 1,
@@ -138,9 +179,10 @@
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "composer_malformed",
"claimCheckCount" : 2,
"toolCallCount" : 1,
"durationMs" : 47000
} ]
}
}