feat(eval): add evidence pipeline acceptance closure
This commit is contained in:
@@ -1,15 +1,49 @@
|
||||
{
|
||||
"totalCases" : 8,
|
||||
"passedCases" : 8,
|
||||
"totalCases" : 10,
|
||||
"passedCases" : 10,
|
||||
"passRate" : 1.0,
|
||||
"verdictDistribution" : {
|
||||
"PASS" : 2,
|
||||
"PASS" : 4,
|
||||
"LOW_CONFID" : 5,
|
||||
"REJECT" : 1
|
||||
},
|
||||
"averageToolCallCount" : 1.625,
|
||||
"averageDurationMs" : 44875.0,
|
||||
"averageToolCallCount" : 1.5,
|
||||
"averageDurationMs" : 39800.0,
|
||||
"results" : [ {
|
||||
"caseId" : "narrow-highcpu-observation",
|
||||
"title" : "Narrow HighCPU observation",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "PASS",
|
||||
"matchedKeywordCount" : 3,
|
||||
"requiredKeywordCount" : 3,
|
||||
"evidenceCoverage" : {
|
||||
"query_metrics" : true
|
||||
},
|
||||
"gatekeeperStatus" : "pass",
|
||||
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
|
||||
"composerStatus" : "valid",
|
||||
"claimCheckCount" : 1,
|
||||
"toolCallCount" : 1,
|
||||
"durationMs" : 18000
|
||||
}, {
|
||||
"caseId" : "hikari-no-evidence-negative-observation",
|
||||
"title" : "Hikari no-evidence negative observation",
|
||||
"passed" : true,
|
||||
"failedChecks" : [ ],
|
||||
"verdict" : "PASS",
|
||||
"matchedKeywordCount" : 3,
|
||||
"requiredKeywordCount" : 3,
|
||||
"evidenceCoverage" : {
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : "pass",
|
||||
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
|
||||
"composerStatus" : "valid",
|
||||
"claimCheckCount" : 1,
|
||||
"toolCallCount" : 1,
|
||||
"durationMs" : 21000
|
||||
}, {
|
||||
"caseId" : "payment-timeout",
|
||||
"title" : "Payment API timeout",
|
||||
"passed" : true,
|
||||
@@ -23,6 +57,7 @@
|
||||
"query_metrics" : true
|
||||
},
|
||||
"gatekeeperStatus" : null,
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : null,
|
||||
"claimCheckCount" : null,
|
||||
"toolCallCount" : 3,
|
||||
@@ -40,6 +75,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : null,
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : null,
|
||||
"claimCheckCount" : null,
|
||||
"toolCallCount" : 2,
|
||||
@@ -56,6 +92,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : null,
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : null,
|
||||
"claimCheckCount" : null,
|
||||
"toolCallCount" : 1,
|
||||
@@ -73,6 +110,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : null,
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : null,
|
||||
"claimCheckCount" : null,
|
||||
"toolCallCount" : 2,
|
||||
@@ -90,6 +128,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : null,
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : null,
|
||||
"claimCheckCount" : null,
|
||||
"toolCallCount" : 2,
|
||||
@@ -106,6 +145,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : "fail",
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : "valid",
|
||||
"claimCheckCount" : 1,
|
||||
"toolCallCount" : 1,
|
||||
@@ -122,6 +162,7 @@
|
||||
"query_logs" : true
|
||||
},
|
||||
"gatekeeperStatus" : "pass",
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : "valid",
|
||||
"claimCheckCount" : 2,
|
||||
"toolCallCount" : 1,
|
||||
@@ -138,9 +179,10 @@
|
||||
"query_metrics" : true
|
||||
},
|
||||
"gatekeeperStatus" : "pass",
|
||||
"gatekeeperRuleSetVersion" : null,
|
||||
"composerStatus" : "composer_malformed",
|
||||
"claimCheckCount" : 2,
|
||||
"toolCallCount" : 1,
|
||||
"durationMs" : 47000
|
||||
} ]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user