feat(eval): add executor audit closure checks

This commit is contained in:
aruo
2026-07-08 10:21:39 +08:00
parent 6015bcbf6f
commit a08672b31e
20 changed files with 1142 additions and 188 deletions
+69 -5
View File
@@ -1,13 +1,14 @@
{
"totalCases" : 5,
"passedCases" : 5,
"totalCases" : 8,
"passedCases" : 8,
"passRate" : 1.0,
"verdictDistribution" : {
"PASS" : 2,
"LOW_CONFID" : 3
"LOW_CONFID" : 5,
"REJECT" : 1
},
"averageToolCallCount" : 2.0,
"averageDurationMs" : 45800.0,
"averageToolCallCount" : 1.625,
"averageDurationMs" : 44875.0,
"results" : [ {
"caseId" : "payment-timeout",
"title" : "Payment API timeout",
@@ -21,6 +22,9 @@
"query_logs" : true,
"query_metrics" : true
},
"gatekeeperStatus" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 3,
"durationMs" : 42000
}, {
@@ -35,6 +39,9 @@
"lookup_knowledge" : true,
"query_logs" : true
},
"gatekeeperStatus" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
"durationMs" : 51000
}, {
@@ -48,6 +55,9 @@
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 1,
"durationMs" : 36000
}, {
@@ -62,6 +72,9 @@
"query_metrics" : true,
"query_logs" : true
},
"gatekeeperStatus" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
"durationMs" : 47000
}, {
@@ -76,7 +89,58 @@
"query_metrics" : true,
"query_logs" : true
},
"gatekeeperStatus" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
"durationMs" : 53000
}, {
"caseId" : "gatekeeper-fabricated-invocation",
"title" : "Gatekeeper fabricated invocation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "REJECT",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "fail",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 39000
}, {
"caseId" : "unsupported-claim-filtering",
"title" : "Unsupported claim filtering",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "LOW_CONFID",
"matchedKeywordCount" : 2,
"requiredKeywordCount" : 2,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"composerStatus" : "valid",
"claimCheckCount" : 2,
"toolCallCount" : 1,
"durationMs" : 44000
}, {
"caseId" : "composer-fallback-no-raw-json",
"title" : "Composer fallback no raw JSON",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "LOW_CONFID",
"matchedKeywordCount" : 2,
"requiredKeywordCount" : 2,
"evidenceCoverage" : {
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"composerStatus" : "composer_malformed",
"claimCheckCount" : 2,
"toolCallCount" : 1,
"durationMs" : 47000
} ]
}