osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(eval): add evidence pipeline acceptance closure

Browse Source
This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mvp/eval/reports/baseline-report.json
+48 -6
View File
@@ -1,15 +1,49 @@
{
"totalCases" : 8,
"passedCases" : 8,
"totalCases" : 10,
"passedCases" : 10,
"passRate" : 1.0,
"verdictDistribution" : {
"PASS" : 2,
"PASS" : 4,
"LOW_CONFID" : 5,
"REJECT" : 1
},
"averageToolCallCount" : 1.625,
"averageDurationMs" : 44875.0,
"averageToolCallCount" : 1.5,
"averageDurationMs" : 39800.0,
"results" : [ {
"caseId" : "narrow-highcpu-observation",
"title" : "Narrow HighCPU observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 18000
}, {
"caseId" : "hikari-no-evidence-negative-observation",
"title" : "Hikari no-evidence negative observation",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
"durationMs" : 21000
}, {
"caseId" : "payment-timeout",
"title" : "Payment API timeout",
"passed" : true,
@@ -23,6 +57,7 @@
"query_metrics" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 3,
@@ -40,6 +75,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -56,6 +92,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 1,
@@ -73,6 +110,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -90,6 +128,7 @@
"query_logs" : true
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"toolCallCount" : 2,
@@ -106,6 +145,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "fail",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"toolCallCount" : 1,
@@ -122,6 +162,7 @@
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 2,
"toolCallCount" : 1,
@@ -138,9 +179,10 @@
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"composerStatus" : "composer_malformed",
"claimCheckCount" : 2,
"toolCallCount" : 1,
"durationMs" : 47000
} ]
}
}
mvp/eval/reports/baseline-report.md
+17 -15
View File
@@ -1,26 +1,28 @@
# Diagnosis Eval Report
- Total cases: 8
- Passed cases: 8
- Total cases: 10
- Passed cases: 10
- Pass rate: 100.00%
- Average tool calls: 1.63
- Average duration ms: 44875.00
- Average tool calls: 1.50
- Average duration ms: 39800.00
## Verdict Distribution
- PASS: 2
- PASS: 4
- LOW_CONFID: 5
- REJECT: 1
## Cases
| Case | Result | Verdict | Gatekeeper | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |
| payment-timeout | PASS | PASS | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | valid | 1 | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | valid | 2 | 2/2 | 1 | 44000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | composer_malformed | 2 | 2/2 | 1 | 47000 | - |
| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |
| narrow-highcpu-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 18000 | - |
| hikari-no-evidence-negative-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 21000 | - |
| payment-timeout | PASS | PASS | - | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | - | valid | 1 | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | - | valid | 2 | 2/2 | 1 | 44000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | - | composer_malformed | 2 | 2/2 | 1 | 47000 | - |