osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): add interview quality audit

Browse Source
This commit is contained in:
zhuyongxin
2026-07-09 11:18:49 +08:00
parent a6c2d4459c
commit 9c9a0024d4
37 changed files with 1162 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mvp/eval/reports/baseline-report.json
+64 -6
View File
@@ -1,14 +1,14 @@
{
"totalCases" : 10,
"passedCases" : 10,
"totalCases" : 12,
"passedCases" : 12,
"passRate" : 1.0,
"verdictDistribution" : {
"PASS" : 4,
"LOW_CONFID" : 5,
"PASS" : 5,
"LOW_CONFID" : 6,
"REJECT" : 1
},
"averageToolCallCount" : 1.5,
"averageDurationMs" : 39800.0,
"averageToolCallCount" : 1.4166666666666667,
"averageDurationMs" : 38500.0,
"results" : [ {
"caseId" : "narrow-highcpu-observation",
"title" : "Narrow HighCPU observation",
@@ -22,10 +22,31 @@
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"promptAuditVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"gatekeeperRuleCount" : 1,
"toolCallCount" : 1,
"durationMs" : 18000
}, {
"caseId" : "prompt-gatekeeper-audit-closure",
"title" : "Prompt and Gatekeeper audit closure",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "PASS",
"matchedKeywordCount" : 3,
"requiredKeywordCount" : 3,
"evidenceCoverage" : {
"query_metrics" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"promptAuditVersion" : "chat-prompts-v1",
"composerStatus" : "valid",
"claimCheckCount" : 1,
"gatekeeperRuleCount" : 2,
"toolCallCount" : 1,
"durationMs" : 19000
}, {
"caseId" : "hikari-no-evidence-negative-observation",
"title" : "Hikari no-evidence negative observation",
@@ -39,8 +60,10 @@
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"promptAuditVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"gatekeeperRuleCount" : 1,
"toolCallCount" : 1,
"durationMs" : 21000
}, {
@@ -58,8 +81,10 @@
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"gatekeeperRuleCount" : null,
"toolCallCount" : 3,
"durationMs" : 42000
}, {
@@ -76,8 +101,10 @@
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"gatekeeperRuleCount" : null,
"toolCallCount" : 2,
"durationMs" : 51000
}, {
@@ -93,8 +120,10 @@
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"gatekeeperRuleCount" : null,
"toolCallCount" : 1,
"durationMs" : 36000
}, {
@@ -111,8 +140,10 @@
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"gatekeeperRuleCount" : null,
"toolCallCount" : 2,
"durationMs" : 47000
}, {
@@ -129,8 +160,10 @@
},
"gatekeeperStatus" : null,
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : null,
"claimCheckCount" : null,
"gatekeeperRuleCount" : null,
"toolCallCount" : 2,
"durationMs" : 53000
}, {
@@ -146,8 +179,10 @@
},
"gatekeeperStatus" : "fail",
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 1,
"gatekeeperRuleCount" : null,
"toolCallCount" : 1,
"durationMs" : 39000
}, {
@@ -163,10 +198,31 @@
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : "valid",
"claimCheckCount" : 2,
"gatekeeperRuleCount" : null,
"toolCallCount" : 1,
"durationMs" : 44000
}, {
"caseId" : "audit-metadata-low-confid",
"title" : "Audit metadata low confidence",
"passed" : true,
"failedChecks" : [ ],
"verdict" : "LOW_CONFID",
"matchedKeywordCount" : 2,
"requiredKeywordCount" : 2,
"evidenceCoverage" : {
"query_logs" : true
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
"promptAuditVersion" : "chat-prompts-v1",
"composerStatus" : "valid",
"claimCheckCount" : 2,
"gatekeeperRuleCount" : 2,
"toolCallCount" : 1,
"durationMs" : 45000
}, {
"caseId" : "composer-fallback-no-raw-json",
"title" : "Composer fallback no raw JSON",
@@ -180,8 +236,10 @@
},
"gatekeeperStatus" : "pass",
"gatekeeperRuleSetVersion" : null,
"promptAuditVersion" : null,
"composerStatus" : "composer_malformed",
"claimCheckCount" : 2,
"gatekeeperRuleCount" : null,
"toolCallCount" : 1,
"durationMs" : 47000
} ]
mvp/eval/reports/baseline-report.md
+20 -18
View File
@@ -1,28 +1,30 @@
# Diagnosis Eval Report
- Total cases: 10
- Passed cases: 10
- Total cases: 12
- Passed cases: 12
- Pass rate: 100.00%
- Average tool calls: 1.50
- Average duration ms: 39800.00
- Average tool calls: 1.42
- Average duration ms: 38500.00
## Verdict Distribution
- PASS: 4
- LOW_CONFID: 5
- PASS: 5
- LOW_CONFID: 6
- REJECT: 1
## Cases
| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |
| narrow-highcpu-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 18000 | - |
| hikari-no-evidence-negative-observation | PASS | PASS | pass | gatekeeper-rules-v1 | valid | 1 | 3/3 | 1 | 21000 | - |
| payment-timeout | PASS | PASS | - | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | - | valid | 1 | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | - | valid | 2 | 2/2 | 1 | 44000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | - | composer_malformed | 2 | 2/2 | 1 | 47000 | - |
| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |
| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |
| narrow-highcpu-observation | PASS | PASS | pass | gatekeeper-rules-v1 | - | valid | 1 | 1 | 3/3 | 1 | 18000 | - |
| prompt-gatekeeper-audit-closure | PASS | PASS | pass | gatekeeper-rules-v1 | chat-prompts-v1 | valid | 1 | 2 | 3/3 | 1 | 19000 | - |
| hikari-no-evidence-negative-observation | PASS | PASS | pass | gatekeeper-rules-v1 | - | valid | 1 | 1 | 3/3 | 1 | 21000 | - |
| payment-timeout | PASS | PASS | - | - | - | - | - | - | 3/3 | 3 | 42000 | - |
| mysql-pool-exhausted | PASS | LOW_CONFID | - | - | - | - | - | - | 3/3 | 2 | 51000 | - |
| redis-timeout | PASS | LOW_CONFID | - | - | - | - | - | - | 2/2 | 1 | 36000 | - |
| slow-response | PASS | PASS | - | - | - | - | - | - | 2/2 | 2 | 47000 | - |
| jvm-memory-risk | PASS | LOW_CONFID | - | - | - | - | - | - | 3/3 | 2 | 53000 | - |
| gatekeeper-fabricated-invocation | PASS | REJECT | fail | - | - | valid | 1 | - | 3/3 | 1 | 39000 | - |
| unsupported-claim-filtering | PASS | LOW_CONFID | pass | - | - | valid | 2 | - | 2/2 | 1 | 44000 | - |
| audit-metadata-low-confid | PASS | LOW_CONFID | pass | gatekeeper-rules-v1 | chat-prompts-v1 | valid | 2 | 2 | 2/2 | 1 | 45000 | - |
| composer-fallback-no-raw-json | PASS | LOW_CONFID | pass | - | - | composer_malformed | 2 | - | 2/2 | 1 | 47000 | - |