osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): add interview quality audit

Browse Source
This commit is contained in:
zhuyongxin
2026-07-09 11:18:49 +08:00
parent a6c2d4459c
commit 9c9a0024d4
37 changed files with 1162 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+86 -4
View File
@@ -24,11 +24,11 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(10, report.getTotalCases());
assertEquals(10, report.getPassedCases());
assertEquals(12, report.getTotalCases());
assertEquals(12, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(4L, report.getVerdictDistribution().get("PASS"));
assertEquals(5L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
@@ -36,6 +36,12 @@ class DiagnosisTraceEvaluatorTest {
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
assertTrue(promptGatekeeperAudit.isPassed());
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
assertTrue(hikariNoEvidence.isPassed());
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
@@ -60,6 +66,12 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
assertTrue(composerFallback.isPassed());
assertEquals("composer_malformed", composerFallback.getComposerStatus());
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
assertTrue(auditMetadataLowConfid.isPassed());
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
}
@Test
@@ -195,6 +207,76 @@ class DiagnosisTraceEvaluatorTest {
"gatekeeper rule set version not expected: old-rules"));
}
@Test
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("prompt-audit")
.title("Prompt audit")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requirePromptAudit(true)
.expectedPromptAuditVersion("chat-prompts-v1")
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"prompt_audit", java.util.Map.of(
"version", "old-prompts",
"prompts", java.util.List.of(java.util.Map.of(
"name", "chat_executor",
"version", "chat-executor-v1"
))
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"prompt audit version not expected: old-prompts"));
assertTrue(result.getFailedChecks().contains(
"prompt version not expected: chat_executor=chat-executor-v1"));
}
@Test
void evaluateFailsWhenGatekeeperRulesAreMissing() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-rules")
.title("Gatekeeper rules")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requireGatekeeperRules(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "gatekeeper-rules-v1"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {