osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): add interview quality audit

Browse Source
This commit is contained in:
zhuyongxin
2026-07-09 11:18:49 +08:00
parent a6c2d4459c
commit 9c9a0024d4
37 changed files with 1162 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/DiagnosisEvalBaselineDiffTest.java
+5 -5
View File
@@ -100,13 +100,13 @@ class DiagnosisEvalBaselineDiffTest {
}
private void degradeRedisCase(DiagnosisEvalReport report) {
report.setPassedCases(9);
report.setPassRate(0.9);
report.setPassedCases(11);
report.setPassRate(11.0 / 12.0);
report.setAverageToolCallCount(3.0);
report.setAverageDurationMs(39800.0);
report.setAverageDurationMs(38500.0);
report.setVerdictDistribution(new LinkedHashMap<>());
report.getVerdictDistribution().put("PASS", 4L);
report.getVerdictDistribution().put("LOW_CONFID", 4L);
report.getVerdictDistribution().put("PASS", 5L);
report.getVerdictDistribution().put("LOW_CONFID", 5L);
report.getVerdictDistribution().put("REJECT", 2L);
DiagnosisEvalResult redis = result(report, "redis-timeout");
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+86 -4
View File
@@ -24,11 +24,11 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(10, report.getTotalCases());
assertEquals(10, report.getPassedCases());
assertEquals(12, report.getTotalCases());
assertEquals(12, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(4L, report.getVerdictDistribution().get("PASS"));
assertEquals(5L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
@@ -36,6 +36,12 @@ class DiagnosisTraceEvaluatorTest {
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
assertTrue(promptGatekeeperAudit.isPassed());
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
assertTrue(hikariNoEvidence.isPassed());
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
@@ -60,6 +66,12 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
assertTrue(composerFallback.isPassed());
assertEquals("composer_malformed", composerFallback.getComposerStatus());
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
assertTrue(auditMetadataLowConfid.isPassed());
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
}
@Test
@@ -195,6 +207,76 @@ class DiagnosisTraceEvaluatorTest {
"gatekeeper rule set version not expected: old-rules"));
}
@Test
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("prompt-audit")
.title("Prompt audit")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requirePromptAudit(true)
.expectedPromptAuditVersion("chat-prompts-v1")
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"prompt_audit", java.util.Map.of(
"version", "old-prompts",
"prompts", java.util.List.of(java.util.Map.of(
"name", "chat_executor",
"version", "chat-executor-v1"
))
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"prompt audit version not expected: old-prompts"));
assertTrue(result.getFailedChecks().contains(
"prompt version not expected: chat_executor=chat-executor-v1"));
}
@Test
void evaluateFailsWhenGatekeeperRulesAreMissing() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-rules")
.title("Gatekeeper rules")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requireGatekeeperRules(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "gatekeeper-rules-v1"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+10
View File
@@ -394,10 +394,20 @@ class ChatServiceSequentialAgentTest {
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
Map<String, Object> verifierEvaluation = captor.getValue();
assertTrue(verifierEvaluation.containsKey("gatekeeper_result"));
assertTrue(verifierEvaluation.containsKey("prompt_audit"));
@SuppressWarnings("unchecked")
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
assertEquals("pass", gatekeeperResult.get("status"));
assertEquals("none", gatekeeperResult.get("severity"));
@SuppressWarnings("unchecked")
Map<String, Object> promptAudit = (Map<String, Object>) verifierEvaluation.get("prompt_audit");
assertEquals("chat-prompts-v1", promptAudit.get("version"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> prompts = (List<Map<String, Object>>) promptAudit.get("prompts");
assertEquals(4, prompts.size());
assertTrue(prompts.stream().anyMatch(prompt ->
"chat_executor".equals(prompt.get("name"))
&& "chat-executor-v2".equals(prompt.get("version"))));
}
@Test