osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): add interview quality audit

Browse Source
This commit is contained in:
zhuyongxin
2026-07-09 11:18:49 +08:00
parent a6c2d4459c
commit 9c9a0024d4
37 changed files with 1162 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java
+5
View File
@@ -6,6 +6,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
@Data
@Builder
@@ -29,4 +30,8 @@ public class DiagnosisEvalCase {
private String expectedGatekeeperRuleSetVersion;
private List<String> expectedComposerStatuses;
private List<String> forbiddenConfirmedClaimKeywords;
private Boolean requirePromptAudit;
private String expectedPromptAuditVersion;
private Map<String, String> expectedPromptVersions;
private Boolean requireGatekeeperRules;
}
src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java
+6 -2
View File
@@ -46,8 +46,8 @@ public class DiagnosisEvalReportWriter {
}
builder.append("## Cases\n\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |\n");
for (DiagnosisEvalResult result : report.getResults()) {
builder.append("| ")
.append(result.getCaseId())
@@ -60,10 +60,14 @@ public class DiagnosisEvalReportWriter {
.append(" | ")
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
.append(" | ")
.append(valueOrDash(result.getPromptAuditVersion()))
.append(" | ")
.append(valueOrDash(result.getComposerStatus()))
.append(" | ")
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
.append(" | ")
.append(result.getGatekeeperRuleCount() == null ? "-" : result.getGatekeeperRuleCount())
.append(" | ")
.append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
.append(" | ")
.append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java
+2
View File
@@ -24,8 +24,10 @@ public class DiagnosisEvalResult {
private Map<String, Boolean> evidenceCoverage;
private String gatekeeperStatus;
private String gatekeeperRuleSetVersion;
private String promptAuditVersion;
private String composerStatus;
private Integer claimCheckCount;
private Integer gatekeeperRuleCount;
private Integer toolCallCount;
private Integer durationMs;
}
src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java
+98 -1
View File
@@ -67,8 +67,10 @@ public class DiagnosisTraceEvaluator {
.evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
.gatekeeperStatus(null)
.gatekeeperRuleSetVersion(null)
.promptAuditVersion(null)
.composerStatus(null)
.claimCheckCount(null)
.gatekeeperRuleCount(null)
.toolCallCount(null)
.durationMs(null)
.build());
@@ -123,10 +125,13 @@ public class DiagnosisTraceEvaluator {
String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
String gatekeeperRuleSetVersion = extractNestedString(trace, "verifier_evaluation",
"gatekeeper_result", "rule_set_version");
String promptAuditVersion = extractNestedString(trace, "verifier_evaluation",
"prompt_audit", "version");
String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
Integer gatekeeperRuleCount = countNestedList(trace, "verifier_evaluation", "gatekeeper_result", "rules");
failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
gatekeeperStatus, gatekeeperRuleSetVersion, composerStatus));
gatekeeperStatus, gatekeeperRuleSetVersion, promptAuditVersion, composerStatus));
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
@@ -142,8 +147,10 @@ public class DiagnosisTraceEvaluator {
.evidenceCoverage(evidenceCoverage)
.gatekeeperStatus(gatekeeperStatus)
.gatekeeperRuleSetVersion(gatekeeperRuleSetVersion)
.promptAuditVersion(promptAuditVersion)
.composerStatus(composerStatus)
.claimCheckCount(claimCheckCount)
.gatekeeperRuleCount(gatekeeperRuleCount)
.toolCallCount(toolCallCount)
.durationMs(durationMs)
.build();
@@ -237,6 +244,7 @@ public class DiagnosisTraceEvaluator {
String verdict,
String gatekeeperStatus,
String gatekeeperRuleSetVersion,
String promptAuditVersion,
String composerStatus) {
List<String> failedChecks = new ArrayList<>();
boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
@@ -259,6 +267,10 @@ public class DiagnosisTraceEvaluator {
failedChecks.add("gatekeeper rule set version not expected: "
+ valueOrMissing(gatekeeperRuleSetVersion));
}
if (Boolean.TRUE.equals(evalCase.getRequireGatekeeperRules())) {
failedChecks.addAll(validateGatekeeperRules(trace));
}
failedChecks.addAll(validatePromptAudit(evalCase, trace, promptAuditVersion));
failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
@@ -290,6 +302,78 @@ public class DiagnosisTraceEvaluator {
return failedChecks;
}
private List<String> validatePromptAudit(DiagnosisEvalCase evalCase,
DiagnosisTraceResponse trace,
String promptAuditVersion) {
List<String> failedChecks = new ArrayList<>();
Object promptAudit = nestedValue(trace, "verifier_evaluation", "prompt_audit");
if (Boolean.TRUE.equals(evalCase.getRequirePromptAudit()) && !(promptAudit instanceof Map<?, ?>)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
if (!isBlank(evalCase.getExpectedPromptAuditVersion())
&& !evalCase.getExpectedPromptAuditVersion().equals(promptAuditVersion)) {
failedChecks.add("prompt audit version not expected: " + valueOrMissing(promptAuditVersion));
}
if (evalCase.getExpectedPromptVersions() == null || evalCase.getExpectedPromptVersions().isEmpty()) {
return failedChecks;
}
if (!(promptAudit instanceof Map<?, ?> audit)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
Object promptsValue = audit.get("prompts");
if (!(promptsValue instanceof List<?> prompts)) {
failedChecks.add("prompt_audit missing prompts");
return failedChecks;
}
Map<String, String> actualVersions = new LinkedHashMap<>();
for (Object promptValue : prompts) {
if (promptValue instanceof Map<?, ?> prompt) {
String name = stringValue(prompt.get("name"));
String version = stringValue(prompt.get("version"));
if (!isBlank(name)) {
actualVersions.put(name, version);
}
}
}
for (Map.Entry<String, String> expected : evalCase.getExpectedPromptVersions().entrySet()) {
String actual = actualVersions.get(expected.getKey());
if (!expected.getValue().equals(actual)) {
failedChecks.add("prompt version not expected: "
+ expected.getKey() + "=" + valueOrMissing(actual));
}
}
return failedChecks;
}
private List<String> validateGatekeeperRules(DiagnosisTraceResponse trace) {
Object rulesValue = nestedNestedValue(trace, "verifier_evaluation", "gatekeeper_result", "rules");
if (!(rulesValue instanceof List<?> rules) || rules.isEmpty()) {
return List.of("gatekeeper_result missing rules");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : rules) {
if (!(item instanceof Map<?, ?> rule)) {
failedChecks.add("gatekeeper rule metadata is not an object");
continue;
}
String id = stringValue(rule.get("id"));
Object enabled = rule.get("enabled");
String severity = stringValue(rule.get("default_severity"));
if (isBlank(id)) {
failedChecks.add("gatekeeper rule metadata missing id");
}
if (!(enabled instanceof Boolean)) {
failedChecks.add("gatekeeper rule metadata missing enabled: " + valueOrMissing(id));
}
if (isBlank(severity)) {
failedChecks.add("gatekeeper rule metadata missing default_severity: " + valueOrMissing(id));
}
}
return failedChecks;
}
private List<String> validateClaimChecks(DiagnosisTraceResponse trace, boolean required) {
Object claimChecks = nestedValue(trace, "verifier_evaluation", "claim_checks");
if (!(claimChecks instanceof List<?> claimCheckList)) {
@@ -347,6 +431,19 @@ public class DiagnosisTraceEvaluator {
return value instanceof List<?> list ? list.size() : null;
}
private Integer countNestedList(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedNestedValue(trace, firstKey, secondKey, thirdKey);
return value instanceof List<?> list ? list.size() : null;
}
private Object nestedNestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedValue(trace, firstKey, secondKey);
if (!(value instanceof Map<?, ?> map)) {
return null;
}
return map.get(thirdKey);
}
private int countMatches(String normalizedAnswer, List<String> keywords) {
int count = 0;
for (String keyword : safeList(keywords)) {
src/main/java/com/superbiz/agent/service/ChatService.java
+22
View File
@@ -60,6 +60,7 @@ public class ChatService {
private static final Logger logger = LoggerFactory.getLogger(ChatService.class);
private static final String LOW_CONFID_DISCLAIMER = "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。";
private static final String DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论，建议人工介入。";
private static final String CHAT_PROMPT_AUDIT_VERSION = "chat-prompts-v1";
/** 封装 answer + 后端生成的 sessionId，用于 feedback 关联 */
public record ChatResult(String answer, String sessionId) {}
@@ -884,6 +885,7 @@ public class ChatService {
verifierEvaluation.put("rationale", decision.rationale());
verifierEvaluation.put("round", round);
verifierEvaluation.put("traceability_version", "v1");
verifierEvaluation.put("prompt_audit", promptAuditSnapshot());
verifierEvaluation.put("executor_output_parse_status",
Optional.ofNullable(VerifierContextHolder.getExecutorOutputParseStatus())
.orElse(Map.of("status", "missing", "detail", "executor parse status unavailable")));
@@ -902,6 +904,26 @@ public class ChatService {
diagnosisSessionRepository.save(session);
}
private Map<String, Object> promptAuditSnapshot() {
Map<String, Object> audit = new LinkedHashMap<>();
audit.put("version", CHAT_PROMPT_AUDIT_VERSION);
audit.put("prompts", List.of(
promptAuditItem("chat_planner", "chat-planner-v1", "prompts/chat-planner-prompt.md"),
promptAuditItem("chat_executor", "chat-executor-v2", "prompts/chat-executor-prompt.md"),
promptAuditItem("chat_verifier", "chat-verifier-v2", "prompts/chat-verifier-prompt.md"),
promptAuditItem("chat_composer", "chat-composer-v1", "prompts/chat-composer-prompt.md")
));
return audit;
}
private Map<String, Object> promptAuditItem(String name, String version, String resource) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("name", name);
item.put("version", version);
item.put("resource", resource);
return item;
}
private Map<String, Object> defaultGatekeeperPass() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
return Map.of(
src/test/java/com/superbiz/agent/eval/DiagnosisEvalBaselineDiffTest.java
+5 -5
View File
@@ -100,13 +100,13 @@ class DiagnosisEvalBaselineDiffTest {
}
private void degradeRedisCase(DiagnosisEvalReport report) {
report.setPassedCases(9);
report.setPassRate(0.9);
report.setPassedCases(11);
report.setPassRate(11.0 / 12.0);
report.setAverageToolCallCount(3.0);
report.setAverageDurationMs(39800.0);
report.setAverageDurationMs(38500.0);
report.setVerdictDistribution(new LinkedHashMap<>());
report.getVerdictDistribution().put("PASS", 4L);
report.getVerdictDistribution().put("LOW_CONFID", 4L);
report.getVerdictDistribution().put("PASS", 5L);
report.getVerdictDistribution().put("LOW_CONFID", 5L);
report.getVerdictDistribution().put("REJECT", 2L);
DiagnosisEvalResult redis = result(report, "redis-timeout");
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+86 -4
View File
@@ -24,11 +24,11 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(10, report.getTotalCases());
assertEquals(10, report.getPassedCases());
assertEquals(12, report.getTotalCases());
assertEquals(12, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(4L, report.getVerdictDistribution().get("PASS"));
assertEquals(5L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
@@ -36,6 +36,12 @@ class DiagnosisTraceEvaluatorTest {
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
assertTrue(promptGatekeeperAudit.isPassed());
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
assertTrue(hikariNoEvidence.isPassed());
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
@@ -60,6 +66,12 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
assertTrue(composerFallback.isPassed());
assertEquals("composer_malformed", composerFallback.getComposerStatus());
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
assertTrue(auditMetadataLowConfid.isPassed());
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
}
@Test
@@ -195,6 +207,76 @@ class DiagnosisTraceEvaluatorTest {
"gatekeeper rule set version not expected: old-rules"));
}
@Test
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("prompt-audit")
.title("Prompt audit")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requirePromptAudit(true)
.expectedPromptAuditVersion("chat-prompts-v1")
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"prompt_audit", java.util.Map.of(
"version", "old-prompts",
"prompts", java.util.List.of(java.util.Map.of(
"name", "chat_executor",
"version", "chat-executor-v1"
))
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"prompt audit version not expected: old-prompts"));
assertTrue(result.getFailedChecks().contains(
"prompt version not expected: chat_executor=chat-executor-v1"));
}
@Test
void evaluateFailsWhenGatekeeperRulesAreMissing() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-rules")
.title("Gatekeeper rules")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requireGatekeeperRules(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "gatekeeper-rules-v1"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+10
View File
@@ -394,10 +394,20 @@ class ChatServiceSequentialAgentTest {
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
Map<String, Object> verifierEvaluation = captor.getValue();
assertTrue(verifierEvaluation.containsKey("gatekeeper_result"));
assertTrue(verifierEvaluation.containsKey("prompt_audit"));
@SuppressWarnings("unchecked")
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
assertEquals("pass", gatekeeperResult.get("status"));
assertEquals("none", gatekeeperResult.get("severity"));
@SuppressWarnings("unchecked")
Map<String, Object> promptAudit = (Map<String, Object>) verifierEvaluation.get("prompt_audit");
assertEquals("chat-prompts-v1", promptAudit.get("version"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> prompts = (List<Map<String, Object>>) promptAudit.get("prompts");
assertEquals(4, prompts.size());
assertTrue(prompts.stream().anyMatch(prompt ->
"chat_executor".equals(prompt.get("name"))
&& "chat-executor-v2".equals(prompt.get("version"))));
}
@Test