osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(demo): add interview quality audit

Browse Source
This commit is contained in:
zhuyongxin
2026-07-09 11:18:49 +08:00
parent a6c2d4459c
commit 9c9a0024d4
37 changed files with 1162 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java
+5
View File
@@ -6,6 +6,7 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
@Data
@Builder
@@ -29,4 +30,8 @@ public class DiagnosisEvalCase {
private String expectedGatekeeperRuleSetVersion;
private List<String> expectedComposerStatuses;
private List<String> forbiddenConfirmedClaimKeywords;
private Boolean requirePromptAudit;
private String expectedPromptAuditVersion;
private Map<String, String> expectedPromptVersions;
private Boolean requireGatekeeperRules;
}
src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java
+6 -2
View File
@@ -46,8 +46,8 @@ public class DiagnosisEvalReportWriter {
}
builder.append("## Cases\n\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |\n");
for (DiagnosisEvalResult result : report.getResults()) {
builder.append("| ")
.append(result.getCaseId())
@@ -60,10 +60,14 @@ public class DiagnosisEvalReportWriter {
.append(" | ")
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
.append(" | ")
.append(valueOrDash(result.getPromptAuditVersion()))
.append(" | ")
.append(valueOrDash(result.getComposerStatus()))
.append(" | ")
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
.append(" | ")
.append(result.getGatekeeperRuleCount() == null ? "-" : result.getGatekeeperRuleCount())
.append(" | ")
.append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
.append(" | ")
.append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java
+2
View File
@@ -24,8 +24,10 @@ public class DiagnosisEvalResult {
private Map<String, Boolean> evidenceCoverage;
private String gatekeeperStatus;
private String gatekeeperRuleSetVersion;
private String promptAuditVersion;
private String composerStatus;
private Integer claimCheckCount;
private Integer gatekeeperRuleCount;
private Integer toolCallCount;
private Integer durationMs;
}
src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java
+98 -1
View File
@@ -67,8 +67,10 @@ public class DiagnosisTraceEvaluator {
.evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
.gatekeeperStatus(null)
.gatekeeperRuleSetVersion(null)
.promptAuditVersion(null)
.composerStatus(null)
.claimCheckCount(null)
.gatekeeperRuleCount(null)
.toolCallCount(null)
.durationMs(null)
.build());
@@ -123,10 +125,13 @@ public class DiagnosisTraceEvaluator {
String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
String gatekeeperRuleSetVersion = extractNestedString(trace, "verifier_evaluation",
"gatekeeper_result", "rule_set_version");
String promptAuditVersion = extractNestedString(trace, "verifier_evaluation",
"prompt_audit", "version");
String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
Integer gatekeeperRuleCount = countNestedList(trace, "verifier_evaluation", "gatekeeper_result", "rules");
failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
gatekeeperStatus, gatekeeperRuleSetVersion, composerStatus));
gatekeeperStatus, gatekeeperRuleSetVersion, promptAuditVersion, composerStatus));
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
@@ -142,8 +147,10 @@ public class DiagnosisTraceEvaluator {
.evidenceCoverage(evidenceCoverage)
.gatekeeperStatus(gatekeeperStatus)
.gatekeeperRuleSetVersion(gatekeeperRuleSetVersion)
.promptAuditVersion(promptAuditVersion)
.composerStatus(composerStatus)
.claimCheckCount(claimCheckCount)
.gatekeeperRuleCount(gatekeeperRuleCount)
.toolCallCount(toolCallCount)
.durationMs(durationMs)
.build();
@@ -237,6 +244,7 @@ public class DiagnosisTraceEvaluator {
String verdict,
String gatekeeperStatus,
String gatekeeperRuleSetVersion,
String promptAuditVersion,
String composerStatus) {
List<String> failedChecks = new ArrayList<>();
boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
@@ -259,6 +267,10 @@ public class DiagnosisTraceEvaluator {
failedChecks.add("gatekeeper rule set version not expected: "
+ valueOrMissing(gatekeeperRuleSetVersion));
}
if (Boolean.TRUE.equals(evalCase.getRequireGatekeeperRules())) {
failedChecks.addAll(validateGatekeeperRules(trace));
}
failedChecks.addAll(validatePromptAudit(evalCase, trace, promptAuditVersion));
failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
@@ -290,6 +302,78 @@ public class DiagnosisTraceEvaluator {
return failedChecks;
}
private List<String> validatePromptAudit(DiagnosisEvalCase evalCase,
DiagnosisTraceResponse trace,
String promptAuditVersion) {
List<String> failedChecks = new ArrayList<>();
Object promptAudit = nestedValue(trace, "verifier_evaluation", "prompt_audit");
if (Boolean.TRUE.equals(evalCase.getRequirePromptAudit()) && !(promptAudit instanceof Map<?, ?>)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
if (!isBlank(evalCase.getExpectedPromptAuditVersion())
&& !evalCase.getExpectedPromptAuditVersion().equals(promptAuditVersion)) {
failedChecks.add("prompt audit version not expected: " + valueOrMissing(promptAuditVersion));
}
if (evalCase.getExpectedPromptVersions() == null || evalCase.getExpectedPromptVersions().isEmpty()) {
return failedChecks;
}
if (!(promptAudit instanceof Map<?, ?> audit)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
Object promptsValue = audit.get("prompts");
if (!(promptsValue instanceof List<?> prompts)) {
failedChecks.add("prompt_audit missing prompts");
return failedChecks;
}
Map<String, String> actualVersions = new LinkedHashMap<>();
for (Object promptValue : prompts) {
if (promptValue instanceof Map<?, ?> prompt) {
String name = stringValue(prompt.get("name"));
String version = stringValue(prompt.get("version"));
if (!isBlank(name)) {
actualVersions.put(name, version);
}
}
}
for (Map.Entry<String, String> expected : evalCase.getExpectedPromptVersions().entrySet()) {
String actual = actualVersions.get(expected.getKey());
if (!expected.getValue().equals(actual)) {
failedChecks.add("prompt version not expected: "
+ expected.getKey() + "=" + valueOrMissing(actual));
}
}
return failedChecks;
}
private List<String> validateGatekeeperRules(DiagnosisTraceResponse trace) {
Object rulesValue = nestedNestedValue(trace, "verifier_evaluation", "gatekeeper_result", "rules");
if (!(rulesValue instanceof List<?> rules) || rules.isEmpty()) {
return List.of("gatekeeper_result missing rules");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : rules) {
if (!(item instanceof Map<?, ?> rule)) {
failedChecks.add("gatekeeper rule metadata is not an object");
continue;
}
String id = stringValue(rule.get("id"));
Object enabled = rule.get("enabled");
String severity = stringValue(rule.get("default_severity"));
if (isBlank(id)) {
failedChecks.add("gatekeeper rule metadata missing id");
}
if (!(enabled instanceof Boolean)) {
failedChecks.add("gatekeeper rule metadata missing enabled: " + valueOrMissing(id));
}
if (isBlank(severity)) {
failedChecks.add("gatekeeper rule metadata missing default_severity: " + valueOrMissing(id));
}
}
return failedChecks;
}
private List<String> validateClaimChecks(DiagnosisTraceResponse trace, boolean required) {
Object claimChecks = nestedValue(trace, "verifier_evaluation", "claim_checks");
if (!(claimChecks instanceof List<?> claimCheckList)) {
@@ -347,6 +431,19 @@ public class DiagnosisTraceEvaluator {
return value instanceof List<?> list ? list.size() : null;
}
private Integer countNestedList(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedNestedValue(trace, firstKey, secondKey, thirdKey);
return value instanceof List<?> list ? list.size() : null;
}
private Object nestedNestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedValue(trace, firstKey, secondKey);
if (!(value instanceof Map<?, ?> map)) {
return null;
}
return map.get(thirdKey);
}
private int countMatches(String normalizedAnswer, List<String> keywords) {
int count = 0;
for (String keyword : safeList(keywords)) {
src/main/java/com/superbiz/agent/service/ChatService.java
+22
View File
@@ -60,6 +60,7 @@ public class ChatService {
private static final Logger logger = LoggerFactory.getLogger(ChatService.class);
private static final String LOW_CONFID_DISCLAIMER = "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。";
private static final String DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论，建议人工介入。";
private static final String CHAT_PROMPT_AUDIT_VERSION = "chat-prompts-v1";
/** 封装 answer + 后端生成的 sessionId，用于 feedback 关联 */
public record ChatResult(String answer, String sessionId) {}
@@ -884,6 +885,7 @@ public class ChatService {
verifierEvaluation.put("rationale", decision.rationale());
verifierEvaluation.put("round", round);
verifierEvaluation.put("traceability_version", "v1");
verifierEvaluation.put("prompt_audit", promptAuditSnapshot());
verifierEvaluation.put("executor_output_parse_status",
Optional.ofNullable(VerifierContextHolder.getExecutorOutputParseStatus())
.orElse(Map.of("status", "missing", "detail", "executor parse status unavailable")));
@@ -902,6 +904,26 @@ public class ChatService {
diagnosisSessionRepository.save(session);
}
private Map<String, Object> promptAuditSnapshot() {
Map<String, Object> audit = new LinkedHashMap<>();
audit.put("version", CHAT_PROMPT_AUDIT_VERSION);
audit.put("prompts", List.of(
promptAuditItem("chat_planner", "chat-planner-v1", "prompts/chat-planner-prompt.md"),
promptAuditItem("chat_executor", "chat-executor-v2", "prompts/chat-executor-prompt.md"),
promptAuditItem("chat_verifier", "chat-verifier-v2", "prompts/chat-verifier-prompt.md"),
promptAuditItem("chat_composer", "chat-composer-v1", "prompts/chat-composer-prompt.md")
));
return audit;
}
private Map<String, Object> promptAuditItem(String name, String version, String resource) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("name", name);
item.put("version", version);
item.put("resource", resource);
return item;
}
private Map<String, Object> defaultGatekeeperPass() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
return Map.of(