feat(eval): add executor audit closure checks

src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java

@@ -22,4 +22,10 @@ public class DiagnosisEvalCase {
     private List<String> requiredEvidenceTools;
     private List<String> allowedVerdicts;
     private List<String> forbiddenAnswerKeywords;
+    private Boolean requireV2AuditClosure;
+    private Boolean requireClaimChecks;
+    private Boolean requireComposerOutput;
+    private List<String> expectedGatekeeperStatuses;
+    private List<String> expectedComposerStatuses;
+    private List<String> forbiddenConfirmedClaimKeywords;
 }

src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java

@@ -46,8 +46,8 @@ public class DiagnosisEvalReportWriter {
         }
 
         builder.append("## Cases\n\n");
-        builder.append("| Case | Result | Verdict | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
-        builder.append("| --- | --- | --- | --- | ---: | ---: | --- |\n");
+        builder.append("| Case | Result | Verdict | Gatekeeper | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
+        builder.append("| --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |\n");
         for (DiagnosisEvalResult result : report.getResults()) {
             builder.append("| ")
                     .append(result.getCaseId())
@@ -56,6 +56,12 @@ public class DiagnosisEvalReportWriter {
                     .append(" | ")
                     .append(valueOrDash(result.getVerdict()))
                     .append(" | ")
+                    .append(valueOrDash(result.getGatekeeperStatus()))
+                    .append(" | ")
+                    .append(valueOrDash(result.getComposerStatus()))
+                    .append(" | ")
+                    .append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
+                    .append(" | ")
                     .append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
                     .append(" | ")
                     .append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())

src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java

@@ -22,6 +22,9 @@ public class DiagnosisEvalResult {
     private int matchedKeywordCount;
     private int requiredKeywordCount;
     private Map<String, Boolean> evidenceCoverage;
+    private String gatekeeperStatus;
+    private String composerStatus;
+    private Integer claimCheckCount;
     private Integer toolCallCount;
     private Integer durationMs;
 }

src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java

@@ -20,6 +20,20 @@ public class DiagnosisTraceEvaluator {
 
     private static final TypeReference<List<DiagnosisEvalCase>> CASE_LIST_TYPE = new TypeReference<>() {};
     private static final String REJECT_DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论";
+    private static final Set<String> RAW_EXECUTOR_MARKERS = Set.of(
+            "executor_evidence_v2",
+            "answer_version",
+            "evidence_bindings",
+            "claim_id"
+    );
+    private static final Set<String> VALID_CLAIM_VERIFICATIONS = Set.of(
+            "direct_observation",
+            "reasonable_inference",
+            "overstated",
+            "unsupported",
+            "external_unknown",
+            "contradicted"
+    );
 
     private final ObjectMapper objectMapper;
 
@@ -51,6 +65,9 @@ public class DiagnosisTraceEvaluator {
                         .matchedKeywordCount(0)
                         .requiredKeywordCount(size(evalCase.getExpectedRootCauseKeywords()))
                         .evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
+                        .gatekeeperStatus(null)
+                        .composerStatus(null)
+                        .claimCheckCount(null)
                         .toolCallCount(null)
                         .durationMs(null)
                         .build());
@@ -102,6 +119,11 @@ public class DiagnosisTraceEvaluator {
         }
 
         failedChecks.addAll(validateExecutorStructuredOutput(trace));
+        String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
+        String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
+        Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
+        failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
+                gatekeeperStatus, composerStatus));
 
         Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
         Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
@@ -115,6 +137,9 @@ public class DiagnosisTraceEvaluator {
                 .matchedKeywordCount(matchedKeywordCount)
                 .requiredKeywordCount(requiredKeywordCount)
                 .evidenceCoverage(evidenceCoverage)
+                .gatekeeperStatus(gatekeeperStatus)
+                .composerStatus(composerStatus)
+                .claimCheckCount(claimCheckCount)
                 .toolCallCount(toolCallCount)
                 .durationMs(durationMs)
                 .build();
@@ -202,6 +227,91 @@ public class DiagnosisTraceEvaluator {
         return failedChecks;
     }
 
+    private List<String> validateV2AuditClosure(DiagnosisEvalCase evalCase,
+                                                DiagnosisTraceResponse trace,
+                                                String normalizedAnswer,
+                                                String verdict,
+                                                String gatekeeperStatus,
+                                                String composerStatus) {
+        List<String> failedChecks = new ArrayList<>();
+        boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
+        boolean requireClaimChecks = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireClaimChecks());
+        boolean requireComposerOutput = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireComposerOutput());
+
+        Object gatekeeperResult = nestedValue(trace, "verifier_evaluation", "gatekeeper_result");
+        if (requireV2AuditClosure && !(gatekeeperResult instanceof Map<?, ?>)) {
+            failedChecks.add("missing gatekeeper_result");
+        }
+        if ("fail".equals(gatekeeperStatus) && "PASS".equals(verdict)) {
+            failedChecks.add("gatekeeper fail cannot have PASS verdict");
+        }
+        if (!safeList(evalCase.getExpectedGatekeeperStatuses()).isEmpty()
+                && !safeList(evalCase.getExpectedGatekeeperStatuses()).contains(gatekeeperStatus)) {
+            failedChecks.add("gatekeeper status not expected: " + valueOrMissing(gatekeeperStatus));
+        }
+
+        failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
+
+        Object composerOutput = nestedValue(trace, "verifier_evaluation", "composer_output");
+        if (requireComposerOutput && !(composerOutput instanceof Map<?, ?>)) {
+            failedChecks.add("missing composer_output");
+        }
+        if (requireComposerOutput && isBlank(composerStatus)) {
+            failedChecks.add("composer_output missing status");
+        }
+        if (!safeList(evalCase.getExpectedComposerStatuses()).isEmpty()
+                && !safeList(evalCase.getExpectedComposerStatuses()).contains(composerStatus)) {
+            failedChecks.add("composer status not expected: " + valueOrMissing(composerStatus));
+        }
+
+        for (String forbidden : safeList(evalCase.getForbiddenConfirmedClaimKeywords())) {
+            if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) {
+                failedChecks.add("answer contains forbidden confirmed claim keyword: " + forbidden);
+            }
+        }
+
+        if (requireV2AuditClosure) {
+            for (String marker : RAW_EXECUTOR_MARKERS) {
+                if (normalizedAnswer.contains(marker.toLowerCase(Locale.ROOT))) {
+                    failedChecks.add("answer leaks raw executor marker: " + marker);
+                }
+            }
+        }
+        return failedChecks;
+    }
+
+    private List<String> validateClaimChecks(DiagnosisTraceResponse trace, boolean required) {
+        Object claimChecks = nestedValue(trace, "verifier_evaluation", "claim_checks");
+        if (!(claimChecks instanceof List<?> claimCheckList)) {
+            return required ? List.of("missing claim_checks") : List.of();
+        }
+        if (required && claimCheckList.isEmpty()) {
+            return List.of("claim_checks is empty");
+        }
+
+        List<String> failedChecks = new ArrayList<>();
+        for (Object item : claimCheckList) {
+            if (!(item instanceof Map<?, ?> claimCheck)) {
+                failedChecks.add("claim_check is not an object");
+                continue;
+            }
+            String claimId = stringValue(claimCheck.get("claim_id"));
+            String verification = stringValue(claimCheck.get("verification"));
+            if (isBlank(claimId)) {
+                failedChecks.add("claim_check missing claim_id");
+            }
+            if (isBlank(verification)) {
+                failedChecks.add("claim_check missing verification: " + valueOrMissing(claimId));
+            } else if (!VALID_CLAIM_VERIFICATIONS.contains(verification)) {
+                failedChecks.add("claim_check verification invalid: " + verification);
+            }
+            if (isBlank(stringValue(claimCheck.get("detail")))) {
+                failedChecks.add("claim_check missing detail: " + valueOrMissing(claimId));
+            }
+        }
+        return failedChecks;
+    }
+
     private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
         if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) {
             return null;
@@ -213,6 +323,20 @@ public class DiagnosisTraceEvaluator {
         return map.get(secondKey);
     }
 
+    private String extractNestedString(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
+        Object value = nestedValue(trace, firstKey, secondKey);
+        if (!(value instanceof Map<?, ?> map)) {
+            return null;
+        }
+        Object nested = map.get(thirdKey);
+        return nested == null ? null : String.valueOf(nested);
+    }
+
+    private Integer countList(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
+        Object value = nestedValue(trace, firstKey, secondKey);
+        return value instanceof List<?> list ? list.size() : null;
+    }
+
     private int countMatches(String normalizedAnswer, List<String> keywords) {
         int count = 0;
         for (String keyword : safeList(keywords)) {
@@ -242,4 +366,16 @@ public class DiagnosisTraceEvaluator {
     private String nullToEmpty(String value) {
         return value == null ? "" : value;
     }
+
+    private String stringValue(Object value) {
+        return value == null ? null : String.valueOf(value);
+    }
+
+    private boolean isBlank(String value) {
+        return value == null || value.isBlank();
+    }
+
+    private String valueOrMissing(String value) {
+        return isBlank(value) ? "missing" : value;
+    }
 }