feat(agent): add verifier claim checks

2026-07-08 02:33:02 +08:00
parent c5e496e715
commit 1b31e78be5
19 changed files with 1247 additions and 115 deletions
@@ -204,19 +204,22 @@ public class AgentLoggingHook extends MessagesModelHook {
    private String summarizeVerifierThought(String verifierOutput) {
        try {
            JsonNode root = objectMapper.readTree(verifierOutput);
+            int claimCount = root.path("claim_checks").isArray() ? root.path("claim_checks").size() : 0;
            int factCount = root.path("facts_checked").isArray() ? root.path("facts_checked").size() : 0;
            int tracedFactCount = 0;
-            if (root.path("facts_checked").isArray()) {
-                for (JsonNode factNode : root.path("facts_checked")) {
-                    if (factNode.path("evidence_refs").isArray() && factNode.path("evidence_refs").size() > 0) {
+            JsonNode tracedNodes = root.path("claim_checks").isArray() ? root.path("claim_checks") : root.path("facts_checked");
+            if (tracedNodes.isArray()) {
+                for (JsonNode node : tracedNodes) {
+                    if (node.path("evidence_refs").isArray() && node.path("evidence_refs").size() > 0) {
                        tracedFactCount++;
                    }
                }
            }
-            return "verdict=%s, score=%s, critical_fact_count=%s, facts_checked=%d, traced_facts=%d".formatted(
+            return "verdict=%s, score=%s, critical_fact_count=%s, claim_checks=%d, facts_checked=%d, traced_facts=%d".formatted(
                    root.path("verdict").asText("UNKNOWN"),
                    root.path("groundedness_score").asText("0.0"),
                    root.path("critical_fact_count").asText("0"),
+                    claimCount,
                    factCount,
                    tracedFactCount
            );
@@ -650,12 +650,18 @@ public class ChatService {

        try {
            JsonNode root = objectMapper.readTree(sanitizeJsonPayload(verifierOutput));
-            List<Map<String, Object>> factsChecked = parseFactsChecked(root.path("facts_checked"));
+            List<Map<String, Object>> claimChecks = parseClaimChecks(root.path("claim_checks"));
+            List<Map<String, Object>> factsChecked = claimChecks.isEmpty()
+                    ? parseFactsChecked(root.path("facts_checked"))
+                    : mapClaimChecksToFactsChecked(claimChecks);
+            String verdict = effectiveVerifierVerdict(root.path("verdict").asText("LOW_CONFID"));
+            int criticalFactCount = root.path("critical_fact_count").asInt(countCriticalFacts(factsChecked));

            return new VerifierDecision(
-                    root.path("verdict").asText("LOW_CONFID"),
+                    verdict,
                    root.path("groundedness_score").asDouble(0.0),
-                    root.path("critical_fact_count").asInt(0),
+                    criticalFactCount,
+                    claimChecks,
                    factsChecked,
                    root.path("rationale").asText(""),
                    round
@@ -678,6 +684,104 @@ public class ChatService {
        return trimmed;
    }

+    private String effectiveVerifierVerdict(String modelVerdict) {
+        String verdict = normalizeVerdict(modelVerdict);
+        Map<String, Object> parseStatus = VerifierContextHolder.getExecutorOutputParseStatus();
+        String parseState = parseStatus == null ? "" : String.valueOf(parseStatus.getOrDefault("status", ""));
+        if (("missing".equals(parseState) || "malformed".equals(parseState)) && "PASS".equals(verdict)) {
+            return "LOW_CONFID";
+        }
+
+        Map<String, Object> gatekeeperResult = VerifierContextHolder.getGatekeeperResult();
+        if (gatekeeperResult == null || !"fail".equals(String.valueOf(gatekeeperResult.get("status")))) {
+            return verdict;
+        }
+        if (containsRule(gatekeeperResult.get("failed_rules"), ExecutorGatekeeperService.RULE_INVOCATION_REF)) {
+            return "REJECT";
+        }
+        return "PASS".equals(verdict) ? "LOW_CONFID" : verdict;
+    }
+
+    private String normalizeVerdict(String verdict) {
+        if ("PASS".equals(verdict) || "LOW_CONFID".equals(verdict) || "REJECT".equals(verdict)) {
+            return verdict;
+        }
+        return "LOW_CONFID";
+    }
+
+    private boolean containsRule(Object rulesValue, String ruleId) {
+        if (!(rulesValue instanceof List<?> rules)) {
+            return false;
+        }
+        return rules.stream().anyMatch(rule -> ruleId.equals(String.valueOf(rule)));
+    }
+
+    private int countCriticalFacts(List<Map<String, Object>> factsChecked) {
+        return (int) factsChecked.stream()
+                .filter(fact -> Boolean.TRUE.equals(fact.get("is_critical")))
+                .count();
+    }
+
+    private List<Map<String, Object>> parseClaimChecks(JsonNode claimChecksNode) {
+        List<Map<String, Object>> claimChecks = new ArrayList<>();
+        if (!claimChecksNode.isArray()) {
+            return claimChecks;
+        }
+        for (JsonNode claimNode : claimChecksNode) {
+            Map<String, Object> claimCheck = new LinkedHashMap<>();
+            claimCheck.put("claim_id", claimNode.path("claim_id").asText(""));
+            claimCheck.put("claim_text", claimNode.path("claim_text").asText(""));
+            claimCheck.put("claim_type", claimNode.path("claim_type").asText(""));
+            claimCheck.put("verification", normalizeClaimVerification(claimNode.path("verification").asText("unsupported")));
+            claimCheck.put("detail", claimNode.path("detail").asText(""));
+            claimCheck.put("evidence_refs", parseEvidenceRefs(claimNode.path("evidence_refs")));
+            claimChecks.add(claimCheck);
+        }
+        return claimChecks;
+    }
+
+    private String normalizeClaimVerification(String verification) {
+        return switch (verification) {
+            case "direct_observation", "reasonable_inference", "overstated", "unsupported",
+                 "external_unknown", "contradicted" -> verification;
+            default -> "unsupported";
+        };
+    }
+
+    private List<Map<String, Object>> mapClaimChecksToFactsChecked(List<Map<String, Object>> claimChecks) {
+        List<Map<String, Object>> factsChecked = new ArrayList<>();
+        for (Map<String, Object> claimCheck : claimChecks) {
+            String claimId = String.valueOf(claimCheck.getOrDefault("claim_id", ""));
+            String claimText = String.valueOf(claimCheck.getOrDefault("claim_text", ""));
+            String claimType = String.valueOf(claimCheck.getOrDefault("claim_type", ""));
+            Map<String, Object> fact = new LinkedHashMap<>();
+            fact.put("fact", claimId.isBlank() ? claimText : claimId + ": " + claimText);
+            fact.put("is_critical", isCriticalClaimType(claimType));
+            fact.put("verification", mapClaimVerificationToFactVerification(
+                    String.valueOf(claimCheck.getOrDefault("verification", "unsupported"))));
+            fact.put("detail", claimCheck.getOrDefault("detail", ""));
+            fact.put("evidence_refs", claimCheck.getOrDefault("evidence_refs", List.of()));
+            factsChecked.add(fact);
+        }
+        return factsChecked;
+    }
+
+    private boolean isCriticalClaimType(String claimType) {
+        return "root_cause".equals(claimType)
+                || "symptom".equals(claimType)
+                || "impact".equals(claimType)
+                || "risk".equals(claimType);
+    }
+
+    private String mapClaimVerificationToFactVerification(String verification) {
+        return switch (verification) {
+            case "direct_observation" -> "direct_evidence";
+            case "reasonable_inference", "overstated" -> "indirect_support";
+            case "contradicted" -> "contradicted";
+            default -> "no_evidence";
+        };
+    }
+
    private List<Map<String, Object>> parseFactsChecked(JsonNode factsNode) {
        List<Map<String, Object>> factsChecked = new ArrayList<>();
        if (!factsNode.isArray()) {
@@ -723,7 +827,7 @@ public class ChatService {
    }

    private VerifierDecision buildVerifierFallbackDecision(int round, String rationale) {
-        return new VerifierDecision("LOW_CONFID", 0.0, 0, List.of(), rationale, round);
+        return new VerifierDecision("LOW_CONFID", 0.0, 0, List.of(), List.of(), rationale, round);
    }

    private String extractStateText(Optional<OverAllState> stateOptional, String key) {
@@ -748,6 +852,7 @@ public class ChatService {
        verifierEvaluation.put("verdict", decision.verdict());
        verifierEvaluation.put("groundedness_score", decision.groundednessScore());
        verifierEvaluation.put("critical_fact_count", decision.criticalFactCount());
+        verifierEvaluation.put("claim_checks", decision.claimChecks());
        verifierEvaluation.put("facts_checked", decision.factsChecked());
        verifierEvaluation.put("rationale", decision.rationale());
        verifierEvaluation.put("round", round);
@@ -1005,6 +1110,7 @@ public class ChatService {
            String verdict,
            double groundednessScore,
            int criticalFactCount,
+            List<Map<String, Object>> claimChecks,
            List<Map<String, Object>> factsChecked,
            String rationale,
            int round