feat: add chat verifier agent

2026-07-03 10:54:33 +08:00
parent 4f5316d473
commit 9050487307
28 changed files with 3200 additions and 209 deletions
@@ -5,6 +5,8 @@ import com.alibaba.cloud.ai.graph.RunnableConfig;
 import com.alibaba.cloud.ai.graph.agent.ReactAgent;
 import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
 import com.alibaba.cloud.ai.graph.exception.GraphRunnerException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.superbiz.agent.agent.tool.DateTimeTools;
 import com.superbiz.agent.agent.tool.InternalDocsTools;
 import com.superbiz.agent.agent.tool.QueryLogsTools;
@@ -13,13 +15,14 @@ import com.superbiz.agent.domain.entity.DiagnosisSession;
 import com.superbiz.agent.hook.AgentLoggingHook;
 import com.superbiz.agent.hook.TokenTrackingChatModel;
 import com.superbiz.agent.hook.TokenUsageHolder;
+import com.superbiz.agent.hook.VerifierInputHook;
 import com.superbiz.agent.repository.AgentStepRepository;
 import com.superbiz.agent.repository.DiagnosisSessionRepository;
 import com.superbiz.agent.tool.LookupKnowledgeTool;
 import com.superbiz.agent.tool.RetrievedDocTracker;
 import com.superbiz.agent.util.QuestionComplexity;
 import com.superbiz.agent.util.SessionContextHolder;
-import com.superbiz.agent.service.KnowledgeDomainService;
+import com.superbiz.agent.util.VerifierContextHolder;

 import jakarta.annotation.PostConstruct;
 import org.slf4j.Logger;
@@ -29,11 +32,14 @@ import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.tool.ToolCallback;
 import org.springframework.ai.tool.ToolCallbackProvider;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.stereotype.Service;

 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -47,6 +53,8 @@ import java.util.UUID;
 public class ChatService {

    private static final Logger logger = LoggerFactory.getLogger(ChatService.class);
+    private static final String LOW_CONFID_DISCLAIMER = "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。";
+    private static final String DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论，建议人工介入。";

    /** 封装 answer + 后端生成的 sessionId，用于 feedback 关联 */
    public record ChatResult(String answer, String sessionId) {}
@@ -87,9 +95,20 @@ public class ChatService {
    @Autowired
    private KnowledgeDomainService knowledgeDomainService;

+    @Autowired
+    private ToolTraceSummaryService toolTraceSummaryService;
+
+    @Autowired
+    private SelfEvaluationMergeService selfEvaluationMergeService;
+
+    @Value("${verifier.low-confidence-threshold:0.5}")
+    private double verifierLowConfidenceThreshold;
+
    /** 多 Agent Chat 的 Prompt */
    private String chatPlannerPrompt;
    private String chatExecutorPrompt;
+    private String chatVerifierPrompt;
+    private final ObjectMapper objectMapper = new ObjectMapper();

    @PostConstruct
    public void init() {
@@ -101,6 +120,9 @@ public class ChatService {
            chatExecutorPrompt = new String(
                new ClassPathResource("prompts/chat-executor-prompt.md").getInputStream().readAllBytes(),
                StandardCharsets.UTF_8);
+            chatVerifierPrompt = new String(
+                new ClassPathResource("prompts/chat-verifier-prompt.md").getInputStream().readAllBytes(),
+                StandardCharsets.UTF_8);
            logger.info("Chat 多 Agent Prompts 加载成功");
        } catch (IOException e) {
            logger.error("加载 Chat Prompt 文件失败", e);
@@ -341,34 +363,71 @@ public class ChatService {
        diagnosisSessionRepository.save(session);

        SessionContextHolder.setSessionId(sessionId);
+        VerifierContextHolder.setOriginalQuery(question);
+        VerifierContextHolder.setRetryContext(null);
+        VerifierContextHolder.setExecutorFinalAnswer(null);

        try {
-            ReactAgent planner = buildChatPlannerAgent(chatModel, toolCallbacks, history);
-            ReactAgent executor = buildChatExecutorAgent(chatModel, toolCallbacks, history);
-
-            SupervisorAgent supervisor = SupervisorAgent.builder()
-                    .name("chat_supervisor")
-                    .description("负责调度 Planner 与 Executor 的多 Agent 控制器")
-                    .model(chatModel)
-                    .systemPrompt("你是一个智能任务调度器。分析用户问题，调用 Planner 拆解步骤，调用 Executor 执行各步骤。")
-                    .subAgents(List.of(planner, executor))
+            VerifierDecision finalDecision = null;
+            String retryContext = null;
+            String answer = null;
+            RunnableConfig config = RunnableConfig.builder()
+                    .addMetadata("sessionId", sessionId)
                    .build();

-            Optional<OverAllState> stateOptional = supervisor.invoke(question);
-            long duration = System.currentTimeMillis() - startTime;
+            for (int round = 1; round <= 2; round++) {
+                VerifierContextHolder.setRetryContext(retryContext);
+                VerifierContextHolder.setToolTraceSummary(null);

-            String answer = null;
-            if (stateOptional.isPresent()) {
-                // 从 state 中提取 Executor 的最终输出
-                OverAllState state = stateOptional.get();
-                Optional<AssistantMessage> executorOutput = state.value("executor_feedback")
-                        .filter(AssistantMessage.class::isInstance)
-                        .map(AssistantMessage.class::cast);
-                if (executorOutput.isPresent()) {
-                    answer = executorOutput.get().getText();
+                ReactAgent planner = buildChatPlannerAgent(chatModel, history, retryContext);
+                ReactAgent executor = buildChatExecutorAgent(chatModel, toolCallbacks, history, retryContext);
+                ReactAgent verifier = buildChatVerifierAgent(chatModel);
+
+                SupervisorAgent supervisor = SupervisorAgent.builder()
+                        .name("chat_supervisor")
+                        .description("负责按单轮顺序调度 Planner、Executor、Verifier 的多 Agent 控制器")
+                        .model(chatModel)
+                        .systemPrompt(buildSupervisorPrompt(round))
+                        .subAgents(List.of(planner, executor, verifier))
+                        .build();
+
+                String plannerPlan = callAgent(planner, buildPlannerInput(question, retryContext), config);
+                answer = callAgent(executor, buildExecutorInput(question, plannerPlan, retryContext), config);
+                VerifierContextHolder.setExecutorFinalAnswer(answer);
+                String verifierOutput = callAgent(verifier, "VERIFY", config);
+                finalDecision = parseVerifierDecision(verifierOutput, round);
+
+                if (finalDecision == null) {
+                    finalDecision = buildVerifierFallbackDecision(round, "verifier_output 缺失或无法解析");
+                    answer = buildLowConfidenceOutput(answer, finalDecision);
+                    persistVerifierEvaluation(session, finalDecision, round);
+                    break;
                }
+
+                if ("PASS".equals(finalDecision.verdict())) {
+                    answer = answer == null || answer.isBlank() ? "抱歉，多 Agent 分析未能生成有效结论。" : answer;
+                    persistVerifierEvaluation(session, finalDecision, round);
+                    break;
+                }
+
+                if ("REJECT".equals(finalDecision.verdict())) {
+                    answer = buildDegradedOutput(finalDecision);
+                    persistVerifierEvaluation(session, finalDecision, round);
+                    break;
+                }
+
+                if (finalDecision.groundednessScore() >= verifierLowConfidenceThreshold || round == 2) {
+                    answer = buildLowConfidenceOutput(answer, finalDecision);
+                    persistVerifierEvaluation(session, finalDecision, round);
+                    break;
+                }
+
+                retryContext = buildRetryContext(finalDecision);
+                persistVerifierEvaluation(session, finalDecision, round);
            }

+            long duration = System.currentTimeMillis() - startTime;
+
            if (answer == null || answer.isBlank()) {
                answer = "抱歉，多 Agent 分析未能生成有效结论。";
            }
@@ -394,11 +453,12 @@ public class ChatService {
        } finally {
            retrievedDocTracker.clearSession(sessionId);
            SessionContextHolder.clear();
+            VerifierContextHolder.clear();
        }
    }

-    private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
-                                             List<Map<String, String>> history) {
+    private ReactAgent buildChatPlannerAgent(ChatModel chatModel, List<Map<String, String>> history,
+                                             String retryContext) {
        StringBuilder prompt = new StringBuilder(chatPlannerPrompt);

        // 注入 knowledge map
@@ -414,6 +474,9 @@ public class ChatService {
            }
            prompt.append("--- 对话历史结束 ---\n");
        }
+        if (retryContext != null && !retryContext.isBlank()) {
+            prompt.append("\n\n--- 本轮补证据约束 ---\n").append(retryContext).append("\n");
+        }
        return ReactAgent.builder()
                .name("chat_planner")
                .description("负责拆解问题、规划步骤")
@@ -424,8 +487,20 @@ public class ChatService {
                .build();
    }

+    private ReactAgent buildChatVerifierAgent(ChatModel chatModel) {
+        return ReactAgent.builder()
+                .name("chat_verifier")
+                .description("负责验证 Executor 答案的事实准确性")
+                .model(chatModel)
+                .systemPrompt(chatVerifierPrompt)
+                .hooks(new AgentLoggingHook(agentStepRepository, "verifier"),
+                        new VerifierInputHook(toolTraceSummaryService))
+                .outputKey("verifier_output")
+                .build();
+    }
+
    private ReactAgent buildChatExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
-                                              List<Map<String, String>> history) {
+                                              List<Map<String, String>> history, String retryContext) {
        StringBuilder prompt = new StringBuilder(chatExecutorPrompt);
        if (!history.isEmpty()) {
            prompt.append("\n\n--- 对话历史 ---\n");
@@ -434,6 +509,9 @@ public class ChatService {
            }
            prompt.append("--- 对话历史结束 ---\n");
        }
+        if (retryContext != null && !retryContext.isBlank()) {
+            prompt.append("\n\n--- 本轮补证据约束 ---\n").append(retryContext).append("\n");
+        }
        return ReactAgent.builder()
                .name("chat_executor")
                .description("负责执行具体步骤并及时反馈")
@@ -446,6 +524,316 @@ public class ChatService {
                .build();
    }

+    private String callAgent(ReactAgent agent, String input, RunnableConfig config) throws GraphRunnerException {
+        return agent.call(input, config).getText();
+    }
+
+    private String buildPlannerInput(String question, String retryContext) {
+        if (retryContext == null || retryContext.isBlank()) {
+            return question;
+        }
+        return question + "\n\n--- 补充约束 ---\n" + retryContext;
+    }
+
+    private String buildExecutorInput(String question, String plannerPlan, String retryContext) {
+        StringBuilder input = new StringBuilder(question);
+        if (plannerPlan != null && !plannerPlan.isBlank()) {
+            input.append("\n\n--- planner_plan ---\n").append(plannerPlan);
+        }
+        if (retryContext != null && !retryContext.isBlank()) {
+            input.append("\n\n--- retry_context ---\n").append(retryContext);
+        }
+        return input.toString();
+    }
+
+    private VerifierDecision parseVerifierDecision(String verifierOutput, int round) {
+        if (verifierOutput == null || verifierOutput.isBlank()) {
+            return null;
+        }
+
+        try {
+            JsonNode root = objectMapper.readTree(sanitizeJsonPayload(verifierOutput));
+            List<Map<String, Object>> factsChecked = parseFactsChecked(root.path("facts_checked"));
+
+            return new VerifierDecision(
+                    root.path("verdict").asText("LOW_CONFID"),
+                    root.path("groundedness_score").asDouble(0.0),
+                    root.path("critical_fact_count").asInt(0),
+                    factsChecked,
+                    root.path("rationale").asText(""),
+                    round
+            );
+        } catch (Exception e) {
+            logger.error("解析 verifier_output 失败: {}", verifierOutput, e);
+            return null;
+        }
+    }
+
+    private String sanitizeJsonPayload(String raw) {
+        String trimmed = raw.trim();
+        if (trimmed.startsWith("```")) {
+            int firstNewline = trimmed.indexOf('\n');
+            int lastFence = trimmed.lastIndexOf("```");
+            if (firstNewline >= 0 && lastFence > firstNewline) {
+                return trimmed.substring(firstNewline + 1, lastFence).trim();
+            }
+        }
+        return trimmed;
+    }
+
+    private List<Map<String, Object>> parseFactsChecked(JsonNode factsNode) {
+        List<Map<String, Object>> factsChecked = new ArrayList<>();
+        if (!factsNode.isArray()) {
+            return factsChecked;
+        }
+        for (JsonNode factNode : factsNode) {
+            Map<String, Object> fact = new LinkedHashMap<>();
+            fact.put("fact", factNode.path("fact").asText(""));
+            fact.put("is_critical", factNode.path("is_critical").asBoolean(false));
+            fact.put("verification", factNode.path("verification").asText(""));
+            fact.put("detail", factNode.path("detail").asText(""));
+            fact.put("evidence_refs", parseEvidenceRefs(factNode.path("evidence_refs")));
+            factsChecked.add(fact);
+        }
+        return factsChecked;
+    }
+
+    private List<Map<String, Object>> parseEvidenceRefs(JsonNode evidenceRefsNode) {
+        List<Map<String, Object>> evidenceRefs = new ArrayList<>();
+        if (!evidenceRefsNode.isArray()) {
+            return evidenceRefs;
+        }
+        for (JsonNode refNode : evidenceRefsNode) {
+            Map<String, Object> evidenceRef = new LinkedHashMap<>();
+            evidenceRef.put("trace_ref", refNode.path("trace_ref").asText(""));
+            evidenceRef.put("tool_name", refNode.path("tool_name").asText(""));
+            evidenceRef.put("topic_domain", refNode.path("topic_domain").asText(""));
+            evidenceRef.put("note", refNode.path("note").asText(""));
+
+            List<Long> sourceInvocationIds = new ArrayList<>();
+            JsonNode idsNode = refNode.path("source_invocation_ids");
+            if (idsNode.isArray()) {
+                for (JsonNode idNode : idsNode) {
+                    if (idNode.canConvertToLong()) {
+                        sourceInvocationIds.add(idNode.asLong());
+                    }
+                }
+            }
+            evidenceRef.put("source_invocation_ids", sourceInvocationIds);
+            evidenceRefs.add(evidenceRef);
+        }
+        return evidenceRefs;
+    }
+
+    private VerifierDecision buildVerifierFallbackDecision(int round, String rationale) {
+        return new VerifierDecision("LOW_CONFID", 0.0, 0, List.of(), rationale, round);
+    }
+
+    private String buildSupervisorPrompt(int round) {
+        return """
+                你是一个多 Agent 调度器。每一轮必须严格按顺序完成以下动作：
+                1. 先调用 chat_planner 生成执行计划
+                2. 再调用 chat_executor 执行计划并形成最终答案
+                3. 最后调用 chat_verifier 对 executor 最终答案做事实核查
+
+                规则：
+                - 本轮只允许完成一次 Planner -> Executor -> Verifier 链路
+                - Verifier 完成后立即停止，不要继续调用任何 Agent
+                - 不要自己编造答案，最终用户输出由外层代码根据 verifier_output 决定
+                - 当前是第 %d 轮，保持单轮内顺序稳定
+                """.formatted(round);
+    }
+
+    private String buildRoundInput(String question, String retryContext) {
+        if (retryContext == null || retryContext.isBlank()) {
+            return question;
+        }
+        return question + "\n\n--- 补充约束 ---\n" + retryContext;
+    }
+
+    private String extractExecutorAnswer(Optional<OverAllState> stateOptional) {
+        if (stateOptional.isEmpty()) {
+            return null;
+        }
+        return stateOptional.get().value("executor_feedback")
+                .filter(AssistantMessage.class::isInstance)
+                .map(AssistantMessage.class::cast)
+                .map(AssistantMessage::getText)
+                .orElse(null);
+    }
+
+    private VerifierDecision parseVerifierDecision(Optional<OverAllState> stateOptional, int round) {
+        if (stateOptional.isEmpty()) {
+            return null;
+        }
+
+        Optional<AssistantMessage> verifierOutput = stateOptional.get().value("verifier_output")
+                .filter(AssistantMessage.class::isInstance)
+                .map(AssistantMessage.class::cast);
+
+        if (verifierOutput.isEmpty() || verifierOutput.get().getText() == null || verifierOutput.get().getText().isBlank()) {
+            return null;
+        }
+
+        try {
+            JsonNode root = objectMapper.readTree(verifierOutput.get().getText());
+            List<Map<String, Object>> factsChecked = parseFactsChecked(root.path("facts_checked"));
+
+            return new VerifierDecision(
+                    root.path("verdict").asText("LOW_CONFID"),
+                    root.path("groundedness_score").asDouble(0.0),
+                    root.path("critical_fact_count").asInt(0),
+                    factsChecked,
+                    root.path("rationale").asText(""),
+                    round
+            );
+        } catch (Exception e) {
+            logger.error("解析 verifier_output 失败: {}", verifierOutput.get().getText(), e);
+            return null;
+        }
+    }
+
+    private void persistVerifierEvaluation(DiagnosisSession session, VerifierDecision decision, int round) {
+        if (decision == null) {
+            return;
+        }
+        Map<String, Object> verifierEvaluation = new LinkedHashMap<>();
+        verifierEvaluation.put("verdict", decision.verdict());
+        verifierEvaluation.put("groundedness_score", decision.groundednessScore());
+        verifierEvaluation.put("critical_fact_count", decision.criticalFactCount());
+        verifierEvaluation.put("facts_checked", decision.factsChecked());
+        verifierEvaluation.put("rationale", decision.rationale());
+        verifierEvaluation.put("round", round);
+        verifierEvaluation.put("traceability_version", "v1");
+        verifierEvaluation.put("tool_trace_summary",
+                Optional.ofNullable(VerifierContextHolder.getToolTraceSummary()).orElse(List.of()));
+
+        String merged = selfEvaluationMergeService.mergeVerifierEvaluation(session.getSelfEvaluation(), verifierEvaluation);
+        session.setSelfEvaluation(merged);
+        diagnosisSessionRepository.save(session);
+    }
+
+    private String buildRetryContext(VerifierDecision decision) {
+        try {
+            List<String> missingFacts = extractEvidenceGaps(decision);
+            Map<String, Object> retryContext = new LinkedHashMap<>();
+            retryContext.put("round", decision.round());
+            retryContext.put("missing_evidence_facts", missingFacts);
+            retryContext.put("instruction", "仅补充以上断言相关证据，不要重复已完成检索");
+            return objectMapper.writeValueAsString(retryContext);
+        } catch (Exception e) {
+            logger.error("构造 retry_context 失败", e);
+            return "{\"round\":1,\"missing_evidence_facts\":[],\"instruction\":\"仅补充缺失证据\"}";
+        }
+    }
+
+    private String buildLowConfidenceOutput(String executorAnswer, VerifierDecision decision) {
+        StringBuilder output = new StringBuilder(LOW_CONFID_DISCLAIMER);
+        output.append("\n\n").append(executorAnswer == null ? "" : executorAnswer);
+
+        List<String> gaps = extractEvidenceGaps(decision);
+        if (!gaps.isEmpty()) {
+            output.append("\n\n当前缺口：");
+            for (String gap : gaps) {
+                output.append("\n- ").append(gap);
+            }
+        }
+        return output.toString();
+    }
+
+    private String buildDegradedOutput(VerifierDecision decision) {
+        StringBuilder output = new StringBuilder(DEGRADED_PREFIX);
+
+        List<String> confirmedFacts = extractConfirmedFacts(decision);
+        List<String> gaps = extractEvidenceGaps(decision);
+        List<String> suggestions = buildNextStepSuggestions(decision);
+
+        output.append("\n\n已确认信息：");
+        if (confirmedFacts.isEmpty()) {
+            output.append("\n- 暂无可稳定确认的信息");
+        } else {
+            for (String fact : confirmedFacts) {
+                output.append("\n- ").append(fact);
+            }
+        }
+
+        output.append("\n\n证据缺口：");
+        if (gaps.isEmpty()) {
+            output.append("\n- 当前缺少足够的直接证据支撑核心结论");
+        } else {
+            for (String gap : gaps) {
+                output.append("\n- ").append(gap);
+            }
+        }
+
+        output.append("\n\n建议下一步：");
+        for (String suggestion : suggestions) {
+            output.append("\n- ").append(suggestion);
+        }
+        return output.toString();
+    }
+
+    private List<String> extractConfirmedFacts(VerifierDecision decision) {
+        List<String> confirmedFacts = new ArrayList<>();
+        for (Map<String, Object> fact : decision.factsChecked()) {
+            String verification = String.valueOf(fact.get("verification"));
+            boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
+            if (critical && ("direct_evidence".equals(verification) || "indirect_support".equals(verification))) {
+                confirmedFacts.add(String.valueOf(fact.get("fact")));
+            }
+        }
+        return confirmedFacts;
+    }
+
+    private List<String> extractEvidenceGaps(VerifierDecision decision) {
+        List<String> gaps = new ArrayList<>();
+        for (Map<String, Object> fact : decision.factsChecked()) {
+            String verification = String.valueOf(fact.get("verification"));
+            boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
+            if (critical && ("no_evidence".equals(verification) || "contradicted".equals(verification))) {
+                gaps.add(String.valueOf(fact.get("fact")) + "：" + String.valueOf(fact.get("detail")));
+            }
+        }
+        if (gaps.isEmpty() && "LOW_CONFID".equals(decision.verdict())) {
+            for (Map<String, Object> fact : decision.factsChecked()) {
+                String verification = String.valueOf(fact.get("verification"));
+                boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
+                if (critical && "indirect_support".equals(verification)) {
+                    gaps.add(String.valueOf(fact.get("fact")) + "：缺少直接证据锚点");
+                }
+            }
+        }
+        return gaps;
+    }
+
+    private List<String> buildNextStepSuggestions(VerifierDecision decision) {
+        List<String> suggestions = new ArrayList<>();
+        List<Map<String, Object>> toolSummary = toolTraceSummaryService.buildVerifierTraceSummary(SessionContextHolder.getSessionId(), null);
+        boolean hasKnowledgeTool = toolSummary.stream().anyMatch(item -> "lookup_knowledge".equals(item.get("tool_name")));
+        boolean hasFailedEvidence = toolSummary.stream().anyMatch(item -> !Boolean.TRUE.equals(item.get("success")));
+
+        if (!hasKnowledgeTool) {
+            suggestions.add("补充知识库或业务文档检索结果，建立可引用的证据锚点");
+        }
+        if (hasFailedEvidence) {
+            suggestions.add("优先重试失败的证据型查询，补齐日志、指标或知识库侧证据");
+        }
+        if (suggestions.isEmpty()) {
+            suggestions.add("围绕上述证据缺口补充只读查询，再由人工复核最终结论");
+        }
+        return suggestions;
+    }
+
+    private record VerifierDecision(
+            String verdict,
+            double groundednessScore,
+            int criticalFactCount,
+            List<Map<String, Object>> factsChecked,
+            String rationale,
+            int round
+    ) {
+    }
+
    /** 从 agent_step 汇总 token、步数等指标回填 diagnosis_session */
    private void backfillSessionMetrics(DiagnosisSession session) {
        try {