feat(agent): add verifier claim checks

2026-07-08 02:33:02 +08:00
parent c5e496e715
commit 1b31e78be5
19 changed files with 1247 additions and 115 deletions
@@ -55,7 +55,8 @@ class ChatServiceSequentialAgentTest {
                "sequential-test-session"
        );

-        assertEquals("EXECUTOR_FINAL_ANSWER", result.answer());
+        assertTrue(result.answer().contains("连接池 active 达到上限"));
+        assertFalse(result.answer().contains("\"answer_version\""));
        assertEquals("sequential-test-session", result.sessionId());
        assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
        assertTrue(chatModel.sawVerifierPrompt);
@@ -225,7 +226,8 @@ class ChatServiceSequentialAgentTest {
                "sequential-workflow-session"
        );

-        assertEquals("EXECUTOR_FINAL_ANSWER", result.answer());
+        assertTrue(result.answer().contains("连接池 active 达到上限"));
+        assertFalse(result.answer().contains("\"answer_version\""));
        assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
        assertTrue(chatModel.sawVerifierPrompt);
    }
@@ -236,8 +238,7 @@ class ChatServiceSequentialAgentTest {
        ScriptedChatModel chatModel = new ScriptedChatModel();
        chatModel.executorOutput = """
                {
-                  "answer_version": "executor_evidence_v1",
-                  "diagnosis_summary": "已确认连接池 active 达到上限。",
+                  "answer_version": "executor_evidence_v2",
                  "claims": [
                    {
                      "claim_id": "claim-1",
@@ -257,8 +258,7 @@ class ChatServiceSequentialAgentTest {
                  ],
                  "hypotheses": [],
                  "recommended_actions": [],
-                  "missing_info": [],
-                  "user_facing_answer": "已确认连接池 active 达到上限。"
+                  "missing_info": []
                }
                """;

@@ -270,7 +270,8 @@ class ChatServiceSequentialAgentTest {
                "sequential-structured-executor-session"
        );

-        assertEquals("已确认连接池 active 达到上限。", result.answer());
+        assertTrue(result.answer().contains("连接池 active 达到上限"));
+        assertFalse(result.answer().contains("\"answer_version\""));
        assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\""));
        assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\""));
        assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\""));
@@ -391,6 +392,160 @@ class ChatServiceSequentialAgentTest {
        assertEquals("pass", gatekeeperResult.get("status"));
    }

+    @Test
+    void executeChatComplexMapsClaimChecksToFactsCheckedAndPersistsBoth() throws Exception {
+        ChatService chatService = createChatService();
+        SelfEvaluationMergeService mergeService =
+                (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
+        ToolInvocationRepository invocationRepository =
+                (ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
+        when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-claim-check-session"))
+                .thenReturn(List.of(ToolInvocation.builder()
+                        .id(101L)
+                        .sessionId("sequential-claim-check-session")
+                        .toolName("query_metrics")
+                        .build()));
+        ScriptedChatModel chatModel = new ScriptedChatModel("""
+                {
+                  "verdict": "LOW_CONFID",
+                  "groundedness_score": 0.32,
+                  "critical_fact_count": 6,
+                  "claim_checks": [
+                    {"claim_id":"claim-1","claim_text":"CPU 使用率 92%","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[{"trace_ref":"trace-1","tool_name":"query_metrics","source_invocation_ids":[101],"note":"cpu"}]},
+                    {"claim_id":"claim-2","claim_text":"CPU 过高可能导致超时","claim_type":"risk","verification":"reasonable_inference","detail":"inference","evidence_refs":[]},
+                    {"claim_id":"claim-3","claim_text":"CPU 是唯一根因","claim_type":"root_cause","verification":"overstated","detail":"too strong","evidence_refs":[]},
+                    {"claim_id":"claim-4","claim_text":"缺少线程池证据","claim_type":"symptom","verification":"unsupported","detail":"missing","evidence_refs":[]},
+                    {"claim_id":"claim-5","claim_text":"出现证据外错误码 ERR_FAKE","claim_type":"symptom","verification":"external_unknown","detail":"external","evidence_refs":[]},
+                    {"claim_id":"claim-6","claim_text":"证据显示 CPU 很低","claim_type":"symptom","verification":"contradicted","detail":"conflict","evidence_refs":[]}
+                  ],
+                  "facts_checked": [],
+                  "rationale": "claim checks drive compatibility"
+                }
+                """);
+        chatModel.executorOutput = validExecutorV2Output();
+
+        chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析 MySQL 连接池耗尽",
+                List.of(),
+                "sequential-claim-check-session"
+        );
+
+        ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
+        verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
+        Map<String, Object> verifierEvaluation = captor.getValue();
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> claimChecks = (List<Map<String, Object>>) verifierEvaluation.get("claim_checks");
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> factsChecked = (List<Map<String, Object>>) verifierEvaluation.get("facts_checked");
+
+        assertEquals(6, claimChecks.size());
+        assertEquals(6, factsChecked.size());
+        assertEquals("direct_evidence", factsChecked.get(0).get("verification"));
+        assertEquals("indirect_support", factsChecked.get(1).get("verification"));
+        assertEquals("indirect_support", factsChecked.get(2).get("verification"));
+        assertEquals("no_evidence", factsChecked.get(3).get("verification"));
+        assertEquals("no_evidence", factsChecked.get(4).get("verification"));
+        assertEquals("contradicted", factsChecked.get(5).get("verification"));
+        assertTrue(String.valueOf(factsChecked.get(0).get("fact")).startsWith("claim-1:"));
+    }
+
+    @Test
+    void executeChatComplexDowngradesPassToRejectWhenGatekeeperInvocationRefFails() throws Exception {
+        ChatService chatService = createChatService();
+        SelfEvaluationMergeService mergeService =
+                (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
+        ToolInvocationRepository invocationRepository =
+                (ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
+        when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-fail-session"))
+                .thenReturn(List.of(ToolInvocation.builder()
+                        .id(101L)
+                        .sessionId("sequential-gatekeeper-fail-session")
+                        .toolName("query_metrics")
+                        .build()));
+        ScriptedChatModel chatModel = new ScriptedChatModel("""
+                {
+                  "verdict": "PASS",
+                  "groundedness_score": 1.0,
+                  "critical_fact_count": 1,
+                  "claim_checks": [
+                    {"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[]}
+                  ],
+                  "facts_checked": [],
+                  "rationale": "model tried pass"
+                }
+                """);
+        chatModel.executorOutput = """
+                {
+                  "answer_version": "executor_evidence_v2",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [999],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": []
+                }
+                """;
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析 MySQL 连接池耗尽",
+                List.of(),
+                "sequential-gatekeeper-fail-session"
+        );
+
+        assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
+        ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
+        verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
+        assertEquals("REJECT", captor.getValue().get("verdict"));
+    }
+
+    @Test
+    void executeChatComplexDowngradesPassToLowConfidenceWhenExecutorOutputMalformed() throws Exception {
+        ChatService chatService = createChatService();
+        SelfEvaluationMergeService mergeService =
+                (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
+        ScriptedChatModel chatModel = new ScriptedChatModel("""
+                {
+                  "verdict": "PASS",
+                  "groundedness_score": 1.0,
+                  "critical_fact_count": 0,
+                  "claim_checks": [],
+                  "facts_checked": [],
+                  "rationale": "model tried pass"
+                }
+                """);
+        chatModel.executorOutput = "{ not-json";
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析 MySQL 连接池耗尽",
+                List.of(),
+                "sequential-malformed-pass-session"
+        );
+
+        assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
+        ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
+        verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
+        assertEquals("LOW_CONFID", captor.getValue().get("verdict"));
+    }
+
    @Test
    void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
        ChatService chatService = new ChatService();
@@ -484,6 +639,10 @@ class ChatServiceSequentialAgentTest {
        when(agentStepRepository.findBySessionIdOrderByStepIndex(anyString())).thenReturn(List.of());
        ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
        when(toolInvocationRepository.countBySessionId(anyString())).thenReturn(0L);
+        when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
+                .id(101L)
+                .toolName("query_metrics")
+                .build()));

        EvaluationService evaluationService = mock(EvaluationService.class);
        RetrievedDocTracker retrievedDocTracker = mock(RetrievedDocTracker.class);
@@ -514,13 +673,65 @@ class ChatServiceSequentialAgentTest {
        return chatService;
    }

+    private String validExecutorV2Output() {
+        return """
+                {
+                  "answer_version": "executor_evidence_v2",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "source_id": "trace-1",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [101],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": []
+                }
+                """;
+    }
+
    private static final class ScriptedChatModel implements ChatModel {
        private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
        private String promptText = "";
        private String plannerPromptText = "";
        private String executorPromptText = "";
        private String verifierPromptText = "";
-        private String executorOutput = "EXECUTOR_FINAL_ANSWER";
+        private String executorOutput = """
+                {
+                  "answer_version": "executor_evidence_v2",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "source_id": "trace-1",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [101],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": []
+                }
+                """;
        private boolean sawVerifierPrompt;
        private final java.util.List<String> verifierOutputs;
        private int verifierOutputIndex;
@@ -531,15 +742,10 @@ class ChatServiceSequentialAgentTest {
                      "verdict": "PASS",
                      "groundedness_score": 1.0,
                      "critical_fact_count": 1,
-                      "facts_checked": [
-                        {
-                          "fact": "executor answer generated",
-                          "is_critical": true,
-                          "verification": "direct_evidence",
-                          "detail": "covered by scripted verifier",
-                          "evidence_refs": []
-                        }
+                      "claim_checks": [
+                        {"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"covered by scripted verifier","evidence_refs":[]}
                      ],
+                      "facts_checked": [],
                      "rationale": "scripted pass"
                    }
                    """);