Harden evidence trace semantics

2026-07-04 22:36:30 +08:00
parent 246c99b954
commit dc6cd32a67
24 changed files with 1096 additions and 141 deletions
@@ -24,6 +24,7 @@ import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;

 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -85,6 +86,73 @@ class ChatServiceSequentialAgentTest {
        assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
    }

+    @Test
+    void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
+        ChatService chatService = createChatService();
+        ScriptedChatModel chatModel = new ScriptedChatModel("", "");
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析订单支付超时的原因，并给出修复建议",
+                List.of(),
+                "sequential-missing-verifier-session"
+        );
+
+        assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
+        assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
+    }
+
+    @Test
+    void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
+        ChatService chatService = createChatService();
+        ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析订单支付超时的原因，并给出修复建议",
+                List.of(),
+                "sequential-invalid-verifier-session"
+        );
+
+        assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
+        assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
+    }
+
+    @Test
+    void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
+        ChatService chatService = createChatService();
+        ScriptedChatModel chatModel = new ScriptedChatModel("""
+                {
+                  "verdict": "REJECT",
+                  "groundedness_score": 0.0,
+                  "critical_fact_count": 1,
+                  "facts_checked": [
+                    {
+                      "fact": "payment timeout root cause",
+                      "is_critical": true,
+                      "verification": "contradicted",
+                      "detail": "scripted contradiction",
+                      "evidence_refs": []
+                    }
+                  ],
+                  "rationale": "scripted reject"
+                }
+                """);
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析订单支付超时的原因，并给出修复建议",
+                List.of(),
+                "sequential-reject-session"
+        );
+
+        assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
+        assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
+    }
+
    @Test
    void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
        ChatService chatService = createChatService();
@@ -174,7 +242,8 @@ class ChatServiceSequentialAgentTest {
        private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
        private String promptText = "";
        private boolean sawVerifierPrompt;
-        private final String verifierOutput;
+        private final java.util.List<String> verifierOutputs;
+        private int verifierOutputIndex;

        private ScriptedChatModel() {
            this("""
@@ -197,7 +266,11 @@ class ChatServiceSequentialAgentTest {
        }

        private ScriptedChatModel(String verifierOutput) {
-            this.verifierOutput = verifierOutput;
+            this.verifierOutputs = java.util.List.of(verifierOutput);
+        }
+
+        private ScriptedChatModel(String... verifierOutputs) {
+            this.verifierOutputs = java.util.List.of(verifierOutputs);
        }

        @Override
@@ -213,7 +286,9 @@ class ChatServiceSequentialAgentTest {
            } else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
                agentCalls.add("chat_verifier");
                sawVerifierPrompt = true;
-                text = verifierOutput;
+                int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
+                text = verifierOutputs.get(index);
+                verifierOutputIndex++;
            } else {
                text = "UNEXPECTED_PROMPT";
            }
@@ -0,0 +1,96 @@
+package com.superbiz.agent.service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.superbiz.agent.domain.entity.ToolInvocation;
+import com.superbiz.agent.repository.ToolInvocationRepository;
+import com.superbiz.agent.util.SessionContextHolder;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class ToolInvocationRecorderTest {
+
+    @Test
+    void recordEvidenceToolPreservesNoEvidenceSemantics() {
+        ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
+        when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
+        ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
+        SessionContextHolder.setSessionId("recorder-test-session");
+
+        try {
+            recorder.recordEvidenceTool(
+                    "query_logs",
+                    Map.of("query", "timeout"),
+                    "{\"success\":false,\"message\":\"未找到匹配的日志\"}",
+                    true,
+                    System.currentTimeMillis() - 10,
+                    null,
+                    "application-logs",
+                    ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
+                    Map.of("log_topic", "application-logs")
+            );
+        } finally {
+            SessionContextHolder.clear();
+        }
+
+        ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
+        verify(repository).save(captor.capture());
+        ToolInvocation saved = captor.getValue();
+
+        assertEquals("query_logs", saved.getToolName());
+        assertEquals(Boolean.TRUE, saved.getSuccess());
+        assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"no_evidence\""));
+        assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
+    }
+
+    @Test
+    void recordLookupKnowledgePreservesRetrievalSpecificFields() {
+        ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
+        when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
+        ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
+        SessionContextHolder.setSessionId("lookup-recorder-session");
+
+        ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
+                .query("ERR_TIMEOUT")
+                .outputPreview("matched payment doc")
+                .outputLength(18)
+                .retrievalLayer("L0")
+                .l0MatchCount(1)
+                .l1MatchCount(null)
+                .truncated(false)
+                .relevanceLevel("PRECISE")
+                .completenessHint("already precise")
+                .domain("payment")
+                .dedupReason("doc_retrieved")
+                .durationMs(42)
+                .success(true)
+                .evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED)
+                .l0Titles(List.of("payment/errors.md"))
+                .build();
+
+        try {
+            recorder.recordLookupKnowledge(record);
+        } finally {
+            SessionContextHolder.clear();
+        }
+
+        ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
+        verify(repository).save(captor.capture());
+        ToolInvocation saved = captor.getValue();
+
+        assertEquals("lookup_knowledge", saved.getToolName());
+        assertEquals("PRECISE", saved.getRelevanceLevel());
+        assertEquals("doc_retrieved", saved.getDedupReason());
+        assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"deduped\""));
+        assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"payment\"]"));
+    }
+}
@@ -0,0 +1,75 @@
+package com.superbiz.agent.service;
+
+import com.superbiz.agent.domain.entity.ToolInvocation;
+import com.superbiz.agent.repository.ToolInvocationRepository;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ToolTraceSummaryServiceTest {
+
+    @Test
+    void buildVerifierTraceSummaryTreatsNoEvidenceAsGapWithoutLosingSuccessfulEvidence() {
+        ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
+        when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
+                ToolInvocation.builder()
+                        .id(1L)
+                        .sessionId("session-1")
+                        .toolName("query_logs")
+                        .inputParams("{\"query\":\"timeout\"}")
+                        .outputPreview("payment timeout stack trace")
+                        .retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
+                        .success(true)
+                        .build(),
+                ToolInvocation.builder()
+                        .id(2L)
+                        .sessionId("session-1")
+                        .toolName("query_logs")
+                        .inputParams("{\"query\":\"timeout\"}")
+                        .outputPreview("{\"success\":false,\"message\":\"未找到匹配的日志\"}")
+                        .retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"no_evidence\"}")
+                        .success(true)
+                        .build(),
+                ToolInvocation.builder()
+                        .id(3L)
+                        .sessionId("session-1")
+                        .toolName("query_metrics")
+                        .inputParams("{\"query\":\"active_prometheus_alerts\"}")
+                        .errorMessage("prometheus timeout")
+                        .retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"failed\"}")
+                        .success(false)
+                        .build()
+        ));
+
+        ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
+
+        List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-1", "application-logs point to timeout");
+
+        assertEquals(2, summaries.size());
+
+        Map<String, Object> logsSummary = summaries.stream()
+                .filter(item -> "query_logs".equals(item.get("tool_name")))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(Boolean.TRUE, logsSummary.get("success"));
+        assertEquals("direct", logsSummary.get("evidence_level"));
+        assertEquals(2, logsSummary.get("invocation_count"));
+        assertEquals(1, logsSummary.get("no_hit_invocation_count"));
+
+        Map<String, Object> metricsSummary = summaries.stream()
+                .filter(item -> "query_metrics".equals(item.get("tool_name")))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(Boolean.FALSE, metricsSummary.get("success"));
+        assertEquals("none", metricsSummary.get("evidence_level"));
+        assertEquals(1, metricsSummary.get("failed_invocation_count"));
+        assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
+    }
+}
@@ -3,7 +3,9 @@ package com.superbiz.agent.tool;
 import com.superbiz.agent.dto.KnowledgeEntry;
 import com.superbiz.agent.dto.LookupResult;
 import com.superbiz.agent.service.KnowledgeIndexService;
+import com.superbiz.agent.service.ToolInvocationRecorder;
 import com.superbiz.agent.service.VectorSearchService;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.InjectMocks;
@@ -28,6 +30,15 @@ class LookupKnowledgeToolTest {
    @Mock
    private VectorSearchService vectorSearchService;

+    @Mock
+    private ToolInvocationRecorder toolInvocationRecorder;
+
+    @Mock
+    private RetrievedDocTracker retrievedDocTracker;
+
+    @Mock
+    private ObjectMapper objectMapper;
+
    @InjectMocks
    private LookupKnowledgeTool tool;