package com.superbiz.agent.service; import com.alibaba.cloud.ai.graph.agent.ReactAgent; import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry; import com.alibaba.cloud.ai.graph.skills.registry.classpath.ClasspathSkillRegistry; import com.superbiz.agent.agent.tool.DateTimeTools; import com.superbiz.agent.agent.tool.QueryLogsTools; import com.superbiz.agent.agent.tool.QueryMetricsTools; import com.superbiz.agent.domain.entity.AgentStep; import com.superbiz.agent.domain.entity.DiagnosisSession; import com.superbiz.agent.domain.entity.ToolInvocation; import com.superbiz.agent.repository.AgentStepRepository; import com.superbiz.agent.repository.DiagnosisSessionRepository; import com.superbiz.agent.repository.ToolInvocationRepository; import com.superbiz.agent.tool.LookupKnowledgeTool; import com.superbiz.agent.tool.RetrievedDocTracker; import org.junit.jupiter.api.Test; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.tool.ToolCallback; import org.springframework.test.util.ReflectionTestUtils; import org.mockito.ArgumentCaptor; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; class ChatServiceSequentialAgentTest { @Test void executeChatComplexInvokesSequentialWorkflow() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-test-session" ); assertTrue(result.answer().contains("连接池 active 达到上限")); assertFalse(result.answer().contains("\"answer_version\"")); assertEquals("sequential-test-session", result.sessionId()); assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls); assertTrue(chatModel.sawVerifierPrompt); } @Test void executeChatComplexDoesNotRetryLowConfidenceByDefault() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "LOW_CONFID", "groundedness_score": 0.1, "critical_fact_count": 1, "facts_checked": [ { "fact": "missing direct evidence", "is_critical": true, "verification": "no_evidence", "detail": "scripted evidence gap", "evidence_refs": [] } ], "rationale": "scripted low confidence" } """); chatModel.composerOutput = "not-json"; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-low-confidence-session" ); assertTrue(result.answer().startsWith("以下结论基于当前已获取证据")); assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER")); assertTrue(result.answer().contains("当前缺口")); assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls); } @Test void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "LOW_CONFID", "groundedness_score": 0.37, "critical_fact_count": 3, "facts_checked": [ { "fact": "连接池耗尽 active=50/50", "is_critical": true, "verification": "direct_evidence", "detail": "log evidence", "evidence_refs": [] }, { "fact": "临时扩容连接池到 80", "is_critical": true, "verification": "indirect_support", "detail": "suggestion inferred from evidence", "evidence_refs": [] }, { "fact": "OOM 导致连接泄漏", "is_critical": true, "verification": "no_evidence", "detail": "missing OOM log", "evidence_refs": [] } ], "rationale": "scripted low confidence" } """); chatModel.composerOutput = "not-json"; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-low-confid-direct-only-session" ); assertTrue(result.answer().contains("已确认信息:\n- 连接池耗尽 active=50/50")); assertTrue(result.answer().contains("80")); assertTrue(result.answer().contains("suggestion inferred from evidence")); assertTrue(result.answer().contains("missing OOM log")); assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER")); } @Test void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel("", ""); ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-missing-verifier-session" ); assertTrue(result.answer().startsWith("以下结论基于当前已获取证据")); assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls); } @Test void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel("not-json"); ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-invalid-verifier-session" ); assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls); } @Test void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "REJECT", "groundedness_score": 0.0, "critical_fact_count": 1, "facts_checked": [ { "fact": "payment timeout root cause", "is_critical": true, "verification": "contradicted", "detail": "scripted contradiction", "evidence_refs": [] } ], "rationale": "scripted reject" } """); chatModel.composerOutput = "not-json"; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-reject-session" ); assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论")); assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER")); } @Test void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析订单支付超时的原因,并给出修复建议", List.of(), "sequential-workflow-session" ); assertTrue(result.answer().contains("连接池 active 达到上限")); assertFalse(result.answer().contains("\"answer_version\"")); assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls); assertTrue(chatModel.sawVerifierPrompt); } @Test void verifierReceivesStructuredExecutorPayloadFields() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); chatModel.executorOutput = """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "source_id": "trace-1", "tool_name": "query_metrics", "source_invocation_id": 101, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [], "recommended_actions": [], "missing_info": [] } """; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-structured-executor-session" ); assertTrue(result.answer().contains("连接池 active 达到上限")); assertFalse(result.answer().contains("\"answer_version\"")); assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\"")); assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\"")); assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\"")); assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限")); } @Test void executeChatComplexRendersExecutorEvidenceV2InsteadOfRawJsonOnPass() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); chatModel.composerOutput = "not-json"; chatModel.executorOutput = """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "source_id": "trace-1", "tool_name": "query_metrics", "source_invocation_id": 101, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [ { "hypothesis_text": "连接泄漏可能参与了连接池耗尽", "basis": "已有连接池满载证据,但缺少泄漏检测日志", "needed_evidence": ["连接泄漏检测日志"] } ], "recommended_actions": [ { "action_text": "补充查询连接池泄漏检测日志", "reason": "用于确认是否存在连接未释放" } ], "missing_info": ["缺少连接泄漏检测日志"] } """; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-v2-render-session" ); assertTrue(result.answer().contains("已确认信息")); assertTrue(result.answer().contains("连接池 active 达到上限")); assertTrue(result.answer().contains("建议下一步")); assertFalse(result.answer().contains("\"answer_version\"")); assertFalse(result.answer().contains("executor_evidence_v2")); } @Test void executeChatComplexPersistsGatekeeperResultInVerifierEvaluation() throws Exception { ChatService chatService = createChatService(); SelfEvaluationMergeService mergeService = (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService"); ToolInvocationRepository invocationRepository = (ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository"); when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-persist-session")) .thenReturn(List.of(ToolInvocation.builder() .id(101L) .sessionId("sequential-gatekeeper-persist-session") .toolName("query_metrics") .retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50")) .build())); ScriptedChatModel chatModel = new ScriptedChatModel(); chatModel.executorOutput = """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "source_id": "trace-1", "tool_name": "query_metrics", "source_invocation_id": 101, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [], "recommended_actions": [], "missing_info": [] } """; chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-gatekeeper-persist-session" ); ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture()); Map verifierEvaluation = captor.getValue(); assertTrue(verifierEvaluation.containsKey("gatekeeper_result")); assertTrue(verifierEvaluation.containsKey("prompt_audit")); @SuppressWarnings("unchecked") Map gatekeeperResult = (Map) verifierEvaluation.get("gatekeeper_result"); assertEquals("pass", gatekeeperResult.get("status")); assertEquals("none", gatekeeperResult.get("severity")); @SuppressWarnings("unchecked") Map promptAudit = (Map) verifierEvaluation.get("prompt_audit"); assertEquals("chat-prompts-v1", promptAudit.get("version")); @SuppressWarnings("unchecked") List> prompts = (List>) promptAudit.get("prompts"); assertEquals(4, prompts.size()); assertTrue(prompts.stream().anyMatch(prompt -> "chat_executor".equals(prompt.get("name")) && "chat-executor-v2".equals(prompt.get("version")))); } @Test void executeChatComplexMapsClaimChecksToFactsCheckedAndPersistsBoth() throws Exception { ChatService chatService = createChatService(); SelfEvaluationMergeService mergeService = (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService"); ToolInvocationRepository invocationRepository = (ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository"); when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-claim-check-session")) .thenReturn(List.of(ToolInvocation.builder() .id(101L) .sessionId("sequential-claim-check-session") .toolName("query_metrics") .retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50")) .build())); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "LOW_CONFID", "groundedness_score": 0.32, "critical_fact_count": 6, "claim_checks": [ {"claim_id":"claim-1","claim_text":"CPU 使用率 92%","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[{"trace_ref":"trace-1","tool_name":"query_metrics","source_invocation_ids":[101],"note":"cpu"}]}, {"claim_id":"claim-2","claim_text":"CPU 过高可能导致超时","claim_type":"risk","verification":"reasonable_inference","detail":"inference","evidence_refs":[]}, {"claim_id":"claim-3","claim_text":"CPU 是唯一根因","claim_type":"root_cause","verification":"overstated","detail":"too strong","evidence_refs":[]}, {"claim_id":"claim-4","claim_text":"缺少线程池证据","claim_type":"symptom","verification":"unsupported","detail":"missing","evidence_refs":[]}, {"claim_id":"claim-5","claim_text":"出现证据外错误码 ERR_FAKE","claim_type":"symptom","verification":"external_unknown","detail":"external","evidence_refs":[]}, {"claim_id":"claim-6","claim_text":"证据显示 CPU 很低","claim_type":"symptom","verification":"contradicted","detail":"conflict","evidence_refs":[]} ], "facts_checked": [], "rationale": "claim checks drive compatibility" } """); chatModel.executorOutput = validExecutorV2Output(); chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-claim-check-session" ); ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture()); Map verifierEvaluation = captor.getValue(); @SuppressWarnings("unchecked") List> claimChecks = (List>) verifierEvaluation.get("claim_checks"); @SuppressWarnings("unchecked") List> factsChecked = (List>) verifierEvaluation.get("facts_checked"); assertEquals(6, claimChecks.size()); assertEquals(6, factsChecked.size()); assertEquals("direct_evidence", factsChecked.get(0).get("verification")); assertEquals("indirect_support", factsChecked.get(1).get("verification")); assertEquals("indirect_support", factsChecked.get(2).get("verification")); assertEquals("no_evidence", factsChecked.get(3).get("verification")); assertEquals("no_evidence", factsChecked.get(4).get("verification")); assertEquals("contradicted", factsChecked.get(5).get("verification")); assertTrue(String.valueOf(factsChecked.get(0).get("fact")).startsWith("claim-1:")); } @Test void executeChatComplexDowngradesPassToRejectWhenGatekeeperInvocationRefFails() throws Exception { ChatService chatService = createChatService(); SelfEvaluationMergeService mergeService = (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService"); ToolInvocationRepository invocationRepository = (ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository"); when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-fail-session")) .thenReturn(List.of(ToolInvocation.builder() .id(101L) .sessionId("sequential-gatekeeper-fail-session") .toolName("query_metrics") .retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50")) .build())); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "PASS", "groundedness_score": 1.0, "critical_fact_count": 1, "claim_checks": [ {"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[]} ], "facts_checked": [], "rationale": "model tried pass" } """); chatModel.composerOutput = "not-json"; chatModel.executorOutput = """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "tool_name": "query_metrics", "source_invocation_id": 999, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [], "recommended_actions": [], "missing_info": [] } """; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-gatekeeper-fail-session" ); assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论")); ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture()); assertEquals("REJECT", captor.getValue().get("verdict")); } @Test void executeChatComplexDowngradesPassToLowConfidenceWhenExecutorOutputMalformed() throws Exception { ChatService chatService = createChatService(); SelfEvaluationMergeService mergeService = (SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService"); ScriptedChatModel chatModel = new ScriptedChatModel(""" { "verdict": "PASS", "groundedness_score": 1.0, "critical_fact_count": 0, "claim_checks": [], "facts_checked": [], "rationale": "model tried pass" } """); chatModel.composerOutput = "not-json"; chatModel.executorOutput = "{ not-json"; ChatService.ChatResult result = chatService.executeChatComplex( chatModel, new ToolCallback[0], "请分析 MySQL 连接池耗尽", List.of(), "sequential-malformed-pass-session" ); assertTrue(result.answer().startsWith("以下结论基于当前已获取证据")); ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture()); assertEquals("LOW_CONFID", captor.getValue().get("verdict")); } @Test void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() { ChatService chatService = new ChatService(); DateTimeTools dateTimeTools = new DateTimeTools(); LookupKnowledgeTool lookupKnowledgeTool = new LookupKnowledgeTool(); QueryLogsTools queryLogsTools = new QueryLogsTools(mock(ToolInvocationRecorder.class)); QueryMetricsTools queryMetricsTools = new QueryMetricsTools(mock(ToolInvocationRecorder.class)); ReflectionTestUtils.setField(chatService, "dateTimeTools", dateTimeTools); ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", lookupKnowledgeTool); ReflectionTestUtils.setField(chatService, "queryLogsTools", queryLogsTools); ReflectionTestUtils.setField(chatService, "queryMetricsTools", queryMetricsTools); Object[] methodTools = chatService.buildMethodToolsArray(); assertEquals(4, methodTools.length); assertSame(dateTimeTools, methodTools[0]); assertSame(lookupKnowledgeTool, methodTools[1]); assertSame(queryLogsTools, methodTools[2]); assertSame(queryMetricsTools, methodTools[3]); } @Test void createReactAgentInjectsSkillCatalogThroughAlibabaHook() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); SkillRegistry skillRegistry = ClasspathSkillRegistry.builder() .classpathPath("skills") .basePath("target/test-skills-cache") .build(); ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry); ReactAgent agent = chatService.createReactAgent(chatModel, "BASE_TEST_PROMPT"); agent.call("diagnose mysql connection pool exhaustion"); assertTrue(chatModel.promptText.contains("BASE_TEST_PROMPT")); assertTrue(chatModel.promptText.contains("## Skills System")); assertTrue(chatModel.promptText.contains("diagnose-mysql-connection-pool")); assertTrue(chatModel.promptText.contains("read_skill")); } @Test void plannerGetsSkillMetadataAndExecutorGetsReadSkillTool() throws Exception { ChatService chatService = createChatService(); ScriptedChatModel chatModel = new ScriptedChatModel(); SkillRegistry skillRegistry = ClasspathSkillRegistry.builder() .classpathPath("skills") .basePath("target/test-skills-cache") .build(); ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry); chatService.executeChatComplex( chatModel, new ToolCallback[0], "diagnose mysql connection pool exhaustion", List.of(), "planner-skill-metadata-session" ); assertTrue(chatModel.plannerPromptText.contains("\"skill_catalog\"")); assertTrue(chatModel.plannerPromptText.contains("diagnose-mysql-connection-pool")); assertTrue(chatModel.plannerPromptText.contains("\"selected_skill\"")); assertFalse(chatModel.plannerPromptText.contains("## Skills System")); assertFalse(chatModel.plannerPromptText.contains("read_skill")); assertTrue(chatModel.executorPromptText.contains("## Skills System")); assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool")); assertTrue(chatModel.executorPromptText.contains("read_skill")); assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill")); assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool")); assertFalse(chatModel.verifierPromptText.contains("read_skill")); } private ChatService createChatService() { ChatService chatService = new ChatService(); DiagnosisSessionRepository diagnosisSessionRepository = mock(DiagnosisSessionRepository.class); when(diagnosisSessionRepository.findBySessionId(anyString())).thenReturn(Optional.empty()); when(diagnosisSessionRepository.save(any(DiagnosisSession.class))).thenAnswer(invocation -> invocation.getArgument(0)); AtomicInteger stepId = new AtomicInteger(1); AgentStepRepository agentStepRepository = mock(AgentStepRepository.class); when(agentStepRepository.save(any(AgentStep.class))).thenAnswer(invocation -> { AgentStep step = invocation.getArgument(0); if (step.getId() == null) { step.setId((long) stepId.getAndIncrement()); } return step; }); when(agentStepRepository.findById(any())).thenReturn(Optional.of(new AgentStep())); when(agentStepRepository.findBySessionIdOrderByStepIndex(anyString())).thenReturn(List.of()); ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class); when(toolInvocationRepository.countBySessionId(anyString())).thenReturn(0L); when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder() .id(101L) .toolName("query_metrics") .retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50")) .build())); EvaluationService evaluationService = mock(EvaluationService.class); RetrievedDocTracker retrievedDocTracker = mock(RetrievedDocTracker.class); KnowledgeDomainService knowledgeDomainService = mock(KnowledgeDomainService.class); when(knowledgeDomainService.buildKnowledgeMap()).thenReturn(""); ToolTraceSummaryService toolTraceSummaryService = mock(ToolTraceSummaryService.class); when(toolTraceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of()); SelfEvaluationMergeService selfEvaluationMergeService = mock(SelfEvaluationMergeService.class); when(selfEvaluationMergeService.mergeVerifierEvaluation(any(), any())).thenReturn("{}"); ExecutorGatekeeperService executorGatekeeperService = new ExecutorGatekeeperService(toolInvocationRepository); ReflectionTestUtils.setField(chatService, "dateTimeTools", new DateTimeTools()); ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", new LookupKnowledgeTool()); ReflectionTestUtils.setField(chatService, "queryLogsTools", new QueryLogsTools(mock(ToolInvocationRecorder.class))); ReflectionTestUtils.setField(chatService, "diagnosisSessionRepository", diagnosisSessionRepository); ReflectionTestUtils.setField(chatService, "agentStepRepository", agentStepRepository); ReflectionTestUtils.setField(chatService, "toolInvocationRepository", toolInvocationRepository); ReflectionTestUtils.setField(chatService, "evaluationService", evaluationService); ReflectionTestUtils.setField(chatService, "retrievedDocTracker", retrievedDocTracker); ReflectionTestUtils.setField(chatService, "knowledgeDomainService", knowledgeDomainService); ReflectionTestUtils.setField(chatService, "toolTraceSummaryService", toolTraceSummaryService); ReflectionTestUtils.setField(chatService, "selfEvaluationMergeService", selfEvaluationMergeService); ReflectionTestUtils.setField(chatService, "executorGatekeeperService", executorGatekeeperService); ReflectionTestUtils.setField(chatService, "verifierLowConfidenceThreshold", 0.5d); ReflectionTestUtils.setField(chatService, "chatPlannerPrompt", "PLANNER_TEST_PROMPT"); ReflectionTestUtils.setField(chatService, "chatExecutorPrompt", "EXECUTOR_TEST_PROMPT"); ReflectionTestUtils.setField(chatService, "chatVerifierPrompt", "VERIFIER_TEST_PROMPT"); ReflectionTestUtils.setField(chatService, "chatComposerPrompt", "COMPOSER_TEST_PROMPT"); return chatService; } private String validExecutorV2Output() { return """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "source_id": "trace-1", "tool_name": "query_metrics", "source_invocation_id": 101, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [], "recommended_actions": [], "missing_info": [] } """; } private String evidenceRefs(String rawPath, String text) { return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}"; } private static final class ScriptedChatModel implements ChatModel { private final java.util.ArrayList agentCalls = new java.util.ArrayList<>(); private String promptText = ""; private String plannerPromptText = ""; private String executorPromptText = ""; private String verifierPromptText = ""; private String composerPromptText = ""; private String executorOutput = """ { "answer_version": "executor_evidence_v2", "claims": [ { "claim_id": "claim-1", "claim_type": "symptom", "claim_text": "连接池 active 达到上限", "support_level": "direct", "evidence_bindings": [ { "source_type": "tool_trace", "source_id": "trace-1", "tool_name": "query_metrics", "source_invocation_id": 101, "raw_path": "$.alerts[0]", "evidence_excerpt": "active=50 max=50" } ] } ], "hypotheses": [], "recommended_actions": [], "missing_info": [] } """; private String composerOutput = """ { "answer_summary": "已确认连接池 active 达到上限。", "recommended_actions": [ { "action_text": "补充查询连接池泄漏检测日志", "reason": "用于确认是否存在连接未释放" } ], "user_facing_answer": "已确认连接池 active 达到上限。建议补充查询连接池泄漏检测日志。" } """; private boolean sawVerifierPrompt; private final java.util.List verifierOutputs; private int verifierOutputIndex; private ScriptedChatModel() { this(""" { "verdict": "PASS", "groundedness_score": 1.0, "critical_fact_count": 1, "claim_checks": [ {"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"covered by scripted verifier","evidence_refs":[]} ], "facts_checked": [], "rationale": "scripted pass" } """); } private ScriptedChatModel(String verifierOutput) { this.verifierOutputs = java.util.List.of(verifierOutput); } private ScriptedChatModel(String... verifierOutputs) { this.verifierOutputs = java.util.List.of(verifierOutputs); } @Override public ChatResponse call(Prompt prompt) { promptText = prompt.getContents(); String text; if (promptText.contains("PLANNER_TEST_PROMPT")) { agentCalls.add("chat_planner"); plannerPromptText = promptText; text = "PLANNER_PLAN"; } else if (promptText.contains("EXECUTOR_TEST_PROMPT")) { agentCalls.add("chat_executor"); executorPromptText = promptText; text = executorOutput; } else if (promptText.contains("VERIFIER_TEST_PROMPT")) { agentCalls.add("chat_verifier"); verifierPromptText = promptText; sawVerifierPrompt = true; int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1); text = verifierOutputs.get(index); verifierOutputIndex++; } else if (promptText.contains("COMPOSER_TEST_PROMPT")) { agentCalls.add("chat_composer"); composerPromptText = promptText; text = composerOutput; } else { text = "UNEXPECTED_PROMPT"; } return new ChatResponse(List.of(new Generation(new AssistantMessage(text)))); } } }