836 lines
40 KiB
Java
836 lines
40 KiB
Java
package com.superbiz.agent.service;
|
|
|
|
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
|
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
|
|
import com.alibaba.cloud.ai.graph.skills.registry.classpath.ClasspathSkillRegistry;
|
|
import com.superbiz.agent.agent.tool.DateTimeTools;
|
|
import com.superbiz.agent.agent.tool.QueryLogsTools;
|
|
import com.superbiz.agent.agent.tool.QueryMetricsTools;
|
|
import com.superbiz.agent.domain.entity.AgentStep;
|
|
import com.superbiz.agent.domain.entity.DiagnosisSession;
|
|
import com.superbiz.agent.domain.entity.ToolInvocation;
|
|
import com.superbiz.agent.repository.AgentStepRepository;
|
|
import com.superbiz.agent.repository.DiagnosisSessionRepository;
|
|
import com.superbiz.agent.repository.ToolInvocationRepository;
|
|
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
|
import com.superbiz.agent.tool.RetrievedDocTracker;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.springframework.ai.chat.messages.AssistantMessage;
|
|
import org.springframework.ai.chat.model.ChatModel;
|
|
import org.springframework.ai.chat.model.ChatResponse;
|
|
import org.springframework.ai.chat.model.Generation;
|
|
import org.springframework.ai.chat.prompt.Prompt;
|
|
import org.springframework.ai.tool.ToolCallback;
|
|
import org.springframework.test.util.ReflectionTestUtils;
|
|
import org.mockito.ArgumentCaptor;
|
|
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Optional;
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
import static org.junit.jupiter.api.Assertions.assertSame;
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
import static org.mockito.ArgumentMatchers.any;
|
|
import static org.mockito.ArgumentMatchers.anyString;
|
|
import static org.mockito.ArgumentMatchers.isNull;
|
|
import static org.mockito.Mockito.verify;
|
|
import static org.mockito.Mockito.mock;
|
|
import static org.mockito.Mockito.when;
|
|
|
|
class ChatServiceSequentialAgentTest {
|
|
|
|
@Test
|
|
void executeChatComplexInvokesSequentialWorkflow() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-test-session"
|
|
);
|
|
|
|
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
|
assertFalse(result.answer().contains("\"answer_version\""));
|
|
assertEquals("sequential-test-session", result.sessionId());
|
|
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
|
assertTrue(chatModel.sawVerifierPrompt);
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexDoesNotRetryLowConfidenceByDefault() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "LOW_CONFID",
|
|
"groundedness_score": 0.1,
|
|
"critical_fact_count": 1,
|
|
"facts_checked": [
|
|
{
|
|
"fact": "missing direct evidence",
|
|
"is_critical": true,
|
|
"verification": "no_evidence",
|
|
"detail": "scripted evidence gap",
|
|
"evidence_refs": []
|
|
}
|
|
],
|
|
"rationale": "scripted low confidence"
|
|
}
|
|
""");
|
|
chatModel.composerOutput = "not-json";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-low-confidence-session"
|
|
);
|
|
|
|
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
|
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
|
assertTrue(result.answer().contains("当前缺口"));
|
|
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "LOW_CONFID",
|
|
"groundedness_score": 0.37,
|
|
"critical_fact_count": 3,
|
|
"facts_checked": [
|
|
{
|
|
"fact": "连接池耗尽 active=50/50",
|
|
"is_critical": true,
|
|
"verification": "direct_evidence",
|
|
"detail": "log evidence",
|
|
"evidence_refs": []
|
|
},
|
|
{
|
|
"fact": "临时扩容连接池到 80",
|
|
"is_critical": true,
|
|
"verification": "indirect_support",
|
|
"detail": "suggestion inferred from evidence",
|
|
"evidence_refs": []
|
|
},
|
|
{
|
|
"fact": "OOM 导致连接泄漏",
|
|
"is_critical": true,
|
|
"verification": "no_evidence",
|
|
"detail": "missing OOM log",
|
|
"evidence_refs": []
|
|
}
|
|
],
|
|
"rationale": "scripted low confidence"
|
|
}
|
|
""");
|
|
chatModel.composerOutput = "not-json";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-low-confid-direct-only-session"
|
|
);
|
|
|
|
assertTrue(result.answer().contains("已确认信息：\n- 连接池耗尽 active=50/50"));
|
|
assertTrue(result.answer().contains("80"));
|
|
assertTrue(result.answer().contains("suggestion inferred from evidence"));
|
|
assertTrue(result.answer().contains("missing OOM log"));
|
|
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-missing-verifier-session"
|
|
);
|
|
|
|
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
|
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-invalid-verifier-session"
|
|
);
|
|
|
|
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "REJECT",
|
|
"groundedness_score": 0.0,
|
|
"critical_fact_count": 1,
|
|
"facts_checked": [
|
|
{
|
|
"fact": "payment timeout root cause",
|
|
"is_critical": true,
|
|
"verification": "contradicted",
|
|
"detail": "scripted contradiction",
|
|
"evidence_refs": []
|
|
}
|
|
],
|
|
"rationale": "scripted reject"
|
|
}
|
|
""");
|
|
chatModel.composerOutput = "not-json";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-reject-session"
|
|
);
|
|
|
|
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
|
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析订单支付超时的原因，并给出修复建议",
|
|
List.of(),
|
|
"sequential-workflow-session"
|
|
);
|
|
|
|
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
|
assertFalse(result.answer().contains("\"answer_version\""));
|
|
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
|
assertTrue(chatModel.sawVerifierPrompt);
|
|
}
|
|
|
|
@Test
|
|
void verifierReceivesStructuredExecutorPayloadFields() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
chatModel.executorOutput = """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"source_id": "trace-1",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 101,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [],
|
|
"missing_info": []
|
|
}
|
|
""";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-structured-executor-session"
|
|
);
|
|
|
|
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
|
assertFalse(result.answer().contains("\"answer_version\""));
|
|
assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\""));
|
|
assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\""));
|
|
assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\""));
|
|
assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexRendersExecutorEvidenceV2InsteadOfRawJsonOnPass() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
chatModel.composerOutput = "not-json";
|
|
chatModel.executorOutput = """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"source_id": "trace-1",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 101,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [
|
|
{
|
|
"hypothesis_text": "连接泄漏可能参与了连接池耗尽",
|
|
"basis": "已有连接池满载证据，但缺少泄漏检测日志",
|
|
"needed_evidence": ["连接泄漏检测日志"]
|
|
}
|
|
],
|
|
"recommended_actions": [
|
|
{
|
|
"action_text": "补充查询连接池泄漏检测日志",
|
|
"reason": "用于确认是否存在连接未释放"
|
|
}
|
|
],
|
|
"missing_info": ["缺少连接泄漏检测日志"]
|
|
}
|
|
""";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-v2-render-session"
|
|
);
|
|
|
|
assertTrue(result.answer().contains("已确认信息"));
|
|
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
|
assertTrue(result.answer().contains("建议下一步"));
|
|
assertFalse(result.answer().contains("\"answer_version\""));
|
|
assertFalse(result.answer().contains("executor_evidence_v2"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexPersistsGatekeeperResultInVerifierEvaluation() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
SelfEvaluationMergeService mergeService =
|
|
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
|
ToolInvocationRepository invocationRepository =
|
|
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
|
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-persist-session"))
|
|
.thenReturn(List.of(ToolInvocation.builder()
|
|
.id(101L)
|
|
.sessionId("sequential-gatekeeper-persist-session")
|
|
.toolName("query_metrics")
|
|
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
|
.build()));
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
chatModel.executorOutput = """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"source_id": "trace-1",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 101,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [],
|
|
"missing_info": []
|
|
}
|
|
""";
|
|
|
|
chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-gatekeeper-persist-session"
|
|
);
|
|
|
|
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
|
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
|
Map<String, Object> verifierEvaluation = captor.getValue();
|
|
assertTrue(verifierEvaluation.containsKey("gatekeeper_result"));
|
|
assertTrue(verifierEvaluation.containsKey("prompt_audit"));
|
|
@SuppressWarnings("unchecked")
|
|
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
|
|
assertEquals("pass", gatekeeperResult.get("status"));
|
|
assertEquals("none", gatekeeperResult.get("severity"));
|
|
@SuppressWarnings("unchecked")
|
|
Map<String, Object> promptAudit = (Map<String, Object>) verifierEvaluation.get("prompt_audit");
|
|
assertEquals("chat-prompts-v1", promptAudit.get("version"));
|
|
@SuppressWarnings("unchecked")
|
|
List<Map<String, Object>> prompts = (List<Map<String, Object>>) promptAudit.get("prompts");
|
|
assertEquals(4, prompts.size());
|
|
assertTrue(prompts.stream().anyMatch(prompt ->
|
|
"chat_executor".equals(prompt.get("name"))
|
|
&& "chat-executor-v2".equals(prompt.get("version"))));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexMapsClaimChecksToFactsCheckedAndPersistsBoth() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
SelfEvaluationMergeService mergeService =
|
|
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
|
ToolInvocationRepository invocationRepository =
|
|
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
|
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-claim-check-session"))
|
|
.thenReturn(List.of(ToolInvocation.builder()
|
|
.id(101L)
|
|
.sessionId("sequential-claim-check-session")
|
|
.toolName("query_metrics")
|
|
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
|
.build()));
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "LOW_CONFID",
|
|
"groundedness_score": 0.32,
|
|
"critical_fact_count": 6,
|
|
"claim_checks": [
|
|
{"claim_id":"claim-1","claim_text":"CPU 使用率 92%","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[{"trace_ref":"trace-1","tool_name":"query_metrics","source_invocation_ids":[101],"note":"cpu"}]},
|
|
{"claim_id":"claim-2","claim_text":"CPU 过高可能导致超时","claim_type":"risk","verification":"reasonable_inference","detail":"inference","evidence_refs":[]},
|
|
{"claim_id":"claim-3","claim_text":"CPU 是唯一根因","claim_type":"root_cause","verification":"overstated","detail":"too strong","evidence_refs":[]},
|
|
{"claim_id":"claim-4","claim_text":"缺少线程池证据","claim_type":"symptom","verification":"unsupported","detail":"missing","evidence_refs":[]},
|
|
{"claim_id":"claim-5","claim_text":"出现证据外错误码 ERR_FAKE","claim_type":"symptom","verification":"external_unknown","detail":"external","evidence_refs":[]},
|
|
{"claim_id":"claim-6","claim_text":"证据显示 CPU 很低","claim_type":"symptom","verification":"contradicted","detail":"conflict","evidence_refs":[]}
|
|
],
|
|
"facts_checked": [],
|
|
"rationale": "claim checks drive compatibility"
|
|
}
|
|
""");
|
|
chatModel.executorOutput = validExecutorV2Output();
|
|
|
|
chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-claim-check-session"
|
|
);
|
|
|
|
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
|
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
|
Map<String, Object> verifierEvaluation = captor.getValue();
|
|
@SuppressWarnings("unchecked")
|
|
List<Map<String, Object>> claimChecks = (List<Map<String, Object>>) verifierEvaluation.get("claim_checks");
|
|
@SuppressWarnings("unchecked")
|
|
List<Map<String, Object>> factsChecked = (List<Map<String, Object>>) verifierEvaluation.get("facts_checked");
|
|
|
|
assertEquals(6, claimChecks.size());
|
|
assertEquals(6, factsChecked.size());
|
|
assertEquals("direct_evidence", factsChecked.get(0).get("verification"));
|
|
assertEquals("indirect_support", factsChecked.get(1).get("verification"));
|
|
assertEquals("indirect_support", factsChecked.get(2).get("verification"));
|
|
assertEquals("no_evidence", factsChecked.get(3).get("verification"));
|
|
assertEquals("no_evidence", factsChecked.get(4).get("verification"));
|
|
assertEquals("contradicted", factsChecked.get(5).get("verification"));
|
|
assertTrue(String.valueOf(factsChecked.get(0).get("fact")).startsWith("claim-1:"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexDowngradesPassToRejectWhenGatekeeperInvocationRefFails() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
SelfEvaluationMergeService mergeService =
|
|
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
|
ToolInvocationRepository invocationRepository =
|
|
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
|
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-fail-session"))
|
|
.thenReturn(List.of(ToolInvocation.builder()
|
|
.id(101L)
|
|
.sessionId("sequential-gatekeeper-fail-session")
|
|
.toolName("query_metrics")
|
|
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
|
.build()));
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "PASS",
|
|
"groundedness_score": 1.0,
|
|
"critical_fact_count": 1,
|
|
"claim_checks": [
|
|
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[]}
|
|
],
|
|
"facts_checked": [],
|
|
"rationale": "model tried pass"
|
|
}
|
|
""");
|
|
chatModel.composerOutput = "not-json";
|
|
chatModel.executorOutput = """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 999,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [],
|
|
"missing_info": []
|
|
}
|
|
""";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-gatekeeper-fail-session"
|
|
);
|
|
|
|
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
|
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
|
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
|
assertEquals("REJECT", captor.getValue().get("verdict"));
|
|
}
|
|
|
|
@Test
|
|
void executeChatComplexDowngradesPassToLowConfidenceWhenExecutorOutputMalformed() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
SelfEvaluationMergeService mergeService =
|
|
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
|
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
|
{
|
|
"verdict": "PASS",
|
|
"groundedness_score": 1.0,
|
|
"critical_fact_count": 0,
|
|
"claim_checks": [],
|
|
"facts_checked": [],
|
|
"rationale": "model tried pass"
|
|
}
|
|
""");
|
|
chatModel.composerOutput = "not-json";
|
|
chatModel.executorOutput = "{ not-json";
|
|
|
|
ChatService.ChatResult result = chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"请分析 MySQL 连接池耗尽",
|
|
List.of(),
|
|
"sequential-malformed-pass-session"
|
|
);
|
|
|
|
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
|
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
|
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
|
assertEquals("LOW_CONFID", captor.getValue().get("verdict"));
|
|
}
|
|
|
|
@Test
|
|
void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
|
|
ChatService chatService = new ChatService();
|
|
DateTimeTools dateTimeTools = new DateTimeTools();
|
|
LookupKnowledgeTool lookupKnowledgeTool = new LookupKnowledgeTool();
|
|
QueryLogsTools queryLogsTools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
|
|
QueryMetricsTools queryMetricsTools = new QueryMetricsTools(mock(ToolInvocationRecorder.class));
|
|
|
|
ReflectionTestUtils.setField(chatService, "dateTimeTools", dateTimeTools);
|
|
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", lookupKnowledgeTool);
|
|
ReflectionTestUtils.setField(chatService, "queryLogsTools", queryLogsTools);
|
|
ReflectionTestUtils.setField(chatService, "queryMetricsTools", queryMetricsTools);
|
|
|
|
Object[] methodTools = chatService.buildMethodToolsArray();
|
|
|
|
assertEquals(4, methodTools.length);
|
|
assertSame(dateTimeTools, methodTools[0]);
|
|
assertSame(lookupKnowledgeTool, methodTools[1]);
|
|
assertSame(queryLogsTools, methodTools[2]);
|
|
assertSame(queryMetricsTools, methodTools[3]);
|
|
}
|
|
|
|
@Test
|
|
void createReactAgentInjectsSkillCatalogThroughAlibabaHook() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
|
|
.classpathPath("skills")
|
|
.basePath("target/test-skills-cache")
|
|
.build();
|
|
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
|
|
|
|
ReactAgent agent = chatService.createReactAgent(chatModel, "BASE_TEST_PROMPT");
|
|
agent.call("diagnose mysql connection pool exhaustion");
|
|
|
|
assertTrue(chatModel.promptText.contains("BASE_TEST_PROMPT"));
|
|
assertTrue(chatModel.promptText.contains("## Skills System"));
|
|
assertTrue(chatModel.promptText.contains("diagnose-mysql-connection-pool"));
|
|
assertTrue(chatModel.promptText.contains("read_skill"));
|
|
}
|
|
|
|
@Test
|
|
void plannerGetsSkillMetadataAndExecutorGetsReadSkillTool() throws Exception {
|
|
ChatService chatService = createChatService();
|
|
ScriptedChatModel chatModel = new ScriptedChatModel();
|
|
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
|
|
.classpathPath("skills")
|
|
.basePath("target/test-skills-cache")
|
|
.build();
|
|
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
|
|
|
|
chatService.executeChatComplex(
|
|
chatModel,
|
|
new ToolCallback[0],
|
|
"diagnose mysql connection pool exhaustion",
|
|
List.of(),
|
|
"planner-skill-metadata-session"
|
|
);
|
|
|
|
assertTrue(chatModel.plannerPromptText.contains("\"skill_catalog\""));
|
|
assertTrue(chatModel.plannerPromptText.contains("diagnose-mysql-connection-pool"));
|
|
assertTrue(chatModel.plannerPromptText.contains("\"selected_skill\""));
|
|
assertFalse(chatModel.plannerPromptText.contains("## Skills System"));
|
|
assertFalse(chatModel.plannerPromptText.contains("read_skill"));
|
|
|
|
assertTrue(chatModel.executorPromptText.contains("## Skills System"));
|
|
assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool"));
|
|
assertTrue(chatModel.executorPromptText.contains("read_skill"));
|
|
assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill"));
|
|
assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool"));
|
|
assertFalse(chatModel.verifierPromptText.contains("read_skill"));
|
|
}
|
|
|
|
private ChatService createChatService() {
|
|
ChatService chatService = new ChatService();
|
|
|
|
DiagnosisSessionRepository diagnosisSessionRepository = mock(DiagnosisSessionRepository.class);
|
|
when(diagnosisSessionRepository.findBySessionId(anyString())).thenReturn(Optional.empty());
|
|
when(diagnosisSessionRepository.save(any(DiagnosisSession.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
|
|
|
AtomicInteger stepId = new AtomicInteger(1);
|
|
AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
|
|
when(agentStepRepository.save(any(AgentStep.class))).thenAnswer(invocation -> {
|
|
AgentStep step = invocation.getArgument(0);
|
|
if (step.getId() == null) {
|
|
step.setId((long) stepId.getAndIncrement());
|
|
}
|
|
return step;
|
|
});
|
|
when(agentStepRepository.findById(any())).thenReturn(Optional.of(new AgentStep()));
|
|
when(agentStepRepository.findBySessionIdOrderByStepIndex(anyString())).thenReturn(List.of());
|
|
ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
|
|
when(toolInvocationRepository.countBySessionId(anyString())).thenReturn(0L);
|
|
when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
|
|
.id(101L)
|
|
.toolName("query_metrics")
|
|
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
|
.build()));
|
|
|
|
EvaluationService evaluationService = mock(EvaluationService.class);
|
|
RetrievedDocTracker retrievedDocTracker = mock(RetrievedDocTracker.class);
|
|
KnowledgeDomainService knowledgeDomainService = mock(KnowledgeDomainService.class);
|
|
when(knowledgeDomainService.buildKnowledgeMap()).thenReturn("");
|
|
ToolTraceSummaryService toolTraceSummaryService = mock(ToolTraceSummaryService.class);
|
|
when(toolTraceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
|
SelfEvaluationMergeService selfEvaluationMergeService = mock(SelfEvaluationMergeService.class);
|
|
when(selfEvaluationMergeService.mergeVerifierEvaluation(any(), any())).thenReturn("{}");
|
|
ExecutorGatekeeperService executorGatekeeperService = new ExecutorGatekeeperService(toolInvocationRepository);
|
|
|
|
ReflectionTestUtils.setField(chatService, "dateTimeTools", new DateTimeTools());
|
|
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", new LookupKnowledgeTool());
|
|
ReflectionTestUtils.setField(chatService, "queryLogsTools", new QueryLogsTools(mock(ToolInvocationRecorder.class)));
|
|
ReflectionTestUtils.setField(chatService, "diagnosisSessionRepository", diagnosisSessionRepository);
|
|
ReflectionTestUtils.setField(chatService, "agentStepRepository", agentStepRepository);
|
|
ReflectionTestUtils.setField(chatService, "toolInvocationRepository", toolInvocationRepository);
|
|
ReflectionTestUtils.setField(chatService, "evaluationService", evaluationService);
|
|
ReflectionTestUtils.setField(chatService, "retrievedDocTracker", retrievedDocTracker);
|
|
ReflectionTestUtils.setField(chatService, "knowledgeDomainService", knowledgeDomainService);
|
|
ReflectionTestUtils.setField(chatService, "toolTraceSummaryService", toolTraceSummaryService);
|
|
ReflectionTestUtils.setField(chatService, "selfEvaluationMergeService", selfEvaluationMergeService);
|
|
ReflectionTestUtils.setField(chatService, "executorGatekeeperService", executorGatekeeperService);
|
|
ReflectionTestUtils.setField(chatService, "verifierLowConfidenceThreshold", 0.5d);
|
|
ReflectionTestUtils.setField(chatService, "chatPlannerPrompt", "PLANNER_TEST_PROMPT");
|
|
ReflectionTestUtils.setField(chatService, "chatExecutorPrompt", "EXECUTOR_TEST_PROMPT");
|
|
ReflectionTestUtils.setField(chatService, "chatVerifierPrompt", "VERIFIER_TEST_PROMPT");
|
|
ReflectionTestUtils.setField(chatService, "chatComposerPrompt", "COMPOSER_TEST_PROMPT");
|
|
return chatService;
|
|
}
|
|
|
|
private String validExecutorV2Output() {
|
|
return """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"source_id": "trace-1",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 101,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [],
|
|
"missing_info": []
|
|
}
|
|
""";
|
|
}
|
|
|
|
private String evidenceRefs(String rawPath, String text) {
|
|
return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}";
|
|
}
|
|
|
|
private static final class ScriptedChatModel implements ChatModel {
|
|
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
|
|
private String promptText = "";
|
|
private String plannerPromptText = "";
|
|
private String executorPromptText = "";
|
|
private String verifierPromptText = "";
|
|
private String composerPromptText = "";
|
|
private String executorOutput = """
|
|
{
|
|
"answer_version": "executor_evidence_v2",
|
|
"claims": [
|
|
{
|
|
"claim_id": "claim-1",
|
|
"claim_type": "symptom",
|
|
"claim_text": "连接池 active 达到上限",
|
|
"support_level": "direct",
|
|
"evidence_bindings": [
|
|
{
|
|
"source_type": "tool_trace",
|
|
"source_id": "trace-1",
|
|
"tool_name": "query_metrics",
|
|
"source_invocation_id": 101,
|
|
"raw_path": "$.alerts[0]",
|
|
"evidence_excerpt": "active=50 max=50"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"hypotheses": [],
|
|
"recommended_actions": [],
|
|
"missing_info": []
|
|
}
|
|
""";
|
|
private String composerOutput = """
|
|
{
|
|
"answer_summary": "已确认连接池 active 达到上限。",
|
|
"recommended_actions": [
|
|
{
|
|
"action_text": "补充查询连接池泄漏检测日志",
|
|
"reason": "用于确认是否存在连接未释放"
|
|
}
|
|
],
|
|
"user_facing_answer": "已确认连接池 active 达到上限。建议补充查询连接池泄漏检测日志。"
|
|
}
|
|
""";
|
|
private boolean sawVerifierPrompt;
|
|
private final java.util.List<String> verifierOutputs;
|
|
private int verifierOutputIndex;
|
|
|
|
private ScriptedChatModel() {
|
|
this("""
|
|
{
|
|
"verdict": "PASS",
|
|
"groundedness_score": 1.0,
|
|
"critical_fact_count": 1,
|
|
"claim_checks": [
|
|
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"covered by scripted verifier","evidence_refs":[]}
|
|
],
|
|
"facts_checked": [],
|
|
"rationale": "scripted pass"
|
|
}
|
|
""");
|
|
}
|
|
|
|
private ScriptedChatModel(String verifierOutput) {
|
|
this.verifierOutputs = java.util.List.of(verifierOutput);
|
|
}
|
|
|
|
private ScriptedChatModel(String... verifierOutputs) {
|
|
this.verifierOutputs = java.util.List.of(verifierOutputs);
|
|
}
|
|
|
|
@Override
|
|
public ChatResponse call(Prompt prompt) {
|
|
promptText = prompt.getContents();
|
|
String text;
|
|
if (promptText.contains("PLANNER_TEST_PROMPT")) {
|
|
agentCalls.add("chat_planner");
|
|
plannerPromptText = promptText;
|
|
text = "PLANNER_PLAN";
|
|
} else if (promptText.contains("EXECUTOR_TEST_PROMPT")) {
|
|
agentCalls.add("chat_executor");
|
|
executorPromptText = promptText;
|
|
text = executorOutput;
|
|
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
|
|
agentCalls.add("chat_verifier");
|
|
verifierPromptText = promptText;
|
|
sawVerifierPrompt = true;
|
|
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
|
|
text = verifierOutputs.get(index);
|
|
verifierOutputIndex++;
|
|
} else if (promptText.contains("COMPOSER_TEST_PROMPT")) {
|
|
agentCalls.add("chat_composer");
|
|
composerPromptText = promptText;
|
|
text = composerOutput;
|
|
} else {
|
|
text = "UNEXPECTED_PROMPT";
|
|
}
|
|
return new ChatResponse(List.of(new Generation(new AssistantMessage(text))));
|
|
}
|
|
}
|
|
}
|