osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add chat verifier agent

Browse Source
This commit is contained in:
zhuyongxin
2026-07-03 10:54:33 +08:00
parent 4f5316d473
commit 9050487307
28 changed files with 3200 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/hook/AgentLoggingHook.java
+140 -166
View File
@@ -1,25 +1,27 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.util.SessionContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.ToolResponseMessage;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Agent 日志 Hook
* 记录 Agent 的思考过程、消息流转 + 持久化 agent_step 到 DB
* Persists per-agent model input/output snapshots into agent_step.
*/
@Slf4j
@HookPositions({HookPosition.BEFORE_MODEL, HookPosition.AFTER_MODEL})
@@ -27,11 +29,9 @@ public class AgentLoggingHook extends MessagesModelHook {
private final AgentStepRepository agentStepRepository;
private final String agentName;
private final ObjectMapper objectMapper = new ObjectMapper();
/** 每个 session 的步数计数器：sessionId → stepIndex */
private final ConcurrentHashMap<String, Integer> stepCounters = new ConcurrentHashMap<>();
/** beforeModel → afterModel 中间状态：sessionId_stepIndex → {stepId, startTime} */
private final ConcurrentHashMap<String, Map<String, Object>> pendingSteps = new ConcurrentHashMap<>();
public AgentLoggingHook(AgentStepRepository agentStepRepository, String agentName) {
@@ -46,60 +46,44 @@ public class AgentLoggingHook extends MessagesModelHook {
@Override
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
// 优先从 config.metadata 取 sessionId（线程安全），兜底 ThreadLocal
String sessionId = config.metadata("sessionId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getSessionId);
boolean hasSession = (sessionId != null);
String sessionId = resolveSessionId(config);
boolean hasSession = sessionId != null;
int stepIndex = 0;
if (hasSession) {
stepIndex = stepCounters.merge(sessionId, 0, (old, one) -> old + 1);
stepIndex = stepCounters.merge(sessionId, 0, (oldValue, ignored) -> oldValue + 1);
}
log.info("========================================");
log.info("*** [Agent 思考] 第 {} 轮思考开始", (hasSession ? stepCounters.get(sessionId) : 0) + 1);
log.info("*** [Agent 思考] 当前消息数量: {}", previousMessages.size());
log.info("*** [AgentTrace] agent={}, phase=before_model, stepIndex={}", agentName, stepIndex);
log.info("*** [AgentTrace] messageCount={}", previousMessages.size());
// 打印最后几条消息
int lastN = Math.min(3, previousMessages.size());
if (lastN > 0) {
log.info("*** [Agent 思考] 最近 {} 条消息:", lastN);
log.info("*** [AgentTrace] recentMessages={}", lastN);
List<Message> recentMessages = previousMessages.subList(previousMessages.size() - lastN, previousMessages.size());
for (int i = 0; i < recentMessages.size(); i++) {
Message msg = recentMessages.get(i);
String role = getMessageRole(msg);
log.info(" [{}] 角色: {}, 类型: {}", i + 1, role, msg.getClass().getSimpleName());
log.info(" [{}] role={}, type={}", i + 1, getMessageRole(msg), msg.getClass().getSimpleName());
}
}
log.info("*** [Agent 思考] 准备调用模型...");
log.info("========================================");
// 持久化 agent_step（beforeModel：先创建，先记 model_input 摘要）
if (sessionId != null) {
try {
String modelInputSummary = buildModelInputSummary(previousMessages);
AgentStep step = AgentStep.builder()
.sessionId(sessionId)
.stepIndex(stepIndex)
.agentName(agentName)
.modelInput(modelInputSummary)
.modelInput(buildModelInputSummary(previousMessages))
.build();
AgentStep saved = agentStepRepository.save(step);
// 记录中间状态供 afterModel 使用
pendingSteps.put(sessionId + "_" + stepIndex, Map.of(
"stepId", saved.getId(),
"startTime", System.currentTimeMillis()
));
log.debug("agent_step 已创建: sessionId={}, stepIndex={}, id={}", sessionId, stepIndex, saved.getId());
} catch (Exception e) {
log.error("保存 agent_step 失败", e);
// 不中断 Agent 执行
log.error("Failed to persist agent_step before model", e);
}
}
@@ -108,58 +92,38 @@ public class AgentLoggingHook extends MessagesModelHook {
@Override
public AgentCommand afterModel(List<Message> previousMessages, RunnableConfig config) {
String sessionId = SessionContextHolder.getSessionId();
boolean hasSession = (sessionId != null);
String sessionId = resolveSessionId(config);
int stepIndex = sessionId == null ? 0 : stepCounters.getOrDefault(sessionId, 0);
log.info("========================================");
log.info("*** [Agent 思考] 第 {} 轮思考完成", (hasSession ? stepCounters.getOrDefault(sessionId, 0) : 0));
// 查找最后一条 AssistantMessage（模型的回复）
AssistantMessage lastAssistant = null;
for (int i = previousMessages.size() - 1; i >= 0; i--) {
if (previousMessages.get(i) instanceof AssistantMessage) {
lastAssistant = (AssistantMessage) previousMessages.get(i);
break;
}
}
log.info("*** [AgentTrace] agent={}, phase=after_model, stepIndex={}", agentName, stepIndex);
AssistantMessage lastAssistant = findLastAssistant(previousMessages);
boolean hasToolCall = false;
if (lastAssistant != null) {
// 打印模型返回的文本内容
String textContent = extractTextContent(lastAssistant);
if (textContent != null && !textContent.isEmpty()) {
log.info("*** [Agent 思考] 模型返回文本: {}",
textContent.length() > 500
? textContent.substring(0, 500) + "... (已截断，总长度: " + textContent.length() + ")"
: textContent);
log.info("*** [AgentTrace] text={}",
textContent.length() > 500
? textContent.substring(0, 500) + "... (len=" + textContent.length() + ")"
: textContent);
}
// 检查是否有工具调用
if (lastAssistant.getToolCalls() != null && !lastAssistant.getToolCalls().isEmpty()) {
hasToolCall = true;
log.info("*** [Agent 思考] 模型决定调用 {} 个工具:",
lastAssistant.getToolCalls().size());
lastAssistant.getToolCalls().forEach(toolCall -> {
log.info(" - 工具: {}, 参数: {}",
toolCall.name(),
toolCall.arguments());
});
log.info("*** [Agent 思考] 等待工具执行结果...");
log.info("*** [AgentTrace] toolCalls={}", lastAssistant.getToolCalls().size());
lastAssistant.getToolCalls().forEach(toolCall ->
log.info(" - tool={}, arguments={}", toolCall.name(), toolCall.arguments()));
} else {
log.info("*** [Agent 思考] 模型决定不调用工具");
log.info("*** [Agent 思考] 这是最终答案，准备返回给用户");
log.info("*** [AgentTrace] no tool call");
}
}
log.info("========================================");
// 更新 agent_step（afterModel：补全 model_output、耗时等）
if (sessionId != null) {
int stepIndex = stepCounters.getOrDefault(sessionId, 0);
String stepKey = sessionId + "_" + stepIndex;
Map<String, Object> pending = pendingSteps.remove(stepKey);
if (pending != null) {
try {
Long stepId = (Long) pending.get("stepId");
@@ -168,20 +132,12 @@ public class AgentLoggingHook extends MessagesModelHook {
AgentStep step = agentStepRepository.findById(stepId).orElse(null);
if (step != null) {
String thought = extractTextContent(lastAssistant);
if (thought != null && thought.length() > 2000) {
thought = thought.substring(0, 2000);
}
step.setThought(thought);
step.setThought(buildStoredThought(lastAssistant));
step.setHasToolCall(hasToolCall);
step.setDurationMs(durationMs);
if (lastAssistant != null) {
String outputSummary = buildModelOutputSummary(lastAssistant);
step.setModelOutput(outputSummary);
// 读取实际 token 用量（由 TokenTrackingChatModel 写入）
step.setModelOutput(buildModelOutputSummary(lastAssistant));
Integer tokenCount = TokenUsageHolder.get();
if (tokenCount != null) {
step.setTokenCount(tokenCount);
@@ -189,24 +145,32 @@ public class AgentLoggingHook extends MessagesModelHook {
}
agentStepRepository.save(step);
log.debug("agent_step 已更新: sessionId={}, stepIndex={}, duration={}ms",
sessionId, stepIndex, durationMs);
}
} catch (Exception e) {
log.error("更新 agent_step 失败", e);
log.error("Failed to update agent_step after model", e);
}
}
}
// 清理 token 上下文
TokenUsageHolder.clear();
return new AgentCommand(previousMessages);
}
/**
* 构建模型输入摘要（前 N 条消息的 role + 截断内容）
*/
private String resolveSessionId(RunnableConfig config) {
return config.metadata("sessionId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getSessionId);
}
private AssistantMessage findLastAssistant(List<Message> previousMessages) {
for (int i = previousMessages.size() - 1; i >= 0; i--) {
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
return assistantMessage;
}
}
return null;
}
private String buildModelInputSummary(List<Message> messages) {
StringBuilder sb = new StringBuilder();
int maxMessages = Math.min(messages.size(), 5);
@@ -226,25 +190,59 @@ public class AgentLoggingHook extends MessagesModelHook {
return result;
}
/**
* 构建模型输出摘要
*/
private String buildStoredThought(AssistantMessage message) {
String text = extractTextContent(message);
if (text == null || text.isBlank()) {
return text;
}
if (!"verifier".equals(agentName)) {
return truncate(text, 2000);
}
return summarizeVerifierThought(text);
}
private String summarizeVerifierThought(String verifierOutput) {
try {
JsonNode root = objectMapper.readTree(verifierOutput);
int factCount = root.path("facts_checked").isArray() ? root.path("facts_checked").size() : 0;
int tracedFactCount = 0;
if (root.path("facts_checked").isArray()) {
for (JsonNode factNode : root.path("facts_checked")) {
if (factNode.path("evidence_refs").isArray() && factNode.path("evidence_refs").size() > 0) {
tracedFactCount++;
}
}
}
return "verdict=%s, score=%s, critical_fact_count=%s, facts_checked=%d, traced_facts=%d".formatted(
root.path("verdict").asText("UNKNOWN"),
root.path("groundedness_score").asText("0.0"),
root.path("critical_fact_count").asText("0"),
factCount,
tracedFactCount
);
} catch (Exception e) {
return truncate(verifierOutput, 300);
}
}
private String buildModelOutputSummary(AssistantMessage message) {
String text = extractTextContent(message);
if (text == null) {
text = "";
}
if (text.length() > 500) {
text = text.substring(0, 500) + "...";
}
int maxTextLength = "verifier".equals(agentName) ? 4000 : 500;
text = truncate(text, maxTextLength);
StringBuilder sb = new StringBuilder();
sb.append("{\"text\":\"").append(escapeJson(text)).append("\"");
if (message.getToolCalls() != null && !message.getToolCalls().isEmpty()) {
sb.append(",\"toolCalls\":[");
for (int i = 0; i < message.getToolCalls().size(); i++) {
if (i > 0) sb.append(",");
if (i > 0) {
sb.append(",");
}
sb.append("{\"name\":\"").append(escapeJson(message.getToolCalls().get(i).name()))
.append("\",\"arguments\":").append(message.getToolCalls().get(i).arguments()).append("}");
.append("\",\"arguments\":").append(message.getToolCalls().get(i).arguments()).append("}");
}
sb.append("]");
}
@@ -253,7 +251,9 @@ public class AgentLoggingHook extends MessagesModelHook {
}
private String escapeJson(String s) {
if (s == null) return "";
if (s == null) {
return "";
}
return s.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
@@ -261,96 +261,70 @@ public class AgentLoggingHook extends MessagesModelHook {
.replace("\t", "\\t");
}
/**
* 提取 AssistantMessage 的文本内容
*/
private String truncate(String text, int maxLength) {
if (text == null || text.length() <= maxLength) {
return text;
}
return text.substring(0, maxLength) + "...";
}
private String extractTextContent(AssistantMessage message) {
if (message == null) return null;
if (message == null) {
return null;
}
try {
// 方法 1: 反射获取 text 字段
try {
java.lang.reflect.Field textField = message.getClass().getDeclaredField("text");
textField.setAccessible(true);
Object value = textField.get(message);
if (value != null) {
log.debug("通过 text 字段提取成功");
return value.toString();
return message.getText();
} catch (Exception ignore) {
// Fallback below.
}
for (String fieldName : List.of("text", "content")) {
try {
java.lang.reflect.Field field = message.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
Object value = field.get(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchFieldException ignore) {
// continue
}
} catch (NoSuchFieldException e) {
// 尝试下一种方法
}
// 方法 2: 反射获取 content 字段
try {
java.lang.reflect.Field contentField = message.getClass().getDeclaredField("content");
contentField.setAccessible(true);
Object value = contentField.get(message);
if (value != null) {
log.debug("通过 content 字段提取成功");
return value.toString();
for (String methodName : List.of("getText", "getContent")) {
try {
java.lang.reflect.Method method = message.getClass().getMethod(methodName);
Object value = method.invoke(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchMethodException ignore) {
// continue
}
} catch (NoSuchFieldException e) {
// 尝试下一种方法
}
// 方法 3: 调用 getText() 方法
try {
java.lang.reflect.Method getTextMethod = message.getClass().getMethod("getText");
Object value = getTextMethod.invoke(message);
if (value != null) {
log.debug("通过 getText() 方法提取成功");
return value.toString();
}
} catch (NoSuchMethodException e) {
// 尝试下一种方法
String fallback = message.toString();
if (fallback != null && !fallback.startsWith("AssistantMessage@")) {
return fallback;
}
// 方法 4: 调用 getContent() 方法
try {
java.lang.reflect.Method getContentMethod = message.getClass().getMethod("getContent");
Object value = getContentMethod.invoke(message);
if (value != null) {
log.debug("通过 getContent() 方法提取成功");
return value.toString();
}
} catch (NoSuchMethodException e) {
// 方法不存在
}
// 方法 5: 打印类结构信息
log.warn("无法提取 AssistantMessage 文本内容，打印类信息:");
log.warn("类名: {}", message.getClass().getName());
log.warn("字段列表:");
for (java.lang.reflect.Field field : message.getClass().getDeclaredFields()) {
log.warn(" - {}: {}", field.getName(), field.getType().getSimpleName());
}
// 方法 6: toString() 兜底
String toString = message.toString();
if (toString != null && !toString.startsWith("AssistantMessage@")) {
log.debug("通过 toString() 提取");
return toString;
}
return null;
} catch (Exception e) {
log.error("提取 AssistantMessage 文本内容时出错", e);
log.error("Failed to extract AssistantMessage text", e);
return null;
}
}
/**
* 获取消息角色
*/
private String getMessageRole(Message message) {
if (message instanceof UserMessage) {
return "User（用户）";
} else if (message instanceof AssistantMessage) {
return "Assistant（模型）";
} else if (message instanceof ToolResponseMessage) {
return "Tool（工具返回）";
} else {
return message.getClass().getSimpleName();
return "user";
}
if (message instanceof AssistantMessage) {
return "assistant";
}
if (message instanceof ToolResponseMessage) {
return "tool";
}
return message.getClass().getSimpleName();
}
}
src/main/java/com/superbiz/agent/hook/VerifierInputHook.java
+105
View File
@@ -0,0 +1,105 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ToolTraceSummaryService;
import com.superbiz.agent.util.SessionContextHolder;
import com.superbiz.agent.util.VerifierContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Replaces verifier history with an explicit structured payload.
*/
@Slf4j
@HookPositions(HookPosition.BEFORE_MODEL)
public class VerifierInputHook extends MessagesModelHook {
private final ToolTraceSummaryService toolTraceSummaryService;
private final ObjectMapper objectMapper = new ObjectMapper();
public VerifierInputHook(ToolTraceSummaryService toolTraceSummaryService) {
this.toolTraceSummaryService = toolTraceSummaryService;
}
@Override
public String getName() {
return "verifier_input_hook";
}
@Override
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
try {
String sessionId = config.metadata("sessionId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getSessionId);
String executorFinalAnswer = VerifierContextHolder.getExecutorFinalAnswer();
if (executorFinalAnswer == null || executorFinalAnswer.isBlank()) {
executorFinalAnswer = extractLastAssistantText(previousMessages);
}
List<Map<String, Object>> toolTraceSummary =
toolTraceSummaryService.buildVerifierTraceSummary(sessionId, executorFinalAnswer);
VerifierContextHolder.setToolTraceSummary(toolTraceSummary);
Map<String, Object> verifierInput = new LinkedHashMap<>();
verifierInput.put("original_query", VerifierContextHolder.getOriginalQuery());
verifierInput.put("executor_final_answer", executorFinalAnswer);
verifierInput.put("tool_trace_summary", toolTraceSummary);
verifierInput.put("retry_context", VerifierContextHolder.getRetryContext());
String payload = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(verifierInput);
return new AgentCommand(List.of(new UserMessage(payload)));
} catch (Exception e) {
log.error("Failed to build verifier input, fallback to original messages", e);
return new AgentCommand(previousMessages);
}
}
private String extractLastAssistantText(List<Message> previousMessages) {
for (int i = previousMessages.size() - 1; i >= 0; i--) {
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
String text = extractTextContent(assistantMessage);
if (text != null && !text.isBlank()) {
return text;
}
}
}
return "";
}
private String extractTextContent(AssistantMessage message) {
try {
try {
return message.getText();
} catch (Exception ignore) {
// Fallback for older implementations.
}
for (String methodName : List.of("getText", "getContent")) {
try {
var method = message.getClass().getMethod(methodName);
Object value = method.invoke(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchMethodException ignore) {
// continue
}
}
} catch (Exception e) {
log.debug("Failed to extract verifier assistant text", e);
}
return message.toString();
}
}
src/main/java/com/superbiz/agent/repository/ToolInvocationRepository.java
+5
View File
@@ -17,6 +17,11 @@ public interface ToolInvocationRepository extends JpaRepository<ToolInvocation,
*/
List<ToolInvocation> findBySessionId(String sessionId);
/**
* 根据会话ID按创建顺序查询所有工具调用
*/
List<ToolInvocation> findBySessionIdOrderByIdAsc(String sessionId);
/**
* 根据工具名查询所有调用
*/
src/main/java/com/superbiz/agent/service/ChatService.java
+412 -24
View File
@@ -5,6 +5,8 @@ import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
import com.alibaba.cloud.ai.graph.exception.GraphRunnerException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.agent.tool.DateTimeTools;
import com.superbiz.agent.agent.tool.InternalDocsTools;
import com.superbiz.agent.agent.tool.QueryLogsTools;
@@ -13,13 +15,14 @@ import com.superbiz.agent.domain.entity.DiagnosisSession;
import com.superbiz.agent.hook.AgentLoggingHook;
import com.superbiz.agent.hook.TokenTrackingChatModel;
import com.superbiz.agent.hook.TokenUsageHolder;
import com.superbiz.agent.hook.VerifierInputHook;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.DiagnosisSessionRepository;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.tool.RetrievedDocTracker;
import com.superbiz.agent.util.QuestionComplexity;
import com.superbiz.agent.util.SessionContextHolder;
import com.superbiz.agent.service.KnowledgeDomainService;
import com.superbiz.agent.util.VerifierContextHolder;
import jakarta.annotation.PostConstruct;
import org.slf4j.Logger;
@@ -29,11 +32,14 @@ import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -47,6 +53,8 @@ import java.util.UUID;
public class ChatService {
private static final Logger logger = LoggerFactory.getLogger(ChatService.class);
private static final String LOW_CONFID_DISCLAIMER = "以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。";
private static final String DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论，建议人工介入。";
/** 封装 answer + 后端生成的 sessionId，用于 feedback 关联 */
public record ChatResult(String answer, String sessionId) {}
@@ -87,9 +95,20 @@ public class ChatService {
@Autowired
private KnowledgeDomainService knowledgeDomainService;
@Autowired
private ToolTraceSummaryService toolTraceSummaryService;
@Autowired
private SelfEvaluationMergeService selfEvaluationMergeService;
@Value("${verifier.low-confidence-threshold:0.5}")
private double verifierLowConfidenceThreshold;
/** 多 Agent Chat 的 Prompt */
private String chatPlannerPrompt;
private String chatExecutorPrompt;
private String chatVerifierPrompt;
private final ObjectMapper objectMapper = new ObjectMapper();
@PostConstruct
public void init() {
@@ -101,6 +120,9 @@ public class ChatService {
chatExecutorPrompt = new String(
new ClassPathResource("prompts/chat-executor-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
chatVerifierPrompt = new String(
new ClassPathResource("prompts/chat-verifier-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
logger.info("Chat 多 Agent Prompts 加载成功");
} catch (IOException e) {
logger.error("加载 Chat Prompt 文件失败", e);
@@ -341,34 +363,71 @@ public class ChatService {
diagnosisSessionRepository.save(session);
SessionContextHolder.setSessionId(sessionId);
VerifierContextHolder.setOriginalQuery(question);
VerifierContextHolder.setRetryContext(null);
VerifierContextHolder.setExecutorFinalAnswer(null);
try {
ReactAgent planner = buildChatPlannerAgent(chatModel, toolCallbacks, history);
ReactAgent executor = buildChatExecutorAgent(chatModel, toolCallbacks, history);
SupervisorAgent supervisor = SupervisorAgent.builder()
.name("chat_supervisor")
.description("负责调度 Planner 与 Executor 的多 Agent 控制器")
.model(chatModel)
.systemPrompt("你是一个智能任务调度器。分析用户问题，调用 Planner 拆解步骤，调用 Executor 执行各步骤。")
.subAgents(List.of(planner, executor))
VerifierDecision finalDecision = null;
String retryContext = null;
String answer = null;
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", sessionId)
.build();
Optional<OverAllState> stateOptional = supervisor.invoke(question);
long duration = System.currentTimeMillis() - startTime;
for (int round = 1; round <= 2; round++) {
VerifierContextHolder.setRetryContext(retryContext);
VerifierContextHolder.setToolTraceSummary(null);
String answer = null;
if (stateOptional.isPresent()) {
// 从 state 中提取 Executor 的最终输出
OverAllState state = stateOptional.get();
Optional<AssistantMessage> executorOutput = state.value("executor_feedback")
.filter(AssistantMessage.class::isInstance)
.map(AssistantMessage.class::cast);
if (executorOutput.isPresent()) {
answer = executorOutput.get().getText();
ReactAgent planner = buildChatPlannerAgent(chatModel, history, retryContext);
ReactAgent executor = buildChatExecutorAgent(chatModel, toolCallbacks, history, retryContext);
ReactAgent verifier = buildChatVerifierAgent(chatModel);
SupervisorAgent supervisor = SupervisorAgent.builder()
.name("chat_supervisor")
.description("负责按单轮顺序调度 Planner、Executor、Verifier 的多 Agent 控制器")
.model(chatModel)
.systemPrompt(buildSupervisorPrompt(round))
.subAgents(List.of(planner, executor, verifier))
.build();
String plannerPlan = callAgent(planner, buildPlannerInput(question, retryContext), config);
answer = callAgent(executor, buildExecutorInput(question, plannerPlan, retryContext), config);
VerifierContextHolder.setExecutorFinalAnswer(answer);
String verifierOutput = callAgent(verifier, "VERIFY", config);
finalDecision = parseVerifierDecision(verifierOutput, round);
if (finalDecision == null) {
finalDecision = buildVerifierFallbackDecision(round, "verifier_output 缺失或无法解析");
answer = buildLowConfidenceOutput(answer, finalDecision);
persistVerifierEvaluation(session, finalDecision, round);
break;
}
if ("PASS".equals(finalDecision.verdict())) {
answer = answer == null || answer.isBlank() ? "抱歉，多 Agent 分析未能生成有效结论。" : answer;
persistVerifierEvaluation(session, finalDecision, round);
break;
}
if ("REJECT".equals(finalDecision.verdict())) {
answer = buildDegradedOutput(finalDecision);
persistVerifierEvaluation(session, finalDecision, round);
break;
}
if (finalDecision.groundednessScore() >= verifierLowConfidenceThreshold || round == 2) {
answer = buildLowConfidenceOutput(answer, finalDecision);
persistVerifierEvaluation(session, finalDecision, round);
break;
}
retryContext = buildRetryContext(finalDecision);
persistVerifierEvaluation(session, finalDecision, round);
}
long duration = System.currentTimeMillis() - startTime;
if (answer == null || answer.isBlank()) {
answer = "抱歉，多 Agent 分析未能生成有效结论。";
}
@@ -394,11 +453,12 @@ public class ChatService {
} finally {
retrievedDocTracker.clearSession(sessionId);
SessionContextHolder.clear();
VerifierContextHolder.clear();
}
}
private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
List<Map<String, String>> history) {
private ReactAgent buildChatPlannerAgent(ChatModel chatModel, List<Map<String, String>> history,
String retryContext) {
StringBuilder prompt = new StringBuilder(chatPlannerPrompt);
// 注入 knowledge map
@@ -414,6 +474,9 @@ public class ChatService {
}
prompt.append("--- 对话历史结束 ---\n");
}
if (retryContext != null && !retryContext.isBlank()) {
prompt.append("\n\n--- 本轮补证据约束 ---\n").append(retryContext).append("\n");
}
return ReactAgent.builder()
.name("chat_planner")
.description("负责拆解问题、规划步骤")
@@ -424,8 +487,20 @@ public class ChatService {
.build();
}
private ReactAgent buildChatVerifierAgent(ChatModel chatModel) {
return ReactAgent.builder()
.name("chat_verifier")
.description("负责验证 Executor 答案的事实准确性")
.model(chatModel)
.systemPrompt(chatVerifierPrompt)
.hooks(new AgentLoggingHook(agentStepRepository, "verifier"),
new VerifierInputHook(toolTraceSummaryService))
.outputKey("verifier_output")
.build();
}
private ReactAgent buildChatExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
List<Map<String, String>> history) {
List<Map<String, String>> history, String retryContext) {
StringBuilder prompt = new StringBuilder(chatExecutorPrompt);
if (!history.isEmpty()) {
prompt.append("\n\n--- 对话历史 ---\n");
@@ -434,6 +509,9 @@ public class ChatService {
}
prompt.append("--- 对话历史结束 ---\n");
}
if (retryContext != null && !retryContext.isBlank()) {
prompt.append("\n\n--- 本轮补证据约束 ---\n").append(retryContext).append("\n");
}
return ReactAgent.builder()
.name("chat_executor")
.description("负责执行具体步骤并及时反馈")
@@ -446,6 +524,316 @@ public class ChatService {
.build();
}
private String callAgent(ReactAgent agent, String input, RunnableConfig config) throws GraphRunnerException {
return agent.call(input, config).getText();
}
private String buildPlannerInput(String question, String retryContext) {
if (retryContext == null || retryContext.isBlank()) {
return question;
}
return question + "\n\n--- 补充约束 ---\n" + retryContext;
}
private String buildExecutorInput(String question, String plannerPlan, String retryContext) {
StringBuilder input = new StringBuilder(question);
if (plannerPlan != null && !plannerPlan.isBlank()) {
input.append("\n\n--- planner_plan ---\n").append(plannerPlan);
}
if (retryContext != null && !retryContext.isBlank()) {
input.append("\n\n--- retry_context ---\n").append(retryContext);
}
return input.toString();
}
private VerifierDecision parseVerifierDecision(String verifierOutput, int round) {
if (verifierOutput == null || verifierOutput.isBlank()) {
return null;
}
try {
JsonNode root = objectMapper.readTree(sanitizeJsonPayload(verifierOutput));
List<Map<String, Object>> factsChecked = parseFactsChecked(root.path("facts_checked"));
return new VerifierDecision(
root.path("verdict").asText("LOW_CONFID"),
root.path("groundedness_score").asDouble(0.0),
root.path("critical_fact_count").asInt(0),
factsChecked,
root.path("rationale").asText(""),
round
);
} catch (Exception e) {
logger.error("解析 verifier_output 失败: {}", verifierOutput, e);
return null;
}
}
private String sanitizeJsonPayload(String raw) {
String trimmed = raw.trim();
if (trimmed.startsWith("```")) {
int firstNewline = trimmed.indexOf('\n');
int lastFence = trimmed.lastIndexOf("```");
if (firstNewline >= 0 && lastFence > firstNewline) {
return trimmed.substring(firstNewline + 1, lastFence).trim();
}
}
return trimmed;
}
private List<Map<String, Object>> parseFactsChecked(JsonNode factsNode) {
List<Map<String, Object>> factsChecked = new ArrayList<>();
if (!factsNode.isArray()) {
return factsChecked;
}
for (JsonNode factNode : factsNode) {
Map<String, Object> fact = new LinkedHashMap<>();
fact.put("fact", factNode.path("fact").asText(""));
fact.put("is_critical", factNode.path("is_critical").asBoolean(false));
fact.put("verification", factNode.path("verification").asText(""));
fact.put("detail", factNode.path("detail").asText(""));
fact.put("evidence_refs", parseEvidenceRefs(factNode.path("evidence_refs")));
factsChecked.add(fact);
}
return factsChecked;
}
private List<Map<String, Object>> parseEvidenceRefs(JsonNode evidenceRefsNode) {
List<Map<String, Object>> evidenceRefs = new ArrayList<>();
if (!evidenceRefsNode.isArray()) {
return evidenceRefs;
}
for (JsonNode refNode : evidenceRefsNode) {
Map<String, Object> evidenceRef = new LinkedHashMap<>();
evidenceRef.put("trace_ref", refNode.path("trace_ref").asText(""));
evidenceRef.put("tool_name", refNode.path("tool_name").asText(""));
evidenceRef.put("topic_domain", refNode.path("topic_domain").asText(""));
evidenceRef.put("note", refNode.path("note").asText(""));
List<Long> sourceInvocationIds = new ArrayList<>();
JsonNode idsNode = refNode.path("source_invocation_ids");
if (idsNode.isArray()) {
for (JsonNode idNode : idsNode) {
if (idNode.canConvertToLong()) {
sourceInvocationIds.add(idNode.asLong());
}
}
}
evidenceRef.put("source_invocation_ids", sourceInvocationIds);
evidenceRefs.add(evidenceRef);
}
return evidenceRefs;
}
private VerifierDecision buildVerifierFallbackDecision(int round, String rationale) {
return new VerifierDecision("LOW_CONFID", 0.0, 0, List.of(), rationale, round);
}
private String buildSupervisorPrompt(int round) {
return """
你是一个多 Agent 调度器。每一轮必须严格按顺序完成以下动作：
1. 先调用 chat_planner 生成执行计划
2. 再调用 chat_executor 执行计划并形成最终答案
3. 最后调用 chat_verifier 对 executor 最终答案做事实核查
规则：
- 本轮只允许完成一次 Planner -> Executor -> Verifier 链路
- Verifier 完成后立即停止，不要继续调用任何 Agent
- 不要自己编造答案，最终用户输出由外层代码根据 verifier_output 决定
- 当前是第 %d 轮，保持单轮内顺序稳定
""".formatted(round);
}
private String buildRoundInput(String question, String retryContext) {
if (retryContext == null || retryContext.isBlank()) {
return question;
}
return question + "\n\n--- 补充约束 ---\n" + retryContext;
}
private String extractExecutorAnswer(Optional<OverAllState> stateOptional) {
if (stateOptional.isEmpty()) {
return null;
}
return stateOptional.get().value("executor_feedback")
.filter(AssistantMessage.class::isInstance)
.map(AssistantMessage.class::cast)
.map(AssistantMessage::getText)
.orElse(null);
}
private VerifierDecision parseVerifierDecision(Optional<OverAllState> stateOptional, int round) {
if (stateOptional.isEmpty()) {
return null;
}
Optional<AssistantMessage> verifierOutput = stateOptional.get().value("verifier_output")
.filter(AssistantMessage.class::isInstance)
.map(AssistantMessage.class::cast);
if (verifierOutput.isEmpty() || verifierOutput.get().getText() == null || verifierOutput.get().getText().isBlank()) {
return null;
}
try {
JsonNode root = objectMapper.readTree(verifierOutput.get().getText());
List<Map<String, Object>> factsChecked = parseFactsChecked(root.path("facts_checked"));
return new VerifierDecision(
root.path("verdict").asText("LOW_CONFID"),
root.path("groundedness_score").asDouble(0.0),
root.path("critical_fact_count").asInt(0),
factsChecked,
root.path("rationale").asText(""),
round
);
} catch (Exception e) {
logger.error("解析 verifier_output 失败: {}", verifierOutput.get().getText(), e);
return null;
}
}
private void persistVerifierEvaluation(DiagnosisSession session, VerifierDecision decision, int round) {
if (decision == null) {
return;
}
Map<String, Object> verifierEvaluation = new LinkedHashMap<>();
verifierEvaluation.put("verdict", decision.verdict());
verifierEvaluation.put("groundedness_score", decision.groundednessScore());
verifierEvaluation.put("critical_fact_count", decision.criticalFactCount());
verifierEvaluation.put("facts_checked", decision.factsChecked());
verifierEvaluation.put("rationale", decision.rationale());
verifierEvaluation.put("round", round);
verifierEvaluation.put("traceability_version", "v1");
verifierEvaluation.put("tool_trace_summary",
Optional.ofNullable(VerifierContextHolder.getToolTraceSummary()).orElse(List.of()));
String merged = selfEvaluationMergeService.mergeVerifierEvaluation(session.getSelfEvaluation(), verifierEvaluation);
session.setSelfEvaluation(merged);
diagnosisSessionRepository.save(session);
}
private String buildRetryContext(VerifierDecision decision) {
try {
List<String> missingFacts = extractEvidenceGaps(decision);
Map<String, Object> retryContext = new LinkedHashMap<>();
retryContext.put("round", decision.round());
retryContext.put("missing_evidence_facts", missingFacts);
retryContext.put("instruction", "仅补充以上断言相关证据，不要重复已完成检索");
return objectMapper.writeValueAsString(retryContext);
} catch (Exception e) {
logger.error("构造 retry_context 失败", e);
return "{\"round\":1,\"missing_evidence_facts\":[],\"instruction\":\"仅补充缺失证据\"}";
}
}
private String buildLowConfidenceOutput(String executorAnswer, VerifierDecision decision) {
StringBuilder output = new StringBuilder(LOW_CONFID_DISCLAIMER);
output.append("\n\n").append(executorAnswer == null ? "" : executorAnswer);
List<String> gaps = extractEvidenceGaps(decision);
if (!gaps.isEmpty()) {
output.append("\n\n当前缺口：");
for (String gap : gaps) {
output.append("\n- ").append(gap);
}
}
return output.toString();
}
private String buildDegradedOutput(VerifierDecision decision) {
StringBuilder output = new StringBuilder(DEGRADED_PREFIX);
List<String> confirmedFacts = extractConfirmedFacts(decision);
List<String> gaps = extractEvidenceGaps(decision);
List<String> suggestions = buildNextStepSuggestions(decision);
output.append("\n\n已确认信息：");
if (confirmedFacts.isEmpty()) {
output.append("\n- 暂无可稳定确认的信息");
} else {
for (String fact : confirmedFacts) {
output.append("\n- ").append(fact);
}
}
output.append("\n\n证据缺口：");
if (gaps.isEmpty()) {
output.append("\n- 当前缺少足够的直接证据支撑核心结论");
} else {
for (String gap : gaps) {
output.append("\n- ").append(gap);
}
}
output.append("\n\n建议下一步：");
for (String suggestion : suggestions) {
output.append("\n- ").append(suggestion);
}
return output.toString();
}
private List<String> extractConfirmedFacts(VerifierDecision decision) {
List<String> confirmedFacts = new ArrayList<>();
for (Map<String, Object> fact : decision.factsChecked()) {
String verification = String.valueOf(fact.get("verification"));
boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
if (critical && ("direct_evidence".equals(verification) || "indirect_support".equals(verification))) {
confirmedFacts.add(String.valueOf(fact.get("fact")));
}
}
return confirmedFacts;
}
private List<String> extractEvidenceGaps(VerifierDecision decision) {
List<String> gaps = new ArrayList<>();
for (Map<String, Object> fact : decision.factsChecked()) {
String verification = String.valueOf(fact.get("verification"));
boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
if (critical && ("no_evidence".equals(verification) || "contradicted".equals(verification))) {
gaps.add(String.valueOf(fact.get("fact")) + "：" + String.valueOf(fact.get("detail")));
}
}
if (gaps.isEmpty() && "LOW_CONFID".equals(decision.verdict())) {
for (Map<String, Object> fact : decision.factsChecked()) {
String verification = String.valueOf(fact.get("verification"));
boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
if (critical && "indirect_support".equals(verification)) {
gaps.add(String.valueOf(fact.get("fact")) + "：缺少直接证据锚点");
}
}
}
return gaps;
}
private List<String> buildNextStepSuggestions(VerifierDecision decision) {
List<String> suggestions = new ArrayList<>();
List<Map<String, Object>> toolSummary = toolTraceSummaryService.buildVerifierTraceSummary(SessionContextHolder.getSessionId(), null);
boolean hasKnowledgeTool = toolSummary.stream().anyMatch(item -> "lookup_knowledge".equals(item.get("tool_name")));
boolean hasFailedEvidence = toolSummary.stream().anyMatch(item -> !Boolean.TRUE.equals(item.get("success")));
if (!hasKnowledgeTool) {
suggestions.add("补充知识库或业务文档检索结果，建立可引用的证据锚点");
}
if (hasFailedEvidence) {
suggestions.add("优先重试失败的证据型查询，补齐日志、指标或知识库侧证据");
}
if (suggestions.isEmpty()) {
suggestions.add("围绕上述证据缺口补充只读查询，再由人工复核最终结论");
}
return suggestions;
}
private record VerifierDecision(
String verdict,
double groundednessScore,
int criticalFactCount,
List<Map<String, Object>> factsChecked,
String rationale,
int round
) {
}
/** 从 agent_step 汇总 token、步数等指标回填 diagnosis_session */
private void backfillSessionMetrics(DiagnosisSession session) {
try {
src/main/java/com/superbiz/agent/service/EvaluationService.java
+14 -19
View File
@@ -1,6 +1,5 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.DiagnosisSession;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.DiagnosisSessionRepository;
@@ -12,6 +11,7 @@ import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -32,17 +32,19 @@ public class EvaluationService {
@Autowired
private ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
@Autowired
private SelfEvaluationMergeService selfEvaluationMergeService;
@Async
public void evaluate(String sessionId, String answer) {
diagnosisSessionRepository.findBySessionId(sessionId).ifPresent(session -> {
try {
List<ToolInvocation> toolInvocations = toolInvocationRepository.findBySessionId(sessionId);
String selfEvaluation = evaluateWithRules(session, toolInvocations);
session.setSelfEvaluation(selfEvaluation);
Map<String, Object> ruleEvaluation = evaluateWithRules(session, toolInvocations);
String merged = selfEvaluationMergeService.mergeRuleEvaluation(session.getSelfEvaluation(), ruleEvaluation);
session.setSelfEvaluation(merged);
diagnosisSessionRepository.save(session);
logger.info("证据评分已写入: sessionId={}, result={}", sessionId, selfEvaluation);
logger.info("证据评分已写入: sessionId={}, result={}", sessionId, merged);
} catch (Exception e) {
logger.error("评分失败: sessionId={}", sessionId, e);
}
@@ -53,7 +55,7 @@ public class EvaluationService {
// 规则引擎（事实层）
// -------------------------------------------------------------------------
private String evaluateWithRules(DiagnosisSession session, List<ToolInvocation> invocations) {
private Map<String, Object> evaluateWithRules(DiagnosisSession session, List<ToolInvocation> invocations) {
List<Map<String, Object>> factors = new ArrayList<>();
if ("FAILED".equals(session.getStatus())) {
@@ -117,19 +119,12 @@ public class EvaluationService {
return Map.of("name", name, "delta", delta, "description", description);
}
private String buildResult(int score, List<Map<String, Object>> factors) {
try {
Map<String, Object> result = Map.of(
"evidence_score", score,
"source", "rule",
"factors", factors
// llm_opinion: null ← 预留字段，LLM 观点叠加时在此处扩展
);
return objectMapper.writeValueAsString(result);
} catch (Exception e) {
logger.error("序列化评分结果失败", e);
return "{\"evidence_score\":0,\"source\":\"rule\",\"factors\":[]}";
}
private Map<String, Object> buildResult(int score, List<Map<String, Object>> factors) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("evidence_score", score);
result.put("source", "rule");
result.put("factors", factors);
return result;
}
// -------------------------------------------------------------------------
src/main/java/com/superbiz/agent/service/SelfEvaluationMergeService.java
+71
View File
@@ -0,0 +1,71 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.LinkedHashMap;
import java.util.Map;
/**
* 统一维护 diagnosis_session.self_evaluation JSON 容器。
*/
@Slf4j
@Service
public class SelfEvaluationMergeService {
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
private final ObjectMapper objectMapper = new ObjectMapper();
public String mergeRuleEvaluation(String existingJson, Map<String, Object> ruleEvaluation) {
return merge(existingJson, "rule_evaluation", ruleEvaluation);
}
public String mergeVerifierEvaluation(String existingJson, Map<String, Object> verifierEvaluation) {
return merge(existingJson, "verifier_evaluation", verifierEvaluation);
}
private String merge(String existingJson, String key, Map<String, Object> value) {
try {
Map<String, Object> root = parseRoot(existingJson);
root.put(key, value);
return objectMapper.writeValueAsString(root);
} catch (Exception e) {
log.error("合并 self_evaluation 失败: key={}", key, e);
return fallbackJson(key, value);
}
}
private Map<String, Object> parseRoot(String existingJson) throws Exception {
if (existingJson == null || existingJson.isBlank()) {
return new LinkedHashMap<>();
}
Map<String, Object> parsed = objectMapper.readValue(existingJson, MAP_TYPE);
if (parsed.containsKey("rule_evaluation") || parsed.containsKey("verifier_evaluation")) {
return new LinkedHashMap<>(parsed);
}
LinkedHashMap<String, Object> wrapped = new LinkedHashMap<>();
if (parsed.containsKey("evidence_score") || parsed.containsKey("source") || parsed.containsKey("factors")) {
wrapped.put("rule_evaluation", parsed);
return wrapped;
}
if (parsed.containsKey("verdict") || parsed.containsKey("groundedness_score") || parsed.containsKey("facts_checked")) {
wrapped.put("verifier_evaluation", parsed);
return wrapped;
}
return new LinkedHashMap<>(parsed);
}
private String fallbackJson(String key, Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(Map.of(key, value));
} catch (Exception ex) {
log.error("兜底序列化 self_evaluation 失败", ex);
return "{}";
}
}
}
src/main/java/com/superbiz/agent/service/ToolTraceSummaryService.java
+307
View File
@@ -0,0 +1,307 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
/**
* Builds a verifier-facing evidence index from persisted tool invocations.
*/
@Slf4j
@Service
public class ToolTraceSummaryService {
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
private static final Set<String> EVIDENCE_TOOLS = Set.of("lookup_knowledge", "query_logs", "query_metrics", "query_order");
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
public ToolTraceSummaryService(ToolInvocationRepository toolInvocationRepository) {
this.toolInvocationRepository = toolInvocationRepository;
}
public List<Map<String, Object>> buildVerifierTraceSummary(String sessionId, String executorFinalAnswer) {
if (sessionId == null || sessionId.isBlank()) {
return List.of();
}
List<ToolInvocation> invocations = toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId);
if (invocations.isEmpty()) {
return List.of();
}
Map<String, AggregateEntry> grouped = new LinkedHashMap<>();
for (ToolInvocation invocation : invocations) {
if (!EVIDENCE_TOOLS.contains(invocation.getToolName())) {
continue;
}
String topicDomain = extractTopicDomain(invocation);
String key = invocation.getToolName() + "|" + topicDomain;
AggregateEntry entry = grouped.computeIfAbsent(
key,
ignored -> new AggregateEntry(invocation.getToolName(), topicDomain));
entry.absorb(invocation);
}
List<AggregateEntry> rankedEntries = grouped.values().stream()
.sorted(Comparator.comparingInt((AggregateEntry entry) -> entry.relevanceScore(executorFinalAnswer)).reversed())
.limit(8)
.toList();
List<Map<String, Object>> summaries = new ArrayList<>();
for (int i = 0; i < rankedEntries.size(); i++) {
summaries.add(rankedEntries.get(i).toSummary("trace-" + (i + 1)));
}
return summaries;
}
private String extractTopicDomain(ToolInvocation invocation) {
try {
if (invocation.getRetrievalDetails() != null && !invocation.getRetrievalDetails().isBlank()) {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
Object domains = details.get("retrieved_domains");
if (domains instanceof List<?> domainList && !domainList.isEmpty()) {
return String.valueOf(domainList.get(0));
}
}
} catch (Exception e) {
log.debug("Failed to parse retrieved_domains, fallback to general", e);
}
return "general";
}
private String extractInputSummary(ToolInvocation invocation) {
String query = extractQuery(invocation);
if (query != null && !query.isBlank()) {
return "query=" + truncate(query, 120);
}
return invocation.getToolName() + " invoked";
}
private String extractQuery(ToolInvocation invocation) {
try {
if (invocation.getInputParams() != null && !invocation.getInputParams().isBlank()) {
Map<String, Object> params = objectMapper.readValue(invocation.getInputParams(), MAP_TYPE);
Object query = params.get("query");
if (query != null) {
return String.valueOf(query);
}
}
} catch (Exception e) {
log.debug("Failed to parse invocation query", e);
}
return null;
}
private String extractOutputSummary(ToolInvocation invocation, String topicDomain) {
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
if (invocation.getErrorMessage() != null && !invocation.getErrorMessage().isBlank()) {
return "call failed: " + truncate(invocation.getErrorMessage(), 120);
}
return "no usable evidence returned";
}
if ("lookup_knowledge".equals(invocation.getToolName())) {
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
? truncate(invocation.getOutputPreview(), 160)
: "no preview";
return "matched domain=" + topicDomain + ", relevance=" + relevance + ", preview=" + preview;
}
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return truncate(invocation.getOutputPreview(), 160);
}
return "evidence retrieved without preview";
}
private String determineEvidenceLevel(ToolInvocation invocation) {
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
return "none";
}
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
return "direct";
}
if ("REFERENCE".equals(invocation.getRelevanceLevel())) {
return "indirect";
}
return "none";
}
private List<String> extractStringList(Object value) {
if (!(value instanceof List<?> list) || list.isEmpty()) {
return List.of();
}
List<String> result = new ArrayList<>();
for (Object item : list) {
if (item != null) {
result.add(String.valueOf(item));
}
}
return result;
}
private List<String> extractSourceDocuments(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return List.of();
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
List<String> paths = extractStringList(details.get("l0_paths"));
if (!paths.isEmpty()) {
return paths;
}
List<String> titles = extractStringList(details.get("l0_titles"));
if (!titles.isEmpty()) {
return titles;
}
} catch (Exception e) {
log.debug("Failed to parse source documents", e);
}
return List.of();
}
private String truncate(String text, int maxLength) {
if (text == null) {
return "";
}
return text.length() <= maxLength ? text : text.substring(0, maxLength) + "...";
}
private final class AggregateEntry {
private final String toolName;
private final String topicDomain;
private String inputSummary;
private String outputSummary;
private boolean success;
private String evidenceLevel = "none";
private int invocationCount;
private int failedCount;
private int noHitCount;
private final List<Long> sourceInvocationIds = new ArrayList<>();
private final LinkedHashSet<String> querySamples = new LinkedHashSet<>();
private final LinkedHashSet<String> retrievalLayers = new LinkedHashSet<>();
private final LinkedHashSet<String> relevanceLevels = new LinkedHashSet<>();
private final LinkedHashSet<String> sourceDocuments = new LinkedHashSet<>();
private AggregateEntry(String toolName, String topicDomain) {
this.toolName = toolName;
this.topicDomain = topicDomain;
}
void absorb(ToolInvocation invocation) {
invocationCount++;
if (invocation.getId() != null) {
sourceInvocationIds.add(invocation.getId());
}
String query = extractQuery(invocation);
if (query != null && !query.isBlank()) {
querySamples.add(query);
}
if (invocation.getRetrievalLayer() != null && !invocation.getRetrievalLayer().isBlank()) {
retrievalLayers.add(invocation.getRetrievalLayer());
}
if (invocation.getRelevanceLevel() != null && !invocation.getRelevanceLevel().isBlank()) {
relevanceLevels.add(invocation.getRelevanceLevel());
}
sourceDocuments.addAll(extractSourceDocuments(invocation));
if (inputSummary == null || inputSummary.isBlank()) {
inputSummary = extractInputSummary(invocation);
}
boolean invocationSuccess = Boolean.TRUE.equals(invocation.getSuccess());
if (!invocationSuccess) {
failedCount++;
return;
}
if (invocation.getDedupReason() != null) {
noHitCount++;
}
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
if (!success || evidenceRank(invocationEvidenceLevel) > evidenceRank(evidenceLevel)) {
success = true;
evidenceLevel = invocationEvidenceLevel;
outputSummary = extractOutputSummary(invocation, topicDomain);
}
}
int relevanceScore(String answer) {
int score = success ? 10 : 0;
if ("direct".equals(evidenceLevel)) {
score += 10;
} else if ("indirect".equals(evidenceLevel)) {
score += 5;
}
if (answer != null) {
String normalized = answer.toLowerCase(Locale.ROOT);
if (normalized.contains(topicDomain.toLowerCase(Locale.ROOT))) {
score += 8;
}
if (normalized.contains(toolName.toLowerCase(Locale.ROOT))) {
score += 3;
}
}
return score;
}
Map<String, Object> toSummary(String traceRef) {
String mergedOutput = outputSummary == null ? "no summarized evidence" : outputSummary;
if (invocationCount > 1) {
StringBuilder builder = new StringBuilder(mergedOutput);
builder.append(" (merged ").append(invocationCount).append(" invocations");
if (failedCount > 0) {
builder.append(", failed=").append(failedCount);
}
if (noHitCount > 0) {
builder.append(", no_hit=").append(noHitCount);
}
builder.append(")");
mergedOutput = builder.toString();
}
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("trace_ref", traceRef);
summary.put("tool_name", toolName);
summary.put("success", success);
summary.put("input_summary", inputSummary);
summary.put("output_summary", mergedOutput);
summary.put("evidence_level", evidenceLevel);
summary.put("topic_domain", topicDomain);
summary.put("source_invocation_ids", new ArrayList<>(sourceInvocationIds));
summary.put("invocation_count", invocationCount);
summary.put("failed_invocation_count", failedCount);
summary.put("no_hit_invocation_count", noHitCount);
summary.put("query_samples", new ArrayList<>(querySamples));
summary.put("retrieval_layers", new ArrayList<>(retrievalLayers));
summary.put("relevance_levels", new ArrayList<>(relevanceLevels));
summary.put("source_documents", new ArrayList<>(sourceDocuments));
return summary;
}
private int evidenceRank(String level) {
if ("direct".equals(level)) {
return 2;
}
if ("indirect".equals(level)) {
return 1;
}
return 0;
}
}
}
src/main/java/com/superbiz/agent/util/VerifierContextHolder.java
+57
View File
@@ -0,0 +1,57 @@
package com.superbiz.agent.util;
import java.util.List;
import java.util.Map;
/**
* Thread-local verifier context shared across one planner/executor/verifier round.
*/
public final class VerifierContextHolder {
private static final ThreadLocal<String> ORIGINAL_QUERY = new ThreadLocal<>();
private static final ThreadLocal<String> RETRY_CONTEXT = new ThreadLocal<>();
private static final ThreadLocal<String> EXECUTOR_FINAL_ANSWER = new ThreadLocal<>();
private static final ThreadLocal<List<Map<String, Object>>> TOOL_TRACE_SUMMARY = new ThreadLocal<>();
private VerifierContextHolder() {
}
public static void setOriginalQuery(String originalQuery) {
ORIGINAL_QUERY.set(originalQuery);
}
public static String getOriginalQuery() {
return ORIGINAL_QUERY.get();
}
public static void setRetryContext(String retryContext) {
RETRY_CONTEXT.set(retryContext);
}
public static String getRetryContext() {
return RETRY_CONTEXT.get();
}
public static void setExecutorFinalAnswer(String executorFinalAnswer) {
EXECUTOR_FINAL_ANSWER.set(executorFinalAnswer);
}
public static String getExecutorFinalAnswer() {
return EXECUTOR_FINAL_ANSWER.get();
}
public static void setToolTraceSummary(List<Map<String, Object>> toolTraceSummary) {
TOOL_TRACE_SUMMARY.set(toolTraceSummary);
}
public static List<Map<String, Object>> getToolTraceSummary() {
return TOOL_TRACE_SUMMARY.get();
}
public static void clear() {
ORIGINAL_QUERY.remove();
RETRY_CONTEXT.remove();
EXECUTOR_FINAL_ANSWER.remove();
TOOL_TRACE_SUMMARY.remove();
}
}