refactor(harness): remove legacy agent architecture
This commit is contained in:
zhuyongxin
@@ -1,27 +0,0 @@
|
||||
package com.superbiz.agent.agent.tool;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.context.i18n.LocaleContextHolder;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Component
|
||||
public class DateTimeTools {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DateTimeTools.class);
|
||||
|
||||
/** 工具名常量，用于动态构建提示词 */
|
||||
public static final String TOOL_GET_CURRENT_DATETIME = "getCurrentDateTime";
|
||||
|
||||
@Tool(description = "Get the current date and time in the user's timezone. " +
|
||||
"IMPORTANT: Time changes constantly. Always call this tool when user asks about time, " +
|
||||
"even if there's a recent time query in the conversation history.")
|
||||
public String getCurrentDateTime() {
|
||||
String currentTime = LocalDateTime.now().atZone(LocaleContextHolder.getTimeZone().toZoneId()).toString();
|
||||
logger.debug("🕐 getCurrentDateTime 调用 - 返回时间: {}", currentTime);
|
||||
return currentTime;
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package com.superbiz.agent.agent.tool;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 内部文档查询工具
|
||||
* 使用 RAG (Retrieval-Augmented Generation) 从内部知识库检索相关文档
|
||||
*
|
||||
* @deprecated 请使用 {@link com.superbiz.agent.tool.LookupKnowledgeTool} 替代。
|
||||
* lookup_knowledge 支持 L0 精确匹配 + L1 语义检索，性能更优且功能更全面。
|
||||
* 计划在下一个版本中移除此工具。
|
||||
*/
|
||||
@Deprecated
|
||||
@Component
|
||||
public class InternalDocsTools {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(InternalDocsTools.class);
|
||||
|
||||
/** 工具名常量，用于动态构建提示词 */
|
||||
public static final String TOOL_QUERY_INTERNAL_DOCS = "queryInternalDocs";
|
||||
|
||||
private final VectorSearchService vectorSearchService;
|
||||
|
||||
@Value("${rag.top-k:3}")
|
||||
private int topK = 3; // 默认值
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
/**
|
||||
* 构造函数注入依赖
|
||||
* Spring 会自动注入 VectorSearchService
|
||||
*/
|
||||
@Autowired
|
||||
public InternalDocsTools(VectorSearchService vectorSearchService) {
|
||||
this.vectorSearchService = vectorSearchService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询内部文档工具
|
||||
*
|
||||
* @param query 搜索查询，描述您要查找的信息
|
||||
* @return JSON 格式的搜索结果，包含相关文档内容、相似度分数和元数据
|
||||
* @deprecated 请使用 {@link com.superbiz.agent.tool.LookupKnowledgeTool#lookupKnowledge(String)} 替代
|
||||
*/
|
||||
@Deprecated
|
||||
@Tool(description = "Use this tool to search internal documentation and knowledge base for relevant information. " +
|
||||
"It performs RAG (Retrieval-Augmented Generation) to find similar documents and extract processing steps. " +
|
||||
"This is useful when you need to understand internal procedures, best practices, or step-by-step guides " +
|
||||
"stored in the company's documentation.")
|
||||
public String queryInternalDocs(
|
||||
@ToolParam(description = "Search query describing what information you are looking for")
|
||||
String query) {
|
||||
|
||||
|
||||
try {
|
||||
// 使用向量搜索服务检索相关文档
|
||||
List<VectorSearchService.SearchResult> searchResults =
|
||||
vectorSearchService.searchSimilarDocuments(query, topK);
|
||||
|
||||
if (searchResults.isEmpty()) {
|
||||
return "{\"status\": \"no_results\", \"message\": \"No relevant documents found in the knowledge base.\"}";
|
||||
}
|
||||
|
||||
// 将搜索结果转换为 JSON 格式
|
||||
String resultJson = objectMapper.writeValueAsString(searchResults);
|
||||
|
||||
|
||||
return resultJson;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("[工具错误] queryInternalDocs 执行失败", e);
|
||||
return String.format("{\"status\": \"error\", \"message\": \"Failed to query internal docs: %s\"}",
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,9 @@ package com.superbiz.agent.agent.tool;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -30,16 +27,7 @@ public class QueryLogsTools {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(QueryLogsTools.class);
|
||||
|
||||
/** 工具名常量，用于动态构建提示词 */
|
||||
public static final String TOOL_QUERY_LOGS = "queryLogs";
|
||||
public static final String TOOL_GET_AVAILABLE_LOG_TOPICS = "getAvailableLogTopics";
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolInvocationRecorder toolInvocationRecorder;
|
||||
|
||||
public QueryLogsTools(ToolInvocationRecorder toolInvocationRecorder) {
|
||||
this.toolInvocationRecorder = toolInvocationRecorder;
|
||||
}
|
||||
|
||||
@Value("${cls.mock-enabled:false}")
|
||||
private boolean mockEnabled;
|
||||
@@ -53,97 +41,6 @@ public class QueryLogsTools {
|
||||
logger.info("✅ QueryLogsTools 初始化成功, Mock模式: {}", mockEnabled);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取可用的日志主题列表
|
||||
* 用于查询前先了解有哪些日志主题可供查询
|
||||
*/
|
||||
@Tool(description = "Get all available log topics and their descriptions. " +
|
||||
"Call this tool first before querying logs to understand what log topics are available. " +
|
||||
"Returns a list of log topics with their names, descriptions, and example queries.")
|
||||
public String getAvailableLogTopics() {
|
||||
long startTime = System.currentTimeMillis();
|
||||
logger.info("获取可用的日志主题列表");
|
||||
|
||||
try {
|
||||
List<LogTopicInfo> topics = new ArrayList<>();
|
||||
|
||||
// 系统指标日志
|
||||
LogTopicInfo systemMetrics = new LogTopicInfo();
|
||||
systemMetrics.setTopicName("system-metrics");
|
||||
systemMetrics.setDescription("系统指标日志，包含 CPU、内存、磁盘使用率等系统资源监控数据");
|
||||
systemMetrics.setExampleQueries(List.of(
|
||||
"cpu_usage:>80",
|
||||
"memory_usage:>85",
|
||||
"disk_usage:>90",
|
||||
"level:WARN AND service:payment-service"
|
||||
));
|
||||
systemMetrics.setRelatedAlerts(List.of("HighCPUUsage", "HighMemoryUsage", "HighDiskUsage"));
|
||||
topics.add(systemMetrics);
|
||||
|
||||
// 应用日志
|
||||
LogTopicInfo applicationLogs = new LogTopicInfo();
|
||||
applicationLogs.setTopicName("application-logs");
|
||||
applicationLogs.setDescription("应用日志，包含应用程序的错误日志、警告日志、慢请求日志、下游依赖调用日志等");
|
||||
applicationLogs.setExampleQueries(List.of(
|
||||
"level:ERROR",
|
||||
"level:FATAL",
|
||||
"http_status:500",
|
||||
"response_time:>3000",
|
||||
"slow",
|
||||
"downstream OR redis OR database OR mq"
|
||||
));
|
||||
applicationLogs.setRelatedAlerts(List.of("ServiceUnavailable", "SlowResponse", "HighMemoryUsage"));
|
||||
topics.add(applicationLogs);
|
||||
|
||||
// 数据库慢查询日志
|
||||
LogTopicInfo dbSlowQuery = new LogTopicInfo();
|
||||
dbSlowQuery.setTopicName("database-slow-query");
|
||||
dbSlowQuery.setDescription("数据库慢查询日志，包含执行时间较长的 SQL 查询，可用于分析数据库性能问题");
|
||||
dbSlowQuery.setExampleQueries(List.of(
|
||||
"query_time:>2",
|
||||
"table:orders",
|
||||
"query_type:SELECT",
|
||||
"*" // 查询所有慢查询
|
||||
));
|
||||
dbSlowQuery.setRelatedAlerts(List.of("SlowResponse", "ServiceUnavailable"));
|
||||
topics.add(dbSlowQuery);
|
||||
|
||||
// 系统事件日志
|
||||
LogTopicInfo systemEvents = new LogTopicInfo();
|
||||
systemEvents.setTopicName("system-events");
|
||||
systemEvents.setDescription("系统事件日志，包含 Kubernetes Pod 重启、OOM Kill、容器崩溃等系统级事件");
|
||||
systemEvents.setExampleQueries(List.of(
|
||||
"restart OR crash",
|
||||
"oom_kill",
|
||||
"event_type:PodRestart",
|
||||
"reason:OOMKilled"
|
||||
));
|
||||
systemEvents.setRelatedAlerts(List.of("ServiceUnavailable", "HighMemoryUsage"));
|
||||
topics.add(systemEvents);
|
||||
|
||||
// 构建输出
|
||||
LogTopicsOutput output = new LogTopicsOutput();
|
||||
output.setSuccess(true);
|
||||
output.setTopics(topics);
|
||||
output.setAvailableRegions(List.of("ap-guangzhou", "ap-shanghai", "ap-beijing", "ap-chengdu"));
|
||||
output.setDefaultRegion("ap-guangzhou");
|
||||
|
||||
output.setMessage(String.format("共有 %d 个可用的日志主题。建议使用默认地域 'ap-guangzhou' 或省略 region 参数", topics.size()));
|
||||
|
||||
String response = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
|
||||
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
|
||||
response, true, null, "logs", ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
|
||||
return response;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("获取日志主题列表失败", e);
|
||||
String response = "{\"success\":false,\"message\":\"获取日志主题列表失败: " + e.getMessage() + "\"}";
|
||||
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
|
||||
response, false, e.getMessage(), "logs", ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询日志
|
||||
* 从云日志服务查询指定条件的日志
|
||||
@@ -160,24 +57,9 @@ public class QueryLogsTools {
|
||||
|
||||
private static final String DEFAULT_REGION = "ap-guangzhou";
|
||||
|
||||
@Tool(description = "Query logs from Cloud Log Service (CLS). " +
|
||||
"Use this tool to search application logs, system metrics, and other log data. " +
|
||||
"IMPORTANT: Before calling this tool, you should call getAvailableLogTopics to understand what log topics are available. " +
|
||||
"Available log topics: " +
|
||||
"1) 'system-metrics' - System metrics logs (CPU, memory, disk usage, etc. Related to HighCPUUsage, HighMemoryUsage, HighDiskUsage alerts); " +
|
||||
"2) 'application-logs' - Application logs (error logs, slow request logs, downstream dependency logs. Related to ServiceUnavailable, SlowResponse alerts); " +
|
||||
"3) 'database-slow-query' - Database slow query logs (SQL queries with long execution time. Related to SlowResponse alerts); " +
|
||||
"4) 'system-events' - System event logs (Pod restart, OOM Kill, container crash. Related to ServiceUnavailable, HighMemoryUsage alerts). " +
|
||||
"logTopic (required, one of the above topics or their CLS topicId), " +
|
||||
"query (optional, defaults to a curated search if empty), " +
|
||||
"limit (optional, default 20, max 100).")
|
||||
public String queryLogs(
|
||||
@ToolParam(description = "地域，可选值: ap-guangzhou, ap-shanghai, ap-beijing, ap-chengdu。默认 ap-guangzhou") String region,
|
||||
@ToolParam(description = "日志主题，如 system-metrics, application-logs, database-slow-query, system-events，也支持 CLS TopicId") String logTopic,
|
||||
@ToolParam(description = "查询条件，支持 Lucene 语法，如 level:ERROR OR cpu_usage:>80；为空时返回该主题近 5 条核心日志") String query,
|
||||
@ToolParam(description = "返回日志条数，默认20，最大100") Integer limit) {
|
||||
String region, String logTopic, String query, Integer limit) {
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
int actualLimit = (limit == null || limit <= 0) ? 20 : Math.min(limit, 100);
|
||||
|
||||
String safeQuery = query == null ? "" : query;
|
||||
@@ -193,15 +75,12 @@ public class QueryLogsTools {
|
||||
} else {
|
||||
// 真实模式：调用 CLS API（这里预留接口，后续实现）
|
||||
String response = buildErrorResponse("CLS 真实查询尚未实现，请启用 mock 模式进行测试");
|
||||
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
|
||||
"CLS 真实查询尚未实现，请启用 mock 模式进行测试", normalizeTopicDomain(logTopic),
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
|
||||
return response;
|
||||
}
|
||||
|
||||
// 构建成功响应
|
||||
QueryLogsOutput output = new QueryLogsOutput();
|
||||
output.setSuccess(!logEntries.isEmpty());
|
||||
output.setSuccess(true);
|
||||
output.setRegion(region);
|
||||
output.setLogTopic(logTopic);
|
||||
output.setQuery(safeQuery.isBlank() ? "DEFAULT_QUERY" : safeQuery);
|
||||
@@ -211,56 +90,15 @@ public class QueryLogsTools {
|
||||
|
||||
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
|
||||
logger.info("日志查询完成: 找到 {} 条日志", logEntries.size());
|
||||
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, jsonResult,
|
||||
true, null, normalizeTopicDomain(logTopic),
|
||||
logEntries.isEmpty()
|
||||
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
|
||||
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
|
||||
|
||||
return jsonResult;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("查询日志失败", e);
|
||||
String response = buildErrorResponse("查询失败: " + e.getMessage());
|
||||
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
|
||||
e.getMessage(), normalizeTopicDomain(logTopic), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private void recordInvocation(String toolName, long startTime, String query, String region, String logTopic, Integer limit,
|
||||
String output, boolean success, String errorMessage, String topicDomain,
|
||||
String evidenceStatus) {
|
||||
Map<String, Object> input = new HashMap<>();
|
||||
input.put("query", query == null || query.isBlank() ? "DEFAULT_QUERY" : query);
|
||||
if (region != null) {
|
||||
input.put("region", region);
|
||||
}
|
||||
if (logTopic != null) {
|
||||
input.put("log_topic", logTopic);
|
||||
}
|
||||
if (limit != null) {
|
||||
input.put("limit", limit);
|
||||
}
|
||||
input.put("mock_enabled", mockEnabled);
|
||||
|
||||
toolInvocationRecorder.recordEvidenceTool(
|
||||
toolName,
|
||||
input,
|
||||
output,
|
||||
success,
|
||||
startTime,
|
||||
errorMessage,
|
||||
topicDomain,
|
||||
evidenceStatus,
|
||||
Map.of("log_topic", logTopic == null ? "" : logTopic)
|
||||
);
|
||||
}
|
||||
|
||||
private String normalizeTopicDomain(String logTopic) {
|
||||
return logTopic == null || logTopic.isBlank() ? "logs" : logTopic;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 Mock 日志数据
|
||||
|
||||
@@ -764,42 +602,4 @@ public class QueryLogsTools {
|
||||
private String message;
|
||||
}
|
||||
|
||||
/**
|
||||
* 日志主题信息
|
||||
*/
|
||||
@Data
|
||||
public static class LogTopicInfo {
|
||||
@JsonProperty("topic_name")
|
||||
private String topicName;
|
||||
|
||||
@JsonProperty("description")
|
||||
private String description;
|
||||
|
||||
@JsonProperty("example_queries")
|
||||
private List<String> exampleQueries;
|
||||
|
||||
@JsonProperty("related_alerts")
|
||||
private List<String> relatedAlerts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 日志主题列表输出
|
||||
*/
|
||||
@Data
|
||||
public static class LogTopicsOutput {
|
||||
@JsonProperty("success")
|
||||
private boolean success;
|
||||
|
||||
@JsonProperty("topics")
|
||||
private List<LogTopicInfo> topics;
|
||||
|
||||
@JsonProperty("available_regions")
|
||||
private List<String> availableRegions;
|
||||
|
||||
@JsonProperty("default_region")
|
||||
private String defaultRegion;
|
||||
|
||||
@JsonProperty("message")
|
||||
private String message;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,332 +0,0 @@
|
||||
package com.superbiz.agent.agent.tool;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import lombok.Data;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Prometheus 告警查询工具
|
||||
* 用于查询 Prometheus 的活动告警信息
|
||||
*/
|
||||
@Component
|
||||
public class QueryMetricsTools {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(QueryMetricsTools.class);
|
||||
|
||||
/** 工具名常量，用于动态构建提示词 */
|
||||
public static final String TOOL_QUERY_PROMETHEUS_ALERTS = "queryPrometheusAlerts";
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolInvocationRecorder toolInvocationRecorder;
|
||||
|
||||
public QueryMetricsTools(ToolInvocationRecorder toolInvocationRecorder) {
|
||||
this.toolInvocationRecorder = toolInvocationRecorder;
|
||||
}
|
||||
|
||||
@Value("${prometheus.base-url}")
|
||||
private String prometheusBaseUrl;
|
||||
|
||||
@Value("${prometheus.timeout:10}")
|
||||
private int timeout;
|
||||
|
||||
@Value("${prometheus.mock-enabled:false}")
|
||||
private boolean mockEnabled;
|
||||
|
||||
private OkHttpClient httpClient;
|
||||
|
||||
@jakarta.annotation.PostConstruct
|
||||
public void init() {
|
||||
this.httpClient = new OkHttpClient.Builder()
|
||||
.connectTimeout(Duration.ofSeconds(timeout))
|
||||
.readTimeout(Duration.ofSeconds(timeout))
|
||||
.build();
|
||||
logger.info("✅ QueryMetricsTools 初始化成功, Prometheus URL: {}, Mock模式: {}", prometheusBaseUrl, mockEnabled);
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询 Prometheus 活动告警
|
||||
* 该工具从 Prometheus 告警系统检索所有当前活动/触发的告警，包括标签、注释、状态和值
|
||||
*/
|
||||
@Tool(description = "Query active alerts from Prometheus alerting system. " +
|
||||
"This tool retrieves all currently active/firing alerts including their labels, annotations, state, and values. " +
|
||||
"Use this tool when you need to check what alerts are currently firing, investigate alert conditions, or monitor alert status.")
|
||||
public String queryPrometheusAlerts() {
|
||||
long startTime = System.currentTimeMillis();
|
||||
logger.info("开始查询 Prometheus 活动告警, Mock模式: {}", mockEnabled);
|
||||
|
||||
try {
|
||||
List<SimplifiedAlert> simplifiedAlerts;
|
||||
|
||||
if (mockEnabled) {
|
||||
// Mock 模式：返回与文档关联的模拟告警数据
|
||||
simplifiedAlerts = buildMockAlerts();
|
||||
logger.info("使用 Mock 数据，返回 {} 个模拟告警", simplifiedAlerts.size());
|
||||
} else {
|
||||
// 真实模式：调用 Prometheus Alerts API
|
||||
PrometheusAlertsResult result = fetchPrometheusAlerts();
|
||||
|
||||
if (!"success".equals(result.getStatus())) {
|
||||
String response = buildErrorResponse("Prometheus API 返回非成功状态: " + result.getStatus(), result.getError());
|
||||
recordInvocation(startTime, response, false, result.getError(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
|
||||
return response;
|
||||
}
|
||||
|
||||
// 转换为简化格式，对于相同的 alertname，只保留第一个
|
||||
Set<String> seenAlertNames = new HashSet<>();
|
||||
simplifiedAlerts = new ArrayList<>();
|
||||
|
||||
for (PrometheusAlert alert : result.getData().getAlerts()) {
|
||||
String alertName = alert.getLabels().get("alertname");
|
||||
|
||||
// 如果这个 alertname 已经存在，跳过
|
||||
if (seenAlertNames.contains(alertName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 标记为已见过
|
||||
seenAlertNames.add(alertName);
|
||||
|
||||
SimplifiedAlert simplified = new SimplifiedAlert();
|
||||
simplified.setAlertName(alertName);
|
||||
simplified.setDescription(alert.getAnnotations().getOrDefault("description", ""));
|
||||
simplified.setState(alert.getState());
|
||||
simplified.setActiveAt(alert.getActiveAt());
|
||||
simplified.setDuration(calculateDuration(alert.getActiveAt()));
|
||||
|
||||
simplifiedAlerts.add(simplified);
|
||||
}
|
||||
}
|
||||
|
||||
// 构建成功响应
|
||||
PrometheusAlertsOutput output = new PrometheusAlertsOutput();
|
||||
output.setSuccess(true);
|
||||
output.setAlerts(simplifiedAlerts);
|
||||
output.setMessage(String.format("成功检索到 %d 个活动告警", simplifiedAlerts.size()));
|
||||
|
||||
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
|
||||
logger.info("Prometheus 告警查询完成: 找到 {} 个告警", simplifiedAlerts.size());
|
||||
recordInvocation(startTime, jsonResult, true, null,
|
||||
simplifiedAlerts.isEmpty()
|
||||
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
|
||||
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
|
||||
|
||||
return jsonResult;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("查询 Prometheus 告警失败", e);
|
||||
String response = buildErrorResponse("查询失败", e.getMessage());
|
||||
recordInvocation(startTime, response, false, e.getMessage(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
private void recordInvocation(long startTime, String output, boolean success, String errorMessage, String evidenceStatus) {
|
||||
toolInvocationRecorder.recordEvidenceTool(
|
||||
"query_metrics",
|
||||
Map.of("query", "active_prometheus_alerts", "mock_enabled", mockEnabled),
|
||||
output,
|
||||
success,
|
||||
startTime,
|
||||
errorMessage,
|
||||
"prometheus_alerts",
|
||||
evidenceStatus,
|
||||
Map.of("metric_family", "prometheus_alerts")
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 Mock 告警数据
|
||||
* 与 aiops-docs 文档中的告警类型对应：
|
||||
* - HighCPUUsage: CPU使用率过高
|
||||
* - HighMemoryUsage: 内存使用率过高
|
||||
* - HighDiskUsage: 磁盘使用率过高
|
||||
* - ServiceUnavailable: 服务不可用
|
||||
* - SlowResponse: 响应时间过长
|
||||
*/
|
||||
private List<SimplifiedAlert> buildMockAlerts() {
|
||||
List<SimplifiedAlert> alerts = new ArrayList<>();
|
||||
Instant now = Instant.now();
|
||||
|
||||
// 告警1: CPU使用率过高 - 持续约25分钟
|
||||
SimplifiedAlert cpuAlert = new SimplifiedAlert();
|
||||
cpuAlert.setAlertName("HighCPUUsage");
|
||||
cpuAlert.setDescription("服务 payment-service 的 CPU 使用率持续超过 80%，当前值为 92%。" +
|
||||
"实例: pod-payment-service-7d8f9c6b5-x2k4m，命名空间: production");
|
||||
cpuAlert.setState("firing");
|
||||
Instant cpuActiveAt = now.minus(25, ChronoUnit.MINUTES);
|
||||
cpuAlert.setActiveAt(cpuActiveAt.toString());
|
||||
cpuAlert.setDuration(calculateDuration(cpuActiveAt.toString()));
|
||||
alerts.add(cpuAlert);
|
||||
|
||||
// 告警2: 内存使用率过高 - 持续约15分钟
|
||||
SimplifiedAlert memoryAlert = new SimplifiedAlert();
|
||||
memoryAlert.setAlertName("HighMemoryUsage");
|
||||
memoryAlert.setDescription("服务 order-service 的内存使用率持续超过 85%，当前值为 91%。" +
|
||||
"JVM堆内存使用: 3.8GB/4GB，可能存在内存泄漏风险。" +
|
||||
"实例: pod-order-service-5c7d8e9f1-m3n2p，命名空间: production");
|
||||
memoryAlert.setState("firing");
|
||||
Instant memoryActiveAt = now.minus(15, ChronoUnit.MINUTES);
|
||||
memoryAlert.setActiveAt(memoryActiveAt.toString());
|
||||
memoryAlert.setDuration(calculateDuration(memoryActiveAt.toString()));
|
||||
alerts.add(memoryAlert);
|
||||
|
||||
// 告警3: 响应时间过长 - 持续约10分钟
|
||||
SimplifiedAlert slowAlert = new SimplifiedAlert();
|
||||
slowAlert.setAlertName("SlowResponse");
|
||||
slowAlert.setDescription("服务 user-service 的 P99 响应时间持续超过 3 秒，当前值为 4.2 秒。" +
|
||||
"受影响接口: /api/v1/users/profile, /api/v1/users/orders。" +
|
||||
"可能原因：数据库慢查询或下游服务延迟");
|
||||
slowAlert.setState("firing");
|
||||
Instant slowActiveAt = now.minus(10, ChronoUnit.MINUTES);
|
||||
slowAlert.setActiveAt(slowActiveAt.toString());
|
||||
slowAlert.setDuration(calculateDuration(slowActiveAt.toString()));
|
||||
alerts.add(slowAlert);
|
||||
|
||||
return alerts;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Prometheus API 获取告警数据
|
||||
*/
|
||||
private PrometheusAlertsResult fetchPrometheusAlerts() throws Exception {
|
||||
String apiUrl = prometheusBaseUrl + "/api/v1/alerts";
|
||||
logger.debug("请求 Prometheus API: {}", apiUrl);
|
||||
|
||||
Request request = new Request.Builder()
|
||||
.url(apiUrl)
|
||||
.get()
|
||||
.build();
|
||||
|
||||
try (Response response = httpClient.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
throw new RuntimeException("HTTP 请求失败: " + response.code());
|
||||
}
|
||||
|
||||
String responseBody = response.body().string();
|
||||
return objectMapper.readValue(responseBody, PrometheusAlertsResult.class);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算从 activeAt 到现在的持续时间
|
||||
*/
|
||||
private String calculateDuration(String activeAtStr) {
|
||||
try {
|
||||
Instant activeAt = Instant.parse(activeAtStr);
|
||||
Duration duration = Duration.between(activeAt, Instant.now());
|
||||
|
||||
long hours = duration.toHours();
|
||||
long minutes = duration.toMinutes() % 60;
|
||||
long seconds = duration.getSeconds() % 60;
|
||||
|
||||
if (hours > 0) {
|
||||
return String.format("%dh%dm%ds", hours, minutes, seconds);
|
||||
} else if (minutes > 0) {
|
||||
return String.format("%dm%ds", minutes, seconds);
|
||||
} else {
|
||||
return String.format("%ds", seconds);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.warn("解析时间失败: {}", activeAtStr, e);
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建错误响应
|
||||
*/
|
||||
private String buildErrorResponse(String message, String error) {
|
||||
try {
|
||||
PrometheusAlertsOutput output = new PrometheusAlertsOutput();
|
||||
output.setSuccess(false);
|
||||
output.setMessage(message);
|
||||
output.setError(error);
|
||||
return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
|
||||
} catch (Exception e) {
|
||||
return String.format("{\"success\":false,\"message\":\"%s\",\"error\":\"%s\"}", message, error);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 数据模型 ====================
|
||||
|
||||
/**
|
||||
* Prometheus 告警信息结构
|
||||
*/
|
||||
@Data
|
||||
public static class PrometheusAlert {
|
||||
private Map<String, String> labels;
|
||||
private Map<String, String> annotations;
|
||||
private String state;
|
||||
private String activeAt;
|
||||
private String value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prometheus 告警查询结果
|
||||
*/
|
||||
@Data
|
||||
public static class PrometheusAlertsResult {
|
||||
private String status;
|
||||
private AlertsData data;
|
||||
private String error;
|
||||
private String errorType;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class AlertsData {
|
||||
private List<PrometheusAlert> alerts = new ArrayList<>();
|
||||
}
|
||||
|
||||
/**
|
||||
* 简化的告警信息
|
||||
*/
|
||||
@Data
|
||||
public static class SimplifiedAlert {
|
||||
@JsonProperty("alert_name")
|
||||
private String alertName;
|
||||
|
||||
@JsonProperty("description")
|
||||
private String description;
|
||||
|
||||
@JsonProperty("state")
|
||||
private String state;
|
||||
|
||||
@JsonProperty("active_at")
|
||||
private String activeAt;
|
||||
|
||||
@JsonProperty("duration")
|
||||
private String duration;
|
||||
}
|
||||
|
||||
/**
|
||||
* 告警查询输出
|
||||
*/
|
||||
@Data
|
||||
public static class PrometheusAlertsOutput {
|
||||
@JsonProperty("success")
|
||||
private boolean success;
|
||||
|
||||
@JsonProperty("alerts")
|
||||
private List<SimplifiedAlert> alerts;
|
||||
|
||||
@JsonProperty("message")
|
||||
private String message;
|
||||
|
||||
@JsonProperty("error")
|
||||
private String error;
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* AI Ops Agent Prompt 配置
|
||||
* 从独立的 Markdown 文件加载 Prompt 模板
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
public class AiOpsPromptProperties {
|
||||
|
||||
private String planner;
|
||||
private String executor;
|
||||
private String supervisor;
|
||||
|
||||
@PostConstruct
|
||||
public void loadPrompts() {
|
||||
try {
|
||||
planner = loadPromptFromFile("prompts/planner-prompt.md");
|
||||
executor = loadPromptFromFile("prompts/executor-prompt.md");
|
||||
supervisor = loadPromptFromFile("prompts/supervisor-prompt.md");
|
||||
|
||||
log.info("AI Ops Prompts 加载成功");
|
||||
log.debug("Planner Prompt 长度: {} 字符", planner.length());
|
||||
log.debug("Executor Prompt 长度: {} 字符", executor.length());
|
||||
log.debug("Supervisor Prompt 长度: {} 字符", supervisor.length());
|
||||
} catch (IOException e) {
|
||||
log.error("加载 Prompt 文件失败", e);
|
||||
throw new RuntimeException("Failed to load AI Ops prompts", e);
|
||||
}
|
||||
}
|
||||
|
||||
private String loadPromptFromFile(String path) throws IOException {
|
||||
ClassPathResource resource = new ClassPathResource(path);
|
||||
return new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public String getPlanner() {
|
||||
return planner;
|
||||
}
|
||||
|
||||
public String getExecutor() {
|
||||
return executor;
|
||||
}
|
||||
|
||||
public String getSupervisor() {
|
||||
return supervisor;
|
||||
}
|
||||
}
|
||||
@@ -49,7 +49,8 @@ import com.superbiz.agent.harness.application.DiagnosisOperation;
|
||||
import com.superbiz.agent.harness.application.KnowledgeQueryOperation;
|
||||
import com.superbiz.agent.harness.application.SystemChatOperation;
|
||||
import com.superbiz.agent.harness.application.IntentRouting;
|
||||
import com.superbiz.agent.hook.AgentLoggingHook;
|
||||
import com.superbiz.agent.harness.audit.HarnessAgentAuditHook;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.agent.tool.QueryLogsTools;
|
||||
@@ -137,8 +138,9 @@ public class HarnessChatConfiguration {
|
||||
|
||||
@Bean
|
||||
public ToolBoundary toolBoundary(DiagnosisHarnessCore core, ToolCallKeyFactory keyFactory,
|
||||
CanonicalInvocationStore store, ObjectMapper objectMapper, Clock clock) {
|
||||
return new ToolBoundary(core, keyFactory, store, objectMapper, clock);
|
||||
CanonicalInvocationStore store, ObjectMapper objectMapper, Clock clock,
|
||||
ToolInvocationAuditSink auditSink) {
|
||||
return new ToolBoundary(core, keyFactory, store, objectMapper, clock, auditSink);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -158,17 +160,17 @@ public class HarnessChatConfiguration {
|
||||
|
||||
@Bean
|
||||
public RagToolAdapter ragToolAdapter(ToolBoundary boundary, ObjectMapper mapper,
|
||||
RagResultProjector projector, LookupKnowledgeTool legacy) {
|
||||
return new RagToolAdapter(boundary, mapper, projector, legacy::lookupKnowledge);
|
||||
RagResultProjector projector, LookupKnowledgeTool backend) {
|
||||
return new RagToolAdapter(boundary, mapper, projector, backend::lookupKnowledge);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public QueryLogsToolAdapter queryLogsToolAdapter(ToolBoundary boundary, ObjectMapper mapper,
|
||||
QueryLogsResultProjector projector,
|
||||
ObjectProvider<QueryLogsTools> legacy, Clock clock) {
|
||||
ObjectProvider<QueryLogsTools> backend, Clock clock) {
|
||||
return new QueryLogsToolAdapter(boundary, mapper, projector,
|
||||
(region, topic, query, limit) -> {
|
||||
QueryLogsTools tools = legacy.getIfAvailable();
|
||||
QueryLogsTools tools = backend.getIfAvailable();
|
||||
if (tools == null) {
|
||||
return "{\"success\":false,\"logs\":[],\"total\":0}";
|
||||
}
|
||||
@@ -230,10 +232,11 @@ public class HarnessChatConfiguration {
|
||||
|
||||
@Bean
|
||||
public DiagnosisAgentFactory diagnosisAgentFactory(ChatModel chatModel, DiagnosisHarnessCore core,
|
||||
HarnessEvidenceTools tools, ObjectMapper mapper,
|
||||
AgentStepRepository steps) {
|
||||
HarnessEvidenceTools tools, ObjectMapper mapper,
|
||||
AgentStepRepository steps) {
|
||||
return new DiagnosisAgentFactory(chatModel, core, tools, mapper,
|
||||
List.of(new AgentLoggingHook(steps, DiagnosisAgentFactory.AGENT_NAME)));
|
||||
List.of(new HarnessAgentAuditHook(
|
||||
steps, mapper, DiagnosisAgentFactory.AGENT_NAME)));
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@@ -33,6 +33,6 @@ public class WebConfig implements WebMvcConfigurer {
|
||||
|
||||
@Bean
|
||||
public ObjectMapper objectMapper() {
|
||||
return new ObjectMapper();
|
||||
return new ObjectMapper().findAndRegisterModules();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.OverAllState;
|
||||
import com.superbiz.agent.dto.AIOpsRequest;
|
||||
import com.superbiz.agent.service.AiOpsService;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
|
||||
/** Preserves the legacy AiOps SSE protocol independently from Chat. */
|
||||
@RestController
|
||||
@RequestMapping("/api")
|
||||
public class AiOpsController {
|
||||
|
||||
private final AiOpsService aiOpsService;
|
||||
private final ThreadPoolExecutor workerExecutor;
|
||||
|
||||
public AiOpsController(AiOpsService aiOpsService,
|
||||
@Qualifier("chatWorkerExecutor") ThreadPoolExecutor workerExecutor) {
|
||||
this.aiOpsService = aiOpsService;
|
||||
this.workerExecutor = workerExecutor;
|
||||
}
|
||||
|
||||
@PostMapping(value = "/ai_ops", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
|
||||
public SseEmitter aiOps(@RequestBody(required = false) AIOpsRequest request) {
|
||||
SseEmitter emitter = new SseEmitter(600_000L);
|
||||
String sessionId = aiOpsService.resolveSessionId(request);
|
||||
String runId = aiOpsService.newRunId();
|
||||
|
||||
emitter.onTimeout(emitter::complete);
|
||||
try {
|
||||
workerExecutor.execute(() -> executeAiOps(request, sessionId, runId, emitter));
|
||||
} catch (RejectedExecutionException rejected) {
|
||||
emitter.completeWithError(rejected);
|
||||
}
|
||||
return emitter;
|
||||
}
|
||||
|
||||
private void executeAiOps(AIOpsRequest request, String sessionId, String runId, SseEmitter emitter) {
|
||||
try {
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.metadata(sessionId, runId), MediaType.APPLICATION_JSON));
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.content("正在读取告警并拆解任务...\n"), MediaType.APPLICATION_JSON));
|
||||
|
||||
Optional<OverAllState> state = aiOpsService.executeAiOpsAnalysis(request, sessionId, runId);
|
||||
if (state.isEmpty()) {
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.error("多 Agent 编排未获取到有效结果"), MediaType.APPLICATION_JSON));
|
||||
emitter.complete();
|
||||
return;
|
||||
}
|
||||
|
||||
Optional<String> report = aiOpsService.extractFinalReport(state.get());
|
||||
if (report.isPresent()) {
|
||||
aiOpsService.persistFinalReport(sessionId, runId, report.get(), request);
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.content(report.get()), MediaType.APPLICATION_JSON));
|
||||
} else {
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.content("多 Agent 流程已完成，但未能生成最终报告。"), MediaType.APPLICATION_JSON));
|
||||
}
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.done(), MediaType.APPLICATION_JSON));
|
||||
emitter.complete();
|
||||
} catch (Exception exception) {
|
||||
try {
|
||||
emitter.send(SseEmitter.event().name("message")
|
||||
.data(SseMessage.error("AI Ops 流程失败"), MediaType.APPLICATION_JSON));
|
||||
} catch (IOException ignored) {
|
||||
// Client has already gone away.
|
||||
}
|
||||
emitter.completeWithError(exception);
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class SseMessage {
|
||||
private String type;
|
||||
private String data;
|
||||
private String sessionId;
|
||||
private String runId;
|
||||
|
||||
public static SseMessage content(String data) {
|
||||
SseMessage message = new SseMessage();
|
||||
message.type = "content";
|
||||
message.data = data;
|
||||
return message;
|
||||
}
|
||||
|
||||
public static SseMessage metadata(String sessionId, String runId) {
|
||||
SseMessage message = new SseMessage();
|
||||
message.type = "metadata";
|
||||
message.sessionId = sessionId;
|
||||
message.runId = runId;
|
||||
return message;
|
||||
}
|
||||
|
||||
public static SseMessage error(String data) {
|
||||
SseMessage message = new SseMessage();
|
||||
message.type = "error";
|
||||
message.data = data;
|
||||
return message;
|
||||
}
|
||||
|
||||
public static SseMessage done() {
|
||||
SseMessage message = new SseMessage();
|
||||
message.type = "done";
|
||||
return message;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
||||
import com.superbiz.agent.service.DiagnosisTraceService;
|
||||
import com.superbiz.agent.service.session.SessionManager;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/** Legacy Chat session management endpoints, isolated from the Chat execution adapter. */
|
||||
@RestController
|
||||
@RequestMapping("/api/chat")
|
||||
public class ChatSessionController {
|
||||
|
||||
private final DiagnosisTraceService diagnosisTraceService;
|
||||
private final SessionManager sessionManager;
|
||||
|
||||
public ChatSessionController(DiagnosisTraceService diagnosisTraceService, SessionManager sessionManager) {
|
||||
this.diagnosisTraceService = diagnosisTraceService;
|
||||
this.sessionManager = sessionManager;
|
||||
}
|
||||
|
||||
@PostMapping("/clear")
|
||||
public ResponseEntity<ApiResponse<String>> clearChatHistory(@RequestBody ClearRequest request) {
|
||||
if (request == null || request.getId() == null || request.getId().isBlank()) {
|
||||
return ResponseEntity.ok(ApiResponse.error("会话ID不能为空"));
|
||||
}
|
||||
try {
|
||||
Optional<com.superbiz.agent.domain.model.SessionContext> session =
|
||||
sessionManager.getSession(request.getId());
|
||||
if (session.isEmpty()) {
|
||||
return ResponseEntity.ok(ApiResponse.error("会话不存在"));
|
||||
}
|
||||
var context = session.get();
|
||||
context.clearMessageHistory();
|
||||
sessionManager.updateSession(context);
|
||||
return ResponseEntity.ok(ApiResponse.success("会话历史已清空"));
|
||||
} catch (RuntimeException exception) {
|
||||
return ResponseEntity.ok(ApiResponse.error("会话历史清理失败"));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/session/{sessionId}")
|
||||
public ResponseEntity<ApiResponse<SessionInfoResponse>> getSessionInfo(@PathVariable String sessionId) {
|
||||
try {
|
||||
Optional<com.superbiz.agent.domain.model.SessionContext> session = sessionManager.getSession(sessionId);
|
||||
if (session.isEmpty()) {
|
||||
return ResponseEntity.ok(ApiResponse.error("会话不存在"));
|
||||
}
|
||||
var context = session.get();
|
||||
SessionInfoResponse response = new SessionInfoResponse();
|
||||
response.setSessionId(sessionId);
|
||||
response.setMessagePairCount(context.getMessagePairCount());
|
||||
response.setCreateTime(toEpochMillis(context.getCreatedAt()));
|
||||
return ResponseEntity.ok(ApiResponse.success(response));
|
||||
} catch (RuntimeException exception) {
|
||||
return ResponseEntity.ok(ApiResponse.error("会话读取失败"));
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/session/{sessionId}/runs")
|
||||
public ResponseEntity<ApiResponse<List<DiagnosisTraceResponse.RunSummary>>> listSessionRuns(
|
||||
@PathVariable String sessionId) {
|
||||
return ResponseEntity.ok(ApiResponse.success(diagnosisTraceService.listRunSummaries(sessionId)));
|
||||
}
|
||||
|
||||
private long toEpochMillis(LocalDateTime time) {
|
||||
return time == null ? 0L : time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class ClearRequest {
|
||||
@JsonProperty("Id")
|
||||
@JsonAlias({"id", "ID"})
|
||||
private String id;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class SessionInfoResponse {
|
||||
private String sessionId;
|
||||
private int messagePairCount;
|
||||
private long createTime;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
public static class ApiResponse<T> {
|
||||
private int code;
|
||||
private String message;
|
||||
private T data;
|
||||
|
||||
public static <T> ApiResponse<T> success(T data) {
|
||||
ApiResponse<T> response = new ApiResponse<>();
|
||||
response.code = 200;
|
||||
response.message = "success";
|
||||
response.data = data;
|
||||
return response;
|
||||
}
|
||||
|
||||
public static <T> ApiResponse<T> error(String message) {
|
||||
ApiResponse<T> response = new ApiResponse<>();
|
||||
response.code = 500;
|
||||
response.message = message;
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* Chat session metadata entity.
|
||||
* Full message history remains in Redis SessionContext.
|
||||
* Stores current Chat Run session metadata; message history is not persisted here.
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "chat_session", indexes = {
|
||||
@@ -61,4 +61,3 @@ public class ChatSession {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -51,11 +51,11 @@ public class DiagnosisRun {
|
||||
private String answer;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "intent", length = 32)
|
||||
@Column(name = "intent", length = 32, columnDefinition = "VARCHAR(32)")
|
||||
private IntentType intent;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "release_outcome", length = 16)
|
||||
@Column(name = "release_outcome", length = 16, columnDefinition = "VARCHAR(16)")
|
||||
private ReleaseOutcome releaseOutcome;
|
||||
|
||||
@JdbcTypeCode(SqlTypes.JSON)
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
package com.superbiz.agent.domain.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 会话上下文数据类
|
||||
* 存储在 Redis 中的会话数据
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class SessionContext implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
/**
|
||||
* 会话ID
|
||||
*/
|
||||
private String sessionId;
|
||||
|
||||
/**
|
||||
* 用户ID
|
||||
*/
|
||||
private String userId;
|
||||
|
||||
/**
|
||||
* 业务ID（订单号/请求ID等）
|
||||
*/
|
||||
private String businessId;
|
||||
|
||||
/**
|
||||
* 链路追踪ID
|
||||
*/
|
||||
private String traceId;
|
||||
|
||||
/**
|
||||
* 会话状态（ACTIVE/COMPLETED/EXPIRED）
|
||||
*/
|
||||
private String status;
|
||||
|
||||
/**
|
||||
* 工具调用历史
|
||||
*/
|
||||
@Builder.Default
|
||||
private List<ToolCall> toolCalls = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 聊天消息历史：[{"role":"user","content":"..."}, {"role":"assistant","content":"..."}]
|
||||
*/
|
||||
@Builder.Default
|
||||
private List<Map<String, String>> messageHistory = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* 会话创建时间
|
||||
*/
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
/**
|
||||
* 最后活跃时间
|
||||
*/
|
||||
private LocalDateTime lastActiveAt;
|
||||
|
||||
/**
|
||||
* 会话过期时间（秒）
|
||||
*/
|
||||
private Long ttl;
|
||||
|
||||
/**
|
||||
* 添加工具调用记录
|
||||
*/
|
||||
public void addToolCall(ToolCall toolCall) {
|
||||
if (this.toolCalls == null) {
|
||||
this.toolCalls = new ArrayList<>();
|
||||
}
|
||||
this.toolCalls.add(toolCall);
|
||||
this.lastActiveAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加一对聊天消息，并按消息对数裁剪窗口。
|
||||
*/
|
||||
public void addChatMessagePair(String userQuestion, String assistantAnswer, int maxPairCount) {
|
||||
if (this.messageHistory == null) {
|
||||
this.messageHistory = new ArrayList<>();
|
||||
}
|
||||
|
||||
Map<String, String> userMessage = new HashMap<>();
|
||||
userMessage.put("role", "user");
|
||||
userMessage.put("content", userQuestion);
|
||||
this.messageHistory.add(userMessage);
|
||||
|
||||
Map<String, String> assistantMessage = new HashMap<>();
|
||||
assistantMessage.put("role", "assistant");
|
||||
assistantMessage.put("content", assistantAnswer);
|
||||
this.messageHistory.add(assistantMessage);
|
||||
|
||||
int maxMessages = Math.max(maxPairCount, 0) * 2;
|
||||
while (maxMessages > 0 && this.messageHistory.size() > maxMessages) {
|
||||
this.messageHistory.remove(0);
|
||||
if (!this.messageHistory.isEmpty()) {
|
||||
this.messageHistory.remove(0);
|
||||
}
|
||||
}
|
||||
this.lastActiveAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取聊天历史副本，避免调用方直接修改内部列表。
|
||||
*/
|
||||
@JsonIgnore
|
||||
public List<Map<String, String>> getMessageHistorySnapshot() {
|
||||
if (this.messageHistory == null || this.messageHistory.isEmpty()) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
List<Map<String, String>> snapshot = new ArrayList<>();
|
||||
for (Map<String, String> message : this.messageHistory) {
|
||||
snapshot.add(new HashMap<>(message));
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空聊天历史。
|
||||
*/
|
||||
public void clearMessageHistory() {
|
||||
if (this.messageHistory == null) {
|
||||
this.messageHistory = new ArrayList<>();
|
||||
} else {
|
||||
this.messageHistory.clear();
|
||||
}
|
||||
this.lastActiveAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
public int getMessagePairCount() {
|
||||
return this.messageHistory == null ? 0 : this.messageHistory.size() / 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新最后活跃时间
|
||||
*/
|
||||
public void updateLastActiveTime() {
|
||||
this.lastActiveAt = LocalDateTime.now();
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/**
|
||||
* AIOps 请求 DTO
|
||||
*/
|
||||
@Data
|
||||
public class AIOpsRequest {
|
||||
|
||||
/**
|
||||
* 诊断会话 ID；为空时后端自动生成。
|
||||
*/
|
||||
private String sessionId;
|
||||
|
||||
/**
|
||||
* 告警名称。
|
||||
*/
|
||||
private String alertName;
|
||||
|
||||
/**
|
||||
* 受影响服务。
|
||||
*/
|
||||
private String service;
|
||||
|
||||
/**
|
||||
* 告警等级，例如 P0/P1/P2。
|
||||
*/
|
||||
private String severity;
|
||||
|
||||
/**
|
||||
* 告警描述。
|
||||
*/
|
||||
private String description;
|
||||
|
||||
/**
|
||||
* 排查时间范围，例如 last_15m。
|
||||
*/
|
||||
private String timeRange;
|
||||
|
||||
/**
|
||||
* 用户请求描述
|
||||
*/
|
||||
private String userRequest;
|
||||
}
|
||||
@@ -140,7 +140,6 @@ public class DiagnosisTraceResponse {
|
||||
private int persistedToolCallCount;
|
||||
private int returnedToolCallCount;
|
||||
private boolean hasVerifierEvaluation;
|
||||
private boolean hasAiOpsRuleEvaluation;
|
||||
private boolean hasFeedback;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class DiagnosisEvalBaselineDiffer {
|
||||
|
||||
private static final String REGRESSION = "REGRESSION";
|
||||
private static final String IMPROVEMENT = "IMPROVEMENT";
|
||||
private static final String CHANGED = "CHANGED";
|
||||
|
||||
public DiagnosisEvalDiffReport compare(DiagnosisEvalReport baseline, DiagnosisEvalReport current) {
|
||||
List<DiagnosisEvalDiffItem> items = new ArrayList<>();
|
||||
|
||||
compareDouble(items, "aggregate", null, "passRate",
|
||||
baseline.getPassRate(), current.getPassRate(), true);
|
||||
compareDouble(items, "aggregate", null, "averageToolCallCount",
|
||||
baseline.getAverageToolCallCount(), current.getAverageToolCallCount(), false);
|
||||
compareDouble(items, "aggregate", null, "averageDurationMs",
|
||||
baseline.getAverageDurationMs(), current.getAverageDurationMs(), false);
|
||||
compareVerdictDistribution(items, baseline.getVerdictDistribution(), current.getVerdictDistribution());
|
||||
compareCases(items, safeResults(baseline), safeResults(current));
|
||||
|
||||
int regressionCount = countType(items, REGRESSION);
|
||||
int improvementCount = countType(items, IMPROVEMENT);
|
||||
int changedCount = countType(items, CHANGED);
|
||||
|
||||
return DiagnosisEvalDiffReport.builder()
|
||||
.baselineTotalCases(baseline.getTotalCases())
|
||||
.currentTotalCases(current.getTotalCases())
|
||||
.baselinePassedCases(baseline.getPassedCases())
|
||||
.currentPassedCases(current.getPassedCases())
|
||||
.baselinePassRate(baseline.getPassRate())
|
||||
.currentPassRate(current.getPassRate())
|
||||
.regressionCount(regressionCount)
|
||||
.improvementCount(improvementCount)
|
||||
.changedCount(changedCount)
|
||||
.hasRegression(regressionCount > 0)
|
||||
.items(items)
|
||||
.build();
|
||||
}
|
||||
|
||||
private void compareVerdictDistribution(List<DiagnosisEvalDiffItem> items,
|
||||
Map<String, Long> baseline,
|
||||
Map<String, Long> current) {
|
||||
Set<String> verdicts = new LinkedHashSet<>();
|
||||
verdicts.addAll(safeMap(baseline).keySet());
|
||||
verdicts.addAll(safeMap(current).keySet());
|
||||
for (String verdict : verdicts) {
|
||||
long baselineCount = safeMap(baseline).getOrDefault(verdict, 0L);
|
||||
long currentCount = safeMap(current).getOrDefault(verdict, 0L);
|
||||
if (baselineCount != currentCount) {
|
||||
items.add(item(CHANGED, "aggregate", null, "verdictDistribution." + verdict,
|
||||
String.valueOf(baselineCount), String.valueOf(currentCount),
|
||||
(double) currentCount - baselineCount,
|
||||
"verdict count changed for " + verdict));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void compareCases(List<DiagnosisEvalDiffItem> items,
|
||||
List<DiagnosisEvalResult> baselineResults,
|
||||
List<DiagnosisEvalResult> currentResults) {
|
||||
Map<String, DiagnosisEvalResult> baselineById = byCaseId(baselineResults);
|
||||
Map<String, DiagnosisEvalResult> currentById = byCaseId(currentResults);
|
||||
Set<String> caseIds = new LinkedHashSet<>();
|
||||
caseIds.addAll(baselineById.keySet());
|
||||
caseIds.addAll(currentById.keySet());
|
||||
|
||||
for (String caseId : caseIds) {
|
||||
DiagnosisEvalResult baseline = baselineById.get(caseId);
|
||||
DiagnosisEvalResult current = currentById.get(caseId);
|
||||
if (baseline == null) {
|
||||
items.add(item(CHANGED, "case", caseId, "casePresence",
|
||||
"missing", "present", null, "new case appears in current report"));
|
||||
continue;
|
||||
}
|
||||
if (current == null) {
|
||||
items.add(item(REGRESSION, "case", caseId, "casePresence",
|
||||
"present", "missing", null, "baseline case is missing from current report"));
|
||||
continue;
|
||||
}
|
||||
|
||||
comparePassState(items, baseline, current);
|
||||
compareVerdict(items, baseline, current);
|
||||
compareInteger(items, caseId, "matchedKeywordCount",
|
||||
baseline.getMatchedKeywordCount(), current.getMatchedKeywordCount(), true);
|
||||
compareInteger(items, caseId, "toolCallCount",
|
||||
baseline.getToolCallCount(), current.getToolCallCount(), false);
|
||||
compareInteger(items, caseId, "durationMs",
|
||||
baseline.getDurationMs(), current.getDurationMs(), false);
|
||||
compareEvidenceCoverage(items, baseline, current);
|
||||
}
|
||||
}
|
||||
|
||||
private void comparePassState(List<DiagnosisEvalDiffItem> items,
|
||||
DiagnosisEvalResult baseline,
|
||||
DiagnosisEvalResult current) {
|
||||
if (baseline.isPassed() == current.isPassed()) {
|
||||
return;
|
||||
}
|
||||
String type = baseline.isPassed() ? REGRESSION : IMPROVEMENT;
|
||||
items.add(item(type, "case", baseline.getCaseId(), "passed",
|
||||
String.valueOf(baseline.isPassed()), String.valueOf(current.isPassed()), null,
|
||||
baseline.getCaseId() + " pass state changed"));
|
||||
}
|
||||
|
||||
private void compareVerdict(List<DiagnosisEvalDiffItem> items,
|
||||
DiagnosisEvalResult baseline,
|
||||
DiagnosisEvalResult current) {
|
||||
if (Objects.equals(baseline.getVerdict(), current.getVerdict())) {
|
||||
return;
|
||||
}
|
||||
int baselineRank = verdictRank(baseline.getVerdict());
|
||||
int currentRank = verdictRank(current.getVerdict());
|
||||
String type = currentRank < baselineRank ? REGRESSION : currentRank > baselineRank ? IMPROVEMENT : CHANGED;
|
||||
items.add(item(type, "case", baseline.getCaseId(), "verdict",
|
||||
value(baseline.getVerdict()), value(current.getVerdict()), (double) currentRank - baselineRank,
|
||||
baseline.getCaseId() + " verdict changed"));
|
||||
}
|
||||
|
||||
private void compareEvidenceCoverage(List<DiagnosisEvalDiffItem> items,
|
||||
DiagnosisEvalResult baseline,
|
||||
DiagnosisEvalResult current) {
|
||||
Set<String> tools = new LinkedHashSet<>();
|
||||
tools.addAll(safeMap(baseline.getEvidenceCoverage()).keySet());
|
||||
tools.addAll(safeMap(current.getEvidenceCoverage()).keySet());
|
||||
for (String tool : tools) {
|
||||
boolean baselineCovered = Boolean.TRUE.equals(safeMap(baseline.getEvidenceCoverage()).get(tool));
|
||||
boolean currentCovered = Boolean.TRUE.equals(safeMap(current.getEvidenceCoverage()).get(tool));
|
||||
if (baselineCovered == currentCovered) {
|
||||
continue;
|
||||
}
|
||||
String type = baselineCovered ? REGRESSION : IMPROVEMENT;
|
||||
items.add(item(type, "case", baseline.getCaseId(), "evidenceCoverage." + tool,
|
||||
String.valueOf(baselineCovered), String.valueOf(currentCovered), null,
|
||||
baseline.getCaseId() + " evidence coverage changed for " + tool));
|
||||
}
|
||||
}
|
||||
|
||||
private void compareDouble(List<DiagnosisEvalDiffItem> items,
|
||||
String scope,
|
||||
String caseId,
|
||||
String metric,
|
||||
double baseline,
|
||||
double current,
|
||||
boolean higherIsBetter) {
|
||||
if (Double.compare(baseline, current) == 0) {
|
||||
return;
|
||||
}
|
||||
double delta = current - baseline;
|
||||
String type = classifyDelta(delta, higherIsBetter);
|
||||
items.add(item(type, scope, caseId, metric,
|
||||
String.valueOf(baseline), String.valueOf(current), delta,
|
||||
metric + " changed"));
|
||||
}
|
||||
|
||||
private void compareInteger(List<DiagnosisEvalDiffItem> items,
|
||||
String caseId,
|
||||
String metric,
|
||||
Integer baseline,
|
||||
Integer current,
|
||||
boolean higherIsBetter) {
|
||||
if (Objects.equals(baseline, current)) {
|
||||
return;
|
||||
}
|
||||
if (baseline == null || current == null) {
|
||||
items.add(item(CHANGED, "case", caseId, metric,
|
||||
value(baseline), value(current), null, caseId + " " + metric + " changed"));
|
||||
return;
|
||||
}
|
||||
int delta = current - baseline;
|
||||
items.add(item(classifyDelta(delta, higherIsBetter), "case", caseId, metric,
|
||||
String.valueOf(baseline), String.valueOf(current), (double) delta,
|
||||
caseId + " " + metric + " changed"));
|
||||
}
|
||||
|
||||
private String classifyDelta(double delta, boolean higherIsBetter) {
|
||||
if (delta == 0.0) {
|
||||
return CHANGED;
|
||||
}
|
||||
boolean improved = higherIsBetter ? delta > 0 : delta < 0;
|
||||
return improved ? IMPROVEMENT : REGRESSION;
|
||||
}
|
||||
|
||||
private DiagnosisEvalDiffItem item(String type,
|
||||
String scope,
|
||||
String caseId,
|
||||
String metric,
|
||||
String baselineValue,
|
||||
String currentValue,
|
||||
Double delta,
|
||||
String message) {
|
||||
return DiagnosisEvalDiffItem.builder()
|
||||
.type(type)
|
||||
.scope(scope)
|
||||
.caseId(caseId)
|
||||
.metric(metric)
|
||||
.baselineValue(baselineValue)
|
||||
.currentValue(currentValue)
|
||||
.delta(delta)
|
||||
.message(message)
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, DiagnosisEvalResult> byCaseId(List<DiagnosisEvalResult> results) {
|
||||
return results.stream()
|
||||
.sorted(Comparator.comparing(DiagnosisEvalResult::getCaseId))
|
||||
.collect(Collectors.toMap(
|
||||
DiagnosisEvalResult::getCaseId,
|
||||
Function.identity(),
|
||||
(left, right) -> right,
|
||||
LinkedHashMap::new));
|
||||
}
|
||||
|
||||
private List<DiagnosisEvalResult> safeResults(DiagnosisEvalReport report) {
|
||||
return report.getResults() == null ? List.of() : report.getResults();
|
||||
}
|
||||
|
||||
private <T> Map<String, T> safeMap(Map<String, T> value) {
|
||||
return value == null ? Map.of() : value;
|
||||
}
|
||||
|
||||
private int countType(List<DiagnosisEvalDiffItem> items, String type) {
|
||||
return (int) items.stream().filter(item -> type.equals(item.getType())).count();
|
||||
}
|
||||
|
||||
private int verdictRank(String verdict) {
|
||||
if ("PASS".equals(verdict)) {
|
||||
return 3;
|
||||
}
|
||||
if ("LOW_CONFID".equals(verdict)) {
|
||||
return 2;
|
||||
}
|
||||
if ("REJECT".equals(verdict)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private String value(Object value) {
|
||||
return value == null ? "-" : String.valueOf(value);
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DiagnosisEvalCase {
|
||||
|
||||
private String id;
|
||||
private String title;
|
||||
private String question;
|
||||
private String traceFixture;
|
||||
private List<String> expectedRootCauseKeywords;
|
||||
private Integer minKeywordMatches;
|
||||
private List<String> requiredEvidenceTools;
|
||||
private List<String> allowedVerdicts;
|
||||
private List<String> forbiddenAnswerKeywords;
|
||||
private Boolean requireV2AuditClosure;
|
||||
private Boolean requireClaimChecks;
|
||||
private Boolean requireComposerOutput;
|
||||
private List<String> expectedGatekeeperStatuses;
|
||||
private String expectedGatekeeperRuleSetVersion;
|
||||
private List<String> expectedComposerStatuses;
|
||||
private List<String> forbiddenConfirmedClaimKeywords;
|
||||
private Boolean requirePromptAudit;
|
||||
private String expectedPromptAuditVersion;
|
||||
private Map<String, String> expectedPromptVersions;
|
||||
private Boolean requireGatekeeperRules;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DiagnosisEvalDiffItem {
|
||||
|
||||
private String type;
|
||||
private String scope;
|
||||
private String caseId;
|
||||
private String metric;
|
||||
private String baselineValue;
|
||||
private String currentValue;
|
||||
private Double delta;
|
||||
private String message;
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DiagnosisEvalDiffReport {
|
||||
|
||||
private int baselineTotalCases;
|
||||
private int currentTotalCases;
|
||||
private int baselinePassedCases;
|
||||
private int currentPassedCases;
|
||||
private double baselinePassRate;
|
||||
private double currentPassRate;
|
||||
private int regressionCount;
|
||||
private int improvementCount;
|
||||
private int changedCount;
|
||||
private boolean hasRegression;
|
||||
private List<DiagnosisEvalDiffItem> items;
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class DiagnosisEvalDiffReportWriter {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public DiagnosisEvalDiffReportWriter(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public void writeJson(DiagnosisEvalDiffReport report, Path outputFile) throws IOException {
|
||||
Files.createDirectories(outputFile.getParent());
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
|
||||
}
|
||||
|
||||
public void writeMarkdown(DiagnosisEvalDiffReport report, Path outputFile) throws IOException {
|
||||
Files.createDirectories(outputFile.getParent());
|
||||
Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public String toMarkdown(DiagnosisEvalDiffReport report) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("# Diagnosis Eval Baseline Diff\n\n");
|
||||
builder.append("- Baseline pass rate: ").append(formatPercent(report.getBaselinePassRate())).append("\n");
|
||||
builder.append("- Current pass rate: ").append(formatPercent(report.getCurrentPassRate())).append("\n");
|
||||
builder.append("- Baseline passed cases: ").append(report.getBaselinePassedCases()).append("/")
|
||||
.append(report.getBaselineTotalCases()).append("\n");
|
||||
builder.append("- Current passed cases: ").append(report.getCurrentPassedCases()).append("/")
|
||||
.append(report.getCurrentTotalCases()).append("\n");
|
||||
builder.append("- Regressions: ").append(report.getRegressionCount()).append("\n");
|
||||
builder.append("- Improvements: ").append(report.getImprovementCount()).append("\n");
|
||||
builder.append("- Other changes: ").append(report.getChangedCount()).append("\n\n");
|
||||
|
||||
builder.append("## Diff Items\n\n");
|
||||
if (report.getItems() == null || report.getItems().isEmpty()) {
|
||||
builder.append("- No differences\n");
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
builder.append("| Type | Scope | Case | Metric | Baseline | Current | Delta | Message |\n");
|
||||
builder.append("| --- | --- | --- | --- | --- | --- | ---: | --- |\n");
|
||||
for (DiagnosisEvalDiffItem item : report.getItems()) {
|
||||
builder.append("| ")
|
||||
.append(valueOrDash(item.getType()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getScope()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getCaseId()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getMetric()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getBaselineValue()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getCurrentValue()))
|
||||
.append(" | ")
|
||||
.append(item.getDelta() == null ? "-" : String.format("%.3f", item.getDelta()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(item.getMessage()))
|
||||
.append(" |\n");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private String formatPercent(double value) {
|
||||
return String.format("%.2f%%", value * 100);
|
||||
}
|
||||
|
||||
private String valueOrDash(String value) {
|
||||
return value == null || value.isBlank() ? "-" : value;
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DiagnosisEvalReport {
|
||||
|
||||
private int totalCases;
|
||||
private int passedCases;
|
||||
private double passRate;
|
||||
private Map<String, Long> verdictDistribution;
|
||||
private double averageToolCallCount;
|
||||
private double averageDurationMs;
|
||||
private List<DiagnosisEvalResult> results;
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
|
||||
public class DiagnosisEvalReportWriter {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public DiagnosisEvalReportWriter(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public void writeJson(DiagnosisEvalReport report, Path outputFile) throws IOException {
|
||||
Files.createDirectories(outputFile.getParent());
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
|
||||
}
|
||||
|
||||
public void writeMarkdown(DiagnosisEvalReport report, Path outputFile) throws IOException {
|
||||
Files.createDirectories(outputFile.getParent());
|
||||
Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
public String toMarkdown(DiagnosisEvalReport report) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("# Diagnosis Eval Report\n\n");
|
||||
builder.append("- Total cases: ").append(report.getTotalCases()).append("\n");
|
||||
builder.append("- Passed cases: ").append(report.getPassedCases()).append("\n");
|
||||
builder.append("- Pass rate: ").append(String.format("%.2f%%", report.getPassRate() * 100)).append("\n");
|
||||
builder.append("- Average tool calls: ").append(String.format("%.2f", report.getAverageToolCallCount())).append("\n");
|
||||
builder.append("- Average duration ms: ").append(String.format("%.2f", report.getAverageDurationMs())).append("\n\n");
|
||||
|
||||
builder.append("## Verdict Distribution\n\n");
|
||||
if (report.getVerdictDistribution() == null || report.getVerdictDistribution().isEmpty()) {
|
||||
builder.append("- None\n\n");
|
||||
} else {
|
||||
for (Map.Entry<String, Long> entry : report.getVerdictDistribution().entrySet()) {
|
||||
builder.append("- ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
|
||||
}
|
||||
builder.append("\n");
|
||||
}
|
||||
|
||||
builder.append("## Cases\n\n");
|
||||
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
|
||||
builder.append("| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |\n");
|
||||
for (DiagnosisEvalResult result : report.getResults()) {
|
||||
builder.append("| ")
|
||||
.append(result.getCaseId())
|
||||
.append(" | ")
|
||||
.append(result.isPassed() ? "PASS" : "FAIL")
|
||||
.append(" | ")
|
||||
.append(valueOrDash(result.getVerdict()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(result.getGatekeeperStatus()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(result.getPromptAuditVersion()))
|
||||
.append(" | ")
|
||||
.append(valueOrDash(result.getComposerStatus()))
|
||||
.append(" | ")
|
||||
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
|
||||
.append(" | ")
|
||||
.append(result.getGatekeeperRuleCount() == null ? "-" : result.getGatekeeperRuleCount())
|
||||
.append(" | ")
|
||||
.append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
|
||||
.append(" | ")
|
||||
.append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
|
||||
.append(" | ")
|
||||
.append(result.getDurationMs() == null ? "-" : result.getDurationMs())
|
||||
.append(" | ")
|
||||
.append(result.getFailedChecks() == null || result.getFailedChecks().isEmpty()
|
||||
? "-"
|
||||
: String.join("; ", result.getFailedChecks()))
|
||||
.append(" |\n");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private String valueOrDash(String value) {
|
||||
return value == null || value.isBlank() ? "-" : value;
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DiagnosisEvalResult {
|
||||
|
||||
private String caseId;
|
||||
private String title;
|
||||
private boolean passed;
|
||||
private List<String> failedChecks;
|
||||
private String verdict;
|
||||
private int matchedKeywordCount;
|
||||
private int requiredKeywordCount;
|
||||
private Map<String, Boolean> evidenceCoverage;
|
||||
private String gatekeeperStatus;
|
||||
private String gatekeeperRuleSetVersion;
|
||||
private String promptAuditVersion;
|
||||
private String composerStatus;
|
||||
private Integer claimCheckCount;
|
||||
private Integer gatekeeperRuleCount;
|
||||
private Integer toolCallCount;
|
||||
private Integer durationMs;
|
||||
}
|
||||
@@ -1,488 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class DiagnosisTraceEvaluator {
|
||||
|
||||
private static final TypeReference<List<DiagnosisEvalCase>> CASE_LIST_TYPE = new TypeReference<>() {};
|
||||
private static final String REJECT_DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论";
|
||||
private static final Set<String> RAW_EXECUTOR_MARKERS = Set.of(
|
||||
"executor_evidence_v2",
|
||||
"answer_version",
|
||||
"evidence_bindings",
|
||||
"claim_id"
|
||||
);
|
||||
private static final Set<String> VALID_CLAIM_VERIFICATIONS = Set.of(
|
||||
"direct_observation",
|
||||
"reasonable_inference",
|
||||
"overstated",
|
||||
"unsupported",
|
||||
"external_unknown",
|
||||
"contradicted"
|
||||
);
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public DiagnosisTraceEvaluator(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public List<DiagnosisEvalCase> loadCases(Path casesFile) throws IOException {
|
||||
return objectMapper.readValue(casesFile.toFile(), CASE_LIST_TYPE);
|
||||
}
|
||||
|
||||
public DiagnosisTraceResponse loadTrace(Path traceFile) throws IOException {
|
||||
return objectMapper.readValue(traceFile.toFile(), DiagnosisTraceResponse.class);
|
||||
}
|
||||
|
||||
public DiagnosisEvalReport evaluate(List<DiagnosisEvalCase> cases, Path fixtureDir) {
|
||||
List<DiagnosisEvalResult> results = new ArrayList<>();
|
||||
for (DiagnosisEvalCase evalCase : cases) {
|
||||
try {
|
||||
DiagnosisTraceResponse trace = loadTrace(fixtureDir.resolve(evalCase.getTraceFixture()));
|
||||
results.add(evaluate(evalCase, trace));
|
||||
} catch (Exception e) {
|
||||
results.add(DiagnosisEvalResult.builder()
|
||||
.caseId(evalCase.getId())
|
||||
.title(evalCase.getTitle())
|
||||
.passed(false)
|
||||
.failedChecks(List.of("trace fixture unavailable: " + e.getMessage()))
|
||||
.verdict(null)
|
||||
.matchedKeywordCount(0)
|
||||
.requiredKeywordCount(size(evalCase.getExpectedRootCauseKeywords()))
|
||||
.evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
|
||||
.gatekeeperStatus(null)
|
||||
.gatekeeperRuleSetVersion(null)
|
||||
.promptAuditVersion(null)
|
||||
.composerStatus(null)
|
||||
.claimCheckCount(null)
|
||||
.gatekeeperRuleCount(null)
|
||||
.toolCallCount(null)
|
||||
.durationMs(null)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
return toReport(results);
|
||||
}
|
||||
|
||||
public DiagnosisEvalResult evaluate(DiagnosisEvalCase evalCase, DiagnosisTraceResponse trace) {
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
String answer = trace.getSession() == null ? "" : nullToEmpty(trace.getSession().getAnswer());
|
||||
String normalizedAnswer = answer.toLowerCase(Locale.ROOT);
|
||||
|
||||
int requiredKeywordCount = size(evalCase.getExpectedRootCauseKeywords());
|
||||
int matchedKeywordCount = countMatches(normalizedAnswer, evalCase.getExpectedRootCauseKeywords());
|
||||
int minKeywordMatches = evalCase.getMinKeywordMatches() == null
|
||||
? requiredKeywordCount
|
||||
: evalCase.getMinKeywordMatches();
|
||||
if (matchedKeywordCount < minKeywordMatches) {
|
||||
failedChecks.add("answer keyword coverage too low: " + matchedKeywordCount + "/" + minKeywordMatches);
|
||||
}
|
||||
|
||||
for (String forbidden : safeList(evalCase.getForbiddenAnswerKeywords())) {
|
||||
if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) {
|
||||
failedChecks.add("answer contains forbidden keyword: " + forbidden);
|
||||
}
|
||||
}
|
||||
|
||||
Set<String> evidenceTools = collectEvidenceTools(trace);
|
||||
Map<String, Boolean> evidenceCoverage = new LinkedHashMap<>();
|
||||
for (String requiredTool : safeList(evalCase.getRequiredEvidenceTools())) {
|
||||
boolean present = evidenceTools.contains(requiredTool);
|
||||
evidenceCoverage.put(requiredTool, present);
|
||||
if (!present) {
|
||||
failedChecks.add("missing required evidence tool: " + requiredTool);
|
||||
}
|
||||
}
|
||||
|
||||
String verdict = extractVerifierVerdict(trace);
|
||||
if (verdict == null || verdict.isBlank()) {
|
||||
failedChecks.add("missing verifier verdict");
|
||||
} else if (!safeList(evalCase.getAllowedVerdicts()).isEmpty()
|
||||
&& !safeList(evalCase.getAllowedVerdicts()).contains(verdict)) {
|
||||
failedChecks.add("verdict not allowed: " + verdict);
|
||||
}
|
||||
|
||||
if ("REJECT".equals(verdict) && !answer.startsWith(REJECT_DEGRADED_PREFIX)) {
|
||||
failedChecks.add("reject output does not use degraded template");
|
||||
}
|
||||
|
||||
failedChecks.addAll(validateExecutorStructuredOutput(trace));
|
||||
String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
|
||||
String gatekeeperRuleSetVersion = extractNestedString(trace, "verifier_evaluation",
|
||||
"gatekeeper_result", "rule_set_version");
|
||||
String promptAuditVersion = extractNestedString(trace, "verifier_evaluation",
|
||||
"prompt_audit", "version");
|
||||
String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
|
||||
Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
|
||||
Integer gatekeeperRuleCount = countNestedList(trace, "verifier_evaluation", "gatekeeper_result", "rules");
|
||||
failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
|
||||
gatekeeperStatus, gatekeeperRuleSetVersion, promptAuditVersion, composerStatus));
|
||||
|
||||
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
|
||||
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
|
||||
|
||||
return DiagnosisEvalResult.builder()
|
||||
.caseId(evalCase.getId())
|
||||
.title(evalCase.getTitle())
|
||||
.passed(failedChecks.isEmpty())
|
||||
.failedChecks(failedChecks)
|
||||
.verdict(verdict)
|
||||
.matchedKeywordCount(matchedKeywordCount)
|
||||
.requiredKeywordCount(requiredKeywordCount)
|
||||
.evidenceCoverage(evidenceCoverage)
|
||||
.gatekeeperStatus(gatekeeperStatus)
|
||||
.gatekeeperRuleSetVersion(gatekeeperRuleSetVersion)
|
||||
.promptAuditVersion(promptAuditVersion)
|
||||
.composerStatus(composerStatus)
|
||||
.claimCheckCount(claimCheckCount)
|
||||
.gatekeeperRuleCount(gatekeeperRuleCount)
|
||||
.toolCallCount(toolCallCount)
|
||||
.durationMs(durationMs)
|
||||
.build();
|
||||
}
|
||||
|
||||
private DiagnosisEvalReport toReport(List<DiagnosisEvalResult> results) {
|
||||
int total = results.size();
|
||||
int passed = (int) results.stream().filter(DiagnosisEvalResult::isPassed).count();
|
||||
Map<String, Long> verdictDistribution = results.stream()
|
||||
.map(DiagnosisEvalResult::getVerdict)
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.groupingBy(value -> value, LinkedHashMap::new, Collectors.counting()));
|
||||
double averageToolCallCount = results.stream()
|
||||
.map(DiagnosisEvalResult::getToolCallCount)
|
||||
.filter(Objects::nonNull)
|
||||
.mapToInt(Integer::intValue)
|
||||
.average()
|
||||
.orElse(0.0);
|
||||
double averageDurationMs = results.stream()
|
||||
.map(DiagnosisEvalResult::getDurationMs)
|
||||
.filter(Objects::nonNull)
|
||||
.mapToInt(Integer::intValue)
|
||||
.average()
|
||||
.orElse(0.0);
|
||||
|
||||
return DiagnosisEvalReport.builder()
|
||||
.totalCases(total)
|
||||
.passedCases(passed)
|
||||
.passRate(total == 0 ? 0.0 : (double) passed / total)
|
||||
.verdictDistribution(verdictDistribution)
|
||||
.averageToolCallCount(averageToolCallCount)
|
||||
.averageDurationMs(averageDurationMs)
|
||||
.results(results)
|
||||
.build();
|
||||
}
|
||||
|
||||
private Set<String> collectEvidenceTools(DiagnosisTraceResponse trace) {
|
||||
Set<String> tools = new LinkedHashSet<>();
|
||||
if (trace.getToolInvocations() != null) {
|
||||
for (DiagnosisTraceResponse.ToolInvocationTrace invocation : trace.getToolInvocations()) {
|
||||
if (invocation.getToolName() != null) {
|
||||
tools.add(invocation.getToolName());
|
||||
}
|
||||
}
|
||||
}
|
||||
Object summaries = nestedValue(trace, "verifier_evaluation", "tool_trace_summary");
|
||||
if (summaries instanceof List<?> list) {
|
||||
for (Object item : list) {
|
||||
if (item instanceof Map<?, ?> map && map.get("tool_name") != null) {
|
||||
tools.add(String.valueOf(map.get("tool_name")));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tools;
|
||||
}
|
||||
|
||||
private String extractVerifierVerdict(DiagnosisTraceResponse trace) {
|
||||
Object value = nestedValue(trace, "verifier_evaluation", "verdict");
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private List<String> validateExecutorStructuredOutput(DiagnosisTraceResponse trace) {
|
||||
Object structuredOutput = nestedValue(trace, "verifier_evaluation", "executor_structured_output");
|
||||
if (!(structuredOutput instanceof Map<?, ?> output)) {
|
||||
return List.of();
|
||||
}
|
||||
Object claims = output.get("claims");
|
||||
if (!(claims instanceof List<?> claimList)) {
|
||||
return List.of("executor structured output missing claims array");
|
||||
}
|
||||
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
for (Object item : claimList) {
|
||||
if (!(item instanceof Map<?, ?> claim)) {
|
||||
failedChecks.add("executor structured claim is not an object");
|
||||
continue;
|
||||
}
|
||||
Object claimIdValue = claim.get("claim_id");
|
||||
String claimId = claimIdValue == null ? "unknown" : String.valueOf(claimIdValue);
|
||||
Object bindings = claim.get("evidence_bindings");
|
||||
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
|
||||
failedChecks.add("executor confirmed claim missing evidence bindings: " + claimId);
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private List<String> validateV2AuditClosure(DiagnosisEvalCase evalCase,
|
||||
DiagnosisTraceResponse trace,
|
||||
String normalizedAnswer,
|
||||
String verdict,
|
||||
String gatekeeperStatus,
|
||||
String gatekeeperRuleSetVersion,
|
||||
String promptAuditVersion,
|
||||
String composerStatus) {
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
|
||||
boolean requireClaimChecks = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireClaimChecks());
|
||||
boolean requireComposerOutput = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireComposerOutput());
|
||||
|
||||
Object gatekeeperResult = nestedValue(trace, "verifier_evaluation", "gatekeeper_result");
|
||||
if (requireV2AuditClosure && !(gatekeeperResult instanceof Map<?, ?>)) {
|
||||
failedChecks.add("missing gatekeeper_result");
|
||||
}
|
||||
if ("fail".equals(gatekeeperStatus) && "PASS".equals(verdict)) {
|
||||
failedChecks.add("gatekeeper fail cannot have PASS verdict");
|
||||
}
|
||||
if (!safeList(evalCase.getExpectedGatekeeperStatuses()).isEmpty()
|
||||
&& !safeList(evalCase.getExpectedGatekeeperStatuses()).contains(gatekeeperStatus)) {
|
||||
failedChecks.add("gatekeeper status not expected: " + valueOrMissing(gatekeeperStatus));
|
||||
}
|
||||
if (!isBlank(evalCase.getExpectedGatekeeperRuleSetVersion())
|
||||
&& !evalCase.getExpectedGatekeeperRuleSetVersion().equals(gatekeeperRuleSetVersion)) {
|
||||
failedChecks.add("gatekeeper rule set version not expected: "
|
||||
+ valueOrMissing(gatekeeperRuleSetVersion));
|
||||
}
|
||||
if (Boolean.TRUE.equals(evalCase.getRequireGatekeeperRules())) {
|
||||
failedChecks.addAll(validateGatekeeperRules(trace));
|
||||
}
|
||||
failedChecks.addAll(validatePromptAudit(evalCase, trace, promptAuditVersion));
|
||||
|
||||
failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
|
||||
|
||||
Object composerOutput = nestedValue(trace, "verifier_evaluation", "composer_output");
|
||||
if (requireComposerOutput && !(composerOutput instanceof Map<?, ?>)) {
|
||||
failedChecks.add("missing composer_output");
|
||||
}
|
||||
if (requireComposerOutput && isBlank(composerStatus)) {
|
||||
failedChecks.add("composer_output missing status");
|
||||
}
|
||||
if (!safeList(evalCase.getExpectedComposerStatuses()).isEmpty()
|
||||
&& !safeList(evalCase.getExpectedComposerStatuses()).contains(composerStatus)) {
|
||||
failedChecks.add("composer status not expected: " + valueOrMissing(composerStatus));
|
||||
}
|
||||
|
||||
for (String forbidden : safeList(evalCase.getForbiddenConfirmedClaimKeywords())) {
|
||||
if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) {
|
||||
failedChecks.add("answer contains forbidden confirmed claim keyword: " + forbidden);
|
||||
}
|
||||
}
|
||||
|
||||
if (requireV2AuditClosure) {
|
||||
for (String marker : RAW_EXECUTOR_MARKERS) {
|
||||
if (normalizedAnswer.contains(marker.toLowerCase(Locale.ROOT))) {
|
||||
failedChecks.add("answer leaks raw executor marker: " + marker);
|
||||
}
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private List<String> validatePromptAudit(DiagnosisEvalCase evalCase,
|
||||
DiagnosisTraceResponse trace,
|
||||
String promptAuditVersion) {
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
Object promptAudit = nestedValue(trace, "verifier_evaluation", "prompt_audit");
|
||||
if (Boolean.TRUE.equals(evalCase.getRequirePromptAudit()) && !(promptAudit instanceof Map<?, ?>)) {
|
||||
failedChecks.add("missing prompt_audit");
|
||||
return failedChecks;
|
||||
}
|
||||
if (!isBlank(evalCase.getExpectedPromptAuditVersion())
|
||||
&& !evalCase.getExpectedPromptAuditVersion().equals(promptAuditVersion)) {
|
||||
failedChecks.add("prompt audit version not expected: " + valueOrMissing(promptAuditVersion));
|
||||
}
|
||||
if (evalCase.getExpectedPromptVersions() == null || evalCase.getExpectedPromptVersions().isEmpty()) {
|
||||
return failedChecks;
|
||||
}
|
||||
if (!(promptAudit instanceof Map<?, ?> audit)) {
|
||||
failedChecks.add("missing prompt_audit");
|
||||
return failedChecks;
|
||||
}
|
||||
Object promptsValue = audit.get("prompts");
|
||||
if (!(promptsValue instanceof List<?> prompts)) {
|
||||
failedChecks.add("prompt_audit missing prompts");
|
||||
return failedChecks;
|
||||
}
|
||||
Map<String, String> actualVersions = new LinkedHashMap<>();
|
||||
for (Object promptValue : prompts) {
|
||||
if (promptValue instanceof Map<?, ?> prompt) {
|
||||
String name = stringValue(prompt.get("name"));
|
||||
String version = stringValue(prompt.get("version"));
|
||||
if (!isBlank(name)) {
|
||||
actualVersions.put(name, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (Map.Entry<String, String> expected : evalCase.getExpectedPromptVersions().entrySet()) {
|
||||
String actual = actualVersions.get(expected.getKey());
|
||||
if (!expected.getValue().equals(actual)) {
|
||||
failedChecks.add("prompt version not expected: "
|
||||
+ expected.getKey() + "=" + valueOrMissing(actual));
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private List<String> validateGatekeeperRules(DiagnosisTraceResponse trace) {
|
||||
Object rulesValue = nestedNestedValue(trace, "verifier_evaluation", "gatekeeper_result", "rules");
|
||||
if (!(rulesValue instanceof List<?> rules) || rules.isEmpty()) {
|
||||
return List.of("gatekeeper_result missing rules");
|
||||
}
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
for (Object item : rules) {
|
||||
if (!(item instanceof Map<?, ?> rule)) {
|
||||
failedChecks.add("gatekeeper rule metadata is not an object");
|
||||
continue;
|
||||
}
|
||||
String id = stringValue(rule.get("id"));
|
||||
Object enabled = rule.get("enabled");
|
||||
String severity = stringValue(rule.get("default_severity"));
|
||||
if (isBlank(id)) {
|
||||
failedChecks.add("gatekeeper rule metadata missing id");
|
||||
}
|
||||
if (!(enabled instanceof Boolean)) {
|
||||
failedChecks.add("gatekeeper rule metadata missing enabled: " + valueOrMissing(id));
|
||||
}
|
||||
if (isBlank(severity)) {
|
||||
failedChecks.add("gatekeeper rule metadata missing default_severity: " + valueOrMissing(id));
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private List<String> validateClaimChecks(DiagnosisTraceResponse trace, boolean required) {
|
||||
Object claimChecks = nestedValue(trace, "verifier_evaluation", "claim_checks");
|
||||
if (!(claimChecks instanceof List<?> claimCheckList)) {
|
||||
return required ? List.of("missing claim_checks") : List.of();
|
||||
}
|
||||
if (required && claimCheckList.isEmpty()) {
|
||||
return List.of("claim_checks is empty");
|
||||
}
|
||||
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
for (Object item : claimCheckList) {
|
||||
if (!(item instanceof Map<?, ?> claimCheck)) {
|
||||
failedChecks.add("claim_check is not an object");
|
||||
continue;
|
||||
}
|
||||
String claimId = stringValue(claimCheck.get("claim_id"));
|
||||
String verification = stringValue(claimCheck.get("verification"));
|
||||
if (isBlank(claimId)) {
|
||||
failedChecks.add("claim_check missing claim_id");
|
||||
}
|
||||
if (isBlank(verification)) {
|
||||
failedChecks.add("claim_check missing verification: " + valueOrMissing(claimId));
|
||||
} else if (!VALID_CLAIM_VERIFICATIONS.contains(verification)) {
|
||||
failedChecks.add("claim_check verification invalid: " + verification);
|
||||
}
|
||||
if (isBlank(stringValue(claimCheck.get("detail")))) {
|
||||
failedChecks.add("claim_check missing detail: " + valueOrMissing(claimId));
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
|
||||
if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) {
|
||||
return null;
|
||||
}
|
||||
Object first = trace.getSession().getSelfEvaluation().get(firstKey);
|
||||
if (!(first instanceof Map<?, ?> map)) {
|
||||
return null;
|
||||
}
|
||||
return map.get(secondKey);
|
||||
}
|
||||
|
||||
private String extractNestedString(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
|
||||
Object value = nestedValue(trace, firstKey, secondKey);
|
||||
if (!(value instanceof Map<?, ?> map)) {
|
||||
return null;
|
||||
}
|
||||
Object nested = map.get(thirdKey);
|
||||
return nested == null ? null : String.valueOf(nested);
|
||||
}
|
||||
|
||||
private Integer countList(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
|
||||
Object value = nestedValue(trace, firstKey, secondKey);
|
||||
return value instanceof List<?> list ? list.size() : null;
|
||||
}
|
||||
|
||||
private Integer countNestedList(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
|
||||
Object value = nestedNestedValue(trace, firstKey, secondKey, thirdKey);
|
||||
return value instanceof List<?> list ? list.size() : null;
|
||||
}
|
||||
|
||||
private Object nestedNestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
|
||||
Object value = nestedValue(trace, firstKey, secondKey);
|
||||
if (!(value instanceof Map<?, ?> map)) {
|
||||
return null;
|
||||
}
|
||||
return map.get(thirdKey);
|
||||
}
|
||||
|
||||
private int countMatches(String normalizedAnswer, List<String> keywords) {
|
||||
int count = 0;
|
||||
for (String keyword : safeList(keywords)) {
|
||||
if (normalizedAnswer.contains(keyword.toLowerCase(Locale.ROOT))) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private Map<String, Boolean> emptyCoverage(List<String> tools) {
|
||||
Map<String, Boolean> coverage = new LinkedHashMap<>();
|
||||
for (String tool : safeList(tools)) {
|
||||
coverage.put(tool, false);
|
||||
}
|
||||
return coverage;
|
||||
}
|
||||
|
||||
private List<String> safeList(List<String> values) {
|
||||
return values == null ? List.of() : values;
|
||||
}
|
||||
|
||||
private int size(List<?> values) {
|
||||
return values == null ? 0 : values.size();
|
||||
}
|
||||
|
||||
private String nullToEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
private String stringValue(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
|
||||
private String valueOrMissing(String value) {
|
||||
return isBlank(value) ? "missing" : value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
@HookPositions({HookPosition.BEFORE_MODEL, HookPosition.AFTER_MODEL})
|
||||
public final class HarnessAgentAuditHook extends MessagesModelHook {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(HarnessAgentAuditHook.class);
|
||||
|
||||
private final AgentStepRepository repository;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final String agentName;
|
||||
private final ConcurrentHashMap<String, Integer> stepCounters = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, PendingStep> pendingSteps = new ConcurrentHashMap<>();
|
||||
|
||||
public HarnessAgentAuditHook(AgentStepRepository repository, ObjectMapper objectMapper, String agentName) {
|
||||
this.repository = Objects.requireNonNull(repository, "repository must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
if (agentName == null || agentName.isBlank()) {
|
||||
throw new IllegalArgumentException("agentName must not be blank");
|
||||
}
|
||||
this.agentName = agentName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "harness_agent_audit";
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand beforeModel(List<Message> messages, RunnableConfig config) {
|
||||
AuditIdentity identity = identity(config);
|
||||
if (identity == null) {
|
||||
log.warn("Skipping AgentStep audit because RunnableConfig identity is missing");
|
||||
return new AgentCommand(messages);
|
||||
}
|
||||
int stepIndex = stepCounters.merge(identity.runId(), 0, (current, ignored) -> current + 1);
|
||||
try {
|
||||
AgentStep saved = repository.save(AgentStep.builder()
|
||||
.sessionId(identity.sessionId())
|
||||
.runId(identity.runId())
|
||||
.stepIndex(stepIndex)
|
||||
.agentName(agentName)
|
||||
.modelInput(write(inputMetadata(messages)))
|
||||
.hasToolCall(false)
|
||||
.build());
|
||||
pendingSteps.put(stepKey(identity.runId(), stepIndex),
|
||||
new PendingStep(saved.getId(), System.nanoTime()));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist AgentStep audit before model: agent={}", agentName);
|
||||
}
|
||||
return new AgentCommand(messages);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand afterModel(List<Message> messages, RunnableConfig config) {
|
||||
AuditIdentity identity = identity(config);
|
||||
if (identity == null) {
|
||||
log.warn("Skipping AgentStep audit completion because RunnableConfig identity is missing");
|
||||
return new AgentCommand(messages);
|
||||
}
|
||||
int stepIndex = stepCounters.getOrDefault(identity.runId(), 0);
|
||||
PendingStep pending = pendingSteps.remove(stepKey(identity.runId(), stepIndex));
|
||||
if (pending == null || pending.id() == null) {
|
||||
return new AgentCommand(messages);
|
||||
}
|
||||
try {
|
||||
AgentStep step = repository.findById(pending.id()).orElse(null);
|
||||
if (step != null) {
|
||||
AssistantMessage assistant = lastAssistant(messages);
|
||||
List<String> toolNames = assistant == null || assistant.getToolCalls() == null
|
||||
? List.of()
|
||||
: assistant.getToolCalls().stream().map(AssistantMessage.ToolCall::name)
|
||||
.distinct().sorted().toList();
|
||||
step.setModelOutput(write(outputMetadata(assistant, toolNames)));
|
||||
step.setThought(null);
|
||||
step.setHasToolCall(!toolNames.isEmpty());
|
||||
step.setDurationMs(durationMillis(pending.startedNanos()));
|
||||
repository.save(step);
|
||||
}
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to complete AgentStep audit: agent={}", agentName);
|
||||
}
|
||||
return new AgentCommand(messages);
|
||||
}
|
||||
|
||||
private Map<String, Object> inputMetadata(List<Message> messages) {
|
||||
List<Message> safeMessages = messages == null ? List.of() : messages;
|
||||
Map<String, Object> metadata = new LinkedHashMap<>();
|
||||
metadata.put("message_count", safeMessages.size());
|
||||
metadata.put("roles", safeMessages.stream()
|
||||
.map(message -> message.getMessageType().getValue())
|
||||
.toList());
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private Map<String, Object> outputMetadata(AssistantMessage assistant, List<String> toolNames) {
|
||||
Map<String, Object> metadata = new LinkedHashMap<>();
|
||||
metadata.put("has_text", assistant != null
|
||||
&& assistant.getText() != null && !assistant.getText().isBlank());
|
||||
metadata.put("tool_names", toolNames);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private AuditIdentity identity(RunnableConfig config) {
|
||||
if (config == null) {
|
||||
return null;
|
||||
}
|
||||
String sessionId = config.metadata("sessionId").map(Object::toString).orElse(null);
|
||||
String runId = config.metadata("runId").map(Object::toString).orElse(null);
|
||||
return hasText(sessionId) && hasText(runId) ? new AuditIdentity(sessionId, runId) : null;
|
||||
}
|
||||
|
||||
private AssistantMessage lastAssistant(List<Message> messages) {
|
||||
if (messages == null) {
|
||||
return null;
|
||||
}
|
||||
for (int index = messages.size() - 1; index >= 0; index--) {
|
||||
if (messages.get(index) instanceof AssistantMessage assistant) {
|
||||
return assistant;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String write(Map<String, Object> value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalStateException("Audit metadata is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static int durationMillis(long startedNanos) {
|
||||
long value = Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
|
||||
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) value;
|
||||
}
|
||||
|
||||
private static String stepKey(String runId, int stepIndex) {
|
||||
return runId + ':' + stepIndex;
|
||||
}
|
||||
|
||||
private static boolean hasText(String value) {
|
||||
return value != null && !value.isBlank();
|
||||
}
|
||||
|
||||
private record AuditIdentity(String sessionId, String runId) {
|
||||
}
|
||||
|
||||
private record PendingStep(Long id, long startedNanos) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
@Component
|
||||
public final class JpaToolInvocationAuditSink implements ToolInvocationAuditSink {
|
||||
|
||||
private final ToolInvocationRepository repository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public JpaToolInvocationAuditSink(ToolInvocationRepository repository, ObjectMapper objectMapper) {
|
||||
this.repository = Objects.requireNonNull(repository, "repository must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void record(ToolInvocationAuditEvent event) {
|
||||
Objects.requireNonNull(event, "event must not be null");
|
||||
repository.save(ToolInvocation.builder()
|
||||
.sessionId(event.sessionId())
|
||||
.runId(event.runId())
|
||||
.toolName(event.toolName())
|
||||
.inputParams(write(inputMetadata(event)))
|
||||
.outputPreview("status=%s,evidence_status=%s".formatted(
|
||||
event.status(), event.evidenceStatus()))
|
||||
.outputLength(event.agentResultBytes())
|
||||
.retrievalLayer("HARNESS")
|
||||
.isTruncated(false)
|
||||
.relevanceLevel(event.evidenceStatus().name())
|
||||
.retrievalDetails(write(resultMetadata(event)))
|
||||
.durationMs(event.durationMs())
|
||||
.success(event.status() == InvocationStatus.READY)
|
||||
.errorMessage(event.errorCode())
|
||||
.build());
|
||||
}
|
||||
|
||||
private Map<String, Object> inputMetadata(ToolInvocationAuditEvent event) {
|
||||
Map<String, Object> metadata = new LinkedHashMap<>();
|
||||
metadata.put("tool_call_id", event.toolCallId());
|
||||
metadata.put("request_bytes", event.requestBytes());
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private Map<String, Object> resultMetadata(ToolInvocationAuditEvent event) {
|
||||
Map<String, Object> metadata = new LinkedHashMap<>();
|
||||
metadata.put("tool_call_id", event.toolCallId());
|
||||
metadata.put("status", event.status().name());
|
||||
metadata.put("evidence_status", event.evidenceStatus().name());
|
||||
metadata.put("agent_result_bytes", event.agentResultBytes());
|
||||
if (event.errorCode() != null) {
|
||||
metadata.put("error_code", event.errorCode());
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private String write(Map<String, Object> value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalStateException("Tool audit metadata is not serializable", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record ToolInvocationAuditEvent(
|
||||
String sessionId,
|
||||
String runId,
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
InvocationStatus status,
|
||||
EvidenceStatus evidenceStatus,
|
||||
String errorCode,
|
||||
int durationMs,
|
||||
int requestBytes,
|
||||
int agentResultBytes) {
|
||||
|
||||
public ToolInvocationAuditEvent {
|
||||
requireText(sessionId, "sessionId");
|
||||
requireText(runId, "runId");
|
||||
requireText(toolCallId, "toolCallId");
|
||||
requireText(toolName, "toolName");
|
||||
Objects.requireNonNull(status, "status must not be null");
|
||||
Objects.requireNonNull(evidenceStatus, "evidenceStatus must not be null");
|
||||
if (durationMs < 0 || requestBytes < 0 || agentResultBytes < 0) {
|
||||
throw new IllegalArgumentException("Audit counters must not be negative");
|
||||
}
|
||||
if (status == InvocationStatus.READY && errorCode != null) {
|
||||
throw new IllegalArgumentException("READY audit must not contain errorCode");
|
||||
}
|
||||
if (status == InvocationStatus.ERROR && (errorCode == null || errorCode.isBlank())) {
|
||||
throw new IllegalArgumentException("ERROR audit requires errorCode");
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ToolInvocationAuditSink {
|
||||
|
||||
void record(ToolInvocationAuditEvent event);
|
||||
|
||||
static ToolInvocationAuditSink noop() {
|
||||
return ignored -> {
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.tool.boundary;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
@@ -18,6 +20,9 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class ToolBoundary {
|
||||
@@ -29,23 +34,44 @@ public final class ToolBoundary {
|
||||
private final CanonicalInvocationStore store;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final Clock clock;
|
||||
private final ToolInvocationAuditSink auditSink;
|
||||
|
||||
public ToolBoundary(DiagnosisHarnessCore core,
|
||||
ToolCallKeyFactory keyFactory,
|
||||
CanonicalInvocationStore store,
|
||||
ObjectMapper objectMapper,
|
||||
Clock clock) {
|
||||
this(core, keyFactory, store, objectMapper, clock, ToolInvocationAuditSink.noop());
|
||||
}
|
||||
|
||||
public ToolBoundary(DiagnosisHarnessCore core,
|
||||
ToolCallKeyFactory keyFactory,
|
||||
CanonicalInvocationStore store,
|
||||
ObjectMapper objectMapper,
|
||||
Clock clock,
|
||||
ToolInvocationAuditSink auditSink) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
|
||||
this.store = Objects.requireNonNull(store, "store must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
this.auditSink = Objects.requireNonNull(auditSink, "auditSink must not be null");
|
||||
}
|
||||
|
||||
public ToolBoundaryResult execute(RunContext context,
|
||||
ToolCallRequestEnvelope request,
|
||||
ToolExecutor executor,
|
||||
ToolResultProjector projector) {
|
||||
Instant startedAt = clock.instant();
|
||||
ToolBoundaryResult result = executeCanonical(context, request, executor, projector);
|
||||
auditSafely(context, request, result, startedAt);
|
||||
return result;
|
||||
}
|
||||
|
||||
private ToolBoundaryResult executeCanonical(RunContext context,
|
||||
ToolCallRequestEnvelope request,
|
||||
ToolExecutor executor,
|
||||
ToolResultProjector projector) {
|
||||
String toolCallId = request == null ? null : request.toolCallId();
|
||||
String key;
|
||||
try {
|
||||
@@ -193,6 +219,36 @@ public final class ToolBoundary {
|
||||
return ToolBoundaryErrorCode.INVALID_REQUEST;
|
||||
}
|
||||
|
||||
private void auditSafely(RunContext context,
|
||||
ToolCallRequestEnvelope request,
|
||||
ToolBoundaryResult result,
|
||||
Instant startedAt) {
|
||||
if (context == null || request == null || result == null
|
||||
|| !context.runId().equals(request.runId())
|
||||
|| request.toolCallId() == null || request.toolCallId().isBlank()
|
||||
|| request.toolName() == null || request.toolName().isBlank()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
auditSink.record(new ToolInvocationAuditEvent(
|
||||
context.sessionId(), context.runId(), request.toolCallId(), request.toolName(),
|
||||
result.status(), result.evidenceStatus(), result.errorCode(),
|
||||
saturatingInt(Math.max(0L, Duration.between(startedAt, clock.instant()).toMillis())),
|
||||
utf8Bytes(request.requestJson()), utf8Bytes(result.agentResult())));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist Tool durable audit: tool={}, status={}",
|
||||
request.toolName(), result.status());
|
||||
}
|
||||
}
|
||||
|
||||
private static int utf8Bytes(String value) {
|
||||
return value == null ? 0 : saturatingInt(value.getBytes(StandardCharsets.UTF_8).length);
|
||||
}
|
||||
|
||||
private static int saturatingInt(long value) {
|
||||
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) value;
|
||||
}
|
||||
|
||||
private static final class InvalidToolCallIdException extends IllegalArgumentException {
|
||||
private InvalidToolCallIdException(Throwable cause) {
|
||||
super("Invalid tool call ID", cause);
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.ToolResponseMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Persists per-agent model input/output snapshots into agent_step.
|
||||
*/
|
||||
@Slf4j
|
||||
@HookPositions({HookPosition.BEFORE_MODEL, HookPosition.AFTER_MODEL})
|
||||
public class AgentLoggingHook extends MessagesModelHook {
|
||||
|
||||
private final AgentStepRepository agentStepRepository;
|
||||
private final String agentName;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
private final ConcurrentHashMap<String, Integer> stepCounters = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Map<String, Object>> pendingSteps = new ConcurrentHashMap<>();
|
||||
|
||||
public AgentLoggingHook(AgentStepRepository agentStepRepository, String agentName) {
|
||||
this.agentStepRepository = agentStepRepository;
|
||||
this.agentName = agentName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "agent_logging_hook";
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
|
||||
String sessionId = resolveSessionId(config);
|
||||
String runId = resolveRunId(config);
|
||||
String traceScopeId = traceScopeId(sessionId, runId);
|
||||
boolean hasSession = sessionId != null;
|
||||
|
||||
int stepIndex = 0;
|
||||
if (traceScopeId != null) {
|
||||
stepIndex = stepCounters.merge(traceScopeId, 0, (oldValue, ignored) -> oldValue + 1);
|
||||
}
|
||||
|
||||
log.info("========================================");
|
||||
log.info("*** [AgentTrace] agent={}, phase=before_model, stepIndex={}", agentName, stepIndex);
|
||||
log.info("*** [AgentTrace] messageCount={}", previousMessages.size());
|
||||
|
||||
int lastN = Math.min(3, previousMessages.size());
|
||||
if (lastN > 0) {
|
||||
log.info("*** [AgentTrace] recentMessages={}", lastN);
|
||||
List<Message> recentMessages = previousMessages.subList(previousMessages.size() - lastN, previousMessages.size());
|
||||
for (int i = 0; i < recentMessages.size(); i++) {
|
||||
Message msg = recentMessages.get(i);
|
||||
log.info(" [{}] role={}, type={}", i + 1, getMessageRole(msg), msg.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
log.info("========================================");
|
||||
|
||||
if (sessionId != null) {
|
||||
try {
|
||||
AgentStep step = AgentStep.builder()
|
||||
.sessionId(sessionId)
|
||||
.runId(runId)
|
||||
.stepIndex(stepIndex)
|
||||
.agentName(agentName)
|
||||
.modelInput(buildModelInputSummary(previousMessages))
|
||||
.build();
|
||||
AgentStep saved = agentStepRepository.save(step);
|
||||
pendingSteps.put(stepKey(traceScopeId, stepIndex), Map.of(
|
||||
"stepId", saved.getId(),
|
||||
"startTime", System.currentTimeMillis()
|
||||
));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to persist agent_step before model", e);
|
||||
}
|
||||
}
|
||||
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand afterModel(List<Message> previousMessages, RunnableConfig config) {
|
||||
String sessionId = resolveSessionId(config);
|
||||
String runId = resolveRunId(config);
|
||||
String traceScopeId = traceScopeId(sessionId, runId);
|
||||
int stepIndex = traceScopeId == null ? 0 : stepCounters.getOrDefault(traceScopeId, 0);
|
||||
|
||||
log.info("========================================");
|
||||
log.info("*** [AgentTrace] agent={}, phase=after_model, stepIndex={}", agentName, stepIndex);
|
||||
|
||||
AssistantMessage lastAssistant = findLastAssistant(previousMessages);
|
||||
boolean hasToolCall = false;
|
||||
|
||||
if (lastAssistant != null) {
|
||||
String textContent = extractTextContent(lastAssistant);
|
||||
if (textContent != null && !textContent.isEmpty()) {
|
||||
log.info("*** [AgentTrace] text={}",
|
||||
textContent.length() > 500
|
||||
? textContent.substring(0, 500) + "... (len=" + textContent.length() + ")"
|
||||
: textContent);
|
||||
}
|
||||
|
||||
if (lastAssistant.getToolCalls() != null && !lastAssistant.getToolCalls().isEmpty()) {
|
||||
hasToolCall = true;
|
||||
log.info("*** [AgentTrace] toolCalls={}", lastAssistant.getToolCalls().size());
|
||||
lastAssistant.getToolCalls().forEach(toolCall ->
|
||||
log.info(" - tool={}, arguments={}", toolCall.name(), toolCall.arguments()));
|
||||
} else {
|
||||
log.info("*** [AgentTrace] no tool call");
|
||||
}
|
||||
}
|
||||
log.info("========================================");
|
||||
|
||||
if (sessionId != null) {
|
||||
String stepKey = stepKey(traceScopeId, stepIndex);
|
||||
Map<String, Object> pending = pendingSteps.remove(stepKey);
|
||||
if (pending != null) {
|
||||
try {
|
||||
Long stepId = (Long) pending.get("stepId");
|
||||
long startTime = (long) pending.get("startTime");
|
||||
int durationMs = (int) (System.currentTimeMillis() - startTime);
|
||||
|
||||
AgentStep step = agentStepRepository.findById(stepId).orElse(null);
|
||||
if (step != null) {
|
||||
step.setThought(buildStoredThought(lastAssistant));
|
||||
step.setHasToolCall(hasToolCall);
|
||||
step.setDurationMs(durationMs);
|
||||
|
||||
if (lastAssistant != null) {
|
||||
step.setModelOutput(buildModelOutputSummary(lastAssistant));
|
||||
Integer tokenCount = TokenUsageHolder.get();
|
||||
if (tokenCount != null) {
|
||||
step.setTokenCount(tokenCount);
|
||||
}
|
||||
}
|
||||
|
||||
agentStepRepository.save(step);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to update agent_step after model", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TokenUsageHolder.clear();
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
|
||||
private String resolveSessionId(RunnableConfig config) {
|
||||
return config.metadata("sessionId")
|
||||
.map(Object::toString)
|
||||
.orElseGet(SessionContextHolder::getSessionId);
|
||||
}
|
||||
|
||||
private String resolveRunId(RunnableConfig config) {
|
||||
return config.metadata("runId")
|
||||
.map(Object::toString)
|
||||
.orElseGet(SessionContextHolder::getRunId);
|
||||
}
|
||||
|
||||
private String traceScopeId(String sessionId, String runId) {
|
||||
if (runId != null && !runId.isBlank()) {
|
||||
return runId;
|
||||
}
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
private String stepKey(String traceScopeId, int stepIndex) {
|
||||
return traceScopeId + "_" + stepIndex;
|
||||
}
|
||||
|
||||
private AssistantMessage findLastAssistant(List<Message> previousMessages) {
|
||||
for (int i = previousMessages.size() - 1; i >= 0; i--) {
|
||||
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
|
||||
return assistantMessage;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String buildModelInputSummary(List<Message> messages) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int maxMessages = Math.min(messages.size(), 5);
|
||||
for (int i = messages.size() - maxMessages; i < messages.size(); i++) {
|
||||
Message msg = messages.get(i);
|
||||
String role = getMessageRole(msg);
|
||||
String content = msg.toString();
|
||||
if (content.length() > 200) {
|
||||
content = content.substring(0, 200) + "...";
|
||||
}
|
||||
sb.append("[").append(role).append("] ").append(content).append("\n");
|
||||
}
|
||||
String result = sb.toString();
|
||||
if (result.length() > 500) {
|
||||
result = result.substring(0, 500) + "...";
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String buildStoredThought(AssistantMessage message) {
|
||||
String text = extractTextContent(message);
|
||||
if (text == null || text.isBlank()) {
|
||||
return text;
|
||||
}
|
||||
if (!"verifier".equals(agentName)) {
|
||||
return truncate(text, 2000);
|
||||
}
|
||||
return summarizeVerifierThought(text);
|
||||
}
|
||||
|
||||
private String summarizeVerifierThought(String verifierOutput) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(verifierOutput);
|
||||
int claimCount = root.path("claim_checks").isArray() ? root.path("claim_checks").size() : 0;
|
||||
int factCount = root.path("facts_checked").isArray() ? root.path("facts_checked").size() : 0;
|
||||
int tracedFactCount = 0;
|
||||
JsonNode tracedNodes = root.path("claim_checks").isArray() ? root.path("claim_checks") : root.path("facts_checked");
|
||||
if (tracedNodes.isArray()) {
|
||||
for (JsonNode node : tracedNodes) {
|
||||
if (node.path("evidence_refs").isArray() && node.path("evidence_refs").size() > 0) {
|
||||
tracedFactCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return "verdict=%s, score=%s, critical_fact_count=%s, claim_checks=%d, facts_checked=%d, traced_facts=%d".formatted(
|
||||
root.path("verdict").asText("UNKNOWN"),
|
||||
root.path("groundedness_score").asText("0.0"),
|
||||
root.path("critical_fact_count").asText("0"),
|
||||
claimCount,
|
||||
factCount,
|
||||
tracedFactCount
|
||||
);
|
||||
} catch (Exception e) {
|
||||
return truncate(verifierOutput, 300);
|
||||
}
|
||||
}
|
||||
|
||||
private String buildModelOutputSummary(AssistantMessage message) {
|
||||
String text = extractTextContent(message);
|
||||
if (text == null) {
|
||||
text = "";
|
||||
}
|
||||
int maxTextLength = "verifier".equals(agentName) ? 4000 : 500;
|
||||
text = truncate(text, maxTextLength);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("{\"text\":\"").append(escapeJson(text)).append("\"");
|
||||
if (message.getToolCalls() != null && !message.getToolCalls().isEmpty()) {
|
||||
sb.append(",\"toolCalls\":[");
|
||||
for (int i = 0; i < message.getToolCalls().size(); i++) {
|
||||
if (i > 0) {
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append("{\"name\":\"").append(escapeJson(message.getToolCalls().get(i).name()))
|
||||
.append("\",\"arguments\":").append(message.getToolCalls().get(i).arguments()).append("}");
|
||||
}
|
||||
sb.append("]");
|
||||
}
|
||||
sb.append("}");
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String escapeJson(String s) {
|
||||
if (s == null) {
|
||||
return "";
|
||||
}
|
||||
return s.replace("\\", "\\\\")
|
||||
.replace("\"", "\\\"")
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r")
|
||||
.replace("\t", "\\t");
|
||||
}
|
||||
|
||||
private String truncate(String text, int maxLength) {
|
||||
if (text == null || text.length() <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
return text.substring(0, maxLength) + "...";
|
||||
}
|
||||
|
||||
private String extractTextContent(AssistantMessage message) {
|
||||
if (message == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
try {
|
||||
return message.getText();
|
||||
} catch (Exception ignore) {
|
||||
// Fallback below.
|
||||
}
|
||||
|
||||
for (String fieldName : List.of("text", "content")) {
|
||||
try {
|
||||
java.lang.reflect.Field field = message.getClass().getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
Object value = field.get(message);
|
||||
if (value != null) {
|
||||
return value.toString();
|
||||
}
|
||||
} catch (NoSuchFieldException ignore) {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
|
||||
for (String methodName : List.of("getText", "getContent")) {
|
||||
try {
|
||||
java.lang.reflect.Method method = message.getClass().getMethod(methodName);
|
||||
Object value = method.invoke(message);
|
||||
if (value != null) {
|
||||
return value.toString();
|
||||
}
|
||||
} catch (NoSuchMethodException ignore) {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
|
||||
String fallback = message.toString();
|
||||
if (fallback != null && !fallback.startsWith("AssistantMessage@")) {
|
||||
return fallback;
|
||||
}
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to extract AssistantMessage text", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getMessageRole(Message message) {
|
||||
if (message instanceof UserMessage) {
|
||||
return "user";
|
||||
}
|
||||
if (message instanceof AssistantMessage) {
|
||||
return "assistant";
|
||||
}
|
||||
if (message instanceof ToolResponseMessage) {
|
||||
return "tool";
|
||||
}
|
||||
return message.getClass().getSimpleName();
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.Prioritized;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
|
||||
import com.alibaba.cloud.ai.graph.skills.SkillMetadata;
|
||||
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Injects planner-visible skill metadata without exposing the full skill loader tool.
|
||||
*/
|
||||
@Slf4j
|
||||
@HookPositions(HookPosition.BEFORE_MODEL)
|
||||
public class PlannerSkillMetadataHook extends MessagesModelHook {
|
||||
|
||||
private static final String CATALOG_MARKER = "\"skill_catalog\"";
|
||||
|
||||
private final SkillRegistry skillRegistry;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
public PlannerSkillMetadataHook(SkillRegistry skillRegistry) {
|
||||
this.skillRegistry = skillRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "planner_skill_metadata_hook";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return Prioritized.HIGHEST_PRECEDENCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
|
||||
if (skillRegistry == null || skillRegistry.size() == 0 || hasCatalog(previousMessages)) {
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
|
||||
try {
|
||||
List<Map<String, String>> skills = skillRegistry.listAll().stream()
|
||||
.map(this::toSkillSummary)
|
||||
.toList();
|
||||
if (skills.isEmpty()) {
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
|
||||
Map<String, Object> catalog = new LinkedHashMap<>();
|
||||
catalog.put("purpose", "Planner-visible diagnosis skill metadata only.");
|
||||
catalog.put("rules", List.of(
|
||||
"Choose at most one primary skill.",
|
||||
"Do not load full skill instructions in Planner.",
|
||||
"Executor reads the selected skill before evidence collection.",
|
||||
"If no skill matches, set selected_skill to null."
|
||||
));
|
||||
catalog.put("skills", skills);
|
||||
catalog.put("required_planner_output", Map.of(
|
||||
"selected_skill", "skill name or null",
|
||||
"selection_reason", "short reason",
|
||||
"plan", "ordered execution step list"
|
||||
));
|
||||
|
||||
Map<String, Object> payload = Map.of("skill_catalog", catalog);
|
||||
String content = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(payload);
|
||||
|
||||
List<Message> updatedMessages = new ArrayList<>(previousMessages.size() + 1);
|
||||
updatedMessages.add(new SystemMessage(content));
|
||||
updatedMessages.addAll(previousMessages);
|
||||
return new AgentCommand(updatedMessages);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to inject planner skill metadata, fallback to original messages", e);
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> toSkillSummary(SkillMetadata skill) {
|
||||
Map<String, String> summary = new LinkedHashMap<>();
|
||||
summary.put("name", skill.getName());
|
||||
summary.put("description", skill.getDescription());
|
||||
return summary;
|
||||
}
|
||||
|
||||
private boolean hasCatalog(List<Message> messages) {
|
||||
return messages.stream()
|
||||
.map(Message::getText)
|
||||
.anyMatch(text -> text != null && text.contains(CATALOG_MARKER));
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
/**
|
||||
* ChatModel 包装器 — 捕获每次模型调用的实际 token 用量
|
||||
* 通过 TokenUsageHolder 传递给 AgentLoggingHook
|
||||
*/
|
||||
public class TokenTrackingChatModel implements ChatModel {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(TokenTrackingChatModel.class);
|
||||
|
||||
private final ChatModel delegate;
|
||||
|
||||
public TokenTrackingChatModel(ChatModel delegate) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
ChatResponse response = delegate.call(prompt);
|
||||
captureTokenUsage(response);
|
||||
return response;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Flux<ChatResponse> stream(Prompt prompt) {
|
||||
return delegate.stream(prompt);
|
||||
}
|
||||
|
||||
private void captureTokenUsage(ChatResponse response) {
|
||||
try {
|
||||
if (response.getMetadata() == null || response.getMetadata().getUsage() == null) {
|
||||
return;
|
||||
}
|
||||
var usage = response.getMetadata().getUsage();
|
||||
Integer total = usage.getTotalTokens();
|
||||
if (total != null && total > 0) {
|
||||
TokenUsageHolder.set(total);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("捕获 token 用量失败", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
/**
|
||||
* Token 用量持有者（基于 ThreadLocal）
|
||||
* ChatModel 调用后写入实际 token 数，AgentLoggingHook 读取
|
||||
*/
|
||||
public class TokenUsageHolder {
|
||||
|
||||
private static final ThreadLocal<Integer> TOKEN_COUNT = new ThreadLocal<>();
|
||||
|
||||
public static void set(Integer count) {
|
||||
TOKEN_COUNT.set(count);
|
||||
}
|
||||
|
||||
public static Integer get() {
|
||||
return TOKEN_COUNT.get();
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
TOKEN_COUNT.remove();
|
||||
}
|
||||
}
|
||||
@@ -1,352 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.ExecutorGatekeeperService;
|
||||
import com.superbiz.agent.service.GatekeeperRuleCatalog;
|
||||
import com.superbiz.agent.service.ToolTraceSummaryService;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import com.superbiz.agent.util.VerifierContextHolder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Replaces verifier history with an explicit structured payload.
|
||||
*/
|
||||
@Slf4j
|
||||
@HookPositions(HookPosition.BEFORE_MODEL)
|
||||
public class VerifierInputHook extends MessagesModelHook {
|
||||
|
||||
private final ToolTraceSummaryService toolTraceSummaryService;
|
||||
private final ExecutorGatekeeperService executorGatekeeperService;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
|
||||
};
|
||||
|
||||
public VerifierInputHook(ToolTraceSummaryService toolTraceSummaryService) {
|
||||
this(toolTraceSummaryService, null);
|
||||
}
|
||||
|
||||
public VerifierInputHook(ToolTraceSummaryService toolTraceSummaryService,
|
||||
ExecutorGatekeeperService executorGatekeeperService) {
|
||||
this.toolTraceSummaryService = toolTraceSummaryService;
|
||||
this.executorGatekeeperService = executorGatekeeperService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "verifier_input_hook";
|
||||
}
|
||||
|
||||
@Override
|
||||
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
|
||||
try {
|
||||
String sessionId = config.metadata("sessionId")
|
||||
.map(Object::toString)
|
||||
.orElseGet(SessionContextHolder::getSessionId);
|
||||
String runId = config.metadata("runId")
|
||||
.map(Object::toString)
|
||||
.orElseGet(SessionContextHolder::getRunId);
|
||||
String executorFinalAnswer = VerifierContextHolder.getExecutorFinalAnswer();
|
||||
if (executorFinalAnswer == null || executorFinalAnswer.isBlank()) {
|
||||
executorFinalAnswer = extractLastAssistantText(previousMessages);
|
||||
}
|
||||
|
||||
List<Map<String, Object>> toolTraceSummary = runId == null || runId.isBlank()
|
||||
? toolTraceSummaryService.buildVerifierTraceSummary(sessionId, executorFinalAnswer)
|
||||
: toolTraceSummaryService.buildVerifierTraceSummaryForRun(runId, executorFinalAnswer);
|
||||
VerifierContextHolder.setToolTraceSummary(toolTraceSummary);
|
||||
|
||||
ExecutorOutputParseResult parseResult = parseExecutorOutput(executorFinalAnswer);
|
||||
parseResult = new ExecutorOutputParseResult(
|
||||
enrichExecutorStructuredOutput(parseResult.structuredOutput(), toolTraceSummary),
|
||||
parseResult.status()
|
||||
);
|
||||
VerifierContextHolder.setExecutorStructuredOutput(parseResult.structuredOutput());
|
||||
VerifierContextHolder.setExecutorOutputParseStatus(parseResult.status());
|
||||
|
||||
Map<String, Object> gatekeeperResult = runGatekeeper(sessionId, runId, parseResult);
|
||||
VerifierContextHolder.setGatekeeperResult(gatekeeperResult);
|
||||
|
||||
Map<String, Object> verifierInput = new LinkedHashMap<>();
|
||||
verifierInput.put("original_query", VerifierContextHolder.getOriginalQuery());
|
||||
verifierInput.put("executor_final_answer", executorFinalAnswer);
|
||||
verifierInput.put("executor_structured_output", parseResult.structuredOutput());
|
||||
verifierInput.put("executor_output_parse_status", parseResult.status());
|
||||
verifierInput.put("tool_trace_summary", toolTraceSummary);
|
||||
verifierInput.put("gatekeeper_result", gatekeeperResult);
|
||||
verifierInput.put("retry_context", VerifierContextHolder.getRetryContext());
|
||||
|
||||
String payload = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(verifierInput);
|
||||
return new AgentCommand(List.of(new UserMessage(payload)));
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to build verifier input, fallback to original messages", e);
|
||||
return new AgentCommand(previousMessages);
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> runGatekeeper(String sessionId, String runId, ExecutorOutputParseResult parseResult) {
|
||||
if (executorGatekeeperService == null) {
|
||||
return passGatekeeperResult();
|
||||
}
|
||||
try {
|
||||
if (runId != null && !runId.isBlank()) {
|
||||
return executorGatekeeperService.validateRun(runId, parseResult.structuredOutput(), parseResult.status());
|
||||
}
|
||||
return executorGatekeeperService.validate(sessionId, parseResult.structuredOutput(), parseResult.status());
|
||||
} catch (Exception e) {
|
||||
log.error("Gatekeeper validation failed unexpectedly", e);
|
||||
return executorGatekeeperService.fail("gatekeeper.internal_error",
|
||||
"gatekeeper",
|
||||
e.getMessage() == null ? "gatekeeper validation failed" : e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> passGatekeeperResult() {
|
||||
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("status", "pass");
|
||||
result.put("severity", "none");
|
||||
result.put("rule_set_version", catalog.version());
|
||||
result.put("rules", catalog.auditRules());
|
||||
result.put("checked_bindings", List.of());
|
||||
result.put("failed_rules", List.of());
|
||||
result.put("warnings", List.of());
|
||||
result.put("errors", List.of());
|
||||
return result;
|
||||
}
|
||||
|
||||
private ExecutorOutputParseResult parseExecutorOutput(String executorFinalAnswer) {
|
||||
if (executorFinalAnswer == null || executorFinalAnswer.isBlank()) {
|
||||
return new ExecutorOutputParseResult(null, status("missing", "executor_final_answer is blank"));
|
||||
}
|
||||
|
||||
String sanitized = sanitizeJsonPayload(executorFinalAnswer);
|
||||
if (!looksJsonLike(sanitized)) {
|
||||
return new ExecutorOutputParseResult(null, status("missing", "executor output is not JSON"));
|
||||
}
|
||||
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(sanitized);
|
||||
if (!root.isObject() || !root.path("claims").isArray()) {
|
||||
return new ExecutorOutputParseResult(null, status("malformed",
|
||||
"executor output JSON does not match evidence-attribution contract"));
|
||||
}
|
||||
Map<String, Object> structuredOutput = objectMapper.convertValue(root, MAP_TYPE);
|
||||
return new ExecutorOutputParseResult(structuredOutput, status("valid", "parsed executor evidence contract"));
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse executor structured output", e);
|
||||
return new ExecutorOutputParseResult(null, status("malformed", e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private Map<String, Object> enrichExecutorStructuredOutput(Map<String, Object> structuredOutput,
|
||||
List<Map<String, Object>> toolTraceSummary) {
|
||||
if (structuredOutput == null) {
|
||||
return null;
|
||||
}
|
||||
Map<String, List<Long>> invocationIdsByTool = invocationIdsByTool(toolTraceSummary);
|
||||
List<Map<String, Object>> warnings = new ArrayList<>();
|
||||
enrichEvidenceBindingsInSection(structuredOutput.get("claims"), invocationIdsByTool, warnings);
|
||||
enrichEvidenceBindingsInSection(structuredOutput.get("recommended_actions"), invocationIdsByTool, warnings);
|
||||
if (!warnings.isEmpty()) {
|
||||
structuredOutput.put("_gatekeeper_warnings", warnings);
|
||||
}
|
||||
return structuredOutput;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void enrichEvidenceBindingsInSection(Object sectionValue,
|
||||
Map<String, List<Long>> invocationIdsByTool,
|
||||
List<Map<String, Object>> warnings) {
|
||||
if (!(sectionValue instanceof List<?> items)) {
|
||||
return;
|
||||
}
|
||||
for (Object itemValue : items) {
|
||||
if (!(itemValue instanceof Map<?, ?> item)) {
|
||||
continue;
|
||||
}
|
||||
Object bindingsValue = item.get("evidence_bindings");
|
||||
if (!(bindingsValue instanceof List<?> bindings)) {
|
||||
continue;
|
||||
}
|
||||
for (Object bindingValue : bindings) {
|
||||
if (!(bindingValue instanceof Map<?, ?> rawBinding)) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> binding = (Map<String, Object>) rawBinding;
|
||||
String normalizedToolName = normalizeToolName(binding.get("tool_name"));
|
||||
if (!normalizedToolName.isBlank()) {
|
||||
binding.put("tool_name", normalizedToolName);
|
||||
}
|
||||
if (!hasInvocationId(binding)) {
|
||||
List<Long> ids = invocationIdsByTool.getOrDefault(normalizedToolName, List.of());
|
||||
if (ids.size() == 1) {
|
||||
binding.put("source_invocation_id", ids.get(0));
|
||||
warnings.add(Map.of(
|
||||
"rule", "evidence.invocation_auto_backfill",
|
||||
"message", "source_invocation_id was auto-filled from the unique tool invocation candidate; raw_path remains missing if Executor did not provide it",
|
||||
"tool_name", normalizedToolName,
|
||||
"source_invocation_id", ids.get(0)
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, List<Long>> invocationIdsByTool(List<Map<String, Object>> toolTraceSummary) {
|
||||
Map<String, Set<Long>> idsByTool = new LinkedHashMap<>();
|
||||
for (Map<String, Object> summary : toolTraceSummary == null ? List.<Map<String, Object>>of() : toolTraceSummary) {
|
||||
String toolName = normalizeToolName(summary.get("tool_name"));
|
||||
if (toolName.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
List<Long> ids = toLongList(summary.get("source_invocation_ids"));
|
||||
if (ids.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
idsByTool.computeIfAbsent(toolName, ignored -> new LinkedHashSet<>()).addAll(ids);
|
||||
}
|
||||
|
||||
Map<String, List<Long>> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, Set<Long>> entry : idsByTool.entrySet()) {
|
||||
result.put(entry.getKey(), new ArrayList<>(entry.getValue()));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean hasInvocationId(Map<String, Object> binding) {
|
||||
if (asLong(binding.get("source_invocation_id")) != null) {
|
||||
return true;
|
||||
}
|
||||
return toLongList(binding.get("source_invocation_ids")).size() == 1;
|
||||
}
|
||||
|
||||
private List<Long> toLongList(Object value) {
|
||||
if (!(value instanceof List<?> values)) {
|
||||
return List.of();
|
||||
}
|
||||
List<Long> ids = new ArrayList<>();
|
||||
for (Object item : values) {
|
||||
Long id = asLong(item);
|
||||
if (id != null) {
|
||||
ids.add(id);
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
private Long asLong(Object value) {
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
if (value instanceof String text) {
|
||||
try {
|
||||
return Long.parseLong(text);
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String normalizeToolName(Object value) {
|
||||
String toolName = value == null ? "" : String.valueOf(value);
|
||||
return switch (toolName) {
|
||||
case "lookupKnowledge" -> "lookup_knowledge";
|
||||
case "queryLogs" -> "query_logs";
|
||||
case "queryPrometheusAlerts" -> "query_metrics";
|
||||
case "getAvailableLogTopics" -> "get_available_log_topics";
|
||||
default -> toolName;
|
||||
};
|
||||
}
|
||||
|
||||
private String sanitizeJsonPayload(String raw) {
|
||||
String trimmed = raw.trim();
|
||||
int fenceStart = trimmed.indexOf("```");
|
||||
if (fenceStart >= 0) {
|
||||
int firstNewline = trimmed.indexOf('\n', fenceStart);
|
||||
int lastFence = trimmed.indexOf("```", firstNewline + 1);
|
||||
if (firstNewline >= 0 && lastFence > firstNewline) {
|
||||
return trimmed.substring(firstNewline + 1, lastFence).trim();
|
||||
}
|
||||
}
|
||||
int objectStart = trimmed.indexOf('{');
|
||||
int objectEnd = trimmed.lastIndexOf('}');
|
||||
if (objectStart >= 0 && objectEnd > objectStart) {
|
||||
return trimmed.substring(objectStart, objectEnd + 1).trim();
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private boolean looksJsonLike(String text) {
|
||||
return text.startsWith("{") && text.endsWith("}");
|
||||
}
|
||||
|
||||
private Map<String, Object> status(String status, String detail) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("status", status);
|
||||
result.put("detail", detail == null ? "" : detail);
|
||||
return result;
|
||||
}
|
||||
|
||||
private String extractLastAssistantText(List<Message> previousMessages) {
|
||||
for (int i = previousMessages.size() - 1; i >= 0; i--) {
|
||||
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
|
||||
String text = extractTextContent(assistantMessage);
|
||||
if (text != null && !text.isBlank()) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private String extractTextContent(AssistantMessage message) {
|
||||
try {
|
||||
try {
|
||||
return message.getText();
|
||||
} catch (Exception ignore) {
|
||||
// Fallback for older implementations.
|
||||
}
|
||||
|
||||
for (String methodName : List.of("getText", "getContent")) {
|
||||
try {
|
||||
var method = message.getClass().getMethod(methodName);
|
||||
Object value = method.invoke(message);
|
||||
if (value != null) {
|
||||
return value.toString();
|
||||
}
|
||||
} catch (NoSuchMethodException ignore) {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to extract verifier assistant text", e);
|
||||
}
|
||||
return message.toString();
|
||||
}
|
||||
|
||||
private record ExecutorOutputParseResult(
|
||||
Map<String, Object> structuredOutput,
|
||||
Map<String, Object> status
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.AIOpsRequest;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
@Service
|
||||
public class AiOpsRuleEvaluationService {
|
||||
|
||||
public static final String PASS = "PASS";
|
||||
public static final String WARN = "WARN";
|
||||
public static final String FAIL = "FAIL";
|
||||
|
||||
public Map<String, Object> evaluate(AIOpsRequest request,
|
||||
String finalReport,
|
||||
List<ToolInvocation> toolInvocations) {
|
||||
List<Map<String, Object>> checks = new ArrayList<>();
|
||||
checks.add(checkReportPresent(finalReport));
|
||||
checks.add(checkPayloadFocus(request, finalReport));
|
||||
checks.add(checkEvidenceCoverage(toolInvocations));
|
||||
|
||||
String verdict = aggregateVerdict(checks);
|
||||
Map<String, Object> evaluation = new LinkedHashMap<>();
|
||||
evaluation.put("verdict", verdict);
|
||||
evaluation.put("checks", checks);
|
||||
evaluation.put("rationale", buildRationale(verdict, checks));
|
||||
evaluation.put("traceability_version", "aiops-rule-v1");
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
private Map<String, Object> checkReportPresent(String finalReport) {
|
||||
boolean passed = finalReport != null && finalReport.trim().length() >= 40;
|
||||
return check(
|
||||
"final_report_present",
|
||||
passed ? PASS : FAIL,
|
||||
passed ? "Final report is present." : "Final report is missing or too short."
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, Object> checkPayloadFocus(AIOpsRequest request, String finalReport) {
|
||||
if (!hasAlertPayload(request)) {
|
||||
return check("payload_focus", PASS, "No alert payload was supplied; payload focus is not required.");
|
||||
}
|
||||
|
||||
String report = lower(finalReport);
|
||||
List<String> missing = new ArrayList<>();
|
||||
if (!contains(report, request.getAlertName())) {
|
||||
missing.add("alertName");
|
||||
}
|
||||
if (!contains(report, request.getService())) {
|
||||
missing.add("service");
|
||||
}
|
||||
|
||||
if (missing.isEmpty()) {
|
||||
return check("payload_focus", PASS, "Final report mentions the supplied alert and service.");
|
||||
}
|
||||
return check(
|
||||
"payload_focus",
|
||||
WARN,
|
||||
"Final report is missing payload focus terms: " + String.join(", ", missing)
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, Object> checkEvidenceCoverage(List<ToolInvocation> toolInvocations) {
|
||||
List<String> evidenceTools = safeTools(toolInvocations).stream()
|
||||
.filter(tool -> tool.equals("lookup_knowledge")
|
||||
|| tool.equals("query_metrics")
|
||||
|| tool.equals("query_logs"))
|
||||
.distinct()
|
||||
.toList();
|
||||
|
||||
if (evidenceTools.isEmpty()) {
|
||||
return check("evidence_tool_coverage", WARN, "No persisted AIOps evidence tool calls were found.");
|
||||
}
|
||||
return check(
|
||||
"evidence_tool_coverage",
|
||||
PASS,
|
||||
"Persisted evidence tools: " + String.join(", ", evidenceTools)
|
||||
);
|
||||
}
|
||||
|
||||
private List<String> safeTools(List<ToolInvocation> toolInvocations) {
|
||||
if (toolInvocations == null) {
|
||||
return List.of();
|
||||
}
|
||||
return toolInvocations.stream()
|
||||
.map(ToolInvocation::getToolName)
|
||||
.filter(name -> name != null && !name.isBlank())
|
||||
.map(name -> name.trim().toLowerCase(Locale.ROOT))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private String aggregateVerdict(List<Map<String, Object>> checks) {
|
||||
boolean hasFail = checks.stream().anyMatch(check -> FAIL.equals(check.get("verdict")));
|
||||
if (hasFail) {
|
||||
return FAIL;
|
||||
}
|
||||
boolean hasWarn = checks.stream().anyMatch(check -> WARN.equals(check.get("verdict")));
|
||||
return hasWarn ? WARN : PASS;
|
||||
}
|
||||
|
||||
private String buildRationale(String verdict, List<Map<String, Object>> checks) {
|
||||
long passCount = checks.stream().filter(check -> PASS.equals(check.get("verdict"))).count();
|
||||
long warnCount = checks.stream().filter(check -> WARN.equals(check.get("verdict"))).count();
|
||||
long failCount = checks.stream().filter(check -> FAIL.equals(check.get("verdict"))).count();
|
||||
return "AIOps rule evaluation %s: pass=%d, warn=%d, fail=%d"
|
||||
.formatted(verdict, passCount, warnCount, failCount);
|
||||
}
|
||||
|
||||
private Map<String, Object> check(String name, String verdict, String detail) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("name", name);
|
||||
result.put("verdict", verdict);
|
||||
result.put("detail", detail);
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean hasAlertPayload(AIOpsRequest request) {
|
||||
if (request == null) {
|
||||
return false;
|
||||
}
|
||||
return !isBlank(request.getAlertName())
|
||||
|| !isBlank(request.getService())
|
||||
|| !isBlank(request.getSeverity())
|
||||
|| !isBlank(request.getDescription())
|
||||
|| !isBlank(request.getTimeRange());
|
||||
}
|
||||
|
||||
private boolean contains(String lowerText, String value) {
|
||||
return isBlank(value) || lowerText.contains(value.trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
private String lower(String value) {
|
||||
return value == null ? "" : value.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private boolean isBlank(String value) {
|
||||
return value == null || value.trim().isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -1,435 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.OverAllState;
|
||||
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
||||
import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.Hook;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.skills.SkillsAgentHook;
|
||||
import com.alibaba.cloud.ai.graph.exception.GraphRunnerException;
|
||||
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
|
||||
import com.superbiz.agent.agent.tool.DateTimeTools;
|
||||
import com.superbiz.agent.agent.tool.InternalDocsTools;
|
||||
import com.superbiz.agent.agent.tool.QueryLogsTools;
|
||||
import com.superbiz.agent.agent.tool.QueryMetricsTools;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisRun;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisSession;
|
||||
import com.superbiz.agent.dto.AIOpsRequest;
|
||||
import com.superbiz.agent.hook.AgentLoggingHook;
|
||||
import com.superbiz.agent.hook.PlannerSkillMetadataHook;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisSessionRepository;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.ToolCallbackProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import com.superbiz.agent.config.AiOpsPromptProperties;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* AI Ops 智能运维服务。
|
||||
* 负责构建 Planner、Executor、Supervisor 多 Agent 编排流程，并持久化诊断会话与执行指标。
|
||||
*/
|
||||
@Service
|
||||
public class AiOpsService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(AiOpsService.class);
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired(required = false)
|
||||
private ToolCallbackProvider toolCallbackProvider;
|
||||
|
||||
@Autowired
|
||||
private DateTimeTools dateTimeTools;
|
||||
|
||||
@Autowired
|
||||
private InternalDocsTools internalDocsTools;
|
||||
|
||||
@Autowired
|
||||
private QueryMetricsTools queryMetricsTools;
|
||||
|
||||
@Autowired(required = false) // Mock 模式下才注册本地日志查询工具。
|
||||
private QueryLogsTools queryLogsTools;
|
||||
|
||||
@Autowired
|
||||
private LookupKnowledgeTool lookupKnowledgeTool;
|
||||
|
||||
@Autowired
|
||||
private AiOpsPromptProperties promptProperties;
|
||||
|
||||
@Autowired
|
||||
private DiagnosisSessionRepository diagnosisSessionRepository;
|
||||
|
||||
@Autowired
|
||||
private DiagnosisRunRepository diagnosisRunRepository;
|
||||
|
||||
@Autowired
|
||||
private AgentStepRepository agentStepRepository;
|
||||
|
||||
@Autowired
|
||||
private ToolInvocationRepository toolInvocationRepository;
|
||||
|
||||
@Autowired
|
||||
private AiOpsRuleEvaluationService aiOpsRuleEvaluationService;
|
||||
|
||||
@Autowired
|
||||
private SelfEvaluationMergeService selfEvaluationMergeService;
|
||||
|
||||
@Autowired(required = false)
|
||||
private SkillRegistry skillRegistry;
|
||||
|
||||
/**
|
||||
* 执行 AI Ops 告警分析流程。
|
||||
*
|
||||
* @param chatModel 大模型实例
|
||||
* @param toolCallbacks Spring AI 工具回调
|
||||
* @return 多 Agent 编排后的最终状态
|
||||
* @throws GraphRunnerException Agent 图执行失败时抛出
|
||||
*/
|
||||
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks) throws GraphRunnerException {
|
||||
return executeAiOpsAnalysis(chatModel, toolCallbacks, null, resolveSessionId(null), newRunId());
|
||||
}
|
||||
|
||||
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks,
|
||||
AIOpsRequest request, String sessionId) throws GraphRunnerException {
|
||||
return executeAiOpsAnalysis(chatModel, toolCallbacks, request, sessionId, newRunId());
|
||||
}
|
||||
|
||||
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks,
|
||||
AIOpsRequest request, String sessionId, String runId) throws GraphRunnerException {
|
||||
logger.info("Starting AI Ops multi-agent analysis");
|
||||
String resolvedSessionId = isBlank(sessionId) ? resolveSessionId(request) : sessionId.trim();
|
||||
String resolvedRunId = isBlank(runId) ? newRunId() : runId.trim();
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
DiagnosisRun run = startDiagnosisRun(resolvedSessionId, resolvedRunId, request);
|
||||
|
||||
// 让工具调用、Hook 和知识库检索能够拿到当前诊断会话和运行 ID。
|
||||
SessionContextHolder.setContext(resolvedSessionId, resolvedRunId);
|
||||
|
||||
try {
|
||||
ReactAgent plannerAgent = buildPlannerAgent(chatModel, toolCallbacks);
|
||||
ReactAgent executorAgent = buildExecutorAgent(chatModel, toolCallbacks);
|
||||
|
||||
SupervisorAgent supervisorAgent = buildSupervisorAgent(chatModel, plannerAgent, executorAgent);
|
||||
|
||||
String taskPrompt = buildTaskPrompt(request);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", resolvedSessionId)
|
||||
.addMetadata("runId", resolvedRunId)
|
||||
.build();
|
||||
|
||||
logger.info("Invoking AI Ops supervisor agent");
|
||||
|
||||
Optional<OverAllState> stateOptional = supervisorAgent.invoke(taskPrompt, config);
|
||||
|
||||
long duration = System.currentTimeMillis() - startTime;
|
||||
|
||||
run.setStatus(stateOptional.isPresent() ? "SUCCESS" : "FAILED");
|
||||
run.setTotalDurationMs((int) duration);
|
||||
backfillRunMetrics(run);
|
||||
diagnosisRunRepository.save(run);
|
||||
|
||||
if (stateOptional.isPresent()) {
|
||||
OverAllState state = stateOptional.get();
|
||||
logger.debug("Final State Keys: {}", state.data().keySet());
|
||||
logger.debug("Planner Plan: {}", state.value("planner_plan"));
|
||||
logger.debug("Executor Feedback: {}", state.value("executor_feedback"));
|
||||
}
|
||||
|
||||
return stateOptional;
|
||||
} catch (Exception e) {
|
||||
run.setStatus("FAILED");
|
||||
run.setTotalDurationMs((int) (System.currentTimeMillis() - startTime));
|
||||
backfillRunMetrics(run);
|
||||
diagnosisRunRepository.save(run);
|
||||
throw e;
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/** Internal dependency-owning overload used by the protocol-only Controller. */
|
||||
public Optional<OverAllState> executeAiOpsAnalysis(AIOpsRequest request,
|
||||
String sessionId,
|
||||
String runId) throws GraphRunnerException {
|
||||
ToolCallback[] callbacks = toolCallbackProvider == null
|
||||
? new ToolCallback[0] : toolCallbackProvider.getToolCallbacks();
|
||||
return executeAiOpsAnalysis(chatModel, callbacks, request, sessionId, runId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从多 Agent 执行状态中提取最终报告文本。
|
||||
*
|
||||
* @param state Agent 图执行状态
|
||||
* @return Planner 输出中的最终报告
|
||||
*/
|
||||
public Optional<String> extractFinalReport(OverAllState state) {
|
||||
logger.info("Extracting final AI Ops report");
|
||||
|
||||
Optional<AssistantMessage> plannerFinalOutput = state.value("planner_plan")
|
||||
.filter(AssistantMessage.class::isInstance)
|
||||
.map(AssistantMessage.class::cast);
|
||||
|
||||
if (plannerFinalOutput.isPresent()) {
|
||||
String reportText = plannerFinalOutput.get().getText();
|
||||
logger.info("Extracted Planner final report, length: {}", reportText.length());
|
||||
return Optional.of(reportText);
|
||||
} else {
|
||||
logger.warn("Unable to extract Planner final report");
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
|
||||
public String resolveSessionId(AIOpsRequest request) {
|
||||
if (request != null && !isBlank(request.getSessionId())) {
|
||||
return request.getSessionId().trim();
|
||||
}
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
public String newRunId() {
|
||||
return "run-" + UUID.randomUUID();
|
||||
}
|
||||
|
||||
public void persistFinalReport(String sessionId, String finalReport) {
|
||||
persistFinalReport(sessionId, finalReport, null);
|
||||
}
|
||||
|
||||
public void persistFinalReport(String sessionId, String finalReport, AIOpsRequest request) {
|
||||
persistLegacyFinalReport(sessionId, finalReport, request);
|
||||
}
|
||||
|
||||
public void persistFinalReport(String sessionId, String runId, String finalReport, AIOpsRequest request) {
|
||||
if (isBlank(sessionId) || isBlank(runId) || isBlank(finalReport)) {
|
||||
return;
|
||||
}
|
||||
diagnosisRunRepository.findBySessionIdAndRunId(sessionId.trim(), runId.trim()).ifPresent(run -> {
|
||||
run.setAnswer(finalReport);
|
||||
List<com.superbiz.agent.domain.entity.ToolInvocation> invocations =
|
||||
toolInvocationRepository.findByRunIdOrderByIdAsc(run.getRunId());
|
||||
Map<String, Object> evaluation = aiOpsRuleEvaluationService.evaluate(request, finalReport, invocations);
|
||||
run.setSelfEvaluation(selfEvaluationMergeService.mergeAiOpsRuleEvaluation(
|
||||
run.getSelfEvaluation(), evaluation));
|
||||
diagnosisRunRepository.save(run);
|
||||
});
|
||||
}
|
||||
|
||||
private void persistLegacyFinalReport(String sessionId, String finalReport, AIOpsRequest request) {
|
||||
if (isBlank(sessionId) || isBlank(finalReport)) {
|
||||
return;
|
||||
}
|
||||
diagnosisSessionRepository.findBySessionId(sessionId.trim()).ifPresent(session -> {
|
||||
session.setAnswer(finalReport);
|
||||
List<com.superbiz.agent.domain.entity.ToolInvocation> invocations =
|
||||
toolInvocationRepository.findBySessionIdOrderByIdAsc(session.getSessionId());
|
||||
Map<String, Object> evaluation = aiOpsRuleEvaluationService.evaluate(request, finalReport, invocations);
|
||||
session.setSelfEvaluation(selfEvaluationMergeService.mergeAiOpsRuleEvaluation(
|
||||
session.getSelfEvaluation(), evaluation));
|
||||
diagnosisSessionRepository.save(session);
|
||||
});
|
||||
}
|
||||
|
||||
String buildQuerySummary(AIOpsRequest request) {
|
||||
if (request == null) {
|
||||
return "AI Ops alert analysis";
|
||||
}
|
||||
|
||||
StringBuilder summary = new StringBuilder("AI Ops alert analysis");
|
||||
appendField(summary, "alert", request.getAlertName());
|
||||
appendField(summary, "service", request.getService());
|
||||
appendField(summary, "severity", request.getSeverity());
|
||||
appendField(summary, "timeRange", request.getTimeRange());
|
||||
appendField(summary, "description", request.getDescription());
|
||||
appendField(summary, "request", request.getUserRequest());
|
||||
return summary.toString();
|
||||
}
|
||||
|
||||
boolean hasAlertPayload(AIOpsRequest request) {
|
||||
if (request == null) {
|
||||
return false;
|
||||
}
|
||||
return !isBlank(request.getAlertName())
|
||||
|| !isBlank(request.getService())
|
||||
|| !isBlank(request.getSeverity())
|
||||
|| !isBlank(request.getDescription())
|
||||
|| !isBlank(request.getTimeRange());
|
||||
}
|
||||
|
||||
String buildKnowledgeRetrievalQuery(AIOpsRequest request) {
|
||||
if (request == null || !hasAlertPayload(request)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
StringBuilder query = new StringBuilder();
|
||||
appendQueryTerm(query, request.getAlertName());
|
||||
appendQueryTerm(query, request.getService());
|
||||
appendQueryTerm(query, request.getSeverity());
|
||||
appendQueryTerm(query, request.getDescription());
|
||||
appendQueryTerm(query, request.getTimeRange());
|
||||
appendQueryTerm(query, request.getUserRequest());
|
||||
return query.toString();
|
||||
}
|
||||
|
||||
String buildTaskPrompt(AIOpsRequest request) {
|
||||
StringBuilder prompt = new StringBuilder();
|
||||
prompt.append("You are an enterprise SRE handling an automated alert diagnosis task. Combine tool evidence, run a plan-execute-replan loop, and output the final alert analysis report. Do not fabricate data; if repeated queries fail, clearly state why the task cannot be completed.");
|
||||
prompt.append("\n\nAlert input:\n");
|
||||
prompt.append(buildQuerySummary(request));
|
||||
if (hasAlertPayload(request)) {
|
||||
String knowledgeQuery = buildKnowledgeRetrievalQuery(request);
|
||||
prompt.append("\n\nAIOps scope mode: PAYLOAD_TARGETED\n");
|
||||
prompt.append("- The request includes an alert payload. Treat the supplied alert payload as the primary and only main diagnosis target.\n");
|
||||
prompt.append("- The final report must focus on the supplied alert fields such as alertName, service, severity, description, and timeRange.\n");
|
||||
prompt.append("- Recommended lookup_knowledge query: ").append(knowledgeQuery).append("\n");
|
||||
prompt.append("- If knowledge-base evidence is needed, call lookup_knowledge with the recommended query or a narrower query that preserves alertName and service.\n");
|
||||
prompt.append("- You may call queryPrometheusAlerts only to verify whether the supplied alert is still active or to identify related risk/context.\n");
|
||||
prompt.append("- If queryPrometheusAlerts returns unrelated active alerts, do not create full root-cause or remediation sections for them.\n");
|
||||
prompt.append("- Mention unrelated active alerts only briefly in a Related Risk section when they help explain the supplied alert.\n");
|
||||
} else {
|
||||
prompt.append("\n\nAIOps scope mode: AUTO_DISCOVERY\n");
|
||||
prompt.append("- The request does not include alert payload fields. First call queryPrometheusAlerts to discover current active/firing alerts.\n");
|
||||
prompt.append("- Prefer P0/P1 alerts or the longest-running firing alerts, then diagnose one or more alerts based on severity and evidence.\n");
|
||||
prompt.append("- Use metrics, logs, and knowledge-base evidence before producing the final alert analysis report.\n");
|
||||
}
|
||||
return prompt.toString();
|
||||
}
|
||||
|
||||
private DiagnosisRun startDiagnosisRun(String sessionId, String runId, AIOpsRequest request) {
|
||||
DiagnosisRun run = DiagnosisRun.builder()
|
||||
.runId(runId)
|
||||
.sessionId(sessionId)
|
||||
.query(buildQuerySummary(request))
|
||||
.status("RUNNING")
|
||||
.agentFlow("AI_OPS")
|
||||
.build();
|
||||
return diagnosisRunRepository.save(run);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 Planner Agent。
|
||||
*/
|
||||
private ReactAgent buildPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks) {
|
||||
return ReactAgent.builder()
|
||||
.name("planner_agent")
|
||||
.description("Plans alert diagnosis steps")
|
||||
.model(chatModel)
|
||||
.systemPrompt(promptProperties.getPlanner())
|
||||
.hooks(buildHooks("planner"))
|
||||
.outputKey("planner_plan")
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 Executor Agent。
|
||||
*/
|
||||
private ReactAgent buildExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks) {
|
||||
return ReactAgent.builder()
|
||||
.name("executor_agent")
|
||||
.description("Executes the current Planner step and reports feedback")
|
||||
.model(chatModel)
|
||||
.systemPrompt(promptProperties.getExecutor())
|
||||
.methodTools(buildMethodToolsArray())
|
||||
.tools(toolCallbacks)
|
||||
.hooks(buildHooks("executor"))
|
||||
.outputKey("executor_feedback")
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据运行模式构建方法工具数组。
|
||||
* Mock 模式注入本地 QueryLogsTools；真实模式下日志查询由外部 MCP 工具提供。
|
||||
*/
|
||||
private Object[] buildMethodToolsArray() {
|
||||
if (queryLogsTools != null) {
|
||||
return new Object[]{dateTimeTools, lookupKnowledgeTool, queryMetricsTools, queryLogsTools};
|
||||
}
|
||||
return new Object[]{dateTimeTools, lookupKnowledgeTool, queryMetricsTools};
|
||||
}
|
||||
|
||||
private Hook[] buildHooks(String agentName) {
|
||||
AgentLoggingHook loggingHook = new AgentLoggingHook(agentStepRepository, agentName);
|
||||
if (skillRegistry == null || skillRegistry.size() == 0) {
|
||||
return new Hook[]{loggingHook};
|
||||
}
|
||||
if ("planner".equals(agentName)) {
|
||||
return new Hook[]{
|
||||
new PlannerSkillMetadataHook(skillRegistry),
|
||||
loggingHook
|
||||
};
|
||||
}
|
||||
return new Hook[]{
|
||||
loggingHook,
|
||||
SkillsAgentHook.builder()
|
||||
.skillRegistry(skillRegistry)
|
||||
.build()
|
||||
};
|
||||
}
|
||||
|
||||
SupervisorAgent buildSupervisorAgent(ChatModel chatModel, ReactAgent plannerAgent, ReactAgent executorAgent) {
|
||||
return SupervisorAgent.builder()
|
||||
.name("ai_ops_supervisor")
|
||||
.description("Coordinates Planner and Executor agents")
|
||||
.model(chatModel)
|
||||
.mainAgent(plannerAgent)
|
||||
.systemPrompt(promptProperties.getSupervisor())
|
||||
.subAgents(List.of(executorAgent))
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 agent_step 和 tool_invocation 回填 diagnosis_run 的汇总指标。
|
||||
*/
|
||||
private void backfillRunMetrics(DiagnosisRun run) {
|
||||
try {
|
||||
List<AgentStep> steps = agentStepRepository.findByRunIdOrderByStepIndex(run.getRunId());
|
||||
|
||||
int totalTokens = 0;
|
||||
int stepCount = 0;
|
||||
for (AgentStep s : steps) {
|
||||
stepCount++;
|
||||
if (s.getTokenCount() != null) totalTokens += s.getTokenCount();
|
||||
}
|
||||
long toolCallCount = toolInvocationRepository.countByRunId(run.getRunId());
|
||||
run.setTotalTokenCount(totalTokens);
|
||||
run.setStepCount(stepCount);
|
||||
run.setToolCallCount(Math.toIntExact(toolCallCount));
|
||||
} catch (Exception e) {
|
||||
logger.warn("Failed to backfill AI Ops run metrics, runId={}", run.getRunId(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendField(StringBuilder builder, String label, String value) {
|
||||
if (!isBlank(value)) {
|
||||
builder.append("\n- ").append(label).append(": ").append(value.trim());
|
||||
}
|
||||
}
|
||||
|
||||
private void appendQueryTerm(StringBuilder builder, String value) {
|
||||
if (!isBlank(value)) {
|
||||
if (!builder.isEmpty()) {
|
||||
builder.append(' ');
|
||||
}
|
||||
builder.append(value.trim());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isBlank(String value) {
|
||||
return value == null || value.trim().isEmpty();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -258,7 +258,6 @@ public class DiagnosisTraceService {
|
||||
.persistedToolCallCount(defaultInt(session.getToolCallCount()))
|
||||
.returnedToolCallCount(toolInvocations.size())
|
||||
.hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation"))
|
||||
.hasAiOpsRuleEvaluation(selfEvaluation != null && selfEvaluation.containsKey("aiops_rule_evaluation"))
|
||||
.hasFeedback(session.getFeedback() != null && !session.getFeedback().isBlank())
|
||||
.build();
|
||||
}
|
||||
@@ -276,7 +275,6 @@ public class DiagnosisTraceService {
|
||||
.persistedToolCallCount(defaultInt(run.getToolCallCount()))
|
||||
.returnedToolCallCount(toolInvocations.size())
|
||||
.hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation"))
|
||||
.hasAiOpsRuleEvaluation(selfEvaluation != null && selfEvaluation.containsKey("aiops_rule_evaluation"))
|
||||
.hasFeedback(run.getFeedback() != null && !run.getFeedback().isBlank())
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -1,614 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Deterministic checks for Executor structured output before verifier reasoning.
|
||||
*/
|
||||
@Service
|
||||
public class ExecutorGatekeeperService {
|
||||
|
||||
public static final String STATUS_PASS = "pass";
|
||||
public static final String STATUS_FAIL = "fail";
|
||||
public static final String SEVERITY_NONE = "none";
|
||||
public static final String SEVERITY_LOW_CONFID = "low_confid";
|
||||
public static final String SEVERITY_REJECT = "reject";
|
||||
public static final String RULE_SCHEMA = "schema.executor_v2";
|
||||
public static final String RULE_INVOCATION_REF = "evidence.invocation_ref";
|
||||
public static final String RULE_RAW_PATH = "evidence.raw_path";
|
||||
public static final String RULE_EXCERPT_MISMATCH = "evidence.excerpt_mismatch";
|
||||
public static final String RULE_EVIDENCE_MISSING = "evidence.missing";
|
||||
|
||||
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
|
||||
};
|
||||
private static final double DEFAULT_MIN_TOKEN_OVERLAP = 0.5;
|
||||
|
||||
private final ToolInvocationRepository toolInvocationRepository;
|
||||
private final GatekeeperRuleCatalog ruleCatalog;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Autowired
|
||||
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository) {
|
||||
this(toolInvocationRepository, null);
|
||||
}
|
||||
|
||||
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository,
|
||||
GatekeeperRuleCatalog ruleCatalog) {
|
||||
this.toolInvocationRepository = toolInvocationRepository;
|
||||
this.ruleCatalog = ruleCatalog == null
|
||||
? GatekeeperRuleCatalog.loadDefault(objectMapper)
|
||||
: ruleCatalog;
|
||||
}
|
||||
|
||||
public Map<String, Object> validate(String sessionId,
|
||||
Map<String, Object> structuredOutput,
|
||||
Map<String, Object> parseStatus) {
|
||||
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
|
||||
validateSchema(structuredOutput, parseStatus, result);
|
||||
if (structuredOutput != null) {
|
||||
List<ToolInvocation> invocations = sessionId == null || sessionId.isBlank()
|
||||
? List.of()
|
||||
: toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId);
|
||||
validateInvocationRefs("session_id", sessionId, invocations, structuredOutput, result);
|
||||
importWarnings(structuredOutput, result);
|
||||
}
|
||||
return result.toMap();
|
||||
}
|
||||
|
||||
public Map<String, Object> validateRun(String runId,
|
||||
Map<String, Object> structuredOutput,
|
||||
Map<String, Object> parseStatus) {
|
||||
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
|
||||
validateSchema(structuredOutput, parseStatus, result);
|
||||
if (structuredOutput != null) {
|
||||
List<ToolInvocation> invocations = runId == null || runId.isBlank()
|
||||
? List.of()
|
||||
: toolInvocationRepository.findByRunIdOrderByIdAsc(runId);
|
||||
validateInvocationRefs("run_id", runId, invocations, structuredOutput, result);
|
||||
importWarnings(structuredOutput, result);
|
||||
}
|
||||
return result.toMap();
|
||||
}
|
||||
|
||||
public Map<String, Object> pass() {
|
||||
return new GatekeeperResult(ruleCatalog).toMap();
|
||||
}
|
||||
|
||||
public Map<String, Object> fail(String ruleId, String target, String message) {
|
||||
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
|
||||
result.fail(ruleId, target, message, SEVERITY_REJECT);
|
||||
return result.toMap();
|
||||
}
|
||||
|
||||
private void validateSchema(Map<String, Object> structuredOutput,
|
||||
Map<String, Object> parseStatus,
|
||||
GatekeeperResult result) {
|
||||
String status = parseStatus == null ? "" : String.valueOf(parseStatus.getOrDefault("status", ""));
|
||||
if (structuredOutput == null) {
|
||||
if ("valid".equals(status)) {
|
||||
result.fail(RULE_SCHEMA, "executor_structured_output",
|
||||
"structured output is missing after valid parse", SEVERITY_LOW_CONFID);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!"executor_evidence_v2".equals(String.valueOf(structuredOutput.get("answer_version")))) {
|
||||
result.fail(RULE_SCHEMA, "answer_version", "answer_version must be executor_evidence_v2",
|
||||
SEVERITY_LOW_CONFID);
|
||||
}
|
||||
if (structuredOutput.containsKey("diagnosis_summary")) {
|
||||
result.fail(RULE_SCHEMA, "diagnosis_summary", "diagnosis_summary is removed from executor_evidence_v2",
|
||||
SEVERITY_REJECT);
|
||||
}
|
||||
if (structuredOutput.containsKey("user_facing_answer")) {
|
||||
result.fail(RULE_SCHEMA, "user_facing_answer", "user_facing_answer is removed from executor_evidence_v2",
|
||||
SEVERITY_REJECT);
|
||||
}
|
||||
|
||||
Object claimsValue = structuredOutput.get("claims");
|
||||
if (!(claimsValue instanceof List<?> claims)) {
|
||||
result.fail(RULE_SCHEMA, "claims", "claims must be an array", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < claims.size(); i++) {
|
||||
String target = "claims[" + i + "]";
|
||||
Object claimValue = claims.get(i);
|
||||
if (!(claimValue instanceof Map<?, ?> claim)) {
|
||||
result.fail(RULE_SCHEMA, target, "claim must be an object", SEVERITY_LOW_CONFID);
|
||||
continue;
|
||||
}
|
||||
requireString(claim, "claim_id", target, result);
|
||||
requireString(claim, "claim_type", target, result);
|
||||
requireString(claim, "claim_text", target, result);
|
||||
String supportLevel = stringValue(claim.get("support_level"));
|
||||
if (!"direct".equals(supportLevel) && !"indirect".equals(supportLevel)) {
|
||||
result.fail(RULE_SCHEMA, target + ".support_level", "support_level must be direct or indirect",
|
||||
SEVERITY_LOW_CONFID);
|
||||
}
|
||||
Object bindings = claim.get("evidence_bindings");
|
||||
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
|
||||
result.fail(RULE_EVIDENCE_MISSING, target + ".evidence_bindings",
|
||||
"claims must include non-empty evidence_bindings", SEVERITY_LOW_CONFID);
|
||||
}
|
||||
}
|
||||
|
||||
requireArray(structuredOutput, "hypotheses", result);
|
||||
requireArray(structuredOutput, "recommended_actions", result);
|
||||
requireArray(structuredOutput, "missing_info", result);
|
||||
}
|
||||
|
||||
private void validateInvocationRefs(String scopeName,
|
||||
String scopeId,
|
||||
List<ToolInvocation> invocations,
|
||||
Map<String, Object> structuredOutput,
|
||||
GatekeeperResult result) {
|
||||
if (scopeId == null || scopeId.isBlank()) {
|
||||
result.fail(RULE_INVOCATION_REF, scopeName, scopeName + " is required to validate source_invocation_id",
|
||||
SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
|
||||
Map<Long, ToolInvocation> validInvocations = invocations.stream()
|
||||
.filter(invocation -> invocation.getId() != null)
|
||||
.collect(Collectors.toMap(ToolInvocation::getId, Function.identity(), (left, right) -> left));
|
||||
Object claimsValue = structuredOutput.get("claims");
|
||||
if (!(claimsValue instanceof List<?> claims)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (int claimIndex = 0; claimIndex < claims.size(); claimIndex++) {
|
||||
Object claimValue = claims.get(claimIndex);
|
||||
if (!(claimValue instanceof Map<?, ?> claim)) {
|
||||
continue;
|
||||
}
|
||||
Object bindingsValue = claim.get("evidence_bindings");
|
||||
if (!(bindingsValue instanceof List<?> bindings)) {
|
||||
continue;
|
||||
}
|
||||
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
|
||||
String target = "claims[" + claimIndex + "].evidence_bindings[" + bindingIndex + "]";
|
||||
Object bindingValue = bindings.get(bindingIndex);
|
||||
if (!(bindingValue instanceof Map<?, ?> binding)) {
|
||||
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
|
||||
SEVERITY_LOW_CONFID);
|
||||
continue;
|
||||
}
|
||||
validateEvidenceBinding(binding, validInvocations, target,
|
||||
claim.get("claim_id"), claim.get("claim_type"), result);
|
||||
}
|
||||
}
|
||||
|
||||
Object actionsValue = structuredOutput.get("recommended_actions");
|
||||
if (!(actionsValue instanceof List<?> actions)) {
|
||||
return;
|
||||
}
|
||||
for (int actionIndex = 0; actionIndex < actions.size(); actionIndex++) {
|
||||
Object actionValue = actions.get(actionIndex);
|
||||
if (!(actionValue instanceof Map<?, ?> action)) {
|
||||
continue;
|
||||
}
|
||||
Object bindingsValue = action.get("evidence_bindings");
|
||||
if (!(bindingsValue instanceof List<?> bindings)) {
|
||||
continue;
|
||||
}
|
||||
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
|
||||
String target = "recommended_actions[" + actionIndex + "].evidence_bindings[" + bindingIndex + "]";
|
||||
Object bindingValue = bindings.get(bindingIndex);
|
||||
if (!(bindingValue instanceof Map<?, ?> binding)) {
|
||||
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
|
||||
SEVERITY_LOW_CONFID);
|
||||
continue;
|
||||
}
|
||||
validateEvidenceBinding(binding, validInvocations, target, action.get("action_id"), null, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validateEvidenceBinding(Map<?, ?> binding,
|
||||
Map<Long, ToolInvocation> validInvocations,
|
||||
String target,
|
||||
Object ownerId,
|
||||
Object ownerType,
|
||||
GatekeeperResult result) {
|
||||
Map<String, Object> checked = new LinkedHashMap<>();
|
||||
checked.put("claim_id", ownerId == null ? "" : String.valueOf(ownerId));
|
||||
checked.put("tool_name", stringValue(binding.get("tool_name")));
|
||||
checked.put("source_invocation_id", binding.get("source_invocation_id"));
|
||||
checked.put("raw_path", stringValue(binding.get("raw_path")));
|
||||
|
||||
String claimedToolName = stringValue(binding.get("tool_name"));
|
||||
if (claimedToolName.isBlank()) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_INVOCATION_REF);
|
||||
checked.put("message", "tool_name is required");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_INVOCATION_REF, target + ".tool_name", "tool_name is required", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
|
||||
String rawPath = stringValue(binding.get("raw_path"));
|
||||
if ("negative_observation".equals(stringValue(ownerType)) && !rawPath.isBlank()
|
||||
&& !"$.no_evidence".equals(rawPath)) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_RAW_PATH);
|
||||
checked.put("message", "negative_observation must only bind $.no_evidence references");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_RAW_PATH, target + ".raw_path",
|
||||
"negative_observation must only bind $.no_evidence references", SEVERITY_REJECT);
|
||||
return;
|
||||
}
|
||||
|
||||
Long id = singleInvocationId(binding);
|
||||
if (id == null) {
|
||||
id = uniqueInvocationIdByToolRawPathAndExcerpt(validInvocations, claimedToolName, rawPath,
|
||||
stringValue(binding.get("evidence_excerpt")));
|
||||
if (id == null) {
|
||||
id = uniqueInvocationIdByToolAndRawPath(validInvocations, claimedToolName, rawPath);
|
||||
}
|
||||
if (id != null) {
|
||||
result.warn(Map.of(
|
||||
"rule", "evidence.invocation_auto_backfill_by_raw_path",
|
||||
"message", "source_invocation_id was auto-filled from the unique evidence reference candidate",
|
||||
"tool_name", claimedToolName,
|
||||
"raw_path", rawPath,
|
||||
"source_invocation_id", id
|
||||
));
|
||||
}
|
||||
}
|
||||
if (id == null) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_INVOCATION_REF);
|
||||
checked.put("message", "source_invocation_id is required");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_INVOCATION_REF, target + ".source_invocation_id",
|
||||
"source_invocation_id is required", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
checked.put("source_invocation_id", id);
|
||||
|
||||
ToolInvocation invocation = validInvocations.get(id);
|
||||
if (invocation == null) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_INVOCATION_REF);
|
||||
checked.put("message", "source_invocation_id not found in current session: " + id);
|
||||
result.checked(checked);
|
||||
result.fail(RULE_INVOCATION_REF, target,
|
||||
"source_invocation_id not found in current session: " + id, SEVERITY_REJECT);
|
||||
return;
|
||||
}
|
||||
if (!Objects.equals(claimedToolName, invocation.getToolName())) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_INVOCATION_REF);
|
||||
checked.put("message", "tool_name does not match invocation " + id + ": expected " + invocation.getToolName());
|
||||
result.checked(checked);
|
||||
result.fail(RULE_INVOCATION_REF, target + ".tool_name",
|
||||
"tool_name does not match invocation " + id + ": expected " + invocation.getToolName(),
|
||||
SEVERITY_REJECT);
|
||||
return;
|
||||
}
|
||||
|
||||
if (rawPath.isBlank()) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_RAW_PATH);
|
||||
checked.put("message", "raw_path is required for precise evidence reference");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_RAW_PATH, target + ".raw_path",
|
||||
"raw_path is required for precise evidence reference", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, String> refs = evidenceRefsByRawPath(invocation.getRetrievalDetails());
|
||||
if (refs.isEmpty()) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_EVIDENCE_MISSING);
|
||||
checked.put("message", "invocation has no retrieval_details.evidence_refs");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_EVIDENCE_MISSING, target,
|
||||
"invocation has no retrieval_details.evidence_refs", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
|
||||
String matchedText = refs.get(rawPath);
|
||||
if (matchedText == null) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_RAW_PATH);
|
||||
checked.put("message", "raw_path not found in retrieval_details.evidence_refs");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_RAW_PATH, target + ".raw_path",
|
||||
"raw_path not found in retrieval_details.evidence_refs", SEVERITY_REJECT);
|
||||
return;
|
||||
}
|
||||
checked.put("matched_text", matchedText);
|
||||
|
||||
String excerpt = stringValue(binding.get("evidence_excerpt"));
|
||||
if (normalized(excerpt).length() < 8) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_EXCERPT_MISMATCH);
|
||||
checked.put("message", "evidence_excerpt is too short to compare");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
|
||||
"evidence_excerpt is too short to compare", SEVERITY_LOW_CONFID);
|
||||
return;
|
||||
}
|
||||
if (!isExcerptSupported(excerpt, matchedText)) {
|
||||
checked.put("status", STATUS_FAIL);
|
||||
checked.put("rule", RULE_EXCERPT_MISMATCH);
|
||||
checked.put("message", "evidence_excerpt is not supported by matched evidence ref text");
|
||||
result.checked(checked);
|
||||
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
|
||||
"evidence_excerpt is not supported by matched evidence ref text", SEVERITY_REJECT);
|
||||
return;
|
||||
}
|
||||
|
||||
checked.put("status", STATUS_PASS);
|
||||
result.checked(checked);
|
||||
}
|
||||
|
||||
private Long uniqueInvocationIdByToolAndRawPath(Map<Long, ToolInvocation> validInvocations,
|
||||
String toolName,
|
||||
String rawPath) {
|
||||
if (toolName == null || toolName.isBlank() || rawPath == null || rawPath.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
Long matchedId = null;
|
||||
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
|
||||
ToolInvocation invocation = entry.getValue();
|
||||
if (!Objects.equals(toolName, invocation.getToolName())) {
|
||||
continue;
|
||||
}
|
||||
if (!evidenceRefsByRawPath(invocation.getRetrievalDetails()).containsKey(rawPath)) {
|
||||
continue;
|
||||
}
|
||||
if (matchedId != null) {
|
||||
return null;
|
||||
}
|
||||
matchedId = entry.getKey();
|
||||
}
|
||||
return matchedId;
|
||||
}
|
||||
|
||||
private Long uniqueInvocationIdByToolRawPathAndExcerpt(Map<Long, ToolInvocation> validInvocations,
|
||||
String toolName,
|
||||
String rawPath,
|
||||
String excerpt) {
|
||||
if (toolName == null || toolName.isBlank()
|
||||
|| rawPath == null || rawPath.isBlank()
|
||||
|| excerpt == null || excerpt.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
Long matchedId = null;
|
||||
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
|
||||
ToolInvocation invocation = entry.getValue();
|
||||
if (!Objects.equals(toolName, invocation.getToolName())) {
|
||||
continue;
|
||||
}
|
||||
String matchedText = evidenceRefsByRawPath(invocation.getRetrievalDetails()).get(rawPath);
|
||||
if (matchedText == null || !isBackfillCandidateSupported(rawPath, excerpt, matchedText)) {
|
||||
continue;
|
||||
}
|
||||
if (matchedId != null) {
|
||||
return null;
|
||||
}
|
||||
matchedId = entry.getKey();
|
||||
}
|
||||
return matchedId;
|
||||
}
|
||||
|
||||
private boolean isBackfillCandidateSupported(String rawPath, String excerpt, String matchedText) {
|
||||
if ("$.no_evidence".equals(rawPath)) {
|
||||
String excerptQuery = semicolonField(excerpt, "query");
|
||||
String matchedQuery = semicolonField(matchedText, "query");
|
||||
if (!excerptQuery.isBlank() && !matchedQuery.isBlank()
|
||||
&& !normalized(excerptQuery).equals(normalized(matchedQuery))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return isExcerptSupported(excerpt, matchedText);
|
||||
}
|
||||
|
||||
private String semicolonField(String text, String field) {
|
||||
if (text == null || text.isBlank() || field == null || field.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
String prefix = field + "=";
|
||||
for (String part : text.split(";")) {
|
||||
String trimmed = part.trim();
|
||||
if (trimmed.regionMatches(true, 0, prefix, 0, prefix.length())) {
|
||||
return trimmed.substring(prefix.length()).trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private Long singleInvocationId(Map<?, ?> binding) {
|
||||
Long singular = asLong(binding.get("source_invocation_id"));
|
||||
if (singular != null) {
|
||||
return singular;
|
||||
}
|
||||
Object idsValue = binding.get("source_invocation_ids");
|
||||
if (!(idsValue instanceof List<?> ids) || ids.size() != 1) {
|
||||
return null;
|
||||
}
|
||||
return asLong(ids.get(0));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void importWarnings(Map<String, Object> structuredOutput, GatekeeperResult result) {
|
||||
Object warningsValue = structuredOutput.remove("_gatekeeper_warnings");
|
||||
if (!(warningsValue instanceof List<?> warnings)) {
|
||||
return;
|
||||
}
|
||||
for (Object warning : warnings) {
|
||||
if (warning instanceof Map<?, ?> map) {
|
||||
result.warn((Map<String, Object>) map);
|
||||
} else if (warning != null) {
|
||||
result.warn(Map.of("message", String.valueOf(warning)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> evidenceRefsByRawPath(String retrievalDetails) {
|
||||
if (retrievalDetails == null || retrievalDetails.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> details = objectMapper.readValue(retrievalDetails, MAP_TYPE);
|
||||
Object refsValue = details.get("evidence_refs");
|
||||
if (!(refsValue instanceof List<?> refs)) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
for (Object refValue : refs) {
|
||||
if (!(refValue instanceof Map<?, ?> ref)) {
|
||||
continue;
|
||||
}
|
||||
String rawPath = stringValue(ref.get("raw_path"));
|
||||
String text = stringValue(ref.get("text"));
|
||||
if (!rawPath.isBlank() && !text.isBlank()) {
|
||||
result.put(rawPath, text);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} catch (Exception ignored) {
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isExcerptSupported(String excerpt, String matchedText) {
|
||||
String normalizedExcerpt = normalized(excerpt);
|
||||
String normalizedMatched = normalized(matchedText);
|
||||
if (normalizedMatched.contains(normalizedExcerpt) || normalizedExcerpt.contains(normalizedMatched)) {
|
||||
return true;
|
||||
}
|
||||
Set<String> excerptTokens = tokens(normalizedExcerpt);
|
||||
if (excerptTokens.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
Set<String> matchedTokens = tokens(normalizedMatched);
|
||||
int overlap = 0;
|
||||
for (String token : excerptTokens) {
|
||||
if (matchedTokens.contains(token)) {
|
||||
overlap++;
|
||||
}
|
||||
}
|
||||
double minTokenOverlap = ruleCatalog.doubleParameter(RULE_EXCERPT_MISMATCH,
|
||||
"min_token_overlap", DEFAULT_MIN_TOKEN_OVERLAP);
|
||||
return (double) overlap / excerptTokens.size() >= minTokenOverlap;
|
||||
}
|
||||
|
||||
private String normalized(String value) {
|
||||
return value == null ? "" : value.toLowerCase()
|
||||
.replaceAll("[\\p{Punct}\\s，。；：、（）【】《》“”‘’]+", " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
private Set<String> tokens(String text) {
|
||||
if (text == null || text.isBlank()) {
|
||||
return Set.of();
|
||||
}
|
||||
Set<String> result = new HashSet<>();
|
||||
for (String token : text.split("\\s+")) {
|
||||
if (token.length() >= 2) {
|
||||
result.add(token);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private void requireArray(Map<String, Object> output, String field, GatekeeperResult result) {
|
||||
if (!(output.get(field) instanceof List<?>)) {
|
||||
result.fail(RULE_SCHEMA, field, field + " must be an array", SEVERITY_LOW_CONFID);
|
||||
}
|
||||
}
|
||||
|
||||
private void requireString(Map<?, ?> object, String field, String target, GatekeeperResult result) {
|
||||
if (stringValue(object.get(field)).isBlank()) {
|
||||
result.fail(RULE_SCHEMA, target + "." + field, field + " is required", SEVERITY_LOW_CONFID);
|
||||
}
|
||||
}
|
||||
|
||||
private String stringValue(Object value) {
|
||||
return value == null ? "" : String.valueOf(value);
|
||||
}
|
||||
|
||||
private Long asLong(Object value) {
|
||||
if (value instanceof Number number) {
|
||||
return number.longValue();
|
||||
}
|
||||
if (value instanceof String text) {
|
||||
try {
|
||||
return Long.parseLong(text);
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static final class GatekeeperResult {
|
||||
private final GatekeeperRuleCatalog ruleCatalog;
|
||||
private final List<String> failedRules = new ArrayList<>();
|
||||
private final List<Map<String, Object>> checkedBindings = new ArrayList<>();
|
||||
private final List<Map<String, Object>> warnings = new ArrayList<>();
|
||||
private final List<Map<String, Object>> errors = new ArrayList<>();
|
||||
private String severity = SEVERITY_NONE;
|
||||
|
||||
GatekeeperResult(GatekeeperRuleCatalog ruleCatalog) {
|
||||
this.ruleCatalog = ruleCatalog == null ? GatekeeperRuleCatalog.fallback() : ruleCatalog;
|
||||
}
|
||||
|
||||
void fail(String ruleId, String target, String message, String failureSeverity) {
|
||||
if (!failedRules.contains(ruleId)) {
|
||||
failedRules.add(ruleId);
|
||||
}
|
||||
if (SEVERITY_REJECT.equals(failureSeverity)) {
|
||||
severity = SEVERITY_REJECT;
|
||||
} else if (!SEVERITY_REJECT.equals(severity)) {
|
||||
severity = SEVERITY_LOW_CONFID;
|
||||
}
|
||||
Map<String, Object> error = new LinkedHashMap<>();
|
||||
error.put("rule_id", ruleId);
|
||||
error.put("rule", ruleId);
|
||||
error.put("target", target);
|
||||
error.put("message", message);
|
||||
error.put("severity", failureSeverity);
|
||||
errors.add(error);
|
||||
}
|
||||
|
||||
void checked(Map<String, Object> checked) {
|
||||
checkedBindings.add(checked);
|
||||
}
|
||||
|
||||
void warn(Map<String, Object> warning) {
|
||||
warnings.add(new LinkedHashMap<>(warning));
|
||||
}
|
||||
|
||||
Map<String, Object> toMap() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("status", failedRules.isEmpty() ? STATUS_PASS : STATUS_FAIL);
|
||||
result.put("severity", failedRules.isEmpty() ? SEVERITY_NONE : severity);
|
||||
result.put("rule_set_version", ruleCatalog.version());
|
||||
result.put("rules", ruleCatalog.auditRules());
|
||||
result.put("checked_bindings", checkedBindings);
|
||||
result.put("failed_rules", failedRules);
|
||||
result.put("warnings", warnings);
|
||||
result.put("errors", errors);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Lightweight metadata catalog for deterministic Gatekeeper rules.
|
||||
*/
|
||||
public class GatekeeperRuleCatalog {
|
||||
|
||||
public static final String DEFAULT_RESOURCE = "gatekeeper/gatekeeper-rules.json";
|
||||
public static final String FALLBACK_VERSION = "gatekeeper-rules-v1";
|
||||
|
||||
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
|
||||
};
|
||||
|
||||
private final String version;
|
||||
private final List<RuleMetadata> rules;
|
||||
|
||||
public GatekeeperRuleCatalog(String version, List<RuleMetadata> rules) {
|
||||
this.version = version == null || version.isBlank() ? FALLBACK_VERSION : version;
|
||||
this.rules = List.copyOf(rules == null ? List.of() : rules);
|
||||
}
|
||||
|
||||
public static GatekeeperRuleCatalog loadDefault(ObjectMapper objectMapper) {
|
||||
try (InputStream input = GatekeeperRuleCatalog.class.getClassLoader()
|
||||
.getResourceAsStream(DEFAULT_RESOURCE)) {
|
||||
if (input == null) {
|
||||
return fallback();
|
||||
}
|
||||
Map<String, Object> root = objectMapper.readValue(input, MAP_TYPE);
|
||||
String version = stringValue(root.get("version"));
|
||||
List<RuleMetadata> rules = new ArrayList<>();
|
||||
Object rulesValue = root.get("rules");
|
||||
if (rulesValue instanceof List<?> ruleList) {
|
||||
for (Object ruleValue : ruleList) {
|
||||
if (ruleValue instanceof Map<?, ?> ruleMap) {
|
||||
rules.add(RuleMetadata.from(ruleMap));
|
||||
}
|
||||
}
|
||||
}
|
||||
return new GatekeeperRuleCatalog(version, rules);
|
||||
} catch (Exception ignored) {
|
||||
return fallback();
|
||||
}
|
||||
}
|
||||
|
||||
public static GatekeeperRuleCatalog fallback() {
|
||||
return new GatekeeperRuleCatalog(FALLBACK_VERSION, List.of(
|
||||
new RuleMetadata("schema.executor_v2", "Executor output must match executor_evidence_v2 schema",
|
||||
true, "low_confid", Map.of()),
|
||||
new RuleMetadata("evidence.invocation_ref", "source_invocation_id must refer to a real current-session tool invocation",
|
||||
true, "reject", Map.of()),
|
||||
new RuleMetadata("evidence.raw_path", "raw_path must exist in retrieval_details.evidence_refs",
|
||||
true, "reject", Map.of()),
|
||||
new RuleMetadata("evidence.excerpt_mismatch", "evidence_excerpt must be supported by the matched evidence ref text",
|
||||
true, "reject", Map.of("min_token_overlap", 0.5)),
|
||||
new RuleMetadata("evidence.missing", "claims must include usable evidence bindings",
|
||||
true, "low_confid", Map.of())
|
||||
));
|
||||
}
|
||||
|
||||
public String version() {
|
||||
return version;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> auditRules() {
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
for (RuleMetadata rule : rules) {
|
||||
if (rule.enabled()) {
|
||||
result.add(rule.toAuditMap());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public double doubleParameter(String ruleId, String parameterName, double fallback) {
|
||||
for (RuleMetadata rule : rules) {
|
||||
if (!rule.id().equals(ruleId) || !rule.enabled()) {
|
||||
continue;
|
||||
}
|
||||
Object value = rule.parameters().get(parameterName);
|
||||
if (value instanceof Number number) {
|
||||
return number.doubleValue();
|
||||
}
|
||||
if (value instanceof String text) {
|
||||
try {
|
||||
return Double.parseDouble(text);
|
||||
} catch (NumberFormatException ignored) {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
private static String stringValue(Object value) {
|
||||
return value == null ? "" : String.valueOf(value);
|
||||
}
|
||||
|
||||
public record RuleMetadata(String id,
|
||||
String description,
|
||||
boolean enabled,
|
||||
String defaultSeverity,
|
||||
Map<String, Object> parameters) {
|
||||
|
||||
static RuleMetadata from(Map<?, ?> raw) {
|
||||
String id = stringValue(raw.get("id"));
|
||||
String description = stringValue(raw.get("description"));
|
||||
boolean enabled = !(raw.get("enabled") instanceof Boolean value) || value;
|
||||
String defaultSeverity = stringValue(raw.get("default_severity"));
|
||||
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||
Object parametersValue = raw.get("parameters");
|
||||
if (parametersValue instanceof Map<?, ?> parameterMap) {
|
||||
for (Map.Entry<?, ?> entry : parameterMap.entrySet()) {
|
||||
if (entry.getKey() != null) {
|
||||
parameters.put(String.valueOf(entry.getKey()), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
return new RuleMetadata(id, description, enabled, defaultSeverity, parameters);
|
||||
}
|
||||
|
||||
Map<String, Object> toAuditMap() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("id", id);
|
||||
result.put("description", description);
|
||||
result.put("enabled", enabled);
|
||||
result.put("default_severity", defaultSeverity);
|
||||
if (!parameters.isEmpty()) {
|
||||
result.put("parameters", parameters);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -169,8 +169,8 @@ public class KnowledgeIndexService {
|
||||
}
|
||||
|
||||
long elapsedTime = System.currentTimeMillis() - startTime;
|
||||
log.debug("L0 Hint分析: query={}, matches={}, domains={}, keywords={}, indexSize={}, time={}ms",
|
||||
query, results.size(), domains, matchedKeywords, knowledgeIndex.size(), elapsedTime);
|
||||
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
|
||||
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
|
||||
|
||||
return new L0Hint(
|
||||
List.copyOf(results),
|
||||
|
||||
@@ -27,10 +27,6 @@ public class SelfEvaluationMergeService {
|
||||
return merge(existingJson, "verifier_evaluation", verifierEvaluation);
|
||||
}
|
||||
|
||||
public String mergeAiOpsRuleEvaluation(String existingJson, Map<String, Object> aiOpsRuleEvaluation) {
|
||||
return merge(existingJson, "aiops_rule_evaluation", aiOpsRuleEvaluation);
|
||||
}
|
||||
|
||||
private String merge(String existingJson, String key, Map<String, Object> value) {
|
||||
try {
|
||||
Map<String, Object> root = parseRoot(existingJson);
|
||||
@@ -49,8 +45,7 @@ public class SelfEvaluationMergeService {
|
||||
|
||||
Map<String, Object> parsed = objectMapper.readValue(existingJson, MAP_TYPE);
|
||||
if (parsed.containsKey("rule_evaluation")
|
||||
|| parsed.containsKey("verifier_evaluation")
|
||||
|| parsed.containsKey("aiops_rule_evaluation")) {
|
||||
|| parsed.containsKey("verifier_evaluation")) {
|
||||
return new LinkedHashMap<>(parsed);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,661 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.ContextPack;
|
||||
import com.superbiz.agent.dto.EvidenceBlock;
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
import com.superbiz.agent.dto.RerankTrace;
|
||||
import com.superbiz.agent.dto.RetrievalTrace;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import lombok.Builder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Central persistence point for agent evidence tool invocations.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ToolInvocationRecorder {
|
||||
|
||||
private static final int OUTPUT_PREVIEW_LIMIT = 500;
|
||||
public static final String EVIDENCE_STATUS_SUPPORTED = "supported";
|
||||
public static final String EVIDENCE_STATUS_NO_EVIDENCE = "no_evidence";
|
||||
public static final String EVIDENCE_STATUS_DEDUPED = "deduped";
|
||||
public static final String EVIDENCE_STATUS_FAILED = "failed";
|
||||
|
||||
private final ToolInvocationRepository toolInvocationRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ToolInvocationRecorder(ToolInvocationRepository toolInvocationRepository, ObjectMapper objectMapper) {
|
||||
this.toolInvocationRepository = toolInvocationRepository;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public void save(ToolInvocation invocation) {
|
||||
try {
|
||||
if (invocation.getSessionId() == null || invocation.getSessionId().isBlank()) {
|
||||
invocation.setSessionId(SessionContextHolder.getSessionId());
|
||||
}
|
||||
if (invocation.getRunId() == null || invocation.getRunId().isBlank()) {
|
||||
invocation.setRunId(SessionContextHolder.getRunId());
|
||||
}
|
||||
if (invocation.getSessionId() == null || invocation.getSessionId().isBlank()) {
|
||||
log.debug("Skip tool_invocation without sessionId: tool={}", invocation.getToolName());
|
||||
return;
|
||||
}
|
||||
toolInvocationRepository.save(invocation);
|
||||
} catch (Exception e) {
|
||||
log.error("保存 tool_invocation 失败: tool={}", invocation.getToolName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public void recordEvidenceTool(String toolName,
|
||||
Map<String, Object> inputParams,
|
||||
String output,
|
||||
boolean success,
|
||||
long startTimeMillis,
|
||||
String errorMessage,
|
||||
String topicDomain) {
|
||||
recordEvidenceTool(toolName, inputParams, output, success, startTimeMillis, errorMessage, topicDomain,
|
||||
success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED, Map.of());
|
||||
}
|
||||
|
||||
public void recordEvidenceTool(String toolName,
|
||||
Map<String, Object> inputParams,
|
||||
String output,
|
||||
boolean success,
|
||||
long startTimeMillis,
|
||||
String errorMessage,
|
||||
String topicDomain,
|
||||
String evidenceStatus,
|
||||
Map<String, Object> extraDetails) {
|
||||
String outputPreview = preview(output);
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("trace_id", UUID.randomUUID().toString());
|
||||
if (topicDomain != null && !topicDomain.isBlank()) {
|
||||
details.put("retrieved_domains", List.of(topicDomain));
|
||||
}
|
||||
details.put("evidence_status", normalizeEvidenceStatus(success, evidenceStatus));
|
||||
if (extraDetails != null && !extraDetails.isEmpty()) {
|
||||
details.putAll(extraDetails);
|
||||
}
|
||||
List<Map<String, Object>> evidenceRefs = extractEvidenceRefs(toolName, inputParams, output, details);
|
||||
if (!evidenceRefs.isEmpty()) {
|
||||
details.put("evidence_refs", evidenceRefs);
|
||||
}
|
||||
|
||||
ToolInvocation invocation = ToolInvocation.builder()
|
||||
.toolName(toolName)
|
||||
.inputParams(toJson(inputParams == null ? Map.of() : inputParams))
|
||||
.outputPreview(outputPreview)
|
||||
.outputLength(output == null ? 0 : output.length())
|
||||
.isTruncated(output != null && output.length() > OUTPUT_PREVIEW_LIMIT)
|
||||
.retrievalDetails(toJson(details))
|
||||
.durationMs((int) Math.max(0, System.currentTimeMillis() - startTimeMillis))
|
||||
.success(success)
|
||||
.errorMessage(errorMessage)
|
||||
.build();
|
||||
save(invocation);
|
||||
}
|
||||
|
||||
public void recordLookupKnowledge(LookupKnowledgeRecord record) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("trace_id", UUID.randomUUID().toString());
|
||||
if (record.l0MatchCount() != null) {
|
||||
details.put("l0_match_count", record.l0MatchCount());
|
||||
}
|
||||
if (record.l0Titles() != null && !record.l0Titles().isEmpty()) {
|
||||
details.put("l0_titles", record.l0Titles());
|
||||
}
|
||||
if (record.l0MatchedKeywords() != null && !record.l0MatchedKeywords().isEmpty()) {
|
||||
details.put("l0_matched_keywords", record.l0MatchedKeywords());
|
||||
}
|
||||
if (record.l0Domains() != null && !record.l0Domains().isEmpty()) {
|
||||
details.put("l0_domains", record.l0Domains());
|
||||
}
|
||||
if (record.l0Entities() != null && !record.l0Entities().isEmpty()) {
|
||||
details.put("l0_entities", record.l0Entities());
|
||||
}
|
||||
if (record.l1TopScore() != null) {
|
||||
details.put("l1_top_score", record.l1TopScore());
|
||||
}
|
||||
if (record.l1TopSimilarity() != null) {
|
||||
details.put("l1_top_similarity", record.l1TopSimilarity());
|
||||
}
|
||||
if (record.l1MatchCount() != null) {
|
||||
details.put("l1_match_count", record.l1MatchCount());
|
||||
}
|
||||
if (record.l1Scores() != null && !record.l1Scores().isEmpty()) {
|
||||
details.put("l1_scores", record.l1Scores());
|
||||
}
|
||||
if (record.evidenceCandidateCount() != null) {
|
||||
details.put("evidence_candidate_count", record.evidenceCandidateCount());
|
||||
}
|
||||
if (record.evidenceBlockCount() != null) {
|
||||
details.put("evidence_block_count", record.evidenceBlockCount());
|
||||
}
|
||||
details.put("evidence_blocks", record.evidenceBlocks() == null ? List.of() : record.evidenceBlocks());
|
||||
List<Map<String, Object>> evidenceRefs = evidenceRefsFromEvidenceBlocks(record.evidenceBlocks());
|
||||
if (evidenceRefs.isEmpty()
|
||||
&& EVIDENCE_STATUS_NO_EVIDENCE.equals(normalizeEvidenceStatus(record.success(), record.evidenceStatus()))) {
|
||||
Map<String, Object> input = new LinkedHashMap<>();
|
||||
input.put("query", record.query());
|
||||
evidenceRefs = List.of(noEvidenceRef("lookup_knowledge", input, null, details));
|
||||
}
|
||||
if (!evidenceRefs.isEmpty()) {
|
||||
details.put("evidence_refs", evidenceRefs);
|
||||
}
|
||||
details.put("query_transform", record.queryTransform() == null ? Map.of() : record.queryTransform());
|
||||
details.put("retrieval_trace", record.retrievalTrace() == null ? Map.of() : record.retrievalTrace());
|
||||
details.put("context_pack_summary", record.contextPack() == null ? Map.of() : record.contextPack());
|
||||
details.put("rerank_trace", record.rerankTrace() == null ? Map.of() : record.rerankTrace());
|
||||
if (record.fallbackReason() != null && !record.fallbackReason().isBlank()) {
|
||||
details.put("fallback_reason", record.fallbackReason());
|
||||
}
|
||||
if (record.relevanceLevel() != null) {
|
||||
details.put("relevance_level", record.relevanceLevel());
|
||||
}
|
||||
if (record.completenessHint() != null) {
|
||||
details.put("completeness_hint", record.completenessHint());
|
||||
}
|
||||
if (record.domain() != null && !record.domain().isBlank()) {
|
||||
details.put("retrieved_domains", List.of(record.domain()));
|
||||
}
|
||||
if (record.dedupReason() != null) {
|
||||
details.put("dedup_reason", record.dedupReason());
|
||||
}
|
||||
details.put("evidence_status", normalizeEvidenceStatus(record.success(), record.evidenceStatus()));
|
||||
|
||||
ToolInvocation invocation = ToolInvocation.builder()
|
||||
.toolName("lookup_knowledge")
|
||||
.inputParams(toJson(Map.of("query", record.query())))
|
||||
.outputPreview(preview(record.outputPreview()))
|
||||
.outputLength(record.outputLength())
|
||||
.retrievalLayer(record.retrievalLayer())
|
||||
.l0MatchCount(record.l0MatchCount())
|
||||
.l1MatchCount(record.l1MatchCount())
|
||||
.isTruncated(Boolean.TRUE.equals(record.truncated()))
|
||||
.retrievalDetails(toJson(details))
|
||||
.relevanceLevel(record.relevanceLevel())
|
||||
.dedupReason(record.dedupReason())
|
||||
.durationMs(record.durationMs())
|
||||
.success(record.success())
|
||||
.errorMessage(record.errorMessage())
|
||||
.build();
|
||||
save(invocation);
|
||||
}
|
||||
|
||||
private String normalizeEvidenceStatus(boolean success, String evidenceStatus) {
|
||||
if (evidenceStatus != null && !evidenceStatus.isBlank()) {
|
||||
return evidenceStatus;
|
||||
}
|
||||
return success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED;
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> extractEvidenceRefs(String toolName,
|
||||
Map<String, Object> inputParams,
|
||||
String output,
|
||||
Map<String, Object> details) {
|
||||
if (output == null || output.isBlank()) {
|
||||
return List.of();
|
||||
}
|
||||
if ("lookup_knowledge".equals(toolName)) {
|
||||
return evidenceRefsFromEvidenceBlocks(asMapList(details.get("evidence_blocks")));
|
||||
}
|
||||
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(output);
|
||||
boolean noEvidence = EVIDENCE_STATUS_NO_EVIDENCE.equals(stringValue(details.get("evidence_status")));
|
||||
if ("query_metrics".equals(toolName)) {
|
||||
List<Map<String, Object>> refs = evidenceRefsFromArray(root.path("alerts"), "$.alerts", this::alertText);
|
||||
return refs.isEmpty() && noEvidence ? List.of(noEvidenceRef(toolName, inputParams, root, details)) : refs;
|
||||
}
|
||||
if ("query_logs".equals(toolName)) {
|
||||
List<Map<String, Object>> refs = evidenceRefsFromArray(root.path("logs"), "$.logs", this::logText);
|
||||
return refs.isEmpty() && noEvidence ? List.of(noEvidenceRef(toolName, inputParams, root, details)) : refs;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("extract evidence_refs failed for tool={}", toolName, e);
|
||||
}
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private Map<String, Object> noEvidenceRef(String toolName,
|
||||
Map<String, Object> inputParams,
|
||||
JsonNode root,
|
||||
Map<String, Object> details) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
addPart(parts, toolName + " returned no evidence");
|
||||
addPart(parts, "evidence_status=" + stringValue(details.get("evidence_status")));
|
||||
String query = firstNonBlank(
|
||||
root == null ? null : textField(root, "query"),
|
||||
inputParams == null ? null : inputParams.get("query")
|
||||
);
|
||||
if (!query.isBlank()) {
|
||||
addPart(parts, "query=" + query);
|
||||
}
|
||||
String topic = firstNonBlank(
|
||||
root == null ? null : textField(root, "log_topic"),
|
||||
inputParams == null ? null : inputParams.get("log_topic"),
|
||||
details == null ? null : details.get("log_topic"),
|
||||
details == null ? null : details.get("metric_family")
|
||||
);
|
||||
if (!topic.isBlank()) {
|
||||
addPart(parts, "topic=" + topic);
|
||||
}
|
||||
if (root != null && root.has("total")) {
|
||||
addPart(parts, "total=" + root.path("total").asText());
|
||||
}
|
||||
String message = root == null ? "" : textField(root, "message");
|
||||
if (!message.isBlank()) {
|
||||
addPart(parts, "message=" + message);
|
||||
}
|
||||
return Map.of(
|
||||
"raw_path", "$.no_evidence",
|
||||
"text", bounded(String.join("; ", parts), 500)
|
||||
);
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> evidenceRefsFromArray(JsonNode arrayNode,
|
||||
String pathPrefix,
|
||||
java.util.function.Function<JsonNode, String> textExtractor) {
|
||||
if (!arrayNode.isArray()) {
|
||||
return List.of();
|
||||
}
|
||||
List<Map<String, Object>> refs = new ArrayList<>();
|
||||
for (int i = 0; i < arrayNode.size(); i++) {
|
||||
String text = textExtractor.apply(arrayNode.get(i));
|
||||
if (text == null || text.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
refs.add(Map.of(
|
||||
"raw_path", pathPrefix + "[" + i + "]",
|
||||
"text", bounded(text, 500)
|
||||
));
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
private String alertText(JsonNode alert) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
addPart(parts, textField(alert, "alert_name"));
|
||||
addPart(parts, textField(alert, "state"));
|
||||
addPart(parts, textField(alert, "description"));
|
||||
addPart(parts, "active_at=" + textField(alert, "active_at"));
|
||||
addPart(parts, "duration=" + textField(alert, "duration"));
|
||||
return String.join(", ", parts);
|
||||
}
|
||||
|
||||
private String logText(JsonNode log) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
addPart(parts, textField(log, "timestamp"));
|
||||
addPart(parts, textField(log, "level"));
|
||||
addPart(parts, textField(log, "service"));
|
||||
addPart(parts, textField(log, "message"));
|
||||
JsonNode metrics = log.path("metrics");
|
||||
if (metrics.isObject() && !metrics.isEmpty()) {
|
||||
addPart(parts, "metrics=" + metrics.toString());
|
||||
}
|
||||
return String.join(" ", parts);
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> evidenceRefsFromEvidenceBlocks(List<Map<String, Object>> blocks) {
|
||||
if (blocks == null || blocks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<Map<String, Object>> refs = new ArrayList<>();
|
||||
for (int i = 0; i < blocks.size(); i++) {
|
||||
Map<String, Object> block = blocks.get(i);
|
||||
String text = firstNonBlank(block.get("content_preview"), block.get("content"),
|
||||
block.get("title"), block.get("source"));
|
||||
if (text.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
refs.add(Map.of(
|
||||
"raw_path", "$.evidence_blocks[" + i + "]",
|
||||
"text", bounded(text, 500)
|
||||
));
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private List<Map<String, Object>> asMapList(Object value) {
|
||||
if (!(value instanceof List<?> list)) {
|
||||
return List.of();
|
||||
}
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
for (Object item : list) {
|
||||
if (item instanceof Map<?, ?> map) {
|
||||
result.add((Map<String, Object>) map);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String textField(JsonNode node, String field) {
|
||||
JsonNode value = node.path(field);
|
||||
return value.isMissingNode() || value.isNull() ? "" : value.asText("");
|
||||
}
|
||||
|
||||
private void addPart(List<String> parts, String value) {
|
||||
if (value != null && !value.isBlank() && !value.endsWith("=")) {
|
||||
parts.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private String firstNonBlank(Object... values) {
|
||||
for (Object value : values) {
|
||||
if (value != null && !String.valueOf(value).isBlank()) {
|
||||
return String.valueOf(value);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private String stringValue(Object value) {
|
||||
return value == null ? "" : String.valueOf(value);
|
||||
}
|
||||
|
||||
private String bounded(String value, int limit) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
}
|
||||
return value.length() <= limit ? value : value.substring(0, limit) + "...";
|
||||
}
|
||||
|
||||
private String preview(String output) {
|
||||
if (output == null) {
|
||||
return null;
|
||||
}
|
||||
return output.length() <= OUTPUT_PREVIEW_LIMIT
|
||||
? output
|
||||
: output.substring(0, OUTPUT_PREVIEW_LIMIT) + "...";
|
||||
}
|
||||
|
||||
private String toJson(Map<String, Object> value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException e) {
|
||||
log.debug("tool_invocation JSON 序列化失败", e);
|
||||
return "{}";
|
||||
}
|
||||
}
|
||||
|
||||
@Builder
|
||||
public record LookupKnowledgeRecord(
|
||||
String query,
|
||||
String outputPreview,
|
||||
Integer outputLength,
|
||||
String retrievalLayer,
|
||||
Integer l0MatchCount,
|
||||
Integer l1MatchCount,
|
||||
Boolean truncated,
|
||||
String relevanceLevel,
|
||||
String completenessHint,
|
||||
String domain,
|
||||
String dedupReason,
|
||||
Integer durationMs,
|
||||
boolean success,
|
||||
String evidenceStatus,
|
||||
String errorMessage,
|
||||
List<String> l0Titles,
|
||||
List<String> l0MatchedKeywords,
|
||||
List<String> l0Domains,
|
||||
List<String> l0Entities,
|
||||
Double l1TopScore,
|
||||
Double l1TopSimilarity,
|
||||
List<Double> l1Scores,
|
||||
Integer evidenceCandidateCount,
|
||||
Integer evidenceBlockCount,
|
||||
List<Map<String, Object>> evidenceBlocks,
|
||||
Map<String, Object> queryTransform,
|
||||
Map<String, Object> retrievalTrace,
|
||||
Map<String, Object> contextPack,
|
||||
Map<String, Object> rerankTrace,
|
||||
String fallbackReason
|
||||
) {
|
||||
public static LookupKnowledgeRecord from(KnowledgeQuery query,
|
||||
LookupResult result,
|
||||
String domain,
|
||||
String dedupReason,
|
||||
int durationMs) {
|
||||
RetrievalTrace trace = result != null ? result.getRetrievalTrace() : null;
|
||||
String layer = recordedRetrievalLayer(query, trace);
|
||||
|
||||
String outputPreview = null;
|
||||
int outputLength = 0;
|
||||
boolean truncated = false;
|
||||
if (result != null && result.getContextPack() != null
|
||||
&& result.getContextPack().getPackedText() != null) {
|
||||
outputPreview = result.getContextPack().getPackedText();
|
||||
outputLength = outputPreview.length();
|
||||
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
|
||||
} else if (result != null && result.getEvidenceBlocks() != null
|
||||
&& !result.getEvidenceBlocks().isEmpty()
|
||||
&& result.getEvidenceBlocks().get(0).getContent() != null) {
|
||||
outputPreview = result.getEvidenceBlocks().get(0).getContent();
|
||||
outputLength = outputPreview.length();
|
||||
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
|
||||
}
|
||||
|
||||
String evidenceStatus = EVIDENCE_STATUS_SUPPORTED;
|
||||
if (dedupReason != null) {
|
||||
evidenceStatus = EVIDENCE_STATUS_DEDUPED;
|
||||
} else if (result == null || !result.isFound()) {
|
||||
evidenceStatus = EVIDENCE_STATUS_NO_EVIDENCE;
|
||||
} else if (trace != null && trace.getEvidenceStatus() != null) {
|
||||
evidenceStatus = trace.getEvidenceStatus();
|
||||
}
|
||||
|
||||
List<Double> l1Scores = collectAttemptScores(trace);
|
||||
|
||||
return LookupKnowledgeRecord.builder()
|
||||
.query(query != null ? query.getOriginalQuery() : null)
|
||||
.outputPreview(outputPreview)
|
||||
.outputLength(outputLength)
|
||||
.retrievalLayer(layer)
|
||||
.l0MatchCount(query != null ? query.getL0MatchCount() : null)
|
||||
.l1MatchCount(totalCandidateCount(trace))
|
||||
.truncated(truncated)
|
||||
.relevanceLevel(result != null ? result.getRelevanceLevel() : null)
|
||||
.completenessHint(result != null ? result.getCompletenessHint() : null)
|
||||
.domain(domain)
|
||||
.dedupReason(dedupReason)
|
||||
.durationMs(durationMs)
|
||||
.success(true)
|
||||
.evidenceStatus(evidenceStatus)
|
||||
.l0Titles(query != null ? query.getL0Titles() : List.of())
|
||||
.l0MatchedKeywords(query != null ? query.getMatchedKeywords() : List.of())
|
||||
.l0Domains(query != null ? query.getDomainHints() : List.of())
|
||||
.l0Entities(query != null ? query.getEntities() : List.of())
|
||||
.l1TopScore(firstAttemptScore(trace))
|
||||
.l1TopSimilarity(firstAttemptSimilarity(trace))
|
||||
.l1Scores(l1Scores)
|
||||
.evidenceCandidateCount(result != null ? result.getEvidenceCandidateCount() : null)
|
||||
.evidenceBlockCount(result != null ? result.getEvidenceBlockCount() : null)
|
||||
.evidenceBlocks(result != null ? summarizeEvidenceBlocks(result.getEvidenceBlocks()) : List.of())
|
||||
.queryTransform(summarizeQueryTransform(query))
|
||||
.retrievalTrace(summarizeRetrievalTrace(trace))
|
||||
.contextPack(summarizeContextPack(result != null ? result.getContextPack() : null))
|
||||
.rerankTrace(summarizeRerankTrace(result != null ? result.getRerankTrace() : null))
|
||||
.fallbackReason(trace != null ? trace.getFallbackReason() : null)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static List<Map<String, Object>> summarizeEvidenceBlocks(List<EvidenceBlock> blocks) {
|
||||
if (blocks == null || blocks.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<Map<String, Object>> summaries = new ArrayList<>();
|
||||
for (int i = 0; i < Math.min(5, blocks.size()); i++) {
|
||||
EvidenceBlock block = blocks.get(i);
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("source", block.getSource());
|
||||
summary.put("title", block.getTitle());
|
||||
summary.put("breadcrumb", block.getBreadcrumb());
|
||||
summary.put("retrieval_layer", block.getRetrievalLayer());
|
||||
summary.put("score", block.getScore());
|
||||
summary.put("hit_reasons", block.getHitReasons());
|
||||
String content = block.getContent();
|
||||
if (content != null) {
|
||||
summary.put("content_preview", content.length() <= 180 ? content : content.substring(0, 180) + "...");
|
||||
}
|
||||
summaries.add(summary);
|
||||
}
|
||||
return summaries;
|
||||
}
|
||||
|
||||
private static Map<String, Object> summarizeQueryTransform(KnowledgeQuery query) {
|
||||
if (query == null) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("original_query", query.getOriginalQuery());
|
||||
summary.put("rewritten_query", query.getRewrittenQuery());
|
||||
summary.put("category_filter", query.getCategoryFilter());
|
||||
summary.put("domain_hints", query.getDomainHints());
|
||||
summary.put("matched_keywords", query.getMatchedKeywords());
|
||||
summary.put("entities", query.getEntities());
|
||||
summary.put("l0_titles", query.getL0Titles());
|
||||
summary.put("l0_match_count", query.getL0MatchCount());
|
||||
return summary;
|
||||
}
|
||||
|
||||
private static Map<String, Object> summarizeRetrievalTrace(RetrievalTrace trace) {
|
||||
if (trace == null) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("selected_attempt", trace.getSelectedAttempt());
|
||||
summary.put("fallback_reason", trace.getFallbackReason());
|
||||
summary.put("evidence_status", trace.getEvidenceStatus());
|
||||
summary.put("category_filter", trace.getCategoryFilter());
|
||||
List<Map<String, Object>> attempts = new ArrayList<>();
|
||||
if (trace.getAttempts() != null) {
|
||||
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
|
||||
Map<String, Object> item = new LinkedHashMap<>();
|
||||
item.put("name", attempt.getName());
|
||||
item.put("category_filter", attempt.getCategoryFilter());
|
||||
item.put("candidate_count", attempt.getCandidateCount());
|
||||
item.put("usable", attempt.getUsable());
|
||||
item.put("duration_ms", attempt.getDurationMs());
|
||||
item.put("top_score", attempt.getTopScore());
|
||||
item.put("top_similarity", attempt.getTopSimilarity());
|
||||
item.put("error_message", attempt.getErrorMessage());
|
||||
attempts.add(item);
|
||||
}
|
||||
}
|
||||
summary.put("attempts", attempts);
|
||||
return summary;
|
||||
}
|
||||
|
||||
private static Map<String, Object> summarizeContextPack(ContextPack contextPack) {
|
||||
if (contextPack == null) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("strategy", contextPack.getStrategy());
|
||||
summary.put("char_budget", contextPack.getCharBudget());
|
||||
summary.put("used_chars", contextPack.getUsedChars());
|
||||
summary.put("included_sources", contextPack.getIncludedSources());
|
||||
summary.put("omitted_sources", contextPack.getOmittedSources());
|
||||
return summary;
|
||||
}
|
||||
|
||||
private static Map<String, Object> summarizeRerankTrace(RerankTrace trace) {
|
||||
if (trace == null || trace.getItems() == null || trace.getItems().isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
List<Map<String, Object>> items = new ArrayList<>();
|
||||
for (int i = 0; i < Math.min(5, trace.getItems().size()); i++) {
|
||||
RerankTrace.Item item = trace.getItems().get(i);
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("final_rank", item.getFinalRank());
|
||||
summary.put("source", item.getSource());
|
||||
summary.put("base_score", item.getBaseScore());
|
||||
summary.put("final_score", item.getFinalScore());
|
||||
summary.put("boost_reasons", item.getBoostReasons());
|
||||
items.add(summary);
|
||||
}
|
||||
return Map.of("items", items);
|
||||
}
|
||||
|
||||
private static Integer totalCandidateCount(RetrievalTrace trace) {
|
||||
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
int total = 0;
|
||||
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
|
||||
if (attempt.getCandidateCount() != null) {
|
||||
total += attempt.getCandidateCount();
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
private static Double firstAttemptScore(RetrievalTrace trace) {
|
||||
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
|
||||
if (attempt.getTopScore() != null) {
|
||||
return attempt.getTopScore();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Double firstAttemptSimilarity(RetrievalTrace trace) {
|
||||
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
|
||||
if (attempt.getTopSimilarity() != null) {
|
||||
return attempt.getTopSimilarity();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static List<Double> collectAttemptScores(RetrievalTrace trace) {
|
||||
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<Double> scores = new ArrayList<>();
|
||||
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
|
||||
if (attempt.getTopScore() != null) {
|
||||
scores.add(attempt.getTopScore());
|
||||
}
|
||||
}
|
||||
return scores;
|
||||
}
|
||||
|
||||
private static String recordedRetrievalLayer(KnowledgeQuery query, RetrievalTrace trace) {
|
||||
if (trace == null || trace.getSelectedAttempt() == null) {
|
||||
return null;
|
||||
}
|
||||
boolean hasL0 = query != null && query.getL0MatchCount() != null && query.getL0MatchCount() > 0;
|
||||
boolean hasL1 = trace.getSelectedAttempt().contains("VECTOR");
|
||||
if (hasL0 && hasL1) {
|
||||
return "L0+L1";
|
||||
}
|
||||
if (hasL1) {
|
||||
return "L1";
|
||||
}
|
||||
return hasL0 ? "L0" : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,546 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Builds a verifier-facing evidence index from persisted tool invocations.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ToolTraceSummaryService {
|
||||
|
||||
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
|
||||
private static final Set<String> EVIDENCE_TOOLS = Set.of("lookup_knowledge", "query_logs", "query_metrics", "query_order");
|
||||
private static final Pattern JSON_STRING_FIELD = Pattern.compile("\"%s\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"");
|
||||
|
||||
private final ToolInvocationRepository toolInvocationRepository;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
public ToolTraceSummaryService(ToolInvocationRepository toolInvocationRepository) {
|
||||
this.toolInvocationRepository = toolInvocationRepository;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> buildVerifierTraceSummary(String sessionId, String executorFinalAnswer) {
|
||||
if (sessionId == null || sessionId.isBlank()) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<ToolInvocation> invocations = toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId);
|
||||
return buildVerifierTraceSummary(invocations, executorFinalAnswer);
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> buildVerifierTraceSummaryForRun(String runId, String executorFinalAnswer) {
|
||||
if (runId == null || runId.isBlank()) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<ToolInvocation> invocations = toolInvocationRepository.findByRunIdOrderByIdAsc(runId);
|
||||
return buildVerifierTraceSummary(invocations, executorFinalAnswer);
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> buildVerifierTraceSummary(List<ToolInvocation> invocations, String executorFinalAnswer) {
|
||||
if (invocations.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
Map<String, AggregateEntry> grouped = new LinkedHashMap<>();
|
||||
for (ToolInvocation invocation : invocations) {
|
||||
if (!EVIDENCE_TOOLS.contains(invocation.getToolName())) {
|
||||
continue;
|
||||
}
|
||||
String topicDomain = extractTopicDomain(invocation);
|
||||
String key = invocation.getToolName() + "|" + topicDomain;
|
||||
AggregateEntry entry = grouped.computeIfAbsent(
|
||||
key,
|
||||
ignored -> new AggregateEntry(invocation.getToolName(), topicDomain));
|
||||
entry.absorb(invocation);
|
||||
}
|
||||
|
||||
List<AggregateEntry> rankedEntries = grouped.values().stream()
|
||||
.sorted(Comparator.comparingInt((AggregateEntry entry) -> entry.relevanceScore(executorFinalAnswer)).reversed())
|
||||
.limit(8)
|
||||
.toList();
|
||||
|
||||
List<Map<String, Object>> summaries = new ArrayList<>();
|
||||
for (int i = 0; i < rankedEntries.size(); i++) {
|
||||
summaries.add(rankedEntries.get(i).toSummary("trace-" + (i + 1)));
|
||||
}
|
||||
return summaries;
|
||||
}
|
||||
|
||||
private String extractTopicDomain(ToolInvocation invocation) {
|
||||
try {
|
||||
if (invocation.getRetrievalDetails() != null && !invocation.getRetrievalDetails().isBlank()) {
|
||||
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
|
||||
Object domains = details.get("retrieved_domains");
|
||||
if (domains instanceof List<?> domainList && !domainList.isEmpty()) {
|
||||
return String.valueOf(domainList.get(0));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse retrieved_domains, fallback to general", e);
|
||||
}
|
||||
return "general";
|
||||
}
|
||||
|
||||
private String extractInputSummary(ToolInvocation invocation) {
|
||||
String query = extractQuery(invocation);
|
||||
if (query != null && !query.isBlank()) {
|
||||
return "query=" + truncate(query, 120);
|
||||
}
|
||||
return invocation.getToolName() + " invoked";
|
||||
}
|
||||
|
||||
private String extractQuery(ToolInvocation invocation) {
|
||||
try {
|
||||
if (invocation.getInputParams() != null && !invocation.getInputParams().isBlank()) {
|
||||
Map<String, Object> params = objectMapper.readValue(invocation.getInputParams(), MAP_TYPE);
|
||||
Object query = params.get("query");
|
||||
if (query != null) {
|
||||
return String.valueOf(query);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse invocation query", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractOutputSummary(ToolInvocation invocation, String topicDomain) {
|
||||
String evidenceStatus = extractEvidenceStatus(invocation);
|
||||
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
|
||||
if (invocation.getErrorMessage() != null && !invocation.getErrorMessage().isBlank()) {
|
||||
return "call failed: " + truncate(invocation.getErrorMessage(), 120);
|
||||
}
|
||||
return "no usable evidence returned";
|
||||
}
|
||||
|
||||
if (ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
|
||||
return "retrieval skipped because the same document was already used in this session";
|
||||
}
|
||||
|
||||
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)) {
|
||||
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
|
||||
return "completed without usable evidence: " + truncate(invocation.getOutputPreview(), 120);
|
||||
}
|
||||
return "completed without usable evidence";
|
||||
}
|
||||
|
||||
if ("lookup_knowledge".equals(invocation.getToolName())) {
|
||||
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
|
||||
String trace = extractLookupTraceSummary(invocation);
|
||||
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
|
||||
? truncate(invocation.getOutputPreview(), 240)
|
||||
: "no preview";
|
||||
return "matched domain=" + topicDomain + ", relevance=" + relevance + trace + ", preview=" + preview;
|
||||
}
|
||||
|
||||
if ("query_logs".equals(invocation.getToolName())) {
|
||||
String concrete = extractLogEvidenceSummary(invocation.getOutputPreview());
|
||||
if (!concrete.isBlank()) {
|
||||
return concrete;
|
||||
}
|
||||
}
|
||||
|
||||
if ("query_metrics".equals(invocation.getToolName())) {
|
||||
String concrete = extractMetricsEvidenceSummary(invocation.getOutputPreview());
|
||||
if (!concrete.isBlank()) {
|
||||
return concrete;
|
||||
}
|
||||
}
|
||||
|
||||
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
|
||||
return truncate(invocation.getOutputPreview(), 240);
|
||||
}
|
||||
return "evidence retrieved without preview";
|
||||
}
|
||||
|
||||
private String extractLookupTraceSummary(ToolInvocation invocation) {
|
||||
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
|
||||
List<String> sources = new ArrayList<>();
|
||||
Object evidenceBlocks = details.get("evidence_blocks");
|
||||
if (evidenceBlocks instanceof List<?> blocks) {
|
||||
for (Object block : blocks) {
|
||||
if (block instanceof Map<?, ?> blockMap) {
|
||||
Object title = blockMap.get("title");
|
||||
Object source = blockMap.get("source");
|
||||
String label = title != null ? String.valueOf(title) : String.valueOf(source);
|
||||
if (label != null && !label.isBlank() && !"null".equals(label)) {
|
||||
sources.add(label);
|
||||
}
|
||||
}
|
||||
if (sources.size() >= 3) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Object retrievalTrace = details.get("retrieval_trace");
|
||||
String selectedAttempt = "";
|
||||
if (retrievalTrace instanceof Map<?, ?> traceMap && traceMap.get("selected_attempt") != null) {
|
||||
selectedAttempt = ", selected_attempt=" + traceMap.get("selected_attempt");
|
||||
}
|
||||
return (sources.isEmpty() ? "" : ", sources=" + truncate(String.join("|", sources), 180)) + selectedAttempt;
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse lookup retrieval details", e);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
private String extractLogEvidenceSummary(String outputPreview) {
|
||||
if (outputPreview == null || outputPreview.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
|
||||
List<String> services = extractJsonStringFields(outputPreview, "service", 3);
|
||||
List<String> levels = extractJsonStringFields(outputPreview, "level", 3);
|
||||
List<String> timestamps = extractJsonStringFields(outputPreview, "timestamp", 3);
|
||||
if (messages.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
List<String> rows = new ArrayList<>();
|
||||
for (int i = 0; i < messages.size(); i++) {
|
||||
String prefix = labelAt(timestamps, i) + labelAt(levels, i) + labelAt(services, i);
|
||||
rows.add((prefix.isBlank() ? "" : prefix + " ") + truncate(messages.get(i), 220));
|
||||
}
|
||||
return "log_evidence: " + truncate(String.join(" | ", rows), 520);
|
||||
}
|
||||
|
||||
private boolean hasOnlyGenericMockLogMessages(String outputPreview) {
|
||||
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
|
||||
if (messages.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return messages.stream()
|
||||
.allMatch(message -> message.startsWith("日志消息 #") && message.contains("查询条件:"));
|
||||
}
|
||||
|
||||
private String extractMetricsEvidenceSummary(String outputPreview) {
|
||||
if (outputPreview == null || outputPreview.isBlank()) {
|
||||
return "";
|
||||
}
|
||||
List<String> alertNames = extractJsonStringFields(outputPreview, "alert_name", 5);
|
||||
List<String> descriptions = extractJsonStringFields(outputPreview, "description", 5);
|
||||
List<String> services = extractJsonStringFields(outputPreview, "service", 5);
|
||||
if (alertNames.isEmpty() && descriptions.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
List<String> rows = new ArrayList<>();
|
||||
int count = Math.max(alertNames.size(), descriptions.size());
|
||||
for (int i = 0; i < Math.min(5, count); i++) {
|
||||
StringBuilder row = new StringBuilder();
|
||||
if (i < alertNames.size()) {
|
||||
row.append(alertNames.get(i));
|
||||
}
|
||||
if (i < services.size()) {
|
||||
if (!row.isEmpty()) {
|
||||
row.append(" ");
|
||||
}
|
||||
row.append("service=").append(services.get(i));
|
||||
}
|
||||
if (i < descriptions.size()) {
|
||||
if (!row.isEmpty()) {
|
||||
row.append(": ");
|
||||
}
|
||||
row.append(descriptions.get(i));
|
||||
}
|
||||
rows.add(truncate(row.toString(), 220));
|
||||
}
|
||||
return "metric_evidence: " + truncate(String.join(" | ", rows), 520);
|
||||
}
|
||||
|
||||
private List<String> extractJsonStringFields(String text, String field, int limit) {
|
||||
Pattern pattern = Pattern.compile(String.format(JSON_STRING_FIELD.pattern(), Pattern.quote(field)));
|
||||
Matcher matcher = pattern.matcher(text);
|
||||
List<String> values = new ArrayList<>();
|
||||
while (matcher.find() && values.size() < limit) {
|
||||
values.add(unescapeJsonString(matcher.group(1)));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private String unescapeJsonString(String value) {
|
||||
return value == null ? "" : value
|
||||
.replace("\\\"", "\"")
|
||||
.replace("\\\\", "\\")
|
||||
.replace("\\n", "\n")
|
||||
.replace("\\r", "\r")
|
||||
.replace("\\t", "\t");
|
||||
}
|
||||
|
||||
private String labelAt(List<String> values, int index) {
|
||||
if (index >= values.size() || values.get(index) == null || values.get(index).isBlank()) {
|
||||
return "";
|
||||
}
|
||||
return "[" + values.get(index) + "]";
|
||||
}
|
||||
|
||||
private String determineEvidenceLevel(ToolInvocation invocation) {
|
||||
String evidenceStatus = extractEvidenceStatus(invocation);
|
||||
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
|
||||
return "none";
|
||||
}
|
||||
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|
||||
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
|
||||
return "none";
|
||||
}
|
||||
if ("query_logs".equals(invocation.getToolName())
|
||||
&& hasOnlyGenericMockLogMessages(invocation.getOutputPreview())) {
|
||||
return "none";
|
||||
}
|
||||
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
|
||||
return "direct";
|
||||
}
|
||||
if ("REFERENCE".equals(invocation.getRelevanceLevel())) {
|
||||
return "indirect";
|
||||
}
|
||||
if (EVIDENCE_TOOLS.contains(invocation.getToolName())) {
|
||||
return "direct";
|
||||
}
|
||||
return "none";
|
||||
}
|
||||
|
||||
private String extractEvidenceStatus(ToolInvocation invocation) {
|
||||
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
|
||||
return Boolean.TRUE.equals(invocation.getSuccess())
|
||||
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
|
||||
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
|
||||
}
|
||||
try {
|
||||
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
|
||||
Object evidenceStatus = details.get("evidence_status");
|
||||
if (evidenceStatus != null) {
|
||||
return String.valueOf(evidenceStatus);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse evidence_status", e);
|
||||
}
|
||||
return Boolean.TRUE.equals(invocation.getSuccess())
|
||||
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
|
||||
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
|
||||
}
|
||||
|
||||
private List<String> extractStringList(Object value) {
|
||||
if (!(value instanceof List<?> list) || list.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<String> result = new ArrayList<>();
|
||||
for (Object item : list) {
|
||||
if (item != null) {
|
||||
result.add(String.valueOf(item));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<String> extractSourceDocuments(ToolInvocation invocation) {
|
||||
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
|
||||
return List.of();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
|
||||
List<String> paths = extractStringList(details.get("l0_paths"));
|
||||
if (!paths.isEmpty()) {
|
||||
return paths;
|
||||
}
|
||||
List<String> titles = extractStringList(details.get("l0_titles"));
|
||||
if (!titles.isEmpty()) {
|
||||
return titles;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse source documents", e);
|
||||
}
|
||||
return List.of();
|
||||
}
|
||||
|
||||
private String truncate(String text, int maxLength) {
|
||||
if (text == null) {
|
||||
return "";
|
||||
}
|
||||
return text.length() <= maxLength ? text : text.substring(0, maxLength) + "...";
|
||||
}
|
||||
|
||||
private final class AggregateEntry {
|
||||
private final String toolName;
|
||||
private final String topicDomain;
|
||||
private String inputSummary;
|
||||
private String outputSummary;
|
||||
private boolean success;
|
||||
private String evidenceLevel = "none";
|
||||
private int invocationCount;
|
||||
private int failedCount;
|
||||
private int noHitCount;
|
||||
private final List<Long> sourceInvocationIds = new ArrayList<>();
|
||||
private final LinkedHashSet<String> querySamples = new LinkedHashSet<>();
|
||||
private final LinkedHashSet<String> retrievalLayers = new LinkedHashSet<>();
|
||||
private final LinkedHashSet<String> relevanceLevels = new LinkedHashSet<>();
|
||||
private final LinkedHashSet<String> sourceDocuments = new LinkedHashSet<>();
|
||||
|
||||
private AggregateEntry(String toolName, String topicDomain) {
|
||||
this.toolName = toolName;
|
||||
this.topicDomain = topicDomain;
|
||||
}
|
||||
|
||||
void absorb(ToolInvocation invocation) {
|
||||
invocationCount++;
|
||||
if (invocation.getId() != null) {
|
||||
sourceInvocationIds.add(invocation.getId());
|
||||
}
|
||||
|
||||
String query = extractQuery(invocation);
|
||||
if (query != null && !query.isBlank()) {
|
||||
querySamples.add(query);
|
||||
}
|
||||
if (invocation.getRetrievalLayer() != null && !invocation.getRetrievalLayer().isBlank()) {
|
||||
retrievalLayers.add(invocation.getRetrievalLayer());
|
||||
}
|
||||
if (invocation.getRelevanceLevel() != null && !invocation.getRelevanceLevel().isBlank()) {
|
||||
relevanceLevels.add(invocation.getRelevanceLevel());
|
||||
}
|
||||
sourceDocuments.addAll(extractSourceDocuments(invocation));
|
||||
|
||||
if (inputSummary == null || inputSummary.isBlank()) {
|
||||
inputSummary = extractInputSummary(invocation);
|
||||
}
|
||||
|
||||
String evidenceStatus = extractEvidenceStatus(invocation);
|
||||
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
|
||||
failedCount++;
|
||||
if (outputSummary == null || outputSummary.isBlank()) {
|
||||
outputSummary = extractOutputSummary(invocation, topicDomain);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|
||||
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
|
||||
noHitCount++;
|
||||
if (outputSummary == null || outputSummary.isBlank()) {
|
||||
outputSummary = extractOutputSummary(invocation, topicDomain);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
|
||||
if ("none".equals(invocationEvidenceLevel)) {
|
||||
noHitCount++;
|
||||
if (outputSummary == null || outputSummary.isBlank()) {
|
||||
outputSummary = extractOutputSummary(invocation, topicDomain);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!success || evidenceRank(invocationEvidenceLevel) > evidenceRank(evidenceLevel)) {
|
||||
success = true;
|
||||
evidenceLevel = invocationEvidenceLevel;
|
||||
outputSummary = extractOutputSummary(invocation, topicDomain);
|
||||
} else if (evidenceRank(invocationEvidenceLevel) == evidenceRank(evidenceLevel)) {
|
||||
String candidateSummary = extractOutputSummary(invocation, topicDomain);
|
||||
if (isMoreConcrete(candidateSummary, outputSummary)) {
|
||||
outputSummary = candidateSummary;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isMoreConcrete(String candidate, String current) {
|
||||
return concretenessScore(candidate) > concretenessScore(current);
|
||||
}
|
||||
|
||||
private int concretenessScore(String summary) {
|
||||
if (summary == null || summary.isBlank()) {
|
||||
return 0;
|
||||
}
|
||||
int score = summary.length() > 160 ? 2 : 1;
|
||||
if (summary.contains("log_evidence") || summary.contains("metric_evidence")) {
|
||||
score += 5;
|
||||
}
|
||||
if (summary.contains("连接池耗尽") || summary.contains("OutOfMemoryError")
|
||||
|| summary.contains("扫描行数") || summary.contains("HighMemoryUsage")
|
||||
|| summary.contains("HighCPUUsage")) {
|
||||
score += 4;
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
int relevanceScore(String answer) {
|
||||
int score = success ? 10 : 0;
|
||||
if ("direct".equals(evidenceLevel)) {
|
||||
score += 10;
|
||||
} else if ("indirect".equals(evidenceLevel)) {
|
||||
score += 5;
|
||||
}
|
||||
if (answer != null) {
|
||||
String normalized = answer.toLowerCase(Locale.ROOT);
|
||||
if (normalized.contains(topicDomain.toLowerCase(Locale.ROOT))) {
|
||||
score += 8;
|
||||
}
|
||||
if (normalized.contains(toolName.toLowerCase(Locale.ROOT))) {
|
||||
score += 3;
|
||||
}
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
Map<String, Object> toSummary(String traceRef) {
|
||||
String mergedOutput = outputSummary == null ? "no summarized evidence" : outputSummary;
|
||||
if (invocationCount > 1) {
|
||||
StringBuilder builder = new StringBuilder(mergedOutput);
|
||||
builder.append(" (merged ").append(invocationCount).append(" invocations");
|
||||
if (failedCount > 0) {
|
||||
builder.append(", failed=").append(failedCount);
|
||||
}
|
||||
if (noHitCount > 0) {
|
||||
builder.append(", no_hit=").append(noHitCount);
|
||||
}
|
||||
builder.append(")");
|
||||
mergedOutput = builder.toString();
|
||||
}
|
||||
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("trace_ref", traceRef);
|
||||
summary.put("tool_name", toolName);
|
||||
summary.put("success", success);
|
||||
summary.put("input_summary", inputSummary);
|
||||
summary.put("output_summary", mergedOutput);
|
||||
summary.put("evidence_level", evidenceLevel);
|
||||
summary.put("topic_domain", topicDomain);
|
||||
summary.put("source_invocation_ids", new ArrayList<>(sourceInvocationIds));
|
||||
summary.put("invocation_count", invocationCount);
|
||||
summary.put("failed_invocation_count", failedCount);
|
||||
summary.put("no_hit_invocation_count", noHitCount);
|
||||
summary.put("query_samples", new ArrayList<>(querySamples));
|
||||
summary.put("retrieval_layers", new ArrayList<>(retrievalLayers));
|
||||
summary.put("relevance_levels", new ArrayList<>(relevanceLevels));
|
||||
summary.put("source_documents", new ArrayList<>(sourceDocuments));
|
||||
return summary;
|
||||
}
|
||||
|
||||
private int evidenceRank(String level) {
|
||||
if ("direct".equals(level)) {
|
||||
return 2;
|
||||
}
|
||||
if ("indirect".equals(level)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,7 +78,8 @@ public class VectorSearchService {
|
||||
try {
|
||||
return searchSimilarDocumentsWithVectorStore(query, topK, category);
|
||||
} catch (Exception e) {
|
||||
logger.warn("Spring AI VectorStore retrieval failed, falling back to Milvus SDK: {}", e.getMessage());
|
||||
logger.warn("Spring AI VectorStore retrieval failed, falling back to Milvus SDK: errorType={}",
|
||||
e.getClass().getSimpleName());
|
||||
return searchSimilarDocumentsWithSdk(query, topK, category);
|
||||
}
|
||||
}
|
||||
@@ -89,8 +90,8 @@ public class VectorSearchService {
|
||||
throw new IllegalStateException("Spring AI VectorStore bean is unavailable");
|
||||
}
|
||||
|
||||
logger.info("Starting Spring AI VectorStore search: query={}, topK={}, category={}, kbScope={}",
|
||||
query, topK, category, effectiveKbScope());
|
||||
logger.info("Starting Spring AI VectorStore search: topK={}, category={}, kbScope={}",
|
||||
topK, category, effectiveKbScope());
|
||||
SearchRequest.Builder builder = SearchRequest.builder()
|
||||
.query(query)
|
||||
.topK(topK)
|
||||
@@ -119,8 +120,8 @@ public class VectorSearchService {
|
||||
|
||||
List<SearchResult> searchSimilarDocumentsWithSdk(String query, int topK, String category) {
|
||||
try {
|
||||
logger.info("Starting Milvus SDK search: query={}, topK={}, category={}, kbScope={}",
|
||||
query, topK, category, effectiveKbScope());
|
||||
logger.info("Starting Milvus SDK search: topK={}, category={}, kbScope={}",
|
||||
topK, category, effectiveKbScope());
|
||||
|
||||
List<Float> queryVector = embeddingService.generateQueryVector(query);
|
||||
logger.debug("Query vector generated, dimension={}", queryVector.size());
|
||||
@@ -166,7 +167,7 @@ public class VectorSearchService {
|
||||
logger.info("Milvus SDK search complete, candidates={}", results.size());
|
||||
return results;
|
||||
} catch (Exception e) {
|
||||
logger.error("Milvus SDK vector search failed", e);
|
||||
logger.error("Milvus SDK vector search failed: errorType={}", e.getClass().getSimpleName());
|
||||
throw new RuntimeException("Vector search failed: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
package com.superbiz.agent.service.session;
|
||||
|
||||
import com.superbiz.agent.domain.model.SessionContext;
|
||||
import com.superbiz.agent.domain.model.ToolCall;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* 会话管理器接口
|
||||
* 负责会话的创建、读取、更新和删除
|
||||
*/
|
||||
public interface SessionManager {
|
||||
|
||||
/**
|
||||
* 创建新会话
|
||||
*
|
||||
* @param sessionContext 会话上下文
|
||||
* @param ttlSeconds 会话过期时间（秒）
|
||||
* @return 会话ID
|
||||
*/
|
||||
String createSession(SessionContext sessionContext, long ttlSeconds);
|
||||
|
||||
/**
|
||||
* 获取会话
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
* @return 会话上下文（如果存在）
|
||||
*/
|
||||
Optional<SessionContext> getSession(String sessionId);
|
||||
|
||||
/**
|
||||
* 更新会话
|
||||
*
|
||||
* @param sessionContext 会话上下文
|
||||
*/
|
||||
void updateSession(SessionContext sessionContext);
|
||||
|
||||
/**
|
||||
* 删除会话
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
*/
|
||||
void deleteSession(String sessionId);
|
||||
|
||||
/**
|
||||
* 检查会话是否存在
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
* @return true 如果会话存在
|
||||
*/
|
||||
boolean exists(String sessionId);
|
||||
|
||||
/**
|
||||
* 刷新会话过期时间
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
* @param ttlSeconds 新的过期时间（秒）
|
||||
* @return true 如果刷新成功
|
||||
*/
|
||||
boolean refreshSession(String sessionId, long ttlSeconds);
|
||||
|
||||
/**
|
||||
* 添加工具调用记录到会话
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
* @param toolCall 工具调用记录
|
||||
*/
|
||||
void addToolCall(String sessionId, ToolCall toolCall);
|
||||
|
||||
/**
|
||||
* 更新会话状态
|
||||
*
|
||||
* @param sessionId 会话ID
|
||||
* @param status 新状态
|
||||
*/
|
||||
void updateStatus(String sessionId, String status);
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.superbiz.agent.service.session.impl;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import com.superbiz.agent.domain.model.SessionContext;
|
||||
import com.superbiz.agent.domain.model.ToolCall;
|
||||
import com.superbiz.agent.service.session.SessionManager;
|
||||
import org.springframework.data.redis.core.RedisTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Redis 会话管理器实现
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class RedisSessionManager implements SessionManager {
|
||||
|
||||
private static final String SESSION_KEY_PREFIX = "session:";
|
||||
|
||||
private final RedisTemplate<String, Object> redisTemplate;
|
||||
|
||||
@Override
|
||||
public String createSession(SessionContext sessionContext, long ttlSeconds) {
|
||||
String sessionId = sessionContext.getSessionId();
|
||||
if (sessionId == null || sessionId.isEmpty()) {
|
||||
throw new IllegalArgumentException("Session ID cannot be null or empty");
|
||||
}
|
||||
|
||||
sessionContext.setCreatedAt(LocalDateTime.now());
|
||||
sessionContext.setLastActiveAt(LocalDateTime.now());
|
||||
sessionContext.setTtl(ttlSeconds);
|
||||
sessionContext.setStatus("ACTIVE");
|
||||
|
||||
String key = buildKey(sessionId);
|
||||
redisTemplate.opsForValue().set(key, sessionContext, ttlSeconds, TimeUnit.SECONDS);
|
||||
|
||||
log.info("创建会话成功: sessionId={}, ttl={}秒", sessionId, ttlSeconds);
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<SessionContext> getSession(String sessionId) {
|
||||
String key = buildKey(sessionId);
|
||||
Object value = redisTemplate.opsForValue().get(key);
|
||||
|
||||
if (value instanceof SessionContext) {
|
||||
SessionContext context = (SessionContext) value;
|
||||
log.debug("获取会话成功: sessionId={}", sessionId);
|
||||
return Optional.of(context);
|
||||
}
|
||||
|
||||
log.debug("会话不存在: sessionId={}", sessionId);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateSession(SessionContext sessionContext) {
|
||||
String sessionId = sessionContext.getSessionId();
|
||||
String key = buildKey(sessionId);
|
||||
|
||||
// 获取剩余 TTL
|
||||
Long ttl = redisTemplate.getExpire(key, TimeUnit.SECONDS);
|
||||
if (ttl == null || ttl <= 0) {
|
||||
ttl = sessionContext.getTtl() != null ? sessionContext.getTtl() : 3600L;
|
||||
}
|
||||
|
||||
sessionContext.setLastActiveAt(LocalDateTime.now());
|
||||
redisTemplate.opsForValue().set(key, sessionContext, ttl, TimeUnit.SECONDS);
|
||||
|
||||
log.debug("更新会话成功: sessionId={}", sessionId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteSession(String sessionId) {
|
||||
String key = buildKey(sessionId);
|
||||
Boolean deleted = redisTemplate.delete(key);
|
||||
|
||||
if (Boolean.TRUE.equals(deleted)) {
|
||||
log.info("删除会话成功: sessionId={}", sessionId);
|
||||
} else {
|
||||
log.warn("删除会话失败，会话可能不存在: sessionId={}", sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String sessionId) {
|
||||
String key = buildKey(sessionId);
|
||||
Boolean exists = redisTemplate.hasKey(key);
|
||||
return Boolean.TRUE.equals(exists);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean refreshSession(String sessionId, long ttlSeconds) {
|
||||
String key = buildKey(sessionId);
|
||||
Boolean refreshed = redisTemplate.expire(key, ttlSeconds, TimeUnit.SECONDS);
|
||||
|
||||
if (Boolean.TRUE.equals(refreshed)) {
|
||||
log.debug("刷新会话过期时间成功: sessionId={}, newTtl={}秒", sessionId, ttlSeconds);
|
||||
return true;
|
||||
}
|
||||
|
||||
log.warn("刷新会话过期时间失败，会话可能不存在: sessionId={}", sessionId);
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addToolCall(String sessionId, ToolCall toolCall) {
|
||||
Optional<SessionContext> sessionOpt = getSession(sessionId);
|
||||
|
||||
if (sessionOpt.isPresent()) {
|
||||
SessionContext context = sessionOpt.get();
|
||||
context.addToolCall(toolCall);
|
||||
updateSession(context);
|
||||
|
||||
log.debug("添加工具调用记录成功: sessionId={}, toolName={}", sessionId, toolCall.getToolName());
|
||||
} else {
|
||||
log.warn("会话不存在，无法添加工具调用记录: sessionId={}", sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateStatus(String sessionId, String status) {
|
||||
Optional<SessionContext> sessionOpt = getSession(sessionId);
|
||||
|
||||
if (sessionOpt.isPresent()) {
|
||||
SessionContext context = sessionOpt.get();
|
||||
context.setStatus(status);
|
||||
updateSession(context);
|
||||
|
||||
log.debug("更新会话状态成功: sessionId={}, status={}", sessionId, status);
|
||||
} else {
|
||||
log.warn("会话不存在，无法更新状态: sessionId={}", sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 Redis key
|
||||
*/
|
||||
private String buildKey(String sessionId) {
|
||||
return SESSION_KEY_PREFIX + sessionId;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import com.superbiz.agent.dto.ContextPack;
|
||||
import com.superbiz.agent.dto.EvidenceBlock;
|
||||
import com.superbiz.agent.dto.EvidencePostprocessResult;
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
@@ -11,10 +10,7 @@ import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
@@ -23,10 +19,9 @@ import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* Agent-facing explicit knowledge retrieval tool.
|
||||
* Harness backend for knowledge retrieval. Agent-facing schema is owned by HarnessEvidenceTools.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@@ -56,40 +51,23 @@ public class LookupKnowledgeTool {
|
||||
@Autowired
|
||||
private LookupResultAssembler resultAssembler;
|
||||
|
||||
@Autowired
|
||||
private ToolInvocationRecorder toolInvocationRecorder;
|
||||
|
||||
@Autowired
|
||||
private RetrievedDocTracker retrievedDocTracker;
|
||||
|
||||
/**
|
||||
* 查询知识库文档。
|
||||
*
|
||||
* @param query 查询关键词
|
||||
* @return 查询结果
|
||||
*/
|
||||
@Tool(description = "查询内部知识库文档，获取错误码定义、接口文档、排障步骤、配置说明和历史案例等背景知识。" +
|
||||
"工具会执行 query understanding、向量检索、证据去重、轻量重排和上下文打包，并返回 evidenceBlocks、contextPack、retrievalTrace、rerankTrace。" +
|
||||
"L0 仅用于领域/关键词/实体 hint 和可选 metadata filter，不作为事实证据兜底。" +
|
||||
"当带 filter 的向量检索低质量或无结果时，会跳过 L0 filter，用原始 query 再做一次无过滤语义检索。" +
|
||||
"不要用此工具查询实时运行状态；日志、指标、告警等实时事实应使用 query_logs、query_metrics 或告警工具。" +
|
||||
"参数 query: 查询关键词或描述")
|
||||
public LookupResult lookupKnowledge(String query) {
|
||||
String requestId = UUID.randomUUID().toString().substring(0, 8);
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
log.info("========================================");
|
||||
log.info(">>> [工具调用] lookup_knowledge");
|
||||
log.info(">>> 参数: query = \"{}\"", query);
|
||||
log.info(">>> RequestId: {}", requestId);
|
||||
log.info(">>> metadata: query_chars={}", query == null ? 0 : query.length());
|
||||
log.info("----------------------------------------");
|
||||
|
||||
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
|
||||
log.info("[QueryTransformer] rewrittenQuery={}, categoryFilter={}, domains={}, keywords={}",
|
||||
knowledgeQuery.getRewrittenQuery(),
|
||||
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
|
||||
knowledgeQuery.getCategoryFilter(),
|
||||
knowledgeQuery.getDomainHints(),
|
||||
knowledgeQuery.getMatchedKeywords());
|
||||
knowledgeQuery.getDomainHints().size(),
|
||||
knowledgeQuery.getMatchedKeywords().size());
|
||||
|
||||
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
|
||||
String fallbackReason = null;
|
||||
@@ -134,31 +112,7 @@ public class LookupKnowledgeTool {
|
||||
fallbackReason, selectedEvidence);
|
||||
LookupResult result = resultAssembler.assemble(selectedEvidence, contextPack, retrievalTrace);
|
||||
|
||||
String sessionId = SessionContextHolder.getSessionId();
|
||||
String domain = extractDomain(knowledgeQuery);
|
||||
if (sessionId != null && result.isFound()) {
|
||||
String docKey = extractDocKey(result);
|
||||
if (docKey != null && retrievedDocTracker.isAlreadyRetrieved(sessionId, docKey)) {
|
||||
log.info("[去重] 文档已在本会话中检索过，跳过: {}", docKey);
|
||||
saveToolInvocation(knowledgeQuery, startTime, result, domain, "doc_retrieved");
|
||||
LookupResult deduped = resultAssembler.deduped(
|
||||
result,
|
||||
retrievedDocTracker.getRetrievedDomains(sessionId),
|
||||
docKey);
|
||||
logReturn(deduped, startTime);
|
||||
return deduped;
|
||||
}
|
||||
if (docKey != null) {
|
||||
retrievedDocTracker.markRetrieved(sessionId, domain, docKey);
|
||||
}
|
||||
}
|
||||
|
||||
if (sessionId != null) {
|
||||
result.setRetrievedDomainsThisSession(retrievedDocTracker.getRetrievedDomains(sessionId));
|
||||
}
|
||||
|
||||
saveToolInvocation(knowledgeQuery, startTime, result, domain, null);
|
||||
logReturn(result, startTime);
|
||||
logReturn(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -188,70 +142,20 @@ public class LookupKnowledgeTool {
|
||||
.selectedAttempt(selectedAttempt)
|
||||
.fallbackReason(fallbackReason)
|
||||
.evidenceStatus(evidence.hasUsableEvidence()
|
||||
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
|
||||
: ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE)
|
||||
? "supported"
|
||||
: "no_evidence")
|
||||
.queryHints(queryHints)
|
||||
.attempts(attempts)
|
||||
.build();
|
||||
}
|
||||
|
||||
private String extractDomain(KnowledgeQuery query) {
|
||||
if (query.getCategoryFilter() != null && !query.getCategoryFilter().isBlank()) {
|
||||
return query.getCategoryFilter();
|
||||
}
|
||||
if (query.getDomainHints() != null && !query.getDomainHints().isEmpty()) {
|
||||
return query.getDomainHints().get(0);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractDocKey(LookupResult result) {
|
||||
if (result.getEvidenceBlocks() == null || result.getEvidenceBlocks().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
EvidenceBlock first = result.getEvidenceBlocks().get(0);
|
||||
if (first.getSource() != null && !first.getSource().isBlank()) {
|
||||
return first.getSource();
|
||||
}
|
||||
if (first.getTitle() != null && !first.getTitle().isBlank()) {
|
||||
return first.getTitle();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void saveToolInvocation(KnowledgeQuery query,
|
||||
long startTime,
|
||||
LookupResult result,
|
||||
String domain,
|
||||
String dedupReason) {
|
||||
try {
|
||||
if (SessionContextHolder.getSessionId() == null) {
|
||||
return;
|
||||
}
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord record =
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord.from(
|
||||
query,
|
||||
result,
|
||||
domain,
|
||||
dedupReason,
|
||||
(int) Math.max(0, System.currentTimeMillis() - startTime)
|
||||
);
|
||||
toolInvocationRecorder.recordLookupKnowledge(record);
|
||||
} catch (Exception e) {
|
||||
log.error("保存 tool_invocation 失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void logReturn(LookupResult result, long startTime) {
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
private void logReturn(LookupResult result) {
|
||||
log.info("----------------------------------------");
|
||||
log.info("<<< [工具返回] lookup_knowledge");
|
||||
log.info("<<< 结果: found={}, relevanceLevel={}, evidenceBlocks={}, 耗时: {}ms",
|
||||
log.info("<<< 结果: found={}, relevanceLevel={}, evidenceBlocks={}",
|
||||
result.isFound(),
|
||||
result.getRelevanceLevel(),
|
||||
result.getEvidenceBlockCount(),
|
||||
totalTime);
|
||||
log.info("<<< 行动记忆: retrievedDomainsThisSession={}", result.getRetrievedDomainsThisSession());
|
||||
result.getEvidenceBlockCount());
|
||||
if (result.getRetrievalTrace() != null) {
|
||||
log.info("<<< 检索路径: selectedAttempt={}, fallbackReason={}",
|
||||
result.getRetrievalTrace().getSelectedAttempt(),
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* session 级已召回文档追踪器
|
||||
* 支持文档级去重 + 域级行动记忆
|
||||
*
|
||||
* 数据结构：sessionId → { domain → Set<filePath> }
|
||||
* - 域级：控制"不要重复查同域"，提供行动记忆给 LLM
|
||||
* - 文档级：控制"不要重复召回同文档"（替代原有单层结构）
|
||||
*/
|
||||
@Component
|
||||
public class RetrievedDocTracker {
|
||||
|
||||
// key: sessionId, value: { domain → Set<filePath> }
|
||||
private final ConcurrentHashMap<String, Map<String, Set<String>>> sessionRetrievals = new ConcurrentHashMap<>();
|
||||
|
||||
/**
|
||||
* 记录一次检索（域级 + 文档级）
|
||||
*/
|
||||
public void markRetrieved(String sessionId, String domain, String filePath) {
|
||||
if (sessionId == null || filePath == null) return;
|
||||
|
||||
sessionRetrievals.computeIfAbsent(sessionId,
|
||||
k -> new ConcurrentHashMap<>())
|
||||
.computeIfAbsent(domain != null ? domain : "_unknown",
|
||||
d -> Collections.newSetFromMap(new ConcurrentHashMap<>()))
|
||||
.add(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 文档级去重：检查 filePath 是否已在本会话中检索过
|
||||
*/
|
||||
public boolean isDocRetrieved(String sessionId, String filePath) {
|
||||
if (sessionId == null || filePath == null) return false;
|
||||
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
|
||||
if (domains == null) return false;
|
||||
return domains.values().stream().anyMatch(docs -> docs.contains(filePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* 域级检查：检查 domain 是否已在本会话中检索过
|
||||
*/
|
||||
public boolean isDomainRetrieved(String sessionId, String domain) {
|
||||
if (sessionId == null || domain == null) return false;
|
||||
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
|
||||
return domains != null && domains.containsKey(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取本次会话已检索的域列表（行动记忆，返回给 LLM）
|
||||
*/
|
||||
public List<String> getRetrievedDomains(String sessionId) {
|
||||
if (sessionId == null) return List.of();
|
||||
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
|
||||
if (domains == null) return List.of();
|
||||
return List.copyOf(domains.keySet());
|
||||
}
|
||||
|
||||
/**
|
||||
* 向后兼容：文档级去重（委托给 isDocRetrieved）
|
||||
*/
|
||||
public boolean isAlreadyRetrieved(String sessionId, String filePath) {
|
||||
return isDocRetrieved(sessionId, filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 向后兼容：旧版 markRetrieved（domain 设为 null，归入 _unknown）
|
||||
*/
|
||||
public void markRetrieved(String sessionId, String filePath) {
|
||||
markRetrieved(sessionId, null, filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理会话
|
||||
*/
|
||||
public void clearSession(String sessionId) {
|
||||
if (sessionId == null) return;
|
||||
sessionRetrievals.remove(sessionId);
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package com.superbiz.agent.util;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 问题复杂度判断
|
||||
* 用于决定使用单 Agent 还是多 Agent（Planner + Executor）处理
|
||||
*/
|
||||
public class QuestionComplexity {
|
||||
|
||||
/** 复杂问题关键词 — 需要多步分析、排查、根因定位 */
|
||||
private static final List<String> COMPLEX_KEYWORDS = List.of(
|
||||
"排查", "分析", "为什么", "根因", "调查", "对比", "影响范围",
|
||||
"原因", "故障", "告警", "诊断", "链路", "流程", "步骤",
|
||||
"root cause", "troubleshoot", "investigate"
|
||||
);
|
||||
|
||||
/** 极简问题关键词 — 快速回答，无需多 Agent */
|
||||
private static final List<String> SIMPLE_KEYWORDS = List.of(
|
||||
"是什么", "查一下", "什么是", "时间", "天气", "定义",
|
||||
"查", "找", "what is", "define", "time"
|
||||
);
|
||||
|
||||
/**
|
||||
* 判断是否为复杂问题
|
||||
*/
|
||||
public static boolean isComplex(String question) {
|
||||
if (question == null || question.isBlank()) return false;
|
||||
String q = question.toLowerCase();
|
||||
|
||||
// 复杂关键词匹配 → 多 Agent
|
||||
for (String kw : COMPLEX_KEYWORDS) {
|
||||
if (q.contains(kw)) return true;
|
||||
}
|
||||
|
||||
// 简单关键词匹配 → 单 Agent
|
||||
for (String kw : SIMPLE_KEYWORDS) {
|
||||
if (q.contains(kw)) return false;
|
||||
}
|
||||
|
||||
// 默认：长问题（>30 字）视为复杂，短问题视为简单
|
||||
return question.length() > 30;
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package com.superbiz.agent.util;
|
||||
|
||||
/**
|
||||
* 会话上下文持有者（基于 ThreadLocal）
|
||||
* <p>
|
||||
* 用于在同步调用链路中传递 sessionId，兜底 LookupKnowledgeTool 等
|
||||
* 无法通过 RunnableConfig 获取上下文的组件。
|
||||
* 优先使用 RunnableConfig.metadata 传递，ThreadLocal 作为同步路径的补充。
|
||||
* <p>
|
||||
* 使用规范：
|
||||
* 1. 调用方在 Agent 执行前调用 setSessionId()
|
||||
* 2. finally 块中调用 clear()
|
||||
*/
|
||||
public class SessionContextHolder {
|
||||
|
||||
private static final ThreadLocal<String> SESSION_ID = new ThreadLocal<>();
|
||||
private static final ThreadLocal<String> RUN_ID = new ThreadLocal<>();
|
||||
|
||||
public static void setContext(String sessionId, String runId) {
|
||||
setSessionId(sessionId);
|
||||
setRunId(runId);
|
||||
}
|
||||
|
||||
public static void setSessionId(String sessionId) {
|
||||
SESSION_ID.set(sessionId);
|
||||
}
|
||||
|
||||
public static void setRunId(String runId) {
|
||||
RUN_ID.set(runId);
|
||||
}
|
||||
|
||||
public static String getSessionId() {
|
||||
return SESSION_ID.get();
|
||||
}
|
||||
|
||||
public static String getRunId() {
|
||||
return RUN_ID.get();
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
SESSION_ID.remove();
|
||||
RUN_ID.remove();
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
package com.superbiz.agent.util;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Thread-local verifier context shared across one planner/executor/verifier round.
|
||||
*/
|
||||
public final class VerifierContextHolder {
|
||||
|
||||
private static final ThreadLocal<String> ORIGINAL_QUERY = new ThreadLocal<>();
|
||||
private static final ThreadLocal<String> RETRY_CONTEXT = new ThreadLocal<>();
|
||||
private static final ThreadLocal<String> EXECUTOR_FINAL_ANSWER = new ThreadLocal<>();
|
||||
private static final ThreadLocal<Map<String, Object>> EXECUTOR_STRUCTURED_OUTPUT = new ThreadLocal<>();
|
||||
private static final ThreadLocal<Map<String, Object>> EXECUTOR_OUTPUT_PARSE_STATUS = new ThreadLocal<>();
|
||||
private static final ThreadLocal<List<Map<String, Object>>> TOOL_TRACE_SUMMARY = new ThreadLocal<>();
|
||||
private static final ThreadLocal<Map<String, Object>> GATEKEEPER_RESULT = new ThreadLocal<>();
|
||||
|
||||
private VerifierContextHolder() {
|
||||
}
|
||||
|
||||
public static void setOriginalQuery(String originalQuery) {
|
||||
ORIGINAL_QUERY.set(originalQuery);
|
||||
}
|
||||
|
||||
public static String getOriginalQuery() {
|
||||
return ORIGINAL_QUERY.get();
|
||||
}
|
||||
|
||||
public static void setRetryContext(String retryContext) {
|
||||
RETRY_CONTEXT.set(retryContext);
|
||||
}
|
||||
|
||||
public static String getRetryContext() {
|
||||
return RETRY_CONTEXT.get();
|
||||
}
|
||||
|
||||
public static void setExecutorFinalAnswer(String executorFinalAnswer) {
|
||||
EXECUTOR_FINAL_ANSWER.set(executorFinalAnswer);
|
||||
}
|
||||
|
||||
public static String getExecutorFinalAnswer() {
|
||||
return EXECUTOR_FINAL_ANSWER.get();
|
||||
}
|
||||
|
||||
public static void setExecutorStructuredOutput(Map<String, Object> executorStructuredOutput) {
|
||||
EXECUTOR_STRUCTURED_OUTPUT.set(executorStructuredOutput);
|
||||
}
|
||||
|
||||
public static Map<String, Object> getExecutorStructuredOutput() {
|
||||
return EXECUTOR_STRUCTURED_OUTPUT.get();
|
||||
}
|
||||
|
||||
public static void setExecutorOutputParseStatus(Map<String, Object> executorOutputParseStatus) {
|
||||
EXECUTOR_OUTPUT_PARSE_STATUS.set(executorOutputParseStatus);
|
||||
}
|
||||
|
||||
public static Map<String, Object> getExecutorOutputParseStatus() {
|
||||
return EXECUTOR_OUTPUT_PARSE_STATUS.get();
|
||||
}
|
||||
|
||||
public static void setToolTraceSummary(List<Map<String, Object>> toolTraceSummary) {
|
||||
TOOL_TRACE_SUMMARY.set(toolTraceSummary);
|
||||
}
|
||||
|
||||
public static List<Map<String, Object>> getToolTraceSummary() {
|
||||
return TOOL_TRACE_SUMMARY.get();
|
||||
}
|
||||
|
||||
public static void setGatekeeperResult(Map<String, Object> gatekeeperResult) {
|
||||
GATEKEEPER_RESULT.set(gatekeeperResult);
|
||||
}
|
||||
|
||||
public static Map<String, Object> getGatekeeperResult() {
|
||||
return GATEKEEPER_RESULT.get();
|
||||
}
|
||||
|
||||
public static void clear() {
|
||||
ORIGINAL_QUERY.remove();
|
||||
RETRY_CONTEXT.remove();
|
||||
EXECUTOR_FINAL_ANSWER.remove();
|
||||
EXECUTOR_STRUCTURED_OUTPUT.remove();
|
||||
EXECUTOR_OUTPUT_PARSE_STATUS.remove();
|
||||
TOOL_TRACE_SUMMARY.remove();
|
||||
GATEKEEPER_RESULT.remove();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user