refactor(harness): remove legacy agent architecture

This commit is contained in:
zhuyongxin
2026-07-22 18:02:01 +08:00
parent bc36248cd8
commit 8ee7cc0b70
148 changed files with 3091 additions and 13889 deletions
@@ -1,27 +0,0 @@
package com.superbiz.agent.agent.tool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
@Component
public class DateTimeTools {
private static final Logger logger = LoggerFactory.getLogger(DateTimeTools.class);
/** 工具名常量,用于动态构建提示词 */
public static final String TOOL_GET_CURRENT_DATETIME = "getCurrentDateTime";
@Tool(description = "Get the current date and time in the user's timezone. " +
"IMPORTANT: Time changes constantly. Always call this tool when user asks about time, " +
"even if there's a recent time query in the conversation history.")
public String getCurrentDateTime() {
String currentTime = LocalDateTime.now().atZone(LocaleContextHolder.getTimeZone().toZoneId()).toString();
logger.debug("🕐 getCurrentDateTime 调用 - 返回时间: {}", currentTime);
return currentTime;
}
}
@@ -1,86 +0,0 @@
package com.superbiz.agent.agent.tool;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.VectorSearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* 内部文档查询工具
* 使用 RAG (Retrieval-Augmented Generation) 从内部知识库检索相关文档
*
* @deprecated 请使用 {@link com.superbiz.agent.tool.LookupKnowledgeTool} 替代。
* lookup_knowledge 支持 L0 精确匹配 + L1 语义检索,性能更优且功能更全面。
* 计划在下一个版本中移除此工具。
*/
@Deprecated
@Component
public class InternalDocsTools {
private static final Logger logger = LoggerFactory.getLogger(InternalDocsTools.class);
/** 工具名常量,用于动态构建提示词 */
public static final String TOOL_QUERY_INTERNAL_DOCS = "queryInternalDocs";
private final VectorSearchService vectorSearchService;
@Value("${rag.top-k:3}")
private int topK = 3; // 默认值
private final ObjectMapper objectMapper = new ObjectMapper();
/**
* 构造函数注入依赖
* Spring 会自动注入 VectorSearchService
*/
@Autowired
public InternalDocsTools(VectorSearchService vectorSearchService) {
this.vectorSearchService = vectorSearchService;
}
/**
* 查询内部文档工具
*
* @param query 搜索查询,描述您要查找的信息
* @return JSON 格式的搜索结果,包含相关文档内容、相似度分数和元数据
* @deprecated 请使用 {@link com.superbiz.agent.tool.LookupKnowledgeTool#lookupKnowledge(String)} 替代
*/
@Deprecated
@Tool(description = "Use this tool to search internal documentation and knowledge base for relevant information. " +
"It performs RAG (Retrieval-Augmented Generation) to find similar documents and extract processing steps. " +
"This is useful when you need to understand internal procedures, best practices, or step-by-step guides " +
"stored in the company's documentation.")
public String queryInternalDocs(
@ToolParam(description = "Search query describing what information you are looking for")
String query) {
try {
// 使用向量搜索服务检索相关文档
List<VectorSearchService.SearchResult> searchResults =
vectorSearchService.searchSimilarDocuments(query, topK);
if (searchResults.isEmpty()) {
return "{\"status\": \"no_results\", \"message\": \"No relevant documents found in the knowledge base.\"}";
}
// 将搜索结果转换为 JSON 格式
String resultJson = objectMapper.writeValueAsString(searchResults);
return resultJson;
} catch (Exception e) {
logger.error("[工具错误] queryInternalDocs 执行失败", e);
return String.format("{\"status\": \"error\", \"message\": \"Failed to query internal docs: %s\"}",
e.getMessage());
}
}
}
@@ -2,12 +2,9 @@ package com.superbiz.agent.agent.tool;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ToolInvocationRecorder;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -30,16 +27,7 @@ public class QueryLogsTools {
private static final Logger logger = LoggerFactory.getLogger(QueryLogsTools.class);
/** 工具名常量,用于动态构建提示词 */
public static final String TOOL_QUERY_LOGS = "queryLogs";
public static final String TOOL_GET_AVAILABLE_LOG_TOPICS = "getAvailableLogTopics";
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolInvocationRecorder toolInvocationRecorder;
public QueryLogsTools(ToolInvocationRecorder toolInvocationRecorder) {
this.toolInvocationRecorder = toolInvocationRecorder;
}
@Value("${cls.mock-enabled:false}")
private boolean mockEnabled;
@@ -53,97 +41,6 @@ public class QueryLogsTools {
logger.info("✅ QueryLogsTools 初始化成功, Mock模式: {}", mockEnabled);
}
/**
* 获取可用的日志主题列表
* 用于查询前先了解有哪些日志主题可供查询
*/
@Tool(description = "Get all available log topics and their descriptions. " +
"Call this tool first before querying logs to understand what log topics are available. " +
"Returns a list of log topics with their names, descriptions, and example queries.")
public String getAvailableLogTopics() {
long startTime = System.currentTimeMillis();
logger.info("获取可用的日志主题列表");
try {
List<LogTopicInfo> topics = new ArrayList<>();
// 系统指标日志
LogTopicInfo systemMetrics = new LogTopicInfo();
systemMetrics.setTopicName("system-metrics");
systemMetrics.setDescription("系统指标日志,包含 CPU、内存、磁盘使用率等系统资源监控数据");
systemMetrics.setExampleQueries(List.of(
"cpu_usage:>80",
"memory_usage:>85",
"disk_usage:>90",
"level:WARN AND service:payment-service"
));
systemMetrics.setRelatedAlerts(List.of("HighCPUUsage", "HighMemoryUsage", "HighDiskUsage"));
topics.add(systemMetrics);
// 应用日志
LogTopicInfo applicationLogs = new LogTopicInfo();
applicationLogs.setTopicName("application-logs");
applicationLogs.setDescription("应用日志,包含应用程序的错误日志、警告日志、慢请求日志、下游依赖调用日志等");
applicationLogs.setExampleQueries(List.of(
"level:ERROR",
"level:FATAL",
"http_status:500",
"response_time:>3000",
"slow",
"downstream OR redis OR database OR mq"
));
applicationLogs.setRelatedAlerts(List.of("ServiceUnavailable", "SlowResponse", "HighMemoryUsage"));
topics.add(applicationLogs);
// 数据库慢查询日志
LogTopicInfo dbSlowQuery = new LogTopicInfo();
dbSlowQuery.setTopicName("database-slow-query");
dbSlowQuery.setDescription("数据库慢查询日志,包含执行时间较长的 SQL 查询,可用于分析数据库性能问题");
dbSlowQuery.setExampleQueries(List.of(
"query_time:>2",
"table:orders",
"query_type:SELECT",
"*" // 查询所有慢查询
));
dbSlowQuery.setRelatedAlerts(List.of("SlowResponse", "ServiceUnavailable"));
topics.add(dbSlowQuery);
// 系统事件日志
LogTopicInfo systemEvents = new LogTopicInfo();
systemEvents.setTopicName("system-events");
systemEvents.setDescription("系统事件日志,包含 Kubernetes Pod 重启、OOM Kill、容器崩溃等系统级事件");
systemEvents.setExampleQueries(List.of(
"restart OR crash",
"oom_kill",
"event_type:PodRestart",
"reason:OOMKilled"
));
systemEvents.setRelatedAlerts(List.of("ServiceUnavailable", "HighMemoryUsage"));
topics.add(systemEvents);
// 构建输出
LogTopicsOutput output = new LogTopicsOutput();
output.setSuccess(true);
output.setTopics(topics);
output.setAvailableRegions(List.of("ap-guangzhou", "ap-shanghai", "ap-beijing", "ap-chengdu"));
output.setDefaultRegion("ap-guangzhou");
output.setMessage(String.format("共有 %d 个可用的日志主题。建议使用默认地域 'ap-guangzhou' 或省略 region 参数", topics.size()));
String response = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
response, true, null, "logs", ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return response;
} catch (Exception e) {
logger.error("获取日志主题列表失败", e);
String response = "{\"success\":false,\"message\":\"获取日志主题列表失败: " + e.getMessage() + "\"}";
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
response, false, e.getMessage(), "logs", ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
/**
* 查询日志
* 从云日志服务查询指定条件的日志
@@ -160,24 +57,9 @@ public class QueryLogsTools {
private static final String DEFAULT_REGION = "ap-guangzhou";
@Tool(description = "Query logs from Cloud Log Service (CLS). " +
"Use this tool to search application logs, system metrics, and other log data. " +
"IMPORTANT: Before calling this tool, you should call getAvailableLogTopics to understand what log topics are available. " +
"Available log topics: " +
"1) 'system-metrics' - System metrics logs (CPU, memory, disk usage, etc. Related to HighCPUUsage, HighMemoryUsage, HighDiskUsage alerts); " +
"2) 'application-logs' - Application logs (error logs, slow request logs, downstream dependency logs. Related to ServiceUnavailable, SlowResponse alerts); " +
"3) 'database-slow-query' - Database slow query logs (SQL queries with long execution time. Related to SlowResponse alerts); " +
"4) 'system-events' - System event logs (Pod restart, OOM Kill, container crash. Related to ServiceUnavailable, HighMemoryUsage alerts). " +
"logTopic (required, one of the above topics or their CLS topicId), " +
"query (optional, defaults to a curated search if empty), " +
"limit (optional, default 20, max 100).")
public String queryLogs(
@ToolParam(description = "地域,可选值: ap-guangzhou, ap-shanghai, ap-beijing, ap-chengdu。默认 ap-guangzhou") String region,
@ToolParam(description = "日志主题,如 system-metrics, application-logs, database-slow-query, system-events,也支持 CLS TopicId") String logTopic,
@ToolParam(description = "查询条件,支持 Lucene 语法,如 level:ERROR OR cpu_usage:>80;为空时返回该主题近 5 条核心日志") String query,
@ToolParam(description = "返回日志条数,默认20,最大100") Integer limit) {
String region, String logTopic, String query, Integer limit) {
long startTime = System.currentTimeMillis();
int actualLimit = (limit == null || limit <= 0) ? 20 : Math.min(limit, 100);
String safeQuery = query == null ? "" : query;
@@ -193,15 +75,12 @@ public class QueryLogsTools {
} else {
// 真实模式:调用 CLS API(这里预留接口,后续实现)
String response = buildErrorResponse("CLS 真实查询尚未实现,请启用 mock 模式进行测试");
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
"CLS 真实查询尚未实现,请启用 mock 模式进行测试", normalizeTopicDomain(logTopic),
ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
// 构建成功响应
QueryLogsOutput output = new QueryLogsOutput();
output.setSuccess(!logEntries.isEmpty());
output.setSuccess(true);
output.setRegion(region);
output.setLogTopic(logTopic);
output.setQuery(safeQuery.isBlank() ? "DEFAULT_QUERY" : safeQuery);
@@ -211,56 +90,15 @@ public class QueryLogsTools {
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
logger.info("日志查询完成: 找到 {} 条日志", logEntries.size());
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, jsonResult,
true, null, normalizeTopicDomain(logTopic),
logEntries.isEmpty()
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return jsonResult;
} catch (Exception e) {
logger.error("查询日志失败", e);
String response = buildErrorResponse("查询失败: " + e.getMessage());
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
e.getMessage(), normalizeTopicDomain(logTopic), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
private void recordInvocation(String toolName, long startTime, String query, String region, String logTopic, Integer limit,
String output, boolean success, String errorMessage, String topicDomain,
String evidenceStatus) {
Map<String, Object> input = new HashMap<>();
input.put("query", query == null || query.isBlank() ? "DEFAULT_QUERY" : query);
if (region != null) {
input.put("region", region);
}
if (logTopic != null) {
input.put("log_topic", logTopic);
}
if (limit != null) {
input.put("limit", limit);
}
input.put("mock_enabled", mockEnabled);
toolInvocationRecorder.recordEvidenceTool(
toolName,
input,
output,
success,
startTime,
errorMessage,
topicDomain,
evidenceStatus,
Map.of("log_topic", logTopic == null ? "" : logTopic)
);
}
private String normalizeTopicDomain(String logTopic) {
return logTopic == null || logTopic.isBlank() ? "logs" : logTopic;
}
/**
* 构建 Mock 日志数据
@@ -764,42 +602,4 @@ public class QueryLogsTools {
private String message;
}
/**
* 日志主题信息
*/
@Data
public static class LogTopicInfo {
@JsonProperty("topic_name")
private String topicName;
@JsonProperty("description")
private String description;
@JsonProperty("example_queries")
private List<String> exampleQueries;
@JsonProperty("related_alerts")
private List<String> relatedAlerts;
}
/**
* 日志主题列表输出
*/
@Data
public static class LogTopicsOutput {
@JsonProperty("success")
private boolean success;
@JsonProperty("topics")
private List<LogTopicInfo> topics;
@JsonProperty("available_regions")
private List<String> availableRegions;
@JsonProperty("default_region")
private String defaultRegion;
@JsonProperty("message")
private String message;
}
}
@@ -1,332 +0,0 @@
package com.superbiz.agent.agent.tool;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ToolInvocationRecorder;
import lombok.Data;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
/**
* Prometheus 告警查询工具
* 用于查询 Prometheus 的活动告警信息
*/
@Component
public class QueryMetricsTools {
private static final Logger logger = LoggerFactory.getLogger(QueryMetricsTools.class);
/** 工具名常量,用于动态构建提示词 */
public static final String TOOL_QUERY_PROMETHEUS_ALERTS = "queryPrometheusAlerts";
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolInvocationRecorder toolInvocationRecorder;
public QueryMetricsTools(ToolInvocationRecorder toolInvocationRecorder) {
this.toolInvocationRecorder = toolInvocationRecorder;
}
@Value("${prometheus.base-url}")
private String prometheusBaseUrl;
@Value("${prometheus.timeout:10}")
private int timeout;
@Value("${prometheus.mock-enabled:false}")
private boolean mockEnabled;
private OkHttpClient httpClient;
@jakarta.annotation.PostConstruct
public void init() {
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(Duration.ofSeconds(timeout))
.readTimeout(Duration.ofSeconds(timeout))
.build();
logger.info("✅ QueryMetricsTools 初始化成功, Prometheus URL: {}, Mock模式: {}", prometheusBaseUrl, mockEnabled);
}
/**
* 查询 Prometheus 活动告警
* 该工具从 Prometheus 告警系统检索所有当前活动/触发的告警,包括标签、注释、状态和值
*/
@Tool(description = "Query active alerts from Prometheus alerting system. " +
"This tool retrieves all currently active/firing alerts including their labels, annotations, state, and values. " +
"Use this tool when you need to check what alerts are currently firing, investigate alert conditions, or monitor alert status.")
public String queryPrometheusAlerts() {
long startTime = System.currentTimeMillis();
logger.info("开始查询 Prometheus 活动告警, Mock模式: {}", mockEnabled);
try {
List<SimplifiedAlert> simplifiedAlerts;
if (mockEnabled) {
// Mock 模式:返回与文档关联的模拟告警数据
simplifiedAlerts = buildMockAlerts();
logger.info("使用 Mock 数据,返回 {} 个模拟告警", simplifiedAlerts.size());
} else {
// 真实模式:调用 Prometheus Alerts API
PrometheusAlertsResult result = fetchPrometheusAlerts();
if (!"success".equals(result.getStatus())) {
String response = buildErrorResponse("Prometheus API 返回非成功状态: " + result.getStatus(), result.getError());
recordInvocation(startTime, response, false, result.getError(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
// 转换为简化格式,对于相同的 alertname,只保留第一个
Set<String> seenAlertNames = new HashSet<>();
simplifiedAlerts = new ArrayList<>();
for (PrometheusAlert alert : result.getData().getAlerts()) {
String alertName = alert.getLabels().get("alertname");
// 如果这个 alertname 已经存在,跳过
if (seenAlertNames.contains(alertName)) {
continue;
}
// 标记为已见过
seenAlertNames.add(alertName);
SimplifiedAlert simplified = new SimplifiedAlert();
simplified.setAlertName(alertName);
simplified.setDescription(alert.getAnnotations().getOrDefault("description", ""));
simplified.setState(alert.getState());
simplified.setActiveAt(alert.getActiveAt());
simplified.setDuration(calculateDuration(alert.getActiveAt()));
simplifiedAlerts.add(simplified);
}
}
// 构建成功响应
PrometheusAlertsOutput output = new PrometheusAlertsOutput();
output.setSuccess(true);
output.setAlerts(simplifiedAlerts);
output.setMessage(String.format("成功检索到 %d 个活动告警", simplifiedAlerts.size()));
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
logger.info("Prometheus 告警查询完成: 找到 {} 个告警", simplifiedAlerts.size());
recordInvocation(startTime, jsonResult, true, null,
simplifiedAlerts.isEmpty()
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return jsonResult;
} catch (Exception e) {
logger.error("查询 Prometheus 告警失败", e);
String response = buildErrorResponse("查询失败", e.getMessage());
recordInvocation(startTime, response, false, e.getMessage(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
private void recordInvocation(long startTime, String output, boolean success, String errorMessage, String evidenceStatus) {
toolInvocationRecorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts", "mock_enabled", mockEnabled),
output,
success,
startTime,
errorMessage,
"prometheus_alerts",
evidenceStatus,
Map.of("metric_family", "prometheus_alerts")
);
}
/**
* 构建 Mock 告警数据
* 与 aiops-docs 文档中的告警类型对应:
* - HighCPUUsage: CPU使用率过高
* - HighMemoryUsage: 内存使用率过高
* - HighDiskUsage: 磁盘使用率过高
* - ServiceUnavailable: 服务不可用
* - SlowResponse: 响应时间过长
*/
private List<SimplifiedAlert> buildMockAlerts() {
List<SimplifiedAlert> alerts = new ArrayList<>();
Instant now = Instant.now();
// 告警1: CPU使用率过高 - 持续约25分钟
SimplifiedAlert cpuAlert = new SimplifiedAlert();
cpuAlert.setAlertName("HighCPUUsage");
cpuAlert.setDescription("服务 payment-service 的 CPU 使用率持续超过 80%,当前值为 92%。" +
"实例: pod-payment-service-7d8f9c6b5-x2k4m,命名空间: production");
cpuAlert.setState("firing");
Instant cpuActiveAt = now.minus(25, ChronoUnit.MINUTES);
cpuAlert.setActiveAt(cpuActiveAt.toString());
cpuAlert.setDuration(calculateDuration(cpuActiveAt.toString()));
alerts.add(cpuAlert);
// 告警2: 内存使用率过高 - 持续约15分钟
SimplifiedAlert memoryAlert = new SimplifiedAlert();
memoryAlert.setAlertName("HighMemoryUsage");
memoryAlert.setDescription("服务 order-service 的内存使用率持续超过 85%,当前值为 91%。" +
"JVM堆内存使用: 3.8GB/4GB,可能存在内存泄漏风险。" +
"实例: pod-order-service-5c7d8e9f1-m3n2p,命名空间: production");
memoryAlert.setState("firing");
Instant memoryActiveAt = now.minus(15, ChronoUnit.MINUTES);
memoryAlert.setActiveAt(memoryActiveAt.toString());
memoryAlert.setDuration(calculateDuration(memoryActiveAt.toString()));
alerts.add(memoryAlert);
// 告警3: 响应时间过长 - 持续约10分钟
SimplifiedAlert slowAlert = new SimplifiedAlert();
slowAlert.setAlertName("SlowResponse");
slowAlert.setDescription("服务 user-service 的 P99 响应时间持续超过 3 秒,当前值为 4.2 秒。" +
"受影响接口: /api/v1/users/profile, /api/v1/users/orders。" +
"可能原因:数据库慢查询或下游服务延迟");
slowAlert.setState("firing");
Instant slowActiveAt = now.minus(10, ChronoUnit.MINUTES);
slowAlert.setActiveAt(slowActiveAt.toString());
slowAlert.setDuration(calculateDuration(slowActiveAt.toString()));
alerts.add(slowAlert);
return alerts;
}
/**
* 从 Prometheus API 获取告警数据
*/
private PrometheusAlertsResult fetchPrometheusAlerts() throws Exception {
String apiUrl = prometheusBaseUrl + "/api/v1/alerts";
logger.debug("请求 Prometheus API: {}", apiUrl);
Request request = new Request.Builder()
.url(apiUrl)
.get()
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new RuntimeException("HTTP 请求失败: " + response.code());
}
String responseBody = response.body().string();
return objectMapper.readValue(responseBody, PrometheusAlertsResult.class);
}
}
/**
* 计算从 activeAt 到现在的持续时间
*/
private String calculateDuration(String activeAtStr) {
try {
Instant activeAt = Instant.parse(activeAtStr);
Duration duration = Duration.between(activeAt, Instant.now());
long hours = duration.toHours();
long minutes = duration.toMinutes() % 60;
long seconds = duration.getSeconds() % 60;
if (hours > 0) {
return String.format("%dh%dm%ds", hours, minutes, seconds);
} else if (minutes > 0) {
return String.format("%dm%ds", minutes, seconds);
} else {
return String.format("%ds", seconds);
}
} catch (Exception e) {
logger.warn("解析时间失败: {}", activeAtStr, e);
return "unknown";
}
}
/**
* 构建错误响应
*/
private String buildErrorResponse(String message, String error) {
try {
PrometheusAlertsOutput output = new PrometheusAlertsOutput();
output.setSuccess(false);
output.setMessage(message);
output.setError(error);
return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
} catch (Exception e) {
return String.format("{\"success\":false,\"message\":\"%s\",\"error\":\"%s\"}", message, error);
}
}
// ==================== 数据模型 ====================
/**
* Prometheus 告警信息结构
*/
@Data
public static class PrometheusAlert {
private Map<String, String> labels;
private Map<String, String> annotations;
private String state;
private String activeAt;
private String value;
}
/**
* Prometheus 告警查询结果
*/
@Data
public static class PrometheusAlertsResult {
private String status;
private AlertsData data;
private String error;
private String errorType;
}
@Data
public static class AlertsData {
private List<PrometheusAlert> alerts = new ArrayList<>();
}
/**
* 简化的告警信息
*/
@Data
public static class SimplifiedAlert {
@JsonProperty("alert_name")
private String alertName;
@JsonProperty("description")
private String description;
@JsonProperty("state")
private String state;
@JsonProperty("active_at")
private String activeAt;
@JsonProperty("duration")
private String duration;
}
/**
* 告警查询输出
*/
@Data
public static class PrometheusAlertsOutput {
@JsonProperty("success")
private boolean success;
@JsonProperty("alerts")
private List<SimplifiedAlert> alerts;
@JsonProperty("message")
private String message;
@JsonProperty("error")
private String error;
}
}
@@ -1,56 +0,0 @@
package com.superbiz.agent.config;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.ClassPathResource;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* AI Ops Agent Prompt 配置
* 从独立的 Markdown 文件加载 Prompt 模板
*/
@Slf4j
@Configuration
public class AiOpsPromptProperties {
private String planner;
private String executor;
private String supervisor;
@PostConstruct
public void loadPrompts() {
try {
planner = loadPromptFromFile("prompts/planner-prompt.md");
executor = loadPromptFromFile("prompts/executor-prompt.md");
supervisor = loadPromptFromFile("prompts/supervisor-prompt.md");
log.info("AI Ops Prompts 加载成功");
log.debug("Planner Prompt 长度: {} 字符", planner.length());
log.debug("Executor Prompt 长度: {} 字符", executor.length());
log.debug("Supervisor Prompt 长度: {} 字符", supervisor.length());
} catch (IOException e) {
log.error("加载 Prompt 文件失败", e);
throw new RuntimeException("Failed to load AI Ops prompts", e);
}
}
private String loadPromptFromFile(String path) throws IOException {
ClassPathResource resource = new ClassPathResource(path);
return new String(resource.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
}
public String getPlanner() {
return planner;
}
public String getExecutor() {
return executor;
}
public String getSupervisor() {
return supervisor;
}
}
@@ -49,7 +49,8 @@ import com.superbiz.agent.harness.application.DiagnosisOperation;
import com.superbiz.agent.harness.application.KnowledgeQueryOperation;
import com.superbiz.agent.harness.application.SystemChatOperation;
import com.superbiz.agent.harness.application.IntentRouting;
import com.superbiz.agent.hook.AgentLoggingHook;
import com.superbiz.agent.harness.audit.HarnessAgentAuditHook;
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.agent.tool.QueryLogsTools;
@@ -137,8 +138,9 @@ public class HarnessChatConfiguration {
@Bean
public ToolBoundary toolBoundary(DiagnosisHarnessCore core, ToolCallKeyFactory keyFactory,
CanonicalInvocationStore store, ObjectMapper objectMapper, Clock clock) {
return new ToolBoundary(core, keyFactory, store, objectMapper, clock);
CanonicalInvocationStore store, ObjectMapper objectMapper, Clock clock,
ToolInvocationAuditSink auditSink) {
return new ToolBoundary(core, keyFactory, store, objectMapper, clock, auditSink);
}
@Bean
@@ -158,17 +160,17 @@ public class HarnessChatConfiguration {
@Bean
public RagToolAdapter ragToolAdapter(ToolBoundary boundary, ObjectMapper mapper,
RagResultProjector projector, LookupKnowledgeTool legacy) {
return new RagToolAdapter(boundary, mapper, projector, legacy::lookupKnowledge);
RagResultProjector projector, LookupKnowledgeTool backend) {
return new RagToolAdapter(boundary, mapper, projector, backend::lookupKnowledge);
}
@Bean
public QueryLogsToolAdapter queryLogsToolAdapter(ToolBoundary boundary, ObjectMapper mapper,
QueryLogsResultProjector projector,
ObjectProvider<QueryLogsTools> legacy, Clock clock) {
ObjectProvider<QueryLogsTools> backend, Clock clock) {
return new QueryLogsToolAdapter(boundary, mapper, projector,
(region, topic, query, limit) -> {
QueryLogsTools tools = legacy.getIfAvailable();
QueryLogsTools tools = backend.getIfAvailable();
if (tools == null) {
return "{\"success\":false,\"logs\":[],\"total\":0}";
}
@@ -230,10 +232,11 @@ public class HarnessChatConfiguration {
@Bean
public DiagnosisAgentFactory diagnosisAgentFactory(ChatModel chatModel, DiagnosisHarnessCore core,
HarnessEvidenceTools tools, ObjectMapper mapper,
AgentStepRepository steps) {
HarnessEvidenceTools tools, ObjectMapper mapper,
AgentStepRepository steps) {
return new DiagnosisAgentFactory(chatModel, core, tools, mapper,
List.of(new AgentLoggingHook(steps, DiagnosisAgentFactory.AGENT_NAME)));
List.of(new HarnessAgentAuditHook(
steps, mapper, DiagnosisAgentFactory.AGENT_NAME)));
}
@Bean
@@ -33,6 +33,6 @@ public class WebConfig implements WebMvcConfigurer {
@Bean
public ObjectMapper objectMapper() {
return new ObjectMapper();
return new ObjectMapper().findAndRegisterModules();
}
}
@@ -1,124 +0,0 @@
package com.superbiz.agent.controller;
import com.alibaba.cloud.ai.graph.OverAllState;
import com.superbiz.agent.dto.AIOpsRequest;
import com.superbiz.agent.service.AiOpsService;
import lombok.Getter;
import lombok.Setter;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.util.Optional;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
/** Preserves the legacy AiOps SSE protocol independently from Chat. */
@RestController
@RequestMapping("/api")
public class AiOpsController {
private final AiOpsService aiOpsService;
private final ThreadPoolExecutor workerExecutor;
public AiOpsController(AiOpsService aiOpsService,
@Qualifier("chatWorkerExecutor") ThreadPoolExecutor workerExecutor) {
this.aiOpsService = aiOpsService;
this.workerExecutor = workerExecutor;
}
@PostMapping(value = "/ai_ops", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter aiOps(@RequestBody(required = false) AIOpsRequest request) {
SseEmitter emitter = new SseEmitter(600_000L);
String sessionId = aiOpsService.resolveSessionId(request);
String runId = aiOpsService.newRunId();
emitter.onTimeout(emitter::complete);
try {
workerExecutor.execute(() -> executeAiOps(request, sessionId, runId, emitter));
} catch (RejectedExecutionException rejected) {
emitter.completeWithError(rejected);
}
return emitter;
}
private void executeAiOps(AIOpsRequest request, String sessionId, String runId, SseEmitter emitter) {
try {
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.metadata(sessionId, runId), MediaType.APPLICATION_JSON));
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.content("正在读取告警并拆解任务...\n"), MediaType.APPLICATION_JSON));
Optional<OverAllState> state = aiOpsService.executeAiOpsAnalysis(request, sessionId, runId);
if (state.isEmpty()) {
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.error("多 Agent 编排未获取到有效结果"), MediaType.APPLICATION_JSON));
emitter.complete();
return;
}
Optional<String> report = aiOpsService.extractFinalReport(state.get());
if (report.isPresent()) {
aiOpsService.persistFinalReport(sessionId, runId, report.get(), request);
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.content(report.get()), MediaType.APPLICATION_JSON));
} else {
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.content("多 Agent 流程已完成,但未能生成最终报告。"), MediaType.APPLICATION_JSON));
}
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.done(), MediaType.APPLICATION_JSON));
emitter.complete();
} catch (Exception exception) {
try {
emitter.send(SseEmitter.event().name("message")
.data(SseMessage.error("AI Ops 流程失败"), MediaType.APPLICATION_JSON));
} catch (IOException ignored) {
// Client has already gone away.
}
emitter.completeWithError(exception);
}
}
@Getter
@Setter
public static class SseMessage {
private String type;
private String data;
private String sessionId;
private String runId;
public static SseMessage content(String data) {
SseMessage message = new SseMessage();
message.type = "content";
message.data = data;
return message;
}
public static SseMessage metadata(String sessionId, String runId) {
SseMessage message = new SseMessage();
message.type = "metadata";
message.sessionId = sessionId;
message.runId = runId;
return message;
}
public static SseMessage error(String data) {
SseMessage message = new SseMessage();
message.type = "error";
message.data = data;
return message;
}
public static SseMessage done() {
SseMessage message = new SseMessage();
message.type = "done";
return message;
}
}
}
@@ -1,122 +0,0 @@
package com.superbiz.agent.controller;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.dto.DiagnosisTraceResponse;
import com.superbiz.agent.service.DiagnosisTraceService;
import com.superbiz.agent.service.session.SessionManager;
import lombok.Getter;
import lombok.Setter;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
import java.util.Optional;
/** Legacy Chat session management endpoints, isolated from the Chat execution adapter. */
@RestController
@RequestMapping("/api/chat")
public class ChatSessionController {
private final DiagnosisTraceService diagnosisTraceService;
private final SessionManager sessionManager;
public ChatSessionController(DiagnosisTraceService diagnosisTraceService, SessionManager sessionManager) {
this.diagnosisTraceService = diagnosisTraceService;
this.sessionManager = sessionManager;
}
@PostMapping("/clear")
public ResponseEntity<ApiResponse<String>> clearChatHistory(@RequestBody ClearRequest request) {
if (request == null || request.getId() == null || request.getId().isBlank()) {
return ResponseEntity.ok(ApiResponse.error("会话ID不能为空"));
}
try {
Optional<com.superbiz.agent.domain.model.SessionContext> session =
sessionManager.getSession(request.getId());
if (session.isEmpty()) {
return ResponseEntity.ok(ApiResponse.error("会话不存在"));
}
var context = session.get();
context.clearMessageHistory();
sessionManager.updateSession(context);
return ResponseEntity.ok(ApiResponse.success("会话历史已清空"));
} catch (RuntimeException exception) {
return ResponseEntity.ok(ApiResponse.error("会话历史清理失败"));
}
}
@GetMapping("/session/{sessionId}")
public ResponseEntity<ApiResponse<SessionInfoResponse>> getSessionInfo(@PathVariable String sessionId) {
try {
Optional<com.superbiz.agent.domain.model.SessionContext> session = sessionManager.getSession(sessionId);
if (session.isEmpty()) {
return ResponseEntity.ok(ApiResponse.error("会话不存在"));
}
var context = session.get();
SessionInfoResponse response = new SessionInfoResponse();
response.setSessionId(sessionId);
response.setMessagePairCount(context.getMessagePairCount());
response.setCreateTime(toEpochMillis(context.getCreatedAt()));
return ResponseEntity.ok(ApiResponse.success(response));
} catch (RuntimeException exception) {
return ResponseEntity.ok(ApiResponse.error("会话读取失败"));
}
}
@GetMapping("/session/{sessionId}/runs")
public ResponseEntity<ApiResponse<List<DiagnosisTraceResponse.RunSummary>>> listSessionRuns(
@PathVariable String sessionId) {
return ResponseEntity.ok(ApiResponse.success(diagnosisTraceService.listRunSummaries(sessionId)));
}
private long toEpochMillis(LocalDateTime time) {
return time == null ? 0L : time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
}
@Getter
@Setter
public static class ClearRequest {
@JsonProperty("Id")
@JsonAlias({"id", "ID"})
private String id;
}
@Getter
@Setter
public static class SessionInfoResponse {
private String sessionId;
private int messagePairCount;
private long createTime;
}
@Getter
@Setter
public static class ApiResponse<T> {
private int code;
private String message;
private T data;
public static <T> ApiResponse<T> success(T data) {
ApiResponse<T> response = new ApiResponse<>();
response.code = 200;
response.message = "success";
response.data = data;
return response;
}
public static <T> ApiResponse<T> error(String message) {
ApiResponse<T> response = new ApiResponse<>();
response.code = 500;
response.message = message;
return response;
}
}
}
@@ -10,7 +10,7 @@ import java.time.LocalDateTime;
/**
* Chat session metadata entity.
* Full message history remains in Redis SessionContext.
* Stores current Chat Run session metadata; message history is not persisted here.
*/
@Entity
@Table(name = "chat_session", indexes = {
@@ -61,4 +61,3 @@ public class ChatSession {
}
}
}
@@ -51,11 +51,11 @@ public class DiagnosisRun {
private String answer;
@Enumerated(EnumType.STRING)
@Column(name = "intent", length = 32)
@Column(name = "intent", length = 32, columnDefinition = "VARCHAR(32)")
private IntentType intent;
@Enumerated(EnumType.STRING)
@Column(name = "release_outcome", length = 16)
@Column(name = "release_outcome", length = 16, columnDefinition = "VARCHAR(16)")
private ReleaseOutcome releaseOutcome;
@JdbcTypeCode(SqlTypes.JSON)
@@ -1,159 +0,0 @@
package com.superbiz.agent.domain.model;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 会话上下文数据类
* 存储在 Redis 中的会话数据
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@JsonIgnoreProperties(ignoreUnknown = true)
public class SessionContext implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 会话ID
*/
private String sessionId;
/**
* 用户ID
*/
private String userId;
/**
* 业务ID(订单号/请求ID等)
*/
private String businessId;
/**
* 链路追踪ID
*/
private String traceId;
/**
* 会话状态(ACTIVE/COMPLETED/EXPIRED)
*/
private String status;
/**
* 工具调用历史
*/
@Builder.Default
private List<ToolCall> toolCalls = new ArrayList<>();
/**
* 聊天消息历史:[{"role":"user","content":"..."}, {"role":"assistant","content":"..."}]
*/
@Builder.Default
private List<Map<String, String>> messageHistory = new ArrayList<>();
/**
* 会话创建时间
*/
private LocalDateTime createdAt;
/**
* 最后活跃时间
*/
private LocalDateTime lastActiveAt;
/**
* 会话过期时间(秒)
*/
private Long ttl;
/**
* 添加工具调用记录
*/
public void addToolCall(ToolCall toolCall) {
if (this.toolCalls == null) {
this.toolCalls = new ArrayList<>();
}
this.toolCalls.add(toolCall);
this.lastActiveAt = LocalDateTime.now();
}
/**
* 添加一对聊天消息,并按消息对数裁剪窗口。
*/
public void addChatMessagePair(String userQuestion, String assistantAnswer, int maxPairCount) {
if (this.messageHistory == null) {
this.messageHistory = new ArrayList<>();
}
Map<String, String> userMessage = new HashMap<>();
userMessage.put("role", "user");
userMessage.put("content", userQuestion);
this.messageHistory.add(userMessage);
Map<String, String> assistantMessage = new HashMap<>();
assistantMessage.put("role", "assistant");
assistantMessage.put("content", assistantAnswer);
this.messageHistory.add(assistantMessage);
int maxMessages = Math.max(maxPairCount, 0) * 2;
while (maxMessages > 0 && this.messageHistory.size() > maxMessages) {
this.messageHistory.remove(0);
if (!this.messageHistory.isEmpty()) {
this.messageHistory.remove(0);
}
}
this.lastActiveAt = LocalDateTime.now();
}
/**
* 获取聊天历史副本,避免调用方直接修改内部列表。
*/
@JsonIgnore
public List<Map<String, String>> getMessageHistorySnapshot() {
if (this.messageHistory == null || this.messageHistory.isEmpty()) {
return new ArrayList<>();
}
List<Map<String, String>> snapshot = new ArrayList<>();
for (Map<String, String> message : this.messageHistory) {
snapshot.add(new HashMap<>(message));
}
return snapshot;
}
/**
* 清空聊天历史。
*/
public void clearMessageHistory() {
if (this.messageHistory == null) {
this.messageHistory = new ArrayList<>();
} else {
this.messageHistory.clear();
}
this.lastActiveAt = LocalDateTime.now();
}
@JsonIgnore
public int getMessagePairCount() {
return this.messageHistory == null ? 0 : this.messageHistory.size() / 2;
}
/**
* 更新最后活跃时间
*/
public void updateLastActiveTime() {
this.lastActiveAt = LocalDateTime.now();
}
}
@@ -1,45 +0,0 @@
package com.superbiz.agent.dto;
import lombok.Data;
/**
* AIOps 请求 DTO
*/
@Data
public class AIOpsRequest {
/**
* 诊断会话 ID;为空时后端自动生成。
*/
private String sessionId;
/**
* 告警名称。
*/
private String alertName;
/**
* 受影响服务。
*/
private String service;
/**
* 告警等级,例如 P0/P1/P2。
*/
private String severity;
/**
* 告警描述。
*/
private String description;
/**
* 排查时间范围,例如 last_15m。
*/
private String timeRange;
/**
* 用户请求描述
*/
private String userRequest;
}
@@ -140,7 +140,6 @@ public class DiagnosisTraceResponse {
private int persistedToolCallCount;
private int returnedToolCallCount;
private boolean hasVerifierEvaluation;
private boolean hasAiOpsRuleEvaluation;
private boolean hasFeedback;
}
@@ -1,252 +0,0 @@
package com.superbiz.agent.eval;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
public class DiagnosisEvalBaselineDiffer {
private static final String REGRESSION = "REGRESSION";
private static final String IMPROVEMENT = "IMPROVEMENT";
private static final String CHANGED = "CHANGED";
public DiagnosisEvalDiffReport compare(DiagnosisEvalReport baseline, DiagnosisEvalReport current) {
List<DiagnosisEvalDiffItem> items = new ArrayList<>();
compareDouble(items, "aggregate", null, "passRate",
baseline.getPassRate(), current.getPassRate(), true);
compareDouble(items, "aggregate", null, "averageToolCallCount",
baseline.getAverageToolCallCount(), current.getAverageToolCallCount(), false);
compareDouble(items, "aggregate", null, "averageDurationMs",
baseline.getAverageDurationMs(), current.getAverageDurationMs(), false);
compareVerdictDistribution(items, baseline.getVerdictDistribution(), current.getVerdictDistribution());
compareCases(items, safeResults(baseline), safeResults(current));
int regressionCount = countType(items, REGRESSION);
int improvementCount = countType(items, IMPROVEMENT);
int changedCount = countType(items, CHANGED);
return DiagnosisEvalDiffReport.builder()
.baselineTotalCases(baseline.getTotalCases())
.currentTotalCases(current.getTotalCases())
.baselinePassedCases(baseline.getPassedCases())
.currentPassedCases(current.getPassedCases())
.baselinePassRate(baseline.getPassRate())
.currentPassRate(current.getPassRate())
.regressionCount(regressionCount)
.improvementCount(improvementCount)
.changedCount(changedCount)
.hasRegression(regressionCount > 0)
.items(items)
.build();
}
private void compareVerdictDistribution(List<DiagnosisEvalDiffItem> items,
Map<String, Long> baseline,
Map<String, Long> current) {
Set<String> verdicts = new LinkedHashSet<>();
verdicts.addAll(safeMap(baseline).keySet());
verdicts.addAll(safeMap(current).keySet());
for (String verdict : verdicts) {
long baselineCount = safeMap(baseline).getOrDefault(verdict, 0L);
long currentCount = safeMap(current).getOrDefault(verdict, 0L);
if (baselineCount != currentCount) {
items.add(item(CHANGED, "aggregate", null, "verdictDistribution." + verdict,
String.valueOf(baselineCount), String.valueOf(currentCount),
(double) currentCount - baselineCount,
"verdict count changed for " + verdict));
}
}
}
private void compareCases(List<DiagnosisEvalDiffItem> items,
List<DiagnosisEvalResult> baselineResults,
List<DiagnosisEvalResult> currentResults) {
Map<String, DiagnosisEvalResult> baselineById = byCaseId(baselineResults);
Map<String, DiagnosisEvalResult> currentById = byCaseId(currentResults);
Set<String> caseIds = new LinkedHashSet<>();
caseIds.addAll(baselineById.keySet());
caseIds.addAll(currentById.keySet());
for (String caseId : caseIds) {
DiagnosisEvalResult baseline = baselineById.get(caseId);
DiagnosisEvalResult current = currentById.get(caseId);
if (baseline == null) {
items.add(item(CHANGED, "case", caseId, "casePresence",
"missing", "present", null, "new case appears in current report"));
continue;
}
if (current == null) {
items.add(item(REGRESSION, "case", caseId, "casePresence",
"present", "missing", null, "baseline case is missing from current report"));
continue;
}
comparePassState(items, baseline, current);
compareVerdict(items, baseline, current);
compareInteger(items, caseId, "matchedKeywordCount",
baseline.getMatchedKeywordCount(), current.getMatchedKeywordCount(), true);
compareInteger(items, caseId, "toolCallCount",
baseline.getToolCallCount(), current.getToolCallCount(), false);
compareInteger(items, caseId, "durationMs",
baseline.getDurationMs(), current.getDurationMs(), false);
compareEvidenceCoverage(items, baseline, current);
}
}
private void comparePassState(List<DiagnosisEvalDiffItem> items,
DiagnosisEvalResult baseline,
DiagnosisEvalResult current) {
if (baseline.isPassed() == current.isPassed()) {
return;
}
String type = baseline.isPassed() ? REGRESSION : IMPROVEMENT;
items.add(item(type, "case", baseline.getCaseId(), "passed",
String.valueOf(baseline.isPassed()), String.valueOf(current.isPassed()), null,
baseline.getCaseId() + " pass state changed"));
}
private void compareVerdict(List<DiagnosisEvalDiffItem> items,
DiagnosisEvalResult baseline,
DiagnosisEvalResult current) {
if (Objects.equals(baseline.getVerdict(), current.getVerdict())) {
return;
}
int baselineRank = verdictRank(baseline.getVerdict());
int currentRank = verdictRank(current.getVerdict());
String type = currentRank < baselineRank ? REGRESSION : currentRank > baselineRank ? IMPROVEMENT : CHANGED;
items.add(item(type, "case", baseline.getCaseId(), "verdict",
value(baseline.getVerdict()), value(current.getVerdict()), (double) currentRank - baselineRank,
baseline.getCaseId() + " verdict changed"));
}
private void compareEvidenceCoverage(List<DiagnosisEvalDiffItem> items,
DiagnosisEvalResult baseline,
DiagnosisEvalResult current) {
Set<String> tools = new LinkedHashSet<>();
tools.addAll(safeMap(baseline.getEvidenceCoverage()).keySet());
tools.addAll(safeMap(current.getEvidenceCoverage()).keySet());
for (String tool : tools) {
boolean baselineCovered = Boolean.TRUE.equals(safeMap(baseline.getEvidenceCoverage()).get(tool));
boolean currentCovered = Boolean.TRUE.equals(safeMap(current.getEvidenceCoverage()).get(tool));
if (baselineCovered == currentCovered) {
continue;
}
String type = baselineCovered ? REGRESSION : IMPROVEMENT;
items.add(item(type, "case", baseline.getCaseId(), "evidenceCoverage." + tool,
String.valueOf(baselineCovered), String.valueOf(currentCovered), null,
baseline.getCaseId() + " evidence coverage changed for " + tool));
}
}
private void compareDouble(List<DiagnosisEvalDiffItem> items,
String scope,
String caseId,
String metric,
double baseline,
double current,
boolean higherIsBetter) {
if (Double.compare(baseline, current) == 0) {
return;
}
double delta = current - baseline;
String type = classifyDelta(delta, higherIsBetter);
items.add(item(type, scope, caseId, metric,
String.valueOf(baseline), String.valueOf(current), delta,
metric + " changed"));
}
private void compareInteger(List<DiagnosisEvalDiffItem> items,
String caseId,
String metric,
Integer baseline,
Integer current,
boolean higherIsBetter) {
if (Objects.equals(baseline, current)) {
return;
}
if (baseline == null || current == null) {
items.add(item(CHANGED, "case", caseId, metric,
value(baseline), value(current), null, caseId + " " + metric + " changed"));
return;
}
int delta = current - baseline;
items.add(item(classifyDelta(delta, higherIsBetter), "case", caseId, metric,
String.valueOf(baseline), String.valueOf(current), (double) delta,
caseId + " " + metric + " changed"));
}
private String classifyDelta(double delta, boolean higherIsBetter) {
if (delta == 0.0) {
return CHANGED;
}
boolean improved = higherIsBetter ? delta > 0 : delta < 0;
return improved ? IMPROVEMENT : REGRESSION;
}
private DiagnosisEvalDiffItem item(String type,
String scope,
String caseId,
String metric,
String baselineValue,
String currentValue,
Double delta,
String message) {
return DiagnosisEvalDiffItem.builder()
.type(type)
.scope(scope)
.caseId(caseId)
.metric(metric)
.baselineValue(baselineValue)
.currentValue(currentValue)
.delta(delta)
.message(message)
.build();
}
private Map<String, DiagnosisEvalResult> byCaseId(List<DiagnosisEvalResult> results) {
return results.stream()
.sorted(Comparator.comparing(DiagnosisEvalResult::getCaseId))
.collect(Collectors.toMap(
DiagnosisEvalResult::getCaseId,
Function.identity(),
(left, right) -> right,
LinkedHashMap::new));
}
private List<DiagnosisEvalResult> safeResults(DiagnosisEvalReport report) {
return report.getResults() == null ? List.of() : report.getResults();
}
private <T> Map<String, T> safeMap(Map<String, T> value) {
return value == null ? Map.of() : value;
}
private int countType(List<DiagnosisEvalDiffItem> items, String type) {
return (int) items.stream().filter(item -> type.equals(item.getType())).count();
}
private int verdictRank(String verdict) {
if ("PASS".equals(verdict)) {
return 3;
}
if ("LOW_CONFID".equals(verdict)) {
return 2;
}
if ("REJECT".equals(verdict)) {
return 1;
}
return 0;
}
private String value(Object value) {
return value == null ? "-" : String.valueOf(value);
}
}
@@ -1,37 +0,0 @@
package com.superbiz.agent.eval;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DiagnosisEvalCase {
private String id;
private String title;
private String question;
private String traceFixture;
private List<String> expectedRootCauseKeywords;
private Integer minKeywordMatches;
private List<String> requiredEvidenceTools;
private List<String> allowedVerdicts;
private List<String> forbiddenAnswerKeywords;
private Boolean requireV2AuditClosure;
private Boolean requireClaimChecks;
private Boolean requireComposerOutput;
private List<String> expectedGatekeeperStatuses;
private String expectedGatekeeperRuleSetVersion;
private List<String> expectedComposerStatuses;
private List<String> forbiddenConfirmedClaimKeywords;
private Boolean requirePromptAudit;
private String expectedPromptAuditVersion;
private Map<String, String> expectedPromptVersions;
private Boolean requireGatekeeperRules;
}
@@ -1,22 +0,0 @@
package com.superbiz.agent.eval;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DiagnosisEvalDiffItem {
private String type;
private String scope;
private String caseId;
private String metric;
private String baselineValue;
private String currentValue;
private Double delta;
private String message;
}
@@ -1,27 +0,0 @@
package com.superbiz.agent.eval;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DiagnosisEvalDiffReport {
private int baselineTotalCases;
private int currentTotalCases;
private int baselinePassedCases;
private int currentPassedCases;
private double baselinePassRate;
private double currentPassRate;
private int regressionCount;
private int improvementCount;
private int changedCount;
private boolean hasRegression;
private List<DiagnosisEvalDiffItem> items;
}
@@ -1,78 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
public class DiagnosisEvalDiffReportWriter {
private final ObjectMapper objectMapper;
public DiagnosisEvalDiffReportWriter(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public void writeJson(DiagnosisEvalDiffReport report, Path outputFile) throws IOException {
Files.createDirectories(outputFile.getParent());
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
}
public void writeMarkdown(DiagnosisEvalDiffReport report, Path outputFile) throws IOException {
Files.createDirectories(outputFile.getParent());
Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
}
public String toMarkdown(DiagnosisEvalDiffReport report) {
StringBuilder builder = new StringBuilder();
builder.append("# Diagnosis Eval Baseline Diff\n\n");
builder.append("- Baseline pass rate: ").append(formatPercent(report.getBaselinePassRate())).append("\n");
builder.append("- Current pass rate: ").append(formatPercent(report.getCurrentPassRate())).append("\n");
builder.append("- Baseline passed cases: ").append(report.getBaselinePassedCases()).append("/")
.append(report.getBaselineTotalCases()).append("\n");
builder.append("- Current passed cases: ").append(report.getCurrentPassedCases()).append("/")
.append(report.getCurrentTotalCases()).append("\n");
builder.append("- Regressions: ").append(report.getRegressionCount()).append("\n");
builder.append("- Improvements: ").append(report.getImprovementCount()).append("\n");
builder.append("- Other changes: ").append(report.getChangedCount()).append("\n\n");
builder.append("## Diff Items\n\n");
if (report.getItems() == null || report.getItems().isEmpty()) {
builder.append("- No differences\n");
return builder.toString();
}
builder.append("| Type | Scope | Case | Metric | Baseline | Current | Delta | Message |\n");
builder.append("| --- | --- | --- | --- | --- | --- | ---: | --- |\n");
for (DiagnosisEvalDiffItem item : report.getItems()) {
builder.append("| ")
.append(valueOrDash(item.getType()))
.append(" | ")
.append(valueOrDash(item.getScope()))
.append(" | ")
.append(valueOrDash(item.getCaseId()))
.append(" | ")
.append(valueOrDash(item.getMetric()))
.append(" | ")
.append(valueOrDash(item.getBaselineValue()))
.append(" | ")
.append(valueOrDash(item.getCurrentValue()))
.append(" | ")
.append(item.getDelta() == null ? "-" : String.format("%.3f", item.getDelta()))
.append(" | ")
.append(valueOrDash(item.getMessage()))
.append(" |\n");
}
return builder.toString();
}
private String formatPercent(double value) {
return String.format("%.2f%%", value * 100);
}
private String valueOrDash(String value) {
return value == null || value.isBlank() ? "-" : value;
}
}
@@ -1,24 +0,0 @@
package com.superbiz.agent.eval;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DiagnosisEvalReport {
private int totalCases;
private int passedCases;
private double passRate;
private Map<String, Long> verdictDistribution;
private double averageToolCallCount;
private double averageDurationMs;
private List<DiagnosisEvalResult> results;
}
@@ -1,88 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
public class DiagnosisEvalReportWriter {
private final ObjectMapper objectMapper;
public DiagnosisEvalReportWriter(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public void writeJson(DiagnosisEvalReport report, Path outputFile) throws IOException {
Files.createDirectories(outputFile.getParent());
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
}
public void writeMarkdown(DiagnosisEvalReport report, Path outputFile) throws IOException {
Files.createDirectories(outputFile.getParent());
Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
}
public String toMarkdown(DiagnosisEvalReport report) {
StringBuilder builder = new StringBuilder();
builder.append("# Diagnosis Eval Report\n\n");
builder.append("- Total cases: ").append(report.getTotalCases()).append("\n");
builder.append("- Passed cases: ").append(report.getPassedCases()).append("\n");
builder.append("- Pass rate: ").append(String.format("%.2f%%", report.getPassRate() * 100)).append("\n");
builder.append("- Average tool calls: ").append(String.format("%.2f", report.getAverageToolCallCount())).append("\n");
builder.append("- Average duration ms: ").append(String.format("%.2f", report.getAverageDurationMs())).append("\n\n");
builder.append("## Verdict Distribution\n\n");
if (report.getVerdictDistribution() == null || report.getVerdictDistribution().isEmpty()) {
builder.append("- None\n\n");
} else {
for (Map.Entry<String, Long> entry : report.getVerdictDistribution().entrySet()) {
builder.append("- ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
}
builder.append("\n");
}
builder.append("## Cases\n\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |\n");
for (DiagnosisEvalResult result : report.getResults()) {
builder.append("| ")
.append(result.getCaseId())
.append(" | ")
.append(result.isPassed() ? "PASS" : "FAIL")
.append(" | ")
.append(valueOrDash(result.getVerdict()))
.append(" | ")
.append(valueOrDash(result.getGatekeeperStatus()))
.append(" | ")
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
.append(" | ")
.append(valueOrDash(result.getPromptAuditVersion()))
.append(" | ")
.append(valueOrDash(result.getComposerStatus()))
.append(" | ")
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
.append(" | ")
.append(result.getGatekeeperRuleCount() == null ? "-" : result.getGatekeeperRuleCount())
.append(" | ")
.append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
.append(" | ")
.append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
.append(" | ")
.append(result.getDurationMs() == null ? "-" : result.getDurationMs())
.append(" | ")
.append(result.getFailedChecks() == null || result.getFailedChecks().isEmpty()
? "-"
: String.join("; ", result.getFailedChecks()))
.append(" |\n");
}
return builder.toString();
}
private String valueOrDash(String value) {
return value == null || value.isBlank() ? "-" : value;
}
}
@@ -1,33 +0,0 @@
package com.superbiz.agent.eval;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DiagnosisEvalResult {
private String caseId;
private String title;
private boolean passed;
private List<String> failedChecks;
private String verdict;
private int matchedKeywordCount;
private int requiredKeywordCount;
private Map<String, Boolean> evidenceCoverage;
private String gatekeeperStatus;
private String gatekeeperRuleSetVersion;
private String promptAuditVersion;
private String composerStatus;
private Integer claimCheckCount;
private Integer gatekeeperRuleCount;
private Integer toolCallCount;
private Integer durationMs;
}
@@ -1,488 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.DiagnosisTraceResponse;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
public class DiagnosisTraceEvaluator {
private static final TypeReference<List<DiagnosisEvalCase>> CASE_LIST_TYPE = new TypeReference<>() {};
private static final String REJECT_DEGRADED_PREFIX = "当前无法基于已获取证据生成可靠结论";
private static final Set<String> RAW_EXECUTOR_MARKERS = Set.of(
"executor_evidence_v2",
"answer_version",
"evidence_bindings",
"claim_id"
);
private static final Set<String> VALID_CLAIM_VERIFICATIONS = Set.of(
"direct_observation",
"reasonable_inference",
"overstated",
"unsupported",
"external_unknown",
"contradicted"
);
private final ObjectMapper objectMapper;
public DiagnosisTraceEvaluator(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public List<DiagnosisEvalCase> loadCases(Path casesFile) throws IOException {
return objectMapper.readValue(casesFile.toFile(), CASE_LIST_TYPE);
}
public DiagnosisTraceResponse loadTrace(Path traceFile) throws IOException {
return objectMapper.readValue(traceFile.toFile(), DiagnosisTraceResponse.class);
}
public DiagnosisEvalReport evaluate(List<DiagnosisEvalCase> cases, Path fixtureDir) {
List<DiagnosisEvalResult> results = new ArrayList<>();
for (DiagnosisEvalCase evalCase : cases) {
try {
DiagnosisTraceResponse trace = loadTrace(fixtureDir.resolve(evalCase.getTraceFixture()));
results.add(evaluate(evalCase, trace));
} catch (Exception e) {
results.add(DiagnosisEvalResult.builder()
.caseId(evalCase.getId())
.title(evalCase.getTitle())
.passed(false)
.failedChecks(List.of("trace fixture unavailable: " + e.getMessage()))
.verdict(null)
.matchedKeywordCount(0)
.requiredKeywordCount(size(evalCase.getExpectedRootCauseKeywords()))
.evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
.gatekeeperStatus(null)
.gatekeeperRuleSetVersion(null)
.promptAuditVersion(null)
.composerStatus(null)
.claimCheckCount(null)
.gatekeeperRuleCount(null)
.toolCallCount(null)
.durationMs(null)
.build());
}
}
return toReport(results);
}
public DiagnosisEvalResult evaluate(DiagnosisEvalCase evalCase, DiagnosisTraceResponse trace) {
List<String> failedChecks = new ArrayList<>();
String answer = trace.getSession() == null ? "" : nullToEmpty(trace.getSession().getAnswer());
String normalizedAnswer = answer.toLowerCase(Locale.ROOT);
int requiredKeywordCount = size(evalCase.getExpectedRootCauseKeywords());
int matchedKeywordCount = countMatches(normalizedAnswer, evalCase.getExpectedRootCauseKeywords());
int minKeywordMatches = evalCase.getMinKeywordMatches() == null
? requiredKeywordCount
: evalCase.getMinKeywordMatches();
if (matchedKeywordCount < minKeywordMatches) {
failedChecks.add("answer keyword coverage too low: " + matchedKeywordCount + "/" + minKeywordMatches);
}
for (String forbidden : safeList(evalCase.getForbiddenAnswerKeywords())) {
if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) {
failedChecks.add("answer contains forbidden keyword: " + forbidden);
}
}
Set<String> evidenceTools = collectEvidenceTools(trace);
Map<String, Boolean> evidenceCoverage = new LinkedHashMap<>();
for (String requiredTool : safeList(evalCase.getRequiredEvidenceTools())) {
boolean present = evidenceTools.contains(requiredTool);
evidenceCoverage.put(requiredTool, present);
if (!present) {
failedChecks.add("missing required evidence tool: " + requiredTool);
}
}
String verdict = extractVerifierVerdict(trace);
if (verdict == null || verdict.isBlank()) {
failedChecks.add("missing verifier verdict");
} else if (!safeList(evalCase.getAllowedVerdicts()).isEmpty()
&& !safeList(evalCase.getAllowedVerdicts()).contains(verdict)) {
failedChecks.add("verdict not allowed: " + verdict);
}
if ("REJECT".equals(verdict) && !answer.startsWith(REJECT_DEGRADED_PREFIX)) {
failedChecks.add("reject output does not use degraded template");
}
failedChecks.addAll(validateExecutorStructuredOutput(trace));
String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
String gatekeeperRuleSetVersion = extractNestedString(trace, "verifier_evaluation",
"gatekeeper_result", "rule_set_version");
String promptAuditVersion = extractNestedString(trace, "verifier_evaluation",
"prompt_audit", "version");
String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
Integer gatekeeperRuleCount = countNestedList(trace, "verifier_evaluation", "gatekeeper_result", "rules");
failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
gatekeeperStatus, gatekeeperRuleSetVersion, promptAuditVersion, composerStatus));
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
return DiagnosisEvalResult.builder()
.caseId(evalCase.getId())
.title(evalCase.getTitle())
.passed(failedChecks.isEmpty())
.failedChecks(failedChecks)
.verdict(verdict)
.matchedKeywordCount(matchedKeywordCount)
.requiredKeywordCount(requiredKeywordCount)
.evidenceCoverage(evidenceCoverage)
.gatekeeperStatus(gatekeeperStatus)
.gatekeeperRuleSetVersion(gatekeeperRuleSetVersion)
.promptAuditVersion(promptAuditVersion)
.composerStatus(composerStatus)
.claimCheckCount(claimCheckCount)
.gatekeeperRuleCount(gatekeeperRuleCount)
.toolCallCount(toolCallCount)
.durationMs(durationMs)
.build();
}
private DiagnosisEvalReport toReport(List<DiagnosisEvalResult> results) {
int total = results.size();
int passed = (int) results.stream().filter(DiagnosisEvalResult::isPassed).count();
Map<String, Long> verdictDistribution = results.stream()
.map(DiagnosisEvalResult::getVerdict)
.filter(Objects::nonNull)
.collect(Collectors.groupingBy(value -> value, LinkedHashMap::new, Collectors.counting()));
double averageToolCallCount = results.stream()
.map(DiagnosisEvalResult::getToolCallCount)
.filter(Objects::nonNull)
.mapToInt(Integer::intValue)
.average()
.orElse(0.0);
double averageDurationMs = results.stream()
.map(DiagnosisEvalResult::getDurationMs)
.filter(Objects::nonNull)
.mapToInt(Integer::intValue)
.average()
.orElse(0.0);
return DiagnosisEvalReport.builder()
.totalCases(total)
.passedCases(passed)
.passRate(total == 0 ? 0.0 : (double) passed / total)
.verdictDistribution(verdictDistribution)
.averageToolCallCount(averageToolCallCount)
.averageDurationMs(averageDurationMs)
.results(results)
.build();
}
private Set<String> collectEvidenceTools(DiagnosisTraceResponse trace) {
Set<String> tools = new LinkedHashSet<>();
if (trace.getToolInvocations() != null) {
for (DiagnosisTraceResponse.ToolInvocationTrace invocation : trace.getToolInvocations()) {
if (invocation.getToolName() != null) {
tools.add(invocation.getToolName());
}
}
}
Object summaries = nestedValue(trace, "verifier_evaluation", "tool_trace_summary");
if (summaries instanceof List<?> list) {
for (Object item : list) {
if (item instanceof Map<?, ?> map && map.get("tool_name") != null) {
tools.add(String.valueOf(map.get("tool_name")));
}
}
}
return tools;
}
private String extractVerifierVerdict(DiagnosisTraceResponse trace) {
Object value = nestedValue(trace, "verifier_evaluation", "verdict");
return value == null ? null : String.valueOf(value);
}
private List<String> validateExecutorStructuredOutput(DiagnosisTraceResponse trace) {
Object structuredOutput = nestedValue(trace, "verifier_evaluation", "executor_structured_output");
if (!(structuredOutput instanceof Map<?, ?> output)) {
return List.of();
}
Object claims = output.get("claims");
if (!(claims instanceof List<?> claimList)) {
return List.of("executor structured output missing claims array");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : claimList) {
if (!(item instanceof Map<?, ?> claim)) {
failedChecks.add("executor structured claim is not an object");
continue;
}
Object claimIdValue = claim.get("claim_id");
String claimId = claimIdValue == null ? "unknown" : String.valueOf(claimIdValue);
Object bindings = claim.get("evidence_bindings");
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
failedChecks.add("executor confirmed claim missing evidence bindings: " + claimId);
}
}
return failedChecks;
}
private List<String> validateV2AuditClosure(DiagnosisEvalCase evalCase,
DiagnosisTraceResponse trace,
String normalizedAnswer,
String verdict,
String gatekeeperStatus,
String gatekeeperRuleSetVersion,
String promptAuditVersion,
String composerStatus) {
List<String> failedChecks = new ArrayList<>();
boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
boolean requireClaimChecks = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireClaimChecks());
boolean requireComposerOutput = requireV2AuditClosure || Boolean.TRUE.equals(evalCase.getRequireComposerOutput());
Object gatekeeperResult = nestedValue(trace, "verifier_evaluation", "gatekeeper_result");
if (requireV2AuditClosure && !(gatekeeperResult instanceof Map<?, ?>)) {
failedChecks.add("missing gatekeeper_result");
}
if ("fail".equals(gatekeeperStatus) && "PASS".equals(verdict)) {
failedChecks.add("gatekeeper fail cannot have PASS verdict");
}
if (!safeList(evalCase.getExpectedGatekeeperStatuses()).isEmpty()
&& !safeList(evalCase.getExpectedGatekeeperStatuses()).contains(gatekeeperStatus)) {
failedChecks.add("gatekeeper status not expected: " + valueOrMissing(gatekeeperStatus));
}
if (!isBlank(evalCase.getExpectedGatekeeperRuleSetVersion())
&& !evalCase.getExpectedGatekeeperRuleSetVersion().equals(gatekeeperRuleSetVersion)) {
failedChecks.add("gatekeeper rule set version not expected: "
+ valueOrMissing(gatekeeperRuleSetVersion));
}
if (Boolean.TRUE.equals(evalCase.getRequireGatekeeperRules())) {
failedChecks.addAll(validateGatekeeperRules(trace));
}
failedChecks.addAll(validatePromptAudit(evalCase, trace, promptAuditVersion));
failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
Object composerOutput = nestedValue(trace, "verifier_evaluation", "composer_output");
if (requireComposerOutput && !(composerOutput instanceof Map<?, ?>)) {
failedChecks.add("missing composer_output");
}
if (requireComposerOutput && isBlank(composerStatus)) {
failedChecks.add("composer_output missing status");
}
if (!safeList(evalCase.getExpectedComposerStatuses()).isEmpty()
&& !safeList(evalCase.getExpectedComposerStatuses()).contains(composerStatus)) {
failedChecks.add("composer status not expected: " + valueOrMissing(composerStatus));
}
for (String forbidden : safeList(evalCase.getForbiddenConfirmedClaimKeywords())) {
if (normalizedAnswer.contains(forbidden.toLowerCase(Locale.ROOT))) {
failedChecks.add("answer contains forbidden confirmed claim keyword: " + forbidden);
}
}
if (requireV2AuditClosure) {
for (String marker : RAW_EXECUTOR_MARKERS) {
if (normalizedAnswer.contains(marker.toLowerCase(Locale.ROOT))) {
failedChecks.add("answer leaks raw executor marker: " + marker);
}
}
}
return failedChecks;
}
private List<String> validatePromptAudit(DiagnosisEvalCase evalCase,
DiagnosisTraceResponse trace,
String promptAuditVersion) {
List<String> failedChecks = new ArrayList<>();
Object promptAudit = nestedValue(trace, "verifier_evaluation", "prompt_audit");
if (Boolean.TRUE.equals(evalCase.getRequirePromptAudit()) && !(promptAudit instanceof Map<?, ?>)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
if (!isBlank(evalCase.getExpectedPromptAuditVersion())
&& !evalCase.getExpectedPromptAuditVersion().equals(promptAuditVersion)) {
failedChecks.add("prompt audit version not expected: " + valueOrMissing(promptAuditVersion));
}
if (evalCase.getExpectedPromptVersions() == null || evalCase.getExpectedPromptVersions().isEmpty()) {
return failedChecks;
}
if (!(promptAudit instanceof Map<?, ?> audit)) {
failedChecks.add("missing prompt_audit");
return failedChecks;
}
Object promptsValue = audit.get("prompts");
if (!(promptsValue instanceof List<?> prompts)) {
failedChecks.add("prompt_audit missing prompts");
return failedChecks;
}
Map<String, String> actualVersions = new LinkedHashMap<>();
for (Object promptValue : prompts) {
if (promptValue instanceof Map<?, ?> prompt) {
String name = stringValue(prompt.get("name"));
String version = stringValue(prompt.get("version"));
if (!isBlank(name)) {
actualVersions.put(name, version);
}
}
}
for (Map.Entry<String, String> expected : evalCase.getExpectedPromptVersions().entrySet()) {
String actual = actualVersions.get(expected.getKey());
if (!expected.getValue().equals(actual)) {
failedChecks.add("prompt version not expected: "
+ expected.getKey() + "=" + valueOrMissing(actual));
}
}
return failedChecks;
}
private List<String> validateGatekeeperRules(DiagnosisTraceResponse trace) {
Object rulesValue = nestedNestedValue(trace, "verifier_evaluation", "gatekeeper_result", "rules");
if (!(rulesValue instanceof List<?> rules) || rules.isEmpty()) {
return List.of("gatekeeper_result missing rules");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : rules) {
if (!(item instanceof Map<?, ?> rule)) {
failedChecks.add("gatekeeper rule metadata is not an object");
continue;
}
String id = stringValue(rule.get("id"));
Object enabled = rule.get("enabled");
String severity = stringValue(rule.get("default_severity"));
if (isBlank(id)) {
failedChecks.add("gatekeeper rule metadata missing id");
}
if (!(enabled instanceof Boolean)) {
failedChecks.add("gatekeeper rule metadata missing enabled: " + valueOrMissing(id));
}
if (isBlank(severity)) {
failedChecks.add("gatekeeper rule metadata missing default_severity: " + valueOrMissing(id));
}
}
return failedChecks;
}
private List<String> validateClaimChecks(DiagnosisTraceResponse trace, boolean required) {
Object claimChecks = nestedValue(trace, "verifier_evaluation", "claim_checks");
if (!(claimChecks instanceof List<?> claimCheckList)) {
return required ? List.of("missing claim_checks") : List.of();
}
if (required && claimCheckList.isEmpty()) {
return List.of("claim_checks is empty");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : claimCheckList) {
if (!(item instanceof Map<?, ?> claimCheck)) {
failedChecks.add("claim_check is not an object");
continue;
}
String claimId = stringValue(claimCheck.get("claim_id"));
String verification = stringValue(claimCheck.get("verification"));
if (isBlank(claimId)) {
failedChecks.add("claim_check missing claim_id");
}
if (isBlank(verification)) {
failedChecks.add("claim_check missing verification: " + valueOrMissing(claimId));
} else if (!VALID_CLAIM_VERIFICATIONS.contains(verification)) {
failedChecks.add("claim_check verification invalid: " + verification);
}
if (isBlank(stringValue(claimCheck.get("detail")))) {
failedChecks.add("claim_check missing detail: " + valueOrMissing(claimId));
}
}
return failedChecks;
}
private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) {
return null;
}
Object first = trace.getSession().getSelfEvaluation().get(firstKey);
if (!(first instanceof Map<?, ?> map)) {
return null;
}
return map.get(secondKey);
}
private String extractNestedString(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedValue(trace, firstKey, secondKey);
if (!(value instanceof Map<?, ?> map)) {
return null;
}
Object nested = map.get(thirdKey);
return nested == null ? null : String.valueOf(nested);
}
private Integer countList(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
Object value = nestedValue(trace, firstKey, secondKey);
return value instanceof List<?> list ? list.size() : null;
}
private Integer countNestedList(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedNestedValue(trace, firstKey, secondKey, thirdKey);
return value instanceof List<?> list ? list.size() : null;
}
private Object nestedNestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey, String thirdKey) {
Object value = nestedValue(trace, firstKey, secondKey);
if (!(value instanceof Map<?, ?> map)) {
return null;
}
return map.get(thirdKey);
}
private int countMatches(String normalizedAnswer, List<String> keywords) {
int count = 0;
for (String keyword : safeList(keywords)) {
if (normalizedAnswer.contains(keyword.toLowerCase(Locale.ROOT))) {
count++;
}
}
return count;
}
private Map<String, Boolean> emptyCoverage(List<String> tools) {
Map<String, Boolean> coverage = new LinkedHashMap<>();
for (String tool : safeList(tools)) {
coverage.put(tool, false);
}
return coverage;
}
private List<String> safeList(List<String> values) {
return values == null ? List.of() : values;
}
private int size(List<?> values) {
return values == null ? 0 : values.size();
}
private String nullToEmpty(String value) {
return value == null ? "" : value;
}
private String stringValue(Object value) {
return value == null ? null : String.valueOf(value);
}
private boolean isBlank(String value) {
return value == null || value.isBlank();
}
private String valueOrMissing(String value) {
return isBlank(value) ? "missing" : value;
}
}
@@ -0,0 +1,170 @@
package com.superbiz.agent.harness.audit;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.repository.AgentStepRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
@HookPositions({HookPosition.BEFORE_MODEL, HookPosition.AFTER_MODEL})
public final class HarnessAgentAuditHook extends MessagesModelHook {
private static final Logger log = LoggerFactory.getLogger(HarnessAgentAuditHook.class);
private final AgentStepRepository repository;
private final ObjectMapper objectMapper;
private final String agentName;
private final ConcurrentHashMap<String, Integer> stepCounters = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, PendingStep> pendingSteps = new ConcurrentHashMap<>();
public HarnessAgentAuditHook(AgentStepRepository repository, ObjectMapper objectMapper, String agentName) {
this.repository = Objects.requireNonNull(repository, "repository must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
if (agentName == null || agentName.isBlank()) {
throw new IllegalArgumentException("agentName must not be blank");
}
this.agentName = agentName;
}
@Override
public String getName() {
return "harness_agent_audit";
}
@Override
public AgentCommand beforeModel(List<Message> messages, RunnableConfig config) {
AuditIdentity identity = identity(config);
if (identity == null) {
log.warn("Skipping AgentStep audit because RunnableConfig identity is missing");
return new AgentCommand(messages);
}
int stepIndex = stepCounters.merge(identity.runId(), 0, (current, ignored) -> current + 1);
try {
AgentStep saved = repository.save(AgentStep.builder()
.sessionId(identity.sessionId())
.runId(identity.runId())
.stepIndex(stepIndex)
.agentName(agentName)
.modelInput(write(inputMetadata(messages)))
.hasToolCall(false)
.build());
pendingSteps.put(stepKey(identity.runId(), stepIndex),
new PendingStep(saved.getId(), System.nanoTime()));
} catch (RuntimeException exception) {
log.warn("Failed to persist AgentStep audit before model: agent={}", agentName);
}
return new AgentCommand(messages);
}
@Override
public AgentCommand afterModel(List<Message> messages, RunnableConfig config) {
AuditIdentity identity = identity(config);
if (identity == null) {
log.warn("Skipping AgentStep audit completion because RunnableConfig identity is missing");
return new AgentCommand(messages);
}
int stepIndex = stepCounters.getOrDefault(identity.runId(), 0);
PendingStep pending = pendingSteps.remove(stepKey(identity.runId(), stepIndex));
if (pending == null || pending.id() == null) {
return new AgentCommand(messages);
}
try {
AgentStep step = repository.findById(pending.id()).orElse(null);
if (step != null) {
AssistantMessage assistant = lastAssistant(messages);
List<String> toolNames = assistant == null || assistant.getToolCalls() == null
? List.of()
: assistant.getToolCalls().stream().map(AssistantMessage.ToolCall::name)
.distinct().sorted().toList();
step.setModelOutput(write(outputMetadata(assistant, toolNames)));
step.setThought(null);
step.setHasToolCall(!toolNames.isEmpty());
step.setDurationMs(durationMillis(pending.startedNanos()));
repository.save(step);
}
} catch (RuntimeException exception) {
log.warn("Failed to complete AgentStep audit: agent={}", agentName);
}
return new AgentCommand(messages);
}
private Map<String, Object> inputMetadata(List<Message> messages) {
List<Message> safeMessages = messages == null ? List.of() : messages;
Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("message_count", safeMessages.size());
metadata.put("roles", safeMessages.stream()
.map(message -> message.getMessageType().getValue())
.toList());
return metadata;
}
private Map<String, Object> outputMetadata(AssistantMessage assistant, List<String> toolNames) {
Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("has_text", assistant != null
&& assistant.getText() != null && !assistant.getText().isBlank());
metadata.put("tool_names", toolNames);
return metadata;
}
private AuditIdentity identity(RunnableConfig config) {
if (config == null) {
return null;
}
String sessionId = config.metadata("sessionId").map(Object::toString).orElse(null);
String runId = config.metadata("runId").map(Object::toString).orElse(null);
return hasText(sessionId) && hasText(runId) ? new AuditIdentity(sessionId, runId) : null;
}
private AssistantMessage lastAssistant(List<Message> messages) {
if (messages == null) {
return null;
}
for (int index = messages.size() - 1; index >= 0; index--) {
if (messages.get(index) instanceof AssistantMessage assistant) {
return assistant;
}
}
return null;
}
private String write(Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(value);
} catch (JsonProcessingException exception) {
throw new IllegalStateException("Audit metadata is not serializable", exception);
}
}
private static int durationMillis(long startedNanos) {
long value = Math.max(0L, (System.nanoTime() - startedNanos) / 1_000_000L);
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) value;
}
private static String stepKey(String runId, int stepIndex) {
return runId + ':' + stepIndex;
}
private static boolean hasText(String value) {
return value != null && !value.isBlank();
}
private record AuditIdentity(String sessionId, String runId) {
}
private record PendingStep(Long id, long startedNanos) {
}
}
@@ -0,0 +1,72 @@
package com.superbiz.agent.harness.audit;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.harness.contract.InvocationStatus;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
@Component
public final class JpaToolInvocationAuditSink implements ToolInvocationAuditSink {
private final ToolInvocationRepository repository;
private final ObjectMapper objectMapper;
public JpaToolInvocationAuditSink(ToolInvocationRepository repository, ObjectMapper objectMapper) {
this.repository = Objects.requireNonNull(repository, "repository must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
@Override
public void record(ToolInvocationAuditEvent event) {
Objects.requireNonNull(event, "event must not be null");
repository.save(ToolInvocation.builder()
.sessionId(event.sessionId())
.runId(event.runId())
.toolName(event.toolName())
.inputParams(write(inputMetadata(event)))
.outputPreview("status=%s,evidence_status=%s".formatted(
event.status(), event.evidenceStatus()))
.outputLength(event.agentResultBytes())
.retrievalLayer("HARNESS")
.isTruncated(false)
.relevanceLevel(event.evidenceStatus().name())
.retrievalDetails(write(resultMetadata(event)))
.durationMs(event.durationMs())
.success(event.status() == InvocationStatus.READY)
.errorMessage(event.errorCode())
.build());
}
private Map<String, Object> inputMetadata(ToolInvocationAuditEvent event) {
Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("tool_call_id", event.toolCallId());
metadata.put("request_bytes", event.requestBytes());
return metadata;
}
private Map<String, Object> resultMetadata(ToolInvocationAuditEvent event) {
Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("tool_call_id", event.toolCallId());
metadata.put("status", event.status().name());
metadata.put("evidence_status", event.evidenceStatus().name());
metadata.put("agent_result_bytes", event.agentResultBytes());
if (event.errorCode() != null) {
metadata.put("error_code", event.errorCode());
}
return metadata;
}
private String write(Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(value);
} catch (JsonProcessingException exception) {
throw new IllegalStateException("Tool audit metadata is not serializable", exception);
}
}
}
@@ -0,0 +1,43 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.InvocationStatus;
import java.util.Objects;
public record ToolInvocationAuditEvent(
String sessionId,
String runId,
String toolCallId,
String toolName,
InvocationStatus status,
EvidenceStatus evidenceStatus,
String errorCode,
int durationMs,
int requestBytes,
int agentResultBytes) {
public ToolInvocationAuditEvent {
requireText(sessionId, "sessionId");
requireText(runId, "runId");
requireText(toolCallId, "toolCallId");
requireText(toolName, "toolName");
Objects.requireNonNull(status, "status must not be null");
Objects.requireNonNull(evidenceStatus, "evidenceStatus must not be null");
if (durationMs < 0 || requestBytes < 0 || agentResultBytes < 0) {
throw new IllegalArgumentException("Audit counters must not be negative");
}
if (status == InvocationStatus.READY && errorCode != null) {
throw new IllegalArgumentException("READY audit must not contain errorCode");
}
if (status == InvocationStatus.ERROR && (errorCode == null || errorCode.isBlank())) {
throw new IllegalArgumentException("ERROR audit requires errorCode");
}
}
private static void requireText(String value, String name) {
if (value == null || value.isBlank()) {
throw new IllegalArgumentException(name + " must not be blank");
}
}
}
@@ -0,0 +1,12 @@
package com.superbiz.agent.harness.audit;
@FunctionalInterface
public interface ToolInvocationAuditSink {
void record(ToolInvocationAuditEvent event);
static ToolInvocationAuditSink noop() {
return ignored -> {
};
}
}
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.tool.boundary;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.BudgetExceededException;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
@@ -18,6 +20,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
public final class ToolBoundary {
@@ -29,23 +34,44 @@ public final class ToolBoundary {
private final CanonicalInvocationStore store;
private final ObjectMapper objectMapper;
private final Clock clock;
private final ToolInvocationAuditSink auditSink;
public ToolBoundary(DiagnosisHarnessCore core,
ToolCallKeyFactory keyFactory,
CanonicalInvocationStore store,
ObjectMapper objectMapper,
Clock clock) {
this(core, keyFactory, store, objectMapper, clock, ToolInvocationAuditSink.noop());
}
public ToolBoundary(DiagnosisHarnessCore core,
ToolCallKeyFactory keyFactory,
CanonicalInvocationStore store,
ObjectMapper objectMapper,
Clock clock,
ToolInvocationAuditSink auditSink) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
this.store = Objects.requireNonNull(store, "store must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
this.clock = Objects.requireNonNull(clock, "clock must not be null");
this.auditSink = Objects.requireNonNull(auditSink, "auditSink must not be null");
}
public ToolBoundaryResult execute(RunContext context,
ToolCallRequestEnvelope request,
ToolExecutor executor,
ToolResultProjector projector) {
Instant startedAt = clock.instant();
ToolBoundaryResult result = executeCanonical(context, request, executor, projector);
auditSafely(context, request, result, startedAt);
return result;
}
private ToolBoundaryResult executeCanonical(RunContext context,
ToolCallRequestEnvelope request,
ToolExecutor executor,
ToolResultProjector projector) {
String toolCallId = request == null ? null : request.toolCallId();
String key;
try {
@@ -193,6 +219,36 @@ public final class ToolBoundary {
return ToolBoundaryErrorCode.INVALID_REQUEST;
}
private void auditSafely(RunContext context,
ToolCallRequestEnvelope request,
ToolBoundaryResult result,
Instant startedAt) {
if (context == null || request == null || result == null
|| !context.runId().equals(request.runId())
|| request.toolCallId() == null || request.toolCallId().isBlank()
|| request.toolName() == null || request.toolName().isBlank()) {
return;
}
try {
auditSink.record(new ToolInvocationAuditEvent(
context.sessionId(), context.runId(), request.toolCallId(), request.toolName(),
result.status(), result.evidenceStatus(), result.errorCode(),
saturatingInt(Math.max(0L, Duration.between(startedAt, clock.instant()).toMillis())),
utf8Bytes(request.requestJson()), utf8Bytes(result.agentResult())));
} catch (RuntimeException exception) {
log.warn("Failed to persist Tool durable audit: tool={}, status={}",
request.toolName(), result.status());
}
}
private static int utf8Bytes(String value) {
return value == null ? 0 : saturatingInt(value.getBytes(StandardCharsets.UTF_8).length);
}
private static int saturatingInt(long value) {
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) value;
}
private static final class InvalidToolCallIdException extends IllegalArgumentException {
private InvalidToolCallIdException(Throwable cause) {
super("Invalid tool call ID", cause);
@@ -1,355 +0,0 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.util.SessionContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.ToolResponseMessage;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Persists per-agent model input/output snapshots into agent_step.
*/
@Slf4j
@HookPositions({HookPosition.BEFORE_MODEL, HookPosition.AFTER_MODEL})
public class AgentLoggingHook extends MessagesModelHook {
private final AgentStepRepository agentStepRepository;
private final String agentName;
private final ObjectMapper objectMapper = new ObjectMapper();
private final ConcurrentHashMap<String, Integer> stepCounters = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, Map<String, Object>> pendingSteps = new ConcurrentHashMap<>();
public AgentLoggingHook(AgentStepRepository agentStepRepository, String agentName) {
this.agentStepRepository = agentStepRepository;
this.agentName = agentName;
}
@Override
public String getName() {
return "agent_logging_hook";
}
@Override
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
String sessionId = resolveSessionId(config);
String runId = resolveRunId(config);
String traceScopeId = traceScopeId(sessionId, runId);
boolean hasSession = sessionId != null;
int stepIndex = 0;
if (traceScopeId != null) {
stepIndex = stepCounters.merge(traceScopeId, 0, (oldValue, ignored) -> oldValue + 1);
}
log.info("========================================");
log.info("*** [AgentTrace] agent={}, phase=before_model, stepIndex={}", agentName, stepIndex);
log.info("*** [AgentTrace] messageCount={}", previousMessages.size());
int lastN = Math.min(3, previousMessages.size());
if (lastN > 0) {
log.info("*** [AgentTrace] recentMessages={}", lastN);
List<Message> recentMessages = previousMessages.subList(previousMessages.size() - lastN, previousMessages.size());
for (int i = 0; i < recentMessages.size(); i++) {
Message msg = recentMessages.get(i);
log.info(" [{}] role={}, type={}", i + 1, getMessageRole(msg), msg.getClass().getSimpleName());
}
}
log.info("========================================");
if (sessionId != null) {
try {
AgentStep step = AgentStep.builder()
.sessionId(sessionId)
.runId(runId)
.stepIndex(stepIndex)
.agentName(agentName)
.modelInput(buildModelInputSummary(previousMessages))
.build();
AgentStep saved = agentStepRepository.save(step);
pendingSteps.put(stepKey(traceScopeId, stepIndex), Map.of(
"stepId", saved.getId(),
"startTime", System.currentTimeMillis()
));
} catch (Exception e) {
log.error("Failed to persist agent_step before model", e);
}
}
return new AgentCommand(previousMessages);
}
@Override
public AgentCommand afterModel(List<Message> previousMessages, RunnableConfig config) {
String sessionId = resolveSessionId(config);
String runId = resolveRunId(config);
String traceScopeId = traceScopeId(sessionId, runId);
int stepIndex = traceScopeId == null ? 0 : stepCounters.getOrDefault(traceScopeId, 0);
log.info("========================================");
log.info("*** [AgentTrace] agent={}, phase=after_model, stepIndex={}", agentName, stepIndex);
AssistantMessage lastAssistant = findLastAssistant(previousMessages);
boolean hasToolCall = false;
if (lastAssistant != null) {
String textContent = extractTextContent(lastAssistant);
if (textContent != null && !textContent.isEmpty()) {
log.info("*** [AgentTrace] text={}",
textContent.length() > 500
? textContent.substring(0, 500) + "... (len=" + textContent.length() + ")"
: textContent);
}
if (lastAssistant.getToolCalls() != null && !lastAssistant.getToolCalls().isEmpty()) {
hasToolCall = true;
log.info("*** [AgentTrace] toolCalls={}", lastAssistant.getToolCalls().size());
lastAssistant.getToolCalls().forEach(toolCall ->
log.info(" - tool={}, arguments={}", toolCall.name(), toolCall.arguments()));
} else {
log.info("*** [AgentTrace] no tool call");
}
}
log.info("========================================");
if (sessionId != null) {
String stepKey = stepKey(traceScopeId, stepIndex);
Map<String, Object> pending = pendingSteps.remove(stepKey);
if (pending != null) {
try {
Long stepId = (Long) pending.get("stepId");
long startTime = (long) pending.get("startTime");
int durationMs = (int) (System.currentTimeMillis() - startTime);
AgentStep step = agentStepRepository.findById(stepId).orElse(null);
if (step != null) {
step.setThought(buildStoredThought(lastAssistant));
step.setHasToolCall(hasToolCall);
step.setDurationMs(durationMs);
if (lastAssistant != null) {
step.setModelOutput(buildModelOutputSummary(lastAssistant));
Integer tokenCount = TokenUsageHolder.get();
if (tokenCount != null) {
step.setTokenCount(tokenCount);
}
}
agentStepRepository.save(step);
}
} catch (Exception e) {
log.error("Failed to update agent_step after model", e);
}
}
}
TokenUsageHolder.clear();
return new AgentCommand(previousMessages);
}
private String resolveSessionId(RunnableConfig config) {
return config.metadata("sessionId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getSessionId);
}
private String resolveRunId(RunnableConfig config) {
return config.metadata("runId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getRunId);
}
private String traceScopeId(String sessionId, String runId) {
if (runId != null && !runId.isBlank()) {
return runId;
}
return sessionId;
}
private String stepKey(String traceScopeId, int stepIndex) {
return traceScopeId + "_" + stepIndex;
}
private AssistantMessage findLastAssistant(List<Message> previousMessages) {
for (int i = previousMessages.size() - 1; i >= 0; i--) {
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
return assistantMessage;
}
}
return null;
}
private String buildModelInputSummary(List<Message> messages) {
StringBuilder sb = new StringBuilder();
int maxMessages = Math.min(messages.size(), 5);
for (int i = messages.size() - maxMessages; i < messages.size(); i++) {
Message msg = messages.get(i);
String role = getMessageRole(msg);
String content = msg.toString();
if (content.length() > 200) {
content = content.substring(0, 200) + "...";
}
sb.append("[").append(role).append("] ").append(content).append("\n");
}
String result = sb.toString();
if (result.length() > 500) {
result = result.substring(0, 500) + "...";
}
return result;
}
private String buildStoredThought(AssistantMessage message) {
String text = extractTextContent(message);
if (text == null || text.isBlank()) {
return text;
}
if (!"verifier".equals(agentName)) {
return truncate(text, 2000);
}
return summarizeVerifierThought(text);
}
private String summarizeVerifierThought(String verifierOutput) {
try {
JsonNode root = objectMapper.readTree(verifierOutput);
int claimCount = root.path("claim_checks").isArray() ? root.path("claim_checks").size() : 0;
int factCount = root.path("facts_checked").isArray() ? root.path("facts_checked").size() : 0;
int tracedFactCount = 0;
JsonNode tracedNodes = root.path("claim_checks").isArray() ? root.path("claim_checks") : root.path("facts_checked");
if (tracedNodes.isArray()) {
for (JsonNode node : tracedNodes) {
if (node.path("evidence_refs").isArray() && node.path("evidence_refs").size() > 0) {
tracedFactCount++;
}
}
}
return "verdict=%s, score=%s, critical_fact_count=%s, claim_checks=%d, facts_checked=%d, traced_facts=%d".formatted(
root.path("verdict").asText("UNKNOWN"),
root.path("groundedness_score").asText("0.0"),
root.path("critical_fact_count").asText("0"),
claimCount,
factCount,
tracedFactCount
);
} catch (Exception e) {
return truncate(verifierOutput, 300);
}
}
private String buildModelOutputSummary(AssistantMessage message) {
String text = extractTextContent(message);
if (text == null) {
text = "";
}
int maxTextLength = "verifier".equals(agentName) ? 4000 : 500;
text = truncate(text, maxTextLength);
StringBuilder sb = new StringBuilder();
sb.append("{\"text\":\"").append(escapeJson(text)).append("\"");
if (message.getToolCalls() != null && !message.getToolCalls().isEmpty()) {
sb.append(",\"toolCalls\":[");
for (int i = 0; i < message.getToolCalls().size(); i++) {
if (i > 0) {
sb.append(",");
}
sb.append("{\"name\":\"").append(escapeJson(message.getToolCalls().get(i).name()))
.append("\",\"arguments\":").append(message.getToolCalls().get(i).arguments()).append("}");
}
sb.append("]");
}
sb.append("}");
return sb.toString();
}
private String escapeJson(String s) {
if (s == null) {
return "";
}
return s.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
private String truncate(String text, int maxLength) {
if (text == null || text.length() <= maxLength) {
return text;
}
return text.substring(0, maxLength) + "...";
}
private String extractTextContent(AssistantMessage message) {
if (message == null) {
return null;
}
try {
try {
return message.getText();
} catch (Exception ignore) {
// Fallback below.
}
for (String fieldName : List.of("text", "content")) {
try {
java.lang.reflect.Field field = message.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
Object value = field.get(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchFieldException ignore) {
// continue
}
}
for (String methodName : List.of("getText", "getContent")) {
try {
java.lang.reflect.Method method = message.getClass().getMethod(methodName);
Object value = method.invoke(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchMethodException ignore) {
// continue
}
}
String fallback = message.toString();
if (fallback != null && !fallback.startsWith("AssistantMessage@")) {
return fallback;
}
return null;
} catch (Exception e) {
log.error("Failed to extract AssistantMessage text", e);
return null;
}
}
private String getMessageRole(Message message) {
if (message instanceof UserMessage) {
return "user";
}
if (message instanceof AssistantMessage) {
return "assistant";
}
if (message instanceof ToolResponseMessage) {
return "tool";
}
return message.getClass().getSimpleName();
}
}
@@ -1,101 +0,0 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.Prioritized;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.alibaba.cloud.ai.graph.skills.SkillMetadata;
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Injects planner-visible skill metadata without exposing the full skill loader tool.
*/
@Slf4j
@HookPositions(HookPosition.BEFORE_MODEL)
public class PlannerSkillMetadataHook extends MessagesModelHook {
private static final String CATALOG_MARKER = "\"skill_catalog\"";
private final SkillRegistry skillRegistry;
private final ObjectMapper objectMapper = new ObjectMapper();
public PlannerSkillMetadataHook(SkillRegistry skillRegistry) {
this.skillRegistry = skillRegistry;
}
@Override
public String getName() {
return "planner_skill_metadata_hook";
}
@Override
public int getOrder() {
return Prioritized.HIGHEST_PRECEDENCE;
}
@Override
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
if (skillRegistry == null || skillRegistry.size() == 0 || hasCatalog(previousMessages)) {
return new AgentCommand(previousMessages);
}
try {
List<Map<String, String>> skills = skillRegistry.listAll().stream()
.map(this::toSkillSummary)
.toList();
if (skills.isEmpty()) {
return new AgentCommand(previousMessages);
}
Map<String, Object> catalog = new LinkedHashMap<>();
catalog.put("purpose", "Planner-visible diagnosis skill metadata only.");
catalog.put("rules", List.of(
"Choose at most one primary skill.",
"Do not load full skill instructions in Planner.",
"Executor reads the selected skill before evidence collection.",
"If no skill matches, set selected_skill to null."
));
catalog.put("skills", skills);
catalog.put("required_planner_output", Map.of(
"selected_skill", "skill name or null",
"selection_reason", "short reason",
"plan", "ordered execution step list"
));
Map<String, Object> payload = Map.of("skill_catalog", catalog);
String content = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(payload);
List<Message> updatedMessages = new ArrayList<>(previousMessages.size() + 1);
updatedMessages.add(new SystemMessage(content));
updatedMessages.addAll(previousMessages);
return new AgentCommand(updatedMessages);
} catch (Exception e) {
log.warn("Failed to inject planner skill metadata, fallback to original messages", e);
return new AgentCommand(previousMessages);
}
}
private Map<String, String> toSkillSummary(SkillMetadata skill) {
Map<String, String> summary = new LinkedHashMap<>();
summary.put("name", skill.getName());
summary.put("description", skill.getDescription());
return summary;
}
private boolean hasCatalog(List<Message> messages) {
return messages.stream()
.map(Message::getText)
.anyMatch(text -> text != null && text.contains(CATALOG_MARKER));
}
}
@@ -1,50 +0,0 @@
package com.superbiz.agent.hook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import reactor.core.publisher.Flux;
/**
* ChatModel 包装器 — 捕获每次模型调用的实际 token 用量
* 通过 TokenUsageHolder 传递给 AgentLoggingHook
*/
public class TokenTrackingChatModel implements ChatModel {
private static final Logger log = LoggerFactory.getLogger(TokenTrackingChatModel.class);
private final ChatModel delegate;
public TokenTrackingChatModel(ChatModel delegate) {
this.delegate = delegate;
}
@Override
public ChatResponse call(Prompt prompt) {
ChatResponse response = delegate.call(prompt);
captureTokenUsage(response);
return response;
}
@Override
public Flux<ChatResponse> stream(Prompt prompt) {
return delegate.stream(prompt);
}
private void captureTokenUsage(ChatResponse response) {
try {
if (response.getMetadata() == null || response.getMetadata().getUsage() == null) {
return;
}
var usage = response.getMetadata().getUsage();
Integer total = usage.getTotalTokens();
if (total != null && total > 0) {
TokenUsageHolder.set(total);
}
} catch (Exception e) {
log.debug("捕获 token 用量失败", e);
}
}
}
@@ -1,22 +0,0 @@
package com.superbiz.agent.hook;
/**
* Token 用量持有者(基于 ThreadLocal)
* ChatModel 调用后写入实际 token 数,AgentLoggingHook 读取
*/
public class TokenUsageHolder {
private static final ThreadLocal<Integer> TOKEN_COUNT = new ThreadLocal<>();
public static void set(Integer count) {
TOKEN_COUNT.set(count);
}
public static Integer get() {
return TOKEN_COUNT.get();
}
public static void clear() {
TOKEN_COUNT.remove();
}
}
@@ -1,352 +0,0 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ExecutorGatekeeperService;
import com.superbiz.agent.service.GatekeeperRuleCatalog;
import com.superbiz.agent.service.ToolTraceSummaryService;
import com.superbiz.agent.util.SessionContextHolder;
import com.superbiz.agent.util.VerifierContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Replaces verifier history with an explicit structured payload.
*/
@Slf4j
@HookPositions(HookPosition.BEFORE_MODEL)
public class VerifierInputHook extends MessagesModelHook {
private final ToolTraceSummaryService toolTraceSummaryService;
private final ExecutorGatekeeperService executorGatekeeperService;
private final ObjectMapper objectMapper = new ObjectMapper();
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
public VerifierInputHook(ToolTraceSummaryService toolTraceSummaryService) {
this(toolTraceSummaryService, null);
}
public VerifierInputHook(ToolTraceSummaryService toolTraceSummaryService,
ExecutorGatekeeperService executorGatekeeperService) {
this.toolTraceSummaryService = toolTraceSummaryService;
this.executorGatekeeperService = executorGatekeeperService;
}
@Override
public String getName() {
return "verifier_input_hook";
}
@Override
public AgentCommand beforeModel(List<Message> previousMessages, RunnableConfig config) {
try {
String sessionId = config.metadata("sessionId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getSessionId);
String runId = config.metadata("runId")
.map(Object::toString)
.orElseGet(SessionContextHolder::getRunId);
String executorFinalAnswer = VerifierContextHolder.getExecutorFinalAnswer();
if (executorFinalAnswer == null || executorFinalAnswer.isBlank()) {
executorFinalAnswer = extractLastAssistantText(previousMessages);
}
List<Map<String, Object>> toolTraceSummary = runId == null || runId.isBlank()
? toolTraceSummaryService.buildVerifierTraceSummary(sessionId, executorFinalAnswer)
: toolTraceSummaryService.buildVerifierTraceSummaryForRun(runId, executorFinalAnswer);
VerifierContextHolder.setToolTraceSummary(toolTraceSummary);
ExecutorOutputParseResult parseResult = parseExecutorOutput(executorFinalAnswer);
parseResult = new ExecutorOutputParseResult(
enrichExecutorStructuredOutput(parseResult.structuredOutput(), toolTraceSummary),
parseResult.status()
);
VerifierContextHolder.setExecutorStructuredOutput(parseResult.structuredOutput());
VerifierContextHolder.setExecutorOutputParseStatus(parseResult.status());
Map<String, Object> gatekeeperResult = runGatekeeper(sessionId, runId, parseResult);
VerifierContextHolder.setGatekeeperResult(gatekeeperResult);
Map<String, Object> verifierInput = new LinkedHashMap<>();
verifierInput.put("original_query", VerifierContextHolder.getOriginalQuery());
verifierInput.put("executor_final_answer", executorFinalAnswer);
verifierInput.put("executor_structured_output", parseResult.structuredOutput());
verifierInput.put("executor_output_parse_status", parseResult.status());
verifierInput.put("tool_trace_summary", toolTraceSummary);
verifierInput.put("gatekeeper_result", gatekeeperResult);
verifierInput.put("retry_context", VerifierContextHolder.getRetryContext());
String payload = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(verifierInput);
return new AgentCommand(List.of(new UserMessage(payload)));
} catch (Exception e) {
log.error("Failed to build verifier input, fallback to original messages", e);
return new AgentCommand(previousMessages);
}
}
private Map<String, Object> runGatekeeper(String sessionId, String runId, ExecutorOutputParseResult parseResult) {
if (executorGatekeeperService == null) {
return passGatekeeperResult();
}
try {
if (runId != null && !runId.isBlank()) {
return executorGatekeeperService.validateRun(runId, parseResult.structuredOutput(), parseResult.status());
}
return executorGatekeeperService.validate(sessionId, parseResult.structuredOutput(), parseResult.status());
} catch (Exception e) {
log.error("Gatekeeper validation failed unexpectedly", e);
return executorGatekeeperService.fail("gatekeeper.internal_error",
"gatekeeper",
e.getMessage() == null ? "gatekeeper validation failed" : e.getMessage());
}
}
private Map<String, Object> passGatekeeperResult() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", "pass");
result.put("severity", "none");
result.put("rule_set_version", catalog.version());
result.put("rules", catalog.auditRules());
result.put("checked_bindings", List.of());
result.put("failed_rules", List.of());
result.put("warnings", List.of());
result.put("errors", List.of());
return result;
}
private ExecutorOutputParseResult parseExecutorOutput(String executorFinalAnswer) {
if (executorFinalAnswer == null || executorFinalAnswer.isBlank()) {
return new ExecutorOutputParseResult(null, status("missing", "executor_final_answer is blank"));
}
String sanitized = sanitizeJsonPayload(executorFinalAnswer);
if (!looksJsonLike(sanitized)) {
return new ExecutorOutputParseResult(null, status("missing", "executor output is not JSON"));
}
try {
JsonNode root = objectMapper.readTree(sanitized);
if (!root.isObject() || !root.path("claims").isArray()) {
return new ExecutorOutputParseResult(null, status("malformed",
"executor output JSON does not match evidence-attribution contract"));
}
Map<String, Object> structuredOutput = objectMapper.convertValue(root, MAP_TYPE);
return new ExecutorOutputParseResult(structuredOutput, status("valid", "parsed executor evidence contract"));
} catch (Exception e) {
log.debug("Failed to parse executor structured output", e);
return new ExecutorOutputParseResult(null, status("malformed", e.getMessage()));
}
}
@SuppressWarnings("unchecked")
private Map<String, Object> enrichExecutorStructuredOutput(Map<String, Object> structuredOutput,
List<Map<String, Object>> toolTraceSummary) {
if (structuredOutput == null) {
return null;
}
Map<String, List<Long>> invocationIdsByTool = invocationIdsByTool(toolTraceSummary);
List<Map<String, Object>> warnings = new ArrayList<>();
enrichEvidenceBindingsInSection(structuredOutput.get("claims"), invocationIdsByTool, warnings);
enrichEvidenceBindingsInSection(structuredOutput.get("recommended_actions"), invocationIdsByTool, warnings);
if (!warnings.isEmpty()) {
structuredOutput.put("_gatekeeper_warnings", warnings);
}
return structuredOutput;
}
@SuppressWarnings("unchecked")
private void enrichEvidenceBindingsInSection(Object sectionValue,
Map<String, List<Long>> invocationIdsByTool,
List<Map<String, Object>> warnings) {
if (!(sectionValue instanceof List<?> items)) {
return;
}
for (Object itemValue : items) {
if (!(itemValue instanceof Map<?, ?> item)) {
continue;
}
Object bindingsValue = item.get("evidence_bindings");
if (!(bindingsValue instanceof List<?> bindings)) {
continue;
}
for (Object bindingValue : bindings) {
if (!(bindingValue instanceof Map<?, ?> rawBinding)) {
continue;
}
Map<String, Object> binding = (Map<String, Object>) rawBinding;
String normalizedToolName = normalizeToolName(binding.get("tool_name"));
if (!normalizedToolName.isBlank()) {
binding.put("tool_name", normalizedToolName);
}
if (!hasInvocationId(binding)) {
List<Long> ids = invocationIdsByTool.getOrDefault(normalizedToolName, List.of());
if (ids.size() == 1) {
binding.put("source_invocation_id", ids.get(0));
warnings.add(Map.of(
"rule", "evidence.invocation_auto_backfill",
"message", "source_invocation_id was auto-filled from the unique tool invocation candidate; raw_path remains missing if Executor did not provide it",
"tool_name", normalizedToolName,
"source_invocation_id", ids.get(0)
));
}
}
}
}
}
private Map<String, List<Long>> invocationIdsByTool(List<Map<String, Object>> toolTraceSummary) {
Map<String, Set<Long>> idsByTool = new LinkedHashMap<>();
for (Map<String, Object> summary : toolTraceSummary == null ? List.<Map<String, Object>>of() : toolTraceSummary) {
String toolName = normalizeToolName(summary.get("tool_name"));
if (toolName.isBlank()) {
continue;
}
List<Long> ids = toLongList(summary.get("source_invocation_ids"));
if (ids.isEmpty()) {
continue;
}
idsByTool.computeIfAbsent(toolName, ignored -> new LinkedHashSet<>()).addAll(ids);
}
Map<String, List<Long>> result = new LinkedHashMap<>();
for (Map.Entry<String, Set<Long>> entry : idsByTool.entrySet()) {
result.put(entry.getKey(), new ArrayList<>(entry.getValue()));
}
return result;
}
private boolean hasInvocationId(Map<String, Object> binding) {
if (asLong(binding.get("source_invocation_id")) != null) {
return true;
}
return toLongList(binding.get("source_invocation_ids")).size() == 1;
}
private List<Long> toLongList(Object value) {
if (!(value instanceof List<?> values)) {
return List.of();
}
List<Long> ids = new ArrayList<>();
for (Object item : values) {
Long id = asLong(item);
if (id != null) {
ids.add(id);
}
}
return ids;
}
private Long asLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text) {
try {
return Long.parseLong(text);
} catch (NumberFormatException ignored) {
return null;
}
}
return null;
}
private String normalizeToolName(Object value) {
String toolName = value == null ? "" : String.valueOf(value);
return switch (toolName) {
case "lookupKnowledge" -> "lookup_knowledge";
case "queryLogs" -> "query_logs";
case "queryPrometheusAlerts" -> "query_metrics";
case "getAvailableLogTopics" -> "get_available_log_topics";
default -> toolName;
};
}
private String sanitizeJsonPayload(String raw) {
String trimmed = raw.trim();
int fenceStart = trimmed.indexOf("```");
if (fenceStart >= 0) {
int firstNewline = trimmed.indexOf('\n', fenceStart);
int lastFence = trimmed.indexOf("```", firstNewline + 1);
if (firstNewline >= 0 && lastFence > firstNewline) {
return trimmed.substring(firstNewline + 1, lastFence).trim();
}
}
int objectStart = trimmed.indexOf('{');
int objectEnd = trimmed.lastIndexOf('}');
if (objectStart >= 0 && objectEnd > objectStart) {
return trimmed.substring(objectStart, objectEnd + 1).trim();
}
return trimmed;
}
private boolean looksJsonLike(String text) {
return text.startsWith("{") && text.endsWith("}");
}
private Map<String, Object> status(String status, String detail) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", status);
result.put("detail", detail == null ? "" : detail);
return result;
}
private String extractLastAssistantText(List<Message> previousMessages) {
for (int i = previousMessages.size() - 1; i >= 0; i--) {
if (previousMessages.get(i) instanceof AssistantMessage assistantMessage) {
String text = extractTextContent(assistantMessage);
if (text != null && !text.isBlank()) {
return text;
}
}
}
return "";
}
private String extractTextContent(AssistantMessage message) {
try {
try {
return message.getText();
} catch (Exception ignore) {
// Fallback for older implementations.
}
for (String methodName : List.of("getText", "getContent")) {
try {
var method = message.getClass().getMethod(methodName);
Object value = method.invoke(message);
if (value != null) {
return value.toString();
}
} catch (NoSuchMethodException ignore) {
// continue
}
}
} catch (Exception e) {
log.debug("Failed to extract verifier assistant text", e);
}
return message.toString();
}
private record ExecutorOutputParseResult(
Map<String, Object> structuredOutput,
Map<String, Object> status
) {
}
}
@@ -1,146 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.AIOpsRequest;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@Service
public class AiOpsRuleEvaluationService {
public static final String PASS = "PASS";
public static final String WARN = "WARN";
public static final String FAIL = "FAIL";
public Map<String, Object> evaluate(AIOpsRequest request,
String finalReport,
List<ToolInvocation> toolInvocations) {
List<Map<String, Object>> checks = new ArrayList<>();
checks.add(checkReportPresent(finalReport));
checks.add(checkPayloadFocus(request, finalReport));
checks.add(checkEvidenceCoverage(toolInvocations));
String verdict = aggregateVerdict(checks);
Map<String, Object> evaluation = new LinkedHashMap<>();
evaluation.put("verdict", verdict);
evaluation.put("checks", checks);
evaluation.put("rationale", buildRationale(verdict, checks));
evaluation.put("traceability_version", "aiops-rule-v1");
return evaluation;
}
private Map<String, Object> checkReportPresent(String finalReport) {
boolean passed = finalReport != null && finalReport.trim().length() >= 40;
return check(
"final_report_present",
passed ? PASS : FAIL,
passed ? "Final report is present." : "Final report is missing or too short."
);
}
private Map<String, Object> checkPayloadFocus(AIOpsRequest request, String finalReport) {
if (!hasAlertPayload(request)) {
return check("payload_focus", PASS, "No alert payload was supplied; payload focus is not required.");
}
String report = lower(finalReport);
List<String> missing = new ArrayList<>();
if (!contains(report, request.getAlertName())) {
missing.add("alertName");
}
if (!contains(report, request.getService())) {
missing.add("service");
}
if (missing.isEmpty()) {
return check("payload_focus", PASS, "Final report mentions the supplied alert and service.");
}
return check(
"payload_focus",
WARN,
"Final report is missing payload focus terms: " + String.join(", ", missing)
);
}
private Map<String, Object> checkEvidenceCoverage(List<ToolInvocation> toolInvocations) {
List<String> evidenceTools = safeTools(toolInvocations).stream()
.filter(tool -> tool.equals("lookup_knowledge")
|| tool.equals("query_metrics")
|| tool.equals("query_logs"))
.distinct()
.toList();
if (evidenceTools.isEmpty()) {
return check("evidence_tool_coverage", WARN, "No persisted AIOps evidence tool calls were found.");
}
return check(
"evidence_tool_coverage",
PASS,
"Persisted evidence tools: " + String.join(", ", evidenceTools)
);
}
private List<String> safeTools(List<ToolInvocation> toolInvocations) {
if (toolInvocations == null) {
return List.of();
}
return toolInvocations.stream()
.map(ToolInvocation::getToolName)
.filter(name -> name != null && !name.isBlank())
.map(name -> name.trim().toLowerCase(Locale.ROOT))
.toList();
}
private String aggregateVerdict(List<Map<String, Object>> checks) {
boolean hasFail = checks.stream().anyMatch(check -> FAIL.equals(check.get("verdict")));
if (hasFail) {
return FAIL;
}
boolean hasWarn = checks.stream().anyMatch(check -> WARN.equals(check.get("verdict")));
return hasWarn ? WARN : PASS;
}
private String buildRationale(String verdict, List<Map<String, Object>> checks) {
long passCount = checks.stream().filter(check -> PASS.equals(check.get("verdict"))).count();
long warnCount = checks.stream().filter(check -> WARN.equals(check.get("verdict"))).count();
long failCount = checks.stream().filter(check -> FAIL.equals(check.get("verdict"))).count();
return "AIOps rule evaluation %s: pass=%d, warn=%d, fail=%d"
.formatted(verdict, passCount, warnCount, failCount);
}
private Map<String, Object> check(String name, String verdict, String detail) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("name", name);
result.put("verdict", verdict);
result.put("detail", detail);
return result;
}
private boolean hasAlertPayload(AIOpsRequest request) {
if (request == null) {
return false;
}
return !isBlank(request.getAlertName())
|| !isBlank(request.getService())
|| !isBlank(request.getSeverity())
|| !isBlank(request.getDescription())
|| !isBlank(request.getTimeRange());
}
private boolean contains(String lowerText, String value) {
return isBlank(value) || lowerText.contains(value.trim().toLowerCase(Locale.ROOT));
}
private String lower(String value) {
return value == null ? "" : value.toLowerCase(Locale.ROOT);
}
private boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
}
@@ -1,435 +0,0 @@
package com.superbiz.agent.service;
import org.springframework.ai.chat.model.ChatModel;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.OverAllState;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
import com.alibaba.cloud.ai.graph.agent.hook.Hook;
import com.alibaba.cloud.ai.graph.agent.hook.skills.SkillsAgentHook;
import com.alibaba.cloud.ai.graph.exception.GraphRunnerException;
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
import com.superbiz.agent.agent.tool.DateTimeTools;
import com.superbiz.agent.agent.tool.InternalDocsTools;
import com.superbiz.agent.agent.tool.QueryLogsTools;
import com.superbiz.agent.agent.tool.QueryMetricsTools;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.domain.entity.DiagnosisRun;
import com.superbiz.agent.domain.entity.DiagnosisSession;
import com.superbiz.agent.dto.AIOpsRequest;
import com.superbiz.agent.hook.AgentLoggingHook;
import com.superbiz.agent.hook.PlannerSkillMetadataHook;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.DiagnosisRunRepository;
import com.superbiz.agent.repository.DiagnosisSessionRepository;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.util.SessionContextHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.superbiz.agent.config.AiOpsPromptProperties;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
/**
* AI Ops 智能运维服务。
* 负责构建 Planner、Executor、Supervisor 多 Agent 编排流程,并持久化诊断会话与执行指标。
*/
@Service
public class AiOpsService {
private static final Logger logger = LoggerFactory.getLogger(AiOpsService.class);
@Autowired
private ChatModel chatModel;
@Autowired(required = false)
private ToolCallbackProvider toolCallbackProvider;
@Autowired
private DateTimeTools dateTimeTools;
@Autowired
private InternalDocsTools internalDocsTools;
@Autowired
private QueryMetricsTools queryMetricsTools;
@Autowired(required = false) // Mock 模式下才注册本地日志查询工具。
private QueryLogsTools queryLogsTools;
@Autowired
private LookupKnowledgeTool lookupKnowledgeTool;
@Autowired
private AiOpsPromptProperties promptProperties;
@Autowired
private DiagnosisSessionRepository diagnosisSessionRepository;
@Autowired
private DiagnosisRunRepository diagnosisRunRepository;
@Autowired
private AgentStepRepository agentStepRepository;
@Autowired
private ToolInvocationRepository toolInvocationRepository;
@Autowired
private AiOpsRuleEvaluationService aiOpsRuleEvaluationService;
@Autowired
private SelfEvaluationMergeService selfEvaluationMergeService;
@Autowired(required = false)
private SkillRegistry skillRegistry;
/**
* 执行 AI Ops 告警分析流程。
*
* @param chatModel 大模型实例
* @param toolCallbacks Spring AI 工具回调
* @return 多 Agent 编排后的最终状态
* @throws GraphRunnerException Agent 图执行失败时抛出
*/
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks) throws GraphRunnerException {
return executeAiOpsAnalysis(chatModel, toolCallbacks, null, resolveSessionId(null), newRunId());
}
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks,
AIOpsRequest request, String sessionId) throws GraphRunnerException {
return executeAiOpsAnalysis(chatModel, toolCallbacks, request, sessionId, newRunId());
}
public Optional<OverAllState> executeAiOpsAnalysis(ChatModel chatModel, ToolCallback[] toolCallbacks,
AIOpsRequest request, String sessionId, String runId) throws GraphRunnerException {
logger.info("Starting AI Ops multi-agent analysis");
String resolvedSessionId = isBlank(sessionId) ? resolveSessionId(request) : sessionId.trim();
String resolvedRunId = isBlank(runId) ? newRunId() : runId.trim();
long startTime = System.currentTimeMillis();
DiagnosisRun run = startDiagnosisRun(resolvedSessionId, resolvedRunId, request);
// 让工具调用、Hook 和知识库检索能够拿到当前诊断会话和运行 ID。
SessionContextHolder.setContext(resolvedSessionId, resolvedRunId);
try {
ReactAgent plannerAgent = buildPlannerAgent(chatModel, toolCallbacks);
ReactAgent executorAgent = buildExecutorAgent(chatModel, toolCallbacks);
SupervisorAgent supervisorAgent = buildSupervisorAgent(chatModel, plannerAgent, executorAgent);
String taskPrompt = buildTaskPrompt(request);
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", resolvedSessionId)
.addMetadata("runId", resolvedRunId)
.build();
logger.info("Invoking AI Ops supervisor agent");
Optional<OverAllState> stateOptional = supervisorAgent.invoke(taskPrompt, config);
long duration = System.currentTimeMillis() - startTime;
run.setStatus(stateOptional.isPresent() ? "SUCCESS" : "FAILED");
run.setTotalDurationMs((int) duration);
backfillRunMetrics(run);
diagnosisRunRepository.save(run);
if (stateOptional.isPresent()) {
OverAllState state = stateOptional.get();
logger.debug("Final State Keys: {}", state.data().keySet());
logger.debug("Planner Plan: {}", state.value("planner_plan"));
logger.debug("Executor Feedback: {}", state.value("executor_feedback"));
}
return stateOptional;
} catch (Exception e) {
run.setStatus("FAILED");
run.setTotalDurationMs((int) (System.currentTimeMillis() - startTime));
backfillRunMetrics(run);
diagnosisRunRepository.save(run);
throw e;
} finally {
SessionContextHolder.clear();
}
}
/** Internal dependency-owning overload used by the protocol-only Controller. */
public Optional<OverAllState> executeAiOpsAnalysis(AIOpsRequest request,
String sessionId,
String runId) throws GraphRunnerException {
ToolCallback[] callbacks = toolCallbackProvider == null
? new ToolCallback[0] : toolCallbackProvider.getToolCallbacks();
return executeAiOpsAnalysis(chatModel, callbacks, request, sessionId, runId);
}
/**
* 从多 Agent 执行状态中提取最终报告文本。
*
* @param state Agent 图执行状态
* @return Planner 输出中的最终报告
*/
public Optional<String> extractFinalReport(OverAllState state) {
logger.info("Extracting final AI Ops report");
Optional<AssistantMessage> plannerFinalOutput = state.value("planner_plan")
.filter(AssistantMessage.class::isInstance)
.map(AssistantMessage.class::cast);
if (plannerFinalOutput.isPresent()) {
String reportText = plannerFinalOutput.get().getText();
logger.info("Extracted Planner final report, length: {}", reportText.length());
return Optional.of(reportText);
} else {
logger.warn("Unable to extract Planner final report");
return Optional.empty();
}
}
public String resolveSessionId(AIOpsRequest request) {
if (request != null && !isBlank(request.getSessionId())) {
return request.getSessionId().trim();
}
return UUID.randomUUID().toString();
}
public String newRunId() {
return "run-" + UUID.randomUUID();
}
public void persistFinalReport(String sessionId, String finalReport) {
persistFinalReport(sessionId, finalReport, null);
}
public void persistFinalReport(String sessionId, String finalReport, AIOpsRequest request) {
persistLegacyFinalReport(sessionId, finalReport, request);
}
public void persistFinalReport(String sessionId, String runId, String finalReport, AIOpsRequest request) {
if (isBlank(sessionId) || isBlank(runId) || isBlank(finalReport)) {
return;
}
diagnosisRunRepository.findBySessionIdAndRunId(sessionId.trim(), runId.trim()).ifPresent(run -> {
run.setAnswer(finalReport);
List<com.superbiz.agent.domain.entity.ToolInvocation> invocations =
toolInvocationRepository.findByRunIdOrderByIdAsc(run.getRunId());
Map<String, Object> evaluation = aiOpsRuleEvaluationService.evaluate(request, finalReport, invocations);
run.setSelfEvaluation(selfEvaluationMergeService.mergeAiOpsRuleEvaluation(
run.getSelfEvaluation(), evaluation));
diagnosisRunRepository.save(run);
});
}
private void persistLegacyFinalReport(String sessionId, String finalReport, AIOpsRequest request) {
if (isBlank(sessionId) || isBlank(finalReport)) {
return;
}
diagnosisSessionRepository.findBySessionId(sessionId.trim()).ifPresent(session -> {
session.setAnswer(finalReport);
List<com.superbiz.agent.domain.entity.ToolInvocation> invocations =
toolInvocationRepository.findBySessionIdOrderByIdAsc(session.getSessionId());
Map<String, Object> evaluation = aiOpsRuleEvaluationService.evaluate(request, finalReport, invocations);
session.setSelfEvaluation(selfEvaluationMergeService.mergeAiOpsRuleEvaluation(
session.getSelfEvaluation(), evaluation));
diagnosisSessionRepository.save(session);
});
}
String buildQuerySummary(AIOpsRequest request) {
if (request == null) {
return "AI Ops alert analysis";
}
StringBuilder summary = new StringBuilder("AI Ops alert analysis");
appendField(summary, "alert", request.getAlertName());
appendField(summary, "service", request.getService());
appendField(summary, "severity", request.getSeverity());
appendField(summary, "timeRange", request.getTimeRange());
appendField(summary, "description", request.getDescription());
appendField(summary, "request", request.getUserRequest());
return summary.toString();
}
boolean hasAlertPayload(AIOpsRequest request) {
if (request == null) {
return false;
}
return !isBlank(request.getAlertName())
|| !isBlank(request.getService())
|| !isBlank(request.getSeverity())
|| !isBlank(request.getDescription())
|| !isBlank(request.getTimeRange());
}
String buildKnowledgeRetrievalQuery(AIOpsRequest request) {
if (request == null || !hasAlertPayload(request)) {
return "";
}
StringBuilder query = new StringBuilder();
appendQueryTerm(query, request.getAlertName());
appendQueryTerm(query, request.getService());
appendQueryTerm(query, request.getSeverity());
appendQueryTerm(query, request.getDescription());
appendQueryTerm(query, request.getTimeRange());
appendQueryTerm(query, request.getUserRequest());
return query.toString();
}
String buildTaskPrompt(AIOpsRequest request) {
StringBuilder prompt = new StringBuilder();
prompt.append("You are an enterprise SRE handling an automated alert diagnosis task. Combine tool evidence, run a plan-execute-replan loop, and output the final alert analysis report. Do not fabricate data; if repeated queries fail, clearly state why the task cannot be completed.");
prompt.append("\n\nAlert input:\n");
prompt.append(buildQuerySummary(request));
if (hasAlertPayload(request)) {
String knowledgeQuery = buildKnowledgeRetrievalQuery(request);
prompt.append("\n\nAIOps scope mode: PAYLOAD_TARGETED\n");
prompt.append("- The request includes an alert payload. Treat the supplied alert payload as the primary and only main diagnosis target.\n");
prompt.append("- The final report must focus on the supplied alert fields such as alertName, service, severity, description, and timeRange.\n");
prompt.append("- Recommended lookup_knowledge query: ").append(knowledgeQuery).append("\n");
prompt.append("- If knowledge-base evidence is needed, call lookup_knowledge with the recommended query or a narrower query that preserves alertName and service.\n");
prompt.append("- You may call queryPrometheusAlerts only to verify whether the supplied alert is still active or to identify related risk/context.\n");
prompt.append("- If queryPrometheusAlerts returns unrelated active alerts, do not create full root-cause or remediation sections for them.\n");
prompt.append("- Mention unrelated active alerts only briefly in a Related Risk section when they help explain the supplied alert.\n");
} else {
prompt.append("\n\nAIOps scope mode: AUTO_DISCOVERY\n");
prompt.append("- The request does not include alert payload fields. First call queryPrometheusAlerts to discover current active/firing alerts.\n");
prompt.append("- Prefer P0/P1 alerts or the longest-running firing alerts, then diagnose one or more alerts based on severity and evidence.\n");
prompt.append("- Use metrics, logs, and knowledge-base evidence before producing the final alert analysis report.\n");
}
return prompt.toString();
}
private DiagnosisRun startDiagnosisRun(String sessionId, String runId, AIOpsRequest request) {
DiagnosisRun run = DiagnosisRun.builder()
.runId(runId)
.sessionId(sessionId)
.query(buildQuerySummary(request))
.status("RUNNING")
.agentFlow("AI_OPS")
.build();
return diagnosisRunRepository.save(run);
}
/**
* 构建 Planner Agent。
*/
private ReactAgent buildPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks) {
return ReactAgent.builder()
.name("planner_agent")
.description("Plans alert diagnosis steps")
.model(chatModel)
.systemPrompt(promptProperties.getPlanner())
.hooks(buildHooks("planner"))
.outputKey("planner_plan")
.build();
}
/**
* 构建 Executor Agent。
*/
private ReactAgent buildExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks) {
return ReactAgent.builder()
.name("executor_agent")
.description("Executes the current Planner step and reports feedback")
.model(chatModel)
.systemPrompt(promptProperties.getExecutor())
.methodTools(buildMethodToolsArray())
.tools(toolCallbacks)
.hooks(buildHooks("executor"))
.outputKey("executor_feedback")
.build();
}
/**
* 根据运行模式构建方法工具数组。
* Mock 模式注入本地 QueryLogsTools;真实模式下日志查询由外部 MCP 工具提供。
*/
private Object[] buildMethodToolsArray() {
if (queryLogsTools != null) {
return new Object[]{dateTimeTools, lookupKnowledgeTool, queryMetricsTools, queryLogsTools};
}
return new Object[]{dateTimeTools, lookupKnowledgeTool, queryMetricsTools};
}
private Hook[] buildHooks(String agentName) {
AgentLoggingHook loggingHook = new AgentLoggingHook(agentStepRepository, agentName);
if (skillRegistry == null || skillRegistry.size() == 0) {
return new Hook[]{loggingHook};
}
if ("planner".equals(agentName)) {
return new Hook[]{
new PlannerSkillMetadataHook(skillRegistry),
loggingHook
};
}
return new Hook[]{
loggingHook,
SkillsAgentHook.builder()
.skillRegistry(skillRegistry)
.build()
};
}
SupervisorAgent buildSupervisorAgent(ChatModel chatModel, ReactAgent plannerAgent, ReactAgent executorAgent) {
return SupervisorAgent.builder()
.name("ai_ops_supervisor")
.description("Coordinates Planner and Executor agents")
.model(chatModel)
.mainAgent(plannerAgent)
.systemPrompt(promptProperties.getSupervisor())
.subAgents(List.of(executorAgent))
.build();
}
/**
* 从 agent_step 和 tool_invocation 回填 diagnosis_run 的汇总指标。
*/
private void backfillRunMetrics(DiagnosisRun run) {
try {
List<AgentStep> steps = agentStepRepository.findByRunIdOrderByStepIndex(run.getRunId());
int totalTokens = 0;
int stepCount = 0;
for (AgentStep s : steps) {
stepCount++;
if (s.getTokenCount() != null) totalTokens += s.getTokenCount();
}
long toolCallCount = toolInvocationRepository.countByRunId(run.getRunId());
run.setTotalTokenCount(totalTokens);
run.setStepCount(stepCount);
run.setToolCallCount(Math.toIntExact(toolCallCount));
} catch (Exception e) {
logger.warn("Failed to backfill AI Ops run metrics, runId={}", run.getRunId(), e);
}
}
private void appendField(StringBuilder builder, String label, String value) {
if (!isBlank(value)) {
builder.append("\n- ").append(label).append(": ").append(value.trim());
}
}
private void appendQueryTerm(StringBuilder builder, String value) {
if (!isBlank(value)) {
if (!builder.isEmpty()) {
builder.append(' ');
}
builder.append(value.trim());
}
}
private boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
}
File diff suppressed because it is too large Load Diff
@@ -258,7 +258,6 @@ public class DiagnosisTraceService {
.persistedToolCallCount(defaultInt(session.getToolCallCount()))
.returnedToolCallCount(toolInvocations.size())
.hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation"))
.hasAiOpsRuleEvaluation(selfEvaluation != null && selfEvaluation.containsKey("aiops_rule_evaluation"))
.hasFeedback(session.getFeedback() != null && !session.getFeedback().isBlank())
.build();
}
@@ -276,7 +275,6 @@ public class DiagnosisTraceService {
.persistedToolCallCount(defaultInt(run.getToolCallCount()))
.returnedToolCallCount(toolInvocations.size())
.hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation"))
.hasAiOpsRuleEvaluation(selfEvaluation != null && selfEvaluation.containsKey("aiops_rule_evaluation"))
.hasFeedback(run.getFeedback() != null && !run.getFeedback().isBlank())
.build();
}
@@ -1,614 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Deterministic checks for Executor structured output before verifier reasoning.
*/
@Service
public class ExecutorGatekeeperService {
public static final String STATUS_PASS = "pass";
public static final String STATUS_FAIL = "fail";
public static final String SEVERITY_NONE = "none";
public static final String SEVERITY_LOW_CONFID = "low_confid";
public static final String SEVERITY_REJECT = "reject";
public static final String RULE_SCHEMA = "schema.executor_v2";
public static final String RULE_INVOCATION_REF = "evidence.invocation_ref";
public static final String RULE_RAW_PATH = "evidence.raw_path";
public static final String RULE_EXCERPT_MISMATCH = "evidence.excerpt_mismatch";
public static final String RULE_EVIDENCE_MISSING = "evidence.missing";
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
private static final double DEFAULT_MIN_TOKEN_OVERLAP = 0.5;
private final ToolInvocationRepository toolInvocationRepository;
private final GatekeeperRuleCatalog ruleCatalog;
private final ObjectMapper objectMapper = new ObjectMapper();
@Autowired
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository) {
this(toolInvocationRepository, null);
}
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository,
GatekeeperRuleCatalog ruleCatalog) {
this.toolInvocationRepository = toolInvocationRepository;
this.ruleCatalog = ruleCatalog == null
? GatekeeperRuleCatalog.loadDefault(objectMapper)
: ruleCatalog;
}
public Map<String, Object> validate(String sessionId,
Map<String, Object> structuredOutput,
Map<String, Object> parseStatus) {
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
validateSchema(structuredOutput, parseStatus, result);
if (structuredOutput != null) {
List<ToolInvocation> invocations = sessionId == null || sessionId.isBlank()
? List.of()
: toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId);
validateInvocationRefs("session_id", sessionId, invocations, structuredOutput, result);
importWarnings(structuredOutput, result);
}
return result.toMap();
}
public Map<String, Object> validateRun(String runId,
Map<String, Object> structuredOutput,
Map<String, Object> parseStatus) {
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
validateSchema(structuredOutput, parseStatus, result);
if (structuredOutput != null) {
List<ToolInvocation> invocations = runId == null || runId.isBlank()
? List.of()
: toolInvocationRepository.findByRunIdOrderByIdAsc(runId);
validateInvocationRefs("run_id", runId, invocations, structuredOutput, result);
importWarnings(structuredOutput, result);
}
return result.toMap();
}
public Map<String, Object> pass() {
return new GatekeeperResult(ruleCatalog).toMap();
}
public Map<String, Object> fail(String ruleId, String target, String message) {
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
result.fail(ruleId, target, message, SEVERITY_REJECT);
return result.toMap();
}
private void validateSchema(Map<String, Object> structuredOutput,
Map<String, Object> parseStatus,
GatekeeperResult result) {
String status = parseStatus == null ? "" : String.valueOf(parseStatus.getOrDefault("status", ""));
if (structuredOutput == null) {
if ("valid".equals(status)) {
result.fail(RULE_SCHEMA, "executor_structured_output",
"structured output is missing after valid parse", SEVERITY_LOW_CONFID);
}
return;
}
if (!"executor_evidence_v2".equals(String.valueOf(structuredOutput.get("answer_version")))) {
result.fail(RULE_SCHEMA, "answer_version", "answer_version must be executor_evidence_v2",
SEVERITY_LOW_CONFID);
}
if (structuredOutput.containsKey("diagnosis_summary")) {
result.fail(RULE_SCHEMA, "diagnosis_summary", "diagnosis_summary is removed from executor_evidence_v2",
SEVERITY_REJECT);
}
if (structuredOutput.containsKey("user_facing_answer")) {
result.fail(RULE_SCHEMA, "user_facing_answer", "user_facing_answer is removed from executor_evidence_v2",
SEVERITY_REJECT);
}
Object claimsValue = structuredOutput.get("claims");
if (!(claimsValue instanceof List<?> claims)) {
result.fail(RULE_SCHEMA, "claims", "claims must be an array", SEVERITY_LOW_CONFID);
return;
}
for (int i = 0; i < claims.size(); i++) {
String target = "claims[" + i + "]";
Object claimValue = claims.get(i);
if (!(claimValue instanceof Map<?, ?> claim)) {
result.fail(RULE_SCHEMA, target, "claim must be an object", SEVERITY_LOW_CONFID);
continue;
}
requireString(claim, "claim_id", target, result);
requireString(claim, "claim_type", target, result);
requireString(claim, "claim_text", target, result);
String supportLevel = stringValue(claim.get("support_level"));
if (!"direct".equals(supportLevel) && !"indirect".equals(supportLevel)) {
result.fail(RULE_SCHEMA, target + ".support_level", "support_level must be direct or indirect",
SEVERITY_LOW_CONFID);
}
Object bindings = claim.get("evidence_bindings");
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
result.fail(RULE_EVIDENCE_MISSING, target + ".evidence_bindings",
"claims must include non-empty evidence_bindings", SEVERITY_LOW_CONFID);
}
}
requireArray(structuredOutput, "hypotheses", result);
requireArray(structuredOutput, "recommended_actions", result);
requireArray(structuredOutput, "missing_info", result);
}
private void validateInvocationRefs(String scopeName,
String scopeId,
List<ToolInvocation> invocations,
Map<String, Object> structuredOutput,
GatekeeperResult result) {
if (scopeId == null || scopeId.isBlank()) {
result.fail(RULE_INVOCATION_REF, scopeName, scopeName + " is required to validate source_invocation_id",
SEVERITY_LOW_CONFID);
return;
}
Map<Long, ToolInvocation> validInvocations = invocations.stream()
.filter(invocation -> invocation.getId() != null)
.collect(Collectors.toMap(ToolInvocation::getId, Function.identity(), (left, right) -> left));
Object claimsValue = structuredOutput.get("claims");
if (!(claimsValue instanceof List<?> claims)) {
return;
}
for (int claimIndex = 0; claimIndex < claims.size(); claimIndex++) {
Object claimValue = claims.get(claimIndex);
if (!(claimValue instanceof Map<?, ?> claim)) {
continue;
}
Object bindingsValue = claim.get("evidence_bindings");
if (!(bindingsValue instanceof List<?> bindings)) {
continue;
}
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
String target = "claims[" + claimIndex + "].evidence_bindings[" + bindingIndex + "]";
Object bindingValue = bindings.get(bindingIndex);
if (!(bindingValue instanceof Map<?, ?> binding)) {
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
SEVERITY_LOW_CONFID);
continue;
}
validateEvidenceBinding(binding, validInvocations, target,
claim.get("claim_id"), claim.get("claim_type"), result);
}
}
Object actionsValue = structuredOutput.get("recommended_actions");
if (!(actionsValue instanceof List<?> actions)) {
return;
}
for (int actionIndex = 0; actionIndex < actions.size(); actionIndex++) {
Object actionValue = actions.get(actionIndex);
if (!(actionValue instanceof Map<?, ?> action)) {
continue;
}
Object bindingsValue = action.get("evidence_bindings");
if (!(bindingsValue instanceof List<?> bindings)) {
continue;
}
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
String target = "recommended_actions[" + actionIndex + "].evidence_bindings[" + bindingIndex + "]";
Object bindingValue = bindings.get(bindingIndex);
if (!(bindingValue instanceof Map<?, ?> binding)) {
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
SEVERITY_LOW_CONFID);
continue;
}
validateEvidenceBinding(binding, validInvocations, target, action.get("action_id"), null, result);
}
}
}
private void validateEvidenceBinding(Map<?, ?> binding,
Map<Long, ToolInvocation> validInvocations,
String target,
Object ownerId,
Object ownerType,
GatekeeperResult result) {
Map<String, Object> checked = new LinkedHashMap<>();
checked.put("claim_id", ownerId == null ? "" : String.valueOf(ownerId));
checked.put("tool_name", stringValue(binding.get("tool_name")));
checked.put("source_invocation_id", binding.get("source_invocation_id"));
checked.put("raw_path", stringValue(binding.get("raw_path")));
String claimedToolName = stringValue(binding.get("tool_name"));
if (claimedToolName.isBlank()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "tool_name is required");
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".tool_name", "tool_name is required", SEVERITY_LOW_CONFID);
return;
}
String rawPath = stringValue(binding.get("raw_path"));
if ("negative_observation".equals(stringValue(ownerType)) && !rawPath.isBlank()
&& !"$.no_evidence".equals(rawPath)) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "negative_observation must only bind $.no_evidence references");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"negative_observation must only bind $.no_evidence references", SEVERITY_REJECT);
return;
}
Long id = singleInvocationId(binding);
if (id == null) {
id = uniqueInvocationIdByToolRawPathAndExcerpt(validInvocations, claimedToolName, rawPath,
stringValue(binding.get("evidence_excerpt")));
if (id == null) {
id = uniqueInvocationIdByToolAndRawPath(validInvocations, claimedToolName, rawPath);
}
if (id != null) {
result.warn(Map.of(
"rule", "evidence.invocation_auto_backfill_by_raw_path",
"message", "source_invocation_id was auto-filled from the unique evidence reference candidate",
"tool_name", claimedToolName,
"raw_path", rawPath,
"source_invocation_id", id
));
}
}
if (id == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "source_invocation_id is required");
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".source_invocation_id",
"source_invocation_id is required", SEVERITY_LOW_CONFID);
return;
}
checked.put("source_invocation_id", id);
ToolInvocation invocation = validInvocations.get(id);
if (invocation == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "source_invocation_id not found in current session: " + id);
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target,
"source_invocation_id not found in current session: " + id, SEVERITY_REJECT);
return;
}
if (!Objects.equals(claimedToolName, invocation.getToolName())) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "tool_name does not match invocation " + id + ": expected " + invocation.getToolName());
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".tool_name",
"tool_name does not match invocation " + id + ": expected " + invocation.getToolName(),
SEVERITY_REJECT);
return;
}
if (rawPath.isBlank()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "raw_path is required for precise evidence reference");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"raw_path is required for precise evidence reference", SEVERITY_LOW_CONFID);
return;
}
Map<String, String> refs = evidenceRefsByRawPath(invocation.getRetrievalDetails());
if (refs.isEmpty()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EVIDENCE_MISSING);
checked.put("message", "invocation has no retrieval_details.evidence_refs");
result.checked(checked);
result.fail(RULE_EVIDENCE_MISSING, target,
"invocation has no retrieval_details.evidence_refs", SEVERITY_LOW_CONFID);
return;
}
String matchedText = refs.get(rawPath);
if (matchedText == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "raw_path not found in retrieval_details.evidence_refs");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"raw_path not found in retrieval_details.evidence_refs", SEVERITY_REJECT);
return;
}
checked.put("matched_text", matchedText);
String excerpt = stringValue(binding.get("evidence_excerpt"));
if (normalized(excerpt).length() < 8) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EXCERPT_MISMATCH);
checked.put("message", "evidence_excerpt is too short to compare");
result.checked(checked);
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
"evidence_excerpt is too short to compare", SEVERITY_LOW_CONFID);
return;
}
if (!isExcerptSupported(excerpt, matchedText)) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EXCERPT_MISMATCH);
checked.put("message", "evidence_excerpt is not supported by matched evidence ref text");
result.checked(checked);
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
"evidence_excerpt is not supported by matched evidence ref text", SEVERITY_REJECT);
return;
}
checked.put("status", STATUS_PASS);
result.checked(checked);
}
private Long uniqueInvocationIdByToolAndRawPath(Map<Long, ToolInvocation> validInvocations,
String toolName,
String rawPath) {
if (toolName == null || toolName.isBlank() || rawPath == null || rawPath.isBlank()) {
return null;
}
Long matchedId = null;
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
ToolInvocation invocation = entry.getValue();
if (!Objects.equals(toolName, invocation.getToolName())) {
continue;
}
if (!evidenceRefsByRawPath(invocation.getRetrievalDetails()).containsKey(rawPath)) {
continue;
}
if (matchedId != null) {
return null;
}
matchedId = entry.getKey();
}
return matchedId;
}
private Long uniqueInvocationIdByToolRawPathAndExcerpt(Map<Long, ToolInvocation> validInvocations,
String toolName,
String rawPath,
String excerpt) {
if (toolName == null || toolName.isBlank()
|| rawPath == null || rawPath.isBlank()
|| excerpt == null || excerpt.isBlank()) {
return null;
}
Long matchedId = null;
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
ToolInvocation invocation = entry.getValue();
if (!Objects.equals(toolName, invocation.getToolName())) {
continue;
}
String matchedText = evidenceRefsByRawPath(invocation.getRetrievalDetails()).get(rawPath);
if (matchedText == null || !isBackfillCandidateSupported(rawPath, excerpt, matchedText)) {
continue;
}
if (matchedId != null) {
return null;
}
matchedId = entry.getKey();
}
return matchedId;
}
private boolean isBackfillCandidateSupported(String rawPath, String excerpt, String matchedText) {
if ("$.no_evidence".equals(rawPath)) {
String excerptQuery = semicolonField(excerpt, "query");
String matchedQuery = semicolonField(matchedText, "query");
if (!excerptQuery.isBlank() && !matchedQuery.isBlank()
&& !normalized(excerptQuery).equals(normalized(matchedQuery))) {
return false;
}
}
return isExcerptSupported(excerpt, matchedText);
}
private String semicolonField(String text, String field) {
if (text == null || text.isBlank() || field == null || field.isBlank()) {
return "";
}
String prefix = field + "=";
for (String part : text.split(";")) {
String trimmed = part.trim();
if (trimmed.regionMatches(true, 0, prefix, 0, prefix.length())) {
return trimmed.substring(prefix.length()).trim();
}
}
return "";
}
private Long singleInvocationId(Map<?, ?> binding) {
Long singular = asLong(binding.get("source_invocation_id"));
if (singular != null) {
return singular;
}
Object idsValue = binding.get("source_invocation_ids");
if (!(idsValue instanceof List<?> ids) || ids.size() != 1) {
return null;
}
return asLong(ids.get(0));
}
@SuppressWarnings("unchecked")
private void importWarnings(Map<String, Object> structuredOutput, GatekeeperResult result) {
Object warningsValue = structuredOutput.remove("_gatekeeper_warnings");
if (!(warningsValue instanceof List<?> warnings)) {
return;
}
for (Object warning : warnings) {
if (warning instanceof Map<?, ?> map) {
result.warn((Map<String, Object>) map);
} else if (warning != null) {
result.warn(Map.of("message", String.valueOf(warning)));
}
}
}
private Map<String, String> evidenceRefsByRawPath(String retrievalDetails) {
if (retrievalDetails == null || retrievalDetails.isBlank()) {
return Map.of();
}
try {
Map<String, Object> details = objectMapper.readValue(retrievalDetails, MAP_TYPE);
Object refsValue = details.get("evidence_refs");
if (!(refsValue instanceof List<?> refs)) {
return Map.of();
}
Map<String, String> result = new LinkedHashMap<>();
for (Object refValue : refs) {
if (!(refValue instanceof Map<?, ?> ref)) {
continue;
}
String rawPath = stringValue(ref.get("raw_path"));
String text = stringValue(ref.get("text"));
if (!rawPath.isBlank() && !text.isBlank()) {
result.put(rawPath, text);
}
}
return result;
} catch (Exception ignored) {
return Map.of();
}
}
private boolean isExcerptSupported(String excerpt, String matchedText) {
String normalizedExcerpt = normalized(excerpt);
String normalizedMatched = normalized(matchedText);
if (normalizedMatched.contains(normalizedExcerpt) || normalizedExcerpt.contains(normalizedMatched)) {
return true;
}
Set<String> excerptTokens = tokens(normalizedExcerpt);
if (excerptTokens.isEmpty()) {
return false;
}
Set<String> matchedTokens = tokens(normalizedMatched);
int overlap = 0;
for (String token : excerptTokens) {
if (matchedTokens.contains(token)) {
overlap++;
}
}
double minTokenOverlap = ruleCatalog.doubleParameter(RULE_EXCERPT_MISMATCH,
"min_token_overlap", DEFAULT_MIN_TOKEN_OVERLAP);
return (double) overlap / excerptTokens.size() >= minTokenOverlap;
}
private String normalized(String value) {
return value == null ? "" : value.toLowerCase()
.replaceAll("[\\p{Punct}\\s,。;:、()【】《》“”‘’]+", " ")
.trim();
}
private Set<String> tokens(String text) {
if (text == null || text.isBlank()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (String token : text.split("\\s+")) {
if (token.length() >= 2) {
result.add(token);
}
}
return result;
}
private void requireArray(Map<String, Object> output, String field, GatekeeperResult result) {
if (!(output.get(field) instanceof List<?>)) {
result.fail(RULE_SCHEMA, field, field + " must be an array", SEVERITY_LOW_CONFID);
}
}
private void requireString(Map<?, ?> object, String field, String target, GatekeeperResult result) {
if (stringValue(object.get(field)).isBlank()) {
result.fail(RULE_SCHEMA, target + "." + field, field + " is required", SEVERITY_LOW_CONFID);
}
}
private String stringValue(Object value) {
return value == null ? "" : String.valueOf(value);
}
private Long asLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text) {
try {
return Long.parseLong(text);
} catch (NumberFormatException ignored) {
return null;
}
}
return null;
}
private static final class GatekeeperResult {
private final GatekeeperRuleCatalog ruleCatalog;
private final List<String> failedRules = new ArrayList<>();
private final List<Map<String, Object>> checkedBindings = new ArrayList<>();
private final List<Map<String, Object>> warnings = new ArrayList<>();
private final List<Map<String, Object>> errors = new ArrayList<>();
private String severity = SEVERITY_NONE;
GatekeeperResult(GatekeeperRuleCatalog ruleCatalog) {
this.ruleCatalog = ruleCatalog == null ? GatekeeperRuleCatalog.fallback() : ruleCatalog;
}
void fail(String ruleId, String target, String message, String failureSeverity) {
if (!failedRules.contains(ruleId)) {
failedRules.add(ruleId);
}
if (SEVERITY_REJECT.equals(failureSeverity)) {
severity = SEVERITY_REJECT;
} else if (!SEVERITY_REJECT.equals(severity)) {
severity = SEVERITY_LOW_CONFID;
}
Map<String, Object> error = new LinkedHashMap<>();
error.put("rule_id", ruleId);
error.put("rule", ruleId);
error.put("target", target);
error.put("message", message);
error.put("severity", failureSeverity);
errors.add(error);
}
void checked(Map<String, Object> checked) {
checkedBindings.add(checked);
}
void warn(Map<String, Object> warning) {
warnings.add(new LinkedHashMap<>(warning));
}
Map<String, Object> toMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", failedRules.isEmpty() ? STATUS_PASS : STATUS_FAIL);
result.put("severity", failedRules.isEmpty() ? SEVERITY_NONE : severity);
result.put("rule_set_version", ruleCatalog.version());
result.put("rules", ruleCatalog.auditRules());
result.put("checked_bindings", checkedBindings);
result.put("failed_rules", failedRules);
result.put("warnings", warnings);
result.put("errors", errors);
return result;
}
}
}
@@ -1,142 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Lightweight metadata catalog for deterministic Gatekeeper rules.
*/
public class GatekeeperRuleCatalog {
public static final String DEFAULT_RESOURCE = "gatekeeper/gatekeeper-rules.json";
public static final String FALLBACK_VERSION = "gatekeeper-rules-v1";
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
private final String version;
private final List<RuleMetadata> rules;
public GatekeeperRuleCatalog(String version, List<RuleMetadata> rules) {
this.version = version == null || version.isBlank() ? FALLBACK_VERSION : version;
this.rules = List.copyOf(rules == null ? List.of() : rules);
}
public static GatekeeperRuleCatalog loadDefault(ObjectMapper objectMapper) {
try (InputStream input = GatekeeperRuleCatalog.class.getClassLoader()
.getResourceAsStream(DEFAULT_RESOURCE)) {
if (input == null) {
return fallback();
}
Map<String, Object> root = objectMapper.readValue(input, MAP_TYPE);
String version = stringValue(root.get("version"));
List<RuleMetadata> rules = new ArrayList<>();
Object rulesValue = root.get("rules");
if (rulesValue instanceof List<?> ruleList) {
for (Object ruleValue : ruleList) {
if (ruleValue instanceof Map<?, ?> ruleMap) {
rules.add(RuleMetadata.from(ruleMap));
}
}
}
return new GatekeeperRuleCatalog(version, rules);
} catch (Exception ignored) {
return fallback();
}
}
public static GatekeeperRuleCatalog fallback() {
return new GatekeeperRuleCatalog(FALLBACK_VERSION, List.of(
new RuleMetadata("schema.executor_v2", "Executor output must match executor_evidence_v2 schema",
true, "low_confid", Map.of()),
new RuleMetadata("evidence.invocation_ref", "source_invocation_id must refer to a real current-session tool invocation",
true, "reject", Map.of()),
new RuleMetadata("evidence.raw_path", "raw_path must exist in retrieval_details.evidence_refs",
true, "reject", Map.of()),
new RuleMetadata("evidence.excerpt_mismatch", "evidence_excerpt must be supported by the matched evidence ref text",
true, "reject", Map.of("min_token_overlap", 0.5)),
new RuleMetadata("evidence.missing", "claims must include usable evidence bindings",
true, "low_confid", Map.of())
));
}
public String version() {
return version;
}
public List<Map<String, Object>> auditRules() {
List<Map<String, Object>> result = new ArrayList<>();
for (RuleMetadata rule : rules) {
if (rule.enabled()) {
result.add(rule.toAuditMap());
}
}
return result;
}
public double doubleParameter(String ruleId, String parameterName, double fallback) {
for (RuleMetadata rule : rules) {
if (!rule.id().equals(ruleId) || !rule.enabled()) {
continue;
}
Object value = rule.parameters().get(parameterName);
if (value instanceof Number number) {
return number.doubleValue();
}
if (value instanceof String text) {
try {
return Double.parseDouble(text);
} catch (NumberFormatException ignored) {
return fallback;
}
}
}
return fallback;
}
private static String stringValue(Object value) {
return value == null ? "" : String.valueOf(value);
}
public record RuleMetadata(String id,
String description,
boolean enabled,
String defaultSeverity,
Map<String, Object> parameters) {
static RuleMetadata from(Map<?, ?> raw) {
String id = stringValue(raw.get("id"));
String description = stringValue(raw.get("description"));
boolean enabled = !(raw.get("enabled") instanceof Boolean value) || value;
String defaultSeverity = stringValue(raw.get("default_severity"));
Map<String, Object> parameters = new LinkedHashMap<>();
Object parametersValue = raw.get("parameters");
if (parametersValue instanceof Map<?, ?> parameterMap) {
for (Map.Entry<?, ?> entry : parameterMap.entrySet()) {
if (entry.getKey() != null) {
parameters.put(String.valueOf(entry.getKey()), entry.getValue());
}
}
}
return new RuleMetadata(id, description, enabled, defaultSeverity, parameters);
}
Map<String, Object> toAuditMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("id", id);
result.put("description", description);
result.put("enabled", enabled);
result.put("default_severity", defaultSeverity);
if (!parameters.isEmpty()) {
result.put("parameters", parameters);
}
return result;
}
}
}
@@ -169,8 +169,8 @@ public class KnowledgeIndexService {
}
long elapsedTime = System.currentTimeMillis() - startTime;
log.debug("L0 Hint分析: query={}, matches={}, domains={}, keywords={}, indexSize={}, time={}ms",
query, results.size(), domains, matchedKeywords, knowledgeIndex.size(), elapsedTime);
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
return new L0Hint(
List.copyOf(results),
@@ -27,10 +27,6 @@ public class SelfEvaluationMergeService {
return merge(existingJson, "verifier_evaluation", verifierEvaluation);
}
public String mergeAiOpsRuleEvaluation(String existingJson, Map<String, Object> aiOpsRuleEvaluation) {
return merge(existingJson, "aiops_rule_evaluation", aiOpsRuleEvaluation);
}
private String merge(String existingJson, String key, Map<String, Object> value) {
try {
Map<String, Object> root = parseRoot(existingJson);
@@ -49,8 +45,7 @@ public class SelfEvaluationMergeService {
Map<String, Object> parsed = objectMapper.readValue(existingJson, MAP_TYPE);
if (parsed.containsKey("rule_evaluation")
|| parsed.containsKey("verifier_evaluation")
|| parsed.containsKey("aiops_rule_evaluation")) {
|| parsed.containsKey("verifier_evaluation")) {
return new LinkedHashMap<>(parsed);
}
@@ -1,661 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.ContextPack;
import com.superbiz.agent.dto.EvidenceBlock;
import com.superbiz.agent.dto.KnowledgeQuery;
import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.dto.RerankTrace;
import com.superbiz.agent.dto.RetrievalTrace;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.util.SessionContextHolder;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* Central persistence point for agent evidence tool invocations.
*/
@Slf4j
@Service
public class ToolInvocationRecorder {
private static final int OUTPUT_PREVIEW_LIMIT = 500;
public static final String EVIDENCE_STATUS_SUPPORTED = "supported";
public static final String EVIDENCE_STATUS_NO_EVIDENCE = "no_evidence";
public static final String EVIDENCE_STATUS_DEDUPED = "deduped";
public static final String EVIDENCE_STATUS_FAILED = "failed";
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper;
public ToolInvocationRecorder(ToolInvocationRepository toolInvocationRepository, ObjectMapper objectMapper) {
this.toolInvocationRepository = toolInvocationRepository;
this.objectMapper = objectMapper;
}
public void save(ToolInvocation invocation) {
try {
if (invocation.getSessionId() == null || invocation.getSessionId().isBlank()) {
invocation.setSessionId(SessionContextHolder.getSessionId());
}
if (invocation.getRunId() == null || invocation.getRunId().isBlank()) {
invocation.setRunId(SessionContextHolder.getRunId());
}
if (invocation.getSessionId() == null || invocation.getSessionId().isBlank()) {
log.debug("Skip tool_invocation without sessionId: tool={}", invocation.getToolName());
return;
}
toolInvocationRepository.save(invocation);
} catch (Exception e) {
log.error("保存 tool_invocation 失败: tool={}", invocation.getToolName(), e);
}
}
public void recordEvidenceTool(String toolName,
Map<String, Object> inputParams,
String output,
boolean success,
long startTimeMillis,
String errorMessage,
String topicDomain) {
recordEvidenceTool(toolName, inputParams, output, success, startTimeMillis, errorMessage, topicDomain,
success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED, Map.of());
}
public void recordEvidenceTool(String toolName,
Map<String, Object> inputParams,
String output,
boolean success,
long startTimeMillis,
String errorMessage,
String topicDomain,
String evidenceStatus,
Map<String, Object> extraDetails) {
String outputPreview = preview(output);
Map<String, Object> details = new LinkedHashMap<>();
details.put("trace_id", UUID.randomUUID().toString());
if (topicDomain != null && !topicDomain.isBlank()) {
details.put("retrieved_domains", List.of(topicDomain));
}
details.put("evidence_status", normalizeEvidenceStatus(success, evidenceStatus));
if (extraDetails != null && !extraDetails.isEmpty()) {
details.putAll(extraDetails);
}
List<Map<String, Object>> evidenceRefs = extractEvidenceRefs(toolName, inputParams, output, details);
if (!evidenceRefs.isEmpty()) {
details.put("evidence_refs", evidenceRefs);
}
ToolInvocation invocation = ToolInvocation.builder()
.toolName(toolName)
.inputParams(toJson(inputParams == null ? Map.of() : inputParams))
.outputPreview(outputPreview)
.outputLength(output == null ? 0 : output.length())
.isTruncated(output != null && output.length() > OUTPUT_PREVIEW_LIMIT)
.retrievalDetails(toJson(details))
.durationMs((int) Math.max(0, System.currentTimeMillis() - startTimeMillis))
.success(success)
.errorMessage(errorMessage)
.build();
save(invocation);
}
public void recordLookupKnowledge(LookupKnowledgeRecord record) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("trace_id", UUID.randomUUID().toString());
if (record.l0MatchCount() != null) {
details.put("l0_match_count", record.l0MatchCount());
}
if (record.l0Titles() != null && !record.l0Titles().isEmpty()) {
details.put("l0_titles", record.l0Titles());
}
if (record.l0MatchedKeywords() != null && !record.l0MatchedKeywords().isEmpty()) {
details.put("l0_matched_keywords", record.l0MatchedKeywords());
}
if (record.l0Domains() != null && !record.l0Domains().isEmpty()) {
details.put("l0_domains", record.l0Domains());
}
if (record.l0Entities() != null && !record.l0Entities().isEmpty()) {
details.put("l0_entities", record.l0Entities());
}
if (record.l1TopScore() != null) {
details.put("l1_top_score", record.l1TopScore());
}
if (record.l1TopSimilarity() != null) {
details.put("l1_top_similarity", record.l1TopSimilarity());
}
if (record.l1MatchCount() != null) {
details.put("l1_match_count", record.l1MatchCount());
}
if (record.l1Scores() != null && !record.l1Scores().isEmpty()) {
details.put("l1_scores", record.l1Scores());
}
if (record.evidenceCandidateCount() != null) {
details.put("evidence_candidate_count", record.evidenceCandidateCount());
}
if (record.evidenceBlockCount() != null) {
details.put("evidence_block_count", record.evidenceBlockCount());
}
details.put("evidence_blocks", record.evidenceBlocks() == null ? List.of() : record.evidenceBlocks());
List<Map<String, Object>> evidenceRefs = evidenceRefsFromEvidenceBlocks(record.evidenceBlocks());
if (evidenceRefs.isEmpty()
&& EVIDENCE_STATUS_NO_EVIDENCE.equals(normalizeEvidenceStatus(record.success(), record.evidenceStatus()))) {
Map<String, Object> input = new LinkedHashMap<>();
input.put("query", record.query());
evidenceRefs = List.of(noEvidenceRef("lookup_knowledge", input, null, details));
}
if (!evidenceRefs.isEmpty()) {
details.put("evidence_refs", evidenceRefs);
}
details.put("query_transform", record.queryTransform() == null ? Map.of() : record.queryTransform());
details.put("retrieval_trace", record.retrievalTrace() == null ? Map.of() : record.retrievalTrace());
details.put("context_pack_summary", record.contextPack() == null ? Map.of() : record.contextPack());
details.put("rerank_trace", record.rerankTrace() == null ? Map.of() : record.rerankTrace());
if (record.fallbackReason() != null && !record.fallbackReason().isBlank()) {
details.put("fallback_reason", record.fallbackReason());
}
if (record.relevanceLevel() != null) {
details.put("relevance_level", record.relevanceLevel());
}
if (record.completenessHint() != null) {
details.put("completeness_hint", record.completenessHint());
}
if (record.domain() != null && !record.domain().isBlank()) {
details.put("retrieved_domains", List.of(record.domain()));
}
if (record.dedupReason() != null) {
details.put("dedup_reason", record.dedupReason());
}
details.put("evidence_status", normalizeEvidenceStatus(record.success(), record.evidenceStatus()));
ToolInvocation invocation = ToolInvocation.builder()
.toolName("lookup_knowledge")
.inputParams(toJson(Map.of("query", record.query())))
.outputPreview(preview(record.outputPreview()))
.outputLength(record.outputLength())
.retrievalLayer(record.retrievalLayer())
.l0MatchCount(record.l0MatchCount())
.l1MatchCount(record.l1MatchCount())
.isTruncated(Boolean.TRUE.equals(record.truncated()))
.retrievalDetails(toJson(details))
.relevanceLevel(record.relevanceLevel())
.dedupReason(record.dedupReason())
.durationMs(record.durationMs())
.success(record.success())
.errorMessage(record.errorMessage())
.build();
save(invocation);
}
private String normalizeEvidenceStatus(boolean success, String evidenceStatus) {
if (evidenceStatus != null && !evidenceStatus.isBlank()) {
return evidenceStatus;
}
return success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED;
}
private List<Map<String, Object>> extractEvidenceRefs(String toolName,
Map<String, Object> inputParams,
String output,
Map<String, Object> details) {
if (output == null || output.isBlank()) {
return List.of();
}
if ("lookup_knowledge".equals(toolName)) {
return evidenceRefsFromEvidenceBlocks(asMapList(details.get("evidence_blocks")));
}
try {
JsonNode root = objectMapper.readTree(output);
boolean noEvidence = EVIDENCE_STATUS_NO_EVIDENCE.equals(stringValue(details.get("evidence_status")));
if ("query_metrics".equals(toolName)) {
List<Map<String, Object>> refs = evidenceRefsFromArray(root.path("alerts"), "$.alerts", this::alertText);
return refs.isEmpty() && noEvidence ? List.of(noEvidenceRef(toolName, inputParams, root, details)) : refs;
}
if ("query_logs".equals(toolName)) {
List<Map<String, Object>> refs = evidenceRefsFromArray(root.path("logs"), "$.logs", this::logText);
return refs.isEmpty() && noEvidence ? List.of(noEvidenceRef(toolName, inputParams, root, details)) : refs;
}
} catch (Exception e) {
log.debug("extract evidence_refs failed for tool={}", toolName, e);
}
return List.of();
}
private Map<String, Object> noEvidenceRef(String toolName,
Map<String, Object> inputParams,
JsonNode root,
Map<String, Object> details) {
List<String> parts = new ArrayList<>();
addPart(parts, toolName + " returned no evidence");
addPart(parts, "evidence_status=" + stringValue(details.get("evidence_status")));
String query = firstNonBlank(
root == null ? null : textField(root, "query"),
inputParams == null ? null : inputParams.get("query")
);
if (!query.isBlank()) {
addPart(parts, "query=" + query);
}
String topic = firstNonBlank(
root == null ? null : textField(root, "log_topic"),
inputParams == null ? null : inputParams.get("log_topic"),
details == null ? null : details.get("log_topic"),
details == null ? null : details.get("metric_family")
);
if (!topic.isBlank()) {
addPart(parts, "topic=" + topic);
}
if (root != null && root.has("total")) {
addPart(parts, "total=" + root.path("total").asText());
}
String message = root == null ? "" : textField(root, "message");
if (!message.isBlank()) {
addPart(parts, "message=" + message);
}
return Map.of(
"raw_path", "$.no_evidence",
"text", bounded(String.join("; ", parts), 500)
);
}
private List<Map<String, Object>> evidenceRefsFromArray(JsonNode arrayNode,
String pathPrefix,
java.util.function.Function<JsonNode, String> textExtractor) {
if (!arrayNode.isArray()) {
return List.of();
}
List<Map<String, Object>> refs = new ArrayList<>();
for (int i = 0; i < arrayNode.size(); i++) {
String text = textExtractor.apply(arrayNode.get(i));
if (text == null || text.isBlank()) {
continue;
}
refs.add(Map.of(
"raw_path", pathPrefix + "[" + i + "]",
"text", bounded(text, 500)
));
}
return refs;
}
private String alertText(JsonNode alert) {
List<String> parts = new ArrayList<>();
addPart(parts, textField(alert, "alert_name"));
addPart(parts, textField(alert, "state"));
addPart(parts, textField(alert, "description"));
addPart(parts, "active_at=" + textField(alert, "active_at"));
addPart(parts, "duration=" + textField(alert, "duration"));
return String.join(", ", parts);
}
private String logText(JsonNode log) {
List<String> parts = new ArrayList<>();
addPart(parts, textField(log, "timestamp"));
addPart(parts, textField(log, "level"));
addPart(parts, textField(log, "service"));
addPart(parts, textField(log, "message"));
JsonNode metrics = log.path("metrics");
if (metrics.isObject() && !metrics.isEmpty()) {
addPart(parts, "metrics=" + metrics.toString());
}
return String.join(" ", parts);
}
private List<Map<String, Object>> evidenceRefsFromEvidenceBlocks(List<Map<String, Object>> blocks) {
if (blocks == null || blocks.isEmpty()) {
return List.of();
}
List<Map<String, Object>> refs = new ArrayList<>();
for (int i = 0; i < blocks.size(); i++) {
Map<String, Object> block = blocks.get(i);
String text = firstNonBlank(block.get("content_preview"), block.get("content"),
block.get("title"), block.get("source"));
if (text.isBlank()) {
continue;
}
refs.add(Map.of(
"raw_path", "$.evidence_blocks[" + i + "]",
"text", bounded(text, 500)
));
}
return refs;
}
@SuppressWarnings("unchecked")
private List<Map<String, Object>> asMapList(Object value) {
if (!(value instanceof List<?> list)) {
return List.of();
}
List<Map<String, Object>> result = new ArrayList<>();
for (Object item : list) {
if (item instanceof Map<?, ?> map) {
result.add((Map<String, Object>) map);
}
}
return result;
}
private String textField(JsonNode node, String field) {
JsonNode value = node.path(field);
return value.isMissingNode() || value.isNull() ? "" : value.asText("");
}
private void addPart(List<String> parts, String value) {
if (value != null && !value.isBlank() && !value.endsWith("=")) {
parts.add(value);
}
}
private String firstNonBlank(Object... values) {
for (Object value : values) {
if (value != null && !String.valueOf(value).isBlank()) {
return String.valueOf(value);
}
}
return "";
}
private String stringValue(Object value) {
return value == null ? "" : String.valueOf(value);
}
private String bounded(String value, int limit) {
if (value == null) {
return "";
}
return value.length() <= limit ? value : value.substring(0, limit) + "...";
}
private String preview(String output) {
if (output == null) {
return null;
}
return output.length() <= OUTPUT_PREVIEW_LIMIT
? output
: output.substring(0, OUTPUT_PREVIEW_LIMIT) + "...";
}
private String toJson(Map<String, Object> value) {
try {
return objectMapper.writeValueAsString(value);
} catch (JsonProcessingException e) {
log.debug("tool_invocation JSON 序列化失败", e);
return "{}";
}
}
@Builder
public record LookupKnowledgeRecord(
String query,
String outputPreview,
Integer outputLength,
String retrievalLayer,
Integer l0MatchCount,
Integer l1MatchCount,
Boolean truncated,
String relevanceLevel,
String completenessHint,
String domain,
String dedupReason,
Integer durationMs,
boolean success,
String evidenceStatus,
String errorMessage,
List<String> l0Titles,
List<String> l0MatchedKeywords,
List<String> l0Domains,
List<String> l0Entities,
Double l1TopScore,
Double l1TopSimilarity,
List<Double> l1Scores,
Integer evidenceCandidateCount,
Integer evidenceBlockCount,
List<Map<String, Object>> evidenceBlocks,
Map<String, Object> queryTransform,
Map<String, Object> retrievalTrace,
Map<String, Object> contextPack,
Map<String, Object> rerankTrace,
String fallbackReason
) {
public static LookupKnowledgeRecord from(KnowledgeQuery query,
LookupResult result,
String domain,
String dedupReason,
int durationMs) {
RetrievalTrace trace = result != null ? result.getRetrievalTrace() : null;
String layer = recordedRetrievalLayer(query, trace);
String outputPreview = null;
int outputLength = 0;
boolean truncated = false;
if (result != null && result.getContextPack() != null
&& result.getContextPack().getPackedText() != null) {
outputPreview = result.getContextPack().getPackedText();
outputLength = outputPreview.length();
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
} else if (result != null && result.getEvidenceBlocks() != null
&& !result.getEvidenceBlocks().isEmpty()
&& result.getEvidenceBlocks().get(0).getContent() != null) {
outputPreview = result.getEvidenceBlocks().get(0).getContent();
outputLength = outputPreview.length();
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
}
String evidenceStatus = EVIDENCE_STATUS_SUPPORTED;
if (dedupReason != null) {
evidenceStatus = EVIDENCE_STATUS_DEDUPED;
} else if (result == null || !result.isFound()) {
evidenceStatus = EVIDENCE_STATUS_NO_EVIDENCE;
} else if (trace != null && trace.getEvidenceStatus() != null) {
evidenceStatus = trace.getEvidenceStatus();
}
List<Double> l1Scores = collectAttemptScores(trace);
return LookupKnowledgeRecord.builder()
.query(query != null ? query.getOriginalQuery() : null)
.outputPreview(outputPreview)
.outputLength(outputLength)
.retrievalLayer(layer)
.l0MatchCount(query != null ? query.getL0MatchCount() : null)
.l1MatchCount(totalCandidateCount(trace))
.truncated(truncated)
.relevanceLevel(result != null ? result.getRelevanceLevel() : null)
.completenessHint(result != null ? result.getCompletenessHint() : null)
.domain(domain)
.dedupReason(dedupReason)
.durationMs(durationMs)
.success(true)
.evidenceStatus(evidenceStatus)
.l0Titles(query != null ? query.getL0Titles() : List.of())
.l0MatchedKeywords(query != null ? query.getMatchedKeywords() : List.of())
.l0Domains(query != null ? query.getDomainHints() : List.of())
.l0Entities(query != null ? query.getEntities() : List.of())
.l1TopScore(firstAttemptScore(trace))
.l1TopSimilarity(firstAttemptSimilarity(trace))
.l1Scores(l1Scores)
.evidenceCandidateCount(result != null ? result.getEvidenceCandidateCount() : null)
.evidenceBlockCount(result != null ? result.getEvidenceBlockCount() : null)
.evidenceBlocks(result != null ? summarizeEvidenceBlocks(result.getEvidenceBlocks()) : List.of())
.queryTransform(summarizeQueryTransform(query))
.retrievalTrace(summarizeRetrievalTrace(trace))
.contextPack(summarizeContextPack(result != null ? result.getContextPack() : null))
.rerankTrace(summarizeRerankTrace(result != null ? result.getRerankTrace() : null))
.fallbackReason(trace != null ? trace.getFallbackReason() : null)
.build();
}
private static List<Map<String, Object>> summarizeEvidenceBlocks(List<EvidenceBlock> blocks) {
if (blocks == null || blocks.isEmpty()) {
return List.of();
}
List<Map<String, Object>> summaries = new ArrayList<>();
for (int i = 0; i < Math.min(5, blocks.size()); i++) {
EvidenceBlock block = blocks.get(i);
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("source", block.getSource());
summary.put("title", block.getTitle());
summary.put("breadcrumb", block.getBreadcrumb());
summary.put("retrieval_layer", block.getRetrievalLayer());
summary.put("score", block.getScore());
summary.put("hit_reasons", block.getHitReasons());
String content = block.getContent();
if (content != null) {
summary.put("content_preview", content.length() <= 180 ? content : content.substring(0, 180) + "...");
}
summaries.add(summary);
}
return summaries;
}
private static Map<String, Object> summarizeQueryTransform(KnowledgeQuery query) {
if (query == null) {
return Map.of();
}
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("original_query", query.getOriginalQuery());
summary.put("rewritten_query", query.getRewrittenQuery());
summary.put("category_filter", query.getCategoryFilter());
summary.put("domain_hints", query.getDomainHints());
summary.put("matched_keywords", query.getMatchedKeywords());
summary.put("entities", query.getEntities());
summary.put("l0_titles", query.getL0Titles());
summary.put("l0_match_count", query.getL0MatchCount());
return summary;
}
private static Map<String, Object> summarizeRetrievalTrace(RetrievalTrace trace) {
if (trace == null) {
return Map.of();
}
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("selected_attempt", trace.getSelectedAttempt());
summary.put("fallback_reason", trace.getFallbackReason());
summary.put("evidence_status", trace.getEvidenceStatus());
summary.put("category_filter", trace.getCategoryFilter());
List<Map<String, Object>> attempts = new ArrayList<>();
if (trace.getAttempts() != null) {
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
Map<String, Object> item = new LinkedHashMap<>();
item.put("name", attempt.getName());
item.put("category_filter", attempt.getCategoryFilter());
item.put("candidate_count", attempt.getCandidateCount());
item.put("usable", attempt.getUsable());
item.put("duration_ms", attempt.getDurationMs());
item.put("top_score", attempt.getTopScore());
item.put("top_similarity", attempt.getTopSimilarity());
item.put("error_message", attempt.getErrorMessage());
attempts.add(item);
}
}
summary.put("attempts", attempts);
return summary;
}
private static Map<String, Object> summarizeContextPack(ContextPack contextPack) {
if (contextPack == null) {
return Map.of();
}
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("strategy", contextPack.getStrategy());
summary.put("char_budget", contextPack.getCharBudget());
summary.put("used_chars", contextPack.getUsedChars());
summary.put("included_sources", contextPack.getIncludedSources());
summary.put("omitted_sources", contextPack.getOmittedSources());
return summary;
}
private static Map<String, Object> summarizeRerankTrace(RerankTrace trace) {
if (trace == null || trace.getItems() == null || trace.getItems().isEmpty()) {
return Map.of();
}
List<Map<String, Object>> items = new ArrayList<>();
for (int i = 0; i < Math.min(5, trace.getItems().size()); i++) {
RerankTrace.Item item = trace.getItems().get(i);
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("final_rank", item.getFinalRank());
summary.put("source", item.getSource());
summary.put("base_score", item.getBaseScore());
summary.put("final_score", item.getFinalScore());
summary.put("boost_reasons", item.getBoostReasons());
items.add(summary);
}
return Map.of("items", items);
}
private static Integer totalCandidateCount(RetrievalTrace trace) {
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
return null;
}
int total = 0;
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
if (attempt.getCandidateCount() != null) {
total += attempt.getCandidateCount();
}
}
return total;
}
private static Double firstAttemptScore(RetrievalTrace trace) {
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
return null;
}
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
if (attempt.getTopScore() != null) {
return attempt.getTopScore();
}
}
return null;
}
private static Double firstAttemptSimilarity(RetrievalTrace trace) {
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
return null;
}
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
if (attempt.getTopSimilarity() != null) {
return attempt.getTopSimilarity();
}
}
return null;
}
private static List<Double> collectAttemptScores(RetrievalTrace trace) {
if (trace == null || trace.getAttempts() == null || trace.getAttempts().isEmpty()) {
return List.of();
}
List<Double> scores = new ArrayList<>();
for (RetrievalTrace.Attempt attempt : trace.getAttempts()) {
if (attempt.getTopScore() != null) {
scores.add(attempt.getTopScore());
}
}
return scores;
}
private static String recordedRetrievalLayer(KnowledgeQuery query, RetrievalTrace trace) {
if (trace == null || trace.getSelectedAttempt() == null) {
return null;
}
boolean hasL0 = query != null && query.getL0MatchCount() != null && query.getL0MatchCount() > 0;
boolean hasL1 = trace.getSelectedAttempt().contains("VECTOR");
if (hasL0 && hasL1) {
return "L0+L1";
}
if (hasL1) {
return "L1";
}
return hasL0 ? "L0" : null;
}
}
}
@@ -1,546 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Builds a verifier-facing evidence index from persisted tool invocations.
*/
@Slf4j
@Service
public class ToolTraceSummaryService {
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
private static final Set<String> EVIDENCE_TOOLS = Set.of("lookup_knowledge", "query_logs", "query_metrics", "query_order");
private static final Pattern JSON_STRING_FIELD = Pattern.compile("\"%s\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"");
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
public ToolTraceSummaryService(ToolInvocationRepository toolInvocationRepository) {
this.toolInvocationRepository = toolInvocationRepository;
}
public List<Map<String, Object>> buildVerifierTraceSummary(String sessionId, String executorFinalAnswer) {
if (sessionId == null || sessionId.isBlank()) {
return List.of();
}
List<ToolInvocation> invocations = toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId);
return buildVerifierTraceSummary(invocations, executorFinalAnswer);
}
public List<Map<String, Object>> buildVerifierTraceSummaryForRun(String runId, String executorFinalAnswer) {
if (runId == null || runId.isBlank()) {
return List.of();
}
List<ToolInvocation> invocations = toolInvocationRepository.findByRunIdOrderByIdAsc(runId);
return buildVerifierTraceSummary(invocations, executorFinalAnswer);
}
private List<Map<String, Object>> buildVerifierTraceSummary(List<ToolInvocation> invocations, String executorFinalAnswer) {
if (invocations.isEmpty()) {
return List.of();
}
Map<String, AggregateEntry> grouped = new LinkedHashMap<>();
for (ToolInvocation invocation : invocations) {
if (!EVIDENCE_TOOLS.contains(invocation.getToolName())) {
continue;
}
String topicDomain = extractTopicDomain(invocation);
String key = invocation.getToolName() + "|" + topicDomain;
AggregateEntry entry = grouped.computeIfAbsent(
key,
ignored -> new AggregateEntry(invocation.getToolName(), topicDomain));
entry.absorb(invocation);
}
List<AggregateEntry> rankedEntries = grouped.values().stream()
.sorted(Comparator.comparingInt((AggregateEntry entry) -> entry.relevanceScore(executorFinalAnswer)).reversed())
.limit(8)
.toList();
List<Map<String, Object>> summaries = new ArrayList<>();
for (int i = 0; i < rankedEntries.size(); i++) {
summaries.add(rankedEntries.get(i).toSummary("trace-" + (i + 1)));
}
return summaries;
}
private String extractTopicDomain(ToolInvocation invocation) {
try {
if (invocation.getRetrievalDetails() != null && !invocation.getRetrievalDetails().isBlank()) {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
Object domains = details.get("retrieved_domains");
if (domains instanceof List<?> domainList && !domainList.isEmpty()) {
return String.valueOf(domainList.get(0));
}
}
} catch (Exception e) {
log.debug("Failed to parse retrieved_domains, fallback to general", e);
}
return "general";
}
private String extractInputSummary(ToolInvocation invocation) {
String query = extractQuery(invocation);
if (query != null && !query.isBlank()) {
return "query=" + truncate(query, 120);
}
return invocation.getToolName() + " invoked";
}
private String extractQuery(ToolInvocation invocation) {
try {
if (invocation.getInputParams() != null && !invocation.getInputParams().isBlank()) {
Map<String, Object> params = objectMapper.readValue(invocation.getInputParams(), MAP_TYPE);
Object query = params.get("query");
if (query != null) {
return String.valueOf(query);
}
}
} catch (Exception e) {
log.debug("Failed to parse invocation query", e);
}
return null;
}
private String extractOutputSummary(ToolInvocation invocation, String topicDomain) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
if (invocation.getErrorMessage() != null && !invocation.getErrorMessage().isBlank()) {
return "call failed: " + truncate(invocation.getErrorMessage(), 120);
}
return "no usable evidence returned";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "retrieval skipped because the same document was already used in this session";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)) {
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return "completed without usable evidence: " + truncate(invocation.getOutputPreview(), 120);
}
return "completed without usable evidence";
}
if ("lookup_knowledge".equals(invocation.getToolName())) {
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
String trace = extractLookupTraceSummary(invocation);
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
? truncate(invocation.getOutputPreview(), 240)
: "no preview";
return "matched domain=" + topicDomain + ", relevance=" + relevance + trace + ", preview=" + preview;
}
if ("query_logs".equals(invocation.getToolName())) {
String concrete = extractLogEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if ("query_metrics".equals(invocation.getToolName())) {
String concrete = extractMetricsEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return truncate(invocation.getOutputPreview(), 240);
}
return "evidence retrieved without preview";
}
private String extractLookupTraceSummary(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return "";
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
List<String> sources = new ArrayList<>();
Object evidenceBlocks = details.get("evidence_blocks");
if (evidenceBlocks instanceof List<?> blocks) {
for (Object block : blocks) {
if (block instanceof Map<?, ?> blockMap) {
Object title = blockMap.get("title");
Object source = blockMap.get("source");
String label = title != null ? String.valueOf(title) : String.valueOf(source);
if (label != null && !label.isBlank() && !"null".equals(label)) {
sources.add(label);
}
}
if (sources.size() >= 3) {
break;
}
}
}
Object retrievalTrace = details.get("retrieval_trace");
String selectedAttempt = "";
if (retrievalTrace instanceof Map<?, ?> traceMap && traceMap.get("selected_attempt") != null) {
selectedAttempt = ", selected_attempt=" + traceMap.get("selected_attempt");
}
return (sources.isEmpty() ? "" : ", sources=" + truncate(String.join("|", sources), 180)) + selectedAttempt;
} catch (Exception e) {
log.debug("Failed to parse lookup retrieval details", e);
return "";
}
}
private String extractLogEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
List<String> services = extractJsonStringFields(outputPreview, "service", 3);
List<String> levels = extractJsonStringFields(outputPreview, "level", 3);
List<String> timestamps = extractJsonStringFields(outputPreview, "timestamp", 3);
if (messages.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
for (int i = 0; i < messages.size(); i++) {
String prefix = labelAt(timestamps, i) + labelAt(levels, i) + labelAt(services, i);
rows.add((prefix.isBlank() ? "" : prefix + " ") + truncate(messages.get(i), 220));
}
return "log_evidence: " + truncate(String.join(" | ", rows), 520);
}
private boolean hasOnlyGenericMockLogMessages(String outputPreview) {
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
if (messages.isEmpty()) {
return false;
}
return messages.stream()
.allMatch(message -> message.startsWith("日志消息 #") && message.contains("查询条件:"));
}
private String extractMetricsEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> alertNames = extractJsonStringFields(outputPreview, "alert_name", 5);
List<String> descriptions = extractJsonStringFields(outputPreview, "description", 5);
List<String> services = extractJsonStringFields(outputPreview, "service", 5);
if (alertNames.isEmpty() && descriptions.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
int count = Math.max(alertNames.size(), descriptions.size());
for (int i = 0; i < Math.min(5, count); i++) {
StringBuilder row = new StringBuilder();
if (i < alertNames.size()) {
row.append(alertNames.get(i));
}
if (i < services.size()) {
if (!row.isEmpty()) {
row.append(" ");
}
row.append("service=").append(services.get(i));
}
if (i < descriptions.size()) {
if (!row.isEmpty()) {
row.append(": ");
}
row.append(descriptions.get(i));
}
rows.add(truncate(row.toString(), 220));
}
return "metric_evidence: " + truncate(String.join(" | ", rows), 520);
}
private List<String> extractJsonStringFields(String text, String field, int limit) {
Pattern pattern = Pattern.compile(String.format(JSON_STRING_FIELD.pattern(), Pattern.quote(field)));
Matcher matcher = pattern.matcher(text);
List<String> values = new ArrayList<>();
while (matcher.find() && values.size() < limit) {
values.add(unescapeJsonString(matcher.group(1)));
}
return values;
}
private String unescapeJsonString(String value) {
return value == null ? "" : value
.replace("\\\"", "\"")
.replace("\\\\", "\\")
.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t");
}
private String labelAt(List<String> values, int index) {
if (index >= values.size() || values.get(index) == null || values.get(index).isBlank()) {
return "";
}
return "[" + values.get(index) + "]";
}
private String determineEvidenceLevel(ToolInvocation invocation) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
return "none";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "none";
}
if ("query_logs".equals(invocation.getToolName())
&& hasOnlyGenericMockLogMessages(invocation.getOutputPreview())) {
return "none";
}
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
return "direct";
}
if ("REFERENCE".equals(invocation.getRelevanceLevel())) {
return "indirect";
}
if (EVIDENCE_TOOLS.contains(invocation.getToolName())) {
return "direct";
}
return "none";
}
private String extractEvidenceStatus(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return Boolean.TRUE.equals(invocation.getSuccess())
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
Object evidenceStatus = details.get("evidence_status");
if (evidenceStatus != null) {
return String.valueOf(evidenceStatus);
}
} catch (Exception e) {
log.debug("Failed to parse evidence_status", e);
}
return Boolean.TRUE.equals(invocation.getSuccess())
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
}
private List<String> extractStringList(Object value) {
if (!(value instanceof List<?> list) || list.isEmpty()) {
return List.of();
}
List<String> result = new ArrayList<>();
for (Object item : list) {
if (item != null) {
result.add(String.valueOf(item));
}
}
return result;
}
private List<String> extractSourceDocuments(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return List.of();
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
List<String> paths = extractStringList(details.get("l0_paths"));
if (!paths.isEmpty()) {
return paths;
}
List<String> titles = extractStringList(details.get("l0_titles"));
if (!titles.isEmpty()) {
return titles;
}
} catch (Exception e) {
log.debug("Failed to parse source documents", e);
}
return List.of();
}
private String truncate(String text, int maxLength) {
if (text == null) {
return "";
}
return text.length() <= maxLength ? text : text.substring(0, maxLength) + "...";
}
private final class AggregateEntry {
private final String toolName;
private final String topicDomain;
private String inputSummary;
private String outputSummary;
private boolean success;
private String evidenceLevel = "none";
private int invocationCount;
private int failedCount;
private int noHitCount;
private final List<Long> sourceInvocationIds = new ArrayList<>();
private final LinkedHashSet<String> querySamples = new LinkedHashSet<>();
private final LinkedHashSet<String> retrievalLayers = new LinkedHashSet<>();
private final LinkedHashSet<String> relevanceLevels = new LinkedHashSet<>();
private final LinkedHashSet<String> sourceDocuments = new LinkedHashSet<>();
private AggregateEntry(String toolName, String topicDomain) {
this.toolName = toolName;
this.topicDomain = topicDomain;
}
void absorb(ToolInvocation invocation) {
invocationCount++;
if (invocation.getId() != null) {
sourceInvocationIds.add(invocation.getId());
}
String query = extractQuery(invocation);
if (query != null && !query.isBlank()) {
querySamples.add(query);
}
if (invocation.getRetrievalLayer() != null && !invocation.getRetrievalLayer().isBlank()) {
retrievalLayers.add(invocation.getRetrievalLayer());
}
if (invocation.getRelevanceLevel() != null && !invocation.getRelevanceLevel().isBlank()) {
relevanceLevels.add(invocation.getRelevanceLevel());
}
sourceDocuments.addAll(extractSourceDocuments(invocation));
if (inputSummary == null || inputSummary.isBlank()) {
inputSummary = extractInputSummary(invocation);
}
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
failedCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
noHitCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
if ("none".equals(invocationEvidenceLevel)) {
noHitCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
if (!success || evidenceRank(invocationEvidenceLevel) > evidenceRank(evidenceLevel)) {
success = true;
evidenceLevel = invocationEvidenceLevel;
outputSummary = extractOutputSummary(invocation, topicDomain);
} else if (evidenceRank(invocationEvidenceLevel) == evidenceRank(evidenceLevel)) {
String candidateSummary = extractOutputSummary(invocation, topicDomain);
if (isMoreConcrete(candidateSummary, outputSummary)) {
outputSummary = candidateSummary;
}
}
}
private boolean isMoreConcrete(String candidate, String current) {
return concretenessScore(candidate) > concretenessScore(current);
}
private int concretenessScore(String summary) {
if (summary == null || summary.isBlank()) {
return 0;
}
int score = summary.length() > 160 ? 2 : 1;
if (summary.contains("log_evidence") || summary.contains("metric_evidence")) {
score += 5;
}
if (summary.contains("连接池耗尽") || summary.contains("OutOfMemoryError")
|| summary.contains("扫描行数") || summary.contains("HighMemoryUsage")
|| summary.contains("HighCPUUsage")) {
score += 4;
}
return score;
}
int relevanceScore(String answer) {
int score = success ? 10 : 0;
if ("direct".equals(evidenceLevel)) {
score += 10;
} else if ("indirect".equals(evidenceLevel)) {
score += 5;
}
if (answer != null) {
String normalized = answer.toLowerCase(Locale.ROOT);
if (normalized.contains(topicDomain.toLowerCase(Locale.ROOT))) {
score += 8;
}
if (normalized.contains(toolName.toLowerCase(Locale.ROOT))) {
score += 3;
}
}
return score;
}
Map<String, Object> toSummary(String traceRef) {
String mergedOutput = outputSummary == null ? "no summarized evidence" : outputSummary;
if (invocationCount > 1) {
StringBuilder builder = new StringBuilder(mergedOutput);
builder.append(" (merged ").append(invocationCount).append(" invocations");
if (failedCount > 0) {
builder.append(", failed=").append(failedCount);
}
if (noHitCount > 0) {
builder.append(", no_hit=").append(noHitCount);
}
builder.append(")");
mergedOutput = builder.toString();
}
Map<String, Object> summary = new LinkedHashMap<>();
summary.put("trace_ref", traceRef);
summary.put("tool_name", toolName);
summary.put("success", success);
summary.put("input_summary", inputSummary);
summary.put("output_summary", mergedOutput);
summary.put("evidence_level", evidenceLevel);
summary.put("topic_domain", topicDomain);
summary.put("source_invocation_ids", new ArrayList<>(sourceInvocationIds));
summary.put("invocation_count", invocationCount);
summary.put("failed_invocation_count", failedCount);
summary.put("no_hit_invocation_count", noHitCount);
summary.put("query_samples", new ArrayList<>(querySamples));
summary.put("retrieval_layers", new ArrayList<>(retrievalLayers));
summary.put("relevance_levels", new ArrayList<>(relevanceLevels));
summary.put("source_documents", new ArrayList<>(sourceDocuments));
return summary;
}
private int evidenceRank(String level) {
if ("direct".equals(level)) {
return 2;
}
if ("indirect".equals(level)) {
return 1;
}
return 0;
}
}
}
@@ -78,7 +78,8 @@ public class VectorSearchService {
try {
return searchSimilarDocumentsWithVectorStore(query, topK, category);
} catch (Exception e) {
logger.warn("Spring AI VectorStore retrieval failed, falling back to Milvus SDK: {}", e.getMessage());
logger.warn("Spring AI VectorStore retrieval failed, falling back to Milvus SDK: errorType={}",
e.getClass().getSimpleName());
return searchSimilarDocumentsWithSdk(query, topK, category);
}
}
@@ -89,8 +90,8 @@ public class VectorSearchService {
throw new IllegalStateException("Spring AI VectorStore bean is unavailable");
}
logger.info("Starting Spring AI VectorStore search: query={}, topK={}, category={}, kbScope={}",
query, topK, category, effectiveKbScope());
logger.info("Starting Spring AI VectorStore search: topK={}, category={}, kbScope={}",
topK, category, effectiveKbScope());
SearchRequest.Builder builder = SearchRequest.builder()
.query(query)
.topK(topK)
@@ -119,8 +120,8 @@ public class VectorSearchService {
List<SearchResult> searchSimilarDocumentsWithSdk(String query, int topK, String category) {
try {
logger.info("Starting Milvus SDK search: query={}, topK={}, category={}, kbScope={}",
query, topK, category, effectiveKbScope());
logger.info("Starting Milvus SDK search: topK={}, category={}, kbScope={}",
topK, category, effectiveKbScope());
List<Float> queryVector = embeddingService.generateQueryVector(query);
logger.debug("Query vector generated, dimension={}", queryVector.size());
@@ -166,7 +167,7 @@ public class VectorSearchService {
logger.info("Milvus SDK search complete, candidates={}", results.size());
return results;
} catch (Exception e) {
logger.error("Milvus SDK vector search failed", e);
logger.error("Milvus SDK vector search failed: errorType={}", e.getClass().getSimpleName());
throw new RuntimeException("Vector search failed: " + e.getMessage(), e);
}
}
@@ -1,77 +0,0 @@
package com.superbiz.agent.service.session;
import com.superbiz.agent.domain.model.SessionContext;
import com.superbiz.agent.domain.model.ToolCall;
import java.util.Optional;
/**
* 会话管理器接口
* 负责会话的创建、读取、更新和删除
*/
public interface SessionManager {
/**
* 创建新会话
*
* @param sessionContext 会话上下文
* @param ttlSeconds 会话过期时间(秒)
* @return 会话ID
*/
String createSession(SessionContext sessionContext, long ttlSeconds);
/**
* 获取会话
*
* @param sessionId 会话ID
* @return 会话上下文(如果存在)
*/
Optional<SessionContext> getSession(String sessionId);
/**
* 更新会话
*
* @param sessionContext 会话上下文
*/
void updateSession(SessionContext sessionContext);
/**
* 删除会话
*
* @param sessionId 会话ID
*/
void deleteSession(String sessionId);
/**
* 检查会话是否存在
*
* @param sessionId 会话ID
* @return true 如果会话存在
*/
boolean exists(String sessionId);
/**
* 刷新会话过期时间
*
* @param sessionId 会话ID
* @param ttlSeconds 新的过期时间(秒)
* @return true 如果刷新成功
*/
boolean refreshSession(String sessionId, long ttlSeconds);
/**
* 添加工具调用记录到会话
*
* @param sessionId 会话ID
* @param toolCall 工具调用记录
*/
void addToolCall(String sessionId, ToolCall toolCall);
/**
* 更新会话状态
*
* @param sessionId 会话ID
* @param status 新状态
*/
void updateStatus(String sessionId, String status);
}
@@ -1,147 +0,0 @@
package com.superbiz.agent.service.session.impl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import com.superbiz.agent.domain.model.SessionContext;
import com.superbiz.agent.domain.model.ToolCall;
import com.superbiz.agent.service.session.SessionManager;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
/**
* Redis 会话管理器实现
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class RedisSessionManager implements SessionManager {
private static final String SESSION_KEY_PREFIX = "session:";
private final RedisTemplate<String, Object> redisTemplate;
@Override
public String createSession(SessionContext sessionContext, long ttlSeconds) {
String sessionId = sessionContext.getSessionId();
if (sessionId == null || sessionId.isEmpty()) {
throw new IllegalArgumentException("Session ID cannot be null or empty");
}
sessionContext.setCreatedAt(LocalDateTime.now());
sessionContext.setLastActiveAt(LocalDateTime.now());
sessionContext.setTtl(ttlSeconds);
sessionContext.setStatus("ACTIVE");
String key = buildKey(sessionId);
redisTemplate.opsForValue().set(key, sessionContext, ttlSeconds, TimeUnit.SECONDS);
log.info("创建会话成功: sessionId={}, ttl={}秒", sessionId, ttlSeconds);
return sessionId;
}
@Override
public Optional<SessionContext> getSession(String sessionId) {
String key = buildKey(sessionId);
Object value = redisTemplate.opsForValue().get(key);
if (value instanceof SessionContext) {
SessionContext context = (SessionContext) value;
log.debug("获取会话成功: sessionId={}", sessionId);
return Optional.of(context);
}
log.debug("会话不存在: sessionId={}", sessionId);
return Optional.empty();
}
@Override
public void updateSession(SessionContext sessionContext) {
String sessionId = sessionContext.getSessionId();
String key = buildKey(sessionId);
// 获取剩余 TTL
Long ttl = redisTemplate.getExpire(key, TimeUnit.SECONDS);
if (ttl == null || ttl <= 0) {
ttl = sessionContext.getTtl() != null ? sessionContext.getTtl() : 3600L;
}
sessionContext.setLastActiveAt(LocalDateTime.now());
redisTemplate.opsForValue().set(key, sessionContext, ttl, TimeUnit.SECONDS);
log.debug("更新会话成功: sessionId={}", sessionId);
}
@Override
public void deleteSession(String sessionId) {
String key = buildKey(sessionId);
Boolean deleted = redisTemplate.delete(key);
if (Boolean.TRUE.equals(deleted)) {
log.info("删除会话成功: sessionId={}", sessionId);
} else {
log.warn("删除会话失败,会话可能不存在: sessionId={}", sessionId);
}
}
@Override
public boolean exists(String sessionId) {
String key = buildKey(sessionId);
Boolean exists = redisTemplate.hasKey(key);
return Boolean.TRUE.equals(exists);
}
@Override
public boolean refreshSession(String sessionId, long ttlSeconds) {
String key = buildKey(sessionId);
Boolean refreshed = redisTemplate.expire(key, ttlSeconds, TimeUnit.SECONDS);
if (Boolean.TRUE.equals(refreshed)) {
log.debug("刷新会话过期时间成功: sessionId={}, newTtl={}秒", sessionId, ttlSeconds);
return true;
}
log.warn("刷新会话过期时间失败,会话可能不存在: sessionId={}", sessionId);
return false;
}
@Override
public void addToolCall(String sessionId, ToolCall toolCall) {
Optional<SessionContext> sessionOpt = getSession(sessionId);
if (sessionOpt.isPresent()) {
SessionContext context = sessionOpt.get();
context.addToolCall(toolCall);
updateSession(context);
log.debug("添加工具调用记录成功: sessionId={}, toolName={}", sessionId, toolCall.getToolName());
} else {
log.warn("会话不存在,无法添加工具调用记录: sessionId={}", sessionId);
}
}
@Override
public void updateStatus(String sessionId, String status) {
Optional<SessionContext> sessionOpt = getSession(sessionId);
if (sessionOpt.isPresent()) {
SessionContext context = sessionOpt.get();
context.setStatus(status);
updateSession(context);
log.debug("更新会话状态成功: sessionId={}, status={}", sessionId, status);
} else {
log.warn("会话不存在,无法更新状态: sessionId={}", sessionId);
}
}
/**
* 构建 Redis key
*/
private String buildKey(String sessionId) {
return SESSION_KEY_PREFIX + sessionId;
}
}
@@ -1,7 +1,6 @@
package com.superbiz.agent.tool;
import com.superbiz.agent.dto.ContextPack;
import com.superbiz.agent.dto.EvidenceBlock;
import com.superbiz.agent.dto.EvidencePostprocessResult;
import com.superbiz.agent.dto.KnowledgeQuery;
import com.superbiz.agent.dto.LookupResult;
@@ -11,10 +10,7 @@ import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.service.ToolInvocationRecorder;
import com.superbiz.agent.util.SessionContextHolder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -23,10 +19,9 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* Agent-facing explicit knowledge retrieval tool.
* Harness backend for knowledge retrieval. Agent-facing schema is owned by HarnessEvidenceTools.
*/
@Slf4j
@Component
@@ -56,40 +51,23 @@ public class LookupKnowledgeTool {
@Autowired
private LookupResultAssembler resultAssembler;
@Autowired
private ToolInvocationRecorder toolInvocationRecorder;
@Autowired
private RetrievedDocTracker retrievedDocTracker;
/**
* 查询知识库文档。
*
* @param query 查询关键词
* @return 查询结果
*/
@Tool(description = "查询内部知识库文档,获取错误码定义、接口文档、排障步骤、配置说明和历史案例等背景知识。" +
"工具会执行 query understanding、向量检索、证据去重、轻量重排和上下文打包,并返回 evidenceBlocks、contextPack、retrievalTrace、rerankTrace。" +
"L0 仅用于领域/关键词/实体 hint 和可选 metadata filter,不作为事实证据兜底。" +
"当带 filter 的向量检索低质量或无结果时,会跳过 L0 filter,用原始 query 再做一次无过滤语义检索。" +
"不要用此工具查询实时运行状态;日志、指标、告警等实时事实应使用 query_logs、query_metrics 或告警工具。" +
"参数 query: 查询关键词或描述")
public LookupResult lookupKnowledge(String query) {
String requestId = UUID.randomUUID().toString().substring(0, 8);
long startTime = System.currentTimeMillis();
log.info("========================================");
log.info(">>> [工具调用] lookup_knowledge");
log.info(">>> 参数: query = \"{}\"", query);
log.info(">>> RequestId: {}", requestId);
log.info(">>> metadata: query_chars={}", query == null ? 0 : query.length());
log.info("----------------------------------------");
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
log.info("[QueryTransformer] rewrittenQuery={}, categoryFilter={}, domains={}, keywords={}",
knowledgeQuery.getRewrittenQuery(),
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
knowledgeQuery.getCategoryFilter(),
knowledgeQuery.getDomainHints(),
knowledgeQuery.getMatchedKeywords());
knowledgeQuery.getDomainHints().size(),
knowledgeQuery.getMatchedKeywords().size());
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
String fallbackReason = null;
@@ -134,31 +112,7 @@ public class LookupKnowledgeTool {
fallbackReason, selectedEvidence);
LookupResult result = resultAssembler.assemble(selectedEvidence, contextPack, retrievalTrace);
String sessionId = SessionContextHolder.getSessionId();
String domain = extractDomain(knowledgeQuery);
if (sessionId != null && result.isFound()) {
String docKey = extractDocKey(result);
if (docKey != null && retrievedDocTracker.isAlreadyRetrieved(sessionId, docKey)) {
log.info("[去重] 文档已在本会话中检索过,跳过: {}", docKey);
saveToolInvocation(knowledgeQuery, startTime, result, domain, "doc_retrieved");
LookupResult deduped = resultAssembler.deduped(
result,
retrievedDocTracker.getRetrievedDomains(sessionId),
docKey);
logReturn(deduped, startTime);
return deduped;
}
if (docKey != null) {
retrievedDocTracker.markRetrieved(sessionId, domain, docKey);
}
}
if (sessionId != null) {
result.setRetrievedDomainsThisSession(retrievedDocTracker.getRetrievedDomains(sessionId));
}
saveToolInvocation(knowledgeQuery, startTime, result, domain, null);
logReturn(result, startTime);
logReturn(result);
return result;
}
@@ -188,70 +142,20 @@ public class LookupKnowledgeTool {
.selectedAttempt(selectedAttempt)
.fallbackReason(fallbackReason)
.evidenceStatus(evidence.hasUsableEvidence()
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
: ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE)
? "supported"
: "no_evidence")
.queryHints(queryHints)
.attempts(attempts)
.build();
}
private String extractDomain(KnowledgeQuery query) {
if (query.getCategoryFilter() != null && !query.getCategoryFilter().isBlank()) {
return query.getCategoryFilter();
}
if (query.getDomainHints() != null && !query.getDomainHints().isEmpty()) {
return query.getDomainHints().get(0);
}
return null;
}
private String extractDocKey(LookupResult result) {
if (result.getEvidenceBlocks() == null || result.getEvidenceBlocks().isEmpty()) {
return null;
}
EvidenceBlock first = result.getEvidenceBlocks().get(0);
if (first.getSource() != null && !first.getSource().isBlank()) {
return first.getSource();
}
if (first.getTitle() != null && !first.getTitle().isBlank()) {
return first.getTitle();
}
return null;
}
private void saveToolInvocation(KnowledgeQuery query,
long startTime,
LookupResult result,
String domain,
String dedupReason) {
try {
if (SessionContextHolder.getSessionId() == null) {
return;
}
ToolInvocationRecorder.LookupKnowledgeRecord record =
ToolInvocationRecorder.LookupKnowledgeRecord.from(
query,
result,
domain,
dedupReason,
(int) Math.max(0, System.currentTimeMillis() - startTime)
);
toolInvocationRecorder.recordLookupKnowledge(record);
} catch (Exception e) {
log.error("保存 tool_invocation 失败", e);
}
}
private void logReturn(LookupResult result, long startTime) {
long totalTime = System.currentTimeMillis() - startTime;
private void logReturn(LookupResult result) {
log.info("----------------------------------------");
log.info("<<< [工具返回] lookup_knowledge");
log.info("<<< 结果: found={}, relevanceLevel={}, evidenceBlocks={}, 耗时: {}ms",
log.info("<<< 结果: found={}, relevanceLevel={}, evidenceBlocks={}",
result.isFound(),
result.getRelevanceLevel(),
result.getEvidenceBlockCount(),
totalTime);
log.info("<<< 行动记忆: retrievedDomainsThisSession={}", result.getRetrievedDomainsThisSession());
result.getEvidenceBlockCount());
if (result.getRetrievalTrace() != null) {
log.info("<<< 检索路径: selectedAttempt={}, fallbackReason={}",
result.getRetrievalTrace().getSelectedAttempt(),
@@ -1,89 +0,0 @@
package com.superbiz.agent.tool;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* session 级已召回文档追踪器
* 支持文档级去重 + 域级行动记忆
*
* 数据结构:sessionId → { domain → Set<filePath> }
* - 域级:控制"不要重复查同域",提供行动记忆给 LLM
* - 文档级:控制"不要重复召回同文档"(替代原有单层结构)
*/
@Component
public class RetrievedDocTracker {
// key: sessionId, value: { domain → Set<filePath> }
private final ConcurrentHashMap<String, Map<String, Set<String>>> sessionRetrievals = new ConcurrentHashMap<>();
/**
* 记录一次检索(域级 + 文档级)
*/
public void markRetrieved(String sessionId, String domain, String filePath) {
if (sessionId == null || filePath == null) return;
sessionRetrievals.computeIfAbsent(sessionId,
k -> new ConcurrentHashMap<>())
.computeIfAbsent(domain != null ? domain : "_unknown",
d -> Collections.newSetFromMap(new ConcurrentHashMap<>()))
.add(filePath);
}
/**
* 文档级去重:检查 filePath 是否已在本会话中检索过
*/
public boolean isDocRetrieved(String sessionId, String filePath) {
if (sessionId == null || filePath == null) return false;
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
if (domains == null) return false;
return domains.values().stream().anyMatch(docs -> docs.contains(filePath));
}
/**
* 域级检查:检查 domain 是否已在本会话中检索过
*/
public boolean isDomainRetrieved(String sessionId, String domain) {
if (sessionId == null || domain == null) return false;
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
return domains != null && domains.containsKey(domain);
}
/**
* 获取本次会话已检索的域列表(行动记忆,返回给 LLM)
*/
public List<String> getRetrievedDomains(String sessionId) {
if (sessionId == null) return List.of();
Map<String, Set<String>> domains = sessionRetrievals.get(sessionId);
if (domains == null) return List.of();
return List.copyOf(domains.keySet());
}
/**
* 向后兼容:文档级去重(委托给 isDocRetrieved)
*/
public boolean isAlreadyRetrieved(String sessionId, String filePath) {
return isDocRetrieved(sessionId, filePath);
}
/**
* 向后兼容:旧版 markRetrieved(domain 设为 null,归入 _unknown)
*/
public void markRetrieved(String sessionId, String filePath) {
markRetrieved(sessionId, null, filePath);
}
/**
* 清理会话
*/
public void clearSession(String sessionId) {
if (sessionId == null) return;
sessionRetrievals.remove(sessionId);
}
}
@@ -1,44 +0,0 @@
package com.superbiz.agent.util;
import java.util.List;
/**
* 问题复杂度判断
* 用于决定使用单 Agent 还是多 Agent(Planner + Executor)处理
*/
public class QuestionComplexity {
/** 复杂问题关键词 — 需要多步分析、排查、根因定位 */
private static final List<String> COMPLEX_KEYWORDS = List.of(
"排查", "分析", "为什么", "根因", "调查", "对比", "影响范围",
"原因", "故障", "告警", "诊断", "链路", "流程", "步骤",
"root cause", "troubleshoot", "investigate"
);
/** 极简问题关键词 — 快速回答,无需多 Agent */
private static final List<String> SIMPLE_KEYWORDS = List.of(
"是什么", "查一下", "什么是", "时间", "天气", "定义",
"查", "找", "what is", "define", "time"
);
/**
* 判断是否为复杂问题
*/
public static boolean isComplex(String question) {
if (question == null || question.isBlank()) return false;
String q = question.toLowerCase();
// 复杂关键词匹配 → 多 Agent
for (String kw : COMPLEX_KEYWORDS) {
if (q.contains(kw)) return true;
}
// 简单关键词匹配 → 单 Agent
for (String kw : SIMPLE_KEYWORDS) {
if (q.contains(kw)) return false;
}
// 默认:长问题(>30 字)视为复杂,短问题视为简单
return question.length() > 30;
}
}
@@ -1,44 +0,0 @@
package com.superbiz.agent.util;
/**
* 会话上下文持有者(基于 ThreadLocal)
* <p>
* 用于在同步调用链路中传递 sessionId,兜底 LookupKnowledgeTool 等
* 无法通过 RunnableConfig 获取上下文的组件。
* 优先使用 RunnableConfig.metadata 传递,ThreadLocal 作为同步路径的补充。
* <p>
* 使用规范:
* 1. 调用方在 Agent 执行前调用 setSessionId()
* 2. finally 块中调用 clear()
*/
public class SessionContextHolder {
private static final ThreadLocal<String> SESSION_ID = new ThreadLocal<>();
private static final ThreadLocal<String> RUN_ID = new ThreadLocal<>();
public static void setContext(String sessionId, String runId) {
setSessionId(sessionId);
setRunId(runId);
}
public static void setSessionId(String sessionId) {
SESSION_ID.set(sessionId);
}
public static void setRunId(String runId) {
RUN_ID.set(runId);
}
public static String getSessionId() {
return SESSION_ID.get();
}
public static String getRunId() {
return RUN_ID.get();
}
public static void clear() {
SESSION_ID.remove();
RUN_ID.remove();
}
}
@@ -1,87 +0,0 @@
package com.superbiz.agent.util;
import java.util.List;
import java.util.Map;
/**
* Thread-local verifier context shared across one planner/executor/verifier round.
*/
public final class VerifierContextHolder {
private static final ThreadLocal<String> ORIGINAL_QUERY = new ThreadLocal<>();
private static final ThreadLocal<String> RETRY_CONTEXT = new ThreadLocal<>();
private static final ThreadLocal<String> EXECUTOR_FINAL_ANSWER = new ThreadLocal<>();
private static final ThreadLocal<Map<String, Object>> EXECUTOR_STRUCTURED_OUTPUT = new ThreadLocal<>();
private static final ThreadLocal<Map<String, Object>> EXECUTOR_OUTPUT_PARSE_STATUS = new ThreadLocal<>();
private static final ThreadLocal<List<Map<String, Object>>> TOOL_TRACE_SUMMARY = new ThreadLocal<>();
private static final ThreadLocal<Map<String, Object>> GATEKEEPER_RESULT = new ThreadLocal<>();
private VerifierContextHolder() {
}
public static void setOriginalQuery(String originalQuery) {
ORIGINAL_QUERY.set(originalQuery);
}
public static String getOriginalQuery() {
return ORIGINAL_QUERY.get();
}
public static void setRetryContext(String retryContext) {
RETRY_CONTEXT.set(retryContext);
}
public static String getRetryContext() {
return RETRY_CONTEXT.get();
}
public static void setExecutorFinalAnswer(String executorFinalAnswer) {
EXECUTOR_FINAL_ANSWER.set(executorFinalAnswer);
}
public static String getExecutorFinalAnswer() {
return EXECUTOR_FINAL_ANSWER.get();
}
public static void setExecutorStructuredOutput(Map<String, Object> executorStructuredOutput) {
EXECUTOR_STRUCTURED_OUTPUT.set(executorStructuredOutput);
}
public static Map<String, Object> getExecutorStructuredOutput() {
return EXECUTOR_STRUCTURED_OUTPUT.get();
}
public static void setExecutorOutputParseStatus(Map<String, Object> executorOutputParseStatus) {
EXECUTOR_OUTPUT_PARSE_STATUS.set(executorOutputParseStatus);
}
public static Map<String, Object> getExecutorOutputParseStatus() {
return EXECUTOR_OUTPUT_PARSE_STATUS.get();
}
public static void setToolTraceSummary(List<Map<String, Object>> toolTraceSummary) {
TOOL_TRACE_SUMMARY.set(toolTraceSummary);
}
public static List<Map<String, Object>> getToolTraceSummary() {
return TOOL_TRACE_SUMMARY.get();
}
public static void setGatekeeperResult(Map<String, Object> gatekeeperResult) {
GATEKEEPER_RESULT.set(gatekeeperResult);
}
public static Map<String, Object> getGatekeeperResult() {
return GATEKEEPER_RESULT.get();
}
public static void clear() {
ORIGINAL_QUERY.remove();
RETRY_CONTEXT.remove();
EXECUTOR_FINAL_ANSWER.remove();
EXECUTOR_STRUCTURED_OUTPUT.remove();
EXECUTOR_OUTPUT_PARSE_STATUS.remove();
TOOL_TRACE_SUMMARY.remove();
GATEKEEPER_RESULT.remove();
}
}
+3
View File
@@ -39,6 +39,9 @@ model-routing:
embedding: siliconflow
spring:
config:
import: optional:file:./application-local.yml
# =====================================================
# 数据源配置 (MySQL)
# =====================================================
@@ -0,0 +1,65 @@
-- V013: reconcile databases that applied an earlier V012 without the release contract.
SET @intent_column_exists = (
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'diagnosis_run'
AND column_name = 'intent'
);
SET @intent_ddl = IF(
@intent_column_exists = 0,
'ALTER TABLE diagnosis_run ADD COLUMN intent VARCHAR(32) NULL COMMENT ''SYSTEM_CHAT/KNOWLEDGE_QUERY/DIAGNOSIS'' AFTER agent_flow',
'SELECT 1'
);
PREPARE intent_statement FROM @intent_ddl;
EXECUTE intent_statement;
DEALLOCATE PREPARE intent_statement;
SET @release_outcome_column_exists = (
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'diagnosis_run'
AND column_name = 'release_outcome'
);
SET @release_outcome_ddl = IF(
@release_outcome_column_exists = 0,
'ALTER TABLE diagnosis_run ADD COLUMN release_outcome VARCHAR(16) NULL COMMENT ''SUCCESS/FALLBACK/FAILED/CANCELLED'' AFTER intent',
'SELECT 1'
);
PREPARE release_outcome_statement FROM @release_outcome_ddl;
EXECUTE release_outcome_statement;
DEALLOCATE PREPARE release_outcome_statement;
SET @published_result_column_exists = (
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'diagnosis_run'
AND column_name = 'published_result'
);
SET @published_result_ddl = IF(
@published_result_column_exists = 0,
'ALTER TABLE diagnosis_run ADD COLUMN published_result JSON NULL COMMENT ''Bounded safe PreviousTurn source; no tool ids or raw evidence'' AFTER answer',
'SELECT 1'
);
PREPARE published_result_statement FROM @published_result_ddl;
EXECUTE published_result_statement;
DEALLOCATE PREPARE published_result_statement;
SET @previous_turn_index_exists = (
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'diagnosis_run'
AND index_name = 'idx_diagnosis_run_previous_turn'
);
SET @previous_turn_index_ddl = IF(
@previous_turn_index_exists = 0,
'ALTER TABLE diagnosis_run ADD INDEX idx_diagnosis_run_previous_turn (session_id, intent, release_outcome, created_at, id)',
'SELECT 1'
);
PREPARE previous_turn_index_statement FROM @previous_turn_index_ddl;
EXECUTE previous_turn_index_statement;
DEALLOCATE PREPARE previous_turn_index_statement;
+1 -47
View File
@@ -50,38 +50,6 @@
</rollingPolicy>
</appender>
<!-- 文件输出 - AI Ops 专用日志 -->
<appender name="FILE_AIOPS" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_PATH}/aiops.log</file>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/aiops-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>10MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>15</maxHistory>
</rollingPolicy>
</appender>
<!-- 文件输出 - Chat 对话日志 -->
<appender name="FILE_CHAT" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_PATH}/chat.log</file>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${LOG_PATH}/chat-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>10MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<maxHistory>15</maxHistory>
</rollingPolicy>
</appender>
<!-- 异步输出(提升性能) -->
<appender name="ASYNC_FILE_ALL" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>0</discardingThreshold>
@@ -102,20 +70,6 @@
<appender-ref ref="ASYNC_FILE_ERROR"/>
</logger>
<!-- AI Ops Service 单独记录 -->
<logger name="com.superbiz.agent.service.AiOpsService" level="DEBUG" additivity="false">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE_AIOPS"/>
<appender-ref ref="ASYNC_FILE_ERROR"/>
</logger>
<!-- Chat Service 单独记录 -->
<logger name="com.superbiz.agent.service.ChatService" level="DEBUG" additivity="false">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="FILE_CHAT"/>
<appender-ref ref="ASYNC_FILE_ERROR"/>
</logger>
<!-- Spring AI 日志 -->
<logger name="org.springframework.ai" level="DEBUG"/>
@@ -136,4 +90,4 @@
<appender-ref ref="ASYNC_FILE_ALL"/>
<appender-ref ref="ASYNC_FILE_ERROR"/>
</root>
</configuration>
</configuration>
@@ -1,64 +0,0 @@
你是 Answer Composer。你的职责是把 Verifier 允许输出的结构化材料组织成用户可读的中文答案。
边界约束:
- 你不是诊断 Agent。
- 你不调用工具。
- 你不重新判断根因。
- 你不补充输入中不存在的新事实。
- 你只能使用输入中的 `allowed_claims`、`allowed_hypotheses`、`missing_info`、`recommended_actions`、`rationale`。
- 禁止使用模型经验添加新的服务名、订单号、时间、指标值、错误码、根因或修复理由。
- 只输出一个合法 JSON 对象,不输出 Markdown,不输出代码块,不输出额外说明。
- 当 `allowed_claims` 中存在 `claim_type=negative_observation`,或证据来自 `$.no_evidence` 时,只能表达“当前查询未检索到 / 本次检索未发现匹配证据”。
- 对 `negative_observation` / `$.no_evidence`,禁止表达“问题不存在”“已排除该问题”“确认没有”“日志层面已排除”等过度结论。
## 输入字段
- `original_query`:用户原始问题
- `verdict`:PASS / LOW_CONFID / REJECT
- `allowed_claims`:允许作为已确认事实表达的结论
- `allowed_hypotheses`:允许作为可能方向表达的内容
- `missing_info`:证据缺口
- `recommended_actions`:建议动作
- `rationale`:Verifier 判定理由
## 表达规则
### PASS
- 可以表达确认结论。
- 只能使用 `allowed_claims` 和 `recommended_actions`。
- 只有当 `allowed_claims` 中存在 `claim_type=root_cause` 的 claim 时,才允许表达“根因已确认”。
- 如果 PASS 的 claim 是 `negative_observation`,只能确认“本次查询没有检索到匹配证据”,不能确认“问题不存在”或“已排除”。
### LOW_CONFID
- 必须说明当前证据仍有缺口。
- 必须区分“已确认信息”和“可能方向”。
- 不得把 `allowed_hypotheses` 写成确认结论。
### REJECT
- 必须说明当前无法基于已获取证据生成可靠结论。
- 不得输出根因结论。
- 只能输出已确认信息、证据缺口和下一步建议。
## 输出协议
必须输出且只能输出以下 JSON 结构:
{
"answer_summary": "...",
"recommended_actions": [
{
"action_text": "...",
"reason": "..."
}
],
"user_facing_answer": "..."
}
输出要求:
- `answer_summary` 用 1-2 句话概括当前可表达结论。
- `recommended_actions` 可以为空数组,但字段不能缺失。
- `user_facing_answer` 是最终给用户看的中文答案。
- 不得输出 schema 之外的字段。
@@ -1,241 +0,0 @@
你是任务执行器。执行 Planner 分配给你的具体步骤,并及时反馈结果。
## 职责
- 按步骤执行具体的查询任务。
- 需要外部信息时调用工具,但必须遵守下方的检索约束。
- 严禁凭记忆回答,必须基于本轮工具返回的真实数据。
- 执行完成后,输出严格的证据归因 JSON,供 Verifier 校验。
- 你是证据收集与微观事实提炼器,不是最终答复生成器。
## 角色边界 HARD-GATE
你只负责证据收集与微观事实提炼,只能输出“当前工具证据可以直接支持的观察事实”。
你不是:
- 根因诊断器。
- 修复方案生成器。
- Runbook 转述器。
- 经验推断器。
- 最终用户答复生成器。
除非本轮工具返回中存在直接证据,否则禁止输出:
- 根因确认。
- 修复建议。
- 扩展排查方向。
- 历史经验。
- 通用知识。
- 与用户问题无关的服务、指标、订单、错误码、组件。
## 规则
- 按顺序执行,不可跳过步骤。
- 所有事实性结论必须来自本轮 evidence tools 的返回。
- runbook、skill、历史案例、知识库中的通用模式只能作为排查指导,不能直接写成本次事故的已确认事实。
- 如果检索内容不足以支撑结论,必须显式声明证据不足,严禁补全事故故事。
- 不要使用“通常情况下”“根据经验”“很可能已经发生”“可能是”“推测”“理论上”等无证据推断词来伪装事实。
- 禁止把根因、修复动作或用户明确排除的服务/主题写成 confirmed claim,除非本轮工具证据直接证明。
## 窄范围确认任务 HARD-GATE
如果用户问题包含以下意图,视为窄范围确认任务:
- “只确认”
- “只排查”
- “只看”
- “不要分析”
- “不要扩展”
- “只回答”
- “是否存在”
- “是否真实存在”
- 明确指定某个服务、告警、日志、错误、订单、时间窗口
窄范围确认任务必须遵守:
1. `claims` 只能输出 `observation` 或 `negative_observation`。
2. claim 数量必须是最少必要数量,通常 1 条,最多 2 条。
3. claim 数量限制不限制 `evidence_bindings` 数量;一条 claim 可以绑定多条直接相关证据。
4. 不得把同一观察事实拆成多条 claim。
5. 只能围绕用户明确要求的目标对象和主题输出 claim。
6. 用户明确排除的对象、服务、告警、订单、数据库、连接池、下游依赖,禁止出现在 claim 中。
7. 禁止输出根因类、风险类或建议类 claim,例如 `root_cause`、`risk`、`recommendation`。
8. 如果证据不足,不要补合理化解释;优先写入 `missing_info`。
9. 如果工具没有返回可被 `source_invocation_id + raw_path + evidence_excerpt` 精确引用的证据,不要生成 confirmed claim。
10. 如果工具明确返回 no-hit / no-evidence 结果,可以输出 `negative_observation`,但必须引用 `raw_path="$.no_evidence"`。
11. 窄范围确认任务中,如果精确查询已经返回 `total=0`、`logs=[]`、`alerts=[]` 或 `evidence_status=no_evidence`,不得为了“再试试”而放宽关键词、去掉服务名、扩大服务范围或追加第二次宽泛查询。
窄范围任务的理想输出是:
- 1 条核心 claim。
- 多条直接相关 `evidence_bindings`。
- 必要的 `missing_info`。
同一条工具数组项只能绑定一次。不要为了引用其中多个字段而拆成多个 `evidence_bindings`。
正确:
```json
{
"raw_path": "$.alerts[0]",
"evidence_excerpt": "HighCPUUsage, service=payment-service, state=firing, current=92%, duration=25m"
}
```
错误:
```json
{ "raw_path": "$.alerts[0].alert_name", "evidence_excerpt": "HighCPUUsage" }
{ "raw_path": "$.alerts[0].state", "evidence_excerpt": "firing" }
```
负向观察示例:
```json
{
"claim_type": "negative_observation",
"claim_text": "未检索到 inventory-service 的 HikariCP 连接池耗尽日志。",
"evidence_bindings": [
{
"tool_name": "query_logs",
"source_invocation_id": 123,
"raw_path": "$.no_evidence",
"evidence_excerpt": "query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"
}
]
}
```
`$.no_evidence` 只表示“该工具对当前查询返回无匹配证据”,不能表示“问题不存在”或“根因被排除”。没有实际调用工具时,禁止使用 `$.no_evidence`。
`negative_observation` 的 `evidence_bindings` 只能绑定 `$.no_evidence`。禁止把其它服务的正向日志或告警绑定到同一个 `negative_observation`,即使这些日志可以说明“不是当前服务”。
输出 `negative_observation` 或基于 `$.no_evidence` 的建议动作时,禁止使用“排除”“确认没有”“不存在该问题”“已证明没有”等过度表达;只能使用“当前查询未检索到”“本次检索未发现匹配日志/告警/证据”。
## 工具使用边界
你只能调用回答当前用户问题所必需的工具。
- 问告警状态:优先使用 `query_metrics`。
- 问日志现象:优先使用 `query_logs`。
- 问知识解释或排查步骤:才使用 `lookup_knowledge`。
- Runbook / Skill / 知识库只能帮助决定“查什么”,不能直接作为“当前环境发生了什么”的证据。
- 如果当前工具结果已经足以回答用户问题,不要继续扩展检索。
- 不要为了补全故事而查询用户没有要求的服务、组件或故障类型。
- 对“只确认某日志/告警是否存在”的问题,精确查询返回 no-evidence 后应停止;不要删除服务名、扩大关键词或查询其它服务来寻找对照样本。
## 检索约束
### 1. 判断重复:基于已检索上下文
每次 lookup_knowledge 返回值中包含 `retrievedDomainsThisSession`,
表示本次会话已检索过的知识域。如果当前问题与已检索域语义重叠,
**禁止再次调用 lookup_knowledge**。
### 2. 重复了该怎么办
如果当前想检索的内容与【已检索上下文】语义相似:
- 禁止换关键词重新检索。
- 直接基于已有事实回答。
- 如果信息不足,先明确指出缺少什么具体维度
(如:“缺少 HikariCP 具体配置参数”、“缺少连接池耗尽的日志样例”),
再针对该维度进行一次定向补充检索,而非盲目换词重查。
### 3. 合法出口:允许信息不全时给出有限结论
如果已有信息足以回答核心问题,即使细节不全,也可以给出有限结论。
但你只能把有证据支撑的内容放入 `claims`。
缺失的细节必须写入 `missing_info`,可疑但未证实的方向必须写入 `hypotheses`。
**不查全不会被追责,重复检索或编造细节才会被惩罚。**
### 4. 利用质量信号判断
- relevanceLevel=PRECISE → 信息精准,直接使用,不再检索。
- relevanceLevel=HIGHLY_RELEVANT + 域已在 retrievedDomainsThisSession → 禁止再次调用。
- relevanceLevel=REFERENCE → 先指出缺什么维度,再定向补充一次。
- completenessHint 是知识库给你的天花板信号,信任它。
- lookup_knowledge 的事实证据以 evidenceBlocks 和 contextPack.packedText 为准,不要假设 L0 hint 本身就是事实证据。
- retrievalTrace 只用于理解检索路径和降级原因,不能单独作为诊断事实。
## 证据归因要求
### confirmed claims
`claims` 只允许放已证实或有明确间接支撑的事实断言。
每条 claim 必须带证据绑定。
支持等级:
- `direct`:工具返回中有直接事实。
- `indirect`:工具返回可支撑方向,但没有直接陈述完整结论。
### hypotheses
`hypotheses` 用来放合理怀疑但未被工具证实的方向。
例如:工具只显示连接池耗尽,但没有泄漏日志,则“可能存在连接泄漏”只能是 hypothesis。
### recommended_actions
`recommended_actions` 用来放下一步排查或修复动作。
建议可以来自 runbook/skill,但必须说明 reason,不能写成“已确认根因”。
本期 recommended_actions 只允许证据收集或继续排查动作,不要输出重启、扩容、修改配置等修复动作,除非用户明确要求执行方案。
### missing_info
`missing_info` 用来列出无法确认结论所缺少的具体证据。
## 输出前自检
在输出 JSON 前,逐项检查:
1. 每条 claim 是否直接回答了用户当前问题?
2. 每条 claim 是否都有真实 `evidence_bindings`?
3. 每个 `evidence_excerpt` 是否来自工具返回原文?
4. 是否出现了用户没有要求的服务、告警、订单、数据库、连接池或下游组件?
5. 是否把 Runbook / Skill / 知识库通用内容写成了当前事实?
6. 是否输出了根因、修复动作、风险判断或经验推断?
只要任一项不通过,删除对应 claim,不要解释。
## 最终输出格式(严格契约)
你必须输出且只能输出一个 JSON 对象,不要输出 Markdown,不要输出代码块,不要输出 JSON 之外的解释文字。
所有用户可读内容必须使用中文。
```json
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "root_cause",
"claim_text": "事实断言或有限结论",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "工具返回中的 evidence block id、trace_ref 或可定位标识,可为空",
"tool_name": "lookup_knowledge/query_logs/query_metrics 等 evidence tool",
"source_invocation_id": null,
"raw_path": "$.alerts[0] / $.logs[0] / $.evidence_blocks[0] / $.no_evidence",
"evidence_excerpt": "从工具返回中摘取的原话、指标值、日志片段或关键数据"
}
]
}
],
"hypotheses": [
{
"hypothesis_text": "未被证实但值得排查的方向",
"basis": "它基于哪些已知证据或为什么只是推测",
"needed_evidence": ["需要补充的证据"]
}
],
"recommended_actions": [
{
"action_text": "建议动作",
"reason": "为什么建议做这个动作",
"evidence_bindings": []
}
],
"missing_info": [
"导致无法确认完整根因的证据缺口"
]
}
```
## 输出校验
- `answer_version` 必须是 `executor_evidence_v2`。
- 不得输出 `diagnosis_summary`。
- 不得输出 `user_facing_answer`。
- `claims[*].support_level` 只能是 `direct` 或 `indirect`。
- `claims[*].evidence_bindings` 不能为空。
- `evidence_excerpt` 必须来自工具返回,不允许编造。
- `raw_path` 必须指向工具返回数组中的具体条目:`query_metrics` 使用 `$.alerts[i]`,`query_logs` 使用 `$.logs[i]`,`lookup_knowledge` 使用 `$.evidence_blocks[i]`。
- 当且仅当工具明确返回 no-hit / no-evidence 结果时,允许使用 `$.no_evidence`;对应 `evidence_excerpt` 必须包含工具名、查询目标、`total=0` 或等价无命中信息、`evidence_status=no_evidence`。
- `raw_path` 禁止指向字段级子路径,例如 `$.alerts[0].alert_name`、`$.alerts[0].state`、`$.logs[0].message` 都是非法路径。需要引用多个字段时,仍然只使用对应数组条目的 `raw_path`,并把必要字段合并进同一个 `evidence_excerpt`。
- `source_invocation_id` 只能填写工具返回中明确给出的真实调用 ID;如果工具返回中没有明确 ID,填写 `null` 或省略该字段,禁止编造数字。系统只会在唯一候选工具调用存在时补齐 ID,但不会补齐 `raw_path`。
- 不要再输出 `source_invocation_ids` 作为主要字段;兼容旧字段不作为精确证据引用。
- 如果没有任何可确认事实,`claims` 返回空数组,并在 `missing_info` 说明缺少什么。
- 不要把其它服务、其它历史案例、其它会话的事实迁移为当前会话事实。
@@ -1,31 +0,0 @@
你是智能任务规划器。分析用户的问题,拆解为具体的执行步骤。
## 职责
- 分析用户问题,拆解为可执行的步骤列表
- **你不能调用任何工具**,你的职责是制定计划,不是执行
- 输出 JSON 格式的计划,不输出其他内容
## 输出格式
```json
{
"selected_skill": "匹配的 skill 名称;如果没有匹配则为 null",
"selection_reason": "选择该 skill 的原因;如果没有匹配则说明不使用 skill",
"plan": ["步骤1描述", "步骤2描述", "步骤3描述"],
"reasoning": "规划思路说明"
}
```
## 规则
- 每个步骤应该是一个可以独立执行的任务
- 步骤要具体可操作,不要模糊
- 如果问题需要查知识库,明确在步骤中说明要查什么
- 如果存在 `skill_catalog`,必须先根据 skill 的 name/description 判断是否匹配用户问题
- 如果匹配某个诊断 Skill,必须在顶层 `selected_skill` 填入 skill 名称,并在计划第一步说明 Executor 需要读取该 skill
- Planner 只能选择 skill 元数据,不能调用 `read_skill`,也不能编造 skill 正文内容
## 知识库检索规则
- 制定步骤前,先查看下方 `available_knowledge_domains`(如果存在)
- 根据每个域的 `when_to_retrieve` 判断是否需要检索该域
- 每个域最多安排一次检索步骤;已覆盖的域不要重复安排
- 如果用户问题与某个域无关,不要安排对该域的检索
@@ -1,205 +0,0 @@
你是质量闸 verifier。你的任务是对 Executor 的结构化 claims 做一次基于现有证据的可推导性校验。
边界约束:
- 不做新的检索
- 不做超出输入证据的推理扩写
- 不补充输入中不存在的新事实
- 只输出一个合法 JSON 对象,不输出 Markdown,不输出代码块,不输出额外说明
## 输入字段
- `original_query`:用户原始问题
- `executor_final_answer`:Executor 原始输出,仅用于 debug/fallback;当结构化输出有效时,不得从这里抽取额外确认事实
- `executor_structured_output`:如果 Executor 输出了合法证据归因 JSON,这里会提供解析后的对象。结构包含 `claims`、`hypotheses`、`recommended_actions`、`missing_info`;兼容旧版时可能包含 `user_facing_answer`
- `executor_output_parse_status`:Executor 输出解析状态,包含 `status` 和 `detail`。`status` 可能是 `valid` / `missing` / `malformed`
- `tool_trace_summary`:基于真实工具调用整理出的全局导航和审计索引。它不是唯一证据源;当 claim 有已核验的 `evidence_bindings[].evidence_excerpt` 时,应优先使用 claim-local excerpt 判断可推导性。每一项都带有:
- `trace_ref`
- `tool_name`
- `topic_domain`
- `source_invocation_ids`
- `input_summary`
- `output_summary`
- `evidence_level`
- `gatekeeper_result`:Executor 结构化输出的确定性校验结果,包含 `status`、`severity`、`checked_bindings`、`failed_rules`、`warnings`、`errors`
- `retry_context`:第二轮可选输入;若为空,按首轮处理
## 任务步骤
### 步骤一:确定校验对象
如果 `executor_output_parse_status.status="valid"` 且 `executor_structured_output.claims` 存在:
- 优先逐条校验 `executor_structured_output.claims`
- 每个 claim 至少形成一条 `claim_checks`
- 如果 `gatekeeper_result.severity="none"`,将 claim 的 `evidence_bindings[].evidence_excerpt` 视为已通过代码核验的主证据,判断 `claim_text` 是否能由这些 excerpt 推出
- `tool_trace_summary` 只用于理解工具调用全貌、补充 trace_ref、识别 no_evidence gap,不要求它逐字包含 excerpt 中已经核验过的全部事实
- 不得从 `executor_final_answer` 中抽取不在 claims 里的额外确认事实
如果 structured output 缺失或 malformed:
- 不得通过扫描 `executor_final_answer` 生成 `PASS`
- 输出 `LOW_CONFID`
- `groundedness_score = 0.0`
- `claim_checks = []`
- `facts_checked = []`
- `rationale` 说明结构化输出不可用
### 步骤二:逐条校验 claim
每条 claim check 必须输出:
- `claim_id`
- `claim_text`
- `claim_type`
- `verification`
- `detail`
- `evidence_refs`
`claim_checks[*].verification` 只允许以下六个值:
- `direct_observation`
- `reasonable_inference`
- `overstated`
- `unsupported`
- `external_unknown`
- `contradicted`
结构化 claim 的校验规则:
- claim 有 Gatekeeper 核验通过的 evidence binding,且 `evidence_excerpt` 直接包含该事实 → `direct_observation`
- claim 有 Gatekeeper 核验通过的 evidence binding,excerpt 没有逐字说明但可以合理推出 → `reasonable_inference`
- claim 有部分依据,但写成唯一根因、确认根因或说得过满 → `overstated`
- claim 无法绑定真实 trace、invocation 或 excerpt → `unsupported`
- claim 引入证据外的新服务名、订单号、错误码、指标值、根因 → `external_unknown`
- claim 与工具摘要冲突 → `contradicted`
`hypotheses` 和 `missing_info` 默认不是 confirmed facts,不应因为它们承认缺证据而惩罚。
### 步骤三:补齐 evidence_refs
`evidence_refs` 必须是数组,数组元素必须引用 `tool_trace_summary` 中真实存在的证据项。每个元素包含:
- `trace_ref`
- `tool_name`
- `topic_domain`
- `source_invocation_ids`
- `note`
规则:
- 有证据支撑时,必须引用支撑该事实的证据项
- `no_evidence` 并不等于不引用
- 如果工具确实查过相关方向,但证据不够,仍应引用对应 trace,并在 `note` 里说明“不足以支撑”
- 只有当确实找不到相关 trace 时,`evidence_refs` 才允许为空数组
- 不允许编造不存在的 `trace_ref` 或 `source_invocation_ids`
### 步骤四:生成 verdict
严格使用以下判定矩阵:
0. 若 `gatekeeper_result.status="fail"`
- 不得输出 `PASS`
- 若 `gatekeeper_result.severity="reject"`,输出 `REJECT`
- 若 `gatekeeper_result.severity="low_confid"`,输出 `LOW_CONFID`
- 兼容旧输入:若缺少 `severity` 且 `failed_rules` 包含 `evidence.invocation_ref`,倾向 `REJECT`
- 兼容旧输入:若缺少 `severity` 且不是明显伪造,至少输出 `LOW_CONFID`
1. 若任一关键 claim 为 `contradicted`
- `verdict = "REJECT"`
- `groundedness_score = 0.0`
2. 否则,若所有关键 claims 均为 `direct_observation` 或 `reasonable_inference`
且至少一条关键 claim 为 `direct_observation`
- `verdict = "PASS"`
3. 否则,若不存在 `contradicted`
且存在关键 claim 为 `unsupported` / `external_unknown` / `overstated`
或所有关键 claim 都只有 `reasonable_inference`
- `verdict = "LOW_CONFID"`
### 步骤五:计算 groundedness_score
只统计关键 claim,映射如下:
- `direct_observation = 1.0`
- `reasonable_inference = 0.6`
- `overstated = 0.3`
- `unsupported = 0.0`
- `external_unknown = 0.0`
- `contradicted = 0.0`
规则:
- 若任一关键事实为 `contradicted`,分数固定为 `0.0`
- 否则对关键事实取平均值
- 保留 2 位小数
- 分数范围必须在 `[0.0, 1.0]`
### 步骤六:facts_checked 兼容输出
你必须同时输出 `facts_checked`,用于旧链路兼容。
映射规则:
- `direct_observation` → `direct_evidence`
- `reasonable_inference` → `indirect_support`
- `overstated` → `indirect_support`
- `unsupported` → `no_evidence`
- `external_unknown` → `no_evidence`
- `contradicted` → `contradicted`
`facts_checked[*].fact` 使用 `{claim_id}: {claim_text}`。
### 步骤七:PASS 前覆盖性自检
在输出 `PASS` 前,必须再次检查:
- `claim_checks` 是否覆盖了 `executor_structured_output.claims` 中的全部 claims
- 是否存在 `gatekeeper_result.status="fail"`
- 是否存在 malformed/missing structured output
如有明显遗漏,即使已校验事实都有证据,也不得输出 `PASS`。
### 步骤八:处理 retry_context
若 `retry_context` 不为空:
- 优先检查上一轮缺失证据点是否已补足
- 不要扩展与缺口无关的新事实
- 不要因为存在 `retry_context` 就自动降低 verdict
## 输出协议
必须输出且只能输出以下 JSON 结构:
{
"verdict": "PASS",
"groundedness_score": 0.8,
"critical_fact_count": 2,
"claim_checks": [
{
"claim_id": "claim-1",
"claim_text": "ERR_TIMEOUT 表示请求超时",
"claim_type": "symptom",
"verification": "direct_observation",
"detail": "知识库文档明确给出该错误码定义",
"evidence_refs": [
{
"trace_ref": "trace-1",
"tool_name": "lookup_knowledge",
"topic_domain": "api",
"source_invocation_ids": [101, 104],
"note": "trace-1 的文档摘要直接给出错误码定义"
}
]
}
],
"hypothesis_checks": [],
"facts_checked": [
{
"fact": "claim-1: ERR_TIMEOUT 表示请求超时",
"is_critical": true,
"verification": "direct_evidence",
"detail": "知识库文档明确给出该错误码定义",
"evidence_refs": [
{
"trace_ref": "trace-1",
"tool_name": "lookup_knowledge",
"topic_domain": "api",
"source_invocation_ids": [101, 104],
"note": "trace-1 的文档摘要直接给出错误码定义"
}
]
}
],
"rationale": "所有关键事实均有支撑,且至少一条具有直接证据"
}
输出要求:
- `verdict` 只能是 `PASS` / `LOW_CONFID` / `REJECT`
- `groundedness_score` 必须是 JSON number
- `critical_fact_count` 必须等于关键 claim 的数量;兼容期也应等于 `facts_checked` 中 `is_critical=true` 的数量
- `claim_checks` 可以为空数组,但字段不能缺失
- `facts_checked` 可以为空数组,但字段不能缺失
- 每条 `claim_checks[*]` 都必须包含 `evidence_refs`
- 每条 `facts_checked[*]` 都必须包含 `evidence_refs`
- 不得输出 schema 之外的字段
@@ -1,76 +0,0 @@
# 执行者 System Prompt
## 角色定位
你是诊断流程的**执行者**。你的任务非常明确:严格遵循规划者下发的任务清单,按步骤调用工具完成任务,并输出最终结果。
---
## 核心行为准则
### 1. 严格按步执行
- 规划者下发的是**有序的任务列表**(如 Step 1 → Step 2 → Step 3)
- 你必须按顺序执行,不可跳过、合并或重排步骤
- 每个步骤完成后,记录该步骤的产出,再进入下一步
### 2. 调用工具而不是凭记忆回答
- 所有需要外部信息的地方,都必须调用对应的工具
- 尤其注意:永远不要凭记忆回答错误码含义、接口定义、排障步骤
- 知识库查询:必须通过 `lookup_knowledge` 工具完成
### 3. 工具调用完毕后,必须结合日志、订单数据等证据综合分析
- 不要把工具的返回结果直接当作最终答案输出
- 你的结论必须基于**至少两个独立证据源**(如错误码+日志、接口文档+实际返回值)
---
## 可用工具
### lookup_knowledge(知识库查询)
用于查询内部知识库,获取错误码定义、接口文档、排障步骤等背景信息。
| 参数 | 说明 |
|------|------|
| `query` | 查询关键词或描述。例如:`ERR_TIMEOUT`、`payment-gateway`、`支付为什么失败` |
**内部机制**:
工具内部会先做 query understanding,使用领域/关键词/实体 hint 控制向量检索;如果带 filter 的向量检索低质量,会用原始 query 再执行一次无过滤语义检索。
**返回结果**:包含 `found`(是否找到)、`evidenceBlocks`(结构化证据)、`contextPack.packedText`(可直接引用的证据上下文)、`retrievalTrace`(检索路径)和 `rerankTrace`(重排解释)等字段。
**使用规则**:
- 当你查到了错误码、接口名、服务名时:**必须**调用此工具
- 当需要查排障步骤、业务流程、最佳实践时:**必须**调用此工具
- 对当前结果没有十足把握时:**建议**调用此工具验证
---
## 任务执行规范
### 1. 每个步骤的产出要求
每完成一个工具调用后,你应该:
- 记录工具返回的关键信息
- 将新信息与已有上下文(日志、订单数据等)进行交叉验证
- 输出该步骤的阶段性结论
### 2. 最终输出的报告格式
```yaml
## 诊断结论
**问题根因**:XXX
**证据链**:
1. 订单状态返回错误码 ERR_TIMEOUT
2. 知识库 lookup_knowledge("ERR_TIMEOUT") 返回:支付网关响应超时(>5秒)
3. 日志确认:14:32:15 请求耗时 5.3s,超过 5s 阈值
**建议方案**:
- 临时方案:重试该笔订单
- 长期方案:优化支付网关超时配置,建议提升至 8s
**引用来源**:
- [来源: interfaces/_errors.md]
```
@@ -1,88 +0,0 @@
你是 Planner Agent,同时承担 Replanner 角色,负责:
1. 读取当前输入任务 {input} 以及 Executor 的最近反馈 {executor_feedback}。
2. 分析 Prometheus 告警、日志、内部文档等信息,制定可执行的下一步步骤。
3. 在执行阶段,输出 JSON,包含 decision (PLAN|EXECUTE|FINISH)、step 描述、预期要调用的工具、以及必要的上下文。
4. 调用任何腾讯云日志/主题相关工具时,region 参数必须使用连字符格式(如 ap-guangzhou),若不确定请省略以使用默认值。
5. 严格禁止编造数据,只能引用工具返回的真实内容;如果连续 3 次调用同一工具仍失败或返回空结果,需停止该方向并在最终报告的结论部分说明"无法完成"的原因。
## 最终报告输出要求(CRITICAL)
当 decision=FINISH 时,你必须:
1. **不要输出 JSON 格式**
2. **直接输出完整的 Markdown 格式报告文本**
3. **报告必须严格遵循以下模板**:
```
# 告警分析报告
---
## 📋 活跃告警清单
| 告警名称 | 级别 | 目标服务 | 首次触发时间 | 最新触发时间 | 状态 |
|---------|------|----------|-------------|-------------|------|
| [告警1名称] | [级别] | [服务名] | [时间] | [时间] | 活跃 |
| [告警2名称] | [级别] | [服务名] | [时间] | [时间] | 活跃 |
---
## 🔍 告警根因分析1 - [告警名称]
### 告警详情
- **告警级别**: [级别]
- **受影响服务**: [服务名]
- **持续时间**: [X分钟]
### 症状描述
[根据监控指标描述症状]
### 日志证据
[引用查询到的关键日志]
### 根因结论
[基于证据得出的根本原因]
---
## 🛠️ 处理方案执行1 - [告警名称]
### 已执行的排查步骤
1. [步骤1]
2. [步骤2]
### 处理建议
[给出具体的处理建议]
### 预期效果
[说明预期的效果]
---
## 🔍 告警根因分析2 - [告警名称]
[如果有第2个告警,重复上述格式]
---
## 📊 结论
### 整体评估
[总结所有告警的整体情况]
### 关键发现
- [发现1]
- [发现2]
### 后续建议
1. [建议1]
2. [建议2]
### 风险评估
[评估当前风险等级和影响范围]
```
**重要提醒**:
- 最终输出必须是纯 Markdown 文本,不要包含 JSON 结构
- 不要使用 "finalReport": "..." 这样的格式
- 直接从 "# 告警分析报告" 开始输出
- 所有内容必须基于工具查询的真实数据,严禁编造
- 如果某个步骤失败,在结论中如实说明,不要跳过
@@ -1,10 +0,0 @@
你是 AI Ops Supervisor,负责调度 planner_agent 与 executor_agent:
1. 当需要拆解任务或重新制定策略时,调用 planner_agent。
2. 当 planner_agent 输出 decision=EXECUTE 时,调用 executor_agent 执行第一步。
3. 根据 executor_agent 的反馈,评估是否需要再次调用 planner_agent,直到 decision=FINISH。
4. FINISH 后,确保向最终用户输出完整的《告警分析报告》,格式必须严格为:
告警分析报告\n---\n# 告警处理详情\n## 活跃告警清单\n## 告警根因分析N\n## 处理方案执行N\n## 结论。
5. 若步骤涉及腾讯云日志/主题工具,请确保使用连字符区域 ID(ap-guangzhou 等),或省略 region 以采用默认值。
6. 如果发现 Planner/Executor 在同一方向连续 3 次调用工具仍失败或没有数据,必须终止流程,直接输出"任务无法完成"的报告,明确告知失败原因,严禁凭空编造结果。
只允许在 planner_agent、executor_agent 与 FINISH 之间做出选择。
+3 -388
View File
@@ -99,7 +99,6 @@ class SuperBizAgentApp {
// 侧边栏元素
this.sidebar = document.querySelector('.sidebar');
this.newChatBtn = document.getElementById('newChatBtn');
this.aiOpsSidebarBtn = document.getElementById('aiOpsSidebarBtn');
// 输入区域元素
this.messageInput = document.getElementById('messageInput');
@@ -128,11 +127,6 @@ class SuperBizAgentApp {
this.newChatBtn.addEventListener('click', () => this.newChat());
}
// AI Ops按钮
if (this.aiOpsSidebarBtn) {
this.aiOpsSidebarBtn.addEventListener('click', () => this.triggerAIOps());
}
// 发送消息
if (this.sendButton) {
this.sendButton.addEventListener('click', () => this.sendMessage());
@@ -1081,349 +1075,6 @@ class SuperBizAgentApp {
return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i];
}
// 发送智能运维请求(SSE 流式模式)
async sendAIOpsRequest(loadingMessageElement) {
try {
const response = await fetch(`${this.apiBaseUrl}/ai_ops`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
});
if (!response.ok) {
throw new Error(`HTTP错误: ${response.status}`);
}
let fullResponse = '';
// 处理 SSE 流式响应
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
let currentEvent = 'message'; // 默认事件类型为 message
try {
while (true) {
const { done, value } = await reader.read();
if (done) {
// 流结束,更新最终内容
if (fullResponse) {
console.log('AI Ops 流结束,更新最终内容,长度:', fullResponse.length);
this.updateAIOpsMessage(loadingMessageElement, fullResponse, []);
}
break;
}
// 解码数据并添加到缓冲区
buffer += decoder.decode(value, { stream: true });
// 按行分割处理
const lines = buffer.split('\n');
// 保留最后一行(可能不完整)
buffer = lines.pop() || '';
for (const line of lines) {
if (line.trim() === '') continue;
console.log('[AI Ops SSE] 收到行:', line);
// 解析 SSE 格式
if (line.startsWith('id:')) {
continue;
} else if (line.startsWith('event:')) {
currentEvent = line.substring(6).trim();
console.log('[AI Ops SSE] 事件类型:', currentEvent);
continue;
} else if (line.startsWith('data:')) {
const rawData = line.substring(5).trim();
console.log('[AI Ops SSE] 数据:', rawData, ', currentEvent:', currentEvent);
// 解析可能包含多个JSON对象的数据
const processJsonMessages = (data) => {
const jsonPattern = /\{"type"\s*:\s*"[^"]+"\s*,\s*"data"\s*:\s*(?:"[^"]*"|null)\}/g;
const matches = data.match(jsonPattern);
if (matches && matches.length > 0) {
console.log('[AI Ops SSE] 匹配到', matches.length, '个JSON对象');
for (const jsonStr of matches) {
try {
const sseMessage = JSON.parse(jsonStr);
if (sseMessage.type === 'metadata') {
this.rememberRunMetadata(sseMessage);
} else if (sseMessage.type === 'session') {
this.lastSessionId = sseMessage.data;
this.rememberTraceTarget(sseMessage.data, null);
} else if (sseMessage.type === 'content') {
fullResponse += sseMessage.data || '';
} else if (sseMessage.type === 'done') {
console.log('AI Ops 流完成,最终内容长度:', fullResponse.length);
this.updateAIOpsMessage(loadingMessageElement, fullResponse, []);
return true;
} else if (sseMessage.type === 'error') {
throw new Error(sseMessage.data || '智能运维分析失败');
}
} catch (e) {
if (e.message.includes('智能运维')) throw e;
console.log('[AI Ops SSE] 单个JSON解析失败:', jsonStr);
}
}
if (loadingMessageElement) {
this.updateAIOpsStreamContent(loadingMessageElement, fullResponse);
}
return false;
}
return null;
};
const result = processJsonMessages(rawData);
if (result === true) {
return; // 流结束
} else if (result === null) {
// 没有匹配到多个JSON,尝试单个JSON解析
try {
const sseMessage = JSON.parse(rawData);
if (sseMessage && sseMessage.type) {
if (sseMessage.type === 'metadata') {
this.rememberRunMetadata(sseMessage);
} else if (sseMessage.type === 'session') {
this.lastSessionId = sseMessage.data;
this.rememberTraceTarget(sseMessage.data, null);
} else if (sseMessage.type === 'content') {
fullResponse += sseMessage.data || '';
if (loadingMessageElement) {
this.updateAIOpsStreamContent(loadingMessageElement, fullResponse);
}
} else if (sseMessage.type === 'done') {
console.log('AI Ops 流完成,最终内容长度:', fullResponse.length);
this.updateAIOpsMessage(loadingMessageElement, fullResponse, []);
return;
} else if (sseMessage.type === 'error') {
throw new Error(sseMessage.data || '智能运维分析失败');
}
} else {
fullResponse += rawData;
if (loadingMessageElement) {
this.updateAIOpsStreamContent(loadingMessageElement, fullResponse);
}
}
} catch (e) {
if (e.message.includes('智能运维')) throw e;
// 非 JSON 格式,直接追加原始数据
fullResponse += rawData;
if (loadingMessageElement) {
this.updateAIOpsStreamContent(loadingMessageElement, fullResponse);
}
}
}
}
}
}
} finally {
reader.releaseLock();
}
} catch (error) {
throw error;
}
}
// 更新智能运维流式内容(实时显示)
updateAIOpsStreamContent(messageElement, content) {
if (!messageElement) return;
// 添加 aiops-message 类
messageElement.classList.add('aiops-message');
const messageContentWrapper = messageElement.querySelector('.message-content-wrapper');
if (messageContentWrapper) {
let messageContent = messageContentWrapper.querySelector('.message-content');
if (!messageContent) {
messageContent = document.createElement('div');
messageContent.className = 'message-content';
messageContentWrapper.appendChild(messageContent);
}
// 流式显示时使用纯文本
messageContent.textContent = content;
this.scrollToBottom();
}
}
// 更新智能运维消息(带折叠详情)
updateAIOpsMessage(messageElement, response, details) {
console.log('updateAIOpsMessage 被调用');
console.log('messageElement:', messageElement);
console.log('response:', response);
console.log('response length:', response ? response.length : 0);
console.log('details:', details);
if (!messageElement) {
// 如果没有传入消息元素,则创建新消息
console.log('messageElement 为空,创建新消息');
return this.addAIOpsMessage(response, details);
}
// 添加aiops-message类
messageElement.classList.add('aiops-message');
// 获取消息内容包装器
const messageContentWrapper = messageElement.querySelector('.message-content-wrapper');
if (!messageContentWrapper) {
console.error('未找到 message-content-wrapper');
return;
}
// 清空现有内容(保留消息内容容器)
const messageContent = messageContentWrapper.querySelector('.message-content');
if (!messageContent) {
console.error('未找到 message-content');
return;
}
// 移除加载动画相关的类和内容
messageContent.classList.remove('loading-message-content');
messageContent.textContent = '';
// 移除加载图标(如果存在)
const loadingIcon = messageContent.querySelector('.loading-spinner-icon');
if (loadingIcon) {
loadingIcon.remove();
}
// 详情部分(可折叠)- 先显示
if (details && details.length > 0) {
// 检查是否已存在详情容器
let detailsContainer = messageElement.querySelector('.aiops-details');
if (!detailsContainer) {
detailsContainer = document.createElement('div');
detailsContainer.className = 'aiops-details';
messageContentWrapper.insertBefore(detailsContainer, messageContent);
} else {
// 清空现有详情
detailsContainer.innerHTML = '';
}
const detailsToggle = document.createElement('div');
detailsToggle.className = 'details-toggle';
detailsToggle.innerHTML = `
<svg class="toggle-icon" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9 18L15 12L9 6" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
<span>查看详细步骤 (${details.length}条)</span>
`;
const detailsContent = document.createElement('div');
detailsContent.className = 'details-content';
details.forEach((detail, index) => {
const detailItem = document.createElement('div');
detailItem.className = 'detail-item';
detailItem.innerHTML = `<strong>步骤 ${index + 1}:</strong> ${this.escapeHtml(detail)}`;
detailsContent.appendChild(detailItem);
});
// 点击切换折叠状态
detailsToggle.addEventListener('click', () => {
detailsContent.classList.toggle('expanded');
detailsToggle.classList.toggle('expanded');
});
detailsContainer.appendChild(detailsToggle);
detailsContainer.appendChild(detailsContent);
}
// 更新主要响应内容(使用Markdown渲染)
console.log('开始渲染 Markdown');
const renderedHtml = this.renderMarkdown(response);
console.log('Markdown 渲染完成,HTML 长度:', renderedHtml ? renderedHtml.length : 0);
messageContent.innerHTML = renderedHtml;
console.log('innerHTML 已设置');
// 高亮代码块
this.highlightCodeBlocks(messageContent);
console.log('代码块高亮完成');
// 保存到历史记录
this.currentChatHistory.push({
type: 'assistant',
content: response,
timestamp: new Date().toISOString()
});
this.scrollToBottom();
return messageElement;
}
// 添加智能运维消息(带折叠详情)- 保留用于兼容性
addAIOpsMessage(response, details) {
const messageDiv = document.createElement('div');
messageDiv.className = 'message assistant aiops-message';
// 添加头像图标
const messageAvatar = document.createElement('div');
messageAvatar.className = 'message-avatar';
messageAvatar.innerHTML = `
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M12 2L15.09 8.26L22 9.27L17 14.14L18.18 21.02L12 17.77L5.82 21.02L7 14.14L2 9.27L8.91 8.26L12 2Z" fill="white"/>
</svg>
`;
messageDiv.appendChild(messageAvatar);
// 创建消息内容包装器
const messageContentWrapper = document.createElement('div');
messageContentWrapper.className = 'message-content-wrapper';
// 详情部分(可折叠)- 先显示
if (details && details.length > 0) {
const detailsContainer = document.createElement('div');
detailsContainer.className = 'aiops-details';
const detailsToggle = document.createElement('div');
detailsToggle.className = 'details-toggle';
detailsToggle.innerHTML = `
<svg class="toggle-icon" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M9 18L15 12L9 6" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
<span>查看详细步骤 (${details.length}条)</span>
`;
const detailsContent = document.createElement('div');
detailsContent.className = 'details-content';
details.forEach((detail, index) => {
const detailItem = document.createElement('div');
detailItem.className = 'detail-item';
detailItem.innerHTML = `<strong>步骤 ${index + 1}:</strong> ${this.escapeHtml(detail)}`;
detailsContent.appendChild(detailItem);
});
// 点击切换折叠状态
detailsToggle.addEventListener('click', () => {
detailsContent.classList.toggle('expanded');
detailsToggle.classList.toggle('expanded');
});
detailsContainer.appendChild(detailsToggle);
detailsContainer.appendChild(detailsContent);
messageContentWrapper.appendChild(detailsContainer);
}
// 主要响应内容 - 后显示(使用Markdown渲染)
const messageContent = document.createElement('div');
messageContent.className = 'message-content';
messageContent.innerHTML = this.renderMarkdown(response);
// 高亮代码块
this.highlightCodeBlocks(messageContent);
messageContentWrapper.appendChild(messageContent);
messageDiv.appendChild(messageContentWrapper);
if (this.chatMessages) {
this.chatMessages.appendChild(messageDiv);
this.scrollToBottom();
}
return messageDiv;
}
// HTML转义
escapeHtml(text) {
const div = document.createElement('div');
@@ -1431,52 +1082,16 @@ class SuperBizAgentApp {
return div.innerHTML;
}
// 触发智能运维(点击智能运维按钮时直接调用)
async triggerAIOps() {
if (this.isStreaming) {
this.showNotification('请等待当前操作完成', 'warning');
return;
}
// 新建对话
this.newChat();
// 添加"分析中..."的消息(带旋转动画)
const loadingMessage = this.addLoadingMessage('分析中...');
this.currentAIOpsMessage = loadingMessage; // 保存消息引用用于后续更新
// 设置发送状态
this.isStreaming = true;
this.updateUI();
try {
await this.sendAIOpsRequest(loadingMessage);
} catch (error) {
console.error('智能运维分析失败:', error);
// 更新消息为错误信息
if (loadingMessage) {
const messageContent = loadingMessage.querySelector('.message-content');
if (messageContent) {
messageContent.textContent = '抱歉,智能运维分析时出现错误:' + error.message;
}
}
} finally {
this.isStreaming = false;
this.currentAIOpsMessage = null;
this.updateUI();
}
}
// 显示/隐藏加载遮罩层
showLoadingOverlay(show) {
if (this.loadingOverlay) {
if (show) {
this.loadingOverlay.style.display = 'flex';
// 更新文字为智能运维
// 更新文字为 Chat 处理状态
const loadingText = this.loadingOverlay.querySelector('.loading-text');
const loadingSubtext = this.loadingOverlay.querySelector('.loading-subtext');
if (loadingText) loadingText.textContent = '智能运维分析中,请稍候...';
if (loadingSubtext) loadingSubtext.textContent = '后端正在处理,请耐心等待';
if (loadingText) loadingText.textContent = '正在处理,请稍候...';
if (loadingSubtext) loadingSubtext.textContent = '后端正在准备安全回答';
// 防止页面滚动
document.body.style.overflow = 'hidden';
} else {
+2 -10
View File
@@ -57,14 +57,6 @@
<!-- 主内容区域 -->
<main class="main-content">
<button class="ai-ops-top-btn" id="aiOpsSidebarBtn">
<svg viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M12 2L2 7L12 12L22 7L12 2Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M2 17L12 22L22 17" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M2 12L12 17L22 12" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>
<span>AI Ops</span>
</button>
<div class="chat-container">
<div class="welcome-greeting" id="welcomeGreeting">
<p>你好!我是智能Oncall小助手</p>
@@ -114,8 +106,8 @@
<div id="loadingOverlay" class="loading-overlay">
<div class="loading-content">
<div class="loading-spinner"></div>
<div class="loading-text">智能运维分析中,请稍候...</div>
<div class="loading-subtext">后端正在处理,请耐心等待</div>
<div class="loading-text">正在处理,请稍候...</div>
<div class="loading-subtext">后端正在准备安全回答</div>
</div>
</div>
-126
View File
@@ -210,39 +210,6 @@ body {
position: relative;
}
/* AI Ops按钮 - 右上角 */
.ai-ops-top-btn {
position: absolute;
top: 16px;
right: 16px;
display: flex;
align-items: center;
gap: 8px;
padding: 10px 16px;
background: #ff9800;
border: none;
border-radius: 24px;
cursor: pointer;
color: #ffffff;
font-size: 14px;
font-weight: 500;
transition: all 0.2s;
z-index: 100;
box-shadow: 0 2px 4px rgba(255, 152, 0, 0.3);
}
.ai-ops-top-btn:hover {
background: #fb8c00;
box-shadow: 0 4px 8px rgba(255, 152, 0, 0.4);
transform: translateY(-1px);
}
.ai-ops-top-btn svg {
width: 18px;
height: 18px;
flex-shrink: 0;
}
.chat-container {
flex: 1;
display: flex;
@@ -879,99 +846,6 @@ body {
to { transform: rotate(360deg); }
}
/* 智能运维消息样式 */
.aiops-message {
max-width: 90%;
}
.aiops-message .message-content {
max-width: 100%;
background: rgba(26, 115, 232, 0.05);
border: 1px solid rgba(26, 115, 232, 0.2);
}
/* 详情折叠区域 */
.aiops-details {
margin-bottom: 16px;
width: 100%;
}
.details-toggle {
display: flex;
align-items: center;
gap: 8px;
padding: 12px 16px;
background: rgba(26, 115, 232, 0.05);
border: 1px solid rgba(26, 115, 232, 0.2);
border-radius: 12px;
cursor: pointer;
transition: all 0.3s ease;
font-size: 0.9rem;
color: #1a73e8;
font-weight: 500;
}
.details-toggle:hover {
background: rgba(26, 115, 232, 0.1);
border-color: rgba(26, 115, 232, 0.3);
}
.details-toggle.expanded {
border-bottom-left-radius: 0;
border-bottom-right-radius: 0;
}
.toggle-icon {
width: 20px;
height: 20px;
transition: transform 0.3s ease;
flex-shrink: 0;
}
.details-toggle.expanded .toggle-icon {
transform: rotate(90deg);
}
.details-content {
max-height: 0;
overflow: hidden;
transition: max-height 0.3s ease;
background: rgba(255, 255, 255, 0.95);
border: 1px solid rgba(26, 115, 232, 0.2);
border-top: none;
border-bottom-left-radius: 12px;
border-bottom-right-radius: 12px;
}
.details-content.expanded {
max-height: 2000px;
padding: 16px;
}
.detail-item {
padding: 12px;
margin-bottom: 8px;
background: rgba(26, 115, 232, 0.03);
border-left: 3px solid #1a73e8;
border-radius: 6px;
font-size: 0.85rem;
color: #202124;
line-height: 1.6;
word-wrap: break-word;
word-break: break-word;
white-space: normal;
overflow-wrap: break-word;
}
.detail-item:last-child {
margin-bottom: 0;
}
.detail-item strong {
color: #1a73e8;
font-weight: 600;
}
/* 加载遮罩层样式 */
.loading-overlay {
display: none;
@@ -2,14 +2,12 @@ package com.superbiz.agent.agent.tool;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ToolInvocationRecorder;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
class QueryLogsToolsTest {
@@ -17,7 +15,7 @@ class QueryLogsToolsTest {
@Test
void queryLogsReturnsHikariPositiveMockForOrderService() throws Exception {
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
QueryLogsTools tools = new QueryLogsTools();
ReflectionTestUtils.setField(tools, "mockEnabled", true);
String output = tools.queryLogs("ap-guangzhou", "application-logs",
@@ -33,14 +31,14 @@ class QueryLogsToolsTest {
@Test
void queryLogsReturnsEmptyNoHitForOtherServiceHikariQuery() throws Exception {
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
QueryLogsTools tools = new QueryLogsTools();
ReflectionTestUtils.setField(tools, "mockEnabled", true);
String output = tools.queryLogs("ap-guangzhou", "application-logs",
"inventory-service HikariCP connection pool active=50/50 waiting", 10);
JsonNode root = objectMapper.readTree(output);
assertFalse(root.path("success").asBoolean());
assertTrue(root.path("success").asBoolean());
assertEquals(0, root.path("logs").size());
assertEquals(0, root.path("total").asInt());
assertFalse(root.toString().contains("generic-service"));
@@ -1,18 +1,17 @@
package com.superbiz.agent.config;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
import com.superbiz.agent.harness.application.persistence.ChatRunStore;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.mysql.MysqlSqlValidator;
import com.superbiz.agent.harness.tool.mysql.MysqlSecurityException;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.tool.RetrievedDocTracker;
import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.service.ToolInvocationRecorder;
import com.superbiz.agent.repository.AgentStepRepository;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
@@ -75,8 +74,7 @@ class HarnessChatConfigurationTest {
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
.withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class))
.withBean(ToolInvocationRecorder.class, () -> mock(ToolInvocationRecorder.class))
.withBean(RetrievedDocTracker.class, () -> mock(RetrievedDocTracker.class))
.withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop)
.withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class))
.withBean(ChatRunStore.class, () -> mock(ChatRunStore.class))
.withBean(MysqlToolProperties.class, MysqlToolProperties::new)
@@ -1,17 +0,0 @@
package com.superbiz.agent.controller;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class AiOpsControllerTest {
@Test
void metadataMessageCarriesSessionAndRunId() {
AiOpsController.SseMessage message = AiOpsController.SseMessage.metadata("session-1", "run-1");
assertEquals("metadata", message.getType());
assertEquals("session-1", message.getSessionId());
assertEquals("run-1", message.getRunId());
}
}
@@ -1,140 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisEvalBaselineDiffTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final DiagnosisEvalBaselineDiffer differ = new DiagnosisEvalBaselineDiffer();
@Test
void compareReportsDetectsAggregateAndCaseRegressions() throws Exception {
DiagnosisEvalReport baseline = readBaselineReport();
DiagnosisEvalReport current = readBaselineReport();
degradeRedisCase(current);
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
assertTrue(diff.isHasRegression());
assertEquals(6, diff.getRegressionCount());
assertEquals(2, diff.getChangedCount());
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "passRate"));
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "averageToolCallCount"));
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "passed"));
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "verdict"));
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "matchedKeywordCount"));
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "evidenceCoverage.query_logs"));
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.LOW_CONFID"));
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.REJECT"));
}
@Test
void compareReportsDetectsMissingAndNewCases() throws Exception {
DiagnosisEvalReport baseline = readBaselineReport();
DiagnosisEvalReport current = readBaselineReport();
DiagnosisEvalResult removed = current.getResults().remove(0);
current.getResults().add(DiagnosisEvalResult.builder()
.caseId("new-case")
.title("New case")
.passed(true)
.failedChecks(List.of())
.verdict("PASS")
.matchedKeywordCount(1)
.requiredKeywordCount(1)
.evidenceCoverage(new LinkedHashMap<>())
.toolCallCount(1)
.durationMs(1000)
.build());
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
assertTrue(hasItem(diff, "REGRESSION", "case", removed.getCaseId(), "casePresence"));
assertTrue(hasItem(diff, "CHANGED", "case", "new-case", "casePresence"));
}
@Test
void compareSameReportHasNoDiff() throws Exception {
DiagnosisEvalReport baseline = readBaselineReport();
DiagnosisEvalDiffReport diff = differ.compare(baseline, readBaselineReport());
assertFalse(diff.isHasRegression());
assertEquals(0, diff.getRegressionCount());
assertTrue(diff.getItems().isEmpty());
}
@Test
void writerOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
DiagnosisEvalReport baseline = readBaselineReport();
DiagnosisEvalReport current = readBaselineReport();
degradeRedisCase(current);
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
DiagnosisEvalDiffReportWriter writer = new DiagnosisEvalDiffReportWriter(objectMapper);
Path json = tempDir.resolve("baseline-diff.json");
Path markdown = tempDir.resolve("baseline-diff.md");
writer.writeJson(diff, json);
writer.writeMarkdown(diff, markdown);
assertTrue(Files.exists(json));
assertTrue(Files.readString(json).contains("\"hasRegression\" : true"));
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Baseline Diff"));
assertTrue(Files.readString(markdown).contains("redis-timeout"));
}
private DiagnosisEvalReport readBaselineReport() throws Exception {
return objectMapper.readValue(Path.of("mvp/eval/reports/baseline-report.json").toFile(),
DiagnosisEvalReport.class);
}
private void degradeRedisCase(DiagnosisEvalReport report) {
report.setPassedCases(11);
report.setPassRate(11.0 / 12.0);
report.setAverageToolCallCount(3.0);
report.setAverageDurationMs(38500.0);
report.setVerdictDistribution(new LinkedHashMap<>());
report.getVerdictDistribution().put("PASS", 5L);
report.getVerdictDistribution().put("LOW_CONFID", 5L);
report.getVerdictDistribution().put("REJECT", 2L);
DiagnosisEvalResult redis = result(report, "redis-timeout");
redis.setPassed(false);
redis.setFailedChecks(new ArrayList<>(List.of("missing required evidence tool: query_logs")));
redis.setVerdict("REJECT");
redis.setMatchedKeywordCount(1);
redis.getEvidenceCoverage().put("query_logs", false);
redis.setToolCallCount(1);
redis.setDurationMs(36000);
}
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
return report.getResults().stream()
.filter(item -> caseId.equals(item.getCaseId()))
.findFirst()
.orElseThrow();
}
private boolean hasItem(DiagnosisEvalDiffReport diff,
String type,
String scope,
String caseId,
String metric) {
return diff.getItems().stream().anyMatch(item ->
type.equals(item.getType())
&& scope.equals(item.getScope())
&& java.util.Objects.equals(caseId, item.getCaseId())
&& metric.equals(item.getMetric()));
}
}
@@ -1,391 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.DiagnosisTraceResponse;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisTraceEvaluatorTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
@Test
void evaluateFixtureReportsFullBaseline() {
List<DiagnosisEvalCase> cases = readCases();
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(12, report.getTotalCases());
assertEquals(12, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
assertTrue(narrowHighCpu.isPassed());
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
assertTrue(promptGatekeeperAudit.isPassed());
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
assertTrue(hikariNoEvidence.isPassed());
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
assertEquals("pass", hikariNoEvidence.getGatekeeperStatus());
DiagnosisEvalResult payment = result(report, "payment-timeout");
assertTrue(payment.isPassed());
assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
assertTrue(payment.getEvidenceCoverage().get("query_logs"));
assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
DiagnosisEvalResult redis = result(report, "redis-timeout");
assertTrue(redis.isPassed());
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
DiagnosisEvalResult fabricatedInvocation = result(report, "gatekeeper-fabricated-invocation");
assertTrue(fabricatedInvocation.isPassed());
assertEquals("fail", fabricatedInvocation.getGatekeeperStatus());
assertEquals("valid", fabricatedInvocation.getComposerStatus());
assertEquals(1, fabricatedInvocation.getClaimCheckCount());
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
assertTrue(composerFallback.isPassed());
assertEquals("composer_malformed", composerFallback.getComposerStatus());
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
assertTrue(auditMetadataLowConfid.isPassed());
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
}
@Test
void everyFixedCaseReferencesExistingFixture() {
for (DiagnosisEvalCase evalCase : readCases()) {
Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture());
assertTrue(Files.exists(fixture), "missing fixture: " + fixture);
}
}
@Test
void evaluateRejectRequiresDegradedOutput() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("reject-case")
.title("Reject case")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("REJECT"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("EXECUTOR_FINAL_ANSWER")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of("verdict", "REJECT")))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
}
@Test
void evaluateFailsWhenStructuredConfirmedClaimHasNoEvidenceBindings() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("structured-claim-case")
.title("Structured claim case")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("LOW_CONFID", "PASS"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("以下结论基于当前已获取证据,仍存在部分证据缺口,请谨慎参考。")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "LOW_CONFID",
"executor_structured_output", java.util.Map.of(
"claims", java.util.List.of(java.util.Map.of(
"claim_id", "claim-unsupported",
"claim_text", "OOM 导致连接泄漏",
"support_level", "direct",
"evidence_bindings", java.util.List.of()
))
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"executor confirmed claim missing evidence bindings: claim-unsupported"));
}
@Test
void evaluateFailsWhenGatekeeperFailStillPassesVerifier() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-pass-leak")
.title("Gatekeeper pass leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS", "LOW_CONFID", "REJECT"))
.requireV2AuditClosure(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of("status", "fail"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "unsupported",
"detail", "evidence ref invalid"
)),
"composer_output", java.util.Map.of("status", "valid")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict"));
}
@Test
void evaluateFailsWhenGatekeeperRuleSetVersionMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("rule-version")
.title("Rule version")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.expectedGatekeeperRuleSetVersion("gatekeeper-rules-v1")
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "old-rules"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"gatekeeper rule set version not expected: old-rules"));
}
@Test
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("prompt-audit")
.title("Prompt audit")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requirePromptAudit(true)
.expectedPromptAuditVersion("chat-prompts-v1")
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"prompt_audit", java.util.Map.of(
"version", "old-prompts",
"prompts", java.util.List.of(java.util.Map.of(
"name", "chat_executor",
"version", "chat-executor-v1"
))
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"prompt audit version not expected: old-prompts"));
assertTrue(result.getFailedChecks().contains(
"prompt version not expected: chat_executor=chat-executor-v1"));
}
@Test
void evaluateFailsWhenGatekeeperRulesAreMissing() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-rules")
.title("Gatekeeper rules")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.requireGatekeeperRules(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "gatekeeper-rules-v1"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("unsupported-leak")
.title("Unsupported leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("LOW_CONFID"))
.requireV2AuditClosure(true)
.forbiddenConfirmedClaimKeywords(List.of("主库故障"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("已经确认主库故障。")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "LOW_CONFID",
"gatekeeper_result", java.util.Map.of("status", "pass"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "unsupported",
"detail", "missing database evidence"
)),
"composer_output", java.util.Map.of("status", "valid")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"answer contains forbidden confirmed claim keyword: 主库故障"));
}
@Test
void evaluateFailsWhenFinalAnswerLeaksRawExecutorMarker() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("raw-json-leak")
.title("Raw json leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("LOW_CONFID"))
.requireV2AuditClosure(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("answer_version=executor_evidence_v2")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "LOW_CONFID",
"gatekeeper_result", java.util.Map.of("status", "pass"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "direct_observation",
"detail", "log evidence"
)),
"composer_output", java.util.Map.of("status", "composer_malformed")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: executor_evidence_v2"));
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: answer_version"));
}
@Test
void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper);
Path json = tempDir.resolve("eval-report.json");
Path markdown = tempDir.resolve("eval-report.md");
writer.writeJson(report, json);
writer.writeMarkdown(report, markdown);
assertTrue(Files.exists(json));
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
assertTrue(Files.readString(markdown).contains("payment-timeout"));
assertEquals(
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))),
comparableReportText(Files.readString(json)));
assertEquals(
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))),
comparableReportText(Files.readString(markdown)));
}
private List<DiagnosisEvalCase> readCases() {
try {
return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json"));
} catch (Exception e) {
throw new AssertionError(e);
}
}
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
return report.getResults().stream()
.filter(item -> caseId.equals(item.getCaseId()))
.findFirst()
.orElseThrow();
}
private String comparableReportText(String value) {
return value.replace("\r\n", "\n").stripTrailing();
}
}
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.superbiz.agent.Main;
import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.util.SessionContextHolder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
@@ -56,13 +55,7 @@ class RagLookupSnapshotGeneratorTest {
String caseId = requiredText(testCase, "caseId");
String query = requiredText(testCase, "query");
SessionContextHolder.clear();
LookupResult lookupResult;
try {
lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
} finally {
SessionContextHolder.clear();
}
LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
ObjectNode fixture = objectMapper.createObjectNode();
fixture.put("caseId", caseId);
@@ -0,0 +1,71 @@
package com.superbiz.agent.harness.audit;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.agent.DiagnosisAgentFactory;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class HarnessAgentAuditHookTest {
@Test
void persistsExactIdentityAndMetadataWithoutContentOrArguments() {
AgentStepRepository repository = mock(AgentStepRepository.class);
AgentStep persisted = AgentStep.builder().id(7L).build();
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
when(repository.findById(7L)).thenReturn(Optional.of(persisted));
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME);
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", "session-audit")
.addMetadata("runId", "run-audit")
.build();
hook.beforeModel(List.of(new UserMessage("secret-query")), config);
AssistantMessage response = AssistantMessage.builder()
.content("secret-model-output")
.toolCalls(List.of(new AssistantMessage.ToolCall(
"call-1", "function", "query_logs", "{\"query\":\"secret-argument\"}")))
.build();
hook.afterModel(List.of(response), config);
ArgumentCaptor<AgentStep> captor = ArgumentCaptor.forClass(AgentStep.class);
verify(repository, org.mockito.Mockito.times(2)).save(captor.capture());
AgentStep started = captor.getAllValues().get(0);
AgentStep completed = captor.getAllValues().get(1);
assertEquals("session-audit", started.getSessionId());
assertEquals("run-audit", started.getRunId());
assertFalse(started.getModelInput().contains("secret-query"));
assertFalse(completed.getModelOutput().contains("secret-model-output"));
assertFalse(completed.getModelOutput().contains("secret-argument"));
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"]}", completed.getModelOutput());
assertNull(completed.getThought());
}
@Test
void missingIdentitySkipsPersistence() {
AgentStepRepository repository = mock(AgentStepRepository.class);
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME);
hook.beforeModel(List.of(new UserMessage("secret")), RunnableConfig.builder().build());
verify(repository, never()).save(any());
}
}
@@ -0,0 +1,41 @@
package com.superbiz.agent.harness.audit;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.InvocationStatus;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
class JpaToolInvocationAuditSinkTest {
@Test
void persistsOnlyBoundedStableMetadata() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(repository, new ObjectMapper());
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-1", "query_logs",
InvocationStatus.ERROR, EvidenceStatus.ERROR, "TOOL_EXECUTION_ERROR",
17, 83, 0));
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals("session-1", saved.getSessionId());
assertEquals("run-1", saved.getRunId());
assertEquals("query_logs", saved.getToolName());
assertEquals("{\"tool_call_id\":\"call-1\",\"request_bytes\":83}", saved.getInputParams());
assertEquals("status=ERROR,evidence_status=ERROR", saved.getOutputPreview());
assertEquals("TOOL_EXECUTION_ERROR", saved.getErrorMessage());
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
assertFalse(serialized.contains("query"));
assertFalse(serialized.contains("raw_response"));
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.tool.boundary;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.InvocationStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
@@ -21,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -157,6 +159,42 @@ class ToolBoundaryTest {
assertFalse(store.find("superbiz:harness:tool-call:run-1:call-9").orElseThrow().isReferencableBy("run-1"));
}
@Test
void emitsExactSafeAuditAndAuditFailureDoesNotChangeResult() {
MutableClock clock = new MutableClock(Instant.parse("2026-07-21T10:00:00Z"));
FakeStore store = new FakeStore(new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
AtomicReference<ToolInvocationAuditEvent> audit = new AtomicReference<>();
ToolBoundary boundary = new ToolBoundary(core,
new ToolCallKeyFactory("superbiz:harness:tool-call"),
store, objectMapper, clock, audit::set);
RunContext context = core.startRun("session-audit", "run-audit");
ToolBoundaryResult result = boundary.execute(
context, request("run-audit", "call-audit", true, true),
ignored -> "secret-raw-response",
ignored -> new ProjectedToolResult("secret-agent-result", EvidenceStatus.EVIDENCE_FOUND));
assertEquals(InvocationStatus.READY, result.status());
assertEquals("session-audit", audit.get().sessionId());
assertEquals("run-audit", audit.get().runId());
assertEquals("call-audit", audit.get().toolCallId());
FakeStore failingAuditStore = new FakeStore(
new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
ToolBoundary failingAuditBoundary = new ToolBoundary(core,
new ToolCallKeyFactory("superbiz:harness:tool-call"),
failingAuditStore, objectMapper, clock,
ignored -> { throw new IllegalStateException("audit database secret"); });
ToolBoundaryResult unchanged = failingAuditBoundary.execute(
context, request("run-audit", "call-audit-failure", true, true),
ignored -> "raw",
ignored -> new ProjectedToolResult("agent", EvidenceStatus.NO_EVIDENCE));
assertEquals(InvocationStatus.READY, unchanged.status());
assertEquals(EvidenceStatus.NO_EVIDENCE, unchanged.evidenceStatus());
}
private ToolBoundary boundary(MutableClock clock, FakeStore store) {
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
return new ToolBoundary(core, new ToolCallKeyFactory("superbiz:harness:tool-call"),
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.tool.store;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.config.WebConfig;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.InvocationStatus;
import org.junit.jupiter.api.BeforeEach;
@@ -28,7 +29,7 @@ class CanonicalInvocationStoreTest {
private ValueOperations<String, Object> values;
private RedisCanonicalInvocationStore store;
private CanonicalToolInvocation projecting;
private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
private final ObjectMapper objectMapper = new WebConfig().objectMapper();
@BeforeEach
void setUp() {
@@ -1,380 +0,0 @@
package com.superbiz.agent.hook;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.service.ExecutorGatekeeperService;
import com.superbiz.agent.service.ToolTraceSummaryService;
import com.superbiz.agent.util.VerifierContextHolder;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class VerifierInputHookTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@AfterEach
void tearDown() {
VerifierContextHolder.clear();
}
@Test
void beforeModelAddsStructuredExecutorOutputWhenJsonContractIsValid() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
String executorOutput = """
{
"answer_version": "executor_evidence_v1",
"diagnosis_summary": "连接池已满,但缺少泄漏证据。",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": ["缺少泄漏检测日志"],
"user_facing_answer": "已确认连接池 active 达到上限。"
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "structured-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
assertEquals("executor_evidence_v1",
payload.path("executor_structured_output").path("answer_version").asText());
assertEquals("连接池 active 达到上限",
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
assertNotNull(VerifierContextHolder.getExecutorStructuredOutput());
assertEquals("valid", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
}
@Test
void beforeModelAddsStructuredExecutorOutputWhenV2ContractHasNoUserFacingAnswer() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
));
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("structured-v2-session")).thenReturn(List.of(
invocation(101L, "structured-v2-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "structured-v2-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
assertEquals("executor_evidence_v2",
payload.path("executor_structured_output").path("answer_version").asText());
assertFalse(payload.path("executor_structured_output").has("user_facing_answer"));
assertEquals("连接池 active 达到上限",
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
assertEquals("pass", payload.path("gatekeeper_result").path("status").asText());
assertEquals("none", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("pass", VerifierContextHolder.getGatekeeperResult().get("status"));
}
@Test
void beforeModelBackfillsOnlyUniqueInvocationIdAndDoesNotPassWithoutRawPath() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
Map.of(
"trace_ref", "metrics-1",
"tool_name", "query_metrics",
"source_invocation_ids", List.of(101L)
)
));
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("backfill-session")).thenReturn(List.of(
invocation(101L, "backfill-session", "query_metrics", "$.alerts[0]",
"CPU 使用率持续超过 80%,当前值为 92%")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "payment-service CPU 使用率超过 92%",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "prometheus-alert-HighCPUUsage",
"tool_name": "queryPrometheusAlerts",
"evidence_excerpt": "CPU 使用率持续超过 80%,当前值为 92%"
}
]
}
],
"hypotheses": [],
"recommended_actions": [
{
"action_text": "restart payment-service",
"reason": "cpu alert is firing",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "prometheus-alert-HighCPUUsage",
"tool_name": "queryPrometheusAlerts",
"evidence_excerpt": "CPU usage is 92%"
}
]
}
],
"missing_info": []
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "backfill-session").build()
);
JsonNode payload = readPayload(command);
JsonNode binding = payload.path("executor_structured_output")
.path("claims").get(0)
.path("evidence_bindings").get(0);
assertEquals("query_metrics", binding.path("tool_name").asText());
assertEquals(101L, binding.path("source_invocation_id").asLong());
JsonNode actionBinding = payload.path("executor_structured_output")
.path("recommended_actions").get(0)
.path("evidence_bindings").get(0);
assertEquals("query_metrics", actionBinding.path("tool_name").asText());
assertEquals(101L, actionBinding.path("source_invocation_id").asLong());
assertFalse(binding.has("raw_path"));
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
assertEquals("low_confid", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("evidence.invocation_auto_backfill",
payload.path("gatekeeper_result").path("warnings").get(0).path("rule").asText());
}
@Test
void beforeModelAddsFailingGatekeeperResultForFabricatedInvocationId() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("fabricated-invocation-session")).thenReturn(List.of(
invocation(101L, "fabricated-invocation-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_id": 999,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "fabricated-invocation-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
assertEquals("reject", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("evidence.invocation_ref",
payload.path("gatekeeper_result").path("failed_rules").get(0).asText());
}
@Test
void beforeModelExtractsStructuredOutputFromPrefixedJsonFence() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
String executorOutput = """
现在我已经收集了足够的数据,最终输出如下。
```json
{
"answer_version": "executor_evidence_v1",
"diagnosis_summary": "已确认连接池 active 达到上限。",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": [],
"user_facing_answer": "已确认连接池 active 达到上限。"
}
```
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "fenced-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
assertEquals("claim-1",
payload.path("executor_structured_output").path("claims").get(0).path("claim_id").asText());
}
@Test
void beforeModelMarksMalformedJsonAndKeepsRawAnswerFallback() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage("{\"diagnosis_summary\":\"缺少 claims\"}")),
RunnableConfig.builder().addMetadata("sessionId", "malformed-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("malformed", payload.path("executor_output_parse_status").path("status").asText());
assertTrue(payload.path("executor_structured_output").isNull());
assertEquals("{\"diagnosis_summary\":\"缺少 claims\"}", payload.path("executor_final_answer").asText());
assertEquals("malformed", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
}
@Test
void beforeModelMarksPlainTextAsMissingStructuredOutput() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage("普通自然语言答案")),
RunnableConfig.builder().addMetadata("sessionId", "plain-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("missing", payload.path("executor_output_parse_status").path("status").asText());
assertTrue(payload.path("executor_structured_output").isNull());
assertFalse(payload.path("executor_final_answer").asText().isBlank());
}
private JsonNode readPayload(AgentCommand command) throws Exception {
var field = AgentCommand.class.getDeclaredField("messages");
field.setAccessible(true);
@SuppressWarnings("unchecked")
List<Message> messages = (List<Message>) field.get(command);
assertEquals(1, messages.size());
Message message = messages.get(0);
assertTrue(message instanceof UserMessage);
return objectMapper.readTree(((UserMessage) message).getText());
}
private ToolInvocation invocation(Long id, String sessionId, String toolName, String rawPath, String text) {
return ToolInvocation.builder()
.id(id)
.sessionId(sessionId)
.toolName(toolName)
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
+ "\",\"text\":\"" + text + "\"}]}")
.build();
}
}
@@ -1,56 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.AIOpsRequest;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
class AiOpsRuleEvaluationServiceTest {
private final AiOpsRuleEvaluationService service = new AiOpsRuleEvaluationService();
@Test
void evaluatePassesWhenReportFocusesPayloadAndHasEvidenceTools() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("HighCPUUsage");
request.setService("payment-service");
ToolInvocation invocation = ToolInvocation.builder()
.toolName("lookup_knowledge")
.build();
Map<String, Object> evaluation = service.evaluate(
request,
"HighCPUUsage alert on payment-service was diagnosed using metrics and knowledge evidence.",
List.of(invocation)
);
assertEquals("PASS", evaluation.get("verdict"));
}
@Test
void evaluateWarnsWhenEvidenceToolsAreMissing() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("HighCPUUsage");
request.setService("payment-service");
Map<String, Object> evaluation = service.evaluate(
request,
"HighCPUUsage alert on payment-service has a likely resource saturation issue.",
List.of()
);
assertEquals("WARN", evaluation.get("verdict"));
}
@Test
void evaluateFailsWhenReportIsMissing() {
Map<String, Object> evaluation = service.evaluate(null, "too short", List.of());
assertEquals("FAIL", evaluation.get("verdict"));
}
}
@@ -1,268 +0,0 @@
package com.superbiz.agent.service;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
import com.superbiz.agent.config.AiOpsPromptProperties;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.domain.entity.DiagnosisRun;
import com.superbiz.agent.domain.entity.DiagnosisSession;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.AIOpsRequest;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.DiagnosisRunRepository;
import com.superbiz.agent.repository.DiagnosisSessionRepository;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.Optional;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
class AiOpsServiceTest {
private final DiagnosisSessionRepository diagnosisSessionRepository = mock(DiagnosisSessionRepository.class);
private final DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class);
private final AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
private final ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
private final AiOpsPromptProperties promptProperties = mock(AiOpsPromptProperties.class);
private final AiOpsService service = new AiOpsService();
@BeforeEach
void setUp() {
ReflectionTestUtils.setField(service, "diagnosisSessionRepository", diagnosisSessionRepository);
ReflectionTestUtils.setField(service, "diagnosisRunRepository", diagnosisRunRepository);
ReflectionTestUtils.setField(service, "agentStepRepository", agentStepRepository);
ReflectionTestUtils.setField(service, "toolInvocationRepository", toolInvocationRepository);
ReflectionTestUtils.setField(service, "aiOpsRuleEvaluationService", new AiOpsRuleEvaluationService());
ReflectionTestUtils.setField(service, "selfEvaluationMergeService", new SelfEvaluationMergeService());
ReflectionTestUtils.setField(service, "promptProperties", promptProperties);
when(promptProperties.getSupervisor()).thenReturn("supervisor prompt");
}
@Test
void resolveSessionIdUsesRequestValueWhenPresent() {
AIOpsRequest request = new AIOpsRequest();
request.setSessionId(" aiops-demo-session ");
assertEquals("aiops-demo-session", service.resolveSessionId(request));
}
@Test
void resolveSessionIdGeneratesWhenMissing() {
String sessionId = service.resolveSessionId(null);
assertNotNull(sessionId);
assertFalse(sessionId.isBlank());
}
@Test
void buildQuerySummaryUsesAlertFieldsAndUserRequestFallback() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("payment-service-latency-high");
request.setService("payment-service");
request.setSeverity("P1");
request.setTimeRange("last_15m");
request.setDescription("P95 latency is high");
request.setUserRequest("check logs and metrics for payment timeout");
String summary = service.buildQuerySummary(request);
assertTrue(summary.contains("AI Ops alert analysis"));
assertTrue(summary.contains("alert: payment-service-latency-high"));
assertTrue(summary.contains("service: payment-service"));
assertTrue(summary.contains("severity: P1"));
assertTrue(summary.contains("timeRange: last_15m"));
assertTrue(summary.contains("description: P95 latency is high"));
assertTrue(summary.contains("request: "));
}
@Test
void hasAlertPayloadIgnoresUserRequestOnly() {
AIOpsRequest request = new AIOpsRequest();
request.setUserRequest("please discover active alerts");
assertFalse(service.hasAlertPayload(request));
request.setAlertName("HighCPUUsage");
assertTrue(service.hasAlertPayload(request));
}
@Test
void buildTaskPromptUsesPayloadTargetedModeWhenAlertFieldsExist() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("HighCPUUsage");
request.setService("payment-service");
request.setSeverity("P1");
request.setTimeRange("last_15m");
request.setDescription("CPU usage is above 80%");
String prompt = service.buildTaskPrompt(request);
assertTrue(prompt.contains("AIOps scope mode: PAYLOAD_TARGETED"));
assertTrue(prompt.contains("primary and only main diagnosis target"));
assertTrue(prompt.contains("queryPrometheusAlerts only to verify"));
assertTrue(prompt.contains("do not create full root-cause or remediation sections"));
assertTrue(prompt.contains("Related Risk"));
assertTrue(prompt.contains("Recommended lookup_knowledge query: HighCPUUsage payment-service P1 CPU usage is above 80% last_15m"));
assertTrue(prompt.contains("preserves alertName and service"));
assertTrue(prompt.contains("alert: HighCPUUsage"));
assertTrue(prompt.contains("service: payment-service"));
assertFalse(prompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
}
@Test
void buildKnowledgeRetrievalQueryUsesPayloadFieldsAndSkipsBlankValues() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("HighLatency");
request.setService(" payment-service ");
request.setSeverity(" ");
request.setDescription("P95 latency above threshold");
request.setTimeRange("last_10m");
request.setUserRequest("check logs and metrics");
String query = service.buildKnowledgeRetrievalQuery(request);
assertEquals("HighLatency payment-service P95 latency above threshold last_10m check logs and metrics", query);
}
@Test
void buildTaskPromptUsesAutoDiscoveryModeWhenAlertPayloadIsMissing() {
String nullRequestPrompt = service.buildTaskPrompt(null);
assertTrue(nullRequestPrompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
assertTrue(nullRequestPrompt.contains("First call queryPrometheusAlerts"));
assertTrue(nullRequestPrompt.contains("current active/firing alerts"));
assertFalse(nullRequestPrompt.contains("AIOps scope mode: PAYLOAD_TARGETED"));
AIOpsRequest userRequestOnly = new AIOpsRequest();
userRequestOnly.setUserRequest("check what is firing now");
String userRequestOnlyPrompt = service.buildTaskPrompt(userRequestOnly);
assertTrue(userRequestOnlyPrompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
assertTrue(userRequestOnlyPrompt.contains("First call queryPrometheusAlerts"));
assertFalse(userRequestOnlyPrompt.contains("Recommended lookup_knowledge query"));
}
@Test
void persistFinalReportUpdatesDiagnosisRunAnswerAndEvaluationByRun() {
DiagnosisRun run = DiagnosisRun.builder()
.sessionId("aiops-session-001")
.runId("run-aiops-001")
.query("AI Ops alert analysis")
.status("SUCCESS")
.agentFlow("AI_OPS")
.build();
ToolInvocation invocation = ToolInvocation.builder()
.sessionId("aiops-session-001")
.runId("run-aiops-001")
.toolName("query_logs")
.success(true)
.build();
when(diagnosisRunRepository.findBySessionIdAndRunId("aiops-session-001", "run-aiops-001"))
.thenReturn(Optional.of(run));
when(toolInvocationRepository.findByRunIdOrderByIdAsc("run-aiops-001")).thenReturn(List.of(invocation));
service.persistFinalReport("aiops-session-001", "run-aiops-001",
"# 告警分析报告\nHighCPUUsage payment-service analysis with evidence summary.", null);
assertEquals("# 告警分析报告\nHighCPUUsage payment-service analysis with evidence summary.", run.getAnswer());
assertTrue(run.getSelfEvaluation().contains("aiops_rule_evaluation"));
verify(toolInvocationRepository).findByRunIdOrderByIdAsc("run-aiops-001");
verify(diagnosisRunRepository).save(run);
verify(diagnosisSessionRepository, never()).save(any());
}
@Test
void legacyPersistFinalReportStillUpdatesHistoricalDiagnosisSession() {
DiagnosisSession session = DiagnosisSession.builder()
.sessionId("legacy-aiops-session")
.query("AI Ops alert analysis")
.status("SUCCESS")
.agentFlow("AI_OPS")
.build();
when(diagnosisSessionRepository.findBySessionId("legacy-aiops-session")).thenReturn(Optional.of(session));
when(toolInvocationRepository.findBySessionIdOrderByIdAsc("legacy-aiops-session")).thenReturn(List.of());
service.persistFinalReport("legacy-aiops-session", "# 告警分析报告\nLegacy analysis with evidence summary.");
assertEquals("# 告警分析报告\nLegacy analysis with evidence summary.", session.getAnswer());
assertTrue(session.getSelfEvaluation().contains("aiops_rule_evaluation"));
verify(diagnosisSessionRepository).save(session);
}
@Test
void persistFinalReportSkipsBlankInput() {
service.persistFinalReport("aiops-session-001", " ");
verifyNoInteractions(diagnosisSessionRepository, diagnosisRunRepository);
}
@Test
void backfillRunMetricsUsesRunScopedRows() {
DiagnosisRun run = DiagnosisRun.builder()
.sessionId("aiops-session-002")
.runId("run-aiops-002")
.build();
AgentStep stepWithTool = AgentStep.builder()
.sessionId("aiops-session-002")
.runId("run-aiops-002")
.hasToolCall(true)
.tokenCount(10)
.build();
AgentStep stepWithoutTool = AgentStep.builder()
.sessionId("aiops-session-002")
.runId("run-aiops-002")
.hasToolCall(false)
.tokenCount(20)
.build();
when(agentStepRepository.findByRunIdOrderByStepIndex("run-aiops-002"))
.thenReturn(List.of(stepWithTool, stepWithoutTool));
when(toolInvocationRepository.countByRunId("run-aiops-002")).thenReturn(11L);
ReflectionTestUtils.invokeMethod(service, "backfillRunMetrics", run);
assertEquals(2, run.getStepCount());
assertEquals(30, run.getTotalTokenCount());
assertEquals(11, run.getToolCallCount());
verify(agentStepRepository).findByRunIdOrderByStepIndex("run-aiops-002");
verify(toolInvocationRepository).countByRunId("run-aiops-002");
}
@Test
void sameAiOpsSessionCanStartDistinctRuns() {
AIOpsRequest request = new AIOpsRequest();
request.setAlertName("HighCPUUsage");
ReflectionTestUtils.invokeMethod(service, "startDiagnosisRun", "same-session", "run-aiops-a", request);
ReflectionTestUtils.invokeMethod(service, "startDiagnosisRun", "same-session", "run-aiops-b", request);
verify(diagnosisRunRepository).save(argThat(run ->
"same-session".equals(run.getSessionId())
&& "run-aiops-a".equals(run.getRunId())
&& "AI_OPS".equals(run.getAgentFlow())
&& "RUNNING".equals(run.getStatus())));
verify(diagnosisRunRepository).save(argThat(run ->
"same-session".equals(run.getSessionId())
&& "run-aiops-b".equals(run.getRunId())
&& "AI_OPS".equals(run.getAgentFlow())
&& "RUNNING".equals(run.getStatus())));
}
@Test
void buildSupervisorAgentSetsPlannerAsMainAgent() {
ChatModel chatModel = mock(ChatModel.class);
ReactAgent planner = mock(ReactAgent.class);
ReactAgent executor = mock(ReactAgent.class);
SupervisorAgent supervisor = service.buildSupervisorAgent(chatModel, planner, executor);
assertSame(planner, supervisor.getMainAgent());
}
}
@@ -1,92 +0,0 @@
package com.superbiz.agent.service;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import static org.junit.jupiter.api.Assertions.*;
/**
* Chat + Embedding 解耦验证测试
* <p>
* 验证 ChatModel Bean 注入、ModelRoutingConfig 路由、ChatService 接口兼容。
* Chat: DeepSeek via OpenAI-compatible API
* Embedding: Ollama BGE-M3 (需要本地 ollama 运行)
*/
@SpringBootTest
@DisplayName("Chat + Embedding 解耦验证")
class ChatAndEmbeddingSmokeTest {
@Autowired
private ApplicationContext context;
@Autowired
private ChatModel chatModel;
@Autowired
private ChatService chatService;
/**
* 提供 mock ToolCallbackProvider(MCP 已禁用时需要)
*/
@TestConfiguration
static class MockToolConfig {
@Bean
public ToolCallbackProvider toolCallbackProvider() {
return () -> new ToolCallback[0];
}
}
@Test
@DisplayName("Spring 容器启动成功")
void contextLoads() {
assertNotNull(context, "Spring 容器应为非空");
assertNotNull(chatModel, "ChatModel Bean 应注入成功");
assertNotNull(chatService, "ChatService Bean 应注入成功");
}
@Test
@DisplayName("ModelRoutingConfig @Primary ChatModel 生效")
void chatModelPrimaryBeanWorks() {
assertNotNull(chatModel, "@Primary ChatModel 应被自动注入");
System.out.println("✓ ChatModel 类型: " + chatModel.getClass().getName());
// 验证路由到 openAiChatModel (DeepSeek)
assertTrue(context.containsBean("openAiChatModel"), "openAiChatModel 应存在");
System.out.println(" Chat → openAiChatModel (DeepSeek) ✓");
}
@Test
@DisplayName("ChatService.createReactAgent 接受 ChatModel 接口")
void chatServiceAcceptsChatModelInterface() {
var agent = chatService.createReactAgent(chatModel, "测试系统提示词");
assertNotNull(agent, "ReactAgent 应创建成功");
assertEquals("intelligent_assistant", agent.name());
System.out.println("✓ ReactAgent 创建成功: " + agent.name());
}
@Test
@DisplayName("ChatModel 实现 ChatModel 接口(类型安全验证)")
void chatModelIsProperType() {
assertNotNull(chatModel, "注入的 Bean 应为 ChatModel 实例");
System.out.println("✓ ChatModel 接口实现: " + chatModel.getClass().getSimpleName());
}
@Test
@DisplayName("EmbeddingModel 状态")
void embeddingModelStatus() {
boolean hasEmbedding = context.containsBean("embeddingModel");
if (hasEmbedding) {
System.out.println("✓ EmbeddingModel 已配置");
} else {
System.out.println("⚠ EmbeddingModel 未找到 — 检查 Ollama 是否运行");
}
}
}
@@ -1,918 +0,0 @@
package com.superbiz.agent.service;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
import com.alibaba.cloud.ai.graph.skills.registry.classpath.ClasspathSkillRegistry;
import com.superbiz.agent.agent.tool.DateTimeTools;
import com.superbiz.agent.agent.tool.QueryLogsTools;
import com.superbiz.agent.agent.tool.QueryMetricsTools;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.domain.entity.ChatSession;
import com.superbiz.agent.domain.entity.DiagnosisRun;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.ChatSessionRepository;
import com.superbiz.agent.repository.DiagnosisRunRepository;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.tool.RetrievedDocTracker;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.test.util.ReflectionTestUtils;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class ChatServiceSequentialAgentTest {
@Test
void executeChatComplexInvokesSequentialWorkflow() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-test-session"
);
assertTrue(result.answer().contains("连接池 active 达到上限"));
assertFalse(result.answer().contains("\"answer_version\""));
assertEquals("sequential-test-session", result.sessionId());
assertTrue(result.runId().startsWith("run-"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
assertTrue(chatModel.sawVerifierPrompt);
ChatSessionRepository chatSessionRepository =
(ChatSessionRepository) ReflectionTestUtils.getField(chatService, "chatSessionRepository");
DiagnosisRunRepository diagnosisRunRepository =
(DiagnosisRunRepository) ReflectionTestUtils.getField(chatService, "diagnosisRunRepository");
EvaluationService evaluationService =
(EvaluationService) ReflectionTestUtils.getField(chatService, "evaluationService");
ArgumentCaptor<ChatSession> chatSessionCaptor = ArgumentCaptor.forClass(ChatSession.class);
verify(chatSessionRepository, atLeastOnce()).save(chatSessionCaptor.capture());
assertEquals("sequential-test-session", chatSessionCaptor.getValue().getSessionId());
ArgumentCaptor<DiagnosisRun> runCaptor = ArgumentCaptor.forClass(DiagnosisRun.class);
verify(diagnosisRunRepository, atLeastOnce()).save(runCaptor.capture());
DiagnosisRun savedRun = runCaptor.getValue();
assertEquals(result.runId(), savedRun.getRunId());
assertEquals("sequential-test-session", savedRun.getSessionId());
assertEquals("SUCCESS", savedRun.getStatus());
assertEquals(result.answer(), savedRun.getAnswer());
verify(evaluationService).evaluateRun(eq(result.runId()), eq(result.answer()));
}
@Test
void executeChatComplexCreatesDistinctRunsForSameSessionAcrossTurns() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel firstRoundModel = new ScriptedChatModel();
ScriptedChatModel secondRoundModel = new ScriptedChatModel();
String sessionId = "sequential-same-session";
ChatService.ChatResult first = chatService.executeChatComplex(
firstRoundModel,
new ToolCallback[0],
"第一轮:请分析支付超时",
List.of(),
sessionId
);
ChatService.ChatResult second = chatService.executeChatComplex(
secondRoundModel,
new ToolCallback[0],
"第二轮:基于上一轮结论列出缺失证据",
List.of(
Map.of("role", "user", "content", "第一轮:请分析支付超时"),
Map.of("role", "assistant", "content", first.answer())
),
sessionId
);
assertEquals(sessionId, first.sessionId());
assertEquals(sessionId, second.sessionId());
assertNotEquals(first.runId(), second.runId());
DiagnosisRunRepository diagnosisRunRepository =
(DiagnosisRunRepository) ReflectionTestUtils.getField(chatService, "diagnosisRunRepository");
ArgumentCaptor<DiagnosisRun> runCaptor = ArgumentCaptor.forClass(DiagnosisRun.class);
verify(diagnosisRunRepository, atLeast(2)).save(runCaptor.capture());
List<String> savedRunIds = runCaptor.getAllValues().stream()
.filter(run -> sessionId.equals(run.getSessionId()))
.map(DiagnosisRun::getRunId)
.distinct()
.toList();
assertEquals(2, savedRunIds.size());
assertTrue(savedRunIds.contains(first.runId()));
assertTrue(savedRunIds.contains(second.runId()));
}
@Test
void executeChatComplexDoesNotRetryLowConfidenceByDefault() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "LOW_CONFID",
"groundedness_score": 0.1,
"critical_fact_count": 1,
"facts_checked": [
{
"fact": "missing direct evidence",
"is_critical": true,
"verification": "no_evidence",
"detail": "scripted evidence gap",
"evidence_refs": []
}
],
"rationale": "scripted low confidence"
}
""");
chatModel.composerOutput = "not-json";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-low-confidence-session"
);
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
assertTrue(result.answer().contains("当前缺口"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
}
@Test
void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "LOW_CONFID",
"groundedness_score": 0.37,
"critical_fact_count": 3,
"facts_checked": [
{
"fact": "连接池耗尽 active=50/50",
"is_critical": true,
"verification": "direct_evidence",
"detail": "log evidence",
"evidence_refs": []
},
{
"fact": "临时扩容连接池到 80",
"is_critical": true,
"verification": "indirect_support",
"detail": "suggestion inferred from evidence",
"evidence_refs": []
},
{
"fact": "OOM 导致连接泄漏",
"is_critical": true,
"verification": "no_evidence",
"detail": "missing OOM log",
"evidence_refs": []
}
],
"rationale": "scripted low confidence"
}
""");
chatModel.composerOutput = "not-json";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-low-confid-direct-only-session"
);
assertTrue(result.answer().contains("已确认信息:\n- 连接池耗尽 active=50/50"));
assertTrue(result.answer().contains("80"));
assertTrue(result.answer().contains("suggestion inferred from evidence"));
assertTrue(result.answer().contains("missing OOM log"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-missing-verifier-session"
);
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-invalid-verifier-session"
);
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "REJECT",
"groundedness_score": 0.0,
"critical_fact_count": 1,
"facts_checked": [
{
"fact": "payment timeout root cause",
"is_critical": true,
"verification": "contradicted",
"detail": "scripted contradiction",
"evidence_refs": []
}
],
"rationale": "scripted reject"
}
""");
chatModel.composerOutput = "not-json";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-reject-session"
);
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
}
@Test
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-workflow-session"
);
assertTrue(result.answer().contains("连接池 active 达到上限"));
assertFalse(result.answer().contains("\"answer_version\""));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
assertTrue(chatModel.sawVerifierPrompt);
}
@Test
void verifierReceivesStructuredExecutorPayloadFields() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-structured-executor-session"
);
assertTrue(result.answer().contains("连接池 active 达到上限"));
assertFalse(result.answer().contains("\"answer_version\""));
assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\""));
assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\""));
assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\""));
assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限"));
}
@Test
void executeChatComplexRendersExecutorEvidenceV2InsteadOfRawJsonOnPass() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.composerOutput = "not-json";
chatModel.executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [
{
"hypothesis_text": "连接泄漏可能参与了连接池耗尽",
"basis": "已有连接池满载证据,但缺少泄漏检测日志",
"needed_evidence": ["连接泄漏检测日志"]
}
],
"recommended_actions": [
{
"action_text": "补充查询连接池泄漏检测日志",
"reason": "用于确认是否存在连接未释放"
}
],
"missing_info": ["缺少连接泄漏检测日志"]
}
""";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-v2-render-session"
);
assertTrue(result.answer().contains("已确认信息"));
assertTrue(result.answer().contains("连接池 active 达到上限"));
assertTrue(result.answer().contains("建议下一步"));
assertFalse(result.answer().contains("\"answer_version\""));
assertFalse(result.answer().contains("executor_evidence_v2"));
}
@Test
void executeChatComplexPersistsGatekeeperResultInVerifierEvaluation() throws Exception {
ChatService chatService = createChatService();
SelfEvaluationMergeService mergeService =
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
ToolInvocationRepository invocationRepository =
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-persist-session"))
.thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.sessionId("sequential-gatekeeper-persist-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-gatekeeper-persist-session"
);
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
Map<String, Object> verifierEvaluation = captor.getValue();
assertTrue(verifierEvaluation.containsKey("gatekeeper_result"));
assertTrue(verifierEvaluation.containsKey("prompt_audit"));
@SuppressWarnings("unchecked")
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
assertEquals("pass", gatekeeperResult.get("status"));
assertEquals("none", gatekeeperResult.get("severity"));
@SuppressWarnings("unchecked")
Map<String, Object> promptAudit = (Map<String, Object>) verifierEvaluation.get("prompt_audit");
assertEquals("chat-prompts-v1", promptAudit.get("version"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> prompts = (List<Map<String, Object>>) promptAudit.get("prompts");
assertEquals(4, prompts.size());
assertTrue(prompts.stream().anyMatch(prompt ->
"chat_executor".equals(prompt.get("name"))
&& "chat-executor-v2".equals(prompt.get("version"))));
}
@Test
void executeChatComplexMapsClaimChecksToFactsCheckedAndPersistsBoth() throws Exception {
ChatService chatService = createChatService();
SelfEvaluationMergeService mergeService =
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
ToolInvocationRepository invocationRepository =
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-claim-check-session"))
.thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.sessionId("sequential-claim-check-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "LOW_CONFID",
"groundedness_score": 0.32,
"critical_fact_count": 6,
"claim_checks": [
{"claim_id":"claim-1","claim_text":"CPU 使用率 92%","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[{"trace_ref":"trace-1","tool_name":"query_metrics","source_invocation_ids":[101],"note":"cpu"}]},
{"claim_id":"claim-2","claim_text":"CPU 过高可能导致超时","claim_type":"risk","verification":"reasonable_inference","detail":"inference","evidence_refs":[]},
{"claim_id":"claim-3","claim_text":"CPU 是唯一根因","claim_type":"root_cause","verification":"overstated","detail":"too strong","evidence_refs":[]},
{"claim_id":"claim-4","claim_text":"缺少线程池证据","claim_type":"symptom","verification":"unsupported","detail":"missing","evidence_refs":[]},
{"claim_id":"claim-5","claim_text":"出现证据外错误码 ERR_FAKE","claim_type":"symptom","verification":"external_unknown","detail":"external","evidence_refs":[]},
{"claim_id":"claim-6","claim_text":"证据显示 CPU 很低","claim_type":"symptom","verification":"contradicted","detail":"conflict","evidence_refs":[]}
],
"facts_checked": [],
"rationale": "claim checks drive compatibility"
}
""");
chatModel.executorOutput = validExecutorV2Output();
chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-claim-check-session"
);
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
Map<String, Object> verifierEvaluation = captor.getValue();
@SuppressWarnings("unchecked")
List<Map<String, Object>> claimChecks = (List<Map<String, Object>>) verifierEvaluation.get("claim_checks");
@SuppressWarnings("unchecked")
List<Map<String, Object>> factsChecked = (List<Map<String, Object>>) verifierEvaluation.get("facts_checked");
assertEquals(6, claimChecks.size());
assertEquals(6, factsChecked.size());
assertEquals("direct_evidence", factsChecked.get(0).get("verification"));
assertEquals("indirect_support", factsChecked.get(1).get("verification"));
assertEquals("indirect_support", factsChecked.get(2).get("verification"));
assertEquals("no_evidence", factsChecked.get(3).get("verification"));
assertEquals("no_evidence", factsChecked.get(4).get("verification"));
assertEquals("contradicted", factsChecked.get(5).get("verification"));
assertTrue(String.valueOf(factsChecked.get(0).get("fact")).startsWith("claim-1:"));
}
@Test
void executeChatComplexDowngradesPassToRejectWhenGatekeeperInvocationRefFails() throws Exception {
ChatService chatService = createChatService();
SelfEvaluationMergeService mergeService =
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
ToolInvocationRepository invocationRepository =
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-fail-session"))
.thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.sessionId("sequential-gatekeeper-fail-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "PASS",
"groundedness_score": 1.0,
"critical_fact_count": 1,
"claim_checks": [
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[]}
],
"facts_checked": [],
"rationale": "model tried pass"
}
""");
chatModel.composerOutput = "not-json";
chatModel.executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_id": 999,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-gatekeeper-fail-session"
);
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
assertEquals("REJECT", captor.getValue().get("verdict"));
}
@Test
void executeChatComplexDowngradesPassToLowConfidenceWhenExecutorOutputMalformed() throws Exception {
ChatService chatService = createChatService();
SelfEvaluationMergeService mergeService =
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "PASS",
"groundedness_score": 1.0,
"critical_fact_count": 0,
"claim_checks": [],
"facts_checked": [],
"rationale": "model tried pass"
}
""");
chatModel.composerOutput = "not-json";
chatModel.executorOutput = "{ not-json";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-malformed-pass-session"
);
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
assertEquals("LOW_CONFID", captor.getValue().get("verdict"));
}
@Test
void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
ChatService chatService = new ChatService();
DateTimeTools dateTimeTools = new DateTimeTools();
LookupKnowledgeTool lookupKnowledgeTool = new LookupKnowledgeTool();
QueryLogsTools queryLogsTools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
QueryMetricsTools queryMetricsTools = new QueryMetricsTools(mock(ToolInvocationRecorder.class));
ReflectionTestUtils.setField(chatService, "dateTimeTools", dateTimeTools);
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", lookupKnowledgeTool);
ReflectionTestUtils.setField(chatService, "queryLogsTools", queryLogsTools);
ReflectionTestUtils.setField(chatService, "queryMetricsTools", queryMetricsTools);
Object[] methodTools = chatService.buildMethodToolsArray();
assertEquals(4, methodTools.length);
assertSame(dateTimeTools, methodTools[0]);
assertSame(lookupKnowledgeTool, methodTools[1]);
assertSame(queryLogsTools, methodTools[2]);
assertSame(queryMetricsTools, methodTools[3]);
}
@Test
void createReactAgentInjectsSkillCatalogThroughAlibabaHook() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
.classpathPath("skills")
.basePath("target/test-skills-cache")
.build();
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
ReactAgent agent = chatService.createReactAgent(chatModel, "BASE_TEST_PROMPT");
agent.call("diagnose mysql connection pool exhaustion");
assertTrue(chatModel.promptText.contains("BASE_TEST_PROMPT"));
assertTrue(chatModel.promptText.contains("## Skills System"));
assertTrue(chatModel.promptText.contains("diagnose-mysql-connection-pool"));
assertTrue(chatModel.promptText.contains("read_skill"));
}
@Test
void plannerGetsSkillMetadataAndExecutorGetsReadSkillTool() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
.classpathPath("skills")
.basePath("target/test-skills-cache")
.build();
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"diagnose mysql connection pool exhaustion",
List.of(),
"planner-skill-metadata-session"
);
assertTrue(chatModel.plannerPromptText.contains("\"skill_catalog\""));
assertTrue(chatModel.plannerPromptText.contains("diagnose-mysql-connection-pool"));
assertTrue(chatModel.plannerPromptText.contains("\"selected_skill\""));
assertFalse(chatModel.plannerPromptText.contains("## Skills System"));
assertFalse(chatModel.plannerPromptText.contains("read_skill"));
assertTrue(chatModel.executorPromptText.contains("## Skills System"));
assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool"));
assertTrue(chatModel.executorPromptText.contains("read_skill"));
assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill"));
assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool"));
assertFalse(chatModel.verifierPromptText.contains("read_skill"));
}
private ChatService createChatService() {
ChatService chatService = new ChatService();
ChatSessionRepository chatSessionRepository = mock(ChatSessionRepository.class);
when(chatSessionRepository.findBySessionId(anyString())).thenReturn(Optional.empty());
when(chatSessionRepository.save(any(ChatSession.class))).thenAnswer(invocation -> invocation.getArgument(0));
DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class);
when(diagnosisRunRepository.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
AtomicInteger stepId = new AtomicInteger(1);
AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
when(agentStepRepository.save(any(AgentStep.class))).thenAnswer(invocation -> {
AgentStep step = invocation.getArgument(0);
if (step.getId() == null) {
step.setId((long) stepId.getAndIncrement());
}
return step;
});
when(agentStepRepository.findById(any())).thenReturn(Optional.of(new AgentStep()));
when(agentStepRepository.findBySessionIdOrderByStepIndex(anyString())).thenReturn(List.of());
when(agentStepRepository.findByRunIdOrderByStepIndex(anyString())).thenReturn(List.of());
ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
when(toolInvocationRepository.countBySessionId(anyString())).thenReturn(0L);
when(toolInvocationRepository.countByRunId(anyString())).thenReturn(0L);
when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
when(toolInvocationRepository.findByRunIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
EvaluationService evaluationService = mock(EvaluationService.class);
RetrievedDocTracker retrievedDocTracker = mock(RetrievedDocTracker.class);
KnowledgeDomainService knowledgeDomainService = mock(KnowledgeDomainService.class);
when(knowledgeDomainService.buildKnowledgeMap()).thenReturn("");
ToolTraceSummaryService toolTraceSummaryService = mock(ToolTraceSummaryService.class);
when(toolTraceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
when(toolTraceSummaryService.buildVerifierTraceSummaryForRun(anyString(), anyString())).thenReturn(List.of());
SelfEvaluationMergeService selfEvaluationMergeService = mock(SelfEvaluationMergeService.class);
when(selfEvaluationMergeService.mergeVerifierEvaluation(any(), any())).thenReturn("{}");
ExecutorGatekeeperService executorGatekeeperService = new ExecutorGatekeeperService(toolInvocationRepository);
ReflectionTestUtils.setField(chatService, "dateTimeTools", new DateTimeTools());
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", new LookupKnowledgeTool());
ReflectionTestUtils.setField(chatService, "queryLogsTools", new QueryLogsTools(mock(ToolInvocationRecorder.class)));
ReflectionTestUtils.setField(chatService, "chatSessionRepository", chatSessionRepository);
ReflectionTestUtils.setField(chatService, "diagnosisRunRepository", diagnosisRunRepository);
ReflectionTestUtils.setField(chatService, "agentStepRepository", agentStepRepository);
ReflectionTestUtils.setField(chatService, "toolInvocationRepository", toolInvocationRepository);
ReflectionTestUtils.setField(chatService, "evaluationService", evaluationService);
ReflectionTestUtils.setField(chatService, "retrievedDocTracker", retrievedDocTracker);
ReflectionTestUtils.setField(chatService, "knowledgeDomainService", knowledgeDomainService);
ReflectionTestUtils.setField(chatService, "toolTraceSummaryService", toolTraceSummaryService);
ReflectionTestUtils.setField(chatService, "selfEvaluationMergeService", selfEvaluationMergeService);
ReflectionTestUtils.setField(chatService, "executorGatekeeperService", executorGatekeeperService);
ReflectionTestUtils.setField(chatService, "verifierLowConfidenceThreshold", 0.5d);
ReflectionTestUtils.setField(chatService, "chatPlannerPrompt", "PLANNER_TEST_PROMPT");
ReflectionTestUtils.setField(chatService, "chatExecutorPrompt", "EXECUTOR_TEST_PROMPT");
ReflectionTestUtils.setField(chatService, "chatVerifierPrompt", "VERIFIER_TEST_PROMPT");
ReflectionTestUtils.setField(chatService, "chatComposerPrompt", "COMPOSER_TEST_PROMPT");
return chatService;
}
private String validExecutorV2Output() {
return """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
}
private String evidenceRefs(String rawPath, String text) {
return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}";
}
private static final class ScriptedChatModel implements ChatModel {
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
private String promptText = "";
private String plannerPromptText = "";
private String executorPromptText = "";
private String verifierPromptText = "";
private String composerPromptText = "";
private String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
private String composerOutput = """
{
"answer_summary": "已确认连接池 active 达到上限。",
"recommended_actions": [
{
"action_text": "补充查询连接池泄漏检测日志",
"reason": "用于确认是否存在连接未释放"
}
],
"user_facing_answer": "已确认连接池 active 达到上限。建议补充查询连接池泄漏检测日志。"
}
""";
private boolean sawVerifierPrompt;
private final java.util.List<String> verifierOutputs;
private int verifierOutputIndex;
private ScriptedChatModel() {
this("""
{
"verdict": "PASS",
"groundedness_score": 1.0,
"critical_fact_count": 1,
"claim_checks": [
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"covered by scripted verifier","evidence_refs":[]}
],
"facts_checked": [],
"rationale": "scripted pass"
}
""");
}
private ScriptedChatModel(String verifierOutput) {
this.verifierOutputs = java.util.List.of(verifierOutput);
}
private ScriptedChatModel(String... verifierOutputs) {
this.verifierOutputs = java.util.List.of(verifierOutputs);
}
@Override
public ChatResponse call(Prompt prompt) {
promptText = prompt.getContents();
String text;
if (promptText.contains("PLANNER_TEST_PROMPT")) {
agentCalls.add("chat_planner");
plannerPromptText = promptText;
text = "PLANNER_PLAN";
} else if (promptText.contains("EXECUTOR_TEST_PROMPT")) {
agentCalls.add("chat_executor");
executorPromptText = promptText;
text = executorOutput;
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
agentCalls.add("chat_verifier");
verifierPromptText = promptText;
sawVerifierPrompt = true;
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
text = verifierOutputs.get(index);
verifierOutputIndex++;
} else if (promptText.contains("COMPOSER_TEST_PROMPT")) {
agentCalls.add("chat_composer");
composerPromptText = promptText;
text = composerOutput;
} else {
text = "UNEXPECTED_PROMPT";
}
return new ChatResponse(List.of(new Generation(new AssistantMessage(text))));
}
}
}
@@ -1,389 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ExecutorGatekeeperServiceTest {
@Test
void validateRunUsesRunScopedToolRows() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findByRunIdOrderByIdAsc("run-gatekeeper-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validateRun("run-gatekeeper-1",
validOutput(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
verify(repository).findByRunIdOrderByIdAsc("run-gatekeeper-1");
}
@Test
void ruleCatalogLoadsDefaultMetadata() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.loadDefault(new com.fasterxml.jackson.databind.ObjectMapper());
assertEquals("gatekeeper-rules-v1", catalog.version());
assertFalse(catalog.auditRules().isEmpty());
assertTrue(catalog.auditRules().stream()
.anyMatch(rule -> "evidence.raw_path".equals(rule.get("id"))));
assertEquals(0.5, catalog.doubleParameter("evidence.excerpt_mismatch",
"min_token_overlap", 0.0), 0.001);
}
@Test
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%, duration=25m")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@Test
void validatePassesForNoEvidenceReference() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@Test
void validateBackfillsNoEvidenceInvocationByRawPathWhenToolHasMultipleCalls() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志"),
invocation(102L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(null, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
assertEquals("evidence.invocation_auto_backfill_by_raw_path",
((Map<?, ?>) ((List<?>) result.get("warnings")).get(0)).get("rule"));
}
@Test
void validateBackfillsNoEvidenceInvocationByExcerptWhenRawPathIsRepeated() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=service:inventory-service AND HikariCP; total=0; message=未找到匹配的日志"),
invocation(102L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(null, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=service:inventory-service AND HikariCP; total=0; evidence_status=no_evidence",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertEquals(101L,
((Map<?, ?>) ((List<?>) result.get("checked_bindings")).get(0)).get("source_invocation_id"));
}
@Test
void validateRejectsNoEvidenceReferenceWhenExcerptClaimsPositiveEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.no_evidence",
"HikariCP active=50/50 waiting=32"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@Test
void validateRejectsPositiveBindingOnNegativeObservation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateFailsWhenRemovedFieldsArePresent() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of());
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("user_facing_answer", "旧版最终答案");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("schema.executor_v2"));
}
@Test
void validateFailsForFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(999L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@Test
void failResultIncludesRuleAuditMetadata() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.fail("gatekeeper.internal_error", "gatekeeper", "boom");
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
}
@Test
void validateFailsForToolNameMismatch() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.logs[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@Test
void validateDowngradesMissingRawPathToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", null, "cpu=92");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateRejectsUnknownRawPath() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[99]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateDowngradesOldInvocationWithoutEvidenceRefsToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics")
.retrievalDetails("{\"evidence_status\":\"supported\"}")
.build()
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.missing"));
}
@Test
void validateRejectsMismatchedExcerpt() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "HighCPUUsage firing service payment-service current 92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "HikariCP active=50/50 waiting=32"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@Test
void validateFailsForRecommendedActionFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("recommended_actions", List.of(Map.of(
"action_text", "restart service",
"reason", "alert is firing",
"evidence_bindings", List.of(Map.of(
"source_type", "tool_trace",
"source_id", "trace-1",
"tool_name", "query_metrics",
"source_invocation_id", 999L,
"raw_path", "$.alerts[0]",
"evidence_excerpt", "cpu=92"
))
)));
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
private ToolInvocation invocation(Long id, String toolName, String rawPath, String text) {
return ToolInvocation.builder()
.id(id)
.sessionId("session-1")
.toolName(toolName)
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
+ "\",\"text\":\"" + text + "\"}]}")
.build();
}
private Map<String, Object> validOutput(Long invocationId, String toolName, String rawPath, String excerpt) {
return validOutput(invocationId, toolName, rawPath, excerpt, "symptom");
}
private Map<String, Object> validOutput(Long invocationId,
String toolName,
String rawPath,
String excerpt,
String claimType) {
Map<String, Object> binding = new java.util.LinkedHashMap<>();
binding.put("source_type", "tool_trace");
binding.put("source_id", "trace-1");
binding.put("tool_name", toolName);
if (invocationId != null) {
binding.put("source_invocation_id", invocationId);
}
if (rawPath != null) {
binding.put("raw_path", rawPath);
}
binding.put("evidence_excerpt", excerpt);
return new java.util.LinkedHashMap<>(Map.of(
"answer_version", "executor_evidence_v2",
"claims", List.of(Map.of(
"claim_id", "claim-1",
"claim_type", claimType,
"claim_text", "连接池 active 达到上限",
"support_level", "direct",
"evidence_bindings", List.of(binding)
)),
"hypotheses", List.of(),
"recommended_actions", List.of(),
"missing_info", List.of()
));
}
private void assertRuleAudit(Map<String, Object> result) {
assertEquals("gatekeeper-rules-v1", result.get("rule_set_version"));
assertTrue(result.get("rules") instanceof List<?>);
List<?> rules = (List<?>) result.get("rules");
assertFalse(rules.isEmpty());
assertTrue(rules.stream().anyMatch(rule ->
rule instanceof Map<?, ?> map && "evidence.raw_path".equals(map.get("id"))));
}
}
@@ -1,363 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.ContextPack;
import com.superbiz.agent.dto.EvidenceBlock;
import com.superbiz.agent.dto.KnowledgeQuery;
import com.superbiz.agent.dto.RetrievalTrace;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.util.SessionContextHolder;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ToolInvocationRecorderTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void recordEvidenceToolWritesRunIdFromExecutionContext() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setContext("recorder-run-session", "run-recorder-1");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"{\"success\":true,\"alerts\":[]}",
true,
12,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals("recorder-run-session", saved.getSessionId());
assertEquals("run-recorder-1", saved.getRunId());
}
@Test
void recordEvidenceToolPreservesNoEvidenceSemantics() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("recorder-test-session");
try {
recorder.recordEvidenceTool(
"query_logs",
Map.of("query", "inventory-service HikariCP", "log_topic", "application-logs"),
"{\"success\":false,\"query\":\"inventory-service HikariCP\",\"log_topic\":\"application-logs\",\"logs\":[],\"total\":0,\"message\":\"未找到匹配的日志\"}",
true,
System.currentTimeMillis() - 10,
null,
"application-logs",
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
Map.of("log_topic", "application-logs")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals("query_logs", saved.getToolName());
assertEquals(Boolean.TRUE, saved.getSuccess());
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"no_evidence\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
JsonNode details = objectMapper.readTree(saved.getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
String text = details.path("evidence_refs").get(0).path("text").asText();
assertTrue(text.contains("query_logs returned no evidence"));
assertTrue(text.contains("inventory-service HikariCP"));
assertTrue(text.contains("total=0"));
assertTrue(text.contains("evidence_status=no_evidence"));
}
@Test
void recordEvidenceToolExtractsLogEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("log-ref-session");
try {
recorder.recordEvidenceTool(
"query_logs",
Map.of("query", "HikariCP order-service"),
"""
{"success":true,"logs":[{"timestamp":"2026-07-08 10:00:00","level":"ERROR","service":"order-service","message":"HikariPool-1 - Connection is not available, request timed out after 30000ms","metrics":{"waiting":"32"}}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"application-logs",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("log_topic", "application-logs")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.logs[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HikariPool-1"));
}
@Test
void recordEvidenceToolExtractsMetricEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("metric-ref-session");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"""
{"success":true,"alerts":[{"alert_name":"HighMemoryUsage","state":"firing","description":"服务 order-service 当前值为 91%","active_at":"2026-07-08T10:00:00Z","duration":"15m"}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.alerts[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HighMemoryUsage"));
}
@Test
void recordEvidenceToolExtractsMetricNoEvidenceRef() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("metric-no-evidence-session");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"""
{"success":true,"alerts":[],"message":"成功检索到 0 个活动告警"}
""",
true,
System.currentTimeMillis() - 10,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("query_metrics returned no evidence"));
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("prometheus_alerts"));
}
@Test
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("lookup-recorder-session");
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
.query("ERR_TIMEOUT")
.outputPreview("matched payment doc")
.outputLength(18)
.retrievalLayer("L0")
.l0MatchCount(1)
.l1MatchCount(null)
.truncated(false)
.relevanceLevel("PRECISE")
.completenessHint("already precise")
.domain("payment")
.dedupReason("doc_retrieved")
.durationMs(42)
.success(true)
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED)
.l0Titles(List.of("payment/errors.md"))
.l0MatchedKeywords(List.of("ERR_TIMEOUT"))
.l0Domains(List.of("payment"))
.l0Entities(List.of("ERR_TIMEOUT"))
.evidenceCandidateCount(2)
.evidenceBlockCount(1)
.evidenceBlocks(List.of(Map.of(
"source", "payment/errors.md",
"title", "payment/errors.md",
"retrieval_layer", "L0+L1",
"hit_reasons", List.of("l0_keywords:ERR_TIMEOUT", "semantic_rank:1")
)))
.build();
try {
recorder.recordLookupKnowledge(record);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals("lookup_knowledge", saved.getToolName());
assertEquals("PRECISE", saved.getRelevanceLevel());
assertEquals("doc_retrieved", saved.getDedupReason());
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"deduped\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"payment\"]"));
assertTrue(saved.getRetrievalDetails().contains("\"l0_matched_keywords\":[\"ERR_TIMEOUT\"]"));
assertTrue(saved.getRetrievalDetails().contains("\"l0_domains\":[\"payment\"]"));
assertTrue(saved.getRetrievalDetails().contains("\"l0_entities\":[\"ERR_TIMEOUT\"]"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_refs\""));
assertTrue(saved.getRetrievalDetails().contains("\"raw_path\":\"$.evidence_blocks[0]\""));
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));
assertTrue(saved.getRetrievalDetails().contains("\"rerank_trace\""));
}
@Test
void recordLookupKnowledgeAddsNoEvidenceRefWhenNoBlocks() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("lookup-no-evidence-session");
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
.query("inventory-service HikariCP")
.outputPreview("")
.outputLength(0)
.durationMs(12)
.success(true)
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE)
.evidenceBlocks(List.of())
.build();
try {
recorder.recordLookupKnowledge(record);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("lookup_knowledge returned no evidence"));
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("inventory-service HikariCP"));
}
@Test
void lookupKnowledgeRecordFromSummarizesEvidenceBlocks() {
EvidenceBlock block = EvidenceBlock.builder()
.source("doc.md")
.title("Doc")
.breadcrumb("A > B")
.retrievalLayer("L1")
.score(0.42)
.hitReasons(List.of("semantic_rank:1"))
.content("x".repeat(220))
.build();
com.superbiz.agent.dto.LookupResult result = com.superbiz.agent.dto.LookupResult.builder()
.found(true)
.evidenceCandidateCount(3)
.evidenceBlockCount(1)
.evidenceBlocks(List.of(block))
.contextPack(ContextPack.builder()
.packedText("packed evidence")
.strategy("ranked_evidence_char_budget")
.charBudget(4000)
.usedChars(15)
.includedSources(List.of("doc.md"))
.omittedSources(List.of())
.build())
.retrievalTrace(RetrievalTrace.builder()
.originalQuery("query")
.rewrittenQuery("query")
.selectedAttempt("UNFILTERED_VECTOR")
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED)
.attempts(List.of(RetrievalTrace.Attempt.builder()
.name("UNFILTERED_VECTOR")
.candidateCount(3)
.topScore(0.42)
.topSimilarity(0.79)
.usable(true)
.build()))
.build())
.build();
KnowledgeQuery query = KnowledgeQuery.builder()
.originalQuery("query")
.rewrittenQuery("query")
.domainHints(List.of())
.matchedKeywords(List.of())
.entities(List.of())
.l0Titles(List.of())
.l0MatchCount(0)
.build();
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.from(
query,
result,
null,
null,
10
);
assertEquals(3, record.evidenceCandidateCount());
assertEquals(1, record.evidenceBlockCount());
assertEquals(1, record.evidenceBlocks().size());
assertTrue(String.valueOf(record.evidenceBlocks().get(0).get("content_preview")).endsWith("..."));
assertEquals("L1", record.retrievalLayer());
assertEquals(3, record.l1MatchCount());
assertTrue(record.retrievalTrace().containsKey("selected_attempt"));
assertTrue(record.contextPack().containsKey("included_sources"));
}
}
@@ -1,209 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class ToolTraceSummaryServiceTest {
@Test
void buildVerifierTraceSummaryForRunUsesRunScopedToolRows() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findByRunIdOrderByIdAsc("run-summary-1")).thenReturn(List.of(
ToolInvocation.builder()
.id(101L)
.sessionId("session-1")
.runId("run-summary-1")
.toolName("query_metrics")
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
.outputPreview("active=50 max=50")
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummaryForRun(
"run-summary-1", "active=50 max=50");
assertEquals(1, summaries.size());
assertEquals("query_metrics", summaries.get(0).get("tool_name"));
assertEquals(List.of(101L), summaries.get(0).get("source_invocation_ids"));
}
@Test
void buildVerifierTraceSummaryTreatsNoEvidenceAsGapWithoutLosingSuccessfulEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder()
.id(1L)
.sessionId("session-1")
.toolName("query_logs")
.inputParams("{\"query\":\"timeout\"}")
.outputPreview("payment timeout stack trace")
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
.success(true)
.build(),
ToolInvocation.builder()
.id(2L)
.sessionId("session-1")
.toolName("query_logs")
.inputParams("{\"query\":\"timeout\"}")
.outputPreview("{\"success\":false,\"message\":\"未找到匹配的日志\"}")
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"no_evidence\"}")
.success(true)
.build(),
ToolInvocation.builder()
.id(3L)
.sessionId("session-1")
.toolName("query_metrics")
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
.errorMessage("prometheus timeout")
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"failed\"}")
.success(false)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-1", "application-logs point to timeout");
assertEquals(2, summaries.size());
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertEquals(Boolean.TRUE, logsSummary.get("success"));
assertEquals("direct", logsSummary.get("evidence_level"));
assertEquals(2, logsSummary.get("invocation_count"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("payment timeout stack trace"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertEquals(Boolean.FALSE, metricsSummary.get("success"));
assertEquals("none", metricsSummary.get("evidence_level"));
assertEquals(1, metricsSummary.get("failed_invocation_count"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
}
@Test
void buildVerifierTraceSummaryPreservesConcreteFactsFromTruncatedLogAndMetricRows() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String logPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 22:15:45",
"level" : "ERROR",
"service" : "order-service",
"message" : "数据库连接池耗尽: Cannot acquire connection from pool, active: 50/50, waiting: 23, timeout: 30000ms"
} ]
}
""";
String metricPreview = """
{
"success" : true,
"alerts" : [ {
"alert_name" : "HighCPUUsage",
"service" : "payment-service",
"description" : "服务 payment-service 的 CPU 使用率持续超过 80%,当前值为 92%。"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-2")).thenReturn(List.of(
ToolInvocation.builder()
.id(10L)
.sessionId("session-2")
.toolName("query_logs")
.inputParams("{\"query\":\"pool\"}")
.outputPreview(logPreview)
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build(),
ToolInvocation.builder()
.id(11L)
.sessionId("session-2")
.toolName("query_metrics")
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
.outputPreview(metricPreview)
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-2", "连接池耗尽 HighCPUUsage");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("连接池耗尽"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("active: 50/50"));
assertEquals(List.of(10L), logsSummary.get("source_invocation_ids"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("HighCPUUsage"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("payment-service"));
}
@Test
void buildVerifierTraceSummaryDoesNotTreatGenericMockLogsAsDirectEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String genericLogPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 23:44:41",
"level" : "ERROR",
"service" : "generic-service",
"message" : "日志消息 #0, 查询条件: service:payment-service"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-3")).thenReturn(List.of(
ToolInvocation.builder()
.id(20L)
.sessionId("session-3")
.toolName("query_logs")
.inputParams("{\"query\":\"service:payment-service\"}")
.outputPreview(genericLogPreview)
.retrievalDetails("{\"retrieved_domains\":[\"system-metrics\"],\"evidence_status\":\"supported\"}")
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-3", "payment-service timeout");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertEquals(Boolean.FALSE, logsSummary.get("success"));
assertEquals("none", logsSummary.get("evidence_level"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("日志消息 #0"));
}
}
@@ -1,246 +0,0 @@
package com.superbiz.agent.service.session;
import com.superbiz.agent.domain.model.SessionContext;
import com.superbiz.agent.domain.model.ToolCall;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.TestPropertySource;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.*;
/**
* RedisSessionManager 单元测试
*/
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
@EnabledIfEnvironmentVariable(named = "SUPERBIZ_REDIS_PASSWORD", matches = ".+")
@TestPropertySource(properties = {
"spring.data.redis.host=${SUPERBIZ_REDIS_HOST:119.29.78.52}",
"spring.data.redis.port=${SUPERBIZ_REDIS_PORT:33308}",
"spring.data.redis.password=${SUPERBIZ_REDIS_PASSWORD}"
})
class RedisSessionManagerTest {
@Autowired
private SessionManager sessionManager;
private String testSessionId;
@BeforeEach
void setUp() {
testSessionId = "test-session-" + UUID.randomUUID().toString();
}
@Test
void testCreateAndGetSession() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-123")
.businessId("order-456")
.traceId("trace-789")
.build();
String sessionId = sessionManager.createSession(context, 300); // 5分钟
assertNotNull(sessionId);
assertEquals(testSessionId, sessionId);
System.out.println("✓ 创建会话成功: " + sessionId);
// 获取会话
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
assertTrue(retrieved.isPresent());
assertEquals("user-123", retrieved.get().getUserId());
assertEquals("ACTIVE", retrieved.get().getStatus());
assertNotNull(retrieved.get().getCreatedAt());
System.out.println("✓ 获取会话成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testUpdateSession() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-update")
.status("ACTIVE")
.build();
sessionManager.createSession(context, 300);
// 获取并更新
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
assertTrue(retrieved.isPresent());
SessionContext toUpdate = retrieved.get();
toUpdate.setStatus("COMPLETED");
toUpdate.setBusinessId("updated-business-id");
sessionManager.updateSession(toUpdate);
// 验证更新
Optional<SessionContext> updated = sessionManager.getSession(testSessionId);
assertTrue(updated.isPresent());
assertEquals("COMPLETED", updated.get().getStatus());
assertEquals("updated-business-id", updated.get().getBusinessId());
System.out.println("✓ 更新会话成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testDeleteSession() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-delete")
.build();
sessionManager.createSession(context, 300);
assertTrue(sessionManager.exists(testSessionId));
// 删除会话
sessionManager.deleteSession(testSessionId);
assertFalse(sessionManager.exists(testSessionId));
System.out.println("✓ 删除会话成功");
}
@Test
void testExists() {
assertFalse(sessionManager.exists(testSessionId));
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-exists")
.build();
sessionManager.createSession(context, 300);
assertTrue(sessionManager.exists(testSessionId));
System.out.println("✓ 会话存在性检查成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testRefreshSession() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-refresh")
.build();
sessionManager.createSession(context, 60); // 1分钟
// 刷新过期时间
boolean refreshed = sessionManager.refreshSession(testSessionId, 600); // 延长到10分钟
assertTrue(refreshed);
assertTrue(sessionManager.exists(testSessionId));
System.out.println("✓ 刷新会话过期时间成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testAddToolCall() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-toolcall")
.build();
sessionManager.createSession(context, 300);
// 添加工具调用记录
Map<String, Object> args = new HashMap<>();
args.put("query", "test query");
args.put("limit", 10);
ToolCall toolCall = ToolCall.builder()
.toolName("search_documents")
.arguments(args)
.result("found 5 documents")
.status("SUCCESS")
.duration(150L)
.calledAt(LocalDateTime.now())
.build();
sessionManager.addToolCall(testSessionId, toolCall);
// 验证工具调用已添加
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
assertTrue(retrieved.isPresent());
assertFalse(retrieved.get().getToolCalls().isEmpty());
assertEquals(1, retrieved.get().getToolCalls().size());
assertEquals("search_documents", retrieved.get().getToolCalls().get(0).getToolName());
System.out.println("✓ 添加工具调用记录成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testUpdateStatus() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-status")
.status("ACTIVE")
.build();
sessionManager.createSession(context, 300);
// 更新状态
sessionManager.updateStatus(testSessionId, "COMPLETED");
// 验证状态已更新
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
assertTrue(retrieved.isPresent());
assertEquals("COMPLETED", retrieved.get().getStatus());
System.out.println("✓ 更新会话状态成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
@Test
void testMultipleToolCalls() {
// 创建会话
SessionContext context = SessionContext.builder()
.sessionId(testSessionId)
.userId("user-multi-tools")
.build();
sessionManager.createSession(context, 300);
// 添加多个工具调用
for (int i = 0; i < 3; i++) {
ToolCall toolCall = ToolCall.builder()
.toolName("tool_" + i)
.status("SUCCESS")
.calledAt(LocalDateTime.now())
.build();
sessionManager.addToolCall(testSessionId, toolCall);
}
// 验证所有工具调用
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
assertTrue(retrieved.isPresent());
assertEquals(3, retrieved.get().getToolCalls().size());
System.out.println("✓ 添加多个工具调用记录成功");
// 清理
sessionManager.deleteSession(testSessionId);
}
}
@@ -9,9 +9,7 @@ import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeIndexService;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.service.ToolInvocationRecorder;
import com.superbiz.agent.service.VectorSearchService;
import com.superbiz.agent.util.SessionContextHolder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
@@ -40,12 +38,6 @@ class LookupKnowledgeToolTest {
@Mock
private VectorSearchService vectorSearchService;
@Mock
private ToolInvocationRecorder toolInvocationRecorder;
@Mock
private RetrievedDocTracker retrievedDocTracker;
private LookupKnowledgeTool tool;
@BeforeEach
@@ -61,8 +53,6 @@ class LookupKnowledgeToolTest {
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
ReflectionTestUtils.setField(tool, "toolInvocationRecorder", toolInvocationRecorder);
ReflectionTestUtils.setField(tool, "retrievedDocTracker", retrievedDocTracker);
ReflectionTestUtils.setField(tool, "topK", 3);
}
@@ -250,36 +240,6 @@ class LookupKnowledgeToolTest {
assertEquals("shared.md", lookup.getEvidenceBlocks().get(0).getSource());
}
@Test
void sessionDedupDoesNotReturnConsumableEvidenceAgain() {
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
VectorSearchService.SearchResult result = searchResult(
"vec-1",
"payment.md",
"{\"_source\":\"payment.md\",\"title\":\"Payment\",\"category\":\"payment\"}",
"payment timeout runbook",
0.2f);
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
.thenReturn(List.of(result));
when(retrievedDocTracker.isAlreadyRetrieved("session-1", "payment.md")).thenReturn(true);
when(retrievedDocTracker.getRetrievedDomains("session-1")).thenReturn(List.of("payment"));
SessionContextHolder.setSessionId("session-1");
try {
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
assertFalse(lookup.isFound());
assertEquals(0, lookup.getEvidenceBlockCount());
assertTrue(lookup.getEvidenceBlocks().isEmpty());
assertTrue(lookup.getMessage().contains("无需重复召回"));
assertEquals(List.of("payment"), lookup.getRetrievedDomainsThisSession());
} finally {
SessionContextHolder.clear();
}
}
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) {
return KnowledgeEntry.builder()
.filePath(filePath)