Harden evidence trace semantics

This commit is contained in:
aruo
2026-07-04 22:36:30 +08:00
parent 246c99b954
commit dc6cd32a67
24 changed files with 1096 additions and 141 deletions
@@ -131,13 +131,15 @@ public class QueryLogsTools {
output.setMessage(String.format("共有 %d 个可用的日志主题。建议使用默认地域 'ap-guangzhou' 或省略 region 参数", topics.size()));
String response = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
recordInvocation(startTime, "get_available_log_topics", null, null, null, response, true, null, "logs");
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
response, true, null, "logs", ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return response;
} catch (Exception e) {
logger.error("获取日志主题列表失败", e);
String response = "{\"success\":false,\"message\":\"获取日志主题列表失败: " + e.getMessage() + "\"}";
recordInvocation(startTime, "get_available_log_topics", null, null, null, response, false, e.getMessage(), "logs");
recordInvocation("get_available_log_topics", startTime, "get_available_log_topics", null, null, null,
response, false, e.getMessage(), "logs", ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
@@ -191,8 +193,9 @@ public class QueryLogsTools {
} else {
// 真实模式:调用 CLS API(这里预留接口,后续实现)
String response = buildErrorResponse("CLS 真实查询尚未实现,请启用 mock 模式进行测试");
recordInvocation(startTime, safeQuery, region, logTopic, actualLimit, response, false,
"CLS 真实查询尚未实现,请启用 mock 模式进行测试", normalizeTopicDomain(logTopic));
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
"CLS 真实查询尚未实现,请启用 mock 模式进行测试", normalizeTopicDomain(logTopic),
ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
@@ -208,23 +211,26 @@ public class QueryLogsTools {
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
logger.info("日志查询完成: 找到 {} 条日志", logEntries.size());
recordInvocation(startTime, safeQuery, region, logTopic, actualLimit, jsonResult,
!logEntries.isEmpty(), logEntries.isEmpty() ? "未找到匹配的日志" : null,
normalizeTopicDomain(logTopic));
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, jsonResult,
true, null, normalizeTopicDomain(logTopic),
logEntries.isEmpty()
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return jsonResult;
} catch (Exception e) {
logger.error("查询日志失败", e);
String response = buildErrorResponse("查询失败: " + e.getMessage());
recordInvocation(startTime, safeQuery, region, logTopic, actualLimit, response, false,
e.getMessage(), normalizeTopicDomain(logTopic));
recordInvocation("query_logs", startTime, safeQuery, region, logTopic, actualLimit, response, false,
e.getMessage(), normalizeTopicDomain(logTopic), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
private void recordInvocation(long startTime, String query, String region, String logTopic, Integer limit,
String output, boolean success, String errorMessage, String topicDomain) {
private void recordInvocation(String toolName, long startTime, String query, String region, String logTopic, Integer limit,
String output, boolean success, String errorMessage, String topicDomain,
String evidenceStatus) {
Map<String, Object> input = new HashMap<>();
input.put("query", query == null || query.isBlank() ? "DEFAULT_QUERY" : query);
if (region != null) {
@@ -239,13 +245,15 @@ public class QueryLogsTools {
input.put("mock_enabled", mockEnabled);
toolInvocationRecorder.recordEvidenceTool(
"query_logs",
toolName,
input,
output,
success,
startTime,
errorMessage,
topicDomain
topicDomain,
evidenceStatus,
Map.of("log_topic", logTopic == null ? "" : logTopic)
);
}
@@ -81,7 +81,7 @@ public class QueryMetricsTools {
if (!"success".equals(result.getStatus())) {
String response = buildErrorResponse("Prometheus API 返回非成功状态: " + result.getStatus(), result.getError());
recordInvocation(startTime, response, false, result.getError());
recordInvocation(startTime, response, false, result.getError(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
@@ -119,19 +119,22 @@ public class QueryMetricsTools {
String jsonResult = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(output);
logger.info("Prometheus 告警查询完成: 找到 {} 个告警", simplifiedAlerts.size());
recordInvocation(startTime, jsonResult, true, null);
recordInvocation(startTime, jsonResult, true, null,
simplifiedAlerts.isEmpty()
? ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE
: ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED);
return jsonResult;
} catch (Exception e) {
logger.error("查询 Prometheus 告警失败", e);
String response = buildErrorResponse("查询失败", e.getMessage());
recordInvocation(startTime, response, false, e.getMessage());
recordInvocation(startTime, response, false, e.getMessage(), ToolInvocationRecorder.EVIDENCE_STATUS_FAILED);
return response;
}
}
private void recordInvocation(long startTime, String output, boolean success, String errorMessage) {
private void recordInvocation(long startTime, String output, boolean success, String errorMessage, String evidenceStatus) {
toolInvocationRecorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts", "mock_enabled", mockEnabled),
@@ -139,7 +142,9 @@ public class QueryMetricsTools {
success,
startTime,
errorMessage,
"prometheus_alerts"
"prometheus_alerts",
evidenceStatus,
Map.of("metric_family", "prometheus_alerts")
);
}
@@ -3,11 +3,16 @@ package com.superbiz.agent.service;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.repository.ToolInvocationRepository;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.service.VectorSearchService;
import com.superbiz.agent.util.SessionContextHolder;
import lombok.Builder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -21,6 +26,10 @@ import java.util.UUID;
public class ToolInvocationRecorder {
private static final int OUTPUT_PREVIEW_LIMIT = 500;
public static final String EVIDENCE_STATUS_SUPPORTED = "supported";
public static final String EVIDENCE_STATUS_NO_EVIDENCE = "no_evidence";
public static final String EVIDENCE_STATUS_DEDUPED = "deduped";
public static final String EVIDENCE_STATUS_FAILED = "failed";
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper;
@@ -52,12 +61,29 @@ public class ToolInvocationRecorder {
long startTimeMillis,
String errorMessage,
String topicDomain) {
recordEvidenceTool(toolName, inputParams, output, success, startTimeMillis, errorMessage, topicDomain,
success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED, Map.of());
}
public void recordEvidenceTool(String toolName,
Map<String, Object> inputParams,
String output,
boolean success,
long startTimeMillis,
String errorMessage,
String topicDomain,
String evidenceStatus,
Map<String, Object> extraDetails) {
String outputPreview = preview(output);
Map<String, Object> details = new LinkedHashMap<>();
details.put("trace_id", UUID.randomUUID().toString());
if (topicDomain != null && !topicDomain.isBlank()) {
details.put("retrieved_domains", List.of(topicDomain));
}
details.put("evidence_status", normalizeEvidenceStatus(success, evidenceStatus));
if (extraDetails != null && !extraDetails.isEmpty()) {
details.putAll(extraDetails);
}
ToolInvocation invocation = ToolInvocation.builder()
.toolName(toolName)
@@ -73,6 +99,67 @@ public class ToolInvocationRecorder {
save(invocation);
}
public void recordLookupKnowledge(LookupKnowledgeRecord record) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("trace_id", UUID.randomUUID().toString());
if (record.l0MatchCount() != null) {
details.put("l0_match_count", record.l0MatchCount());
}
if (record.l0Titles() != null && !record.l0Titles().isEmpty()) {
details.put("l0_titles", record.l0Titles());
}
if (record.l1TopScore() != null) {
details.put("l1_top_score", record.l1TopScore());
}
if (record.l1TopSimilarity() != null) {
details.put("l1_top_similarity", record.l1TopSimilarity());
}
if (record.l1MatchCount() != null) {
details.put("l1_match_count", record.l1MatchCount());
}
if (record.l1Scores() != null && !record.l1Scores().isEmpty()) {
details.put("l1_scores", record.l1Scores());
}
if (record.relevanceLevel() != null) {
details.put("relevance_level", record.relevanceLevel());
}
if (record.completenessHint() != null) {
details.put("completeness_hint", record.completenessHint());
}
if (record.domain() != null && !record.domain().isBlank()) {
details.put("retrieved_domains", List.of(record.domain()));
}
if (record.dedupReason() != null) {
details.put("dedup_reason", record.dedupReason());
}
details.put("evidence_status", normalizeEvidenceStatus(record.success(), record.evidenceStatus()));
ToolInvocation invocation = ToolInvocation.builder()
.toolName("lookup_knowledge")
.inputParams(toJson(Map.of("query", record.query())))
.outputPreview(preview(record.outputPreview()))
.outputLength(record.outputLength())
.retrievalLayer(record.retrievalLayer())
.l0MatchCount(record.l0MatchCount())
.l1MatchCount(record.l1MatchCount())
.isTruncated(Boolean.TRUE.equals(record.truncated()))
.retrievalDetails(toJson(details))
.relevanceLevel(record.relevanceLevel())
.dedupReason(record.dedupReason())
.durationMs(record.durationMs())
.success(record.success())
.errorMessage(record.errorMessage())
.build();
save(invocation);
}
private String normalizeEvidenceStatus(boolean success, String evidenceStatus) {
if (evidenceStatus != null && !evidenceStatus.isBlank()) {
return evidenceStatus;
}
return success ? EVIDENCE_STATUS_SUPPORTED : EVIDENCE_STATUS_FAILED;
}
private String preview(String output) {
if (output == null) {
return null;
@@ -90,4 +177,105 @@ public class ToolInvocationRecorder {
return "{}";
}
}
@Builder
public record LookupKnowledgeRecord(
String query,
String outputPreview,
Integer outputLength,
String retrievalLayer,
Integer l0MatchCount,
Integer l1MatchCount,
Boolean truncated,
String relevanceLevel,
String completenessHint,
String domain,
String dedupReason,
Integer durationMs,
boolean success,
String evidenceStatus,
String errorMessage,
List<String> l0Titles,
Double l1TopScore,
Double l1TopSimilarity,
List<Double> l1Scores
) {
public static LookupKnowledgeRecord from(String query,
List<KnowledgeEntry> l0Matches,
List<VectorSearchService.SearchResult> l1Results,
boolean highConfidence,
LookupResult result,
String domain,
String dedupReason,
int durationMs,
double l1TopSimilarity) {
boolean hasL0 = l0Matches != null && !l0Matches.isEmpty();
boolean hasL1 = l1Results != null && !l1Results.isEmpty();
String layer;
if (hasL0 && !highConfidence) {
layer = "L0+L1";
} else if (hasL0) {
layer = "L0";
} else if (hasL1) {
layer = "L1";
} else {
layer = null;
}
String outputPreview = null;
int outputLength = 0;
boolean truncated = false;
if (result != null && result.getPrimary() != null && result.getPrimary().getContent() != null) {
outputPreview = result.getPrimary().getContent();
outputLength = outputPreview.length();
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
} else if (hasL1 && l1Results.get(0).getContent() != null) {
outputPreview = l1Results.get(0).getContent();
outputLength = outputPreview.length();
truncated = outputLength > OUTPUT_PREVIEW_LIMIT;
}
String evidenceStatus = EVIDENCE_STATUS_SUPPORTED;
if (dedupReason != null) {
evidenceStatus = EVIDENCE_STATUS_DEDUPED;
} else if (result == null || !result.isFound()) {
evidenceStatus = EVIDENCE_STATUS_NO_EVIDENCE;
}
List<String> l0Titles = new ArrayList<>();
if (hasL0) {
for (int i = 0; i < Math.min(3, l0Matches.size()); i++) {
l0Titles.add(l0Matches.get(i).getTitle());
}
}
List<Double> l1Scores = new ArrayList<>();
if (hasL1) {
for (int i = 0; i < Math.min(3, l1Results.size()); i++) {
l1Scores.add((double) l1Results.get(i).getScore());
}
}
return LookupKnowledgeRecord.builder()
.query(query)
.outputPreview(outputPreview)
.outputLength(outputLength)
.retrievalLayer(layer)
.l0MatchCount(hasL0 ? l0Matches.size() : null)
.l1MatchCount(hasL1 ? l1Results.size() : null)
.truncated(truncated)
.relevanceLevel(result != null ? result.getRelevanceLevel() : null)
.completenessHint(result != null ? result.getCompletenessHint() : null)
.domain(domain)
.dedupReason(dedupReason)
.durationMs(durationMs)
.success(true)
.evidenceStatus(evidenceStatus)
.l0Titles(l0Titles)
.l1TopScore(hasL1 ? (double) l1Results.get(0).getScore() : null)
.l1TopSimilarity(hasL1 ? l1TopSimilarity : null)
.l1Scores(l1Scores)
.build();
}
}
}
@@ -107,6 +107,7 @@ public class ToolTraceSummaryService {
}
private String extractOutputSummary(ToolInvocation invocation, String topicDomain) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
if (invocation.getErrorMessage() != null && !invocation.getErrorMessage().isBlank()) {
return "call failed: " + truncate(invocation.getErrorMessage(), 120);
@@ -114,6 +115,17 @@ public class ToolTraceSummaryService {
return "no usable evidence returned";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "retrieval skipped because the same document was already used in this session";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)) {
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return "completed without usable evidence: " + truncate(invocation.getOutputPreview(), 120);
}
return "completed without usable evidence";
}
if ("lookup_knowledge".equals(invocation.getToolName())) {
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
@@ -129,18 +141,46 @@ public class ToolTraceSummaryService {
}
private String determineEvidenceLevel(ToolInvocation invocation) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
return "none";
}
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "none";
}
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
return "direct";
}
if ("REFERENCE".equals(invocation.getRelevanceLevel())) {
return "indirect";
}
if (EVIDENCE_TOOLS.contains(invocation.getToolName())) {
return "direct";
}
return "none";
}
private String extractEvidenceStatus(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return Boolean.TRUE.equals(invocation.getSuccess())
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
Object evidenceStatus = details.get("evidence_status");
if (evidenceStatus != null) {
return String.valueOf(evidenceStatus);
}
} catch (Exception e) {
log.debug("Failed to parse evidence_status", e);
}
return Boolean.TRUE.equals(invocation.getSuccess())
? ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED
: ToolInvocationRecorder.EVIDENCE_STATUS_FAILED;
}
private List<String> extractStringList(Object value) {
if (!(value instanceof List<?> list) || list.isEmpty()) {
return List.of();
@@ -224,13 +264,22 @@ public class ToolTraceSummaryService {
inputSummary = extractInputSummary(invocation);
}
boolean invocationSuccess = Boolean.TRUE.equals(invocation.getSuccess());
if (!invocationSuccess) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
failedCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
if (invocation.getDedupReason() != null) {
if (ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE.equals(evidenceStatus)
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
noHitCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
@@ -14,6 +14,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
/**
@@ -309,131 +310,30 @@ public class LookupKnowledgeTool {
String sessionId = SessionContextHolder.getSessionId();
if (sessionId == null) return;
boolean hasL0 = l0Matches != null && !l0Matches.isEmpty();
boolean hasL1 = l1Results != null && !l1Results.isEmpty();
long duration = System.currentTimeMillis() - startTime;
double l1TopSimilarity = (l1Results != null && !l1Results.isEmpty())
? normalizeL2(l1Results.get(0).getScore())
: -1;
String layer;
String outputPreview = null;
int outputLength = 0;
int l0Count = 0;
int l1Count = 0;
boolean truncated = false;
if (hasL0 && !highConfidence) {
layer = "L0+L1";
l0Count = l0Matches.size();
l1Count = l1Results.size();
} else if (hasL0) {
layer = "L0";
l0Count = l0Matches.size();
} else if (hasL1) {
layer = "L1";
l1Count = l1Results.size();
} else {
layer = null;
}
// output_preview
if (result != null && result.getPrimary() != null && result.getPrimary().getContent() != null) {
String content = result.getPrimary().getContent();
outputLength = content.length();
if (content.length() > 500) {
outputPreview = content.substring(0, 500) + "...";
truncated = true;
} else {
outputPreview = content;
}
} else if (l1Results != null && !l1Results.isEmpty() && l1Results.get(0).getContent() != null) {
String content = l1Results.get(0).getContent();
outputLength = content.length();
if (content.length() > 500) {
outputPreview = content.substring(0, 500) + "...";
truncated = true;
} else {
outputPreview = content;
}
}
// L1 top score + similarity
float l1TopScore = (hasL1) ? l1Results.get(0).getScore() : -1;
double l1TopSimilarity = (hasL1) ? normalizeL2(l1TopScore) : -1;
// 构建检索明细 JSON(扩展版)
StringBuilder details = new StringBuilder("{");
if (hasL0) {
details.append("\"l0_match_count\":").append(l0Count).append(",");
details.append("\"l0_titles\":[");
for (int i = 0; i < Math.min(3, l0Matches.size()); i++) {
if (i > 0) details.append(",");
details.append("\"").append(escapeJson(l0Matches.get(i).getTitle())).append("\"");
}
details.append("],");
}
if (hasL1) {
details.append("\"l1_top_score\":").append(String.format("%.4f", l1TopScore)).append(",");
details.append("\"l1_top_similarity\":").append(String.format("%.4f", l1TopSimilarity)).append(",");
details.append("\"l1_match_count\":").append(l1Count).append(",");
details.append("\"l1_scores\":[");
for (int i = 0; i < Math.min(3, l1Results.size()); i++) {
if (i > 0) details.append(",");
details.append(String.format("%.4f", l1Results.get(i).getScore()));
}
details.append("],");
}
// 归一化信息
if (result != null && result.getRelevanceLevel() != null) {
details.append("\"relevance_level\":\"").append(result.getRelevanceLevel()).append("\",");
details.append("\"completeness_hint\":\"").append(escapeJson(result.getCompletenessHint())).append("\",");
}
// 域信息
if (domain != null) {
details.append("\"retrieved_domains\":[\"").append(escapeJson(domain)).append("\"],");
}
// 去重原因
if (dedupReason != null) {
details.append("\"dedup_reason\":\"").append(dedupReason).append("\",");
}
// 移除末尾逗号
if (details.charAt(details.length() - 1) == ',') {
details.setLength(details.length() - 1);
}
details.append("}");
ToolInvocation inv = ToolInvocation.builder()
.sessionId(sessionId)
.toolName("lookup_knowledge")
.inputParams("{\"query\":\"" + escapeJson(query) + "\"}")
.outputPreview(outputPreview)
.outputLength(outputLength)
.retrievalLayer(layer)
.l0MatchCount(hasL0 ? l0Count : null)
.l1MatchCount(hasL1 ? l1Count : null)
.isTruncated(truncated)
.retrievalDetails(details.toString())
.relevanceLevel(result != null ? result.getRelevanceLevel() : null)
.dedupReason(dedupReason)
.durationMs((int) duration)
.success(true)
.build();
toolInvocationRecorder.save(inv);
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.from(
query,
l0Matches,
l1Results,
highConfidence,
result,
domain,
dedupReason,
(int) duration,
l1TopSimilarity
);
toolInvocationRecorder.recordLookupKnowledge(record);
log.debug("tool_invocation 已保存: sessionId={}, layer={}, relevanceLevel={}, duration={}ms",
sessionId, layer, result != null ? result.getRelevanceLevel() : null, duration);
sessionId, record.retrievalLayer(), record.relevanceLevel(), duration);
} catch (Exception e) {
log.error("保存 tool_invocation 失败", e);
}
}
private String escapeJson(String s) {
if (s == null) return "";
return s.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
// ==================== 结果组装 ====================
private LookupResult buildResult(