osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): harden live diagnosis skill observability

Browse Source
This commit is contained in:
aruo
2026-07-07 00:20:34 +08:00
parent 64adb998cf
commit b3315ead52
20 changed files with 792 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/service/ChatService.java
+22 -3
View File
@@ -544,6 +544,10 @@ public class ChatService {
private ReactAgent buildChatExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
List<Map<String, String>> history, String retryContext) {
StringBuilder prompt = new StringBuilder(chatExecutorPrompt);
prompt.append("\n\n--- Skill 读取约束 ---\n")
.append("如果 planner_plan 已给出 selected_skill，本轮 Executor 只允许对该 skill 调用一次 read_skill。")
.append("读取后必须复用已加载的 playbook 指令继续执行证据工具，不要为了检查支持文件、确认流程或生成报告再次读取同一个 skill。")
.append("只有 ChatService 启动新的补证据 retry round 时，才可以重新读取 selected_skill。\n");
if (!history.isEmpty()) {
prompt.append("\n\n--- 对话历史 ---\n");
for (Map<String, String> msg : history) {
@@ -764,14 +768,29 @@ public class ChatService {
private String buildLowConfidenceOutput(String executorAnswer, VerifierDecision decision) {
StringBuilder output = new StringBuilder(LOW_CONFID_DISCLAIMER);
output.append("\n\n").append(executorAnswer == null ? "" : executorAnswer);
List<String> confirmedFacts = extractConfirmedFacts(decision);
output.append("\n\n已确认信息：");
if (confirmedFacts.isEmpty()) {
output.append("\n- 暂无可稳定确认的信息");
} else {
for (String fact : confirmedFacts) {
output.append("\n- ").append(fact);
}
}
List<String> gaps = extractEvidenceGaps(decision);
output.append("\n\n当前缺口：");
if (!gaps.isEmpty()) {
output.append("\n\n当前缺口：");
for (String gap : gaps) {
output.append("\n- ").append(gap);
}
} else {
output.append("\n- 当前缺少足够的直接证据支撑核心结论");
}
output.append("\n\n建议下一步：");
for (String suggestion : buildNextStepSuggestions(decision)) {
output.append("\n- ").append(suggestion);
}
return output.toString();
}
@@ -813,7 +832,7 @@ public class ChatService {
for (Map<String, Object> fact : decision.factsChecked()) {
String verification = String.valueOf(fact.get("verification"));
boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
if (critical && ("direct_evidence".equals(verification) || "indirect_support".equals(verification))) {
if (critical && "direct_evidence".equals(verification)) {
confirmedFacts.add(String.valueOf(fact.get("fact")));
}
}
src/main/java/com/superbiz/agent/service/ToolInvocationRecorder.java
+21 -16
View File
@@ -138,21 +138,11 @@ public class ToolInvocationRecorder {
if (record.evidenceBlockCount() != null) {
details.put("evidence_block_count", record.evidenceBlockCount());
}
if (record.evidenceBlocks() != null && !record.evidenceBlocks().isEmpty()) {
details.put("evidence_blocks", record.evidenceBlocks());
}
if (record.queryTransform() != null && !record.queryTransform().isEmpty()) {
details.put("query_transform", record.queryTransform());
}
if (record.retrievalTrace() != null && !record.retrievalTrace().isEmpty()) {
details.put("retrieval_trace", record.retrievalTrace());
}
if (record.contextPack() != null && !record.contextPack().isEmpty()) {
details.put("context_pack_summary", record.contextPack());
}
if (record.rerankTrace() != null && !record.rerankTrace().isEmpty()) {
details.put("rerank_trace", record.rerankTrace());
}
details.put("evidence_blocks", record.evidenceBlocks() == null ? List.of() : record.evidenceBlocks());
details.put("query_transform", record.queryTransform() == null ? Map.of() : record.queryTransform());
details.put("retrieval_trace", record.retrievalTrace() == null ? Map.of() : record.retrievalTrace());
details.put("context_pack_summary", record.contextPack() == null ? Map.of() : record.contextPack());
details.put("rerank_trace", record.rerankTrace() == null ? Map.of() : record.rerankTrace());
if (record.fallbackReason() != null && !record.fallbackReason().isBlank()) {
details.put("fallback_reason", record.fallbackReason());
}
@@ -253,7 +243,7 @@ public class ToolInvocationRecorder {
String dedupReason,
int durationMs) {
RetrievalTrace trace = result != null ? result.getRetrievalTrace() : null;
String layer = trace != null ? trace.getSelectedAttempt() : null;
String layer = recordedRetrievalLayer(query, trace);
String outputPreview = null;
int outputLength = 0;
@@ -462,5 +452,20 @@ public class ToolInvocationRecorder {
}
return scores;
}
private static String recordedRetrievalLayer(KnowledgeQuery query, RetrievalTrace trace) {
if (trace == null || trace.getSelectedAttempt() == null) {
return null;
}
boolean hasL0 = query != null && query.getL0MatchCount() != null && query.getL0MatchCount() > 0;
boolean hasL1 = trace.getSelectedAttempt().contains("VECTOR");
if (hasL0 && hasL1) {
return "L0+L1";
}
if (hasL1) {
return "L1";
}
return hasL0 ? "L0" : null;
}
}
}
src/main/java/com/superbiz/agent/service/ToolTraceSummaryService.java
+180 -3
View File
@@ -15,6 +15,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Builds a verifier-facing evidence index from persisted tool invocations.
@@ -25,6 +27,7 @@ public class ToolTraceSummaryService {
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
private static final Set<String> EVIDENCE_TOOLS = Set.of("lookup_knowledge", "query_logs", "query_metrics", "query_order");
private static final Pattern JSON_STRING_FIELD = Pattern.compile("\"%s\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"");
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
@@ -128,18 +131,156 @@ public class ToolTraceSummaryService {
if ("lookup_knowledge".equals(invocation.getToolName())) {
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
String trace = extractLookupTraceSummary(invocation);
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
? truncate(invocation.getOutputPreview(), 160)
? truncate(invocation.getOutputPreview(), 240)
: "no preview";
return "matched domain=" + topicDomain + ", relevance=" + relevance + ", preview=" + preview;
return "matched domain=" + topicDomain + ", relevance=" + relevance + trace + ", preview=" + preview;
}
if ("query_logs".equals(invocation.getToolName())) {
String concrete = extractLogEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if ("query_metrics".equals(invocation.getToolName())) {
String concrete = extractMetricsEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return truncate(invocation.getOutputPreview(), 160);
return truncate(invocation.getOutputPreview(), 240);
}
return "evidence retrieved without preview";
}
private String extractLookupTraceSummary(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return "";
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
List<String> sources = new ArrayList<>();
Object evidenceBlocks = details.get("evidence_blocks");
if (evidenceBlocks instanceof List<?> blocks) {
for (Object block : blocks) {
if (block instanceof Map<?, ?> blockMap) {
Object title = blockMap.get("title");
Object source = blockMap.get("source");
String label = title != null ? String.valueOf(title) : String.valueOf(source);
if (label != null && !label.isBlank() && !"null".equals(label)) {
sources.add(label);
}
}
if (sources.size() >= 3) {
break;
}
}
}
Object retrievalTrace = details.get("retrieval_trace");
String selectedAttempt = "";
if (retrievalTrace instanceof Map<?, ?> traceMap && traceMap.get("selected_attempt") != null) {
selectedAttempt = ", selected_attempt=" + traceMap.get("selected_attempt");
}
return (sources.isEmpty() ? "" : ", sources=" + truncate(String.join("|", sources), 180)) + selectedAttempt;
} catch (Exception e) {
log.debug("Failed to parse lookup retrieval details", e);
return "";
}
}
private String extractLogEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
List<String> services = extractJsonStringFields(outputPreview, "service", 3);
List<String> levels = extractJsonStringFields(outputPreview, "level", 3);
List<String> timestamps = extractJsonStringFields(outputPreview, "timestamp", 3);
if (messages.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
for (int i = 0; i < messages.size(); i++) {
String prefix = labelAt(timestamps, i) + labelAt(levels, i) + labelAt(services, i);
rows.add((prefix.isBlank() ? "" : prefix + " ") + truncate(messages.get(i), 220));
}
return "log_evidence: " + truncate(String.join(" | ", rows), 520);
}
private boolean hasOnlyGenericMockLogMessages(String outputPreview) {
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
if (messages.isEmpty()) {
return false;
}
return messages.stream()
.allMatch(message -> message.startsWith("日志消息 #") && message.contains("查询条件:"));
}
private String extractMetricsEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> alertNames = extractJsonStringFields(outputPreview, "alert_name", 5);
List<String> descriptions = extractJsonStringFields(outputPreview, "description", 5);
List<String> services = extractJsonStringFields(outputPreview, "service", 5);
if (alertNames.isEmpty() && descriptions.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
int count = Math.max(alertNames.size(), descriptions.size());
for (int i = 0; i < Math.min(5, count); i++) {
StringBuilder row = new StringBuilder();
if (i < alertNames.size()) {
row.append(alertNames.get(i));
}
if (i < services.size()) {
if (!row.isEmpty()) {
row.append(" ");
}
row.append("service=").append(services.get(i));
}
if (i < descriptions.size()) {
if (!row.isEmpty()) {
row.append(": ");
}
row.append(descriptions.get(i));
}
rows.add(truncate(row.toString(), 220));
}
return "metric_evidence: " + truncate(String.join(" | ", rows), 520);
}
private List<String> extractJsonStringFields(String text, String field, int limit) {
Pattern pattern = Pattern.compile(String.format(JSON_STRING_FIELD.pattern(), Pattern.quote(field)));
Matcher matcher = pattern.matcher(text);
List<String> values = new ArrayList<>();
while (matcher.find() && values.size() < limit) {
values.add(unescapeJsonString(matcher.group(1)));
}
return values;
}
private String unescapeJsonString(String value) {
return value == null ? "" : value
.replace("\\\"", "\"")
.replace("\\\\", "\\")
.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t");
}
private String labelAt(List<String> values, int index) {
if (index >= values.size() || values.get(index) == null || values.get(index).isBlank()) {
return "";
}
return "[" + values.get(index) + "]";
}
private String determineEvidenceLevel(ToolInvocation invocation) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
@@ -149,6 +290,10 @@ public class ToolTraceSummaryService {
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "none";
}
if ("query_logs".equals(invocation.getToolName())
&& hasOnlyGenericMockLogMessages(invocation.getOutputPreview())) {
return "none";
}
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
return "direct";
}
@@ -283,13 +428,45 @@ public class ToolTraceSummaryService {
}
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
if ("none".equals(invocationEvidenceLevel)) {
noHitCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
if (!success || evidenceRank(invocationEvidenceLevel) > evidenceRank(evidenceLevel)) {
success = true;
evidenceLevel = invocationEvidenceLevel;
outputSummary = extractOutputSummary(invocation, topicDomain);
} else if (evidenceRank(invocationEvidenceLevel) == evidenceRank(evidenceLevel)) {
String candidateSummary = extractOutputSummary(invocation, topicDomain);
if (isMoreConcrete(candidateSummary, outputSummary)) {
outputSummary = candidateSummary;
}
}
}
private boolean isMoreConcrete(String candidate, String current) {
return concretenessScore(candidate) > concretenessScore(current);
}
private int concretenessScore(String summary) {
if (summary == null || summary.isBlank()) {
return 0;
}
int score = summary.length() > 160 ? 2 : 1;
if (summary.contains("log_evidence") || summary.contains("metric_evidence")) {
score += 5;
}
if (summary.contains("连接池耗尽") || summary.contains("OutOfMemoryError")
|| summary.contains("扫描行数") || summary.contains("HighMemoryUsage")
|| summary.contains("HighCPUUsage")) {
score += 4;
}
return score;
}
int relevanceScore(String answer) {
int score = success ? 10 : 0;
if ("direct".equals(evidenceLevel)) {