osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): harden live diagnosis skill observability

Browse Source
This commit is contained in:
aruo
2026-07-07 00:20:34 +08:00
parent 64adb998cf
commit b3315ead52
20 changed files with 792 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/service/ChatService.java
+22 -3
View File
@@ -544,6 +544,10 @@ public class ChatService {
private ReactAgent buildChatExecutorAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
List<Map<String, String>> history, String retryContext) {
StringBuilder prompt = new StringBuilder(chatExecutorPrompt);
prompt.append("\n\n--- Skill 读取约束 ---\n")
.append("如果 planner_plan 已给出 selected_skill，本轮 Executor 只允许对该 skill 调用一次 read_skill。")
.append("读取后必须复用已加载的 playbook 指令继续执行证据工具，不要为了检查支持文件、确认流程或生成报告再次读取同一个 skill。")
.append("只有 ChatService 启动新的补证据 retry round 时，才可以重新读取 selected_skill。\n");
if (!history.isEmpty()) {
prompt.append("\n\n--- 对话历史 ---\n");
for (Map<String, String> msg : history) {
@@ -764,14 +768,29 @@ public class ChatService {
private String buildLowConfidenceOutput(String executorAnswer, VerifierDecision decision) {
StringBuilder output = new StringBuilder(LOW_CONFID_DISCLAIMER);
output.append("\n\n").append(executorAnswer == null ? "" : executorAnswer);
List<String> confirmedFacts = extractConfirmedFacts(decision);
output.append("\n\n已确认信息：");
if (confirmedFacts.isEmpty()) {
output.append("\n- 暂无可稳定确认的信息");
} else {
for (String fact : confirmedFacts) {
output.append("\n- ").append(fact);
}
}
List<String> gaps = extractEvidenceGaps(decision);
output.append("\n\n当前缺口：");
if (!gaps.isEmpty()) {
output.append("\n\n当前缺口：");
for (String gap : gaps) {
output.append("\n- ").append(gap);
}
} else {
output.append("\n- 当前缺少足够的直接证据支撑核心结论");
}
output.append("\n\n建议下一步：");
for (String suggestion : buildNextStepSuggestions(decision)) {
output.append("\n- ").append(suggestion);
}
return output.toString();
}
@@ -813,7 +832,7 @@ public class ChatService {
for (Map<String, Object> fact : decision.factsChecked()) {
String verification = String.valueOf(fact.get("verification"));
boolean critical = Boolean.TRUE.equals(fact.get("is_critical"));
if (critical && ("direct_evidence".equals(verification) || "indirect_support".equals(verification))) {
if (critical && "direct_evidence".equals(verification)) {
confirmedFacts.add(String.valueOf(fact.get("fact")));
}
}
src/main/java/com/superbiz/agent/service/ToolInvocationRecorder.java
+21 -16
View File
@@ -138,21 +138,11 @@ public class ToolInvocationRecorder {
if (record.evidenceBlockCount() != null) {
details.put("evidence_block_count", record.evidenceBlockCount());
}
if (record.evidenceBlocks() != null && !record.evidenceBlocks().isEmpty()) {
details.put("evidence_blocks", record.evidenceBlocks());
}
if (record.queryTransform() != null && !record.queryTransform().isEmpty()) {
details.put("query_transform", record.queryTransform());
}
if (record.retrievalTrace() != null && !record.retrievalTrace().isEmpty()) {
details.put("retrieval_trace", record.retrievalTrace());
}
if (record.contextPack() != null && !record.contextPack().isEmpty()) {
details.put("context_pack_summary", record.contextPack());
}
if (record.rerankTrace() != null && !record.rerankTrace().isEmpty()) {
details.put("rerank_trace", record.rerankTrace());
}
details.put("evidence_blocks", record.evidenceBlocks() == null ? List.of() : record.evidenceBlocks());
details.put("query_transform", record.queryTransform() == null ? Map.of() : record.queryTransform());
details.put("retrieval_trace", record.retrievalTrace() == null ? Map.of() : record.retrievalTrace());
details.put("context_pack_summary", record.contextPack() == null ? Map.of() : record.contextPack());
details.put("rerank_trace", record.rerankTrace() == null ? Map.of() : record.rerankTrace());
if (record.fallbackReason() != null && !record.fallbackReason().isBlank()) {
details.put("fallback_reason", record.fallbackReason());
}
@@ -253,7 +243,7 @@ public class ToolInvocationRecorder {
String dedupReason,
int durationMs) {
RetrievalTrace trace = result != null ? result.getRetrievalTrace() : null;
String layer = trace != null ? trace.getSelectedAttempt() : null;
String layer = recordedRetrievalLayer(query, trace);
String outputPreview = null;
int outputLength = 0;
@@ -462,5 +452,20 @@ public class ToolInvocationRecorder {
}
return scores;
}
private static String recordedRetrievalLayer(KnowledgeQuery query, RetrievalTrace trace) {
if (trace == null || trace.getSelectedAttempt() == null) {
return null;
}
boolean hasL0 = query != null && query.getL0MatchCount() != null && query.getL0MatchCount() > 0;
boolean hasL1 = trace.getSelectedAttempt().contains("VECTOR");
if (hasL0 && hasL1) {
return "L0+L1";
}
if (hasL1) {
return "L1";
}
return hasL0 ? "L0" : null;
}
}
}
src/main/java/com/superbiz/agent/service/ToolTraceSummaryService.java
+180 -3
View File
@@ -15,6 +15,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Builds a verifier-facing evidence index from persisted tool invocations.
@@ -25,6 +27,7 @@ public class ToolTraceSummaryService {
private static final TypeReference<LinkedHashMap<String, Object>> MAP_TYPE = new TypeReference<>() {};
private static final Set<String> EVIDENCE_TOOLS = Set.of("lookup_knowledge", "query_logs", "query_metrics", "query_order");
private static final Pattern JSON_STRING_FIELD = Pattern.compile("\"%s\"\\s*:\\s*\"((?:\\\\.|[^\"])*)\"");
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
@@ -128,18 +131,156 @@ public class ToolTraceSummaryService {
if ("lookup_knowledge".equals(invocation.getToolName())) {
String relevance = invocation.getRelevanceLevel() != null ? invocation.getRelevanceLevel() : "UNKNOWN";
String trace = extractLookupTraceSummary(invocation);
String preview = invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()
? truncate(invocation.getOutputPreview(), 160)
? truncate(invocation.getOutputPreview(), 240)
: "no preview";
return "matched domain=" + topicDomain + ", relevance=" + relevance + ", preview=" + preview;
return "matched domain=" + topicDomain + ", relevance=" + relevance + trace + ", preview=" + preview;
}
if ("query_logs".equals(invocation.getToolName())) {
String concrete = extractLogEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if ("query_metrics".equals(invocation.getToolName())) {
String concrete = extractMetricsEvidenceSummary(invocation.getOutputPreview());
if (!concrete.isBlank()) {
return concrete;
}
}
if (invocation.getOutputPreview() != null && !invocation.getOutputPreview().isBlank()) {
return truncate(invocation.getOutputPreview(), 160);
return truncate(invocation.getOutputPreview(), 240);
}
return "evidence retrieved without preview";
}
private String extractLookupTraceSummary(ToolInvocation invocation) {
if (invocation.getRetrievalDetails() == null || invocation.getRetrievalDetails().isBlank()) {
return "";
}
try {
Map<String, Object> details = objectMapper.readValue(invocation.getRetrievalDetails(), MAP_TYPE);
List<String> sources = new ArrayList<>();
Object evidenceBlocks = details.get("evidence_blocks");
if (evidenceBlocks instanceof List<?> blocks) {
for (Object block : blocks) {
if (block instanceof Map<?, ?> blockMap) {
Object title = blockMap.get("title");
Object source = blockMap.get("source");
String label = title != null ? String.valueOf(title) : String.valueOf(source);
if (label != null && !label.isBlank() && !"null".equals(label)) {
sources.add(label);
}
}
if (sources.size() >= 3) {
break;
}
}
}
Object retrievalTrace = details.get("retrieval_trace");
String selectedAttempt = "";
if (retrievalTrace instanceof Map<?, ?> traceMap && traceMap.get("selected_attempt") != null) {
selectedAttempt = ", selected_attempt=" + traceMap.get("selected_attempt");
}
return (sources.isEmpty() ? "" : ", sources=" + truncate(String.join("|", sources), 180)) + selectedAttempt;
} catch (Exception e) {
log.debug("Failed to parse lookup retrieval details", e);
return "";
}
}
private String extractLogEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
List<String> services = extractJsonStringFields(outputPreview, "service", 3);
List<String> levels = extractJsonStringFields(outputPreview, "level", 3);
List<String> timestamps = extractJsonStringFields(outputPreview, "timestamp", 3);
if (messages.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
for (int i = 0; i < messages.size(); i++) {
String prefix = labelAt(timestamps, i) + labelAt(levels, i) + labelAt(services, i);
rows.add((prefix.isBlank() ? "" : prefix + " ") + truncate(messages.get(i), 220));
}
return "log_evidence: " + truncate(String.join(" | ", rows), 520);
}
private boolean hasOnlyGenericMockLogMessages(String outputPreview) {
List<String> messages = extractJsonStringFields(outputPreview, "message", 3);
if (messages.isEmpty()) {
return false;
}
return messages.stream()
.allMatch(message -> message.startsWith("日志消息 #") && message.contains("查询条件:"));
}
private String extractMetricsEvidenceSummary(String outputPreview) {
if (outputPreview == null || outputPreview.isBlank()) {
return "";
}
List<String> alertNames = extractJsonStringFields(outputPreview, "alert_name", 5);
List<String> descriptions = extractJsonStringFields(outputPreview, "description", 5);
List<String> services = extractJsonStringFields(outputPreview, "service", 5);
if (alertNames.isEmpty() && descriptions.isEmpty()) {
return "";
}
List<String> rows = new ArrayList<>();
int count = Math.max(alertNames.size(), descriptions.size());
for (int i = 0; i < Math.min(5, count); i++) {
StringBuilder row = new StringBuilder();
if (i < alertNames.size()) {
row.append(alertNames.get(i));
}
if (i < services.size()) {
if (!row.isEmpty()) {
row.append(" ");
}
row.append("service=").append(services.get(i));
}
if (i < descriptions.size()) {
if (!row.isEmpty()) {
row.append(": ");
}
row.append(descriptions.get(i));
}
rows.add(truncate(row.toString(), 220));
}
return "metric_evidence: " + truncate(String.join(" | ", rows), 520);
}
private List<String> extractJsonStringFields(String text, String field, int limit) {
Pattern pattern = Pattern.compile(String.format(JSON_STRING_FIELD.pattern(), Pattern.quote(field)));
Matcher matcher = pattern.matcher(text);
List<String> values = new ArrayList<>();
while (matcher.find() && values.size() < limit) {
values.add(unescapeJsonString(matcher.group(1)));
}
return values;
}
private String unescapeJsonString(String value) {
return value == null ? "" : value
.replace("\\\"", "\"")
.replace("\\\\", "\\")
.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t");
}
private String labelAt(List<String> values, int index) {
if (index >= values.size() || values.get(index) == null || values.get(index).isBlank()) {
return "";
}
return "[" + values.get(index) + "]";
}
private String determineEvidenceLevel(ToolInvocation invocation) {
String evidenceStatus = extractEvidenceStatus(invocation);
if (!Boolean.TRUE.equals(invocation.getSuccess())) {
@@ -149,6 +290,10 @@ public class ToolTraceSummaryService {
|| ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED.equals(evidenceStatus)) {
return "none";
}
if ("query_logs".equals(invocation.getToolName())
&& hasOnlyGenericMockLogMessages(invocation.getOutputPreview())) {
return "none";
}
if ("PRECISE".equals(invocation.getRelevanceLevel()) || "HIGHLY_RELEVANT".equals(invocation.getRelevanceLevel())) {
return "direct";
}
@@ -283,13 +428,45 @@ public class ToolTraceSummaryService {
}
String invocationEvidenceLevel = determineEvidenceLevel(invocation);
if ("none".equals(invocationEvidenceLevel)) {
noHitCount++;
if (outputSummary == null || outputSummary.isBlank()) {
outputSummary = extractOutputSummary(invocation, topicDomain);
}
return;
}
if (!success || evidenceRank(invocationEvidenceLevel) > evidenceRank(evidenceLevel)) {
success = true;
evidenceLevel = invocationEvidenceLevel;
outputSummary = extractOutputSummary(invocation, topicDomain);
} else if (evidenceRank(invocationEvidenceLevel) == evidenceRank(evidenceLevel)) {
String candidateSummary = extractOutputSummary(invocation, topicDomain);
if (isMoreConcrete(candidateSummary, outputSummary)) {
outputSummary = candidateSummary;
}
}
}
private boolean isMoreConcrete(String candidate, String current) {
return concretenessScore(candidate) > concretenessScore(current);
}
private int concretenessScore(String summary) {
if (summary == null || summary.isBlank()) {
return 0;
}
int score = summary.length() > 160 ? 2 : 1;
if (summary.contains("log_evidence") || summary.contains("metric_evidence")) {
score += 5;
}
if (summary.contains("连接池耗尽") || summary.contains("OutOfMemoryError")
|| summary.contains("扫描行数") || summary.contains("HighMemoryUsage")
|| summary.contains("HighCPUUsage")) {
score += 4;
}
return score;
}
int relevanceScore(String answer) {
int score = success ? 10 : 0;
if ("direct".equals(evidenceLevel)) {
src/main/resources/prompts/chat-planner-prompt.md
+5
View File
@@ -9,6 +9,8 @@
```json
{
"selected_skill": "匹配的 skill 名称；如果没有匹配则为 null",
"selection_reason": "选择该 skill 的原因；如果没有匹配则说明不使用 skill",
"plan": ["步骤1描述", "步骤2描述", "步骤3描述"],
"reasoning": "规划思路说明"
}
@@ -18,6 +20,9 @@
- 每个步骤应该是一个可以独立执行的任务
- 步骤要具体可操作，不要模糊
- 如果问题需要查知识库，明确在步骤中说明要查什么
- 如果存在 `skill_catalog`，必须先根据 skill 的 name/description 判断是否匹配用户问题
- 如果匹配某个诊断 Skill，必须在顶层 `selected_skill` 填入 skill 名称，并在计划第一步说明 Executor 需要读取该 skill
- Planner 只能选择 skill 元数据，不能调用 `read_skill`，也不能编造 skill 正文内容
## 知识库检索规则
- 制定步骤前，先查看下方 `available_knowledge_domains`（如果存在）
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+53 -1
View File
@@ -86,10 +86,61 @@ class ChatServiceSequentialAgentTest {
"sequential-low-confidence-session"
);
assertTrue(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
assertTrue(result.answer().contains("当前缺口"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "LOW_CONFID",
"groundedness_score": 0.37,
"critical_fact_count": 3,
"facts_checked": [
{
"fact": "连接池耗尽 active=50/50",
"is_critical": true,
"verification": "direct_evidence",
"detail": "log evidence",
"evidence_refs": []
},
{
"fact": "临时扩容连接池到 80",
"is_critical": true,
"verification": "indirect_support",
"detail": "suggestion inferred from evidence",
"evidence_refs": []
},
{
"fact": "OOM 导致连接泄漏",
"is_critical": true,
"verification": "no_evidence",
"detail": "missing OOM log",
"evidence_refs": []
}
],
"rationale": "scripted low confidence"
}
""");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-low-confid-direct-only-session"
);
assertTrue(result.answer().contains("已确认信息：\n- 连接池耗尽 active=50/50"));
assertFalse(result.answer().contains("临时扩容连接池到 80"));
assertTrue(result.answer().contains("当前缺口：\n- OOM 导致连接泄漏：missing OOM log"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
ChatService chatService = createChatService();
@@ -243,6 +294,7 @@ class ChatServiceSequentialAgentTest {
assertTrue(chatModel.executorPromptText.contains("## Skills System"));
assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool"));
assertTrue(chatModel.executorPromptText.contains("read_skill"));
assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill"));
assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool"));
assertFalse(chatModel.verifierPromptText.contains("read_skill"));
}
src/test/java/com/superbiz/agent/service/ToolInvocationRecorderTest.java
+7 -1
View File
@@ -113,6 +113,10 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));
assertTrue(saved.getRetrievalDetails().contains("\"rerank_trace\""));
}
@Test
@@ -176,7 +180,9 @@ class ToolInvocationRecorderTest {
assertEquals(1, record.evidenceBlockCount());
assertEquals(1, record.evidenceBlocks().size());
assertTrue(String.valueOf(record.evidenceBlocks().get(0).get("content_preview")).endsWith("..."));
assertEquals("UNFILTERED_VECTOR", record.retrievalLayer());
assertEquals("L1", record.retrievalLayer());
assertEquals(3, record.l1MatchCount());
assertTrue(record.retrievalTrace().containsKey("selected_attempt"));
assertTrue(record.contextPack().containsKey("included_sources"));
}
}
src/test/java/com/superbiz/agent/service/ToolTraceSummaryServiceTest.java
+108
View File
@@ -62,6 +62,7 @@ class ToolTraceSummaryServiceTest {
assertEquals("direct", logsSummary.get("evidence_level"));
assertEquals(2, logsSummary.get("invocation_count"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("payment timeout stack trace"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
@@ -72,4 +73,111 @@ class ToolTraceSummaryServiceTest {
assertEquals(1, metricsSummary.get("failed_invocation_count"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
}
@Test
void buildVerifierTraceSummaryPreservesConcreteFactsFromTruncatedLogAndMetricRows() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String logPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 22:15:45",
"level" : "ERROR",
"service" : "order-service",
"message" : "数据库连接池耗尽: Cannot acquire connection from pool, active: 50/50, waiting: 23, timeout: 30000ms"
} ]
}
""";
String metricPreview = """
{
"success" : true,
"alerts" : [ {
"alert_name" : "HighCPUUsage",
"service" : "payment-service",
"description" : "服务 payment-service 的 CPU 使用率持续超过 80%，当前值为 92%。"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-2")).thenReturn(List.of(
ToolInvocation.builder()
.id(10L)
.sessionId("session-2")
.toolName("query_logs")
.inputParams("{\"query\":\"pool\"}")
.outputPreview(logPreview)
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build(),
ToolInvocation.builder()
.id(11L)
.sessionId("session-2")
.toolName("query_metrics")
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
.outputPreview(metricPreview)
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-2", "连接池耗尽 HighCPUUsage");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("连接池耗尽"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("active: 50/50"));
assertEquals(List.of(10L), logsSummary.get("source_invocation_ids"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("HighCPUUsage"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("payment-service"));
}
@Test
void buildVerifierTraceSummaryDoesNotTreatGenericMockLogsAsDirectEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String genericLogPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 23:44:41",
"level" : "ERROR",
"service" : "generic-service",
"message" : "日志消息 #0, 查询条件: service:payment-service"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-3")).thenReturn(List.of(
ToolInvocation.builder()
.id(20L)
.sessionId("session-3")
.toolName("query_logs")
.inputParams("{\"query\":\"service:payment-service\"}")
.outputPreview(genericLogPreview)
.retrievalDetails("{\"retrieved_domains\":[\"system-metrics\"],\"evidence_status\":\"supported\"}")
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-3", "payment-service timeout");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertEquals(Boolean.FALSE, logsSummary.get("success"));
assertEquals("none", logsSummary.get("evidence_level"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("日志消息 #0"));
}
}