osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): harden live diagnosis skill observability

Browse Source
This commit is contained in:
aruo
2026-07-07 00:20:34 +08:00
parent 64adb998cf
commit b3315ead52
20 changed files with 792 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+53 -1
View File
@@ -86,10 +86,61 @@ class ChatServiceSequentialAgentTest {
"sequential-low-confidence-session"
);
assertTrue(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
assertTrue(result.answer().contains("当前缺口"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "LOW_CONFID",
"groundedness_score": 0.37,
"critical_fact_count": 3,
"facts_checked": [
{
"fact": "连接池耗尽 active=50/50",
"is_critical": true,
"verification": "direct_evidence",
"detail": "log evidence",
"evidence_refs": []
},
{
"fact": "临时扩容连接池到 80",
"is_critical": true,
"verification": "indirect_support",
"detail": "suggestion inferred from evidence",
"evidence_refs": []
},
{
"fact": "OOM 导致连接泄漏",
"is_critical": true,
"verification": "no_evidence",
"detail": "missing OOM log",
"evidence_refs": []
}
],
"rationale": "scripted low confidence"
}
""");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-low-confid-direct-only-session"
);
assertTrue(result.answer().contains("已确认信息：\n- 连接池耗尽 active=50/50"));
assertFalse(result.answer().contains("临时扩容连接池到 80"));
assertTrue(result.answer().contains("当前缺口：\n- OOM 导致连接泄漏：missing OOM log"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
ChatService chatService = createChatService();
@@ -243,6 +294,7 @@ class ChatServiceSequentialAgentTest {
assertTrue(chatModel.executorPromptText.contains("## Skills System"));
assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool"));
assertTrue(chatModel.executorPromptText.contains("read_skill"));
assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill"));
assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool"));
assertFalse(chatModel.verifierPromptText.contains("read_skill"));
}
src/test/java/com/superbiz/agent/service/ToolInvocationRecorderTest.java
+7 -1
View File
@@ -113,6 +113,10 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));
assertTrue(saved.getRetrievalDetails().contains("\"rerank_trace\""));
}
@Test
@@ -176,7 +180,9 @@ class ToolInvocationRecorderTest {
assertEquals(1, record.evidenceBlockCount());
assertEquals(1, record.evidenceBlocks().size());
assertTrue(String.valueOf(record.evidenceBlocks().get(0).get("content_preview")).endsWith("..."));
assertEquals("UNFILTERED_VECTOR", record.retrievalLayer());
assertEquals("L1", record.retrievalLayer());
assertEquals(3, record.l1MatchCount());
assertTrue(record.retrievalTrace().containsKey("selected_attempt"));
assertTrue(record.contextPack().containsKey("included_sources"));
}
}
src/test/java/com/superbiz/agent/service/ToolTraceSummaryServiceTest.java
+108
View File
@@ -62,6 +62,7 @@ class ToolTraceSummaryServiceTest {
assertEquals("direct", logsSummary.get("evidence_level"));
assertEquals(2, logsSummary.get("invocation_count"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("payment timeout stack trace"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
@@ -72,4 +73,111 @@ class ToolTraceSummaryServiceTest {
assertEquals(1, metricsSummary.get("failed_invocation_count"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
}
@Test
void buildVerifierTraceSummaryPreservesConcreteFactsFromTruncatedLogAndMetricRows() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String logPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 22:15:45",
"level" : "ERROR",
"service" : "order-service",
"message" : "数据库连接池耗尽: Cannot acquire connection from pool, active: 50/50, waiting: 23, timeout: 30000ms"
} ]
}
""";
String metricPreview = """
{
"success" : true,
"alerts" : [ {
"alert_name" : "HighCPUUsage",
"service" : "payment-service",
"description" : "服务 payment-service 的 CPU 使用率持续超过 80%，当前值为 92%。"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-2")).thenReturn(List.of(
ToolInvocation.builder()
.id(10L)
.sessionId("session-2")
.toolName("query_logs")
.inputParams("{\"query\":\"pool\"}")
.outputPreview(logPreview)
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build(),
ToolInvocation.builder()
.id(11L)
.sessionId("session-2")
.toolName("query_metrics")
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
.outputPreview(metricPreview)
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
.isTruncated(true)
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-2", "连接池耗尽 HighCPUUsage");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("连接池耗尽"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("active: 50/50"));
assertEquals(List.of(10L), logsSummary.get("source_invocation_ids"));
Map<String, Object> metricsSummary = summaries.stream()
.filter(item -> "query_metrics".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("HighCPUUsage"));
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("payment-service"));
}
@Test
void buildVerifierTraceSummaryDoesNotTreatGenericMockLogsAsDirectEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
String genericLogPreview = """
{
"success" : true,
"logs" : [ {
"timestamp" : "2026-07-06 23:44:41",
"level" : "ERROR",
"service" : "generic-service",
"message" : "日志消息 #0, 查询条件: service:payment-service"
} ]
}
""";
when(repository.findBySessionIdOrderByIdAsc("session-3")).thenReturn(List.of(
ToolInvocation.builder()
.id(20L)
.sessionId("session-3")
.toolName("query_logs")
.inputParams("{\"query\":\"service:payment-service\"}")
.outputPreview(genericLogPreview)
.retrievalDetails("{\"retrieved_domains\":[\"system-metrics\"],\"evidence_status\":\"supported\"}")
.success(true)
.build()
));
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-3", "payment-service timeout");
Map<String, Object> logsSummary = summaries.stream()
.filter(item -> "query_logs".equals(item.get("tool_name")))
.findFirst()
.orElseThrow();
assertEquals(Boolean.FALSE, logsSummary.get("success"));
assertEquals("none", logsSummary.get("evidence_level"));
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("日志消息 #0"));
}
}