osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): harden verifier evidence references

Browse Source
This commit is contained in:
aruo
2026-07-08 16:12:56 +08:00
parent a08672b31e
commit 7b8c75e571
29 changed files with 3426 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/agent/tool/QueryLogsToolsTest.java
+48
View File
@@ -0,0 +1,48 @@
package com.superbiz.agent.agent.tool;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ToolInvocationRecorder;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
class QueryLogsToolsTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void queryLogsReturnsHikariPositiveMockForOrderService() throws Exception {
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
ReflectionTestUtils.setField(tools, "mockEnabled", true);
String output = tools.queryLogs("ap-guangzhou", "application-logs",
"order-service HikariCP connection pool active=50/50 waiting", 10);
JsonNode root = objectMapper.readTree(output);
assertTrue(root.path("success").asBoolean());
assertEquals(2, root.path("logs").size());
assertEquals("order-service", root.path("logs").get(0).path("service").asText());
assertTrue(root.toString().contains("HikariPool-1"));
assertFalse(root.toString().contains("generic-service"));
}
@Test
void queryLogsReturnsEmptyNoHitForOtherServiceHikariQuery() throws Exception {
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
ReflectionTestUtils.setField(tools, "mockEnabled", true);
String output = tools.queryLogs("ap-guangzhou", "application-logs",
"inventory-service HikariCP connection pool active=50/50 waiting", 10);
JsonNode root = objectMapper.readTree(output);
assertFalse(root.path("success").asBoolean());
assertEquals(0, root.path("logs").size());
assertEquals(0, root.path("total").asInt());
assertFalse(root.toString().contains("generic-service"));
}
}
src/test/java/com/superbiz/agent/hook/VerifierInputHookTest.java
+97 -4
View File
@@ -95,7 +95,7 @@ class VerifierInputHookTest {
));
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("structured-v2-session")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("structured-v2-session").toolName("query_metrics").build()
invocation(101L, "structured-v2-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
@@ -115,7 +115,8 @@ class VerifierInputHookTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -140,16 +141,96 @@ class VerifierInputHookTest {
assertEquals("连接池 active 达到上限",
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
assertEquals("pass", payload.path("gatekeeper_result").path("status").asText());
assertEquals("none", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("pass", VerifierContextHolder.getGatekeeperResult().get("status"));
}
@Test
void beforeModelBackfillsOnlyUniqueInvocationIdAndDoesNotPassWithoutRawPath() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
Map.of(
"trace_ref", "metrics-1",
"tool_name", "query_metrics",
"source_invocation_ids", List.of(101L)
)
));
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("backfill-session")).thenReturn(List.of(
invocation(101L, "backfill-session", "query_metrics", "$.alerts[0]",
"CPU 使用率持续超过 80%，当前值为 92%")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "payment-service CPU 使用率超过 92%",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "prometheus-alert-HighCPUUsage",
"tool_name": "queryPrometheusAlerts",
"evidence_excerpt": "CPU 使用率持续超过 80%，当前值为 92%"
}
]
}
],
"hypotheses": [],
"recommended_actions": [
{
"action_text": "restart payment-service",
"reason": "cpu alert is firing",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "prometheus-alert-HighCPUUsage",
"tool_name": "queryPrometheusAlerts",
"evidence_excerpt": "CPU usage is 92%"
}
]
}
],
"missing_info": []
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "backfill-session").build()
);
JsonNode payload = readPayload(command);
JsonNode binding = payload.path("executor_structured_output")
.path("claims").get(0)
.path("evidence_bindings").get(0);
assertEquals("query_metrics", binding.path("tool_name").asText());
assertEquals(101L, binding.path("source_invocation_id").asLong());
JsonNode actionBinding = payload.path("executor_structured_output")
.path("recommended_actions").get(0)
.path("evidence_bindings").get(0);
assertEquals("query_metrics", actionBinding.path("tool_name").asText());
assertEquals(101L, actionBinding.path("source_invocation_id").asLong());
assertFalse(binding.has("raw_path"));
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
assertEquals("low_confid", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("evidence.invocation_auto_backfill",
payload.path("gatekeeper_result").path("warnings").get(0).path("rule").asText());
}
@Test
void beforeModelAddsFailingGatekeeperResultForFabricatedInvocationId() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
when(invocationRepository.findBySessionIdOrderByIdAsc("fabricated-invocation-session")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("fabricated-invocation-session").toolName("query_metrics").build()
invocation(101L, "fabricated-invocation-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
new ExecutorGatekeeperService(invocationRepository));
@@ -167,7 +248,8 @@ class VerifierInputHookTest {
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_ids": [999],
"source_invocation_id": 999,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -186,6 +268,7 @@ class VerifierInputHookTest {
JsonNode payload = readPayload(command);
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
assertEquals("reject", payload.path("gatekeeper_result").path("severity").asText());
assertEquals("evidence.invocation_ref",
payload.path("gatekeeper_result").path("failed_rules").get(0).asText());
}
@@ -284,4 +367,14 @@ class VerifierInputHookTest {
assertTrue(message instanceof UserMessage);
return objectMapper.readTree(((UserMessage) message).getText());
}
private ToolInvocation invocation(Long id, String sessionId, String toolName, String rawPath, String text) {
return ToolInvocation.builder()
.id(id)
.sessionId(sessionId)
.toolName(toolName)
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
+ "\",\"text\":\"" + text + "\"}]}")
.build();
}
}
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+21 -6
View File
@@ -253,7 +253,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -300,7 +301,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -350,6 +352,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-gatekeeper-persist-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.executorOutput = """
@@ -366,7 +369,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -393,6 +397,7 @@ class ChatServiceSequentialAgentTest {
@SuppressWarnings("unchecked")
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
assertEquals("pass", gatekeeperResult.get("status"));
assertEquals("none", gatekeeperResult.get("severity"));
}
@Test
@@ -407,6 +412,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-claim-check-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
@@ -466,6 +472,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-gatekeeper-fail-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
@@ -493,7 +500,8 @@ class ChatServiceSequentialAgentTest {
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_ids": [999],
"source_invocation_id": 999,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -647,6 +655,7 @@ class ChatServiceSequentialAgentTest {
when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
EvaluationService evaluationService = mock(EvaluationService.class);
@@ -694,7 +703,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -707,6 +717,10 @@ class ChatServiceSequentialAgentTest {
""";
}
private String evidenceRefs(String rawPath, String text) {
return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}";
}
private static final class ScriptedChatModel implements ChatModel {
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
private String promptText = "";
@@ -728,7 +742,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
src/test/java/com/superbiz/agent/service/ExecutorGatekeeperServiceTest.java
+135 -16
View File
@@ -18,14 +18,18 @@ class ExecutorGatekeeperServiceTest {
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics").build()
invocation(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%, duration=25m")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(101L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@@ -34,12 +38,13 @@ class ExecutorGatekeeperServiceTest {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of());
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics");
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("user_facing_answer", "旧版最终答案");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("schema.executor_v2"));
}
@@ -47,14 +52,16 @@ class ExecutorGatekeeperServiceTest {
void validateFailsForFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics").build()
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(999L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(999L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@@ -62,19 +69,137 @@ class ExecutorGatekeeperServiceTest {
void validateFailsForToolNameMismatch() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_logs").build()
invocation(101L, "query_logs", "$.logs[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(101L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@SuppressWarnings("unchecked")
private Map<String, Object> validOutput(Long invocationId, String toolName) {
@Test
void validateDowngradesMissingRawPathToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", null, "cpu=92");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateRejectsUnknownRawPath() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[99]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateDowngradesOldInvocationWithoutEvidenceRefsToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics")
.retrievalDetails("{\"evidence_status\":\"supported\"}")
.build()
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.missing"));
}
@Test
void validateRejectsMismatchedExcerpt() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "HighCPUUsage firing service payment-service current 92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "HikariCP active=50/50 waiting=32"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@Test
void validateFailsForRecommendedActionFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("recommended_actions", List.of(Map.of(
"action_text", "restart service",
"reason", "alert is firing",
"evidence_bindings", List.of(Map.of(
"source_type", "tool_trace",
"source_id", "trace-1",
"tool_name", "query_metrics",
"source_invocation_id", 999L,
"raw_path", "$.alerts[0]",
"evidence_excerpt", "cpu=92"
))
)));
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
private ToolInvocation invocation(Long id, String toolName, String rawPath, String text) {
return ToolInvocation.builder()
.id(id)
.sessionId("session-1")
.toolName(toolName)
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
+ "\",\"text\":\"" + text + "\"}]}")
.build();
}
private Map<String, Object> validOutput(Long invocationId, String toolName, String rawPath, String excerpt) {
Map<String, Object> binding = new java.util.LinkedHashMap<>();
binding.put("source_type", "tool_trace");
binding.put("source_id", "trace-1");
binding.put("tool_name", toolName);
binding.put("source_invocation_id", invocationId);
if (rawPath != null) {
binding.put("raw_path", rawPath);
}
binding.put("evidence_excerpt", excerpt);
return new java.util.LinkedHashMap<>(Map.of(
"answer_version", "executor_evidence_v2",
"claims", List.of(Map.of(
@@ -82,13 +207,7 @@ class ExecutorGatekeeperServiceTest {
"claim_type", "symptom",
"claim_text", "连接池 active 达到上限",
"support_level", "direct",
"evidence_bindings", List.of(Map.of(
"source_type", "tool_trace",
"source_id", "trace-1",
"tool_name", toolName,
"source_invocation_ids", List.of(invocationId),
"evidence_excerpt", "active=50 max=50"
))
"evidence_bindings", List.of(binding)
)),
"hypotheses", List.of(),
"recommended_actions", List.of(),
src/test/java/com/superbiz/agent/service/ToolInvocationRecorderTest.java
+71
View File
@@ -1,5 +1,6 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.ContextPack;
@@ -23,6 +24,8 @@ import static org.mockito.Mockito.when;
class ToolInvocationRecorderTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void recordEvidenceToolPreservesNoEvidenceSemantics() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -56,6 +59,72 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
}
@Test
void recordEvidenceToolExtractsLogEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("log-ref-session");
try {
recorder.recordEvidenceTool(
"query_logs",
Map.of("query", "HikariCP order-service"),
"""
{"success":true,"logs":[{"timestamp":"2026-07-08 10:00:00","level":"ERROR","service":"order-service","message":"HikariPool-1 - Connection is not available, request timed out after 30000ms","metrics":{"waiting":"32"}}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"application-logs",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("log_topic", "application-logs")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.logs[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HikariPool-1"));
}
@Test
void recordEvidenceToolExtractsMetricEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("metric-ref-session");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"""
{"success":true,"alerts":[{"alert_name":"HighMemoryUsage","state":"firing","description":"服务 order-service 当前值为 91%","active_at":"2026-07-08T10:00:00Z","duration":"15m"}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.alerts[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HighMemoryUsage"));
}
@Test
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -113,6 +182,8 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_refs\""));
assertTrue(saved.getRetrievalDetails().contains("\"raw_path\":\"$.evidence_blocks[0]\""));
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));