osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): harden verifier evidence references

Browse Source
This commit is contained in:
aruo
2026-07-08 16:12:56 +08:00
parent a08672b31e
commit 7b8c75e571
29 changed files with 3426 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+21 -6
View File
@@ -253,7 +253,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -300,7 +301,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -350,6 +352,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-gatekeeper-persist-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.executorOutput = """
@@ -366,7 +369,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -393,6 +397,7 @@ class ChatServiceSequentialAgentTest {
@SuppressWarnings("unchecked")
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
assertEquals("pass", gatekeeperResult.get("status"));
assertEquals("none", gatekeeperResult.get("severity"));
}
@Test
@@ -407,6 +412,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-claim-check-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
@@ -466,6 +472,7 @@ class ChatServiceSequentialAgentTest {
.id(101L)
.sessionId("sequential-gatekeeper-fail-session")
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
@@ -493,7 +500,8 @@ class ChatServiceSequentialAgentTest {
{
"source_type": "tool_trace",
"tool_name": "query_metrics",
"source_invocation_ids": [999],
"source_invocation_id": 999,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -647,6 +655,7 @@ class ChatServiceSequentialAgentTest {
when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
.id(101L)
.toolName("query_metrics")
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
.build()));
EvaluationService evaluationService = mock(EvaluationService.class);
@@ -694,7 +703,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
@@ -707,6 +717,10 @@ class ChatServiceSequentialAgentTest {
""";
}
private String evidenceRefs(String rawPath, String text) {
return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}";
}
private static final class ScriptedChatModel implements ChatModel {
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
private String promptText = "";
@@ -728,7 +742,8 @@ class ChatServiceSequentialAgentTest {
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"source_invocation_id": 101,
"raw_path": "$.alerts[0]",
"evidence_excerpt": "active=50 max=50"
}
]
src/test/java/com/superbiz/agent/service/ExecutorGatekeeperServiceTest.java
+135 -16
View File
@@ -18,14 +18,18 @@ class ExecutorGatekeeperServiceTest {
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics").build()
invocation(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%, duration=25m")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(101L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]",
"HighCPUUsage firing, service=payment-service, current=92%"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@@ -34,12 +38,13 @@ class ExecutorGatekeeperServiceTest {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of());
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics");
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("user_facing_answer", "旧版最终答案");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("schema.executor_v2"));
}
@@ -47,14 +52,16 @@ class ExecutorGatekeeperServiceTest {
void validateFailsForFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics").build()
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(999L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(999L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@@ -62,19 +69,137 @@ class ExecutorGatekeeperServiceTest {
void validateFailsForToolNameMismatch() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_logs").build()
invocation(101L, "query_logs", "$.logs[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1", validOutput(101L, "query_metrics"),
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@SuppressWarnings("unchecked")
private Map<String, Object> validOutput(Long invocationId, String toolName) {
@Test
void validateDowngradesMissingRawPathToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", null, "cpu=92");
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateRejectsUnknownRawPath() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[99]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateDowngradesOldInvocationWithoutEvidenceRefsToLowConfidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics")
.retrievalDetails("{\"evidence_status\":\"supported\"}")
.build()
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("low_confid", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.missing"));
}
@Test
void validateRejectsMismatchedExcerpt() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "HighCPUUsage firing service payment-service current 92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_metrics", "$.alerts[0]", "HikariCP active=50/50 waiting=32"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@Test
void validateFailsForRecommendedActionFabricatedInvocationId() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
output.put("recommended_actions", List.of(Map.of(
"action_text", "restart service",
"reason", "alert is firing",
"evidence_bindings", List.of(Map.of(
"source_type", "tool_trace",
"source_id", "trace-1",
"tool_name", "query_metrics",
"source_invocation_id", 999L,
"raw_path", "$.alerts[0]",
"evidence_excerpt", "cpu=92"
))
)));
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
private ToolInvocation invocation(Long id, String toolName, String rawPath, String text) {
return ToolInvocation.builder()
.id(id)
.sessionId("session-1")
.toolName(toolName)
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
+ "\",\"text\":\"" + text + "\"}]}")
.build();
}
private Map<String, Object> validOutput(Long invocationId, String toolName, String rawPath, String excerpt) {
Map<String, Object> binding = new java.util.LinkedHashMap<>();
binding.put("source_type", "tool_trace");
binding.put("source_id", "trace-1");
binding.put("tool_name", toolName);
binding.put("source_invocation_id", invocationId);
if (rawPath != null) {
binding.put("raw_path", rawPath);
}
binding.put("evidence_excerpt", excerpt);
return new java.util.LinkedHashMap<>(Map.of(
"answer_version", "executor_evidence_v2",
"claims", List.of(Map.of(
@@ -82,13 +207,7 @@ class ExecutorGatekeeperServiceTest {
"claim_type", "symptom",
"claim_text", "连接池 active 达到上限",
"support_level", "direct",
"evidence_bindings", List.of(Map.of(
"source_type", "tool_trace",
"source_id", "trace-1",
"tool_name", toolName,
"source_invocation_ids", List.of(invocationId),
"evidence_excerpt", "active=50 max=50"
))
"evidence_bindings", List.of(binding)
)),
"hypotheses", List.of(),
"recommended_actions", List.of(),
src/test/java/com/superbiz/agent/service/ToolInvocationRecorderTest.java
+71
View File
@@ -1,5 +1,6 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.dto.ContextPack;
@@ -23,6 +24,8 @@ import static org.mockito.Mockito.when;
class ToolInvocationRecorderTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void recordEvidenceToolPreservesNoEvidenceSemantics() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -56,6 +59,72 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
}
@Test
void recordEvidenceToolExtractsLogEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("log-ref-session");
try {
recorder.recordEvidenceTool(
"query_logs",
Map.of("query", "HikariCP order-service"),
"""
{"success":true,"logs":[{"timestamp":"2026-07-08 10:00:00","level":"ERROR","service":"order-service","message":"HikariPool-1 - Connection is not available, request timed out after 30000ms","metrics":{"waiting":"32"}}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"application-logs",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("log_topic", "application-logs")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.logs[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HikariPool-1"));
}
@Test
void recordEvidenceToolExtractsMetricEvidenceRefs() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("metric-ref-session");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"""
{"success":true,"alerts":[{"alert_name":"HighMemoryUsage","state":"firing","description":"服务 order-service 当前值为 91%","active_at":"2026-07-08T10:00:00Z","duration":"15m"}]}
""",
true,
System.currentTimeMillis() - 10,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.alerts[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HighMemoryUsage"));
}
@Test
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -113,6 +182,8 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
assertTrue(saved.getRetrievalDetails().contains("\"evidence_refs\""));
assertTrue(saved.getRetrievalDetails().contains("\"raw_path\":\"$.evidence_blocks[0]\""));
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));