osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): support no-evidence references

Browse Source
This commit is contained in:
aruo
2026-07-08 23:30:00 +08:00
parent 7b8c75e571
commit 9a84b3de34
9 changed files with 1040 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/service/ExecutorGatekeeperServiceTest.java
+117 -2
View File
@@ -33,6 +33,111 @@ class ExecutorGatekeeperServiceTest {
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@Test
void validatePassesForNoEvidenceReference() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@Test
void validateBackfillsNoEvidenceInvocationByRawPathWhenToolHasMultipleCalls() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志"),
invocation(102L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(null, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
assertEquals("evidence.invocation_auto_backfill_by_raw_path",
((Map<?, ?>) ((List<?>) result.get("warnings")).get(0)).get("rule"));
}
@Test
void validateBackfillsNoEvidenceInvocationByExcerptWhenRawPathIsRepeated() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=service:inventory-service AND HikariCP; total=0; message=未找到匹配的日志"),
invocation(102L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(null, "query_logs", "$.no_evidence",
"query_logs returned no evidence; query=service:inventory-service AND HikariCP; total=0; evidence_status=no_evidence",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertEquals(101L,
((Map<?, ?>) ((List<?>) result.get("checked_bindings")).get(0)).get("source_invocation_id"));
}
@Test
void validateRejectsNoEvidenceReferenceWhenExcerptClaimsPositiveEvidence() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.no_evidence",
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.no_evidence",
"HikariCP active=50/50 waiting=32"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@Test
void validateRejectsPositiveBindingOnNegativeObservation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
invocation(101L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32")
));
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.validate("session-1",
validOutput(101L, "query_logs", "$.logs[0]",
"order-service HikariCP active=50/50 waiting=32",
"negative_observation"),
Map.of("status", "valid"));
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
}
@Test
void validateFailsWhenRemovedFieldsArePresent() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -191,11 +296,21 @@ class ExecutorGatekeeperServiceTest {
}
private Map<String, Object> validOutput(Long invocationId, String toolName, String rawPath, String excerpt) {
return validOutput(invocationId, toolName, rawPath, excerpt, "symptom");
}
private Map<String, Object> validOutput(Long invocationId,
String toolName,
String rawPath,
String excerpt,
String claimType) {
Map<String, Object> binding = new java.util.LinkedHashMap<>();
binding.put("source_type", "tool_trace");
binding.put("source_id", "trace-1");
binding.put("tool_name", toolName);
binding.put("source_invocation_id", invocationId);
if (invocationId != null) {
binding.put("source_invocation_id", invocationId);
}
if (rawPath != null) {
binding.put("raw_path", rawPath);
}
@@ -204,7 +319,7 @@ class ExecutorGatekeeperServiceTest {
"answer_version", "executor_evidence_v2",
"claims", List.of(Map.of(
"claim_id", "claim-1",
"claim_type", "symptom",
"claim_type", claimType,
"claim_text", "连接池 active 达到上限",
"support_level", "direct",
"evidence_bindings", List.of(binding)
src/test/java/com/superbiz/agent/service/ToolInvocationRecorderTest.java
+77 -3
View File
@@ -27,7 +27,7 @@ class ToolInvocationRecorderTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void recordEvidenceToolPreservesNoEvidenceSemantics() {
void recordEvidenceToolPreservesNoEvidenceSemantics() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
@@ -36,8 +36,8 @@ class ToolInvocationRecorderTest {
try {
recorder.recordEvidenceTool(
"query_logs",
Map.of("query", "timeout"),
"{\"success\":false,\"message\":\"未找到匹配的日志\"}",
Map.of("query", "inventory-service HikariCP", "log_topic", "application-logs"),
"{\"success\":false,\"query\":\"inventory-service HikariCP\",\"log_topic\":\"application-logs\",\"logs\":[],\"total\":0,\"message\":\"未找到匹配的日志\"}",
true,
System.currentTimeMillis() - 10,
null,
@@ -57,6 +57,14 @@ class ToolInvocationRecorderTest {
assertEquals(Boolean.TRUE, saved.getSuccess());
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"no_evidence\""));
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
JsonNode details = objectMapper.readTree(saved.getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
String text = details.path("evidence_refs").get(0).path("text").asText();
assertTrue(text.contains("query_logs returned no evidence"));
assertTrue(text.contains("inventory-service HikariCP"));
assertTrue(text.contains("total=0"));
assertTrue(text.contains("evidence_status=no_evidence"));
}
@Test
@@ -125,6 +133,40 @@ class ToolInvocationRecorderTest {
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HighMemoryUsage"));
}
@Test
void recordEvidenceToolExtractsMetricNoEvidenceRef() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("metric-no-evidence-session");
try {
recorder.recordEvidenceTool(
"query_metrics",
Map.of("query", "active_prometheus_alerts"),
"""
{"success":true,"alerts":[],"message":"成功检索到 0 个活动告警"}
""",
true,
System.currentTimeMillis() - 10,
null,
"prometheus_alerts",
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
Map.of("metric_family", "prometheus_alerts")
);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("query_metrics returned no evidence"));
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("prometheus_alerts"));
}
@Test
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -190,6 +232,38 @@ class ToolInvocationRecorderTest {
assertTrue(saved.getRetrievalDetails().contains("\"rerank_trace\""));
}
@Test
void recordLookupKnowledgeAddsNoEvidenceRefWhenNoBlocks() throws Exception {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
SessionContextHolder.setSessionId("lookup-no-evidence-session");
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
.query("inventory-service HikariCP")
.outputPreview("")
.outputLength(0)
.durationMs(12)
.success(true)
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE)
.evidenceBlocks(List.of())
.build();
try {
recorder.recordLookupKnowledge(record);
} finally {
SessionContextHolder.clear();
}
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("lookup_knowledge returned no evidence"));
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("inventory-service HikariCP"));
}
@Test
void lookupKnowledgeRecordFromSummarizesEvidenceBlocks() {
EvidenceBlock block = EvidenceBlock.builder()