osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality

Browse Source 
Persist provider reasoning and assistant text separately on agent_reasoning_audit
(DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG
tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools,
and align MVP docs after live E2E verification.
This commit is contained in:
zhuyongxin
2026-07-28 19:43:13 +08:00
parent 2f40536248
commit 7ae9707a3b
116 changed files with 8364 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSinkTest.java
+93 -5
View File
@@ -10,17 +10,19 @@ import org.mockito.ArgumentCaptor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
class JpaToolInvocationAuditSinkTest {
@Test
void persistsOnlyBoundedStableMetadata() {
void persistsOnlyBoundedStableMetadataForGenericTools() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(
repository, new ObjectMapper(), traceRecorder);
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, null);
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-1", "query_logs",
@@ -36,14 +38,100 @@ class JpaToolInvocationAuditSinkTest {
assertEquals("{\"tool_call_id\":\"call-1\",\"request_bytes\":83}", saved.getInputParams());
assertEquals("status=ERROR,evidence_status=ERROR", saved.getOutputPreview());
assertEquals("TOOL_EXECUTION_ERROR", saved.getErrorMessage());
// generic tools must not put evidence_status into relevance_level column
assertNull(saved.getRelevanceLevel());
assertNull(saved.getStepId());
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
assertFalse(serialized.contains("query"));
assertFalse(serialized.contains("raw_response"));
assertTrue(saved.getRetrievalDetails().contains("evidence_status"));
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
verify(traceRecorder).record(traceCaptor.capture());
assertEquals(TraceEventType.TOOL_INVOCATION, traceCaptor.getValue().eventType());
assertEquals("call-1", traceCaptor.getValue().details().get("tool_call_id"));
assertFalse(traceCaptor.getValue().details().toString().contains("raw_response"));
}
@Test
void persistsStepIdAndSafeQueryPreviewInInputParams() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, null);
String longQuery = "q".repeat(200);
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-q", "lookup_knowledge",
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
12, 50, 80, null, null, 99L,
"{\"query\":\"" + longQuery + "\",\"password\":\"should-not-store\",\"limit\":5}"));
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals(99L, saved.getStepId());
assertTrue(saved.getInputParams().contains("\"step_id\":99"));
assertTrue(saved.getInputParams().contains("\"query\":\"" + "q".repeat(160) + "\""));
assertTrue(saved.getInputParams().contains("\"query_truncated\":true"));
assertTrue(saved.getInputParams().contains("\"query_chars\":200"));
assertTrue(saved.getInputParams().contains("\"limit\":5"));
assertFalse(saved.getInputParams().contains("should-not-store"));
assertFalse(saved.getInputParams().contains("password"));
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
verify(traceRecorder).record(traceCaptor.capture());
assertEquals(99L, traceCaptor.getValue().details().get("step_id"));
}
@Test
void enrichesLookupKnowledgeWithRagFieldsAndTrueRelevanceLevel() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
RagLookupAuditEnricher enricher = new RagLookupAuditEnricher(new ObjectMapper(), "hybrid");
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, enricher);
String lookup = """
{
"found": true,
"relevanceLevel": "REFERENCE",
"evidenceBlockCount": 1,
"evidenceBlocks": [
{"evidenceKey":"doc#chunk-0","source":"doc","retrievalLayer":"L1","content":"body"}
],
"retrievalTrace": {
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"queryHints": {"l0_match_count": 2, "domains": ["mysql"]},
"attempts": [{"name":"FILTERED_VECTOR","candidateCount":3,"usable":true,"topSimilarity":0.8}]
}
}
""";
String agent = """
{"evidence_status":"EVIDENCE_FOUND","returned_count":1,"truncated":false,"relevance_level":"REFERENCE"}
""";
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-rag", "lookup_knowledge",
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
42, 10, 100, lookup, agent, 7L,
"{\"query\":\"MySQL HikariCP pool exhausted\"}"));
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals(7L, saved.getStepId());
assertEquals("L1", saved.getRetrievalLayer());
assertEquals("REFERENCE", saved.getRelevanceLevel());
assertEquals(2, saved.getL0MatchCount());
assertEquals(1, saved.getL1MatchCount());
assertTrue(saved.getRetrievalDetails().contains("FILTERED_VECTOR"));
assertTrue(saved.getRetrievalDetails().contains("search_mode"));
assertTrue(saved.getRetrievalDetails().contains("EVIDENCE_FOUND"));
assertTrue(saved.getOutputPreview().contains("REFERENCE"));
assertTrue(saved.getInputParams().contains("MySQL HikariCP pool exhausted"));
assertTrue(saved.getInputParams().contains("\"step_id\":7"));
// must not store full excerpt dump as sole content; body may appear only if tiny — ensure attempt present
assertTrue(saved.getRetrievalDetails().contains("attempts"));
}
}