feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality

Persist provider reasoning and assistant text separately on agent_reasoning_audit
(DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG
tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools,
and align MVP docs after live E2E verification.
This commit is contained in:
zhuyongxin
2026-07-28 19:43:13 +08:00
parent 2f40536248
commit 7ae9707a3b
116 changed files with 8364 additions and 1141 deletions
@@ -91,6 +91,8 @@ class HarnessChatConfigurationTest {
.withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class))
.withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop)
.withBean(DiagnosisTraceRecorder.class, DiagnosisTraceRecorder::noop)
.withBean(com.superbiz.agent.harness.audit.AgentStepAuditTracker.class,
com.superbiz.agent.harness.audit.AgentStepAuditTracker::new)
.withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class))
.withBean(AgentReasoningAuditRepository.class,
() -> mock(AgentReasoningAuditRepository.class))
@@ -10,18 +10,29 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Locale;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Generates RAG retrieval fixtures from the real LookupKnowledgeTool bean.
* Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
*
* <p>This class is disabled by default because it writes repository files and
* depends on the configured runtime retrieval stack.</p>
* <p>Disabled by default: writes repository files and needs the live retrieval stack
* (embedding + Milvus hybrid collection + optional MySQL/L0).</p>
*
* <p>System properties (via Maven {@code -D}):</p>
* <ul>
* <li>{@code rag.snapshot.enabled=true} — required to run</li>
* <li>{@code retrieval.search.mode=hybrid|dense} — default hybrid</li>
* <li>{@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})</li>
* <li>{@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}</li>
* </ul>
*/
@SpringBootTest(
classes = Main.class,
@@ -40,11 +51,31 @@ class RagLookupSnapshotGeneratorTest {
@Autowired
private ObjectMapper objectMapper;
/**
* Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
*/
@DynamicPropertySource
static void retrievalProperties(DynamicPropertyRegistry registry) {
String mode = System.getProperty("retrieval.search.mode", "hybrid");
if (mode == null || mode.isBlank()) {
mode = "hybrid";
}
String normalized = mode.trim().toLowerCase(Locale.ROOT);
registry.add("retrieval.search.mode", () -> normalized);
String kbScope = System.getProperty("retrieval.kb-scope", "");
if (kbScope != null && !kbScope.isBlank()) {
registry.add("retrieval.kb-scope", kbScope::trim);
}
}
@Test
void generateLookupResultFixtures() throws Exception {
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
JsonNode root = objectMapper.readTree(casesPath.toFile());
JsonNode cases = root.path("cases");
@@ -61,6 +92,10 @@ class RagLookupSnapshotGeneratorTest {
fixture.put("caseId", caseId);
fixture.put("query", query);
fixture.put("retrievedAt", retrievedAt);
fixture.put("searchMode", searchMode);
if (kbScope != null) {
fixture.put("kbScope", kbScope);
}
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
Path output = fixturesDir.resolve(caseId + ".json");
@@ -68,6 +103,20 @@ class RagLookupSnapshotGeneratorTest {
}
}
private static String normalizeMode(String mode) {
if (mode == null || mode.isBlank()) {
return "hybrid";
}
return mode.trim().toLowerCase(Locale.ROOT);
}
private static String blankToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
private String requiredText(JsonNode node, String fieldName) {
JsonNode value = node.get(fieldName);
if (value == null || value.asText().isBlank()) {
@@ -0,0 +1,44 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import org.junit.jupiter.api.Test;
import org.springframework.ai.tool.ToolCallback;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class HarnessEvidenceToolsTest {
@Test
void omitsQueryMysqlWhenInvokerIsNull() {
EvidenceToolInvoker unused = (context, id, args) ->
ToolBoundaryResult.ready(id, "{}", EvidenceStatus.NO_EVIDENCE);
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, null);
assertTrue(tools.supports(AgentToolContracts.LOOKUP_KNOWLEDGE));
assertTrue(tools.supports(AgentToolContracts.QUERY_LOGS));
assertFalse(tools.supports(AgentToolContracts.QUERY_MYSQL));
List<String> names = tools.callbacks().stream().map(ToolCallback::getToolDefinition)
.map(def -> def.name()).toList();
assertEquals(2, names.size());
assertTrue(names.contains(AgentToolContracts.LOOKUP_KNOWLEDGE));
assertTrue(names.contains(AgentToolContracts.QUERY_LOGS));
assertFalse(names.contains(AgentToolContracts.QUERY_MYSQL));
}
@Test
void registersQueryMysqlWhenInvokerPresent() {
EvidenceToolInvoker unused = (context, id, args) ->
ToolBoundaryResult.ready(id, "{}", EvidenceStatus.NO_EVIDENCE);
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, unused);
assertTrue(tools.supports(AgentToolContracts.QUERY_MYSQL));
assertEquals(3, tools.callbacks().size());
}
}
@@ -33,6 +33,7 @@ import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -106,7 +107,7 @@ class PublishedResultPersistenceTest {
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
"{\"fallback\":{\"type\":\"SEMANTIC_UNAVAILABLE\",\"message\":\"语义校验不可用\",\"conclusion\":null}}",
new PublishedResult("q", "c", "scope", List.of(),
List.of(new SourceDocument("doc", "title"))), 12);
@@ -115,6 +116,32 @@ class PublishedResultPersistenceTest {
assertNull(entity.getPublishedResult());
assertEquals(12, entity.getTotalDurationMs());
assertEquals(2, entity.getStepCount());
assertNotNull(entity.getConclusion());
assertTrue(entity.getConclusion().contains("SEMANTIC_UNAVAILABLE"));
assertTrue(entity.getConclusion().contains("语义校验不可用"));
}
@Test
void successCompletionExtractsConclusionBesideQuery() {
DiagnosisRunRepository runs = mock(DiagnosisRunRepository.class);
ChatSessionRepository sessions = mock(ChatSessionRepository.class);
DiagnosisRun entity = DiagnosisRun.builder()
.runId("run-ok").sessionId("session-1").query("pool?").status("RUNNING").build();
when(runs.findByRunId("run-ok")).thenReturn(Optional.of(entity));
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
when(sessions.findBySessionId("session-1")).thenReturn(Optional.empty());
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
RunContext context = core().startRun("session-1", "run-ok");
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.SUCCESS,
"{\"report\":{\"conclusion\":{\"text\":\"Pool exhausted per runbook.\"}}}",
new PublishedResult("pool?", "Pool exhausted per runbook.", "scope", List.of(),
List.of(new SourceDocument("doc", "title"))), 20);
assertEquals("Pool exhausted per runbook.", entity.getConclusion());
assertEquals("pool?", entity.getQuery());
assertNotNull(entity.getAnswer());
assertNotNull(entity.getPublishedResult());
}
@Test
@@ -2,23 +2,24 @@ package com.superbiz.agent.harness.audit;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.domain.entity.AgentReasoningAudit;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.agent.DiagnosisAgentFactory;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -30,7 +31,7 @@ import static org.mockito.Mockito.when;
class HarnessAgentAuditHookTest {
@Test
void persistsMetadataAndReasoningInSeparateAudit() {
void persistsBothProviderReasoningAndAssistantText() {
AgentStepRepository repository = mock(AgentStepRepository.class);
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
AgentStep persisted = AgentStep.builder().id(7L).build();
@@ -47,7 +48,7 @@ class HarnessAgentAuditHookTest {
hook.beforeModel(List.of(new UserMessage("secret-query")), config);
AssistantMessage response = AssistantMessage.builder()
.content("secret-model-output")
.content("public assistant conclusion text")
.properties(java.util.Map.of(
"reasoning_content", "inspect bounded evidence before selecting query_logs"))
.toolCalls(List.of(new AssistantMessage.ToolCall(
@@ -61,13 +62,16 @@ class HarnessAgentAuditHookTest {
AgentStep completed = captor.getAllValues().get(1);
assertEquals("session-audit", started.getSessionId());
assertEquals("run-audit", started.getRunId());
// model_input stays summary-only (no raw user text)
assertFalse(started.getModelInput().contains("secret-query"));
assertFalse(completed.getModelOutput().contains("secret-model-output"));
// model_output stays summary-only
assertFalse(completed.getModelOutput().contains("public assistant conclusion text"));
assertFalse(completed.getModelOutput().contains("secret-argument"));
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"],"
+ "\"reasoning_available\":true,\"reasoning_bytes\":52}",
completed.getModelOutput());
assertNull(completed.getThought());
assertTrue(completed.getModelOutput().contains("\"reasoning_available\":true"));
assertTrue(completed.getModelOutput().contains("PROVIDER_REASONING+ASSISTANT_TEXT"));
// thought keeps provider reasoning preferentially
assertEquals("inspect bounded evidence before selecting query_logs", completed.getThought());
ArgumentCaptor<AgentReasoningAudit> reasoningCaptor =
ArgumentCaptor.forClass(AgentReasoningAudit.class);
verify(reasoningRepository).save(reasoningCaptor.capture());
@@ -78,16 +82,88 @@ class HarnessAgentAuditHookTest {
assertTrue(reasoning.getReasoningAvailable());
assertEquals("inspect bounded evidence before selecting query_logs",
reasoning.getReasoningContent());
assertTrue(reasoning.getAssistantText().contains("public assistant conclusion text"));
assertTrue(reasoning.getAssistantText().contains("tool_calls:"));
assertTrue(reasoning.getAssistantText().contains("query_logs"));
assertEquals(HarnessAgentAuditHook.SOURCE_BOTH, reasoning.getContentSource());
assertTrue(reasoning.getContentBytes() > 0);
assertNotNull(trace.get());
assertEquals(TraceEventType.AGENT_MODEL_STEP, trace.get().eventType());
String traceText = trace.get().details().toString();
assertFalse(traceText.contains("secret-query"));
assertFalse(traceText.contains("secret-model-output"));
assertFalse(traceText.contains("secret-argument"));
assertFalse(traceText.contains("public assistant conclusion text"));
assertFalse(traceText.contains("inspect bounded evidence"));
assertTrue(traceText.contains("reasoning_available=true"));
}
@Test
void extractsReasoningFromDeepSeekAssistantMessageField() {
AgentStepRepository repository = mock(AgentStepRepository.class);
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
AgentStep persisted = AgentStep.builder().id(9L).build();
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
when(repository.findById(9L)).thenReturn(Optional.of(persisted));
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
DiagnosisTraceRecorder.noop(), reasoningRepository);
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", "session-ds")
.addMetadata("runId", "run-ds")
.build();
// Production path: Spring AI DeepSeekChatModel returns DeepSeekAssistantMessage
// with reasoning on the dedicated field, NOT metadata.
DeepSeekAssistantMessage response = new DeepSeekAssistantMessage.Builder()
.content("final answer body")
.reasoningContent("step1: inspect evidence\nstep2: call lookup_knowledge")
.properties(Map.of())
.build();
hook.beforeModel(List.of(new UserMessage("q")), config);
hook.afterModel(List.of(response), config);
ArgumentCaptor<AgentReasoningAudit> captor = ArgumentCaptor.forClass(AgentReasoningAudit.class);
verify(reasoningRepository).save(captor.capture());
AgentReasoningAudit row = captor.getValue();
assertTrue(row.getReasoningAvailable());
assertEquals("step1: inspect evidence\nstep2: call lookup_knowledge",
row.getReasoningContent());
assertEquals("final answer body", row.getAssistantText());
assertEquals(HarnessAgentAuditHook.SOURCE_BOTH, row.getContentSource());
// metadata-only path must not be required
assertTrue(HarnessAgentAuditHook.providerReasoning(response).contains("inspect evidence"));
}
@Test
void withoutProviderReasoningStillStoresAssistantText() {
AgentStepRepository repository = mock(AgentStepRepository.class);
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
AgentStep persisted = AgentStep.builder().id(8L).build();
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
when(repository.findById(8L)).thenReturn(Optional.of(persisted));
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
DiagnosisTraceRecorder.noop(), reasoningRepository);
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", "session-2")
.addMetadata("runId", "run-2")
.build();
hook.beforeModel(List.of(new UserMessage("q")), config);
hook.afterModel(List.of(AssistantMessage.builder()
.content("only assistant body")
.build()), config);
ArgumentCaptor<AgentReasoningAudit> captor = ArgumentCaptor.forClass(AgentReasoningAudit.class);
verify(reasoningRepository).save(captor.capture());
AgentReasoningAudit row = captor.getValue();
assertFalse(row.getReasoningAvailable());
assertNullSafe(row.getReasoningContent());
assertEquals("only assistant body", row.getAssistantText());
assertEquals(HarnessAgentAuditHook.SOURCE_ASSISTANT, row.getContentSource());
}
@Test
void missingIdentitySkipsPersistence() {
AgentStepRepository repository = mock(AgentStepRepository.class);
@@ -98,4 +174,27 @@ class HarnessAgentAuditHookTest {
verify(repository, never()).save(any());
}
@Test
void bindsStepIdToTrackerForToolAuditLinkage() {
AgentStepRepository repository = mock(AgentStepRepository.class);
AgentStep persisted = AgentStep.builder().id(42L).build();
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
AgentStepAuditTracker tracker = new AgentStepAuditTracker();
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
DiagnosisTraceRecorder.noop(), null, tracker);
RunnableConfig config = RunnableConfig.builder()
.addMetadata("sessionId", "session-bind")
.addMetadata("runId", "run-bind")
.build();
hook.beforeModel(List.of(new UserMessage("q")), config);
assertEquals(42L, tracker.currentStepId("run-bind"));
}
private static void assertNullSafe(String value) {
assertTrue(value == null || value.isBlank());
}
}
@@ -10,17 +10,19 @@ import org.mockito.ArgumentCaptor;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
class JpaToolInvocationAuditSinkTest {
@Test
void persistsOnlyBoundedStableMetadata() {
void persistsOnlyBoundedStableMetadataForGenericTools() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(
repository, new ObjectMapper(), traceRecorder);
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, null);
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-1", "query_logs",
@@ -36,14 +38,100 @@ class JpaToolInvocationAuditSinkTest {
assertEquals("{\"tool_call_id\":\"call-1\",\"request_bytes\":83}", saved.getInputParams());
assertEquals("status=ERROR,evidence_status=ERROR", saved.getOutputPreview());
assertEquals("TOOL_EXECUTION_ERROR", saved.getErrorMessage());
// generic tools must not put evidence_status into relevance_level column
assertNull(saved.getRelevanceLevel());
assertNull(saved.getStepId());
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
assertFalse(serialized.contains("query"));
assertFalse(serialized.contains("raw_response"));
assertTrue(saved.getRetrievalDetails().contains("evidence_status"));
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
verify(traceRecorder).record(traceCaptor.capture());
assertEquals(TraceEventType.TOOL_INVOCATION, traceCaptor.getValue().eventType());
assertEquals("call-1", traceCaptor.getValue().details().get("tool_call_id"));
assertFalse(traceCaptor.getValue().details().toString().contains("raw_response"));
}
@Test
void persistsStepIdAndSafeQueryPreviewInInputParams() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, null);
String longQuery = "q".repeat(200);
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-q", "lookup_knowledge",
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
12, 50, 80, null, null, 99L,
"{\"query\":\"" + longQuery + "\",\"password\":\"should-not-store\",\"limit\":5}"));
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals(99L, saved.getStepId());
assertTrue(saved.getInputParams().contains("\"step_id\":99"));
assertTrue(saved.getInputParams().contains("\"query\":\"" + "q".repeat(160) + "\""));
assertTrue(saved.getInputParams().contains("\"query_truncated\":true"));
assertTrue(saved.getInputParams().contains("\"query_chars\":200"));
assertTrue(saved.getInputParams().contains("\"limit\":5"));
assertFalse(saved.getInputParams().contains("should-not-store"));
assertFalse(saved.getInputParams().contains("password"));
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
verify(traceRecorder).record(traceCaptor.capture());
assertEquals(99L, traceCaptor.getValue().details().get("step_id"));
}
@Test
void enrichesLookupKnowledgeWithRagFieldsAndTrueRelevanceLevel() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
RagLookupAuditEnricher enricher = new RagLookupAuditEnricher(new ObjectMapper(), "hybrid");
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
repository, new ObjectMapper(), traceRecorder, enricher);
String lookup = """
{
"found": true,
"relevanceLevel": "REFERENCE",
"evidenceBlockCount": 1,
"evidenceBlocks": [
{"evidenceKey":"doc#chunk-0","source":"doc","retrievalLayer":"L1","content":"body"}
],
"retrievalTrace": {
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"queryHints": {"l0_match_count": 2, "domains": ["mysql"]},
"attempts": [{"name":"FILTERED_VECTOR","candidateCount":3,"usable":true,"topSimilarity":0.8}]
}
}
""";
String agent = """
{"evidence_status":"EVIDENCE_FOUND","returned_count":1,"truncated":false,"relevance_level":"REFERENCE"}
""";
sink.record(new ToolInvocationAuditEvent(
"session-1", "run-1", "call-rag", "lookup_knowledge",
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
42, 10, 100, lookup, agent, 7L,
"{\"query\":\"MySQL HikariCP pool exhausted\"}"));
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
verify(repository).save(captor.capture());
ToolInvocation saved = captor.getValue();
assertEquals(7L, saved.getStepId());
assertEquals("L1", saved.getRetrievalLayer());
assertEquals("REFERENCE", saved.getRelevanceLevel());
assertEquals(2, saved.getL0MatchCount());
assertEquals(1, saved.getL1MatchCount());
assertTrue(saved.getRetrievalDetails().contains("FILTERED_VECTOR"));
assertTrue(saved.getRetrievalDetails().contains("search_mode"));
assertTrue(saved.getRetrievalDetails().contains("EVIDENCE_FOUND"));
assertTrue(saved.getOutputPreview().contains("REFERENCE"));
assertTrue(saved.getInputParams().contains("MySQL HikariCP pool exhausted"));
assertTrue(saved.getInputParams().contains("\"step_id\":7"));
// must not store full excerpt dump as sole content; body may appear only if tiny — ensure attempt present
assertTrue(saved.getRetrievalDetails().contains("attempts"));
}
}
@@ -0,0 +1,90 @@
package com.superbiz.agent.harness.audit;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class RagLookupAuditEnricherTest {
private final RagLookupAuditEnricher enricher =
new RagLookupAuditEnricher(new ObjectMapper(), "hybrid");
@Test
void extractsBoundedRagFieldsWithoutFullQueryOrExcerpt() {
String lookup = """
{
"found": true,
"relevanceLevel": "PRECISE",
"evidenceCandidateCount": 8,
"evidenceBlockCount": 2,
"evidenceBlocks": [
{
"docId": "mysql-pool",
"evidenceKey": "mysql-pool#chunk-0",
"source": "mysql-pool",
"retrievalLayer": "L1",
"content": "HikariCP details should not be required in audit details dump"
}
],
"retrievalTrace": {
"selectedAttempt": "UNFILTERED_VECTOR_RETRY",
"fallbackReason": "filtered_vector_low_quality",
"categoryFilter": "overfilter-decoy",
"evidenceStatus": "supported",
"originalQuery": "secret user query should not be stored",
"queryHints": {
"l0_match_count": 1,
"domains": ["rag"],
"matched_keywords": ["pool"]
},
"attempts": [
{
"name": "FILTERED_VECTOR",
"categoryFilter": "overfilter-decoy",
"candidateCount": 2,
"usable": false,
"topSimilarity": 0.3,
"durationMs": 12
},
{
"name": "UNFILTERED_VECTOR_RETRY",
"candidateCount": 5,
"usable": true,
"topSimilarity": 0.9,
"durationMs": 20
}
]
}
}
""";
String agent = """
{"evidence_status":"EVIDENCE_FOUND","returned_count":2,"truncated":true,"relevance_level":"PRECISE"}
""";
RagLookupAuditEnricher.Enrichment e = enricher.enrich(lookup, agent);
assertEquals("L1", e.retrievalLayer());
assertEquals(1, e.l0MatchCount());
assertEquals(1, e.l1MatchCount()); // one evidence block in fixture
assertEquals(8, e.retrievalDetails().get("evidence_candidate_count"));
assertEquals("PRECISE", e.relevanceLevel());
assertTrue(e.truncated());
assertEquals("UNFILTERED_VECTOR_RETRY", e.retrievalDetails().get("selected_attempt"));
assertEquals("filtered_vector_low_quality", e.retrievalDetails().get("fallback_reason"));
assertEquals("hybrid", e.retrievalDetails().get("search_mode"));
assertEquals(List.of("mysql-pool#chunk-0"), e.retrievalDetails().get("evidence_keys"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> attempts = (List<Map<String, Object>>) e.retrievalDetails().get("attempts");
assertEquals(2, attempts.size());
String blob = e.retrievalDetails().toString() + e.outputPreview();
assertFalse(blob.contains("secret user query"));
assertFalse(blob.contains("HikariCP details should not"));
}
}
@@ -0,0 +1,43 @@
package com.superbiz.agent.harness.audit;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class RunConclusionExtractorTest {
private final ObjectMapper mapper = new ObjectMapper();
@Test
void extractsReportConclusion() {
String json = """
{"report":{"conclusion":{"text":"Pool is exhausted.","based_on_analysis_ids":["a1"]}}}
""";
assertEquals("Pool is exhausted.", RunConclusionExtractor.extract(mapper, json));
}
@Test
void extractsFallbackMessageWithType() {
String json = """
{"fallback":{"type":"INSUFFICIENT_EVIDENCE","message":"证据不足","conclusion":null}}
""";
String out = RunConclusionExtractor.extract(mapper, json);
assertTrue(out.contains("INSUFFICIENT_EVIDENCE"));
assertTrue(out.contains("证据不足"));
}
@Test
void extractsBareFallbackTypeOnly() {
assertEquals("SEMANTIC_UNAVAILABLE",
RunConclusionExtractor.extract(mapper, "{\"type\":\"SEMANTIC_UNAVAILABLE\"}"));
}
@Test
void nullOrBlankReturnsNull() {
assertNull(RunConclusionExtractor.extract(mapper, null));
assertNull(RunConclusionExtractor.extract(mapper, " "));
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.tool.boundary;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.AgentStepAuditTracker;
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.InvocationStatus;
@@ -179,6 +180,8 @@ class ToolBoundaryTest {
assertEquals("session-audit", audit.get().sessionId());
assertEquals("run-audit", audit.get().runId());
assertEquals("call-audit", audit.get().toolCallId());
assertEquals("{\"query\":\"timeout\"}", audit.get().requestJson());
assertNull(audit.get().stepId());
FakeStore failingAuditStore = new FakeStore(
new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
@@ -195,6 +198,28 @@ class ToolBoundaryTest {
assertEquals(EvidenceStatus.NO_EVIDENCE, unchanged.evidenceStatus());
}
@Test
void auditEventCarriesBoundStepIdFromTracker() {
MutableClock clock = new MutableClock(Instant.parse("2026-07-21T10:00:00Z"));
FakeStore store = new FakeStore(new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
AgentStepAuditTracker tracker = new AgentStepAuditTracker();
tracker.bind("run-step", 55L);
AtomicReference<ToolInvocationAuditEvent> audit = new AtomicReference<>();
ToolBoundary boundary = new ToolBoundary(core,
new ToolCallKeyFactory("superbiz:harness:tool-call"),
store, objectMapper, clock, audit::set, tracker);
RunContext context = core.startRun("session-step", "run-step");
boundary.execute(
context, request("run-step", "call-step", true, true),
ignored -> "raw",
ignored -> new ProjectedToolResult("agent", EvidenceStatus.EVIDENCE_FOUND));
assertEquals(55L, audit.get().stepId());
assertEquals("{\"query\":\"timeout\"}", audit.get().requestJson());
}
private ToolBoundary boundary(MutableClock clock, FakeStore store) {
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
return new ToolBoundary(core, new ToolCallKeyFactory("superbiz:harness:tool-call"),
@@ -3,6 +3,7 @@ package com.superbiz.agent.service;
import com.superbiz.agent.dto.EvidencePostprocessResult;
import com.superbiz.agent.dto.KnowledgeQuery;
import com.superbiz.agent.dto.RetrievedEvidenceCandidate;
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
@@ -22,26 +23,78 @@ class KnowledgeEvidencePostProcessorTest {
processor = new KnowledgeEvidencePostProcessor();
ReflectionTestUtils.setField(processor, "maxChunksPerDocument", 2);
ReflectionTestUtils.setField(processor, "returnN", 5);
ReflectionTestUtils.setField(processor, "maxL2Distance", 2.0);
ReflectionTestUtils.setField(processor, "highlyRelevantThreshold", 0.75);
ReflectionTestUtils.setField(processor, "referenceThreshold", 0.5);
}
@Test
void keepsDistinctChunksFromSameDocument() {
EvidencePostprocessResult result = processor.process(query(), List.of(
candidate("shared", 0, "shared#chunk-0", "c0", 0.2),
candidate("shared", 1, "shared#chunk-1", "c1", 0.3)
candidate("shared", 0, "shared#chunk-0", "c0", 0.2, 1),
candidate("shared", 1, "shared#chunk-1", "c1", 0.3, 2)
));
assertEquals(2, result.getEvidenceBlockCount());
assertEquals("shared#chunk-0", result.getEvidenceBlocks().get(0).getEvidenceKey());
assertEquals("shared#chunk-1", result.getEvidenceBlocks().get(1).getEvidenceKey());
assertEquals("c0", result.getEvidenceBlocks().get(0).getContent());
assertEquals("c1", result.getEvidenceBlocks().get(1).getContent());
}
@Test
void preservesRetrievalRankOverKeywordOverlap() {
KnowledgeQuery q = KnowledgeQuery.builder()
.originalQuery("payment")
.rewrittenQuery("payment")
.domainHints(List.of("payment"))
.matchedKeywords(List.of("ERR_TIMEOUT"))
.entities(List.of())
.build();
RetrievedEvidenceCandidate first = RetrievedEvidenceCandidate.builder()
.id("a")
.docId("a")
.chunkIndex(0)
.evidenceKey("a#chunk-0")
.source("a.md")
.title("Generic")
.content("generic troubleshooting")
.score(0.4)
.scoreLabel(RetrievalScoreLabels.DENSE)
.originalRank(1)
.hitReasons(List.of("base"))
.metadata(Map.of("category", "other"))
.build();
RetrievedEvidenceCandidate second = RetrievedEvidenceCandidate.builder()
.id("b")
.docId("b")
.chunkIndex(0)
.evidenceKey("b#chunk-0")
.source("b.md")
.title("Payment ERR_TIMEOUT")
.content("payment ERR_TIMEOUT timeout diagnosis")
.score(0.45)
.scoreLabel(RetrievalScoreLabels.DENSE)
.originalRank(2)
.hitReasons(List.of("base"))
.metadata(Map.of("category", "payment"))
.build();
EvidencePostprocessResult result = processor.process(q, List.of(first, second));
assertEquals("a.md", result.getEvidenceBlocks().get(0).getSource());
assertEquals("b.md", result.getEvidenceBlocks().get(1).getSource());
// explain-only overlap on second, no score delta boost strings
assertTrue(result.getEvidenceBlocks().get(1).getHitReasons().stream()
.anyMatch(r -> r.equals("l0_domain_overlap") || r.equals("l0_keyword_overlap")));
assertTrue(result.getRerankTrace().getItems().stream()
.flatMap(i -> i.getBoostReasons() == null ? java.util.stream.Stream.empty() : i.getBoostReasons().stream())
.noneMatch(r -> r.contains(":+0.")));
}
@Test
void mergesTrueDuplicateEvidenceKeysWithoutReplacingContent() {
EvidencePostprocessResult result = processor.process(query(), List.of(
candidate("shared", 0, "shared#chunk-0", "keep-me", 0.2),
candidate("shared", 0, "shared#chunk-0", "keep-me", 0.2, 1),
RetrievedEvidenceCandidate.builder()
.id("dup")
.docId("shared")
@@ -50,6 +103,7 @@ class KnowledgeEvidencePostProcessorTest {
.source("shared.md")
.content("drop-me")
.score(0.25)
.scoreLabel(RetrievalScoreLabels.DENSE)
.originalRank(2)
.hitReasons(List.of("extra"))
.metadata(Map.of())
@@ -65,10 +119,10 @@ class KnowledgeEvidencePostProcessorTest {
void enforcesMaxChunksPerDocument() {
ReflectionTestUtils.setField(processor, "maxChunksPerDocument", 2);
EvidencePostprocessResult result = processor.process(query(), List.of(
candidate("shared", 0, "shared#chunk-0", "c0", 0.1),
candidate("shared", 1, "shared#chunk-1", "c1", 0.2),
candidate("shared", 2, "shared#chunk-2", "c2", 0.3),
candidate("other", 0, "other#chunk-0", "o0", 0.15)
candidate("shared", 0, "shared#chunk-0", "c0", 0.1, 1),
candidate("shared", 1, "shared#chunk-1", "c1", 0.2, 2),
candidate("shared", 2, "shared#chunk-2", "c2", 0.3, 3),
candidate("other", 0, "other#chunk-0", "o0", 0.15, 4)
));
assertEquals(3, result.getEvidenceBlockCount());
@@ -83,14 +137,36 @@ class KnowledgeEvidencePostProcessorTest {
void enforcesReturnN() {
ReflectionTestUtils.setField(processor, "returnN", 1);
EvidencePostprocessResult result = processor.process(query(), List.of(
candidate("a", 0, "a#chunk-0", "a0", 0.1),
candidate("b", 0, "b#chunk-0", "b0", 0.2)
candidate("a", 0, "a#chunk-0", "a0", 0.1, 1),
candidate("b", 0, "b#chunk-0", "b0", 0.2, 2)
));
assertEquals(1, result.getEvidenceBlockCount());
assertEquals("a#chunk-0", result.getEvidenceBlocks().get(0).getEvidenceKey());
}
@Test
void hybridQualityUsesRankNotScoreMagnitude() {
EvidencePostprocessResult result = processor.process(query(), List.of(
RetrievedEvidenceCandidate.builder()
.id("h1")
.docId("h1")
.chunkIndex(0)
.evidenceKey("h1#chunk-0")
.source("h1.md")
.content("top hybrid")
.score(0.01)
.scoreLabel(RetrievalScoreLabels.HYBRID)
.originalRank(1)
.hitReasons(List.of())
.metadata(Map.of())
.build()
));
assertEquals(1.0, result.getTopSimilarity(), 1e-9);
assertEquals("PRECISE", result.getRelevanceLevel());
}
private static KnowledgeQuery query() {
return KnowledgeQuery.builder()
.originalQuery("q")
@@ -105,7 +181,8 @@ class KnowledgeEvidencePostProcessorTest {
int chunkIndex,
String evidenceKey,
String content,
double score) {
double score,
int originalRank) {
return RetrievedEvidenceCandidate.builder()
.id(evidenceKey)
.docId(docId)
@@ -114,7 +191,8 @@ class KnowledgeEvidencePostProcessorTest {
.source(docId + ".md")
.content(content)
.score(score)
.originalRank(chunkIndex + 1)
.scoreLabel(RetrievalScoreLabels.DENSE)
.originalRank(originalRank)
.hitReasons(List.of("base"))
.metadata(Map.of("docId", docId, "chunkIndex", String.valueOf(chunkIndex)))
.build();
@@ -63,7 +63,7 @@ class VectorSearchServiceTest {
result.setId(id);
result.setScore(score);
result.setRawScore((double) score);
result.setScoreLabel("l2_distance");
result.setScoreLabel("dense");
result.setContent("content");
result.setMetadata("{}");
return result;
@@ -0,0 +1,56 @@
package com.superbiz.agent.service.retrieval;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class RetrievalScoreNormalizerTest {
@Test
void denseL2MapsToSimilarity() {
assertEquals(1.0, RetrievalScoreNormalizer.l2ToQuality(0.0, 2.0), 1e-9);
assertEquals(0.0, RetrievalScoreNormalizer.l2ToQuality(2.0, 2.0), 1e-9);
assertEquals(0.9, RetrievalScoreNormalizer.l2ToQuality(0.2, 2.0), 1e-9);
assertEquals(0.0, RetrievalScoreNormalizer.l2ToQuality(null, 2.0), 1e-9);
}
@Test
void hybridRankIsMonotonicAndRankOneIsBest() {
double r1 = RetrievalScoreNormalizer.rankToQuality(1, 5);
double r3 = RetrievalScoreNormalizer.rankToQuality(3, 5);
double r5 = RetrievalScoreNormalizer.rankToQuality(5, 5);
assertEquals(1.0, r1, 1e-9);
assertTrue(r1 > r3 && r3 > r5);
assertTrue(r5 > 0.0);
}
@Test
void toQualityScoreUsesLabel() {
double dense = RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.DENSE, 0.2, 1, 3, 2.0);
assertEquals(0.9, dense, 1e-9);
// hybrid without denseDistance falls back to rank (rank1 → 1.0)
double hybridRank = RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.HYBRID, 99.0, 1, 4, 2.0, null);
assertEquals(1.0, hybridRank, 1e-9);
// hybrid with dense L2 uses absolute quality (gate / relevance)
double hybridDenseFar = RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.HYBRID, 99.0, 1, 4, 2.0, 1.4);
assertEquals(0.3, hybridDenseFar, 1e-9);
assertTrue(hybridDenseFar < 0.5);
double hybridRank2 = RetrievalScoreNormalizer.toQualityScore(
"rrf_fused", 0.01, 2, 4, 2.0, null);
assertTrue(hybridRank2 < hybridRank);
}
@Test
void canonicalizeAliases() {
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
}
}
@@ -42,7 +42,7 @@ class VectorKnowledgeSearchAdapterHybridTest {
result.setContent(content);
result.setScore(score);
result.setRawScore((double) score);
result.setScoreLabel("l2_distance");
result.setScoreLabel("dense");
return result;
}
}
@@ -182,7 +182,7 @@ class LookupKnowledgeToolTest {
}
@Test
void rerankUsesHintMatchesAndContextPackPreservesMetadata() {
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
VectorSearchService.SearchResult first = searchResult(
"a",
@@ -202,9 +202,14 @@ class LookupKnowledgeToolTest {
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
assertTrue(lookup.isFound());
assertEquals("b.md", lookup.getEvidenceBlocks().get(0).getSource());
assertTrue(lookup.getRerankTrace().getItems().get(0).getBoostReasons().stream()
.anyMatch(reason -> reason.startsWith("domain_match")));
// originalRank order wins; keyword/domain boost must not promote second over first
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
assertTrue(lookup.getRerankTrace().getItems().stream()
.flatMap(item -> item.getBoostReasons() == null
? java.util.stream.Stream.empty()
: item.getBoostReasons().stream())
.noneMatch(reason -> reason.contains(":+0.")));
assertTrue(lookup.getContextPack().getPackedText().contains("Payment > Timeout"));
assertTrue(lookup.getContextPack().getPackedText().contains("reasons:"));
}
@@ -258,7 +263,7 @@ class LookupKnowledgeToolTest {
result.setContent(content);
result.setScore(score);
result.setRawScore((double) score);
result.setScoreLabel("l2_distance");
result.setScoreLabel("dense");
return result;
}