feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
This commit is contained in:
@@ -91,6 +91,8 @@ class HarnessChatConfigurationTest {
|
||||
.withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class))
|
||||
.withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop)
|
||||
.withBean(DiagnosisTraceRecorder.class, DiagnosisTraceRecorder::noop)
|
||||
.withBean(com.superbiz.agent.harness.audit.AgentStepAuditTracker.class,
|
||||
com.superbiz.agent.harness.audit.AgentStepAuditTracker::new)
|
||||
.withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class))
|
||||
.withBean(AgentReasoningAuditRepository.class,
|
||||
() -> mock(AgentReasoningAuditRepository.class))
|
||||
|
||||
@@ -10,18 +10,29 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Generates RAG retrieval fixtures from the real LookupKnowledgeTool bean.
|
||||
* Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
|
||||
*
|
||||
* <p>This class is disabled by default because it writes repository files and
|
||||
* depends on the configured runtime retrieval stack.</p>
|
||||
* <p>Disabled by default: writes repository files and needs the live retrieval stack
|
||||
* (embedding + Milvus hybrid collection + optional MySQL/L0).</p>
|
||||
*
|
||||
* <p>System properties (via Maven {@code -D}):</p>
|
||||
* <ul>
|
||||
* <li>{@code rag.snapshot.enabled=true} — required to run</li>
|
||||
* <li>{@code retrieval.search.mode=hybrid|dense} — default hybrid</li>
|
||||
* <li>{@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})</li>
|
||||
* <li>{@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}</li>
|
||||
* </ul>
|
||||
*/
|
||||
@SpringBootTest(
|
||||
classes = Main.class,
|
||||
@@ -40,11 +51,31 @@ class RagLookupSnapshotGeneratorTest {
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
|
||||
*/
|
||||
@DynamicPropertySource
|
||||
static void retrievalProperties(DynamicPropertyRegistry registry) {
|
||||
String mode = System.getProperty("retrieval.search.mode", "hybrid");
|
||||
if (mode == null || mode.isBlank()) {
|
||||
mode = "hybrid";
|
||||
}
|
||||
String normalized = mode.trim().toLowerCase(Locale.ROOT);
|
||||
registry.add("retrieval.search.mode", () -> normalized);
|
||||
|
||||
String kbScope = System.getProperty("retrieval.kb-scope", "");
|
||||
if (kbScope != null && !kbScope.isBlank()) {
|
||||
registry.add("retrieval.kb-scope", kbScope::trim);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void generateLookupResultFixtures() throws Exception {
|
||||
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
|
||||
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
|
||||
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
|
||||
String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
|
||||
String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
|
||||
|
||||
JsonNode root = objectMapper.readTree(casesPath.toFile());
|
||||
JsonNode cases = root.path("cases");
|
||||
@@ -61,6 +92,10 @@ class RagLookupSnapshotGeneratorTest {
|
||||
fixture.put("caseId", caseId);
|
||||
fixture.put("query", query);
|
||||
fixture.put("retrievedAt", retrievedAt);
|
||||
fixture.put("searchMode", searchMode);
|
||||
if (kbScope != null) {
|
||||
fixture.put("kbScope", kbScope);
|
||||
}
|
||||
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
|
||||
|
||||
Path output = fixturesDir.resolve(caseId + ".json");
|
||||
@@ -68,6 +103,20 @@ class RagLookupSnapshotGeneratorTest {
|
||||
}
|
||||
}
|
||||
|
||||
private static String normalizeMode(String mode) {
|
||||
if (mode == null || mode.isBlank()) {
|
||||
return "hybrid";
|
||||
}
|
||||
return mode.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static String blankToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private String requiredText(JsonNode node, String fieldName) {
|
||||
JsonNode value = node.get(fieldName);
|
||||
if (value == null || value.asText().isBlank()) {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class HarnessEvidenceToolsTest {
|
||||
|
||||
@Test
|
||||
void omitsQueryMysqlWhenInvokerIsNull() {
|
||||
EvidenceToolInvoker unused = (context, id, args) ->
|
||||
ToolBoundaryResult.ready(id, "{}", EvidenceStatus.NO_EVIDENCE);
|
||||
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, null);
|
||||
|
||||
assertTrue(tools.supports(AgentToolContracts.LOOKUP_KNOWLEDGE));
|
||||
assertTrue(tools.supports(AgentToolContracts.QUERY_LOGS));
|
||||
assertFalse(tools.supports(AgentToolContracts.QUERY_MYSQL));
|
||||
|
||||
List<String> names = tools.callbacks().stream().map(ToolCallback::getToolDefinition)
|
||||
.map(def -> def.name()).toList();
|
||||
assertEquals(2, names.size());
|
||||
assertTrue(names.contains(AgentToolContracts.LOOKUP_KNOWLEDGE));
|
||||
assertTrue(names.contains(AgentToolContracts.QUERY_LOGS));
|
||||
assertFalse(names.contains(AgentToolContracts.QUERY_MYSQL));
|
||||
}
|
||||
|
||||
@Test
|
||||
void registersQueryMysqlWhenInvokerPresent() {
|
||||
EvidenceToolInvoker unused = (context, id, args) ->
|
||||
ToolBoundaryResult.ready(id, "{}", EvidenceStatus.NO_EVIDENCE);
|
||||
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, unused);
|
||||
|
||||
assertTrue(tools.supports(AgentToolContracts.QUERY_MYSQL));
|
||||
assertEquals(3, tools.callbacks().size());
|
||||
}
|
||||
}
|
||||
+28
-1
@@ -33,6 +33,7 @@ import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -106,7 +107,7 @@ class PublishedResultPersistenceTest {
|
||||
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
|
||||
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
|
||||
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
|
||||
"{\"fallback\":{\"type\":\"SEMANTIC_UNAVAILABLE\",\"message\":\"语义校验不可用\",\"conclusion\":null}}",
|
||||
new PublishedResult("q", "c", "scope", List.of(),
|
||||
List.of(new SourceDocument("doc", "title"))), 12);
|
||||
|
||||
@@ -115,6 +116,32 @@ class PublishedResultPersistenceTest {
|
||||
assertNull(entity.getPublishedResult());
|
||||
assertEquals(12, entity.getTotalDurationMs());
|
||||
assertEquals(2, entity.getStepCount());
|
||||
assertNotNull(entity.getConclusion());
|
||||
assertTrue(entity.getConclusion().contains("SEMANTIC_UNAVAILABLE"));
|
||||
assertTrue(entity.getConclusion().contains("语义校验不可用"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void successCompletionExtractsConclusionBesideQuery() {
|
||||
DiagnosisRunRepository runs = mock(DiagnosisRunRepository.class);
|
||||
ChatSessionRepository sessions = mock(ChatSessionRepository.class);
|
||||
DiagnosisRun entity = DiagnosisRun.builder()
|
||||
.runId("run-ok").sessionId("session-1").query("pool?").status("RUNNING").build();
|
||||
when(runs.findByRunId("run-ok")).thenReturn(Optional.of(entity));
|
||||
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
when(sessions.findBySessionId("session-1")).thenReturn(Optional.empty());
|
||||
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
|
||||
RunContext context = core().startRun("session-1", "run-ok");
|
||||
|
||||
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.SUCCESS,
|
||||
"{\"report\":{\"conclusion\":{\"text\":\"Pool exhausted per runbook.\"}}}",
|
||||
new PublishedResult("pool?", "Pool exhausted per runbook.", "scope", List.of(),
|
||||
List.of(new SourceDocument("doc", "title"))), 20);
|
||||
|
||||
assertEquals("Pool exhausted per runbook.", entity.getConclusion());
|
||||
assertEquals("pool?", entity.getQuery());
|
||||
assertNotNull(entity.getAnswer());
|
||||
assertNotNull(entity.getPublishedResult());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -2,23 +2,24 @@ package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.AgentReasoningAudit;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentFactory;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.deepseek.DeepSeekAssistantMessage;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -30,7 +31,7 @@ import static org.mockito.Mockito.when;
|
||||
class HarnessAgentAuditHookTest {
|
||||
|
||||
@Test
|
||||
void persistsMetadataAndReasoningInSeparateAudit() {
|
||||
void persistsBothProviderReasoningAndAssistantText() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(7L).build();
|
||||
@@ -47,7 +48,7 @@ class HarnessAgentAuditHookTest {
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("secret-query")), config);
|
||||
AssistantMessage response = AssistantMessage.builder()
|
||||
.content("secret-model-output")
|
||||
.content("public assistant conclusion text")
|
||||
.properties(java.util.Map.of(
|
||||
"reasoning_content", "inspect bounded evidence before selecting query_logs"))
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
@@ -61,13 +62,16 @@ class HarnessAgentAuditHookTest {
|
||||
AgentStep completed = captor.getAllValues().get(1);
|
||||
assertEquals("session-audit", started.getSessionId());
|
||||
assertEquals("run-audit", started.getRunId());
|
||||
// model_input stays summary-only (no raw user text)
|
||||
assertFalse(started.getModelInput().contains("secret-query"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-model-output"));
|
||||
// model_output stays summary-only
|
||||
assertFalse(completed.getModelOutput().contains("public assistant conclusion text"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-argument"));
|
||||
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"],"
|
||||
+ "\"reasoning_available\":true,\"reasoning_bytes\":52}",
|
||||
completed.getModelOutput());
|
||||
assertNull(completed.getThought());
|
||||
assertTrue(completed.getModelOutput().contains("\"reasoning_available\":true"));
|
||||
assertTrue(completed.getModelOutput().contains("PROVIDER_REASONING+ASSISTANT_TEXT"));
|
||||
// thought keeps provider reasoning preferentially
|
||||
assertEquals("inspect bounded evidence before selecting query_logs", completed.getThought());
|
||||
|
||||
ArgumentCaptor<AgentReasoningAudit> reasoningCaptor =
|
||||
ArgumentCaptor.forClass(AgentReasoningAudit.class);
|
||||
verify(reasoningRepository).save(reasoningCaptor.capture());
|
||||
@@ -78,16 +82,88 @@ class HarnessAgentAuditHookTest {
|
||||
assertTrue(reasoning.getReasoningAvailable());
|
||||
assertEquals("inspect bounded evidence before selecting query_logs",
|
||||
reasoning.getReasoningContent());
|
||||
assertTrue(reasoning.getAssistantText().contains("public assistant conclusion text"));
|
||||
assertTrue(reasoning.getAssistantText().contains("tool_calls:"));
|
||||
assertTrue(reasoning.getAssistantText().contains("query_logs"));
|
||||
assertEquals(HarnessAgentAuditHook.SOURCE_BOTH, reasoning.getContentSource());
|
||||
assertTrue(reasoning.getContentBytes() > 0);
|
||||
|
||||
assertNotNull(trace.get());
|
||||
assertEquals(TraceEventType.AGENT_MODEL_STEP, trace.get().eventType());
|
||||
String traceText = trace.get().details().toString();
|
||||
assertFalse(traceText.contains("secret-query"));
|
||||
assertFalse(traceText.contains("secret-model-output"));
|
||||
assertFalse(traceText.contains("secret-argument"));
|
||||
assertFalse(traceText.contains("public assistant conclusion text"));
|
||||
assertFalse(traceText.contains("inspect bounded evidence"));
|
||||
assertTrue(traceText.contains("reasoning_available=true"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void extractsReasoningFromDeepSeekAssistantMessageField() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(9L).build();
|
||||
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
|
||||
when(repository.findById(9L)).thenReturn(Optional.of(persisted));
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
|
||||
DiagnosisTraceRecorder.noop(), reasoningRepository);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", "session-ds")
|
||||
.addMetadata("runId", "run-ds")
|
||||
.build();
|
||||
|
||||
// Production path: Spring AI DeepSeekChatModel returns DeepSeekAssistantMessage
|
||||
// with reasoning on the dedicated field, NOT metadata.
|
||||
DeepSeekAssistantMessage response = new DeepSeekAssistantMessage.Builder()
|
||||
.content("final answer body")
|
||||
.reasoningContent("step1: inspect evidence\nstep2: call lookup_knowledge")
|
||||
.properties(Map.of())
|
||||
.build();
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("q")), config);
|
||||
hook.afterModel(List.of(response), config);
|
||||
|
||||
ArgumentCaptor<AgentReasoningAudit> captor = ArgumentCaptor.forClass(AgentReasoningAudit.class);
|
||||
verify(reasoningRepository).save(captor.capture());
|
||||
AgentReasoningAudit row = captor.getValue();
|
||||
assertTrue(row.getReasoningAvailable());
|
||||
assertEquals("step1: inspect evidence\nstep2: call lookup_knowledge",
|
||||
row.getReasoningContent());
|
||||
assertEquals("final answer body", row.getAssistantText());
|
||||
assertEquals(HarnessAgentAuditHook.SOURCE_BOTH, row.getContentSource());
|
||||
// metadata-only path must not be required
|
||||
assertTrue(HarnessAgentAuditHook.providerReasoning(response).contains("inspect evidence"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void withoutProviderReasoningStillStoresAssistantText() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(8L).build();
|
||||
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
|
||||
when(repository.findById(8L)).thenReturn(Optional.of(persisted));
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
|
||||
DiagnosisTraceRecorder.noop(), reasoningRepository);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", "session-2")
|
||||
.addMetadata("runId", "run-2")
|
||||
.build();
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("q")), config);
|
||||
hook.afterModel(List.of(AssistantMessage.builder()
|
||||
.content("only assistant body")
|
||||
.build()), config);
|
||||
|
||||
ArgumentCaptor<AgentReasoningAudit> captor = ArgumentCaptor.forClass(AgentReasoningAudit.class);
|
||||
verify(reasoningRepository).save(captor.capture());
|
||||
AgentReasoningAudit row = captor.getValue();
|
||||
assertFalse(row.getReasoningAvailable());
|
||||
assertNullSafe(row.getReasoningContent());
|
||||
assertEquals("only assistant body", row.getAssistantText());
|
||||
assertEquals(HarnessAgentAuditHook.SOURCE_ASSISTANT, row.getContentSource());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingIdentitySkipsPersistence() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
@@ -98,4 +174,27 @@ class HarnessAgentAuditHookTest {
|
||||
|
||||
verify(repository, never()).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void bindsStepIdToTrackerForToolAuditLinkage() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(42L).build();
|
||||
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
|
||||
AgentStepAuditTracker tracker = new AgentStepAuditTracker();
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
|
||||
DiagnosisTraceRecorder.noop(), null, tracker);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", "session-bind")
|
||||
.addMetadata("runId", "run-bind")
|
||||
.build();
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("q")), config);
|
||||
|
||||
assertEquals(42L, tracker.currentStepId("run-bind"));
|
||||
}
|
||||
|
||||
private static void assertNullSafe(String value) {
|
||||
assertTrue(value == null || value.isBlank());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,17 +10,19 @@ import org.mockito.ArgumentCaptor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
class JpaToolInvocationAuditSinkTest {
|
||||
|
||||
@Test
|
||||
void persistsOnlyBoundedStableMetadata() {
|
||||
void persistsOnlyBoundedStableMetadataForGenericTools() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
|
||||
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(
|
||||
repository, new ObjectMapper(), traceRecorder);
|
||||
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
|
||||
repository, new ObjectMapper(), traceRecorder, null);
|
||||
|
||||
sink.record(new ToolInvocationAuditEvent(
|
||||
"session-1", "run-1", "call-1", "query_logs",
|
||||
@@ -36,14 +38,100 @@ class JpaToolInvocationAuditSinkTest {
|
||||
assertEquals("{\"tool_call_id\":\"call-1\",\"request_bytes\":83}", saved.getInputParams());
|
||||
assertEquals("status=ERROR,evidence_status=ERROR", saved.getOutputPreview());
|
||||
assertEquals("TOOL_EXECUTION_ERROR", saved.getErrorMessage());
|
||||
// generic tools must not put evidence_status into relevance_level column
|
||||
assertNull(saved.getRelevanceLevel());
|
||||
assertNull(saved.getStepId());
|
||||
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
|
||||
assertFalse(serialized.contains("query"));
|
||||
assertFalse(serialized.contains("raw_response"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("evidence_status"));
|
||||
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
|
||||
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
|
||||
verify(traceRecorder).record(traceCaptor.capture());
|
||||
assertEquals(TraceEventType.TOOL_INVOCATION, traceCaptor.getValue().eventType());
|
||||
assertEquals("call-1", traceCaptor.getValue().details().get("tool_call_id"));
|
||||
assertFalse(traceCaptor.getValue().details().toString().contains("raw_response"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistsStepIdAndSafeQueryPreviewInInputParams() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
|
||||
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
|
||||
repository, new ObjectMapper(), traceRecorder, null);
|
||||
|
||||
String longQuery = "q".repeat(200);
|
||||
sink.record(new ToolInvocationAuditEvent(
|
||||
"session-1", "run-1", "call-q", "lookup_knowledge",
|
||||
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
|
||||
12, 50, 80, null, null, 99L,
|
||||
"{\"query\":\"" + longQuery + "\",\"password\":\"should-not-store\",\"limit\":5}"));
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
assertEquals(99L, saved.getStepId());
|
||||
assertTrue(saved.getInputParams().contains("\"step_id\":99"));
|
||||
assertTrue(saved.getInputParams().contains("\"query\":\"" + "q".repeat(160) + "\""));
|
||||
assertTrue(saved.getInputParams().contains("\"query_truncated\":true"));
|
||||
assertTrue(saved.getInputParams().contains("\"query_chars\":200"));
|
||||
assertTrue(saved.getInputParams().contains("\"limit\":5"));
|
||||
assertFalse(saved.getInputParams().contains("should-not-store"));
|
||||
assertFalse(saved.getInputParams().contains("password"));
|
||||
|
||||
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
|
||||
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
|
||||
verify(traceRecorder).record(traceCaptor.capture());
|
||||
assertEquals(99L, traceCaptor.getValue().details().get("step_id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void enrichesLookupKnowledgeWithRagFieldsAndTrueRelevanceLevel() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
|
||||
RagLookupAuditEnricher enricher = new RagLookupAuditEnricher(new ObjectMapper(), "hybrid");
|
||||
JpaToolInvocationAuditSink sink = JpaToolInvocationAuditSink.forTest(
|
||||
repository, new ObjectMapper(), traceRecorder, enricher);
|
||||
|
||||
String lookup = """
|
||||
{
|
||||
"found": true,
|
||||
"relevanceLevel": "REFERENCE",
|
||||
"evidenceBlockCount": 1,
|
||||
"evidenceBlocks": [
|
||||
{"evidenceKey":"doc#chunk-0","source":"doc","retrievalLayer":"L1","content":"body"}
|
||||
],
|
||||
"retrievalTrace": {
|
||||
"selectedAttempt": "FILTERED_VECTOR",
|
||||
"fallbackReason": null,
|
||||
"queryHints": {"l0_match_count": 2, "domains": ["mysql"]},
|
||||
"attempts": [{"name":"FILTERED_VECTOR","candidateCount":3,"usable":true,"topSimilarity":0.8}]
|
||||
}
|
||||
}
|
||||
""";
|
||||
String agent = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","returned_count":1,"truncated":false,"relevance_level":"REFERENCE"}
|
||||
""";
|
||||
|
||||
sink.record(new ToolInvocationAuditEvent(
|
||||
"session-1", "run-1", "call-rag", "lookup_knowledge",
|
||||
InvocationStatus.READY, EvidenceStatus.EVIDENCE_FOUND, null,
|
||||
42, 10, 100, lookup, agent, 7L,
|
||||
"{\"query\":\"MySQL HikariCP pool exhausted\"}"));
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
assertEquals(7L, saved.getStepId());
|
||||
assertEquals("L1", saved.getRetrievalLayer());
|
||||
assertEquals("REFERENCE", saved.getRelevanceLevel());
|
||||
assertEquals(2, saved.getL0MatchCount());
|
||||
assertEquals(1, saved.getL1MatchCount());
|
||||
assertTrue(saved.getRetrievalDetails().contains("FILTERED_VECTOR"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("search_mode"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("EVIDENCE_FOUND"));
|
||||
assertTrue(saved.getOutputPreview().contains("REFERENCE"));
|
||||
assertTrue(saved.getInputParams().contains("MySQL HikariCP pool exhausted"));
|
||||
assertTrue(saved.getInputParams().contains("\"step_id\":7"));
|
||||
// must not store full excerpt dump as sole content; body may appear only if tiny — ensure attempt present
|
||||
assertTrue(saved.getRetrievalDetails().contains("attempts"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RagLookupAuditEnricherTest {
|
||||
|
||||
private final RagLookupAuditEnricher enricher =
|
||||
new RagLookupAuditEnricher(new ObjectMapper(), "hybrid");
|
||||
|
||||
@Test
|
||||
void extractsBoundedRagFieldsWithoutFullQueryOrExcerpt() {
|
||||
String lookup = """
|
||||
{
|
||||
"found": true,
|
||||
"relevanceLevel": "PRECISE",
|
||||
"evidenceCandidateCount": 8,
|
||||
"evidenceBlockCount": 2,
|
||||
"evidenceBlocks": [
|
||||
{
|
||||
"docId": "mysql-pool",
|
||||
"evidenceKey": "mysql-pool#chunk-0",
|
||||
"source": "mysql-pool",
|
||||
"retrievalLayer": "L1",
|
||||
"content": "HikariCP details should not be required in audit details dump"
|
||||
}
|
||||
],
|
||||
"retrievalTrace": {
|
||||
"selectedAttempt": "UNFILTERED_VECTOR_RETRY",
|
||||
"fallbackReason": "filtered_vector_low_quality",
|
||||
"categoryFilter": "overfilter-decoy",
|
||||
"evidenceStatus": "supported",
|
||||
"originalQuery": "secret user query should not be stored",
|
||||
"queryHints": {
|
||||
"l0_match_count": 1,
|
||||
"domains": ["rag"],
|
||||
"matched_keywords": ["pool"]
|
||||
},
|
||||
"attempts": [
|
||||
{
|
||||
"name": "FILTERED_VECTOR",
|
||||
"categoryFilter": "overfilter-decoy",
|
||||
"candidateCount": 2,
|
||||
"usable": false,
|
||||
"topSimilarity": 0.3,
|
||||
"durationMs": 12
|
||||
},
|
||||
{
|
||||
"name": "UNFILTERED_VECTOR_RETRY",
|
||||
"candidateCount": 5,
|
||||
"usable": true,
|
||||
"topSimilarity": 0.9,
|
||||
"durationMs": 20
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
""";
|
||||
String agent = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","returned_count":2,"truncated":true,"relevance_level":"PRECISE"}
|
||||
""";
|
||||
|
||||
RagLookupAuditEnricher.Enrichment e = enricher.enrich(lookup, agent);
|
||||
|
||||
assertEquals("L1", e.retrievalLayer());
|
||||
assertEquals(1, e.l0MatchCount());
|
||||
assertEquals(1, e.l1MatchCount()); // one evidence block in fixture
|
||||
assertEquals(8, e.retrievalDetails().get("evidence_candidate_count"));
|
||||
assertEquals("PRECISE", e.relevanceLevel());
|
||||
assertTrue(e.truncated());
|
||||
assertEquals("UNFILTERED_VECTOR_RETRY", e.retrievalDetails().get("selected_attempt"));
|
||||
assertEquals("filtered_vector_low_quality", e.retrievalDetails().get("fallback_reason"));
|
||||
assertEquals("hybrid", e.retrievalDetails().get("search_mode"));
|
||||
assertEquals(List.of("mysql-pool#chunk-0"), e.retrievalDetails().get("evidence_keys"));
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Map<String, Object>> attempts = (List<Map<String, Object>>) e.retrievalDetails().get("attempts");
|
||||
assertEquals(2, attempts.size());
|
||||
|
||||
String blob = e.retrievalDetails().toString() + e.outputPreview();
|
||||
assertFalse(blob.contains("secret user query"));
|
||||
assertFalse(blob.contains("HikariCP details should not"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RunConclusionExtractorTest {
|
||||
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void extractsReportConclusion() {
|
||||
String json = """
|
||||
{"report":{"conclusion":{"text":"Pool is exhausted.","based_on_analysis_ids":["a1"]}}}
|
||||
""";
|
||||
assertEquals("Pool is exhausted.", RunConclusionExtractor.extract(mapper, json));
|
||||
}
|
||||
|
||||
@Test
|
||||
void extractsFallbackMessageWithType() {
|
||||
String json = """
|
||||
{"fallback":{"type":"INSUFFICIENT_EVIDENCE","message":"证据不足","conclusion":null}}
|
||||
""";
|
||||
String out = RunConclusionExtractor.extract(mapper, json);
|
||||
assertTrue(out.contains("INSUFFICIENT_EVIDENCE"));
|
||||
assertTrue(out.contains("证据不足"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void extractsBareFallbackTypeOnly() {
|
||||
assertEquals("SEMANTIC_UNAVAILABLE",
|
||||
RunConclusionExtractor.extract(mapper, "{\"type\":\"SEMANTIC_UNAVAILABLE\"}"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void nullOrBlankReturnsNull() {
|
||||
assertNull(RunConclusionExtractor.extract(mapper, null));
|
||||
assertNull(RunConclusionExtractor.extract(mapper, " "));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.tool.boundary;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.AgentStepAuditTracker;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
@@ -179,6 +180,8 @@ class ToolBoundaryTest {
|
||||
assertEquals("session-audit", audit.get().sessionId());
|
||||
assertEquals("run-audit", audit.get().runId());
|
||||
assertEquals("call-audit", audit.get().toolCallId());
|
||||
assertEquals("{\"query\":\"timeout\"}", audit.get().requestJson());
|
||||
assertNull(audit.get().stepId());
|
||||
|
||||
FakeStore failingAuditStore = new FakeStore(
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
|
||||
@@ -195,6 +198,28 @@ class ToolBoundaryTest {
|
||||
assertEquals(EvidenceStatus.NO_EVIDENCE, unchanged.evidenceStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
void auditEventCarriesBoundStepIdFromTracker() {
|
||||
MutableClock clock = new MutableClock(Instant.parse("2026-07-21T10:00:00Z"));
|
||||
FakeStore store = new FakeStore(new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
|
||||
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
|
||||
AgentStepAuditTracker tracker = new AgentStepAuditTracker();
|
||||
tracker.bind("run-step", 55L);
|
||||
AtomicReference<ToolInvocationAuditEvent> audit = new AtomicReference<>();
|
||||
ToolBoundary boundary = new ToolBoundary(core,
|
||||
new ToolCallKeyFactory("superbiz:harness:tool-call"),
|
||||
store, objectMapper, clock, audit::set, tracker);
|
||||
RunContext context = core.startRun("session-step", "run-step");
|
||||
|
||||
boundary.execute(
|
||||
context, request("run-step", "call-step", true, true),
|
||||
ignored -> "raw",
|
||||
ignored -> new ProjectedToolResult("agent", EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
assertEquals(55L, audit.get().stepId());
|
||||
assertEquals("{\"query\":\"timeout\"}", audit.get().requestJson());
|
||||
}
|
||||
|
||||
private ToolBoundary boundary(MutableClock clock, FakeStore store) {
|
||||
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
|
||||
return new ToolBoundary(core, new ToolCallKeyFactory("superbiz:harness:tool-call"),
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.superbiz.agent.service;
|
||||
import com.superbiz.agent.dto.EvidencePostprocessResult;
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import com.superbiz.agent.dto.RetrievedEvidenceCandidate;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
@@ -22,26 +23,78 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
processor = new KnowledgeEvidencePostProcessor();
|
||||
ReflectionTestUtils.setField(processor, "maxChunksPerDocument", 2);
|
||||
ReflectionTestUtils.setField(processor, "returnN", 5);
|
||||
ReflectionTestUtils.setField(processor, "maxL2Distance", 2.0);
|
||||
ReflectionTestUtils.setField(processor, "highlyRelevantThreshold", 0.75);
|
||||
ReflectionTestUtils.setField(processor, "referenceThreshold", 0.5);
|
||||
}
|
||||
|
||||
@Test
|
||||
void keepsDistinctChunksFromSameDocument() {
|
||||
EvidencePostprocessResult result = processor.process(query(), List.of(
|
||||
candidate("shared", 0, "shared#chunk-0", "c0", 0.2),
|
||||
candidate("shared", 1, "shared#chunk-1", "c1", 0.3)
|
||||
candidate("shared", 0, "shared#chunk-0", "c0", 0.2, 1),
|
||||
candidate("shared", 1, "shared#chunk-1", "c1", 0.3, 2)
|
||||
));
|
||||
|
||||
assertEquals(2, result.getEvidenceBlockCount());
|
||||
assertEquals("shared#chunk-0", result.getEvidenceBlocks().get(0).getEvidenceKey());
|
||||
assertEquals("shared#chunk-1", result.getEvidenceBlocks().get(1).getEvidenceKey());
|
||||
assertEquals("c0", result.getEvidenceBlocks().get(0).getContent());
|
||||
assertEquals("c1", result.getEvidenceBlocks().get(1).getContent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesRetrievalRankOverKeywordOverlap() {
|
||||
KnowledgeQuery q = KnowledgeQuery.builder()
|
||||
.originalQuery("payment")
|
||||
.rewrittenQuery("payment")
|
||||
.domainHints(List.of("payment"))
|
||||
.matchedKeywords(List.of("ERR_TIMEOUT"))
|
||||
.entities(List.of())
|
||||
.build();
|
||||
|
||||
RetrievedEvidenceCandidate first = RetrievedEvidenceCandidate.builder()
|
||||
.id("a")
|
||||
.docId("a")
|
||||
.chunkIndex(0)
|
||||
.evidenceKey("a#chunk-0")
|
||||
.source("a.md")
|
||||
.title("Generic")
|
||||
.content("generic troubleshooting")
|
||||
.score(0.4)
|
||||
.scoreLabel(RetrievalScoreLabels.DENSE)
|
||||
.originalRank(1)
|
||||
.hitReasons(List.of("base"))
|
||||
.metadata(Map.of("category", "other"))
|
||||
.build();
|
||||
RetrievedEvidenceCandidate second = RetrievedEvidenceCandidate.builder()
|
||||
.id("b")
|
||||
.docId("b")
|
||||
.chunkIndex(0)
|
||||
.evidenceKey("b#chunk-0")
|
||||
.source("b.md")
|
||||
.title("Payment ERR_TIMEOUT")
|
||||
.content("payment ERR_TIMEOUT timeout diagnosis")
|
||||
.score(0.45)
|
||||
.scoreLabel(RetrievalScoreLabels.DENSE)
|
||||
.originalRank(2)
|
||||
.hitReasons(List.of("base"))
|
||||
.metadata(Map.of("category", "payment"))
|
||||
.build();
|
||||
|
||||
EvidencePostprocessResult result = processor.process(q, List.of(first, second));
|
||||
|
||||
assertEquals("a.md", result.getEvidenceBlocks().get(0).getSource());
|
||||
assertEquals("b.md", result.getEvidenceBlocks().get(1).getSource());
|
||||
// explain-only overlap on second, no score delta boost strings
|
||||
assertTrue(result.getEvidenceBlocks().get(1).getHitReasons().stream()
|
||||
.anyMatch(r -> r.equals("l0_domain_overlap") || r.equals("l0_keyword_overlap")));
|
||||
assertTrue(result.getRerankTrace().getItems().stream()
|
||||
.flatMap(i -> i.getBoostReasons() == null ? java.util.stream.Stream.empty() : i.getBoostReasons().stream())
|
||||
.noneMatch(r -> r.contains(":+0.")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mergesTrueDuplicateEvidenceKeysWithoutReplacingContent() {
|
||||
EvidencePostprocessResult result = processor.process(query(), List.of(
|
||||
candidate("shared", 0, "shared#chunk-0", "keep-me", 0.2),
|
||||
candidate("shared", 0, "shared#chunk-0", "keep-me", 0.2, 1),
|
||||
RetrievedEvidenceCandidate.builder()
|
||||
.id("dup")
|
||||
.docId("shared")
|
||||
@@ -50,6 +103,7 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
.source("shared.md")
|
||||
.content("drop-me")
|
||||
.score(0.25)
|
||||
.scoreLabel(RetrievalScoreLabels.DENSE)
|
||||
.originalRank(2)
|
||||
.hitReasons(List.of("extra"))
|
||||
.metadata(Map.of())
|
||||
@@ -65,10 +119,10 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
void enforcesMaxChunksPerDocument() {
|
||||
ReflectionTestUtils.setField(processor, "maxChunksPerDocument", 2);
|
||||
EvidencePostprocessResult result = processor.process(query(), List.of(
|
||||
candidate("shared", 0, "shared#chunk-0", "c0", 0.1),
|
||||
candidate("shared", 1, "shared#chunk-1", "c1", 0.2),
|
||||
candidate("shared", 2, "shared#chunk-2", "c2", 0.3),
|
||||
candidate("other", 0, "other#chunk-0", "o0", 0.15)
|
||||
candidate("shared", 0, "shared#chunk-0", "c0", 0.1, 1),
|
||||
candidate("shared", 1, "shared#chunk-1", "c1", 0.2, 2),
|
||||
candidate("shared", 2, "shared#chunk-2", "c2", 0.3, 3),
|
||||
candidate("other", 0, "other#chunk-0", "o0", 0.15, 4)
|
||||
));
|
||||
|
||||
assertEquals(3, result.getEvidenceBlockCount());
|
||||
@@ -83,14 +137,36 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
void enforcesReturnN() {
|
||||
ReflectionTestUtils.setField(processor, "returnN", 1);
|
||||
EvidencePostprocessResult result = processor.process(query(), List.of(
|
||||
candidate("a", 0, "a#chunk-0", "a0", 0.1),
|
||||
candidate("b", 0, "b#chunk-0", "b0", 0.2)
|
||||
candidate("a", 0, "a#chunk-0", "a0", 0.1, 1),
|
||||
candidate("b", 0, "b#chunk-0", "b0", 0.2, 2)
|
||||
));
|
||||
|
||||
assertEquals(1, result.getEvidenceBlockCount());
|
||||
assertEquals("a#chunk-0", result.getEvidenceBlocks().get(0).getEvidenceKey());
|
||||
}
|
||||
|
||||
@Test
|
||||
void hybridQualityUsesRankNotScoreMagnitude() {
|
||||
EvidencePostprocessResult result = processor.process(query(), List.of(
|
||||
RetrievedEvidenceCandidate.builder()
|
||||
.id("h1")
|
||||
.docId("h1")
|
||||
.chunkIndex(0)
|
||||
.evidenceKey("h1#chunk-0")
|
||||
.source("h1.md")
|
||||
.content("top hybrid")
|
||||
.score(0.01)
|
||||
.scoreLabel(RetrievalScoreLabels.HYBRID)
|
||||
.originalRank(1)
|
||||
.hitReasons(List.of())
|
||||
.metadata(Map.of())
|
||||
.build()
|
||||
));
|
||||
|
||||
assertEquals(1.0, result.getTopSimilarity(), 1e-9);
|
||||
assertEquals("PRECISE", result.getRelevanceLevel());
|
||||
}
|
||||
|
||||
private static KnowledgeQuery query() {
|
||||
return KnowledgeQuery.builder()
|
||||
.originalQuery("q")
|
||||
@@ -105,7 +181,8 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
int chunkIndex,
|
||||
String evidenceKey,
|
||||
String content,
|
||||
double score) {
|
||||
double score,
|
||||
int originalRank) {
|
||||
return RetrievedEvidenceCandidate.builder()
|
||||
.id(evidenceKey)
|
||||
.docId(docId)
|
||||
@@ -114,7 +191,8 @@ class KnowledgeEvidencePostProcessorTest {
|
||||
.source(docId + ".md")
|
||||
.content(content)
|
||||
.score(score)
|
||||
.originalRank(chunkIndex + 1)
|
||||
.scoreLabel(RetrievalScoreLabels.DENSE)
|
||||
.originalRank(originalRank)
|
||||
.hitReasons(List.of("base"))
|
||||
.metadata(Map.of("docId", docId, "chunkIndex", String.valueOf(chunkIndex)))
|
||||
.build();
|
||||
|
||||
@@ -63,7 +63,7 @@ class VectorSearchServiceTest {
|
||||
result.setId(id);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("l2_distance");
|
||||
result.setScoreLabel("dense");
|
||||
result.setContent("content");
|
||||
result.setMetadata("{}");
|
||||
return result;
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RetrievalScoreNormalizerTest {
|
||||
|
||||
@Test
|
||||
void denseL2MapsToSimilarity() {
|
||||
assertEquals(1.0, RetrievalScoreNormalizer.l2ToQuality(0.0, 2.0), 1e-9);
|
||||
assertEquals(0.0, RetrievalScoreNormalizer.l2ToQuality(2.0, 2.0), 1e-9);
|
||||
assertEquals(0.9, RetrievalScoreNormalizer.l2ToQuality(0.2, 2.0), 1e-9);
|
||||
assertEquals(0.0, RetrievalScoreNormalizer.l2ToQuality(null, 2.0), 1e-9);
|
||||
}
|
||||
|
||||
@Test
|
||||
void hybridRankIsMonotonicAndRankOneIsBest() {
|
||||
double r1 = RetrievalScoreNormalizer.rankToQuality(1, 5);
|
||||
double r3 = RetrievalScoreNormalizer.rankToQuality(3, 5);
|
||||
double r5 = RetrievalScoreNormalizer.rankToQuality(5, 5);
|
||||
assertEquals(1.0, r1, 1e-9);
|
||||
assertTrue(r1 > r3 && r3 > r5);
|
||||
assertTrue(r5 > 0.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void toQualityScoreUsesLabel() {
|
||||
double dense = RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.DENSE, 0.2, 1, 3, 2.0);
|
||||
assertEquals(0.9, dense, 1e-9);
|
||||
|
||||
// hybrid without denseDistance falls back to rank (rank1 → 1.0)
|
||||
double hybridRank = RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.HYBRID, 99.0, 1, 4, 2.0, null);
|
||||
assertEquals(1.0, hybridRank, 1e-9);
|
||||
|
||||
// hybrid with dense L2 uses absolute quality (gate / relevance)
|
||||
double hybridDenseFar = RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.HYBRID, 99.0, 1, 4, 2.0, 1.4);
|
||||
assertEquals(0.3, hybridDenseFar, 1e-9);
|
||||
assertTrue(hybridDenseFar < 0.5);
|
||||
|
||||
double hybridRank2 = RetrievalScoreNormalizer.toQualityScore(
|
||||
"rrf_fused", 0.01, 2, 4, 2.0, null);
|
||||
assertTrue(hybridRank2 < hybridRank);
|
||||
}
|
||||
|
||||
@Test
|
||||
void canonicalizeAliases() {
|
||||
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
|
||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
|
||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -42,7 +42,7 @@ class VectorKnowledgeSearchAdapterHybridTest {
|
||||
result.setContent(content);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("l2_distance");
|
||||
result.setScoreLabel("dense");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ class LookupKnowledgeToolTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void rerankUsesHintMatchesAndContextPackPreservesMetadata() {
|
||||
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
|
||||
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
||||
VectorSearchService.SearchResult first = searchResult(
|
||||
"a",
|
||||
@@ -202,9 +202,14 @@ class LookupKnowledgeToolTest {
|
||||
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
||||
|
||||
assertTrue(lookup.isFound());
|
||||
assertEquals("b.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
assertTrue(lookup.getRerankTrace().getItems().get(0).getBoostReasons().stream()
|
||||
.anyMatch(reason -> reason.startsWith("domain_match")));
|
||||
// originalRank order wins; keyword/domain boost must not promote second over first
|
||||
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
|
||||
assertTrue(lookup.getRerankTrace().getItems().stream()
|
||||
.flatMap(item -> item.getBoostReasons() == null
|
||||
? java.util.stream.Stream.empty()
|
||||
: item.getBoostReasons().stream())
|
||||
.noneMatch(reason -> reason.contains(":+0.")));
|
||||
assertTrue(lookup.getContextPack().getPackedText().contains("Payment > Timeout"));
|
||||
assertTrue(lookup.getContextPack().getPackedText().contains("reasons:"));
|
||||
}
|
||||
@@ -258,7 +263,7 @@ class LookupKnowledgeToolTest {
|
||||
result.setContent(content);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("l2_distance");
|
||||
result.setScoreLabel("dense");
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user