329 lines
15 KiB
Java
329 lines
15 KiB
Java
package com.superbiz.agent.tool;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
import com.superbiz.agent.dto.LookupResult;
|
|
import com.superbiz.agent.service.KnowledgeContextPacker;
|
|
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
|
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
|
import com.superbiz.agent.service.KnowledgeIndexService;
|
|
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
|
import com.superbiz.agent.service.LookupResultAssembler;
|
|
import com.superbiz.agent.service.ToolInvocationRecorder;
|
|
import com.superbiz.agent.service.VectorSearchService;
|
|
import com.superbiz.agent.util.SessionContextHolder;
|
|
import org.junit.jupiter.api.BeforeEach;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.mockito.Mock;
|
|
import org.mockito.MockitoAnnotations;
|
|
import org.springframework.test.util.ReflectionTestUtils;
|
|
|
|
import java.util.Collections;
|
|
import java.util.List;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
import static org.mockito.Mockito.never;
|
|
import static org.mockito.Mockito.verify;
|
|
import static org.mockito.Mockito.when;
|
|
|
|
/**
|
|
* LookupKnowledgeTool evidence-first contract tests.
|
|
*/
|
|
class LookupKnowledgeToolTest {
|
|
|
|
@Mock
|
|
private KnowledgeIndexService knowledgeIndexService;
|
|
|
|
@Mock
|
|
private VectorSearchService vectorSearchService;
|
|
|
|
@Mock
|
|
private ToolInvocationRecorder toolInvocationRecorder;
|
|
|
|
@Mock
|
|
private RetrievedDocTracker retrievedDocTracker;
|
|
|
|
private LookupKnowledgeTool tool;
|
|
|
|
@BeforeEach
|
|
void setUp() {
|
|
MockitoAnnotations.openMocks(this);
|
|
|
|
KnowledgeEvidencePostProcessor postProcessor = new KnowledgeEvidencePostProcessor();
|
|
KnowledgeContextPacker contextPacker = new KnowledgeContextPacker();
|
|
tool = new LookupKnowledgeTool();
|
|
ReflectionTestUtils.setField(tool, "queryTransformer", new KnowledgeQueryTransformer(knowledgeIndexService));
|
|
ReflectionTestUtils.setField(tool, "documentRetriever",
|
|
new KnowledgeDocumentRetriever(vectorSearchService, new ObjectMapper()));
|
|
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
|
|
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
|
|
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
|
|
ReflectionTestUtils.setField(tool, "toolInvocationRecorder", toolInvocationRecorder);
|
|
ReflectionTestUtils.setField(tool, "retrievedDocTracker", retrievedDocTracker);
|
|
ReflectionTestUtils.setField(tool, "topK", 3);
|
|
}
|
|
|
|
@Test
|
|
void filteredL1SuccessDoesNotRetry() {
|
|
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
|
VectorSearchService.SearchResult result = searchResult(
|
|
"vec-1",
|
|
"db.md",
|
|
"{\"_source\":\"db.md\",\"title\":\"Database Doc\",\"category\":\"database\"}",
|
|
"mysql timeout runbook",
|
|
0.2f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
.thenReturn(List.of(result));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals(1, lookup.getEvidenceBlockCount());
|
|
assertEquals("db.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
assertNotNull(lookup.getContextPack());
|
|
assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook"));
|
|
assertEquals("FILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
assertEquals(1, lookup.getRetrievalTrace().getAttempts().size());
|
|
assertEquals("PRECISE", lookup.getRelevanceLevel());
|
|
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
|
verify(vectorSearchService, never()).searchSimilarDocuments("mysql timeout", 3, null);
|
|
}
|
|
|
|
@Test
|
|
void filteredLowQualityTriggersRawUnfilteredRetry() {
|
|
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
|
VectorSearchService.SearchResult weak = searchResult(
|
|
"weak",
|
|
"weak.md",
|
|
"{\"_source\":\"weak.md\",\"title\":\"Weak\"}",
|
|
"weak candidate",
|
|
1.4f);
|
|
VectorSearchService.SearchResult strong = searchResult(
|
|
"strong",
|
|
"strong.md",
|
|
"{\"_source\":\"strong.md\",\"title\":\"Strong\"}",
|
|
"mysql timeout strong runbook",
|
|
0.2f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
.thenReturn(List.of(weak));
|
|
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
|
.thenReturn(List.of(strong));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
assertEquals("filtered_vector_low_quality", lookup.getRetrievalTrace().getFallbackReason());
|
|
assertEquals(2, lookup.getRetrievalTrace().getAttempts().size());
|
|
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
|
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, null);
|
|
}
|
|
|
|
@Test
|
|
void filteredNoEvidenceTriggersRawUnfilteredRetry() {
|
|
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
|
VectorSearchService.SearchResult strong = searchResult(
|
|
"strong",
|
|
"strong.md",
|
|
"{\"_source\":\"strong.md\",\"title\":\"Strong\"}",
|
|
"mysql timeout strong runbook",
|
|
0.2f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
.thenReturn(Collections.emptyList());
|
|
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
|
.thenReturn(List.of(strong));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
assertEquals("filtered_vector_no_evidence", lookup.getRetrievalTrace().getFallbackReason());
|
|
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
}
|
|
|
|
@Test
|
|
void l0HintsDoNotBecomeStandaloneEvidenceWhenL1Fails() {
|
|
KnowledgeEntry entry = entry("fallback.md", "Fallback Doc", "fallback", "database");
|
|
|
|
when(knowledgeIndexService.analyzeQuery("fallback")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("fallback", 3, "database"))
|
|
.thenReturn(Collections.emptyList());
|
|
when(vectorSearchService.searchSimilarDocuments("fallback", 3, null))
|
|
.thenReturn(Collections.emptyList());
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("fallback");
|
|
|
|
assertFalse(lookup.isFound());
|
|
assertEquals(0, lookup.getEvidenceBlockCount());
|
|
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
|
assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus());
|
|
assertTrue(String.valueOf(lookup.getRetrievalTrace().getQueryHints()).contains("Fallback Doc"));
|
|
}
|
|
|
|
@Test
|
|
void noL0HintUsesUnfilteredVectorSearch() {
|
|
VectorSearchService.SearchResult result = searchResult(
|
|
"vec-1",
|
|
"perf.md",
|
|
"{\"_source\":\"perf.md\",\"title\":\"Perf\"}",
|
|
"performance tuning guide",
|
|
0.3f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("性能优化"))
|
|
.thenReturn(KnowledgeIndexService.L0Hint.empty());
|
|
when(vectorSearchService.searchSimilarDocuments("性能优化", 3, null))
|
|
.thenReturn(List.of(result));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("性能优化");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
assertEquals("perf.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
verify(vectorSearchService).searchSimilarDocuments("性能优化", 3, null);
|
|
}
|
|
|
|
@Test
|
|
void rerankUsesHintMatchesAndContextPackPreservesMetadata() {
|
|
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
|
VectorSearchService.SearchResult first = searchResult(
|
|
"a",
|
|
"a.md",
|
|
"{\"_source\":\"a.md\",\"title\":\"Generic\",\"category\":\"other\"}",
|
|
"generic troubleshooting",
|
|
0.4f);
|
|
VectorSearchService.SearchResult second = searchResult(
|
|
"b",
|
|
"b.md",
|
|
"{\"_source\":\"b.md\",\"title\":\"Payment ERR_TIMEOUT\",\"breadcrumb\":\"Payment > Timeout\",\"category\":\"payment\"}",
|
|
"payment ERR_TIMEOUT timeout diagnosis",
|
|
0.45f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
|
|
.thenReturn(List.of(first, second));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals("b.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
assertTrue(lookup.getRerankTrace().getItems().get(0).getBoostReasons().stream()
|
|
.anyMatch(reason -> reason.startsWith("domain_match")));
|
|
assertTrue(lookup.getContextPack().getPackedText().contains("Payment > Timeout"));
|
|
assertTrue(lookup.getContextPack().getPackedText().contains("reasons:"));
|
|
}
|
|
|
|
@Test
|
|
void deduplicatesEvidenceBlocksBySource() {
|
|
KnowledgeEntry entry = entry("shared.md", "Shared", "shared", "payment");
|
|
VectorSearchService.SearchResult first = searchResult(
|
|
"a",
|
|
"shared.md",
|
|
"{\"_source\":\"shared.md\",\"title\":\"Shared\"}",
|
|
"shared content 1",
|
|
0.2f);
|
|
VectorSearchService.SearchResult second = searchResult(
|
|
"b",
|
|
"shared.md",
|
|
"{\"_source\":\"shared.md\",\"title\":\"Shared\"}",
|
|
"shared content 2",
|
|
0.25f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("shared")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("shared", 3, "payment"))
|
|
.thenReturn(List.of(first, second));
|
|
|
|
LookupResult lookup = tool.lookupKnowledge("shared");
|
|
|
|
assertTrue(lookup.isFound());
|
|
assertEquals(2, lookup.getEvidenceCandidateCount());
|
|
assertEquals(1, lookup.getEvidenceBlockCount());
|
|
assertEquals("shared.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
}
|
|
|
|
@Test
|
|
void sessionDedupDoesNotReturnConsumableEvidenceAgain() {
|
|
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
|
VectorSearchService.SearchResult result = searchResult(
|
|
"vec-1",
|
|
"payment.md",
|
|
"{\"_source\":\"payment.md\",\"title\":\"Payment\",\"category\":\"payment\"}",
|
|
"payment timeout runbook",
|
|
0.2f);
|
|
|
|
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
|
|
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
|
|
.thenReturn(List.of(result));
|
|
when(retrievedDocTracker.isAlreadyRetrieved("session-1", "payment.md")).thenReturn(true);
|
|
when(retrievedDocTracker.getRetrievedDomains("session-1")).thenReturn(List.of("payment"));
|
|
|
|
SessionContextHolder.setSessionId("session-1");
|
|
try {
|
|
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
|
|
|
assertFalse(lookup.isFound());
|
|
assertEquals(0, lookup.getEvidenceBlockCount());
|
|
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
|
assertTrue(lookup.getMessage().contains("无需重复召回"));
|
|
assertEquals(List.of("payment"), lookup.getRetrievedDomainsThisSession());
|
|
} finally {
|
|
SessionContextHolder.clear();
|
|
}
|
|
}
|
|
|
|
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) {
|
|
return KnowledgeEntry.builder()
|
|
.filePath(filePath)
|
|
.title(title)
|
|
.keywords(List.of(keyword))
|
|
.summary(title + " summary")
|
|
.category(category)
|
|
.build();
|
|
}
|
|
|
|
private VectorSearchService.SearchResult searchResult(String id,
|
|
String source,
|
|
String metadata,
|
|
String content,
|
|
float score) {
|
|
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
|
result.setId(id);
|
|
result.setMetadata(metadata);
|
|
result.setContent(content);
|
|
result.setScore(score);
|
|
result.setRawScore((double) score);
|
|
result.setScoreLabel("l2_distance");
|
|
return result;
|
|
}
|
|
|
|
private KnowledgeIndexService.L0Hint hint(KnowledgeEntry... entries) {
|
|
List<KnowledgeEntry> matches = List.of(entries);
|
|
List<String> keywords = matches.stream()
|
|
.flatMap(entry -> entry.getKeywords() == null
|
|
? java.util.stream.Stream.empty()
|
|
: entry.getKeywords().stream())
|
|
.distinct()
|
|
.toList();
|
|
List<String> domains = matches.stream()
|
|
.map(KnowledgeEntry::getCategory)
|
|
.filter(category -> category != null && !category.isBlank())
|
|
.distinct()
|
|
.toList();
|
|
List<String> titles = matches.stream()
|
|
.map(KnowledgeEntry::getTitle)
|
|
.filter(title -> title != null && !title.isBlank())
|
|
.distinct()
|
|
.toList();
|
|
return new KnowledgeIndexService.L0Hint(matches, keywords, domains, keywords, titles);
|
|
}
|
|
}
|