osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): close eval pipeline with live snapshots

Browse Source
This commit is contained in:
zhuyongxin
2026-07-06 21:39:27 +08:00
parent cf3333d607
commit ed7efc58b7
47 changed files with 2613 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/RagEvalSeedImporterTest.java
+93
View File
@@ -0,0 +1,93 @@
package com.superbiz.agent.eval;
import com.superbiz.agent.Main;
import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.dto.DocumentUploadRequest;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.service.DocumentManagementService;
import com.superbiz.agent.service.FrontmatterParser;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.mock.web.MockMultipartFile;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
/**
* Imports canonical RAG eval documents through the real document pipeline.
*
* <p>Disabled by default because it writes DB rows, local knowledge files, and
* vector index records in the configured runtime environment.</p>
*/
@SpringBootTest(
classes = Main.class,
webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = "spring.main.web-application-type=none"
)
@EnabledIfSystemProperty(named = "rag.seed.enabled", matches = "true")
class RagEvalSeedImporterTest {
private static final Path DEFAULT_SEED_DOCS = Path.of("eval/rag-retrieval/seed-docs");
@Autowired
private DocumentManagementService documentManagementService;
@Autowired
private FrontmatterParser frontmatterParser;
@Autowired
private ApiDocumentRepository apiDocumentRepository;
@Test
void importSeedDocuments() throws Exception {
Path seedDir = Path.of(System.getProperty("rag.seed.docs", DEFAULT_SEED_DOCS.toString()));
List<Path> docs;
try (var stream = Files.list(seedDir)) {
docs = stream
.filter(path -> path.getFileName().toString().endsWith(".md"))
.sorted()
.toList();
}
assertFalse(docs.isEmpty(), "seed docs directory must contain markdown files");
for (Path docPath : docs) {
String content = Files.readString(docPath, StandardCharsets.UTF_8);
Frontmatter frontmatter = frontmatterParser.parse(content);
if (frontmatter == null || frontmatter.getSource() == null || frontmatter.getSource().isBlank()) {
throw new IllegalArgumentException("seed doc must include frontmatter source: " + docPath);
}
apiDocumentRepository.findByDocId(frontmatter.getSource().trim())
.map(ApiDocument::getDocId)
.ifPresent(documentManagementService::deleteDocument);
String fileName = docPath.getFileName().toString();
MockMultipartFile file = new MockMultipartFile(
"file",
fileName,
"text/markdown",
content.getBytes(StandardCharsets.UTF_8)
);
DocumentUploadRequest request = DocumentUploadRequest.builder()
.file(file)
.category(resolveCategory(frontmatter))
.build();
documentManagementService.uploadDocument(request);
}
}
private String resolveCategory(Frontmatter frontmatter) {
if (frontmatter.getCategory() != null && !frontmatter.getCategory().isBlank()) {
return frontmatter.getCategory().trim();
}
return "rag-eval";
}
}
src/test/java/com/superbiz/agent/eval/RagLookupSnapshotGeneratorTest.java
+85
View File
@@ -0,0 +1,85 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.superbiz.agent.Main;
import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.util.SessionContextHolder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Generates RAG retrieval fixtures from the real LookupKnowledgeTool bean.
*
* <p>This class is disabled by default because it writes repository files and
* depends on the configured runtime retrieval stack.</p>
*/
@SpringBootTest(
classes = Main.class,
webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = "spring.main.web-application-type=none"
)
@EnabledIfSystemProperty(named = "rag.snapshot.enabled", matches = "true")
class RagLookupSnapshotGeneratorTest {
private static final Path DEFAULT_CASES = Path.of("eval/rag-retrieval/cases/golden-cases.json");
private static final Path DEFAULT_FIXTURES = Path.of("eval/rag-retrieval/fixtures");
@Autowired
private LookupKnowledgeTool lookupKnowledgeTool;
@Autowired
private ObjectMapper objectMapper;
@Test
void generateLookupResultFixtures() throws Exception {
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
JsonNode root = objectMapper.readTree(casesPath.toFile());
JsonNode cases = root.path("cases");
assertTrue(cases.isArray(), "golden cases file must contain a cases array");
Files.createDirectories(fixturesDir);
for (JsonNode testCase : cases) {
String caseId = requiredText(testCase, "caseId");
String query = requiredText(testCase, "query");
SessionContextHolder.clear();
LookupResult lookupResult;
try {
lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
} finally {
SessionContextHolder.clear();
}
ObjectNode fixture = objectMapper.createObjectNode();
fixture.put("caseId", caseId);
fixture.put("query", query);
fixture.put("retrievedAt", retrievedAt);
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
Path output = fixturesDir.resolve(caseId + ".json");
objectMapper.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), fixture);
}
}
private String requiredText(JsonNode node, String fieldName) {
JsonNode value = node.get(fieldName);
if (value == null || value.asText().isBlank()) {
throw new IllegalArgumentException("golden case is missing required field: " + fieldName);
}
return value.asText();
}
}
src/test/java/com/superbiz/agent/service/DocumentManagementServiceTest.java
+13
View File
@@ -1,5 +1,6 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.mock.web.MockMultipartFile;
@@ -38,4 +39,16 @@ class DocumentManagementServiceTest {
assertEquals("payment/runbook.md", storedPath);
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
}
@Test
void resolveDocumentIdUsesFrontmatterSourceWhenItFitsDatabaseColumn() {
DocumentManagementService service = new DocumentManagementService();
Frontmatter frontmatter = Frontmatter.builder()
.source("mysql-connection-pool")
.build();
String docId = ReflectionTestUtils.invokeMethod(service, "resolveDocumentId", frontmatter);
assertEquals("mysql-connection-pool", docId);
}
}
src/test/java/com/superbiz/agent/service/FrontmatterParserTest.java
+44
View File
@@ -146,4 +146,48 @@ class FrontmatterParserTest {
assertEquals("1.0.0", result.getVersion());
assertEquals("Test Author", result.getAuthor());
}
@Test
void testParse_withRetrievalMetadata() {
String content = """
---
title: MySQL Connection Pool
keywords: [connection pool, HikariCP]
summary: Diagnose exhausted MySQL connection pools
category: database
source: mysql-connection-pool
breadcrumb: Database > MySQL > Connection Pool
kb_scope: rag-eval
---
Content
""";
Frontmatter result = parser.parse(content);
assertNotNull(result);
assertEquals("mysql-connection-pool", result.getSource());
assertEquals("Database > MySQL > Connection Pool", result.getBreadcrumb());
assertEquals("rag-eval", result.getKbScope());
}
@Test
void testStripFrontmatter_returnsMarkdownBodyOnly() {
String content = """
---
title: Test
keywords: [frontmatter-only]
summary: Summary
---
# Body
Body content
""";
String body = parser.stripFrontmatter(content);
assertFalse(body.contains("frontmatter-only"));
assertTrue(body.startsWith("# Body"));
assertTrue(body.contains("Body content"));
}
}
src/test/java/com/superbiz/agent/service/KnowledgeIndexServiceTest.java
+43
View File
@@ -138,6 +138,49 @@ class KnowledgeIndexServiceTest {
assertNull(hint.singleDomainOrNull());
}
@Test
void testAnalyzeQuery_filtersByConfiguredKbScope() {
ReflectionTestUtils.setField(service, "kbScope", "rag-eval");
service.addToIndex(KnowledgeEntry.builder()
.filePath("legacy.md")
.keywords(List.of("timeout"))
.category("legacy")
.build());
service.addToIndex(KnowledgeEntry.builder()
.filePath("eval.md")
.keywords(List.of("timeout"))
.category("eval")
.kbScope("rag-eval")
.build());
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
assertEquals(1, hint.matches().size());
assertEquals(List.of("eval"), hint.domains());
assertEquals("eval", hint.singleDomainOrNull());
}
@Test
void testAnalyzeQuery_keepsLegacyEntriesWhenNoScopeConfigured() {
ReflectionTestUtils.setField(service, "kbScope", "");
service.addToIndex(KnowledgeEntry.builder()
.filePath("legacy.md")
.keywords(List.of("timeout"))
.category("legacy")
.build());
service.addToIndex(KnowledgeEntry.builder()
.filePath("eval.md")
.keywords(List.of("timeout"))
.category("eval")
.kbScope("rag-eval")
.build());
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
assertEquals(2, hint.matches().size());
assertNull(hint.singleDomainOrNull());
}
@Test
void testExactMatch_noMatch() {
KnowledgeEntry entry = KnowledgeEntry.builder()
src/test/java/com/superbiz/agent/service/VectorIndexServiceTest.java
+34
View File
@@ -1,8 +1,11 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.DocumentChunk;
import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
class VectorIndexServiceTest {
@@ -34,4 +37,35 @@ class VectorIndexServiceTest {
assertEquals("Plain chunk content.", VectorIndexService.buildEmbeddingText(chunk));
}
@Test
void buildDocumentMetadataUsesFrontmatterRetrievalFields() {
DocumentChunk chunk = DocumentChunk.builder()
.chunkIndex(0)
.title("Chunk Title")
.breadcrumb("Chunk > Path")
.content("content")
.build();
Frontmatter frontmatter = Frontmatter.builder()
.title("Document Title")
.source("mysql-connection-pool")
.breadcrumb("Database > MySQL > Connection Pool")
.kbScope("rag-eval")
.build();
Map<String, Object> metadata = VectorIndexService.buildDocumentMetadata(
"mysql-connection-pool",
chunk,
2,
"database",
frontmatter
);
assertEquals("mysql-connection-pool", metadata.get("docId"));
assertEquals("mysql-connection-pool", metadata.get("_source"));
assertEquals("mysql-connection-pool", metadata.get("source"));
assertEquals("database", metadata.get("category"));
assertEquals("rag-eval", metadata.get("kb_scope"));
assertEquals("Database > MySQL > Connection Pool", metadata.get("breadcrumb"));
}
}
src/test/java/com/superbiz/agent/service/VectorSearchServiceTest.java
+44
View File
@@ -13,6 +13,7 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
@@ -151,6 +152,45 @@ class VectorSearchServiceTest {
assertTrue(request.toString().contains("api"));
}
@Test
void springModeUsesVectorStoreAlias() {
VectorStore vectorStore = mock(VectorStore.class);
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(vectorStore);
when(vectorStore.similaritySearch(any(SearchRequest.class))).thenReturn(List.of());
VectorSearchService service = new VectorSearchService();
setMode(service, "spring");
setVectorStore(service, provider);
service.searchSimilarDocuments("query", 5, "api");
verify(vectorStore).similaritySearch(any(SearchRequest.class));
}
@Test
void defaultScopeDoesNotAddMetadataFilter() {
VectorSearchService service = new VectorSearchService();
setKbScope(service, "");
assertNull(service.buildSpringAiFilterExpression(null));
assertNull(service.buildSdkFilterExpression(null));
assertEquals("category == 'api'", service.buildSpringAiFilterExpression("api"));
assertEquals("metadata[\"category\"] == \"api\"", service.buildSdkFilterExpression("api"));
}
@Test
void configuredScopeCombinesWithCategoryFilter() {
VectorSearchService service = new VectorSearchService();
setKbScope(service, "rag-eval");
assertEquals("kb_scope == 'rag-eval'", service.buildSpringAiFilterExpression(null));
assertEquals("category == 'api' && kb_scope == 'rag-eval'",
service.buildSpringAiFilterExpression("api"));
assertEquals("metadata[\"category\"] == \"api\" && metadata[\"kb_scope\"] == \"rag-eval\"",
service.buildSdkFilterExpression("api"));
}
private static void setMode(VectorSearchService service, String mode) {
ReflectionTestUtils.setField(service, "vectorStoreMode", mode);
}
@@ -161,6 +201,10 @@ class VectorSearchServiceTest {
ReflectionTestUtils.setField(service, "maxL2Distance", 2.0);
}
private static void setKbScope(VectorSearchService service, String kbScope) {
ReflectionTestUtils.setField(service, "kbScope", kbScope);
}
private static VectorSearchService.SearchResult result(String id, float score) {
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
result.setId(id);