chore(rag): rebuild script in Python and use biz collection

Switch default knowledge collection back to biz (drop+recreate on rebuild),
replace PowerShell rebuild runner with Python, skip README.md imports, and
merge duplicate rag keys in application.yml.
This commit is contained in:
zhuyongxin
2026-07-28 09:49:43 +08:00
parent 38f781b157
commit 1e532fb851
8 changed files with 225 additions and 36 deletions
@@ -8,14 +8,15 @@ public class MilvusConstants {
public static final String MILVUS_DB_NAME = "default";
/**
* Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid).
* Default knowledge collection name (dense + BM25).
* Overridable via {@code milvus.collection}.
*/
public static final String MILVUS_COLLECTION_NAME = "biz";
/**
* Default hybrid dense+BM25 collection.
* Alias kept for readability in hybrid-related code.
*/
public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid";
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
/**
* 向量维度(豆包 embedding 模型的维度)
@@ -67,7 +67,7 @@ public class KnowledgeBaseController {
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
* 再从 knowledge_base 全量重建 dense+BM25 索引。
*
* <p>危险操作:会删除 {@code milvus.collection}(默认 biz_hybrid)与文档元数据表数据。
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
* 需要显式 confirm=REBUILD。</p>
*/
@PostMapping("/rebuild-hybrid")
@@ -57,7 +57,7 @@ public class KnowledgeBaseInitService {
/**
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
* then force-import all markdown under knowledge.base-path into biz_hybrid.
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
*/
@Transactional(rollbackFor = Exception.class)
public RebuildResult rebuildHybridFromKnowledgeBase() {
@@ -82,6 +82,8 @@ public class KnowledgeBaseInitService {
rebuild.setInit(init);
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
// Success when at least one doc indexed and no hard failures.
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
rebuild.isSuccess(), init.getInserted(), init.getFailed());
@@ -257,7 +259,11 @@ public class KnowledgeBaseInitService {
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (file.toString().endsWith(".md")) {
String name = file.getFileName() == null ? "" : file.getFileName().toString();
// Import content docs only; skip README/index markdown without frontmatter.
if (name.endsWith(".md")
&& !name.equalsIgnoreCase("README.md")
&& !name.equalsIgnoreCase("readme.md")) {
files.add(file);
}
return FileVisitResult.CONTINUE;
@@ -61,8 +61,8 @@ public class MilvusHybridKnowledgeStore {
private final MilvusProperties milvusProperties;
@Value("${milvus.collection:biz_hybrid}")
private String collectionName = "biz_hybrid";
@Value("${milvus.collection:biz}")
private String collectionName = "biz";
@Value("${retrieval.hybrid.rrf-k:60}")
private int rrfK = 60;
@@ -144,8 +144,8 @@ public class MilvusHybridKnowledgeStore {
}
/**
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
* Drop the configured knowledge collection (if present) and recreate empty dense+BM25 schema.
* Used by knowledge rebuild scripts. Existing vectors in this collection are destroyed.
*/
public synchronized Map<String, Object> dropAndRecreateCollection() {
Map<String, Object> result = new LinkedHashMap<>();
+6 -9
View File
@@ -25,8 +25,8 @@ milvus:
token: ${MILVUS_TOKEN}
secure: true
vector-dim: 1024 # BGE-M3 = 1024,换模型时同步改
# dense+BM25 hybrid collection (requires reindex; legacy "biz" is not used by knowledge path)
collection: biz_hybrid
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
collection: biz
# =====================================================
# 模型路由配置
@@ -158,7 +158,10 @@ document:
# RAG 配置
rag:
top-k: 3 # 检索返回的最相似文档数量
top-k: 3 # legacy fallback when retrieve-k/return-n absent
retrieve-k: 20
return-n: 5
max-chunks-per-document: 2
sidecar:
spring-ai:
enabled: false
@@ -176,12 +179,6 @@ retrieval:
highly-relevant-threshold: 0.75 # similarity >= 0.75 → HIGHLY_RELEVANT
reference-threshold: 0.5 # similarity >= 0.5 → REFERENCE
# RAG 召回宽度(Delivery 1)
rag:
retrieve-k: 20
return-n: 5
max-chunks-per-document: 2
# Prometheus 配置
prometheus:
base-url: http://localhost:9090
@@ -23,7 +23,7 @@ class VectorSearchServiceTest {
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
.thenReturn(List.of(expected));
when(store.collectionName()).thenReturn("biz_hybrid");
when(store.collectionName()).thenReturn("biz");
VectorSearchService service = new VectorSearchService();
ReflectionTestUtils.setField(service, "knowledgeStore", store);
@@ -44,7 +44,7 @@ class VectorSearchServiceTest {
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
.thenReturn(List.of(expected));
when(store.collectionName()).thenReturn("biz_hybrid");
when(store.collectionName()).thenReturn("biz");
VectorSearchService service = new VectorSearchService();
ReflectionTestUtils.setField(service, "knowledgeStore", store);