chore(rag): rebuild script in Python and use biz collection

Switch default knowledge collection back to biz (drop+recreate on rebuild),
replace PowerShell rebuild runner with Python, skip README.md imports, and
merge duplicate rag keys in application.yml.
This commit is contained in:
zhuyongxin
2026-07-28 09:49:43 +08:00
parent 38f781b157
commit 1e532fb851
8 changed files with 225 additions and 36 deletions
@@ -8,14 +8,15 @@ public class MilvusConstants {
public static final String MILVUS_DB_NAME = "default";
/**
* Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid).
* Default knowledge collection name (dense + BM25).
* Overridable via {@code milvus.collection}.
*/
public static final String MILVUS_COLLECTION_NAME = "biz";
/**
* Default hybrid dense+BM25 collection.
* Alias kept for readability in hybrid-related code.
*/
public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid";
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
/**
* 向量维度(豆包 embedding 模型的维度)
@@ -67,7 +67,7 @@ public class KnowledgeBaseController {
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
* 再从 knowledge_base 全量重建 dense+BM25 索引。
*
* <p>危险操作:会删除 {@code milvus.collection}(默认 biz_hybrid)与文档元数据表数据。
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
* 需要显式 confirm=REBUILD。</p>
*/
@PostMapping("/rebuild-hybrid")
@@ -57,7 +57,7 @@ public class KnowledgeBaseInitService {
/**
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
* then force-import all markdown under knowledge.base-path into biz_hybrid.
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
*/
@Transactional(rollbackFor = Exception.class)
public RebuildResult rebuildHybridFromKnowledgeBase() {
@@ -82,6 +82,8 @@ public class KnowledgeBaseInitService {
rebuild.setInit(init);
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
// Success when at least one doc indexed and no hard failures.
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
rebuild.isSuccess(), init.getInserted(), init.getFailed());
@@ -257,7 +259,11 @@ public class KnowledgeBaseInitService {
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (file.toString().endsWith(".md")) {
String name = file.getFileName() == null ? "" : file.getFileName().toString();
// Import content docs only; skip README/index markdown without frontmatter.
if (name.endsWith(".md")
&& !name.equalsIgnoreCase("README.md")
&& !name.equalsIgnoreCase("readme.md")) {
files.add(file);
}
return FileVisitResult.CONTINUE;
@@ -61,8 +61,8 @@ public class MilvusHybridKnowledgeStore {
private final MilvusProperties milvusProperties;
@Value("${milvus.collection:biz_hybrid}")
private String collectionName = "biz_hybrid";
@Value("${milvus.collection:biz}")
private String collectionName = "biz";
@Value("${retrieval.hybrid.rrf-k:60}")
private int rrfK = 60;
@@ -144,8 +144,8 @@ public class MilvusHybridKnowledgeStore {
}
/**
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
* Drop the configured knowledge collection (if present) and recreate empty dense+BM25 schema.
* Used by knowledge rebuild scripts. Existing vectors in this collection are destroyed.
*/
public synchronized Map<String, Object> dropAndRecreateCollection() {
Map<String, Object> result = new LinkedHashMap<>();