chore(rag): rebuild script in Python and use biz collection
Switch default knowledge collection back to biz (drop+recreate on rebuild), replace PowerShell rebuild runner with Python, skip README.md imports, and merge duplicate rag keys in application.yml.
This commit is contained in:
@@ -8,14 +8,15 @@ public class MilvusConstants {
|
||||
public static final String MILVUS_DB_NAME = "default";
|
||||
|
||||
/**
|
||||
* Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid).
|
||||
* Default knowledge collection name (dense + BM25).
|
||||
* Overridable via {@code milvus.collection}.
|
||||
*/
|
||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
||||
|
||||
/**
|
||||
* Default hybrid dense+BM25 collection.
|
||||
* Alias kept for readability in hybrid-related code.
|
||||
*/
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid";
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
||||
|
||||
/**
|
||||
* 向量维度(豆包 embedding 模型的维度)
|
||||
|
||||
@@ -67,7 +67,7 @@ public class KnowledgeBaseController {
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作:会删除 {@code milvus.collection}(默认 biz_hybrid)与文档元数据表数据。
|
||||
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
|
||||
@@ -57,7 +57,7 @@ public class KnowledgeBaseInitService {
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into biz_hybrid.
|
||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
@@ -82,6 +82,8 @@ public class KnowledgeBaseInitService {
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
// Success when at least one doc indexed and no hard failures.
|
||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
@@ -257,7 +259,11 @@ public class KnowledgeBaseInitService {
|
||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
if (file.toString().endsWith(".md")) {
|
||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
||||
// Import content docs only; skip README/index markdown without frontmatter.
|
||||
if (name.endsWith(".md")
|
||||
&& !name.equalsIgnoreCase("README.md")
|
||||
&& !name.equalsIgnoreCase("readme.md")) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
|
||||
@@ -61,8 +61,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
|
||||
private final MilvusProperties milvusProperties;
|
||||
|
||||
@Value("${milvus.collection:biz_hybrid}")
|
||||
private String collectionName = "biz_hybrid";
|
||||
@Value("${milvus.collection:biz}")
|
||||
private String collectionName = "biz";
|
||||
|
||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
||||
private int rrfK = 60;
|
||||
@@ -144,8 +144,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
|
||||
* Drop the configured knowledge collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Existing vectors in this collection are destroyed.
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
|
||||
Reference in New Issue
Block a user