feat(rag): add hybrid knowledge rebuild API and script

Add confirm-gated rebuild-hybrid endpoint that drops biz_hybrid, clears
api_document and L0, then force-imports knowledge_base markdown into the
dense+BM25 store. Include PowerShell runner and ops README.
This commit is contained in:
zhuyongxin
2026-07-27 18:55:00 +08:00
parent f035538531
commit 5c369f3b6c
6 changed files with 407 additions and 3 deletions
@@ -52,6 +52,42 @@ public class KnowledgeBaseInitService {
@Autowired
private KnowledgeIndexService knowledgeIndexService;
@Autowired
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
/**
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
* then force-import all markdown under knowledge.base-path into biz_hybrid.
*/
@Transactional(rollbackFor = Exception.class)
public RebuildResult rebuildHybridFromKnowledgeBase() {
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
RebuildResult rebuild = new RebuildResult();
rebuild.setBasePath(knowledgeBasePath);
rebuild.setCollection(hybridKnowledgeStore.collectionName());
long mysqlBefore = apiDocumentRepository.count();
rebuild.setMysqlDocumentsBefore(mysqlBefore);
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
rebuild.setMilvus(milvus);
apiDocumentRepository.deleteAll();
apiDocumentRepository.flush();
knowledgeIndexService.clearIndex();
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
InitResult init = initializeKnowledgeBase(true);
rebuild.setInit(init);
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
rebuild.isSuccess(), init.getInserted(), init.getFailed());
return rebuild;
}
/**
* 初始化知识库
*
@@ -131,16 +167,16 @@ public class KnowledgeBaseInitService {
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
// 上传到 Milvus
// 上传到 Milvus hybrid collection(dense + BM25 search_text)
try {
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
document.setStatus("INDEXED");
document.setChunkCount(chunks.size());
document.setIndexedAt(LocalDateTime.now());
apiDocumentRepository.save(document);
logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})",
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
title, document.getDocId(), chunks.size());
} catch (Exception e) {
logger.error("上传到 Milvus 失败: {}", relativePath, e);
@@ -161,6 +197,7 @@ public class KnowledgeBaseInitService {
.keywords(keywords)
.summary(summary)
.category(category)
.kbScope(frontmatter.getKbScope())
.build();
knowledgeIndexService.addToIndex(entry);
@@ -344,4 +381,21 @@ public class KnowledgeBaseInitService {
private long totalVectors; // Milvus 中的向量总数
private Map<String, Long> categoryCount; // 按分类统计
}
/**
* Full hybrid rebuild result.
*/
@Data
public static class RebuildResult {
private boolean success;
private String basePath;
private String collection;
private long mysqlDocumentsBefore;
private long mysqlDocumentsAfterClear;
private long mysqlDocumentsAfterInit;
private int l0IndexSizeAfterClear;
private int l0IndexSizeAfterInit;
private Map<String, Object> milvus;
private InitResult init;
}
}