feat(rag): add hybrid knowledge rebuild API and script
Add confirm-gated rebuild-hybrid endpoint that drops biz_hybrid, clears api_document and L0, then force-imports knowledge_base markdown into the dense+BM25 store. Include PowerShell runner and ops README.
This commit is contained in:
@@ -52,6 +52,42 @@ public class KnowledgeBaseInitService {
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into biz_hybrid.
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
|
||||
RebuildResult rebuild = new RebuildResult();
|
||||
rebuild.setBasePath(knowledgeBasePath);
|
||||
rebuild.setCollection(hybridKnowledgeStore.collectionName());
|
||||
|
||||
long mysqlBefore = apiDocumentRepository.count();
|
||||
rebuild.setMysqlDocumentsBefore(mysqlBefore);
|
||||
|
||||
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
|
||||
rebuild.setMilvus(milvus);
|
||||
|
||||
apiDocumentRepository.deleteAll();
|
||||
apiDocumentRepository.flush();
|
||||
knowledgeIndexService.clearIndex();
|
||||
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
|
||||
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
|
||||
|
||||
InitResult init = initializeKnowledgeBase(true);
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
*
|
||||
@@ -131,16 +167,16 @@ public class KnowledgeBaseInitService {
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
||||
|
||||
// 上传到 Milvus
|
||||
// 上传到 Milvus hybrid collection(dense + BM25 search_text)
|
||||
try {
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
|
||||
|
||||
document.setStatus("INDEXED");
|
||||
document.setChunkCount(chunks.size());
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})",
|
||||
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
|
||||
title, document.getDocId(), chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
||||
@@ -161,6 +197,7 @@ public class KnowledgeBaseInitService {
|
||||
.keywords(keywords)
|
||||
.summary(summary)
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.build();
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
|
||||
@@ -344,4 +381,21 @@ public class KnowledgeBaseInitService {
|
||||
private long totalVectors; // Milvus 中的向量总数
|
||||
private Map<String, Long> categoryCount; // 按分类统计
|
||||
}
|
||||
|
||||
/**
|
||||
* Full hybrid rebuild result.
|
||||
*/
|
||||
@Data
|
||||
public static class RebuildResult {
|
||||
private boolean success;
|
||||
private String basePath;
|
||||
private String collection;
|
||||
private long mysqlDocumentsBefore;
|
||||
private long mysqlDocumentsAfterClear;
|
||||
private long mysqlDocumentsAfterInit;
|
||||
private int l0IndexSizeAfterClear;
|
||||
private int l0IndexSizeAfterInit;
|
||||
private Map<String, Object> milvus;
|
||||
private InitResult init;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user