feat(rag): add hybrid knowledge rebuild API and script
Add confirm-gated rebuild-hybrid endpoint that drops biz_hybrid, clears api_document and L0, then force-imports knowledge_base markdown into the dense+BM25 store. Include PowerShell runner and ops README.
This commit is contained in:
zhuyongxin
@@ -63,6 +63,59 @@ public class KnowledgeBaseController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引，
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作：会删除 {@code milvus.collection}（默认 biz_hybrid）与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
public ResponseEntity<?> rebuildHybrid(
|
||||
@RequestParam(defaultValue = "") String confirm) {
|
||||
if (!"REBUILD".equals(confirm)) {
|
||||
Map<String, Object> rejected = new HashMap<>();
|
||||
rejected.put("success", false);
|
||||
rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建");
|
||||
rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
|
||||
return ResponseEntity.badRequest().body(rejected);
|
||||
}
|
||||
|
||||
logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
|
||||
try {
|
||||
KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", result.isSuccess());
|
||||
response.put("message", result.isSuccess()
|
||||
? "hybrid 知识库重建完成"
|
||||
: "hybrid 知识库重建结束，但存在失败项");
|
||||
response.put("collection", result.getCollection());
|
||||
response.put("basePath", result.getBasePath());
|
||||
response.put("milvus", result.getMilvus());
|
||||
response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
|
||||
response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
|
||||
response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
|
||||
response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
|
||||
response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
|
||||
if (result.getInit() != null) {
|
||||
response.put("scanned", result.getInit().getScanned());
|
||||
response.put("skipped", result.getInit().getSkipped());
|
||||
response.put("inserted", result.getInit().getInserted());
|
||||
response.put("failed", result.getInit().getFailed());
|
||||
response.put("details", result.getInit().getDetails());
|
||||
}
|
||||
return result.isSuccess()
|
||||
? ResponseEntity.ok(response)
|
||||
: ResponseEntity.status(500).body(response);
|
||||
} catch (Exception e) {
|
||||
logger.error("hybrid 知识库重建失败", e);
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "重建失败: " + e.getMessage());
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询知识库统计信息
|
||||
*
|
||||
|
||||
@@ -52,6 +52,42 @@ public class KnowledgeBaseInitService {
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into biz_hybrid.
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
|
||||
RebuildResult rebuild = new RebuildResult();
|
||||
rebuild.setBasePath(knowledgeBasePath);
|
||||
rebuild.setCollection(hybridKnowledgeStore.collectionName());
|
||||
|
||||
long mysqlBefore = apiDocumentRepository.count();
|
||||
rebuild.setMysqlDocumentsBefore(mysqlBefore);
|
||||
|
||||
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
|
||||
rebuild.setMilvus(milvus);
|
||||
|
||||
apiDocumentRepository.deleteAll();
|
||||
apiDocumentRepository.flush();
|
||||
knowledgeIndexService.clearIndex();
|
||||
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
|
||||
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
|
||||
|
||||
InitResult init = initializeKnowledgeBase(true);
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
*
|
||||
@@ -131,16 +167,16 @@ public class KnowledgeBaseInitService {
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
||||
|
||||
// 上传到 Milvus
|
||||
// 上传到 Milvus hybrid collection（dense + BM25 search_text）
|
||||
try {
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
|
||||
|
||||
document.setStatus("INDEXED");
|
||||
document.setChunkCount(chunks.size());
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})",
|
||||
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
|
||||
title, document.getDocId(), chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
||||
@@ -161,6 +197,7 @@ public class KnowledgeBaseInitService {
|
||||
.keywords(keywords)
|
||||
.summary(summary)
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.build();
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
|
||||
@@ -344,4 +381,21 @@ public class KnowledgeBaseInitService {
|
||||
private long totalVectors; // Milvus 中的向量总数
|
||||
private Map<String, Long> categoryCount; // 按分类统计
|
||||
}
|
||||
|
||||
/**
|
||||
* Full hybrid rebuild result.
|
||||
*/
|
||||
@Data
|
||||
public static class RebuildResult {
|
||||
private boolean success;
|
||||
private String basePath;
|
||||
private String collection;
|
||||
private long mysqlDocumentsBefore;
|
||||
private long mysqlDocumentsAfterClear;
|
||||
private long mysqlDocumentsAfterInit;
|
||||
private int l0IndexSizeAfterClear;
|
||||
private int l0IndexSizeAfterInit;
|
||||
private Map<String, Object> milvus;
|
||||
private InitResult init;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -296,6 +296,12 @@ public class KnowledgeIndexService {
|
||||
log.debug("文档已从 L0 索引移除: {}", filePath);
|
||||
}
|
||||
|
||||
/** Clear in-memory L0 entries (used by knowledge rebuild). */
|
||||
public void clearIndex() {
|
||||
knowledgeIndex.clear();
|
||||
log.info("L0 knowledge index cleared");
|
||||
}
|
||||
|
||||
public int getIndexSize() {
|
||||
return knowledgeIndex.size();
|
||||
}
|
||||
|
||||
@@ -12,8 +12,10 @@ import io.milvus.v2.common.DataType;
|
||||
import io.milvus.v2.common.IndexParam;
|
||||
import io.milvus.v2.service.collection.request.AddFieldReq;
|
||||
import io.milvus.v2.service.collection.request.CreateCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.DropCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.HasCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.LoadCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
|
||||
import io.milvus.v2.service.index.request.CreateIndexReq;
|
||||
import io.milvus.v2.service.vector.request.AnnSearchReq;
|
||||
import io.milvus.v2.service.vector.request.DeleteReq;
|
||||
@@ -141,6 +143,41 @@ public class MilvusHybridKnowledgeStore {
|
||||
.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("collection", collectionName);
|
||||
MilvusClientV2 milvusClient = client();
|
||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
result.put("existedBefore", Boolean.TRUE.equals(exists));
|
||||
if (Boolean.TRUE.equals(exists)) {
|
||||
try {
|
||||
milvusClient.releaseCollection(ReleaseCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
} catch (Exception e) {
|
||||
log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
|
||||
}
|
||||
milvusClient.dropCollection(DropCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
log.info("Dropped hybrid collection '{}'", collectionName);
|
||||
result.put("dropped", true);
|
||||
} else {
|
||||
result.put("dropped", false);
|
||||
}
|
||||
ensureCollection(milvusClient);
|
||||
loadCollection(milvusClient);
|
||||
result.put("recreated", true);
|
||||
result.put("loaded", true);
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
|
||||
List<Float> queryVector,
|
||||
int topK,
|
||||
|
||||
Reference in New Issue
Block a user