From 5c369f3b6cff63678cdb2c9958dab6e50e86e533 Mon Sep 17 00:00:00 2001 From: zhuyongxin Date: Mon, 27 Jul 2026 18:55:00 +0800 Subject: [PATCH] feat(rag): add hybrid knowledge rebuild API and script Add confirm-gated rebuild-hybrid endpoint that drops biz_hybrid, clears api_document and L0, then force-imports knowledge_base markdown into the dense+BM25 store. Include PowerShell runner and ops README. --- scripts/README-rebuild-hybrid-knowledge.md | 97 +++++++++++ scripts/rebuild-hybrid-knowledge.ps1 | 157 ++++++++++++++++++ .../controller/KnowledgeBaseController.java | 53 ++++++ .../service/KnowledgeBaseInitService.java | 60 ++++++- .../agent/service/KnowledgeIndexService.java | 6 + .../milvus/MilvusHybridKnowledgeStore.java | 37 +++++ 6 files changed, 407 insertions(+), 3 deletions(-) create mode 100644 scripts/README-rebuild-hybrid-knowledge.md create mode 100644 scripts/rebuild-hybrid-knowledge.ps1 diff --git a/scripts/README-rebuild-hybrid-knowledge.md b/scripts/README-rebuild-hybrid-knowledge.md new file mode 100644 index 0000000..d4eb4db --- /dev/null +++ b/scripts/README-rebuild-hybrid-knowledge.md @@ -0,0 +1,97 @@ +# 重建 hybrid 知识库（dense + BM25） + +面向当前 `knowledge_base/` 目录文档，清空并重建 `biz_hybrid` collection。 + +## 前提 + +1. 应用已启动（默认 `http://localhost:9900`） +2. `MILVUS_TOKEN` 等连接配置可用 +3. `application.yml` 已配置： + +```yaml +milvus: + collection: biz_hybrid +retrieval: + search: + mode: hybrid +knowledge: + base-path: knowledge_base/ +``` + +## 一键脚本 + +在项目根目录执行： + +```powershell +.\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD +``` + +指定服务地址： + +```powershell +.\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD +``` + +## 脚本会做什么 + +| 步骤 | 动作 | +|---|---| +| 1 | 检查 `/milvus/health` | +| 2 | 打印重建前 `/api/knowledge/stats` | +| 3 | `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD` | +| 4 | 打印重建后 stats | + +服务端 `rebuild-hybrid` 内部顺序： + +1. **Drop + recreate** Milvus collection（`milvus.collection`，默认 `biz_hybrid`） +2. **清空** MySQL `api_document` +3. **清空** 内存 L0 索引 +4. **扫描** `knowledge_base/**/*.md` 并 `force` 全量导入 + - 写 MySQL 元数据 + - 切片 + - 写 dense 向量 + BM25 `search_text` + - 更新 L0 + +## 不会做什么 + +- **不会**删除旧 collection `biz`（需你在 Zilliz 控制台自行决定是否删） +- **不会**动 `knowledge_base/` 源文件 +- **不会**在未传 `confirm=REBUILD` 时执行 + +## 手动 curl 等价命令 + +```bash +# 重建（危险） +curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD" + +# 仅增量/强制导入（不 drop collection） +curl -X POST "http://localhost:9900/api/knowledge/init?force=true" + +# 统计 +curl "http://localhost:9900/api/knowledge/stats" +``` + +## 成功判据 + +响应中大致应有： + +```json +{ + "success": true, + "collection": "biz_hybrid", + "inserted": <大于0>, + "failed": 0, + "milvus": { "recreated": true, "loaded": true } +} +``` + +然后用一条知识库里真实存在的术语/故障词走 `lookup_knowledge` 或 chat 验证 hybrid 命中。 + +## 失败排查 + +| 现象 | 可能原因 | +|---|---| +| connect / token 错误 | `MILVUS_TOKEN`、host、database | +| BM25 / analyzer 相关报错 | 云端 Milvus/Zilliz 版本不支持 BM25 Function | +| inserted=0 | `knowledge_base` 路径不对，或 md 缺 frontmatter/title | +| failed>0 | 看响应 `details` 与应用日志 | diff --git a/scripts/rebuild-hybrid-knowledge.ps1 b/scripts/rebuild-hybrid-knowledge.ps1 new file mode 100644 index 0000000..d359888 --- /dev/null +++ b/scripts/rebuild-hybrid-knowledge.ps1 @@ -0,0 +1,157 @@ +<# +.SYNOPSIS + 清空 hybrid 知识库（Milvus biz_hybrid + MySQL api_document + L0），并从 knowledge_base 全量重建。 + +.DESCRIPTION + 对应 dense+BM25 混合检索上线后的数据迁移步骤： + + 1) 检查服务健康 + 2) 可选：查看当前 /api/knowledge/stats + 3) POST /api/knowledge/rebuild-hybrid?confirm=REBUILD + - drop + recreate milvus.collection（默认 biz_hybrid） + - 清空 MySQL api_document + - 清空内存 L0 索引 + - 扫描 knowledge_base/**/*.md 强制导入并写入 dense+BM25 + + 不会删除旧的 legacy collection `biz`（知识主路径已不再使用它）。 + +.PARAMETER BaseUrl + 服务根地址，默认 http://localhost:9900 + +.PARAMETER Confirm + 必须为 REBUILD 才会真正执行（防止误触） + +.PARAMETER SkipStats + 跳过重建前后的 stats 查询 + +.EXAMPLE + # 先启动 Spring Boot，再执行： + .\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD + +.EXAMPLE + .\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD +#> + +[CmdletBinding()] +param( + [string]$BaseUrl = "http://localhost:9900", + [ValidateSet("REBUILD")] + [Parameter(Mandatory = $true)] + [string]$Confirm, + [switch]$SkipStats +) + +$ErrorActionPreference = "Stop" + +function Write-Step([string]$Message) { + Write-Host "" + Write-Host "==> $Message" -ForegroundColor Cyan +} + +function Invoke-Json { + param( + [Parameter(Mandatory = $true)][string]$Method, + [Parameter(Mandatory = $true)][string]$Url, + [int]$TimeoutSec = 3600 + ) + try { + $resp = Invoke-WebRequest -Method $Method -Uri $Url -TimeoutSec $TimeoutSec + $body = $resp.Content + if ([string]::IsNullOrWhiteSpace($body)) { + return @{ statusCode = [int]$resp.StatusCode; body = $null } + } + return @{ + statusCode = [int]$resp.StatusCode + body = $body | ConvertFrom-Json + } + } + catch { + $status = $null + $raw = $null + if ($_.Exception.Response) { + $status = [int]$_.Exception.Response.StatusCode + try { + $stream = $_.Exception.Response.GetResponseStream() + $reader = New-Object System.IO.StreamReader($stream) + $raw = $reader.ReadToEnd() + } + catch { } + } + throw "HTTP $Method $Url failed (status=$status): $($_.Exception.Message)`n$raw" + } +} + +$BaseUrl = $BaseUrl.TrimEnd("/") + +Write-Host "Hybrid knowledge rebuild" -ForegroundColor Green +Write-Host " BaseUrl : $BaseUrl" +Write-Host " Confirm : $Confirm" +Write-Host " Source : knowledge_base/ (server-side knowledge.base-path)" +Write-Host "" +Write-Host "This will DESTROY data in:" -ForegroundColor Yellow +Write-Host " - Milvus collection milvus.collection (default: biz_hybrid)" +Write-Host " - MySQL table api_document" +Write-Host " - In-memory L0 knowledge index" +Write-Host "Then re-import all markdown under knowledge_base." +Write-Host "" + +# 1) health +Write-Step "Check service health" +try { + $health = Invoke-Json -Method GET -Url "$BaseUrl/milvus/health" -TimeoutSec 30 + Write-Host (" milvus health status={0}" -f $health.statusCode) + if ($health.body) { + $health.body | ConvertTo-Json -Depth 6 | Write-Host + } +} +catch { + Write-Host " WARN: /milvus/health failed: $_" -ForegroundColor Yellow + Write-Host " Continue if app is up but milvus health endpoint has issues." +} + +# 2) stats before +if (-not $SkipStats) { + Write-Step "Knowledge stats (before)" + try { + $before = Invoke-Json -Method GET -Url "$BaseUrl/api/knowledge/stats" -TimeoutSec 30 + $before.body | ConvertTo-Json -Depth 6 | Write-Host + } + catch { + Write-Host " WARN: stats before failed: $_" -ForegroundColor Yellow + } +} + +# 3) rebuild +Write-Step "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD" +$rebuildUrl = "$BaseUrl/api/knowledge/rebuild-hybrid?confirm=$Confirm" +$rebuild = Invoke-Json -Method POST -Url $rebuildUrl -TimeoutSec 7200 +Write-Host (" HTTP {0}" -f $rebuild.statusCode) +$rebuild.body | ConvertTo-Json -Depth 12 | Write-Host + +if (-not $rebuild.body.success) { + Write-Host "" + Write-Host "Rebuild reported failure. Inspect details above." -ForegroundColor Red + exit 2 +} + +# 4) stats after +if (-not $SkipStats) { + Write-Step "Knowledge stats (after)" + try { + $after = Invoke-Json -Method GET -Url "$BaseUrl/api/knowledge/stats" -TimeoutSec 30 + $after.body | ConvertTo-Json -Depth 6 | Write-Host + } + catch { + Write-Host " WARN: stats after failed: $_" -ForegroundColor Yellow + } +} + +Write-Host "" +Write-Host "Done." -ForegroundColor Green +Write-Host "Next:" +Write-Host " 1) Ensure application.yml has:" +Write-Host " milvus.collection: biz_hybrid" +Write-Host " retrieval.search.mode: hybrid" +Write-Host " 2) Smoke test lookup_knowledge / chat with a known doc query" +Write-Host " 3) Optional: drop legacy collection 'biz' manually in Zilliz console if no longer needed" +exit 0 diff --git a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java b/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java index 23a61e2..80e3680 100644 --- a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java +++ b/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java @@ -63,6 +63,59 @@ public class KnowledgeBaseController { } } + /** + * 清空 hybrid collection + MySQL api_document + L0 内存索引， + * 再从 knowledge_base 全量重建 dense+BM25 索引。 + * + *

危险操作：会删除 {@code milvus.collection}（默认 biz_hybrid）与文档元数据表数据。 + * 需要显式 confirm=REBUILD。

+ */ + @PostMapping("/rebuild-hybrid") + public ResponseEntity rebuildHybrid( + @RequestParam(defaultValue = "") String confirm) { + if (!"REBUILD".equals(confirm)) { + Map rejected = new HashMap<>(); + rejected.put("success", false); + rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建"); + rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD"); + return ResponseEntity.badRequest().body(rejected); + } + + logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm); + try { + KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase(); + Map response = new HashMap<>(); + response.put("success", result.isSuccess()); + response.put("message", result.isSuccess() + ? "hybrid 知识库重建完成" + : "hybrid 知识库重建结束，但存在失败项"); + response.put("collection", result.getCollection()); + response.put("basePath", result.getBasePath()); + response.put("milvus", result.getMilvus()); + response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore()); + response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear()); + response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit()); + response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear()); + response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit()); + if (result.getInit() != null) { + response.put("scanned", result.getInit().getScanned()); + response.put("skipped", result.getInit().getSkipped()); + response.put("inserted", result.getInit().getInserted()); + response.put("failed", result.getInit().getFailed()); + response.put("details", result.getInit().getDetails()); + } + return result.isSuccess() + ? ResponseEntity.ok(response) + : ResponseEntity.status(500).body(response); + } catch (Exception e) { + logger.error("hybrid 知识库重建失败", e); + Map response = new HashMap<>(); + response.put("success", false); + response.put("message", "重建失败: " + e.getMessage()); + return ResponseEntity.internalServerError().body(response); + } + } + /** * 查询知识库统计信息 * diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java b/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java index bc76c11..becf396 100644 --- a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java +++ b/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java @@ -52,6 +52,42 @@ public class KnowledgeBaseInitService { @Autowired private KnowledgeIndexService knowledgeIndexService; + @Autowired + private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore; + + /** + * Drop hybrid collection, clear MySQL api_document + L0 memory index, + * then force-import all markdown under knowledge.base-path into biz_hybrid. + */ + @Transactional(rollbackFor = Exception.class) + public RebuildResult rebuildHybridFromKnowledgeBase() { + logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath); + RebuildResult rebuild = new RebuildResult(); + rebuild.setBasePath(knowledgeBasePath); + rebuild.setCollection(hybridKnowledgeStore.collectionName()); + + long mysqlBefore = apiDocumentRepository.count(); + rebuild.setMysqlDocumentsBefore(mysqlBefore); + + Map milvus = hybridKnowledgeStore.dropAndRecreateCollection(); + rebuild.setMilvus(milvus); + + apiDocumentRepository.deleteAll(); + apiDocumentRepository.flush(); + knowledgeIndexService.clearIndex(); + rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count()); + rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize()); + + InitResult init = initializeKnowledgeBase(true); + rebuild.setInit(init); + rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize()); + rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count()); + rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0); + logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}", + rebuild.isSuccess(), init.getInserted(), init.getFailed()); + return rebuild; + } + /** * 初始化知识库 * @@ -131,16 +167,16 @@ public class KnowledgeBaseInitService { List chunks = documentChunkService.chunkDocument(body, relativePath); logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size()); - // 上传到 Milvus + // 上传到 Milvus hybrid collection（dense + BM25 search_text） try { - vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category); + vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter); document.setStatus("INDEXED"); document.setChunkCount(chunks.size()); document.setIndexedAt(LocalDateTime.now()); apiDocumentRepository.save(document); - logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})", + logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})", title, document.getDocId(), chunks.size()); } catch (Exception e) { logger.error("上传到 Milvus 失败: {}", relativePath, e); @@ -161,6 +197,7 @@ public class KnowledgeBaseInitService { .keywords(keywords) .summary(summary) .category(category) + .kbScope(frontmatter.getKbScope()) .build(); knowledgeIndexService.addToIndex(entry); @@ -344,4 +381,21 @@ public class KnowledgeBaseInitService { private long totalVectors; // Milvus 中的向量总数 private Map categoryCount; // 按分类统计 } + + /** + * Full hybrid rebuild result. + */ + @Data + public static class RebuildResult { + private boolean success; + private String basePath; + private String collection; + private long mysqlDocumentsBefore; + private long mysqlDocumentsAfterClear; + private long mysqlDocumentsAfterInit; + private int l0IndexSizeAfterClear; + private int l0IndexSizeAfterInit; + private Map milvus; + private InitResult init; + } } diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java b/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java index fdb47dc..509a086 100644 --- a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java +++ b/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java @@ -296,6 +296,12 @@ public class KnowledgeIndexService { log.debug("文档已从 L0 索引移除: {}", filePath); } + /** Clear in-memory L0 entries (used by knowledge rebuild). */ + public void clearIndex() { + knowledgeIndex.clear(); + log.info("L0 knowledge index cleared"); + } + public int getIndexSize() { return knowledgeIndex.size(); } diff --git a/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java b/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java index 1622157..c119b35 100644 --- a/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java +++ b/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java @@ -12,8 +12,10 @@ import io.milvus.v2.common.DataType; import io.milvus.v2.common.IndexParam; import io.milvus.v2.service.collection.request.AddFieldReq; import io.milvus.v2.service.collection.request.CreateCollectionReq; +import io.milvus.v2.service.collection.request.DropCollectionReq; import io.milvus.v2.service.collection.request.HasCollectionReq; import io.milvus.v2.service.collection.request.LoadCollectionReq; +import io.milvus.v2.service.collection.request.ReleaseCollectionReq; import io.milvus.v2.service.index.request.CreateIndexReq; import io.milvus.v2.service.vector.request.AnnSearchReq; import io.milvus.v2.service.vector.request.DeleteReq; @@ -141,6 +143,41 @@ public class MilvusHybridKnowledgeStore { .build()); } + /** + * Drop the hybrid collection (if present) and recreate empty dense+BM25 schema. + * Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection. + */ + public synchronized Map dropAndRecreateCollection() { + Map result = new LinkedHashMap<>(); + result.put("collection", collectionName); + MilvusClientV2 milvusClient = client(); + Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder() + .collectionName(collectionName) + .build()); + result.put("existedBefore", Boolean.TRUE.equals(exists)); + if (Boolean.TRUE.equals(exists)) { + try { + milvusClient.releaseCollection(ReleaseCollectionReq.builder() + .collectionName(collectionName) + .build()); + } catch (Exception e) { + log.warn("Release collection before drop failed (continuing): {}", e.getMessage()); + } + milvusClient.dropCollection(DropCollectionReq.builder() + .collectionName(collectionName) + .build()); + log.info("Dropped hybrid collection '{}'", collectionName); + result.put("dropped", true); + } else { + result.put("dropped", false); + } + ensureCollection(milvusClient); + loadCollection(milvusClient); + result.put("recreated", true); + result.put("loaded", true); + return result; + } + public List searchDense(String queryEmbeddingText, List queryVector, int topK,