feat(rag): add hybrid knowledge rebuild API and script

Add confirm-gated rebuild-hybrid endpoint that drops biz_hybrid, clears api_document and L0, then force-imports knowledge_base markdown into the dense+BM25 store. Include PowerShell runner and ops README.
2026-07-27 18:55:00 +08:00
parent f035538531
commit 5c369f3b6c
6 changed files with 407 additions and 3 deletions
@@ -0,0 +1,97 @@
+# 重建 hybrid 知识库（dense + BM25）
+
+面向当前 `knowledge_base/` 目录文档，清空并重建 `biz_hybrid` collection。
+
+## 前提
+
+1. 应用已启动（默认 `http://localhost:9900`）
+2. `MILVUS_TOKEN` 等连接配置可用
+3. `application.yml` 已配置：
+
+```yaml
+milvus:
+  collection: biz_hybrid
+retrieval:
+  search:
+    mode: hybrid
+knowledge:
+  base-path: knowledge_base/
+```
+
+## 一键脚本
+
+在项目根目录执行：
+
+```powershell
+.\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD
+```
+
+指定服务地址：
+
+```powershell
+.\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD
+```
+
+## 脚本会做什么
+
+| 步骤 | 动作 |
+|---|---|
+| 1 | 检查 `/milvus/health` |
+| 2 | 打印重建前 `/api/knowledge/stats` |
+| 3 | `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD` |
+| 4 | 打印重建后 stats |
+
+服务端 `rebuild-hybrid` 内部顺序：
+
+1. **Drop + recreate** Milvus collection（`milvus.collection`，默认 `biz_hybrid`）
+2. **清空** MySQL `api_document`
+3. **清空** 内存 L0 索引
+4. **扫描** `knowledge_base/**/*.md` 并 `force` 全量导入  
+   - 写 MySQL 元数据  
+   - 切片  
+   - 写 dense 向量 + BM25 `search_text`  
+   - 更新 L0
+
+## 不会做什么
+
+- **不会**删除旧 collection `biz`（需你在 Zilliz 控制台自行决定是否删）
+- **不会**动 `knowledge_base/` 源文件
+- **不会**在未传 `confirm=REBUILD` 时执行
+
+## 手动 curl 等价命令
+
+```bash
+# 重建（危险）
+curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD"
+
+# 仅增量/强制导入（不 drop collection）
+curl -X POST "http://localhost:9900/api/knowledge/init?force=true"
+
+# 统计
+curl "http://localhost:9900/api/knowledge/stats"
+```
+
+## 成功判据
+
+响应中大致应有：
+
+```json
+{
+  "success": true,
+  "collection": "biz_hybrid",
+  "inserted": <大于0>,
+  "failed": 0,
+  "milvus": { "recreated": true, "loaded": true }
+}
+```
+
+然后用一条知识库里真实存在的术语/故障词走 `lookup_knowledge` 或 chat 验证 hybrid 命中。
+
+## 失败排查
+
+| 现象 | 可能原因 |
+|---|---|
+| connect / token 错误 | `MILVUS_TOKEN`、host、database |
+| BM25 / analyzer 相关报错 | 云端 Milvus/Zilliz 版本不支持 BM25 Function |
+| inserted=0 | `knowledge_base` 路径不对，或 md 缺 frontmatter/title |
+| failed>0 | 看响应 `details` 与应用日志 |
@@ -0,0 +1,157 @@
+<#
+.SYNOPSIS
+  清空 hybrid 知识库（Milvus biz_hybrid + MySQL api_document + L0），并从 knowledge_base 全量重建。
+
+.DESCRIPTION
+  对应 dense+BM25 混合检索上线后的数据迁移步骤：
+
+    1) 检查服务健康
+    2) 可选：查看当前 /api/knowledge/stats
+    3) POST /api/knowledge/rebuild-hybrid?confirm=REBUILD
+       - drop + recreate milvus.collection（默认 biz_hybrid）
+       - 清空 MySQL api_document
+       - 清空内存 L0 索引
+       - 扫描 knowledge_base/**/*.md 强制导入并写入 dense+BM25
+
+  不会删除旧的 legacy collection `biz`（知识主路径已不再使用它）。
+
+.PARAMETER BaseUrl
+  服务根地址，默认 http://localhost:9900
+
+.PARAMETER Confirm
+  必须为 REBUILD 才会真正执行（防止误触）
+
+.PARAMETER SkipStats
+  跳过重建前后的 stats 查询
+
+.EXAMPLE
+  # 先启动 Spring Boot，再执行：
+  .\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD
+
+.EXAMPLE
+  .\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD
+#>
+
+[CmdletBinding()]
+param(
+    [string]$BaseUrl = "http://localhost:9900",
+    [ValidateSet("REBUILD")]
+    [Parameter(Mandatory = $true)]
+    [string]$Confirm,
+    [switch]$SkipStats
+)
+
+$ErrorActionPreference = "Stop"
+
+function Write-Step([string]$Message) {
+    Write-Host ""
+    Write-Host "==> $Message" -ForegroundColor Cyan
+}
+
+function Invoke-Json {
+    param(
+        [Parameter(Mandatory = $true)][string]$Method,
+        [Parameter(Mandatory = $true)][string]$Url,
+        [int]$TimeoutSec = 3600
+    )
+    try {
+        $resp = Invoke-WebRequest -Method $Method -Uri $Url -TimeoutSec $TimeoutSec
+        $body = $resp.Content
+        if ([string]::IsNullOrWhiteSpace($body)) {
+            return @{ statusCode = [int]$resp.StatusCode; body = $null }
+        }
+        return @{
+            statusCode = [int]$resp.StatusCode
+            body       = $body | ConvertFrom-Json
+        }
+    }
+    catch {
+        $status = $null
+        $raw = $null
+        if ($_.Exception.Response) {
+            $status = [int]$_.Exception.Response.StatusCode
+            try {
+                $stream = $_.Exception.Response.GetResponseStream()
+                $reader = New-Object System.IO.StreamReader($stream)
+                $raw = $reader.ReadToEnd()
+            }
+            catch { }
+        }
+        throw "HTTP $Method $Url failed (status=$status): $($_.Exception.Message)`n$raw"
+    }
+}
+
+$BaseUrl = $BaseUrl.TrimEnd("/")
+
+Write-Host "Hybrid knowledge rebuild" -ForegroundColor Green
+Write-Host "  BaseUrl : $BaseUrl"
+Write-Host "  Confirm : $Confirm"
+Write-Host "  Source  : knowledge_base/ (server-side knowledge.base-path)"
+Write-Host ""
+Write-Host "This will DESTROY data in:" -ForegroundColor Yellow
+Write-Host "  - Milvus collection milvus.collection (default: biz_hybrid)"
+Write-Host "  - MySQL table api_document"
+Write-Host "  - In-memory L0 knowledge index"
+Write-Host "Then re-import all markdown under knowledge_base."
+Write-Host ""
+
+# 1) health
+Write-Step "Check service health"
+try {
+    $health = Invoke-Json -Method GET -Url "$BaseUrl/milvus/health" -TimeoutSec 30
+    Write-Host ("  milvus health status={0}" -f $health.statusCode)
+    if ($health.body) {
+        $health.body | ConvertTo-Json -Depth 6 | Write-Host
+    }
+}
+catch {
+    Write-Host "  WARN: /milvus/health failed: $_" -ForegroundColor Yellow
+    Write-Host "  Continue if app is up but milvus health endpoint has issues."
+}
+
+# 2) stats before
+if (-not $SkipStats) {
+    Write-Step "Knowledge stats (before)"
+    try {
+        $before = Invoke-Json -Method GET -Url "$BaseUrl/api/knowledge/stats" -TimeoutSec 30
+        $before.body | ConvertTo-Json -Depth 6 | Write-Host
+    }
+    catch {
+        Write-Host "  WARN: stats before failed: $_" -ForegroundColor Yellow
+    }
+}
+
+# 3) rebuild
+Write-Step "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD"
+$rebuildUrl = "$BaseUrl/api/knowledge/rebuild-hybrid?confirm=$Confirm"
+$rebuild = Invoke-Json -Method POST -Url $rebuildUrl -TimeoutSec 7200
+Write-Host ("  HTTP {0}" -f $rebuild.statusCode)
+$rebuild.body | ConvertTo-Json -Depth 12 | Write-Host
+
+if (-not $rebuild.body.success) {
+    Write-Host ""
+    Write-Host "Rebuild reported failure. Inspect details above." -ForegroundColor Red
+    exit 2
+}
+
+# 4) stats after
+if (-not $SkipStats) {
+    Write-Step "Knowledge stats (after)"
+    try {
+        $after = Invoke-Json -Method GET -Url "$BaseUrl/api/knowledge/stats" -TimeoutSec 30
+        $after.body | ConvertTo-Json -Depth 6 | Write-Host
+    }
+    catch {
+        Write-Host "  WARN: stats after failed: $_" -ForegroundColor Yellow
+    }
+}
+
+Write-Host ""
+Write-Host "Done." -ForegroundColor Green
+Write-Host "Next:"
+Write-Host "  1) Ensure application.yml has:"
+Write-Host "       milvus.collection: biz_hybrid"
+Write-Host "       retrieval.search.mode: hybrid"
+Write-Host "  2) Smoke test lookup_knowledge / chat with a known doc query"
+Write-Host "  3) Optional: drop legacy collection 'biz' manually in Zilliz console if no longer needed"
+exit 0
@@ -63,6 +63,59 @@ public class KnowledgeBaseController {
        }
    }

+    /**
+     * 清空 hybrid collection + MySQL api_document + L0 内存索引，
+     * 再从 knowledge_base 全量重建 dense+BM25 索引。
+     *
+     * <p>危险操作：会删除 {@code milvus.collection}（默认 biz_hybrid）与文档元数据表数据。
+     * 需要显式 confirm=REBUILD。</p>
+     */
+    @PostMapping("/rebuild-hybrid")
+    public ResponseEntity<?> rebuildHybrid(
+            @RequestParam(defaultValue = "") String confirm) {
+        if (!"REBUILD".equals(confirm)) {
+            Map<String, Object> rejected = new HashMap<>();
+            rejected.put("success", false);
+            rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建");
+            rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
+            return ResponseEntity.badRequest().body(rejected);
+        }
+
+        logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
+        try {
+            KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
+            Map<String, Object> response = new HashMap<>();
+            response.put("success", result.isSuccess());
+            response.put("message", result.isSuccess()
+                    ? "hybrid 知识库重建完成"
+                    : "hybrid 知识库重建结束，但存在失败项");
+            response.put("collection", result.getCollection());
+            response.put("basePath", result.getBasePath());
+            response.put("milvus", result.getMilvus());
+            response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
+            response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
+            response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
+            response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
+            response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
+            if (result.getInit() != null) {
+                response.put("scanned", result.getInit().getScanned());
+                response.put("skipped", result.getInit().getSkipped());
+                response.put("inserted", result.getInit().getInserted());
+                response.put("failed", result.getInit().getFailed());
+                response.put("details", result.getInit().getDetails());
+            }
+            return result.isSuccess()
+                    ? ResponseEntity.ok(response)
+                    : ResponseEntity.status(500).body(response);
+        } catch (Exception e) {
+            logger.error("hybrid 知识库重建失败", e);
+            Map<String, Object> response = new HashMap<>();
+            response.put("success", false);
+            response.put("message", "重建失败: " + e.getMessage());
+            return ResponseEntity.internalServerError().body(response);
+        }
+    }
+
    /**
     * 查询知识库统计信息
     *
@@ -52,6 +52,42 @@ public class KnowledgeBaseInitService {
    @Autowired
    private KnowledgeIndexService knowledgeIndexService;

+    @Autowired
+    private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
+
+    /**
+     * Drop hybrid collection, clear MySQL api_document + L0 memory index,
+     * then force-import all markdown under knowledge.base-path into biz_hybrid.
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public RebuildResult rebuildHybridFromKnowledgeBase() {
+        logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
+        RebuildResult rebuild = new RebuildResult();
+        rebuild.setBasePath(knowledgeBasePath);
+        rebuild.setCollection(hybridKnowledgeStore.collectionName());
+
+        long mysqlBefore = apiDocumentRepository.count();
+        rebuild.setMysqlDocumentsBefore(mysqlBefore);
+
+        Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
+        rebuild.setMilvus(milvus);
+
+        apiDocumentRepository.deleteAll();
+        apiDocumentRepository.flush();
+        knowledgeIndexService.clearIndex();
+        rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
+        rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
+
+        InitResult init = initializeKnowledgeBase(true);
+        rebuild.setInit(init);
+        rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
+        rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
+        rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
+        logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
+                rebuild.isSuccess(), init.getInserted(), init.getFailed());
+        return rebuild;
+    }
+
    /**
     * 初始化知识库
     *
@@ -131,16 +167,16 @@ public class KnowledgeBaseInitService {
                List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
                logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());

-                // 上传到 Milvus
+                // 上传到 Milvus hybrid collection（dense + BM25 search_text）
                try {
-                    vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
+                    vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);

                    document.setStatus("INDEXED");
                    document.setChunkCount(chunks.size());
                    document.setIndexedAt(LocalDateTime.now());
                    apiDocumentRepository.save(document);

-                    logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})",
+                    logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
                            title, document.getDocId(), chunks.size());
                } catch (Exception e) {
                    logger.error("上传到 Milvus 失败: {}", relativePath, e);
@@ -161,6 +197,7 @@ public class KnowledgeBaseInitService {
                        .keywords(keywords)
                        .summary(summary)
                        .category(category)
+                        .kbScope(frontmatter.getKbScope())
                        .build();
                knowledgeIndexService.addToIndex(entry);

@@ -344,4 +381,21 @@ public class KnowledgeBaseInitService {
        private long totalVectors;                // Milvus 中的向量总数
        private Map<String, Long> categoryCount;  // 按分类统计
    }
+
+    /**
+     * Full hybrid rebuild result.
+     */
+    @Data
+    public static class RebuildResult {
+        private boolean success;
+        private String basePath;
+        private String collection;
+        private long mysqlDocumentsBefore;
+        private long mysqlDocumentsAfterClear;
+        private long mysqlDocumentsAfterInit;
+        private int l0IndexSizeAfterClear;
+        private int l0IndexSizeAfterInit;
+        private Map<String, Object> milvus;
+        private InitResult init;
+    }
 }
@@ -296,6 +296,12 @@ public class KnowledgeIndexService {
        log.debug("文档已从 L0 索引移除: {}", filePath);
    }

+    /** Clear in-memory L0 entries (used by knowledge rebuild). */
+    public void clearIndex() {
+        knowledgeIndex.clear();
+        log.info("L0 knowledge index cleared");
+    }
+
    public int getIndexSize() {
        return knowledgeIndex.size();
    }
@@ -12,8 +12,10 @@ import io.milvus.v2.common.DataType;
 import io.milvus.v2.common.IndexParam;
 import io.milvus.v2.service.collection.request.AddFieldReq;
 import io.milvus.v2.service.collection.request.CreateCollectionReq;
+import io.milvus.v2.service.collection.request.DropCollectionReq;
 import io.milvus.v2.service.collection.request.HasCollectionReq;
 import io.milvus.v2.service.collection.request.LoadCollectionReq;
+import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
 import io.milvus.v2.service.index.request.CreateIndexReq;
 import io.milvus.v2.service.vector.request.AnnSearchReq;
 import io.milvus.v2.service.vector.request.DeleteReq;
@@ -141,6 +143,41 @@ public class MilvusHybridKnowledgeStore {
                .build());
    }

+    /**
+     * Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
+     * Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
+     */
+    public synchronized Map<String, Object> dropAndRecreateCollection() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("collection", collectionName);
+        MilvusClientV2 milvusClient = client();
+        Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
+                .collectionName(collectionName)
+                .build());
+        result.put("existedBefore", Boolean.TRUE.equals(exists));
+        if (Boolean.TRUE.equals(exists)) {
+            try {
+                milvusClient.releaseCollection(ReleaseCollectionReq.builder()
+                        .collectionName(collectionName)
+                        .build());
+            } catch (Exception e) {
+                log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
+            }
+            milvusClient.dropCollection(DropCollectionReq.builder()
+                    .collectionName(collectionName)
+                    .build());
+            log.info("Dropped hybrid collection '{}'", collectionName);
+            result.put("dropped", true);
+        } else {
+            result.put("dropped", false);
+        }
+        ensureCollection(milvusClient);
+        loadCollection(milvusClient);
+        result.put("recreated", true);
+        result.put("loaded", true);
+        return result;
+    }
+
    public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
                                                              List<Float> queryVector,
                                                              int topK,