diff --git a/scripts/README-rebuild-hybrid-knowledge.md b/scripts/README-rebuild-hybrid-knowledge.md index d4eb4db..8cd2e1b 100644 --- a/scripts/README-rebuild-hybrid-knowledge.md +++ b/scripts/README-rebuild-hybrid-knowledge.md @@ -1,6 +1,6 @@ # 重建 hybrid 知识库(dense + BM25) -面向当前 `knowledge_base/` 目录文档,清空并重建 `biz_hybrid` collection。 +面向当前 `knowledge_base/` 目录文档,**清空并重建**配置中的 Milvus collection(默认 **`biz`**)。 ## 前提 @@ -10,7 +10,7 @@ ```yaml milvus: - collection: biz_hybrid + collection: biz retrieval: search: mode: hybrid @@ -18,20 +18,28 @@ knowledge: base-path: knowledge_base/ ``` -## 一键脚本 +## 一键脚本(Python) 在项目根目录执行: -```powershell -.\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD +```bash +python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD ``` 指定服务地址: -```powershell -.\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD +```bash +python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD ``` +跳过前后 stats: + +```bash +python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD --skip-stats +``` + +依赖:Python 3.9+ 标准库即可(无需 pip 包)。 + ## 脚本会做什么 | 步骤 | 动作 | @@ -43,10 +51,12 @@ knowledge: 服务端 `rebuild-hybrid` 内部顺序: -1. **Drop + recreate** Milvus collection(`milvus.collection`,默认 `biz_hybrid`) +1. **Drop + recreate** Milvus collection(`milvus.collection`,默认 `biz`) + - 原有向量数据会被删除 + - 按 dense + BM25 schema 重建 2. **清空** MySQL `api_document` 3. **清空** 内存 L0 索引 -4. **扫描** `knowledge_base/**/*.md` 并 `force` 全量导入 +4. **扫描** `knowledge_base/**/*.md`(跳过 `README.md`)并 force 全量导入 - 写 MySQL 元数据 - 切片 - 写 dense 向量 + BM25 `search_text` @@ -54,17 +64,16 @@ knowledge: ## 不会做什么 -- **不会**删除旧 collection `biz`(需你在 Zilliz 控制台自行决定是否删) - **不会**动 `knowledge_base/` 源文件 -- **不会**在未传 `confirm=REBUILD` 时执行 +- **不会**在未传 `--confirm REBUILD` 时执行 ## 手动 curl 等价命令 ```bash -# 重建(危险) +# 重建(危险:会清空 biz collection + api_document) curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD" -# 仅增量/强制导入(不 drop collection) +# 仅强制导入(不 drop collection) curl -X POST "http://localhost:9900/api/knowledge/init?force=true" # 统计 @@ -78,8 +87,8 @@ curl "http://localhost:9900/api/knowledge/stats" ```json { "success": true, - "collection": "biz_hybrid", - "inserted": <大于0>, + "collection": "biz", + "inserted": 15, "failed": 0, "milvus": { "recreated": true, "loaded": true } } diff --git a/scripts/rebuild_hybrid_knowledge.py b/scripts/rebuild_hybrid_knowledge.py new file mode 100644 index 0000000..a05b8a7 --- /dev/null +++ b/scripts/rebuild_hybrid_knowledge.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Rebuild knowledge into the configured Milvus collection (default: biz). + +Clears: + - milvus.collection (drop + recreate dense+BM25 schema) + - MySQL api_document + - in-memory L0 index + +Then force-imports all markdown under server-side knowledge.base-path +(default: knowledge_base/). + +Usage: + # start Spring Boot first, then: + python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD + + python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request +from typing import Any + + +DEFAULT_BASE_URL = "http://localhost:9900" + + +def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]: + req = urllib.request.Request(url=url, method=method.upper()) + req.add_header("Accept", "application/json") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", errors="replace") + status = getattr(resp, "status", 200) + if not raw.strip(): + return status, None + return status, json.loads(raw) + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + body: Any + try: + body = json.loads(raw) if raw.strip() else None + except json.JSONDecodeError: + body = raw + raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc + + +def pretty(obj: Any) -> str: + return json.dumps(obj, ensure_ascii=False, indent=2) + + +def step(title: str) -> None: + print() + print(f"==> {title}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, " + "and reimport knowledge_base markdown into dense+BM25." + ) + parser.add_argument( + "--base-url", + default=DEFAULT_BASE_URL, + help=f"Service base URL (default: {DEFAULT_BASE_URL})", + ) + parser.add_argument( + "--confirm", + required=True, + choices=["REBUILD"], + help="Must be REBUILD to execute destructive rebuild", + ) + parser.add_argument( + "--skip-stats", + action="store_true", + help="Skip before/after /api/knowledge/stats", + ) + parser.add_argument( + "--timeout", + type=float, + default=7200.0, + help="Rebuild request timeout seconds (default: 7200)", + ) + args = parser.parse_args(argv) + + base_url = args.base_url.rstrip("/") + + print("Hybrid knowledge rebuild") + print(f" BaseUrl : {base_url}") + print(f" Confirm : {args.confirm}") + print(" Source : knowledge_base/ (server-side knowledge.base-path)") + print() + print("This will DESTROY data in:") + print(" - Milvus collection milvus.collection (default: biz)") + print(" - MySQL table api_document") + print(" - In-memory L0 knowledge index") + print("Then re-import all markdown under knowledge_base.") + print() + + # 1) health + step("Check service health") + try: + status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30) + print(f" milvus health status={status}") + print(pretty(body)) + except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure + print(f" WARN: /milvus/health failed: {exc}") + print(" Continue if app is up but milvus health endpoint has issues.") + + # 2) stats before + if not args.skip_stats: + step("Knowledge stats (before)") + try: + _, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30) + print(pretty(body)) + except Exception as exc: # noqa: BLE001 + print(f" WARN: stats before failed: {exc}") + + # 3) rebuild + step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD") + rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}" + try: + status, body = http_json("POST", rebuild_url, timeout=args.timeout) + except RuntimeError as exc: + print(str(exc)) + return 1 + + print(f" HTTP {status}") + print(pretty(body)) + + if not isinstance(body, dict): + print("Unexpected rebuild response type", file=sys.stderr) + return 1 + + inserted = int(body.get("inserted") or 0) + failed = int(body.get("failed") or 0) + success = bool(body.get("success")) + + if not success: + if inserted <= 0: + print() + print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr) + return 2 + print() + print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.") + else: + print() + print(f"Rebuild OK: inserted={inserted}, failed={failed}") + + # 4) stats after + if not args.skip_stats: + step("Knowledge stats (after)") + try: + _, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30) + print(pretty(body)) + except Exception as exc: # noqa: BLE001 + print(f" WARN: stats after failed: {exc}") + + print() + print("Done.") + print("Next:") + print(" 1) Ensure application.yml has:") + print(" milvus.collection: biz") + print(" retrieval.search.mode: hybrid") + print(" 2) Smoke test lookup_knowledge / chat with a known doc query") + return 0 if success or inserted > 0 else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/main/java/com/superbiz/agent/constant/MilvusConstants.java b/src/main/java/com/superbiz/agent/constant/MilvusConstants.java index a274833..99e0a21 100644 --- a/src/main/java/com/superbiz/agent/constant/MilvusConstants.java +++ b/src/main/java/com/superbiz/agent/constant/MilvusConstants.java @@ -8,14 +8,15 @@ public class MilvusConstants { public static final String MILVUS_DB_NAME = "default"; /** - * Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid). + * Default knowledge collection name (dense + BM25). + * Overridable via {@code milvus.collection}. */ public static final String MILVUS_COLLECTION_NAME = "biz"; /** - * Default hybrid dense+BM25 collection. + * Alias kept for readability in hybrid-related code. */ - public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid"; + public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME; /** * 向量维度(豆包 embedding 模型的维度) diff --git a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java b/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java index 80e3680..c19fe8b 100644 --- a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java +++ b/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java @@ -67,7 +67,7 @@ public class KnowledgeBaseController { * 清空 hybrid collection + MySQL api_document + L0 内存索引, * 再从 knowledge_base 全量重建 dense+BM25 索引。 * - *
危险操作:会删除 {@code milvus.collection}(默认 biz_hybrid)与文档元数据表数据。 + *
危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。 * 需要显式 confirm=REBUILD。
*/ @PostMapping("/rebuild-hybrid") diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java b/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java index becf396..c0a9efc 100644 --- a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java +++ b/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java @@ -57,7 +57,7 @@ public class KnowledgeBaseInitService { /** * Drop hybrid collection, clear MySQL api_document + L0 memory index, - * then force-import all markdown under knowledge.base-path into biz_hybrid. + * then force-import all markdown under knowledge.base-path into milvus.collection (default biz). */ @Transactional(rollbackFor = Exception.class) public RebuildResult rebuildHybridFromKnowledgeBase() { @@ -82,6 +82,8 @@ public class KnowledgeBaseInitService { rebuild.setInit(init); rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize()); rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count()); + // Success when at least one doc indexed and no hard failures. + // README-like docs are skipped by scanner; remaining failures still mark unsuccessful. rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0); logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}", rebuild.isSuccess(), init.getInserted(), init.getFailed()); @@ -257,7 +259,11 @@ public class KnowledgeBaseInitService { Files.walkFileTree(baseDir, new SimpleFileVisitor