chore(rag): rebuild script in Python and use biz collection
Switch default knowledge collection back to biz (drop+recreate on rebuild), replace PowerShell rebuild runner with Python, skip README.md imports, and merge duplicate rag keys in application.yml.
This commit is contained in:
zhuyongxin
@@ -1,6 +1,6 @@
|
||||
# 重建 hybrid 知识库（dense + BM25）
|
||||
|
||||
面向当前 `knowledge_base/` 目录文档，清空并重建 `biz_hybrid` collection。
|
||||
面向当前 `knowledge_base/` 目录文档，**清空并重建**配置中的 Milvus collection（默认 **`biz`**）。
|
||||
|
||||
## 前提
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
```yaml
|
||||
milvus:
|
||||
collection: biz_hybrid
|
||||
collection: biz
|
||||
retrieval:
|
||||
search:
|
||||
mode: hybrid
|
||||
@@ -18,20 +18,28 @@ knowledge:
|
||||
base-path: knowledge_base/
|
||||
```
|
||||
|
||||
## 一键脚本
|
||||
## 一键脚本（Python）
|
||||
|
||||
在项目根目录执行：
|
||||
|
||||
```powershell
|
||||
.\scripts\rebuild-hybrid-knowledge.ps1 -Confirm REBUILD
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
```
|
||||
|
||||
指定服务地址：
|
||||
|
||||
```powershell
|
||||
.\scripts\rebuild-hybrid-knowledge.ps1 -BaseUrl http://127.0.0.1:9900 -Confirm REBUILD
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
||||
```
|
||||
|
||||
跳过前后 stats：
|
||||
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD --skip-stats
|
||||
```
|
||||
|
||||
依赖：Python 3.9+ 标准库即可（无需 pip 包）。
|
||||
|
||||
## 脚本会做什么
|
||||
|
||||
| 步骤 | 动作 |
|
||||
@@ -43,10 +51,12 @@ knowledge:
|
||||
|
||||
服务端 `rebuild-hybrid` 内部顺序：
|
||||
|
||||
1. **Drop + recreate** Milvus collection（`milvus.collection`，默认 `biz_hybrid`）
|
||||
1. **Drop + recreate** Milvus collection（`milvus.collection`，默认 `biz`）
|
||||
- 原有向量数据会被删除
|
||||
- 按 dense + BM25 schema 重建
|
||||
2. **清空** MySQL `api_document`
|
||||
3. **清空** 内存 L0 索引
|
||||
4. **扫描** `knowledge_base/**/*.md` 并 `force` 全量导入
|
||||
4. **扫描** `knowledge_base/**/*.md`（跳过 `README.md`）并 force 全量导入
|
||||
- 写 MySQL 元数据
|
||||
- 切片
|
||||
- 写 dense 向量 + BM25 `search_text`
|
||||
@@ -54,17 +64,16 @@ knowledge:
|
||||
|
||||
## 不会做什么
|
||||
|
||||
- **不会**删除旧 collection `biz`（需你在 Zilliz 控制台自行决定是否删）
|
||||
- **不会**动 `knowledge_base/` 源文件
|
||||
- **不会**在未传 `confirm=REBUILD` 时执行
|
||||
- **不会**在未传 `--confirm REBUILD` 时执行
|
||||
|
||||
## 手动 curl 等价命令
|
||||
|
||||
```bash
|
||||
# 重建（危险）
|
||||
# 重建（危险：会清空 biz collection + api_document）
|
||||
curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD"
|
||||
|
||||
# 仅增量/强制导入（不 drop collection）
|
||||
# 仅强制导入（不 drop collection）
|
||||
curl -X POST "http://localhost:9900/api/knowledge/init?force=true"
|
||||
|
||||
# 统计
|
||||
@@ -78,8 +87,8 @@ curl "http://localhost:9900/api/knowledge/stats"
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"collection": "biz_hybrid",
|
||||
"inserted": <大于0>,
|
||||
"collection": "biz",
|
||||
"inserted": 15,
|
||||
"failed": 0,
|
||||
"milvus": { "recreated": true, "loaded": true }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild knowledge into the configured Milvus collection (default: biz).
|
||||
|
||||
Clears:
|
||||
- milvus.collection (drop + recreate dense+BM25 schema)
|
||||
- MySQL api_document
|
||||
- in-memory L0 index
|
||||
|
||||
Then force-imports all markdown under server-side knowledge.base-path
|
||||
(default: knowledge_base/).
|
||||
|
||||
Usage:
|
||||
# start Spring Boot first, then:
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
|
||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://localhost:9900"
|
||||
|
||||
|
||||
def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]:
|
||||
req = urllib.request.Request(url=url, method=method.upper())
|
||||
req.add_header("Accept", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
status = getattr(resp, "status", 200)
|
||||
if not raw.strip():
|
||||
return status, None
|
||||
return status, json.loads(raw)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
body: Any
|
||||
try:
|
||||
body = json.loads(raw) if raw.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
body = raw
|
||||
raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc
|
||||
|
||||
|
||||
def pretty(obj: Any) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def step(title: str) -> None:
|
||||
print()
|
||||
print(f"==> {title}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, "
|
||||
"and reimport knowledge_base markdown into dense+BM25."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=DEFAULT_BASE_URL,
|
||||
help=f"Service base URL (default: {DEFAULT_BASE_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirm",
|
||||
required=True,
|
||||
choices=["REBUILD"],
|
||||
help="Must be REBUILD to execute destructive rebuild",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-stats",
|
||||
action="store_true",
|
||||
help="Skip before/after /api/knowledge/stats",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=7200.0,
|
||||
help="Rebuild request timeout seconds (default: 7200)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
|
||||
print("Hybrid knowledge rebuild")
|
||||
print(f" BaseUrl : {base_url}")
|
||||
print(f" Confirm : {args.confirm}")
|
||||
print(" Source : knowledge_base/ (server-side knowledge.base-path)")
|
||||
print()
|
||||
print("This will DESTROY data in:")
|
||||
print(" - Milvus collection milvus.collection (default: biz)")
|
||||
print(" - MySQL table api_document")
|
||||
print(" - In-memory L0 knowledge index")
|
||||
print("Then re-import all markdown under knowledge_base.")
|
||||
print()
|
||||
|
||||
# 1) health
|
||||
step("Check service health")
|
||||
try:
|
||||
status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30)
|
||||
print(f" milvus health status={status}")
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure
|
||||
print(f" WARN: /milvus/health failed: {exc}")
|
||||
print(" Continue if app is up but milvus health endpoint has issues.")
|
||||
|
||||
# 2) stats before
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (before)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats before failed: {exc}")
|
||||
|
||||
# 3) rebuild
|
||||
step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD")
|
||||
rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}"
|
||||
try:
|
||||
status, body = http_json("POST", rebuild_url, timeout=args.timeout)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc))
|
||||
return 1
|
||||
|
||||
print(f" HTTP {status}")
|
||||
print(pretty(body))
|
||||
|
||||
if not isinstance(body, dict):
|
||||
print("Unexpected rebuild response type", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
inserted = int(body.get("inserted") or 0)
|
||||
failed = int(body.get("failed") or 0)
|
||||
success = bool(body.get("success"))
|
||||
|
||||
if not success:
|
||||
if inserted <= 0:
|
||||
print()
|
||||
print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr)
|
||||
return 2
|
||||
print()
|
||||
print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.")
|
||||
else:
|
||||
print()
|
||||
print(f"Rebuild OK: inserted={inserted}, failed={failed}")
|
||||
|
||||
# 4) stats after
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (after)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats after failed: {exc}")
|
||||
|
||||
print()
|
||||
print("Done.")
|
||||
print("Next:")
|
||||
print(" 1) Ensure application.yml has:")
|
||||
print(" milvus.collection: biz")
|
||||
print(" retrieval.search.mode: hybrid")
|
||||
print(" 2) Smoke test lookup_knowledge / chat with a known doc query")
|
||||
return 0 if success or inserted > 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -8,14 +8,15 @@ public class MilvusConstants {
|
||||
public static final String MILVUS_DB_NAME = "default";
|
||||
|
||||
/**
|
||||
* Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid).
|
||||
* Default knowledge collection name (dense + BM25).
|
||||
* Overridable via {@code milvus.collection}.
|
||||
*/
|
||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
||||
|
||||
/**
|
||||
* Default hybrid dense+BM25 collection.
|
||||
* Alias kept for readability in hybrid-related code.
|
||||
*/
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid";
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
||||
|
||||
/**
|
||||
* 向量维度（豆包 embedding 模型的维度）
|
||||
|
||||
@@ -67,7 +67,7 @@ public class KnowledgeBaseController {
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引，
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作：会删除 {@code milvus.collection}（默认 biz_hybrid）与文档元数据表数据。
|
||||
* <p>危险操作：会删除 {@code milvus.collection}（默认 {@code biz}）与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
|
||||
@@ -57,7 +57,7 @@ public class KnowledgeBaseInitService {
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into biz_hybrid.
|
||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
@@ -82,6 +82,8 @@ public class KnowledgeBaseInitService {
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
// Success when at least one doc indexed and no hard failures.
|
||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
@@ -257,7 +259,11 @@ public class KnowledgeBaseInitService {
|
||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
if (file.toString().endsWith(".md")) {
|
||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
||||
// Import content docs only; skip README/index markdown without frontmatter.
|
||||
if (name.endsWith(".md")
|
||||
&& !name.equalsIgnoreCase("README.md")
|
||||
&& !name.equalsIgnoreCase("readme.md")) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
|
||||
@@ -61,8 +61,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
|
||||
private final MilvusProperties milvusProperties;
|
||||
|
||||
@Value("${milvus.collection:biz_hybrid}")
|
||||
private String collectionName = "biz_hybrid";
|
||||
@Value("${milvus.collection:biz}")
|
||||
private String collectionName = "biz";
|
||||
|
||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
||||
private int rrfK = 60;
|
||||
@@ -144,8 +144,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
|
||||
* Drop the configured knowledge collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Existing vectors in this collection are destroyed.
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
|
||||
@@ -25,8 +25,8 @@ milvus:
|
||||
token: ${MILVUS_TOKEN}
|
||||
secure: true
|
||||
vector-dim: 1024 # BGE-M3 = 1024，换模型时同步改
|
||||
# dense+BM25 hybrid collection (requires reindex; legacy "biz" is not used by knowledge path)
|
||||
collection: biz_hybrid
|
||||
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
|
||||
collection: biz
|
||||
|
||||
# =====================================================
|
||||
# 模型路由配置
|
||||
@@ -158,7 +158,10 @@ document:
|
||||
|
||||
# RAG 配置
|
||||
rag:
|
||||
top-k: 3 # 检索返回的最相似文档数量
|
||||
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
sidecar:
|
||||
spring-ai:
|
||||
enabled: false
|
||||
@@ -176,12 +179,6 @@ retrieval:
|
||||
highly-relevant-threshold: 0.75 # similarity >= 0.75 → HIGHLY_RELEVANT
|
||||
reference-threshold: 0.5 # similarity >= 0.5 → REFERENCE
|
||||
|
||||
# RAG 召回宽度（Delivery 1）
|
||||
rag:
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
|
||||
# Prometheus 配置
|
||||
prometheus:
|
||||
base-url: http://localhost:9090
|
||||
|
||||
@@ -23,7 +23,7 @@ class VectorSearchServiceTest {
|
||||
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
|
||||
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz_hybrid");
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
@@ -44,7 +44,7 @@ class VectorSearchServiceTest {
|
||||
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
|
||||
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz_hybrid");
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
|
||||
Reference in New Issue
Block a user