chore(rag): rebuild script in Python and use biz collection
Switch default knowledge collection back to biz (drop+recreate on rebuild), replace PowerShell rebuild runner with Python, skip README.md imports, and merge duplicate rag keys in application.yml.
This commit is contained in:
zhuyongxin
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild knowledge into the configured Milvus collection (default: biz).
|
||||
|
||||
Clears:
|
||||
- milvus.collection (drop + recreate dense+BM25 schema)
|
||||
- MySQL api_document
|
||||
- in-memory L0 index
|
||||
|
||||
Then force-imports all markdown under server-side knowledge.base-path
|
||||
(default: knowledge_base/).
|
||||
|
||||
Usage:
|
||||
# start Spring Boot first, then:
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
|
||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://localhost:9900"
|
||||
|
||||
|
||||
def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]:
|
||||
req = urllib.request.Request(url=url, method=method.upper())
|
||||
req.add_header("Accept", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
status = getattr(resp, "status", 200)
|
||||
if not raw.strip():
|
||||
return status, None
|
||||
return status, json.loads(raw)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
body: Any
|
||||
try:
|
||||
body = json.loads(raw) if raw.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
body = raw
|
||||
raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc
|
||||
|
||||
|
||||
def pretty(obj: Any) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def step(title: str) -> None:
|
||||
print()
|
||||
print(f"==> {title}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, "
|
||||
"and reimport knowledge_base markdown into dense+BM25."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=DEFAULT_BASE_URL,
|
||||
help=f"Service base URL (default: {DEFAULT_BASE_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirm",
|
||||
required=True,
|
||||
choices=["REBUILD"],
|
||||
help="Must be REBUILD to execute destructive rebuild",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-stats",
|
||||
action="store_true",
|
||||
help="Skip before/after /api/knowledge/stats",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=7200.0,
|
||||
help="Rebuild request timeout seconds (default: 7200)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
|
||||
print("Hybrid knowledge rebuild")
|
||||
print(f" BaseUrl : {base_url}")
|
||||
print(f" Confirm : {args.confirm}")
|
||||
print(" Source : knowledge_base/ (server-side knowledge.base-path)")
|
||||
print()
|
||||
print("This will DESTROY data in:")
|
||||
print(" - Milvus collection milvus.collection (default: biz)")
|
||||
print(" - MySQL table api_document")
|
||||
print(" - In-memory L0 knowledge index")
|
||||
print("Then re-import all markdown under knowledge_base.")
|
||||
print()
|
||||
|
||||
# 1) health
|
||||
step("Check service health")
|
||||
try:
|
||||
status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30)
|
||||
print(f" milvus health status={status}")
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure
|
||||
print(f" WARN: /milvus/health failed: {exc}")
|
||||
print(" Continue if app is up but milvus health endpoint has issues.")
|
||||
|
||||
# 2) stats before
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (before)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats before failed: {exc}")
|
||||
|
||||
# 3) rebuild
|
||||
step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD")
|
||||
rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}"
|
||||
try:
|
||||
status, body = http_json("POST", rebuild_url, timeout=args.timeout)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc))
|
||||
return 1
|
||||
|
||||
print(f" HTTP {status}")
|
||||
print(pretty(body))
|
||||
|
||||
if not isinstance(body, dict):
|
||||
print("Unexpected rebuild response type", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
inserted = int(body.get("inserted") or 0)
|
||||
failed = int(body.get("failed") or 0)
|
||||
success = bool(body.get("success"))
|
||||
|
||||
if not success:
|
||||
if inserted <= 0:
|
||||
print()
|
||||
print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr)
|
||||
return 2
|
||||
print()
|
||||
print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.")
|
||||
else:
|
||||
print()
|
||||
print(f"Rebuild OK: inserted={inserted}, failed={failed}")
|
||||
|
||||
# 4) stats after
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (after)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats after failed: {exc}")
|
||||
|
||||
print()
|
||||
print("Done.")
|
||||
print("Next:")
|
||||
print(" 1) Ensure application.yml has:")
|
||||
print(" milvus.collection: biz")
|
||||
print(" retrieval.search.mode: hybrid")
|
||||
print(" 2) Smoke test lookup_knowledge / chat with a known doc query")
|
||||
return 0 if success or inserted > 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user