refactor(rag): extract retrieval and ingest to py-rag service

- replace in-process Milvus stack with PyRagClient + PyRagKnowledgeSearchAdapter behind KnowledgeSearchPort (RERANK score passthrough)
- move document ingest to py-rag /documents:ingest; DocumentManagementService keeps MySQL ledger + local files
- sink L0 query understanding to py-rag; drop KnowledgeQueryTransformer, single UNFILTERED_VECTOR attempt
- remove Milvus deps, config classes, dead demo services and obsolete rebuild scripts
- compose/Makefile reduced to MySQL/Redis; add pyrag.* config
This commit is contained in:
zhuyongxin
2026-09-30 17:03:21 +08:00
parent 83193bdf4a
commit 9cf162482d
74 changed files with 837 additions and 8285 deletions
+16 -54
View File
@@ -5,9 +5,9 @@
SERVER_URL = http://localhost:9900 SERVER_URL = http://localhost:9900
UPLOAD_API = $(SERVER_URL)/api/upload UPLOAD_API = $(SERVER_URL)/api/upload
DOCS_DIR = aiops-docs DOCS_DIR = aiops-docs
HEALTH_CHECK_API = $(SERVER_URL)/milvus/health # 服务就绪探测:9900 端口有 HTTP 响应即视为就绪
DOCKER_COMPOSE_FILE = vector-database.yml HEALTH_CHECK = curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL)
MILVUS_CONTAINER = milvus-standalone DOCKER_COMPOSE_FILE = docker-compose.yml
# 颜色输出 # 颜色输出
GREEN = \033[0;32m GREEN = \033[0;32m
@@ -23,7 +23,7 @@ help:
@echo "" @echo ""
@echo "可用命令:" @echo "可用命令:"
@echo " $(YELLOW)make init$(NC) - 🚀 一键初始化(启动Docker → 启动服务 → 上传文档)" @echo " $(YELLOW)make init$(NC) - 🚀 一键初始化(启动Docker → 启动服务 → 上传文档)"
@echo " $(YELLOW)make up$(NC) - 启动 Docker Compose(Milvus 向量数据库)" @echo " $(YELLOW)make up$(NC) - 启动 Docker Compose(MySQL/Redis)"
@echo " $(YELLOW)make down$(NC) - 停止 Docker Compose" @echo " $(YELLOW)make down$(NC) - 停止 Docker Compose"
@echo " $(YELLOW)make status$(NC) - 查看 Docker 容器状态" @echo " $(YELLOW)make status$(NC) - 查看 Docker 容器状态"
@echo " $(YELLOW)make start$(NC) - 启动 Spring Boot 服务(后台运行)" @echo " $(YELLOW)make start$(NC) - 启动 Spring Boot 服务(后台运行)"
@@ -42,7 +42,7 @@ help:
init: init:
@echo "$(GREEN)🚀 开始一键初始化 SuperBizAgent...$(NC)" @echo "$(GREEN)🚀 开始一键初始化 SuperBizAgent...$(NC)"
@echo "" @echo ""
@echo "$(YELLOW)步骤 1/4: 启动 Docker Compose(Milvus 向量数据库)$(NC)" @echo "$(YELLOW)步骤 1/4: 启动 Docker Compose(MySQL/Redis)$(NC)"
@$(MAKE) up @$(MAKE) up
@echo "" @echo ""
@echo "$(YELLOW)步骤 2/4: 启动 Spring Boot 服务$(NC)" @echo "$(YELLOW)步骤 2/4: 启动 Spring Boot 服务$(NC)"
@@ -51,23 +51,23 @@ init:
@echo "$(YELLOW)步骤 3/4: 等待服务就绪$(NC)" @echo "$(YELLOW)步骤 3/4: 等待服务就绪$(NC)"
@$(MAKE) wait @$(MAKE) wait
@echo "" @echo ""
@echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档到向量数据库$(NC)" @echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档(经 py-rag 入库)$(NC)"
@$(MAKE) upload @$(MAKE) upload
@echo "" @echo ""
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)" @echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
@echo "$(GREEN)✅ 初始化完成!所有文档已成功向量化存储到数据库$(NC)" @echo "$(GREEN)✅ 初始化完成!所有文档已成功入库(py-rag)$(NC)"
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)" @echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
@echo "" @echo ""
@echo "$(GREEN)🌐 服务访问地址:$(NC)" @echo "$(GREEN)🌐 服务访问地址:$(NC)"
@echo " API 服务: $(SERVER_URL)" @echo " API 服务: $(SERVER_URL)"
@echo " Attu (Web UI): http://localhost:8000" @echo "$(YELLOW)💡 提示: 知识检索/入库由 py-rag 服务承担,请在其仓库单独启动$(NC)"
@echo "" @echo ""
@echo "$(YELLOW)💡 提示: 服务正在后台运行,查看日志: tail -f server.log$(NC)" @echo "$(YELLOW)💡 提示: 服务正在后台运行,查看日志: tail -f server.log$(NC)"
# 启动 Spring Boot 服务(后台运行) # 启动 Spring Boot 服务(后台运行)
start: start:
@echo "$(YELLOW)🚀 启动 Spring Boot 服务...$(NC)" @echo "$(YELLOW)🚀 启动 Spring Boot 服务...$(NC)"
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \ @if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
echo "$(GREEN)✅ 服务已经在运行中 ($(SERVER_URL))$(NC)"; \ echo "$(GREEN)✅ 服务已经在运行中 ($(SERVER_URL))$(NC)"; \
else \ else \
echo "$(YELLOW)📦 正在启动服务(后台运行)...$(NC)"; \ echo "$(YELLOW)📦 正在启动服务(后台运行)...$(NC)"; \
@@ -84,7 +84,7 @@ wait:
@max_attempts=60; \ @max_attempts=60; \
attempt=0; \ attempt=0; \
while [ $$attempt -lt $$max_attempts ]; do \ while [ $$attempt -lt $$max_attempts ]; do \
if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \ if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
echo "$(GREEN)✅ 服务器已就绪!($(SERVER_URL))$(NC)"; \ echo "$(GREEN)✅ 服务器已就绪!($(SERVER_URL))$(NC)"; \
exit 0; \ exit 0; \
fi; \ fi; \
@@ -100,7 +100,7 @@ wait:
# 检查服务器是否运行 # 检查服务器是否运行
check: check:
@echo "$(YELLOW)🔍 检查服务器状态...$(NC)" @echo "$(YELLOW)🔍 检查服务器状态...$(NC)"
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \ @if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
echo "$(GREEN)✅ 服务器运行正常 ($(SERVER_URL))$(NC)"; \ echo "$(GREEN)✅ 服务器运行正常 ($(SERVER_URL))$(NC)"; \
else \ else \
echo "$(RED)❌ 服务器未运行或无法连接!$(NC)"; \ echo "$(RED)❌ 服务器未运行或无法连接!$(NC)"; \
@@ -205,38 +205,14 @@ test-upload:
echo "$(RED)测试文件不存在$(NC)"; \ echo "$(RED)测试文件不存在$(NC)"; \
fi fi
# 启动 Docker Compose(智能检测,避免重复启动) # 启动 Docker Compose(MySQL/Redis;py-rag 服务在其仓库单独启动)
up: up:
@echo "$(YELLOW)🐳 检查 Docker 容器状态...$(NC)" @echo "$(YELLOW)🐳 启动 Docker Compose(MySQL/Redis)...$(NC)"
@if [ ! -f "$(DOCKER_COMPOSE_FILE)" ]; then \ @if [ ! -f "$(DOCKER_COMPOSE_FILE)" ]; then \
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \ echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
exit 1; \ exit 1; \
fi fi
@if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \ @docker-compose -f $(DOCKER_COMPOSE_FILE) up -d && echo "$(GREEN)✅ Docker Compose 启动完成$(NC)"
echo "$(GREEN)✅ Milvus 容器已经在运行中$(NC)"; \
echo "$(YELLOW)📋 当前运行的容器:$(NC)"; \
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
else \
echo "$(YELLOW)🚀 启动 Docker Compose...$(NC)"; \
docker-compose -f $(DOCKER_COMPOSE_FILE) up -d; \
echo ""; \
echo "$(YELLOW)⏳ 等待容器启动...$(NC)"; \
sleep 5; \
if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \
echo "$(GREEN)✅ Docker Compose 启动成功!$(NC)"; \
echo ""; \
echo "$(GREEN)📋 运行中的容器:$(NC)"; \
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
echo ""; \
echo "$(GREEN)🌐 服务访问地址:$(NC)"; \
echo " Milvus: localhost:19530"; \
echo " Attu (Web UI): http://localhost:8000"; \
echo " MinIO: http://localhost:9001 (admin/minioadmin)"; \
else \
echo "$(RED)❌ 容器启动失败,请检查日志: docker-compose -f $(DOCKER_COMPOSE_FILE) logs$(NC)"; \
exit 1; \
fi; \
fi
# 停止 Docker Compose # 停止 Docker Compose
down: down:
@@ -245,24 +221,10 @@ down:
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \ echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
exit 1; \ exit 1; \
fi fi
@if docker ps --format '{{.Names}}' | grep -q "milvus"; then \ @docker-compose -f $(DOCKER_COMPOSE_FILE) down && echo "$(GREEN)✅ Docker Compose 已停止$(NC)"
docker-compose -f $(DOCKER_COMPOSE_FILE) down; \
echo "$(GREEN)✅ Docker Compose 已停止$(NC)"; \
else \
echo "$(YELLOW)⚠️ 没有运行中的 Milvus 容器$(NC)"; \
fi
# 查看 Docker 容器状态 # 查看 Docker 容器状态
status: status:
@echo "$(YELLOW)📊 Docker 容器状态:$(NC)" @echo "$(YELLOW)📊 Docker 容器状态:$(NC)"
@echo "" @echo ""
@if docker ps -a --format '{{.Names}}' | grep -q "milvus"; then \ @docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
docker ps -a --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
echo ""; \
running=$$(docker ps --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
total=$$(docker ps -a --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
echo "$(GREEN)运行中: $$running / $$total$(NC)"; \
else \
echo "$(YELLOW)⚠️ 没有找到 Milvus 相关容器$(NC)"; \
echo "$(YELLOW)提示: 运行 'make docker-up' 启动容器$(NC)"; \
fi
+2 -58
View File
@@ -40,68 +40,12 @@ services:
timeout: 5s timeout: 5s
retries: 5 retries: 5
# Milvus 向量数据库(Standalone 模式) # 向量检索与知识入库由独立的 py-rag 服务承担(见 py-rag 仓库),
# 注意:生产环境建议使用 Zilliz Cloud 或 Milvus 集群 # 其依赖的 Milvus/etcd/MinIO 随 py-rag 部署,不再由本 compose 管理。
etcd:
image: quay.io/coreos/etcd:v3.5.5
container_name: superbiz-etcd
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
volumes:
- etcd-data:/etcd
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
healthcheck:
test: ["CMD", "etcdctl", "endpoint", "health"]
interval: 30s
timeout: 20s
retries: 3
minio:
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
container_name: superbiz-minio
environment:
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
volumes:
- minio-data:/minio_data
command: minio server /minio_data --console-address ":9001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
milvus:
image: milvusdb/milvus:v2.3.3
container_name: superbiz-milvus
depends_on:
- etcd
- minio
environment:
ETCD_ENDPOINTS: etcd:2379
MINIO_ADDRESS: minio:9000
volumes:
- milvus-data:/var/lib/milvus
ports:
- "19530:19530"
- "9091:9091"
command: ["milvus", "run", "standalone"]
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
interval: 30s
start_period: 90s
timeout: 20s
retries: 3
volumes: volumes:
mysql-data: mysql-data:
redis-data: redis-data:
etcd-data:
minio-data:
milvus-data:
networks: networks:
default: default:
+6 -17
View File
@@ -76,12 +76,6 @@
<artifactId>spring-ai-starter-model-deepseek</artifactId> <artifactId>spring-ai-starter-model-deepseek</artifactId>
</dependency> </dependency>
<!-- Embedding: SiliconFlow BGE-M3 (需要 OpenAI 模块的 OpenAiEmbeddingModel) -->
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency> <dependency>
<groupId>com.alibaba.cloud.ai</groupId> <groupId>com.alibaba.cloud.ai</groupId>
<artifactId>spring-ai-alibaba-agent-framework</artifactId> <artifactId>spring-ai-alibaba-agent-framework</artifactId>
@@ -91,19 +85,14 @@
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
</dependency> </dependency>
<!-- <!--
spring-boot-devtools removed on purpose. spring-boot-devtools removed on purpose (bean recreation on classpath restart
Classpath restart (restartedMain) recreates beans without reliably closing is unreliable). Prefer full process restart: stop then `mvn spring-boot:run`.
MilvusClientV2 gRPC channels, causing orphan channels and long hybrid RPC retries.
Prefer full process restart: stop then `mvn spring-boot:run`.
--> -->
<!-- okhttp:DashScopeConfig 的 RestClient.Builder 使用(此前由 milvus-sdk 传递引入) -->
<dependency> <dependency>
<groupId>io.milvus</groupId> <groupId>com.squareup.okhttp3</groupId>
<artifactId>milvus-sdk-java</artifactId> <artifactId>okhttp</artifactId>
<version>2.6.10</version> <version>4.12.0</version>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-vector-store-milvus</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
-106
View File
@@ -1,106 +0,0 @@
# 重建 hybrid 知识库(dense + BM25)
面向当前 `knowledge_base/` 目录文档,**清空并重建**配置中的 Milvus collection(默认 **`biz`**)。
## 前提
1. 应用已启动(默认 `http://localhost:9900`)
2. `MILVUS_TOKEN` 等连接配置可用
3. `application.yml` 已配置:
```yaml
milvus:
collection: biz
retrieval:
search:
mode: hybrid
knowledge:
base-path: knowledge_base/
```
## 一键脚本(Python)
在项目根目录执行:
```bash
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
```
指定服务地址:
```bash
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
```
跳过前后 stats:
```bash
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD --skip-stats
```
依赖:Python 3.9+ 标准库即可(无需 pip 包)。
## 脚本会做什么
| 步骤 | 动作 |
|---|---|
| 1 | 检查 `/milvus/health` |
| 2 | 打印重建前 `/api/knowledge/stats` |
| 3 | `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD` |
| 4 | 打印重建后 stats |
服务端 `rebuild-hybrid` 内部顺序:
1. **Drop + recreate** Milvus collection(`milvus.collection`,默认 `biz`)
- 原有向量数据会被删除
- 按 dense + BM25 schema 重建
2. **清空** MySQL `api_document`
3. **清空** 内存 L0 索引
4. **扫描** `knowledge_base/**/*.md`(跳过 `README.md`)并 force 全量导入
- 写 MySQL 元数据
- 切片
- 写 dense 向量 + BM25 `search_text`
- 更新 L0
## 不会做什么
- **不会**动 `knowledge_base/` 源文件
- **不会**在未传 `--confirm REBUILD` 时执行
## 手动 curl 等价命令
```bash
# 重建(危险:会清空 biz collection + api_document)
curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD"
# 仅强制导入(不 drop collection)
curl -X POST "http://localhost:9900/api/knowledge/init?force=true"
# 统计
curl "http://localhost:9900/api/knowledge/stats"
```
## 成功判据
响应中大致应有:
```json
{
"success": true,
"collection": "biz",
"inserted": 15,
"failed": 0,
"milvus": { "recreated": true, "loaded": true }
}
```
然后用一条知识库里真实存在的术语/故障词走 `lookup_knowledge` 或 chat 验证 hybrid 命中。
## 失败排查
| 现象 | 可能原因 |
|---|---|
| connect / token 错误 | `MILVUS_TOKEN`、host、database |
| BM25 / analyzer 相关报错 | 云端 Milvus/Zilliz 版本不支持 BM25 Function |
| inserted=0 | `knowledge_base` 路径不对,或 md 缺 frontmatter/title |
| failed>0 | 看响应 `details` 与应用日志 |
-292
View File
@@ -1,292 +0,0 @@
#!/usr/bin/env python3
"""Live acceptance runner for post-reindex RAG retrieval checks.
This script calls the running Spring Boot retrieval endpoint. It is intentionally
separate from the offline fixture baseline because it depends on live service and
Milvus/Zilliz state.
"""
from __future__ import annotations
import argparse
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
DEFAULT_BASE_URL = "http://127.0.0.1:9900"
DEFAULT_JSON_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.json")
DEFAULT_MD_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.md")
DEFAULT_CASES: list[dict[str, Any]] = [
{
"caseId": "breadcrumb-rag-chunk-context",
"query": "If a long RAG section is split into multiple chunks, how do we keep retrieval context?",
"topK": 5,
"purpose": "Breadcrumb-sensitive RAG chunk context retrieval.",
},
{
"caseId": "breadcrumb-diagnosis-flow",
"query": "What is the standard troubleshooting flow for an application incident?",
"topK": 5,
"purpose": "Process-style retrieval where section path matters.",
},
{
"caseId": "core-err-timeout",
"query": "ERR_TIMEOUT",
"topK": 3,
"purpose": "Exact error-code retrieval should remain stable.",
},
{
"caseId": "core-mysql-connection-pool",
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
"topK": 3,
"purpose": "Core infrastructure troubleshooting retrieval.",
},
{
"caseId": "aiops-payment-latency",
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
"topK": 3,
"purpose": "AIOps alert-style retrieval.",
},
]
@dataclass
class LiveCase:
case_id: str
query: str
top_k: int
purpose: str
category: str | None = None
@classmethod
def from_json(cls, raw: dict[str, Any]) -> "LiveCase":
return cls(
case_id=str(raw["caseId"]),
query=str(raw["query"]),
top_k=int(raw.get("topK") or 3),
purpose=str(raw.get("purpose") or raw.get("notes") or ""),
category=(
str(raw.get("category"))
if raw.get("category") not in (None, "")
else None
),
)
def load_cases(path: Path | None) -> list[LiveCase]:
if path is None:
return [LiveCase.from_json(item) for item in DEFAULT_CASES]
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
raw_cases = payload.get("cases", payload)
return [LiveCase.from_json(item) for item in raw_cases]
def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8", newline="\n") as handle:
json.dump(payload, handle, ensure_ascii=False, indent=2)
handle.write("\n")
def write_text(path: Path, content: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8", newline="\n") as handle:
handle.write(content)
def request_case(base_url: str, case: LiveCase, timeout_seconds: float) -> dict[str, Any]:
endpoint = base_url.rstrip("/") + "/api/search/similar"
params: dict[str, str] = {
"query": case.query,
"topK": str(case.top_k),
}
if case.category:
params["category"] = case.category
url = endpoint + "?" + urllib.parse.urlencode(params)
started_at = datetime.now(timezone.utc)
try:
with urllib.request.urlopen(url, timeout=timeout_seconds) as response:
body = response.read().decode("utf-8")
payload = json.loads(body)
status = int(getattr(response, "status", 200))
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
return {
"caseId": case.case_id,
"query": case.query,
"topK": case.top_k,
"category": case.category,
"purpose": case.purpose,
"url": url,
"ok": False,
"error": str(exc),
"resultCount": 0,
"topCandidates": [],
"rawResponse": None,
"startedAt": started_at.isoformat(),
}
data = payload.get("data") if isinstance(payload, dict) else None
if not isinstance(data, list):
data = []
ok = status == 200 and payload.get("code") == 200
return {
"caseId": case.case_id,
"query": case.query,
"topK": case.top_k,
"category": case.category,
"purpose": case.purpose,
"url": url,
"ok": ok,
"httpStatus": status,
"responseCode": payload.get("code"),
"responseMessage": payload.get("message"),
"resultCount": len(data),
"topCandidates": [summarize_candidate(item, index + 1) for index, item in enumerate(data)],
"rawResponse": payload,
"startedAt": started_at.isoformat(),
}
def summarize_candidate(raw: dict[str, Any], rank: int) -> dict[str, Any]:
metadata = parse_metadata(raw.get("metadata"))
return {
"rank": rank,
"id": raw.get("id"),
"title": metadata.get("title"),
"breadcrumb": metadata.get("breadcrumb"),
"category": metadata.get("category"),
"source": metadata.get("_source") or metadata.get("source"),
"score": raw.get("score"),
"rawScore": raw.get("rawScore"),
"scoreLabel": raw.get("scoreLabel"),
"contentPreview": preview(raw.get("content")),
}
def parse_metadata(value: Any) -> dict[str, Any]:
if isinstance(value, dict):
return value
if isinstance(value, str) and value.strip():
try:
parsed = json.loads(value)
return parsed if isinstance(parsed, dict) else {}
except json.JSONDecodeError:
return {}
return {}
def preview(value: Any, limit: int = 180) -> str:
text = " ".join(str(value or "").split())
if len(text) <= limit:
return text
return text[: limit - 3] + "..."
def render_markdown(report: dict[str, Any]) -> str:
lines = [
"# RAG Live Post-Reindex Acceptance",
"",
f"Generated at: `{report['generatedAt']}`",
f"Base URL: `{report['baseUrl']}`",
"",
"> Reindex prerequisite: this report only reflects breadcrumb-aware embedding if the knowledge base was reindexed after the embedding-text change.",
"",
"## Summary",
"",
"| Metric | Value |",
"|---|---:|",
f"| Cases | {report['caseCount']} |",
f"| Successful calls | {report['successfulCalls']} |",
f"| Empty result cases | {report['emptyResultCases']} |",
"",
"## Cases",
"",
"| Case | Purpose | Results | Top Candidates |",
"|---|---|---:|---|",
]
for item in report["results"]:
top = "<br>".join(format_candidate(candidate) for candidate in item["topCandidates"])
if not top and item.get("error"):
top = "ERROR: " + str(item["error"])
lines.append(
"| {case} | {purpose} | {count} | {top} |".format(
case=item["caseId"],
purpose=item.get("purpose") or "",
count=item["resultCount"],
top=top,
)
)
lines.append("")
return "\n".join(lines)
def format_candidate(candidate: dict[str, Any]) -> str:
label = candidate.get("title") or candidate.get("source") or candidate.get("id") or ""
breadcrumb = candidate.get("breadcrumb") or ""
score_label = candidate.get("scoreLabel") or ""
score = candidate.get("score")
raw_score = candidate.get("rawScore")
details = f"score={score}"
if raw_score is not None:
details += f", raw={raw_score}"
if score_label:
details += f", label={score_label}"
if breadcrumb:
return f"{candidate['rank']}. {label} ({breadcrumb}; {details})"
return f"{candidate['rank']}. {label} ({details})"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
parser.add_argument("--cases", type=Path, default=None)
parser.add_argument("--json-report", type=Path, default=DEFAULT_JSON_REPORT)
parser.add_argument("--markdown-report", type=Path, default=DEFAULT_MD_REPORT)
parser.add_argument("--timeout-seconds", type=float, default=10.0)
return parser.parse_args()
def main() -> int:
args = parse_args()
cases = load_cases(args.cases)
results = [
request_case(args.base_url, case, args.timeout_seconds)
for case in cases
]
successful = [item for item in results if item["ok"]]
empty = [item for item in results if item["ok"] and item["resultCount"] == 0]
report = {
"generatedAt": datetime.now(timezone.utc).isoformat(),
"baseUrl": args.base_url,
"caseCount": len(results),
"successfulCalls": len(successful),
"emptyResultCases": len(empty),
"reindexPrerequisite": "Run or trigger knowledge-base reindex before treating this as breadcrumb-aware embedding evidence.",
"results": results,
}
write_json(args.json_report, report)
write_text(args.markdown_report, render_markdown(report))
print(
"Ran {total} live cases: successful={successful}, empty={empty}".format(
total=len(results),
successful=len(successful),
empty=len(empty),
)
)
return 1 if len(successful) != len(results) else 0
if __name__ == "__main__":
raise SystemExit(main())
-176
View File
@@ -1,176 +0,0 @@
#!/usr/bin/env python3
"""Rebuild knowledge into the configured Milvus collection (default: biz).
Clears:
- milvus.collection (drop + recreate dense+BM25 schema)
- MySQL api_document
- in-memory L0 index
Then force-imports all markdown under server-side knowledge.base-path
(default: knowledge_base/).
Usage:
# start Spring Boot first, then:
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
"""
from __future__ import annotations
import argparse
import json
import sys
import urllib.error
import urllib.request
from typing import Any
DEFAULT_BASE_URL = "http://localhost:9900"
def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]:
req = urllib.request.Request(url=url, method=method.upper())
req.add_header("Accept", "application/json")
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", errors="replace")
status = getattr(resp, "status", 200)
if not raw.strip():
return status, None
return status, json.loads(raw)
except urllib.error.HTTPError as exc:
raw = exc.read().decode("utf-8", errors="replace")
body: Any
try:
body = json.loads(raw) if raw.strip() else None
except json.JSONDecodeError:
body = raw
raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc
except urllib.error.URLError as exc:
raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc
def pretty(obj: Any) -> str:
return json.dumps(obj, ensure_ascii=False, indent=2)
def step(title: str) -> None:
print()
print(f"==> {title}")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, "
"and reimport knowledge_base markdown into dense+BM25."
)
parser.add_argument(
"--base-url",
default=DEFAULT_BASE_URL,
help=f"Service base URL (default: {DEFAULT_BASE_URL})",
)
parser.add_argument(
"--confirm",
required=True,
choices=["REBUILD"],
help="Must be REBUILD to execute destructive rebuild",
)
parser.add_argument(
"--skip-stats",
action="store_true",
help="Skip before/after /api/knowledge/stats",
)
parser.add_argument(
"--timeout",
type=float,
default=7200.0,
help="Rebuild request timeout seconds (default: 7200)",
)
args = parser.parse_args(argv)
base_url = args.base_url.rstrip("/")
print("Hybrid knowledge rebuild")
print(f" BaseUrl : {base_url}")
print(f" Confirm : {args.confirm}")
print(" Source : knowledge_base/ (server-side knowledge.base-path)")
print()
print("This will DESTROY data in:")
print(" - Milvus collection milvus.collection (default: biz)")
print(" - MySQL table api_document")
print(" - In-memory L0 knowledge index")
print("Then re-import all markdown under knowledge_base.")
print()
# 1) health
step("Check service health")
try:
status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30)
print(f" milvus health status={status}")
print(pretty(body))
except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure
print(f" WARN: /milvus/health failed: {exc}")
print(" Continue if app is up but milvus health endpoint has issues.")
# 2) stats before
if not args.skip_stats:
step("Knowledge stats (before)")
try:
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
print(pretty(body))
except Exception as exc: # noqa: BLE001
print(f" WARN: stats before failed: {exc}")
# 3) rebuild
step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD")
rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}"
try:
status, body = http_json("POST", rebuild_url, timeout=args.timeout)
except RuntimeError as exc:
print(str(exc))
return 1
print(f" HTTP {status}")
print(pretty(body))
if not isinstance(body, dict):
print("Unexpected rebuild response type", file=sys.stderr)
return 1
inserted = int(body.get("inserted") or 0)
failed = int(body.get("failed") or 0)
success = bool(body.get("success"))
if not success:
if inserted <= 0:
print()
print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr)
return 2
print()
print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.")
else:
print()
print(f"Rebuild OK: inserted={inserted}, failed={failed}")
# 4) stats after
if not args.skip_stats:
step("Knowledge stats (after)")
try:
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
print(pretty(body))
except Exception as exc: # noqa: BLE001
print(f" WARN: stats after failed: {exc}")
print()
print("Done.")
print("Next:")
print(" 1) Ensure application.yml has:")
print(" milvus.collection: biz")
print(" retrieval.search.mode: hybrid")
print(" 2) Smoke test lookup_knowledge / chat with a known doc query")
return 0 if success or inserted > 0 else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -1,196 +0,0 @@
package com.superbiz.agent.client;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.DataType;
import io.milvus.param.ConnectParam;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.*;
import io.milvus.param.index.CreateIndexParam;
import com.superbiz.agent.config.MilvusProperties;
import com.superbiz.agent.constant.MilvusConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.concurrent.TimeUnit;
/**
* Milvus 客户端工厂类
* 负责创建和初始化 Milvus 客户端连接
*/
@Component
public class MilvusClientFactory {
private static final Logger logger = LoggerFactory.getLogger(MilvusClientFactory.class);
@Autowired
private MilvusProperties milvusProperties;
/**
* 创建并初始化 Milvus 客户端
*
* 简化版本:直接连接并创建 collection
*
* @return MilvusServiceClient 实例
* @throws RuntimeException 如果连接或初始化失败
*/
public MilvusServiceClient createClient() {
MilvusServiceClient client = null;
try {
// 1. 连接到 Milvus
logger.info("正在连接到 Milvus: {}:{}", milvusProperties.getHost(), milvusProperties.getPort());
client = connectToMilvus();
logger.info("成功连接到 Milvus");
// 2. 检查并创建 biz collection(如果不存在)
if (!collectionExists(client, MilvusConstants.MILVUS_COLLECTION_NAME)) {
logger.info("collection '{}' 不存在,正在创建...", MilvusConstants.MILVUS_COLLECTION_NAME);
createBizCollection(client);
logger.info("成功创建 collection '{}'", MilvusConstants.MILVUS_COLLECTION_NAME);
// 创建索引
createIndexes(client);
logger.info("成功创建索引");
} else {
logger.info("collection '{}' 已存在", MilvusConstants.MILVUS_COLLECTION_NAME);
}
// 3. 加载 collection 到内存(搜索必须)
logger.info("正在加载 collection '{}' 到内存...", MilvusConstants.MILVUS_COLLECTION_NAME);
R<RpcStatus> loadResp = client.loadCollection(LoadCollectionParam.newBuilder()
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
.build());
if (loadResp.getStatus() == 0) {
logger.info("collection '{}' 已加载", MilvusConstants.MILVUS_COLLECTION_NAME);
} else {
logger.warn("collection '{}' 加载失败: {}", MilvusConstants.MILVUS_COLLECTION_NAME, loadResp.getMessage());
}
return client;
} catch (Exception e) {
logger.error("创建 Milvus 客户端失败", e);
if (client != null) {
client.close();
}
throw new RuntimeException("创建 Milvus 客户端失败: " + e.getMessage(), e);
}
}
/**
* 连接到 Milvus
*/
private MilvusServiceClient connectToMilvus() {
ConnectParam.Builder builder = ConnectParam.newBuilder()
.withHost(milvusProperties.getHost())
.withPort(milvusProperties.getPort())
.withDatabaseName(milvusProperties.getDatabase())
.withConnectTimeout(milvusProperties.getTimeout(), TimeUnit.MILLISECONDS);
// Zilliz Cloud: token + SSL
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isEmpty()) {
builder.withToken(milvusProperties.getToken());
builder.withSecure(true);
}
// 本地 Milvus: username + password
else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isEmpty()) {
builder.withAuthorization(milvusProperties.getUsername(), milvusProperties.getPassword());
}
return new MilvusServiceClient(builder.build());
}
/**
* 检查 collection 是否存在
*/
private boolean collectionExists(MilvusServiceClient client, String collectionName) {
R<Boolean> response = client.hasCollection(HasCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build());
if (response.getStatus() != 0) {
throw new RuntimeException("检查 collection 失败: " + response.getMessage());
}
return response.getData();
}
/**
* 创建 biz collection
*/
private void createBizCollection(MilvusServiceClient client) {
// 定义字段
FieldType idField = FieldType.newBuilder()
.withName("id")
.withDataType(DataType.VarChar)
.withMaxLength(MilvusConstants.ID_MAX_LENGTH)
.withPrimaryKey(true)
.build();
FieldType vectorField = FieldType.newBuilder()
.withName("vector")
.withDataType(DataType.FloatVector) // 改为 FloatVector
.withDimension(milvusProperties.getVectorDim())
.build();
FieldType contentField = FieldType.newBuilder()
.withName("content")
.withDataType(DataType.VarChar)
.withMaxLength(MilvusConstants.CONTENT_MAX_LENGTH)
.build();
FieldType metadataField = FieldType.newBuilder()
.withName("metadata")
.withDataType(DataType.JSON)
.build();
// 创建 collection schema
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
.withEnableDynamicField(false)
.addFieldType(idField)
.addFieldType(vectorField)
.addFieldType(contentField)
.addFieldType(metadataField)
.build();
// 创建 collection
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
.withDescription("Business knowledge collection")
.withSchema(schema)
.withShardsNum(MilvusConstants.DEFAULT_SHARD_NUMBER)
.build();
R<RpcStatus> response = client.createCollection(createParam);
if (response.getStatus() != 0) {
throw new RuntimeException("创建 collection 失败: " + response.getMessage());
}
}
/**
* 为 collection 创建索引
*/
private void createIndexes(MilvusServiceClient client) {
// 为 vector 字段创建索引(FloatVector 使用 IVF_FLAT 和 L2 距离)
CreateIndexParam vectorIndexParam = CreateIndexParam.newBuilder()
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
.withFieldName("vector")
.withIndexType(IndexType.IVF_FLAT)
.withMetricType(MetricType.L2) // L2 距离(欧氏距离)
.withExtraParam("{\"nlist\":128}")
.withSyncMode(Boolean.FALSE)
.build();
R<RpcStatus> response = client.createIndex(vectorIndexParam);
if (response.getStatus() != 0) {
throw new RuntimeException("创建 vector 索引失败: " + response.getMessage());
}
logger.info("成功为 vector 字段创建索引");
}
}
@@ -0,0 +1,286 @@
package com.superbiz.agent.client;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import com.superbiz.agent.config.PyRagProperties;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.MediaType;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestClient;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
/**
* py-rag 知识服务 HTTP 客户端(API v1,契约见 py-rag 仓库 docs/Java接入文档.md)。
*
* <p>错误信封:4xx/5xx 一律 {@code {"error":{"code":"E_XXX","message":…,"details":[…]}}},
* 统一抛出 {@link PyRagClientException};网络异常包装为 {@code E_NETWORK}。
* {@code evidence_status=no_evidence} 是 200 正常业务响应,不作为错误。</p>
*
* <p>每个请求携带 {@code X-Request-ID}(UUID)用于跨服务日志关联;
* 超时按接入文档矩阵分端点配置(见 {@link PyRagProperties})。</p>
*/
@Slf4j
@Component
public class PyRagClient {
private static final String REQUEST_ID_HEADER = "X-Request-ID";
private final PyRagProperties properties;
private final ObjectMapper objectMapper;
private final RestClient searchClient;
private final RestClient ingestClient;
private final RestClient defaultClient;
public PyRagClient(PyRagProperties properties, ObjectMapper objectMapper) {
this.properties = properties;
this.objectMapper = objectMapper;
this.searchClient = buildRestClient(properties.getSearchReadTimeoutMs());
this.ingestClient = buildRestClient(properties.getIngestReadTimeoutMs());
this.defaultClient = buildRestClient(properties.getDefaultReadTimeoutMs());
}
private RestClient buildRestClient(int readTimeoutMs) {
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
factory.setConnectTimeout(properties.getConnectTimeoutMs());
factory.setReadTimeout(readTimeoutMs);
return RestClient.builder()
.baseUrl(properties.getBaseUrl())
.requestFactory(factory)
.build();
}
// ── 检索 ────────────────────────────────────────────────
public PyRagSearchResponse search(PyRagSearchRequest request) {
return exchange(() -> searchClient.post()
.uri("/api/v1/search")
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
.contentType(MediaType.APPLICATION_JSON)
.body(request), PyRagSearchResponse.class);
}
// ── 文档入库 ────────────────────────────────────────────
/**
* multipart 文档入库。category 为契约必填;title/breadcrumb/kbScope 可选(null 不传)。
* 同内容重复上传返回 unchanged(幂等),网络超时可安全重试。
*/
public PyRagIngestResponse ingest(String filename,
byte[] content,
String contentType,
String category,
String title,
String breadcrumb,
String kbScope) {
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
body.add("file", new ByteArrayResource(content) {
@Override
public String getFilename() {
return filename;
}
});
body.add("category", category);
if (title != null && !title.isBlank()) {
body.add("title", title);
}
if (breadcrumb != null && !breadcrumb.isBlank()) {
body.add("breadcrumb", breadcrumb);
}
if (kbScope != null && !kbScope.isBlank()) {
body.add("kb_scope", kbScope);
}
return exchange(() -> ingestClient.post()
.uri("/api/v1/documents:ingest")
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
.contentType(MediaType.MULTIPART_FORM_DATA)
.body(body), PyRagIngestResponse.class);
}
// ── 全量重建(异步任务) ────────────────────────────────
/** 202 返回任务号;已有 rebuild 执行中抛 E_REBUILD_IN_PROGRESS。 */
public PyRagTaskAccepted rebuild() {
return exchange(() -> defaultClient.post()
.uri("/api/v1/collections:rebuild?confirm=REBUILD")
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
.contentType(MediaType.APPLICATION_JSON)
.body(Map.of()), PyRagTaskAccepted.class);
}
public PyRagTaskStatus task(String taskId) {
return exchange(() -> defaultClient.get()
.uri("/api/v1/tasks/{id}", taskId)
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagTaskStatus.class);
}
// ── 统计与健康 ──────────────────────────────────────────
public PyRagStats stats() {
return exchange(() -> defaultClient.get()
.uri("/api/v1/stats")
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagStats.class);
}
public PyRagHealth health() {
return exchange(() -> defaultClient.get()
.uri("/api/v1/health")
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagHealth.class);
}
// ── 内部 ────────────────────────────────────────────────
private <T> T exchange(Supplier<RestClient.RequestHeadersSpec<?>> spec, Class<T> type) {
try {
return spec.get().exchange((request, response) -> {
if (response.getStatusCode().isError()) {
throw toClientException(response.getStatusCode().value(), response.getBody());
}
return objectMapper.readValue(response.getBody(), type);
});
} catch (PyRagClientException e) {
throw e;
} catch (Exception e) {
log.error("py-rag 调用失败: {}", e.getMessage(), e);
throw new PyRagClientException("E_NETWORK",
"py-rag 调用失败: " + e.getMessage(), null, e);
}
}
private PyRagClientException toClientException(int httpStatus, java.io.InputStream body) {
String code = "E_HTTP_" + httpStatus;
String message = "HTTP " + httpStatus;
try {
PyRagErrorEnvelope envelope = objectMapper.readValue(body, PyRagErrorEnvelope.class);
if (envelope != null && envelope.error() != null) {
code = envelope.error().code() == null ? code : envelope.error().code();
message = envelope.error().message() == null ? message : envelope.error().message();
}
} catch (Exception ignored) {
// 错误响应体不是契约信封(如网关 502 页面),保留 HTTP 默认语义
}
return new PyRagClientException(code, message, httpStatus, null);
}
// ── 契约 DTO(snake_case 对齐 py-rag API) ──────────────
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
public record PyRagSearchRequest(
String query,
/** hybrid=dense+BM25 融合;semantic=纯向量 */
String mode,
Integer retrieveK,
Integer returnN,
Integer maxChunksPerDocument,
String category,
String kbScope
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagSearchHit(
String evidenceKey,
String documentId,
String source,
String title,
String breadcrumb,
String excerpt,
Double qualityScore,
String relevanceLevel
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagRetrievalTrace(
String mode,
Map<String, Object> filters,
Integer recallCount,
String rerankModel,
String noEvidenceBasis
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagSearchResponse(
String query,
String mode,
List<PyRagSearchHit> hits,
String relevanceLevel,
/** supported | no_evidence(no_evidence 时 hits=[],属正常业务响应) */
String evidenceStatus,
PyRagRetrievalTrace retrievalTrace
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagIngestResponse(
String docId,
String source,
/** created | updated | unchanged */
String status,
Integer chunkCount,
List<String> warnings,
Map<String, Object> frontmatter
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagTaskAccepted(
String taskId,
String status
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagTaskStatus(
String taskId,
String status,
Integer documents,
String detail,
String createdAt,
String finishedAt
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagStats(
String collection,
Integer rowCount
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagHealth(
String status,
String version,
Map<String, String> checks
) {
}
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public record PyRagErrorEnvelope(ErrorBody error) {
@JsonIgnoreProperties(ignoreUnknown = true)
public record ErrorBody(String code, String message, List<Map<String, Object>> details) {
}
}
}
@@ -0,0 +1,25 @@
package com.superbiz.agent.client;
import lombok.Getter;
/**
* py-rag 调用异常:携带契约错误码(E_*)与 HTTP 状态。
*
* <p>调用方按错误码分支:422 参数/数据问题不重试;
* 409 E_REBUILD_IN_PROGRESS 延迟重试;网络异常(E_NETWORK)可安全重试。</p>
*/
@Getter
public class PyRagClientException extends RuntimeException {
/** 契约错误码:E_INVALID_REQUEST / E_FRONTMATTER_INVALID / E_REBUILD_IN_PROGRESS / E_NETWORK 等 */
private final String code;
/** HTTP 状态码;网络层异常(未拿到响应)为 null */
private final Integer httpStatus;
public PyRagClientException(String code, String message, Integer httpStatus, Throwable cause) {
super("[" + code + "] " + message, cause);
this.code = code;
this.httpStatus = httpStatus;
}
}
@@ -1,52 +0,0 @@
package com.superbiz.agent.config;
import lombok.Getter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* 文档分片配置
*/
@Getter
@Configuration
@ConfigurationProperties(prefix = "document.chunk")
public class DocumentChunkConfig {
/**
* 每个分片的最大字符数(保留向后兼容)
*/
private int maxSize = 800;
/**
* 分片之间的重叠字符数
*/
private int overlap = 100;
/**
* 每个分片的最大 token 数(中文~1:1,英文~0.25:1)
* 替代 maxSize 作为切割触发器
*/
private int maxTokens = 500;
/**
* 硬上限 token 数 = maxTokens × 1.2
* 仅在不可中断上下文(列表、代码块)内触发
*/
private int maxTokensHard = 600;
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
public void setOverlap(int overlap) {
this.overlap = overlap;
}
public void setMaxTokens(int maxTokens) {
this.maxTokens = maxTokens;
}
public void setMaxTokensHard(int maxTokensHard) {
this.maxTokensHard = maxTokensHard;
}
}
@@ -1,27 +0,0 @@
package com.superbiz.agent.config;
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Configuration;
/**
* Milvus 知识路径配置说明(无额外 Bean 装配)。
*
* <p>知识库 RAG 唯一实现:{@link MilvusHybridKnowledgeStore}({@code MilvusClientV2})。</p>
* <ul>
* <li>支持 dense 与 dense+BM25 {@code hybridSearch}+RRF。</li>
* <li>不再为知识路径创建 legacy {@code MilvusServiceClient} Bean。</li>
* <li>Spring AI {@code VectorStore} starter 仍可存在于 classpath,但只作 sidecar,
* 不作 lookup_knowledge 主路径(starter 无 BM25 hybrid API)。</li>
* </ul>
*/
@Configuration
public class MilvusConfig {
private static final Logger logger = LoggerFactory.getLogger(MilvusConfig.class);
public MilvusConfig() {
logger.info("Milvus knowledge path: MilvusClientV2 hybrid store only (legacy SDK search disabled)");
}
}
@@ -1,95 +0,0 @@
package com.superbiz.agent.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@ConfigurationProperties(prefix = "milvus")
public class MilvusProperties {
private String host = "localhost";
private Integer port = 19530;
private String username = "";
private String password = "";
private String database = "default";
private Long timeout = 10000L;
private String token = "";
private boolean secure = false;
private int vectorDim = 1024;
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
public Integer getPort() {
return port;
}
public void setPort(Integer port) {
this.port = port;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public Long getTimeout() {
return timeout;
}
public void setTimeout(Long timeout) {
this.timeout = timeout;
}
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
public boolean isSecure() {
return secure;
}
public void setSecure(boolean secure) {
this.secure = secure;
}
public int getVectorDim() {
return vectorDim;
}
public void setVectorDim(int vectorDim) {
this.vectorDim = vectorDim;
}
public String getAddress() {
return host + ":" + port;
}
}
@@ -1,12 +1,10 @@
package com.superbiz.agent.config; package com.superbiz.agent.config;
import java.util.List; import java.util.List;
import java.util.Map;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@@ -19,11 +17,11 @@ import org.springframework.context.annotation.Primary;
* <pre>{@code * <pre>{@code
* model-routing: * model-routing:
* chat: deepseek * chat: deepseek
* embedding: siliconflow
* }</pre> * }</pre>
* <p> * <p>
* 匹配优先级:Bean 名 > 类名(均不区分大小写)。 * 匹配优先级:Bean 名 > 类名(均不区分大小写)。
* 切换模型只改 yml + pom + 对应 api-key,Java 代码不动。 * 切换模型只改 yml + pom + 对应 api-key,Java 代码不动。
* (Embedding 路由已随 RAG 模块抽离至 py-rag 服务端,此处仅路由 Chat。)
*/ */
@Configuration @Configuration
public class ModelRoutingConfig { public class ModelRoutingConfig {
@@ -33,9 +31,6 @@ public class ModelRoutingConfig {
@Value("${model-routing.chat:deepseek}") @Value("${model-routing.chat:deepseek}")
private String chatKeyword; private String chatKeyword;
@Value("${model-routing.embedding:siliconflow}")
private String embeddingKeyword;
@Bean @Bean
@Primary @Primary
public ChatModel chatModel(List<ChatModel> chatModels) { public ChatModel chatModel(List<ChatModel> chatModels) {
@@ -53,33 +48,6 @@ public class ModelRoutingConfig {
return chatModels.get(0); return chatModels.get(0);
} }
@Bean
@Primary
public EmbeddingModel embeddingModel(Map<String, EmbeddingModel> embeddingBeans) {
log.info("Embedding 路由: keyword='{}', 可用: {}", embeddingKeyword, embeddingBeans.keySet());
// 先按 Bean 名匹配
for (Map.Entry<String, EmbeddingModel> entry : embeddingBeans.entrySet()) {
if (containsIgnoreCase(entry.getKey(), embeddingKeyword)) {
log.info(" → Bean 名匹配: {} → {}", entry.getKey(),
entry.getValue().getClass().getSimpleName());
return entry.getValue();
}
}
// 再按类名匹配
for (EmbeddingModel em : embeddingBeans.values()) {
if (matches(em.getClass(), embeddingKeyword)) {
log.info(" → 类名匹配: {}", em.getClass().getSimpleName());
return em;
}
}
var first = embeddingBeans.values().iterator().next();
log.warn(" → 未匹配, 回退到 {}", first.getClass().getSimpleName());
return first;
}
private boolean matches(Class<?> clazz, String keyword) { private boolean matches(Class<?> clazz, String keyword) {
return containsIgnoreCase(clazz.getName(), keyword) return containsIgnoreCase(clazz.getName(), keyword)
|| containsIgnoreCase(clazz.getSimpleName(), keyword); || containsIgnoreCase(clazz.getSimpleName(), keyword);
@@ -0,0 +1,34 @@
package com.superbiz.agent.config;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
/**
* py-rag 知识服务接入配置。
*
* <p>超时矩阵来自《py-rag 知识服务 · Java 接入文档》第 6 节:
* 服务含 embedding/rerank 外呼,search 正常 300–800ms、ingest 正常 1–5s。</p>
*/
@Getter
@Setter
@Configuration
@ConfigurationProperties(prefix = "pyrag")
public class PyRagProperties {
/** py-rag 服务根地址,如 http://py-rag:8000 */
private String baseUrl = "http://localhost:8000";
/** 连接超时(毫秒),全端点统一 */
private int connectTimeoutMs = 3000;
/** /api/v1/search 读取超时(毫秒) */
private int searchReadTimeoutMs = 5000;
/** /api/v1/documents:ingest 读取超时(毫秒) */
private int ingestReadTimeoutMs = 30000;
/** rebuild/tasks/stats/health 读取超时(毫秒) */
private int defaultReadTimeoutMs = 10000;
}
@@ -1,23 +0,0 @@
package com.superbiz.agent.config;
import lombok.Getter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Getter
@Configuration
@ConfigurationProperties(prefix = "rag.sidecar.spring-ai")
public class RagSidecarProperties {
private boolean enabled = false;
private int contentPreviewLimit = 300;
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public void setContentPreviewLimit(int contentPreviewLimit) {
this.contentPreviewLimit = contentPreviewLimit;
}
}
@@ -1,54 +0,0 @@
package com.superbiz.agent.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.document.MetadataMode;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingModel;
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
import org.springframework.ai.openai.api.OpenAiApi;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.client.RestClient;
import org.springframework.web.reactive.function.client.WebClient;
/**
* SiliconFlow Embedding 配置(BGE-M3, OpenAI 兼容协议, 1024维)
* <p>
* Chat 走 DeepSeek、Embedding 走 SiliconFlow,两者都是 OpenAI 兼容但地址不同,
* 因此单独为 SiliconFlow 创建 OpenAiApi + EmbeddingModel Bean。
*/
@Configuration
public class SiliconFlowEmbeddingConfig {
private static final Logger log = LoggerFactory.getLogger(SiliconFlowEmbeddingConfig.class);
@Value("${siliconflow.api-key}")
private String apiKey;
@Value("${siliconflow.base-url}")
private String baseUrl;
@Value("${siliconflow.embedding.model}")
private String model;
@Bean
public OpenAiApi siliconFlowApi(RestClient.Builder restClientBuilder, WebClient.Builder webClientBuilder) {
log.info("创建 SiliconFlow OpenAiApi: {}", baseUrl);
return OpenAiApi.builder()
.baseUrl(baseUrl)
.apiKey(apiKey)
.restClientBuilder(restClientBuilder)
.build();
}
@Bean
public EmbeddingModel siliconFlowEmbeddingModel(OpenAiApi siliconFlowApi) {
log.info("创建 SiliconFlow EmbeddingModel, model: {}", model);
return new OpenAiEmbeddingModel(siliconFlowApi, MetadataMode.EMBED,
OpenAiEmbeddingOptions.builder()
.model(model)
.build());
}
}
@@ -1,44 +0,0 @@
package com.superbiz.agent.constant;
public class MilvusConstants {
/**
* Milvus 数据库名称
*/
public static final String MILVUS_DB_NAME = "default";
/**
* Default knowledge collection name (dense + BM25).
* Overridable via {@code milvus.collection}.
*/
public static final String MILVUS_COLLECTION_NAME = "biz";
/**
* Alias kept for readability in hybrid-related code.
*/
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
/**
* 向量维度(豆包 embedding 模型的维度)
*/
public static final int VECTOR_DIM = 1024; // 豆包模型返回1024维向量
/**
* ID字段最大长度
*/
public static final int ID_MAX_LENGTH = 256;
/**
* Content字段最大长度
*/
public static final int CONTENT_MAX_LENGTH = 8192;
/**
* 默认分片数
*/
public static final int DEFAULT_SHARD_NUMBER = 2;
private MilvusConstants() {
// 工具类,禁止实例化
}
}
@@ -1,8 +1,8 @@
package com.superbiz.agent.controller; package com.superbiz.agent.controller;
import com.superbiz.agent.client.PyRagClient;
import com.superbiz.agent.config.FileUploadConfig; import com.superbiz.agent.config.FileUploadConfig;
import com.superbiz.agent.dto.FileUploadRes; import com.superbiz.agent.dto.FileUploadRes;
import com.superbiz.agent.service.VectorIndexService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -29,10 +29,11 @@ public class FileUploadController {
private FileUploadConfig fileUploadConfig; private FileUploadConfig fileUploadConfig;
@Autowired @Autowired
private VectorIndexService vectorIndexService; private PyRagClient pyRagClient;
@PostMapping(value = "/api/upload", consumes = "multipart/form-data") @PostMapping(value = "/api/upload", consumes = "multipart/form-data")
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file) { public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file,
@RequestParam(value = "category", required = false) String category) {
if (file.isEmpty()) { if (file.isEmpty()) {
return ResponseEntity.badRequest().body("文件不能为空"); return ResponseEntity.badRequest().body("文件不能为空");
} }
@@ -68,15 +69,17 @@ public class FileUploadController {
logger.info("文件上传成功: {}", filePath); logger.info("文件上传成功: {}", filePath);
// 文件上传成功后,自动调用向量索引服务 // 转发 py-rag 入库(同内容重传返回 unchanged)。入库失败不影响上传成功语义。
try { try {
logger.info("开始为上传文件创建向量索引: {}", filePath); String ingestCategory = (category == null || category.isBlank()) ? "default" : category;
vectorIndexService.indexSingleFile(filePath.toString()); logger.info("开始 py-rag 入库: {}, category={}", filePath, ingestCategory);
logger.info("向量索引创建成功: {}", filePath); var ingest = pyRagClient.ingest(originalFilename, file.getBytes(), file.getContentType(),
ingestCategory, null, null, null);
logger.info("py-rag 入库完成: docId={}, status={}, chunks={}",
ingest.docId(), ingest.status(), ingest.chunkCount());
} catch (Exception e) { } catch (Exception e) {
logger.error("向量索引创建失败: {}, 错误: {}", filePath, e.getMessage(), e); logger.error("py-rag 入库失败: {}, 错误: {}", filePath, e.getMessage(), e);
// 注意:即使索引失败,文件上传仍然成功,只是记录错误日志 // 注意:即使入库失败,文件上传仍然成功,只是记录错误日志
// 可以根据业务需求决定是否要删除文件或返回错误
} }
FileUploadRes response = new FileUploadRes( FileUploadRes response = new FileUploadRes(
@@ -1,147 +0,0 @@
package com.superbiz.agent.controller;
import com.superbiz.agent.service.KnowledgeBaseInitService;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.HashMap;
import java.util.Map;
/**
* 知识库管理控制器
* 提供知识库初始化、查询等接口
*/
@RestController
@RequestMapping("/api/knowledge")
public class KnowledgeBaseController {
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
@Autowired
private KnowledgeBaseInitService initService;
/**
* 初始化知识库
* 扫描 knowledge_base 目录下的所有文档,去重后批量导入到数据库和 Milvus
*
* @param force 是否强制重新导入(跳过去重检查)
* @return 初始化结果
*/
@PostMapping("/init")
public ResponseEntity<?> initKnowledgeBase(@RequestParam(defaultValue = "false") boolean force) {
logger.info("收到知识库初始化请求, force={}", force);
try {
KnowledgeBaseInitService.InitResult result = initService.initializeKnowledgeBase(force);
Map<String, Object> response = new HashMap<>();
response.put("success", true);
response.put("message", "知识库初始化完成");
response.put("scanned", result.getScanned());
response.put("skipped", result.getSkipped());
response.put("inserted", result.getInserted());
response.put("failed", result.getFailed());
response.put("details", result.getDetails());
logger.info("知识库初始化成功: 扫描={}, 跳过={}, 新增={}, 失败={}",
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
return ResponseEntity.ok(response);
} catch (Exception e) {
logger.error("知识库初始化失败", e);
Map<String, Object> response = new HashMap<>();
response.put("success", false);
response.put("message", "初始化失败: " + e.getMessage());
return ResponseEntity.internalServerError().body(response);
}
}
/**
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
* 再从 knowledge_base 全量重建 dense+BM25 索引。
*
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
* 需要显式 confirm=REBUILD。</p>
*/
@PostMapping("/rebuild-hybrid")
public ResponseEntity<?> rebuildHybrid(
@RequestParam(defaultValue = "") String confirm) {
if (!"REBUILD".equals(confirm)) {
Map<String, Object> rejected = new HashMap<>();
rejected.put("success", false);
rejected.put("message", "拒绝执行:请传 confirm=REBUILD 以确认清空并重建");
rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
return ResponseEntity.badRequest().body(rejected);
}
logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
try {
KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
Map<String, Object> response = new HashMap<>();
response.put("success", result.isSuccess());
response.put("message", result.isSuccess()
? "hybrid 知识库重建完成"
: "hybrid 知识库重建结束,但存在失败项");
response.put("collection", result.getCollection());
response.put("basePath", result.getBasePath());
response.put("milvus", result.getMilvus());
response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
if (result.getInit() != null) {
response.put("scanned", result.getInit().getScanned());
response.put("skipped", result.getInit().getSkipped());
response.put("inserted", result.getInit().getInserted());
response.put("failed", result.getInit().getFailed());
response.put("details", result.getInit().getDetails());
}
return result.isSuccess()
? ResponseEntity.ok(response)
: ResponseEntity.status(500).body(response);
} catch (Exception e) {
logger.error("hybrid 知识库重建失败", e);
Map<String, Object> response = new HashMap<>();
response.put("success", false);
response.put("message", "重建失败: " + e.getMessage());
return ResponseEntity.internalServerError().body(response);
}
}
/**
* 查询知识库统计信息
*
* @return 统计信息
*/
@GetMapping("/stats")
public ResponseEntity<?> getStats() {
try {
KnowledgeBaseInitService.Stats stats = initService.getStats();
Map<String, Object> response = new HashMap<>();
response.put("success", true);
response.put("totalDocuments", stats.getTotalDocuments());
response.put("totalVectors", stats.getTotalVectors());
response.put("categories", stats.getCategoryCount());
return ResponseEntity.ok(response);
} catch (Exception e) {
logger.error("查询统计信息失败", e);
Map<String, Object> response = new HashMap<>();
response.put("success", false);
response.put("message", "查询失败: " + e.getMessage());
return ResponseEntity.internalServerError().body(response);
}
}
}
@@ -1,39 +0,0 @@
package com.superbiz.agent.controller;
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
import io.milvus.v2.service.collection.response.ListCollectionsResp;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.HashMap;
import java.util.Map;
/**
* Milvus health check using the single V2 knowledge backend.
*/
@RestController
@RequestMapping("/milvus")
public class MilvusCheckController {
@Autowired
private MilvusHybridKnowledgeStore knowledgeStore;
@GetMapping("/health")
public ResponseEntity<Map<String, Object>> simpleHealth() {
Map<String, Object> result = new HashMap<>();
try {
ListCollectionsResp response = knowledgeStore.client().listCollections();
result.put("message", "ok");
result.put("backend", "milvus-client-v2");
result.put("knowledgeCollection", knowledgeStore.collectionName());
result.put("collections", response == null ? null : response.getCollectionNames());
return ResponseEntity.ok(result);
} catch (Exception e) {
result.put("error", e.getMessage());
return ResponseEntity.status(503).body(result);
}
}
}
@@ -1,42 +0,0 @@
package com.superbiz.agent.controller;
import com.superbiz.agent.dto.Result;
import com.superbiz.agent.service.VectorSearchService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* 文档检索控制器(测试用)
*/
@Slf4j
@RestController
@RequestMapping("/api/search")
public class SearchController {
@Autowired
private VectorSearchService vectorSearchService;
/**
* 搜索相似文档
*/
@GetMapping("/similar")
public Result<List<VectorSearchService.SearchResult>> searchSimilar(
@RequestParam("query") String query,
@RequestParam(value = "topK", defaultValue = "5") int topK,
@RequestParam(value = "category", required = false) String category
) {
try {
log.info("收到检索请求,query: {}, topK: {}, category: {}", query, topK, category);
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(query, topK, category);
log.info("检索完成,返回 {} 条结果", results.size());
return Result.success(results);
} catch (Exception e) {
log.error("检索失败", e);
return Result.error(500, "检索失败: " + e.getMessage());
}
}
}
@@ -1,47 +0,0 @@
package com.superbiz.agent.dto;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 文档分片
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class DocumentChunk {
/**
* 分片内容
*/
private String content;
/**
* 分片在原文档中的起始位置
*/
private int startOffset;
/**
* 分片在原文档中的结束位置
*/
private int endOffset;
/**
* 分片序号(从0开始)
*/
private int chunkIndex;
/**
* 分片标题或上下文信息
*/
private String title;
/**
* 面包屑导航(完整标题层级路径)
* 例如: "故障诊断流程规范 > 应急响应流程 > 1. 初步评估"
*/
private String breadcrumb;
}
@@ -1,80 +0,0 @@
package com.superbiz.agent.dto;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDate;
import java.util.List;
import java.util.Map;
/**
* Frontmatter 数据模型
* 用于解析 Markdown 文件头的 YAML frontmatter
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class Frontmatter {
/**
* 文档标题(必填)
*/
private String title;
/**
* 关键词列表(必填,用于 L0 精确匹配)
*/
private List<String> keywords;
/**
* 文档摘要(必填)
*/
private String summary;
/**
* 文档类别(可选)
*/
private String category;
private String source;
private String breadcrumb;
@JsonProperty("kb_scope")
private String kbScope;
/**
* 章节锚点(预留字段,MVP 不使用)
* Key: 章节标题,Value: 章节 Markdown 标题
*/
private Map<String, String> sections;
/**
* 版本号(预留字段)
*/
private String version;
/**
* 作者(预留字段)
*/
private String author;
/**
* 最后更新日期(预留字段)
*/
private LocalDate lastUpdated;
/**
* 业务场景标签,供 Planner 决策用(LLM 上传时自动生成)
*/
private List<String> covers;
/**
* 文档级检索时机(LLM 上传时自动生成)
*/
private String whenToRetrieve;
}
@@ -1,58 +0,0 @@
package com.superbiz.agent.dto;
import lombok.Builder;
import lombok.Data;
import java.util.List;
import java.util.Map;
/**
* 知识库索引条目
* L0 内存索引使用的数据结构
*/
@Data
@Builder
public class KnowledgeEntry {
/**
* 文件路径(如:knowledge_base/api/payment-errors.md)
*/
private String filePath;
/**
* 文档标题
*/
private String title;
/**
* 关键词列表(用于精确匹配)
*/
private List<String> keywords;
/**
* 文档摘要
*/
private String summary;
/**
* 文档类别(如:api、domain、troubleshooting)
*/
private String category;
private String kbScope;
/**
* 章节锚点(预留字段,MVP 不使用)
*/
private Map<String, String> sections;
/**
* 业务场景标签,供 Planner 决策用
*/
private List<String> covers;
/**
* 文档级检索时机
*/
private String whenToRetrieve;
}
@@ -6,9 +6,10 @@ import lombok.Data;
import java.util.List; import java.util.List;
/** /**
* 检索前 query understanding 的输出(L0 -&gt; pipeline 控制面)。 * 检索 pipeline 控制面参数。
* *
* <p>由 {@code KnowledgeQueryTransformer} 生成,供 L1 过滤、规则 rerank 与 trace 使用。 * <p>L0 query 理解已下沉 py-rag 服务端;当前 {@code originalQuery} = {@code rewrittenQuery}、
* hint 字段恒为空、{@code categoryFilter} 恒为 null,结构保留供后处理与 trace 使用。
* 不是 Agent 可见契约。</p> * 不是 Agent 可见契约。</p>
*/ */
@Data @Data
@@ -1,446 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.config.DocumentChunkConfig;
import com.superbiz.agent.dto.DocumentChunk;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* 文档切片服务(RAG 入库前处理)。
*
* <p>把长 Markdown/文本切成带 title/breadcrumb 的 {@link com.superbiz.agent.dto.DocumentChunk},
* 供 {@link VectorIndexService} 向量化。</p>
*
* <h3>策略摘要</h3>
* <ol>
* <li>先按 Markdown 标题分 section,并维护 breadcrumb 层级</li>
* <li>section 过长再按段落累积;用 token 估算做软边界 / 硬上限</li>
* <li>尽量不在有序/无序列表或未闭合代码块中间切断</li>
* <li>相邻 chunk 保留 overlap,减轻边界语义断裂</li>
* </ol>
*
* <p>检索命中单个 chunk 后,当前主链路不会自动回补同章节相邻 chunk
* (上下文重建仍是后续增强点)。</p>
*/
@Service
public class DocumentChunkService {
private static final Logger logger = LoggerFactory.getLogger(DocumentChunkService.class);
@Autowired
private DocumentChunkConfig chunkConfig;
/**
* 智能分片文档
* 优先按照标题、段落边界进行分割,保持语义完整性
*
* @param content 文档内容
* @param filePath 文件路径(用于日志)
* @return 文档分片列表
*/
public List<DocumentChunk> chunkDocument(String content, String filePath) {
List<DocumentChunk> chunks = new ArrayList<>();
if (content == null || content.trim().isEmpty()) {
logger.warn("文档内容为空: {}", filePath);
return chunks;
}
// 1. 首先尝试按标题分割(Markdown格式)
List<Section> sections = splitByHeadings(content);
// 2. 对每个章节进行进一步分片
int globalChunkIndex = 0;
for (Section section : sections) {
List<DocumentChunk> sectionChunks = chunkSection(section, globalChunkIndex);
chunks.addAll(sectionChunks);
globalChunkIndex += sectionChunks.size();
}
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
return chunks;
}
/**
* 按照 Markdown 标题分割文档,同时构建面包屑层级路径
*/
private List<Section> splitByHeadings(String content) {
List<Section> sections = new ArrayList<>();
// 匹配 Markdown 标题:# 标题, ## 标题, ### 标题等
Pattern headingPattern = Pattern.compile("^(#{1,6})\\s+(.+)$", Pattern.MULTILINE);
Matcher matcher = headingPattern.matcher(content);
// 标题层级栈:维护当前标题的完整路径
List<String> headingStack = new ArrayList<>();
int lastEnd = 0;
String currentBreadcrumb = null;
while (matcher.find()) {
int level = matcher.group(1).length(); // #→1, ##→2, ###→3 ...
String title = matcher.group(2).trim();
// 保存上一个章节
if (lastEnd < matcher.start()) {
String sectionContent = content.substring(lastEnd, matcher.start()).trim();
if (!sectionContent.isEmpty()) {
sections.add(new Section(
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
level,
currentBreadcrumb,
sectionContent,
lastEnd));
}
}
// 维护层级栈:同级别或更高级别 → 弹出,低级 → 追加
while (!headingStack.isEmpty() && headingStack.size() >= level) {
headingStack.remove(headingStack.size() - 1);
}
headingStack.add(title);
currentBreadcrumb = String.join(" > ", headingStack);
lastEnd = matcher.start();
}
// 添加最后一个章节
if (lastEnd < content.length()) {
String sectionContent = content.substring(lastEnd).trim();
if (!sectionContent.isEmpty()) {
sections.add(new Section(
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
headingStack.size(),
currentBreadcrumb,
sectionContent,
lastEnd));
}
}
// 如果没有找到任何标题,将整个文档作为一个章节
if (sections.isEmpty()) {
sections.add(new Section(null, 0, null, content, 0));
}
return sections;
}
/**
* 对单个章节进行分片
* <p>
* 核心改造(Phase 1):
* - Token 估算替代字符计数
* - 感知有序/无序列表结构,不在列表中间切断
* - 软边界(maxTokens)+ 硬上限(maxTokensHard)双重控制
* - 修复 currentStartIndex 漂移:用段落原始位置而非手工推算
*/
private List<DocumentChunk> chunkSection(Section section, int startChunkIndex) {
List<DocumentChunk> chunks = new ArrayList<>();
String content = section.content;
String title = section.title;
String breadcrumb = section.breadcrumb;
// 短章节直接作为一个分片(用 token 估算替代字符数做短路判断)
if (content.length() <= chunkConfig.getMaxSize()
&& estimateTokens(content) <= chunkConfig.getMaxTokens()) {
DocumentChunk chunk = DocumentChunk.builder()
.content(content)
.startOffset(section.startIndex)
.endOffset(section.startIndex + content.length())
.chunkIndex(startChunkIndex)
.title(title)
.breadcrumb(breadcrumb)
.build();
chunks.add(chunk);
return chunks;
}
// 章节内容较长,需要进一步分片
List<String> paragraphs = splitByParagraphs(content);
if (paragraphs.isEmpty()) {
return chunks;
}
// 定位每个段落在 section.content 中的位置(修复 index 漂移)
List<ParagraphPos> paraPositions = locateParagraphPositions(paragraphs, content);
// 当前分片的段落范围
int chunkParaStart = 0; // 当前分片第一个段落的索引(在 paragraphs 中)
StringBuilder buffer = new StringBuilder();
int tokenCount = 0;
int chunkIndex = startChunkIndex;
for (int i = 0; i < paragraphs.size(); i++) {
String paragraph = paragraphs.get(i);
int paraTokens = estimateTokens(paragraph);
// 判断是否需要切分
if (buffer.length() > 0 && tokenCount + paraTokens > chunkConfig.getMaxTokens()) {
// 检查是否处于不可中断的上下文中
if (isInUnbreakableContext(buffer.toString(), paragraph)) {
// 硬上限保护:即使不可中断也不能无限膨胀
if (tokenCount + paraTokens > chunkConfig.getMaxTokensHard()) {
logger.debug(" 触及硬上限 ({} tokens),强制切分", tokenCount + paraTokens);
chunkParaStart = saveChunkAndGetNextStart(
chunks, section, paraPositions,
chunkParaStart, i, title, breadcrumb, chunkIndex);
chunkIndex++;
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
String overlap = getOverlapText(prevChunkContent);
buffer = new StringBuilder(overlap);
tokenCount = estimateTokens(overlap);
}
// 否则:容忍超出(软边界)
} else {
// 安全切点:段落边界
chunkParaStart = saveChunkAndGetNextStart(
chunks, section, paraPositions,
chunkParaStart, i, title, breadcrumb, chunkIndex);
chunkIndex++;
// 新分片以重叠文本开头
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
String overlap = getOverlapText(prevChunkContent);
buffer = new StringBuilder(overlap);
tokenCount = estimateTokens(overlap);
}
}
buffer.append(paragraph).append("\n\n");
tokenCount += paraTokens;
}
// 保存最后一个分片
if (buffer.length() > 0 && chunkParaStart < paragraphs.size()) {
String chunkContent = buffer.toString().trim();
int actualStart = paraPositions.get(chunkParaStart).start;
int actualEnd = paraPositions.get(paragraphs.size() - 1).end;
DocumentChunk chunk = DocumentChunk.builder()
.content(chunkContent)
.startOffset(section.startIndex + actualStart)
.endOffset(section.startIndex + actualEnd)
.chunkIndex(chunkIndex)
.title(title)
.breadcrumb(breadcrumb)
.build();
chunks.add(chunk);
}
return chunks;
}
/**
* 保存当前分块,返回下一个分块的起始段落索引
* <p>
* 从 section.content 中提取原始文本(而非手工拼装),修复 index 漂移问题
*/
private int saveChunkAndGetNextStart(
List<DocumentChunk> chunks,
Section section,
List<ParagraphPos> paraPositions,
int fromPara,
int toPara,
String title,
String breadcrumb,
int chunkIndex) {
int actualStart = paraPositions.get(fromPara).start;
int actualEnd = paraPositions.get(toPara - 1).end;
String originalText = section.content.substring(actualStart, actualEnd);
DocumentChunk chunk = DocumentChunk.builder()
.content(originalText)
.startOffset(section.startIndex + actualStart)
.endOffset(section.startIndex + actualEnd)
.chunkIndex(chunkIndex)
.title(title)
.breadcrumb(breadcrumb)
.build();
chunks.add(chunk);
return toPara; // 下一个分块的起始段落索引
}
/**
* 按段落分割文本
*/
private List<String> splitByParagraphs(String content) {
List<String> paragraphs = new ArrayList<>();
// 按双换行符分割段落
String[] parts = content.split("\n\n+");
for (String part : parts) {
String trimmed = part.trim();
if (!trimmed.isEmpty()) {
paragraphs.add(trimmed);
}
}
return paragraphs;
}
/**
* 定位每个段落在原始文本中的字符偏移
*/
private List<ParagraphPos> locateParagraphPositions(List<String> paragraphs, String sectionContent) {
List<ParagraphPos> positions = new ArrayList<>();
int searchFrom = 0;
for (String p : paragraphs) {
int idx = sectionContent.indexOf(p, searchFrom);
if (idx >= 0) {
positions.add(new ParagraphPos(idx, idx + p.length()));
searchFrom = idx + p.length();
} else {
// fallback: 段落在原文中找不到(不应该发生)
positions.add(new ParagraphPos(searchFrom, searchFrom + p.length()));
searchFrom += p.length();
}
}
return positions;
}
/**
* 启发式 token 估算(无需外部依赖)
* <p>
* 中文(BMP): ~1 字符/token
* 英文/数字/标点: ~4 字符/token
* 空白字符忽略
*/
private int estimateTokens(String text) {
int nonCjkCount = 0;
int cjkCount = 0;
for (char c : text.toCharArray()) {
if (Character.isWhitespace(c)) {
continue;
}
Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
|| block == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS) {
cjkCount++;
} else {
nonCjkCount++;
}
}
return cjkCount + (nonCjkCount + 3) / 4; // 非中文每 4 字符算 1 token,向上取整
}
/**
* 判断当前段落是否属于不可中断的结构
* <p>
* 不可中断结构包括:
* - 有序列表项("1. ", "2. " 格式)
* - 无序列表项("- " 或 "* " 格式)
* - 未闭合的代码块(``` 内)
*/
private boolean isInUnbreakableContext(String buffer, String nextParagraph) {
// 有序列表:判断 buffer 末尾和下一段是否都是列表项
if (nextParagraph.matches("^\\d{1,2}\\.\\s.*")) {
String lastLine = getLastNonEmptyLine(buffer);
if (lastLine != null && lastLine.matches("^\\d{1,2}\\.\\s.*")) {
return true;
}
}
// 无序列表:"- " 或 "* " 格式
if (nextParagraph.matches("^[-*]\\s.*")) {
String lastLine = getLastNonEmptyLine(buffer);
if (lastLine != null && lastLine.matches("^[-*]\\s.*")) {
return true;
}
}
// 代码块:``` 未闭合
if (buffer.contains("```")) {
int count = 0;
for (int i = 0; i <= buffer.length() - 3; i++) {
if (buffer.substring(i).startsWith("```")) {
count++;
i += 2;
}
}
if (count % 2 == 1) {
return true; // 奇数个 ``` → 在代码块内部
}
}
return false;
}
/**
* 获取 buffer 中最后一行非空白文本
*/
private String getLastNonEmptyLine(String buffer) {
String[] lines = buffer.split("\n");
for (int i = lines.length - 1; i >= 0; i--) {
String line = lines[i].trim();
if (!line.isEmpty()) {
return line;
}
}
return null;
}
/**
* 获取重叠文本
* 从文本末尾提取指定长度的内容作为下一个分片的开头
*/
private String getOverlapText(String text) {
int overlapSize = Math.min(chunkConfig.getOverlap(), text.length());
if (overlapSize <= 0) {
return "";
}
// 从末尾提取重叠内容
String overlap = text.substring(text.length() - overlapSize);
// 尝试在句子边界截断(查找最后一个句号、问号、感叹号)
int lastSentenceEnd = Math.max(
overlap.lastIndexOf('。'),
Math.max(overlap.lastIndexOf('?'), overlap.lastIndexOf('!'))
);
if (lastSentenceEnd > overlapSize / 2) {
return overlap.substring(lastSentenceEnd + 1).trim();
}
return overlap.trim();
}
/**
* 段落在原文中的位置
*/
private static class ParagraphPos {
final int start;
final int end;
ParagraphPos(int start, int end) {
this.start = start;
this.end = end;
}
}
/**
* 章节数据类
*/
private static class Section {
String title; // 最近一级标题名称
int level; // 标题级别(1-6),0=无标题
String breadcrumb; // 完整面包屑路径
String content; // 章节内容
int startIndex; // 在原文中的起始偏移
Section(String title, int level, String breadcrumb, String content, int startIndex) {
this.title = title;
this.level = level;
this.breadcrumb = breadcrumb;
this.content = content;
this.startIndex = startIndex;
}
}
}
@@ -1,135 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.KnowledgeEntry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* 文档字段补全服务
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
*/
@Slf4j
@Service
public class DocumentFieldEnricher {
@Autowired
private ChatModel chatModel;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
private String promptTemplate;
@PostConstruct
public void init() {
try {
promptTemplate = new String(
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
log.info("DocumentFieldEnricher prompt 加载成功");
} catch (IOException e) {
log.error("加载 doc-field-enricher-prompt.md 失败", e);
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
}
}
public void enrich(Frontmatter frontmatter, String bodyText) {
enrich(frontmatter, bodyText, null);
}
/**
* 为 Frontmatter 补全 covers 和 whenToRetrieve
* 若已有值则跳过;LLM 失败时降级,不阻断主流程
*
* @param frontmatter 待补全的 frontmatter
* @param bodyText 文档正文
* @param category 文档所属域(用于查找同域其他文档)
*/
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
if (frontmatter == null) return;
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
if (!needsCovers && !needsWhen) {
log.debug("covers 和 whenToRetrieve 已存在,跳过 LLM 生成");
return;
}
try {
String snippet = bodyText != null && bodyText.length() > 1000
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
String promptText = String.format(promptTemplate,
frontmatter.getTitle(),
frontmatter.getSummary(),
sameDomainDocs,
snippet);
String response = chatModel.call(new Prompt(promptText))
.getResult().getOutput().getText();
// 提取 JSON 部分(防止模型输出多余文本)
String json = extractJson(response);
JsonNode node = objectMapper.readTree(json);
if (needsCovers && node.has("covers")) {
List<String> covers = new ArrayList<>();
node.get("covers").forEach(n -> covers.add(n.asText()));
frontmatter.setCovers(covers);
log.debug("LLM 生成 covers: {}", covers);
}
if (needsWhen && node.has("whenToRetrieve")) {
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
}
} catch (Exception e) {
log.warn("LLM 生成文档字段失败,降级处理: title={}", frontmatter.getTitle(), e);
if (needsCovers) frontmatter.setCovers(List.of());
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
}
}
private String extractJson(String text) {
if (text == null) return "{}";
int start = text.indexOf('{');
int end = text.lastIndexOf('}');
if (start == -1 || end == -1 || end <= start) return "{}";
return text.substring(start, end + 1);
}
/**
* 构建同域其他文档标题列表(供 LLM 做排除判断)
*/
private String buildSameDomainDocs(String currentTitle, String category) {
if (category == null || category.isBlank()) return "(无同域文档信息)";
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
.filter(e -> category.equals(e.getCategory()))
.map(KnowledgeEntry::getTitle)
.filter(t -> t != null && !t.equals(currentTitle))
.collect(Collectors.toList());
if (otherTitles.isEmpty()) return "(无同域其他文档)";
return String.join("、", otherTitles);
}
}
@@ -1,13 +1,13 @@
package com.superbiz.agent.service; package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.client.PyRagClient;
import com.superbiz.agent.client.PyRagClient.PyRagIngestResponse;
import com.superbiz.agent.client.PyRagClientException;
import com.superbiz.agent.domain.entity.ApiDocument; import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.domain.enums.FaultCategory; import com.superbiz.agent.domain.enums.FaultCategory;
import com.superbiz.agent.dto.DocumentChunk;
import com.superbiz.agent.dto.DocumentQueryResponse; import com.superbiz.agent.dto.DocumentQueryResponse;
import com.superbiz.agent.dto.DocumentUploadRequest; import com.superbiz.agent.dto.DocumentUploadRequest;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.exception.DocumentProcessException; import com.superbiz.agent.exception.DocumentProcessException;
import com.superbiz.agent.repository.ApiDocumentRepository; import com.superbiz.agent.repository.ApiDocumentRepository;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -27,11 +27,13 @@ import java.security.MessageDigest;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
* 文档管理服务 * 文档管理服务。
*
* <p>负责 MySQL 元数据({@link ApiDocument})、本地原件保存与业务查询/删除;
* 文档解析、frontmatter 校验、分块与向量索引全部由 py-rag 服务端 ingest 完成。</p>
*/ */
@Slf4j @Slf4j
@Service @Service
@@ -41,34 +43,19 @@ public class DocumentManagementService {
private String knowledgeBasePath; private String knowledgeBasePath;
@Autowired @Autowired
private TextExtractorService textExtractorService; private PyRagClient pyRagClient;
@Autowired
private DocumentChunkService documentChunkService;
@Autowired
private VectorIndexService vectorIndexService;
@Autowired @Autowired
private ApiDocumentRepository apiDocumentRepository; private ApiDocumentRepository apiDocumentRepository;
@Autowired
private FrontmatterParser frontmatterParser;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
@Autowired
private DocumentFieldEnricher documentFieldEnricher;
@Autowired
private KnowledgeDomainService knowledgeDomainService;
@Autowired @Autowired
private ObjectMapper objectMapper; private ObjectMapper objectMapper;
/** /**
* 上传文档 * 上传文档。
*
* <p>流程:hash 去重 → 本地原件保存 → py-rag ingest(同步,服务端解析/分块/索引)→
* MySQL 元数据落库。docId 取 py-rag 返回的 doc_id,与检索 evidence_key 的 docId 段对齐。</p>
* *
* @param request 上传请求 * @param request 上传请求
* @return 文档ID * @return 文档ID
@@ -83,15 +70,7 @@ public class DocumentManagementService {
log.info("开始上传文档,文件名: {}, 大小: {} bytes", fileName, file.getSize()); log.info("开始上传文档,文件名: {}, 大小: {} bytes", fileName, file.getSize());
try { try {
// 1. 验证文件格式 // 1. 计算文件 hash(去重)
if (!textExtractorService.isSupportedFormat(fileName)) {
throw new DocumentProcessException(
fileName, "upload",
"不支持的文件格式,仅支持 .md 和 .txt"
);
}
// 2. 计算文件 hash(去重)
long hashStart = System.currentTimeMillis(); long hashStart = System.currentTimeMillis();
String fileHash = calculateFileHash(file); String fileHash = calculateFileHash(file);
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart); log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
@@ -105,16 +84,7 @@ public class DocumentManagementService {
); );
} }
// 3. 提取文本 // 2. category 缺省处理 + 保存原始文件到本地
long extractStart = System.currentTimeMillis();
String text = textExtractorService.extractText(file, fileName);
log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart);
if (text == null || text.isBlank()) {
throw new DocumentProcessException(fileName, "upload", "文档内容为空");
}
// 4. 保存原始文件到本地
String category = request.getCategory(); String category = request.getCategory();
if (category == null || category.isBlank()) { if (category == null || category.isBlank()) {
category = "default"; category = "default";
@@ -123,47 +93,33 @@ public class DocumentManagementService {
localPath = saveToLocal(file, fileName, category); localPath = saveToLocal(file, fileName, category);
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart); log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
// 5. 解析 frontmatter // 3. py-rag 入库(格式校验/frontmatter/分块/向量索引都在服务端;同内容重传返回 unchanged)
long frontmatterStart = System.currentTimeMillis(); long ingestStart = System.currentTimeMillis();
Frontmatter frontmatter = null; PyRagIngestResponse ingest;
String bodyText = text; try {
if (frontmatterParser.hasFrontmatter(text)) { ingest = pyRagClient.ingest(fileName, file.getBytes(), file.getContentType(),
frontmatter = frontmatterParser.parse(text); category, null, null, null);
if (frontmatter != null) { } catch (PyRagClientException | IOException e) {
// LLM 补全 covers / whenToRetrieve(已有值则跳过) throw new DocumentProcessException(
bodyText = frontmatterParser.stripFrontmatter(text); fileName, "ingest", "py-rag 入库失败: " + e.getMessage(), e
documentFieldEnricher.enrich(frontmatter, bodyText, category); );
log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
} else {
log.warn("frontmatter解析失败,文件名: {}", fileName);
}
} else {
log.debug("文件不包含frontmatter: {}", fileName);
} }
log.info("py-rag 入库完成: docId={}, status={}, chunks={}, time={}ms",
ingest.docId(), ingest.status(), ingest.chunkCount(),
System.currentTimeMillis() - ingestStart);
// 6. 分块 // 4. 保存文档元数据
long chunkStart = System.currentTimeMillis();
List<DocumentChunk> chunks = documentChunkService.chunkDocument(bodyText, fileName);
if (chunks.isEmpty()) {
throw new DocumentProcessException(fileName, "upload", "文档分块失败");
}
log.info("文档分块完成: fileName={}, chunks={}, time={}ms",
fileName, chunks.size(), System.currentTimeMillis() - chunkStart);
// 7. 创建文档元数据
String docId = resolveDocumentId(frontmatter);
String metadataJson = null; String metadataJson = null;
if (frontmatter != null) { if (ingest.frontmatter() != null) {
try { try {
metadataJson = objectMapper.writeValueAsString(frontmatter); metadataJson = objectMapper.writeValueAsString(ingest.frontmatter());
} catch (Exception e) { } catch (Exception e) {
log.warn("Frontmatter序列化失败", e); log.warn("frontmatter 序列化失败", e);
} }
} }
ApiDocument document = ApiDocument.builder() ApiDocument document = ApiDocument.builder()
.docId(docId) .docId(ingest.docId())
.fileName(fileName) .fileName(fileName)
.filePath(localPath) .filePath(localPath)
.metadata(metadataJson) .metadata(metadataJson)
@@ -173,56 +129,17 @@ public class DocumentManagementService {
.version(request.getVersion()) .version(request.getVersion())
.fileSize(file.getSize()) .fileSize(file.getSize())
.fileHash(fileHash) .fileHash(fileHash)
.status("PROCESSING") .status("INDEXED")
.chunkCount(chunks.size()) .chunkCount(ingest.chunkCount())
.build(); .build();
document.setIndexedAt(LocalDateTime.now());
apiDocumentRepository.save(document); apiDocumentRepository.save(document);
log.info("文档元数据已保存: docId={}", docId); log.info("文档元数据已保存: docId={}", document.getDocId());
// 8. 向量化并索引 log.info("文档上传完成: docId={}, fileName={}, ingestStatus={}, totalTime={}ms",
try { document.getDocId(), fileName, ingest.status(), System.currentTimeMillis() - startTime);
long vectorStart = System.currentTimeMillis();
vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter);
document.setStatus("INDEXED");
document.setIndexedAt(LocalDateTime.now());
apiDocumentRepository.save(document);
log.info("文档向量索引完成: docId={}, category={}, time={}ms",
docId, category, System.currentTimeMillis() - vectorStart);
} catch (Exception e) { return document.getDocId();
log.error("文档索引失败: docId={}", docId, e);
document.setStatus("FAILED");
apiDocumentRepository.save(document);
throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
}
// 9. 更新 L0 索引
if (frontmatter != null) {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath(localPath)
.title(frontmatter.getTitle())
.keywords(frontmatter.getKeywords())
.summary(frontmatter.getSummary())
.category(category)
.kbScope(frontmatter.getKbScope())
.sections(frontmatter.getSections())
.covers(frontmatter.getCovers())
.whenToRetrieve(frontmatter.getWhenToRetrieve())
.build();
knowledgeIndexService.addToIndex(entry);
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
}
// 触发域级聚合重算
knowledgeDomainService.onDocumentChange(category);
long totalTime = System.currentTimeMillis() - startTime;
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
docId, fileName, frontmatter != null, totalTime);
return docId;
} catch (Exception e) { } catch (Exception e) {
// 失败时清理本地文件 // 失败时清理本地文件
@@ -314,16 +231,6 @@ public class DocumentManagementService {
} }
} }
private String resolveDocumentId(Frontmatter frontmatter) {
if (frontmatter != null && frontmatter.getSource() != null) {
String source = frontmatter.getSource().trim();
if (!source.isEmpty() && source.length() <= 64) {
return source;
}
}
return UUID.randomUUID().toString();
}
/** /**
* 根据 docId 查询文档 * 根据 docId 查询文档
*/ */
@@ -358,7 +265,10 @@ public class DocumentManagementService {
} }
/** /**
* 删除文档 * 删除文档(仅 MySQL 元数据与本地原件)。
*
* <p>py-rag v1 契约没有单文档删除端点:已入库内容需在其服务端
* 全量重建({@code /api/v1/collections:rebuild})后才会从知识库消失。</p>
*/ */
@Transactional @Transactional
public void deleteDocument(String docId) { public void deleteDocument(String docId) {
@@ -379,47 +289,9 @@ public class DocumentManagementService {
} }
} }
// 删除 L0 索引 // 删除元数据(py-rag 侧索引留存,重建后失效)
if (doc.getFilePath() != null) {
knowledgeIndexService.removeFromIndex(doc.getFilePath());
}
// 删除向量索引
try {
vectorIndexService.deleteDocumentChunks(docId);
log.info("文档向量索引已删除,docId: {}", docId);
} catch (Exception e) {
log.warn("删除向量索引失败,docId: {}", docId, e);
}
// 删除元数据
apiDocumentRepository.delete(doc); apiDocumentRepository.delete(doc);
log.info("文档已删除,docId: {}", docId); log.info("文档已删除,docId={}(py-rag 侧需全量重建后生效)", docId);
// 触发域级聚合重算
String category = doc.getFilePath() != null
? resolveCategory(doc.getFilePath()) : null;
if (category != null) {
knowledgeDomainService.onDocumentChange(category);
}
}
/**
* 转换为响应 DTO
*/
/**
* 从 filePath 解析 category(取 knowledge_base/{category}/... 中的 category 段)
*/
private String resolveCategory(String filePath) {
try {
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
// filePath 形如 knowledge_base/payment/xxx.md,取倒数第二段
int nameCount = p.getNameCount();
if (nameCount >= 2) {
return p.getName(nameCount - 2).toString();
}
} catch (Exception ignored) {}
return null;
} }
private Path resolveLocalPath(String filePath) { private Path resolveLocalPath(String filePath) {
@@ -1,160 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.Frontmatter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.yaml.snakeyaml.Yaml;
import java.util.Map;
/**
* Frontmatter 解析器
* 解析 Markdown 文件头的 YAML frontmatter
*/
@Slf4j
@Service
public class FrontmatterParser {
private final Yaml yaml = new Yaml();
/**
* 检查文件是否包含 frontmatter
*
* @param content 文件内容
* @return true 如果包含 frontmatter
*/
public boolean hasFrontmatter(String content) {
if (content == null || content.isEmpty()) {
return false;
}
return content.trim().startsWith("---");
}
/**
* 解析 Markdown frontmatter
*
* @param content 完整文件内容
* @return Frontmatter 对象,如果不存在或解析失败返回 null
*/
public Frontmatter parse(String content) {
if (!hasFrontmatter(content)) {
return null;
}
try {
// 1. 提取 frontmatter 部分(两个 --- 之间)
String frontmatterText = extractFrontmatter(content);
if (frontmatterText == null) {
log.warn("未找到有效的 frontmatter 结束标记");
return null;
}
// 2. 使用 SnakeYAML 解析
Map<String, Object> map = yaml.load(frontmatterText);
if (map == null || map.isEmpty()) {
log.warn("Frontmatter 解析结果为空");
return null;
}
// 3. 映射到 Frontmatter 对象
Frontmatter frontmatter = Frontmatter.builder()
.title((String) map.get("title"))
.keywords((java.util.List<String>) map.get("keywords"))
.summary((String) map.get("summary"))
.category((String) map.get("category"))
.source((String) map.get("source"))
.breadcrumb((String) map.get("breadcrumb"))
.kbScope(firstString(map, "kb_scope", "kbScope"))
.sections((Map<String, String>) map.get("sections"))
.version((String) map.get("version"))
.author((String) map.get("author"))
.covers((java.util.List<String>) map.get("covers"))
.whenToRetrieve((String) map.get("when_to_retrieve"))
.build();
// 4. 验证必填字段
if (frontmatter.getTitle() == null || frontmatter.getKeywords() == null ||
frontmatter.getSummary() == null) {
log.warn("Frontmatter 缺少必填字段: title={}, keywords={}, summary={}",
frontmatter.getTitle(), frontmatter.getKeywords(), frontmatter.getSummary());
return null;
}
log.debug("Frontmatter 解析成功: title={}, keywords=",
frontmatter.getTitle(), frontmatter.getKeywords());
return frontmatter;
} catch (Exception e) {
log.warn("Frontmatter 解析失败", e);
return null;
}
}
public String stripFrontmatter(String content) {
if (!hasFrontmatter(content)) {
return content;
}
String trimmed = content.trim();
int secondDelimiter = trimmed.indexOf("\n---", 3);
int delimiterLength = 4;
if (secondDelimiter == -1) {
secondDelimiter = trimmed.indexOf("\r\n---", 3);
delimiterLength = 5;
}
if (secondDelimiter == -1) {
return content;
}
int bodyStart = secondDelimiter + delimiterLength;
if (bodyStart < trimmed.length()) {
char next = trimmed.charAt(bodyStart);
if (next == '\r') {
bodyStart++;
}
if (bodyStart < trimmed.length() && trimmed.charAt(bodyStart) == '\n') {
bodyStart++;
}
}
return trimmed.substring(Math.min(bodyStart, trimmed.length())).stripLeading();
}
/**
* 提取 frontmatter 文本(两个 --- 之间的内容)
*
* @param content 完整文件内容
* @return frontmatter 文本,如果格式错误返回 null
*/
private String extractFrontmatter(String content) {
// 去除开头的空白
content = content.trim();
// 检查是否以 --- 开头
if (!content.startsWith("---")) {
return null;
}
// 查找第二个 ---(结束标记)
int secondDelimiter = content.indexOf("\n---", 3);
if (secondDelimiter == -1) {
// 尝试查找 Windows 风格换行
secondDelimiter = content.indexOf("\r\n---", 3);
if (secondDelimiter == -1) {
return null;
}
}
// 提取 frontmatter(不包含 --- 标记)
return content.substring(3, secondDelimiter).trim();
}
private String firstString(Map<String, Object> map, String... keys) {
for (String key : keys) {
Object value = map.get(key);
if (value instanceof String text && !text.isBlank()) {
return text;
}
}
return null;
}
}
@@ -1,407 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.domain.enums.FaultCategory;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.DocumentChunk;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.LocalDateTime;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Collectors;
/**
* 知识库初始化服务
* 负责批量导入 knowledge_base 目录下的文档到数据库和 Milvus
*/
@Service
public class KnowledgeBaseInitService {
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseInitService.class);
@Value("${knowledge.base-path:knowledge_base}")
private String knowledgeBasePath;
@Autowired
private ApiDocumentRepository apiDocumentRepository;
@Autowired
private FrontmatterParser frontmatterParser;
@Autowired
private DocumentChunkService documentChunkService;
@Autowired
private VectorIndexService vectorIndexService;
@Autowired
private VectorEmbeddingService vectorEmbeddingService;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
@Autowired
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
/**
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
*/
@Transactional(rollbackFor = Exception.class)
public RebuildResult rebuildHybridFromKnowledgeBase() {
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
RebuildResult rebuild = new RebuildResult();
rebuild.setBasePath(knowledgeBasePath);
rebuild.setCollection(hybridKnowledgeStore.collectionName());
long mysqlBefore = apiDocumentRepository.count();
rebuild.setMysqlDocumentsBefore(mysqlBefore);
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
rebuild.setMilvus(milvus);
apiDocumentRepository.deleteAll();
apiDocumentRepository.flush();
knowledgeIndexService.clearIndex();
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
InitResult init = initializeKnowledgeBase(true);
rebuild.setInit(init);
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
// Success when at least one doc indexed and no hard failures.
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
rebuild.isSuccess(), init.getInserted(), init.getFailed());
return rebuild;
}
/**
* 初始化知识库
*
* @param force 是否强制重新导入(跳过去重检查)
* @return 初始化结果
*/
@Transactional(rollbackFor = Exception.class)
public InitResult initializeKnowledgeBase(boolean force) {
logger.info("开始初始化知识库: basePath={}, force={}", knowledgeBasePath, force);
InitResult result = new InitResult();
Path baseDir = Paths.get(knowledgeBasePath);
if (!Files.exists(baseDir)) {
logger.error("知识库目录不存在: {}", knowledgeBasePath);
throw new RuntimeException("知识库目录不存在: " + knowledgeBasePath);
}
// 1. 扫描所有 Markdown 文件
List<Path> markdownFiles = scanMarkdownFiles(baseDir);
result.setScanned(markdownFiles.size());
logger.info("扫描到 {} 个 Markdown 文件", markdownFiles.size());
// 2. 如果非强制模式,获取已存在的文档(用于去重)
Set<String> existingFilePaths = new HashSet<>();
if (!force) {
existingFilePaths = apiDocumentRepository.findAll().stream()
.map(ApiDocument::getFilePath)
.collect(Collectors.toSet());
logger.info("已存在 个文档记录", existingFilePaths.size());
}
// 3. 逐个处理文档
for (Path file : markdownFiles) {
String relativePath = baseDir.relativize(file).toString().replace("\\", "/");
try {
// 去重检查
if (!force && existingFilePaths.contains(relativePath)) {
logger.debug("跳过已存在的文档: {}", relativePath);
result.incrementSkipped();
result.addDetail(relativePath, "已存在,跳过");
continue;
}
// 解析文档
String content = Files.readString(file);
Frontmatter frontmatter = frontmatterParser.parse(content);
if (frontmatter == null) {
logger.warn("文档格式无效: {}, frontmatter 解析失败", relativePath);
result.incrementFailed();
result.addDetail(relativePath, "格式无效: frontmatter 解析失败");
continue;
}
// 提取字段
String title = frontmatter.getTitle();
String summary = frontmatter.getSummary();
String category = frontmatter.getCategory() != null ? frontmatter.getCategory() : "general";
List<String> keywords = frontmatter.getKeywords();
if (title == null || title.isBlank()) {
logger.warn("文档缺少标题: {}", relativePath);
result.incrementFailed();
result.addDetail(relativePath, "缺少标题");
continue;
}
// 保存到数据库
ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
// 提取文档正文(去除 frontmatter)
String body = extractBody(content);
// 文档分块
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
// 上传到 Milvus hybrid collection(dense + BM25 search_text)
try {
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
document.setStatus("INDEXED");
document.setChunkCount(chunks.size());
document.setIndexedAt(LocalDateTime.now());
apiDocumentRepository.save(document);
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
title, document.getDocId(), chunks.size());
} catch (Exception e) {
logger.error("上传到 Milvus 失败: {}", relativePath, e);
document.setStatus("FAILED");
document.setErrorMessage(e.getMessage());
apiDocumentRepository.save(document);
result.incrementFailed();
result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
continue; // 跳过该文档,继续处理下一个
}
// 添加到 L0 内存索引
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath(relativePath)
.title(title)
.keywords(keywords)
.summary(summary)
.category(category)
.kbScope(frontmatter.getKbScope())
.build();
knowledgeIndexService.addToIndex(entry);
result.incrementInserted();
result.addDetail(relativePath, "导入成功(L0+L1)");
logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
} catch (Exception e) {
logger.error("处理文档失败: {}", relativePath, e);
result.incrementFailed();
result.addDetail(relativePath, "处理失败: " + e.getMessage());
}
}
logger.info("知识库初始化完成: 扫描={}, 跳过={}, 新增={}, 失败={}",
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
return result;
}
/**
* 获取知识库统计信息
*/
public Stats getStats() {
Stats stats = new Stats();
// 数据库中的文档数量
long totalDocuments = apiDocumentRepository.count();
stats.setTotalDocuments(totalDocuments);
// L0 索引中的文档数量
int indexSize = knowledgeIndexService.getIndexSize();
logger.debug("L0 索引大小: {}", indexSize);
// 按分类统计(从 fault_category 字段读取)
Map<String, Long> categoryCount = apiDocumentRepository.findAll().stream()
.collect(Collectors.groupingBy(
doc -> doc.getFaultCategory() != null ? doc.getFaultCategory().name() : "GENERAL",
Collectors.counting()
));
stats.setCategoryCount(categoryCount);
// Milvus 中的向量数量(需要实现)
// TODO: 查询 Milvus collection 的实体数量
stats.setTotalVectors(0L);
return stats;
}
/**
* 扫描目录下所有 Markdown 文件
*/
private List<Path> scanMarkdownFiles(Path baseDir) {
List<Path> files = new ArrayList<>();
try {
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
String name = file.getFileName() == null ? "" : file.getFileName().toString();
// Import content docs only; skip README/index markdown without frontmatter.
if (name.endsWith(".md")
&& !name.equalsIgnoreCase("README.md")
&& !name.equalsIgnoreCase("readme.md")) {
files.add(file);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
logger.warn("访问文件失败: {}", file, exc);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
logger.error("扫描目录失败: {}", baseDir, e);
throw new RuntimeException("扫描目录失败", e);
}
return files;
}
/**
* 保存文档到数据库
*/
private ApiDocument saveToDatabase(String filePath, String title, String summary,
String category, String content, List<String> keywords) {
ApiDocument document = new ApiDocument();
document.setDocId(UUID.randomUUID().toString());
document.setFileName(Paths.get(filePath).getFileName().toString());
document.setFilePath(filePath);
document.setApiName(title); // 使用 title 作为 apiName
document.setStatus("PENDING"); // 初始状态为 PENDING,索引成功后更新为 INDEXED
// 映射 category 到 FaultCategory 枚举
FaultCategory faultCategory = FaultCategory.fromString(category);
document.setFaultCategory(faultCategory);
// 将 frontmatter 信息保存到 metadata(JSON 格式)
String metadataJson = String.format(
"{\"title\":\"%s\",\"summary\":\"%s\",\"category\":\"%s\",\"keywords\":%s}",
escapeJson(title),
escapeJson(summary),
escapeJson(category),
"[\"" + String.join("\",\"", keywords.stream().map(this::escapeJson).toArray(String[]::new)) + "\"]"
);
document.setMetadata(metadataJson);
document.setFileSize((long) content.length());
return apiDocumentRepository.save(document);
}
/**
* JSON 转义
*/
private String escapeJson(String str) {
if (str == null) {
return "";
}
return str.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r");
}
/**
* 提取文档正文(去除 frontmatter)
*/
private String extractBody(String content) {
if (!content.trim().startsWith("---")) {
return content;
}
int firstEnd = content.indexOf("---", 3);
if (firstEnd == -1) {
return content;
}
int secondEnd = content.indexOf("---", firstEnd + 3);
if (secondEnd == -1) {
return content.substring(firstEnd + 3).trim();
}
return content.substring(secondEnd + 3).trim();
}
// ==================== 数据模型 ====================
/**
* 初始化结果
*/
@Data
public static class InitResult {
private int scanned; // 扫描到的文件数量
private int skipped; // 跳过的文件数量(已存在)
private int inserted; // 成功导入的文件数量
private int failed; // 失败的文件数量
private Map<String, String> details = new LinkedHashMap<>(); // 详细信息
public void incrementSkipped() {
this.skipped++;
}
public void incrementInserted() {
this.inserted++;
}
public void incrementFailed() {
this.failed++;
}
public void addDetail(String filePath, String message) {
this.details.put(filePath, message);
}
}
/**
* 统计信息
*/
@Data
public static class Stats {
private long totalDocuments; // 数据库中的文档总数
private long totalVectors; // Milvus 中的向量总数
private Map<String, Long> categoryCount; // 按分类统计
}
/**
* Full hybrid rebuild result.
*/
@Data
public static class RebuildResult {
private boolean success;
private String basePath;
private String collection;
private long mysqlDocumentsBefore;
private long mysqlDocumentsAfterClear;
private long mysqlDocumentsAfterInit;
private int l0IndexSizeAfterClear;
private int l0IndexSizeAfterInit;
private Map<String, Object> milvus;
private InitResult init;
}
}
@@ -1,188 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.KnowledgeDomain;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.repository.KnowledgeDomainRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* 知识域服务
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
*/
@Slf4j
@Service
public class KnowledgeDomainService {
@Autowired
private KnowledgeDomainRepository knowledgeDomainRepository;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
@Autowired
private ChatModel chatModel;
@Autowired
private ObjectMapper objectMapper;
private String domainPromptTemplate;
@PostConstruct
public void init() {
try {
domainPromptTemplate = new String(
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
log.info("KnowledgeDomainService prompt 加载成功");
} catch (IOException e) {
log.error("加载 domain-summary-prompt.md 失败", e);
throw new RuntimeException("Failed to load domain-summary prompt", e);
}
}
/**
* 文档变更后重算指定域的 when_to_retrieve
*/
public void onDocumentChange(String category) {
if (category == null || category.isBlank()) return;
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
.filter(e -> category.equals(e.getCategory()))
.collect(Collectors.toList());
buildDomainSummary(category, entries);
}
/**
* 聚合同域文档,调用 LLM 生成域级摘要,写入 DB
*/
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
if (entries.isEmpty()) {
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
d.setDocumentCount(0);
knowledgeDomainRepository.save(d);
});
return;
}
// 构建文档列表描述
StringBuilder docList = new StringBuilder();
for (KnowledgeEntry entry : entries) {
docList.append("- 文档:").append(entry.getTitle()).append("\n");
if (entry.getWhenToRetrieve() != null) {
docList.append(" 适用场景:").append(entry.getWhenToRetrieve()).append("\n");
}
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
docList.append(" 覆盖:").append(String.join("、", entry.getCovers())).append("\n");
}
}
String description = entries.stream()
.map(KnowledgeEntry::getSummary)
.filter(s -> s != null && !s.isBlank())
.findFirst().orElse(category);
String whenToRetrieve = null;
try {
String otherDomainsInfo = buildOtherDomainsInfo(category);
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
whenToRetrieve = chatModel.call(new Prompt(promptText))
.getResult().getOutput().getText();
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
} catch (Exception e) {
log.warn("LLM 生成域级 when_to_retrieve 失败,保留旧值: domain={}", category, e);
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
}
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
.orElse(KnowledgeDomain.builder().domainId(category).build());
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
domain.setWhenToRetrieve(whenToRetrieve);
domain.setDocumentCount(entries.size());
knowledgeDomainRepository.save(domain);
}
/**
* 构建注入 Planner 的 knowledge map YAML 文本
*/
public String buildKnowledgeMap() {
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
if (domains.isEmpty()) return "";
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
.filter(e -> e.getCategory() != null)
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
for (KnowledgeDomain domain : domains) {
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
if (domain.getDescription() != null) {
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
}
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
yaml.append(" when_to_retrieve: \"")
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
}
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
if (!domainEntries.isEmpty()) {
yaml.append(" documents:\n");
for (KnowledgeEntry entry : domainEntries) {
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
}
}
}
}
return yaml.toString();
}
/**
* 构建其他域的摘要信息(用于 LLM 域级 prompt 的边界判断)
* 优先使用其他域的 when_to_retrieve(边界信号),而非 description
*/
private String buildOtherDomainsInfo(String currentCategory) {
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
StringBuilder sb = new StringBuilder();
for (KnowledgeDomain d : allDomains) {
if (d.getDomainId().equals(currentCategory)) continue;
sb.append("- ").append(d.getDomainId());
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
sb.append(":").append(d.getWhenToRetrieve());
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
sb.append("(").append(d.getDescription()).append(")");
}
sb.append("\n");
}
// 如果 DB 里还没有其他域的记录(首次启动),从 L0 索引补充
if (sb.isEmpty()) {
knowledgeIndexService.getAllEntries().stream()
.map(KnowledgeEntry::getCategory)
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
.distinct()
.forEach(c -> sb.append("- ").append(c).append("\n"));
}
return sb.isEmpty() ? "(无其他域信息)" : sb.toString();
}
}
@@ -1,341 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.repository.KnowledgeDomainRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* L0 知识索引服务(关键词 / domain hint,不是向量库)。
*
* <h3>定位</h3>
* 从 MySQL {@code api_document.metadata}(frontmatter)加载文档级关键词与 category,
* 供检索前 query understanding 使用。L0 输出只作为:
* <ul>
* <li>可选 category filter(唯一 domain 时)</li>
* <li>rerank 的 domain/keyword/entity boost 信号</li>
* <li>trace 可解释信息</li>
* </ul>
* <b>L0 命中文档不会直接当作事实 evidence</b>;证据正文只来自 L1 向量召回。
*
* <h3>匹配方式(当前较粗)</h3>
* {@code query.contains(keyword) || keyword.contains(query)},大小写不敏感。
* 没有分词、别名归一或停用词;短词/泛词可能误命中。
*/
@Slf4j
@Service
public class KnowledgeIndexService {
@Value("${knowledge.base-path:knowledge_base}")
private String knowledgeBasePath;
@Value("${retrieval.kb-scope:}")
private String kbScope = "";
@Autowired
private ApiDocumentRepository apiDocumentRepository;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private KnowledgeDomainRepository knowledgeDomainRepository;
@Lazy
@Autowired
private KnowledgeDomainService knowledgeDomainService;
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
@PostConstruct
public void loadIndex() {
log.info("开始从数据库加载知识库索引");
try {
List<ApiDocument> documents = apiDocumentRepository.findAll();
int loaded = 0;
for (ApiDocument doc : documents) {
try {
KnowledgeEntry entry = parseDocumentToEntry(doc);
if (entry != null) {
knowledgeIndex.add(entry);
loaded++;
}
} catch (Exception e) {
log.warn("解析文档失败: docId={}, error={}", doc.getDocId(), e.getMessage());
}
}
log.info("知识库索引加载完成,共 {} 个文档", loaded);
} catch (Exception e) {
log.error("知识库索引加载失败", e);
}
}
/**
* 应用就绪后,检查各域是否有 knowledge_domain 记录,无则触发生成
* 使用 ApplicationReadyEvent 而非 PostConstruct,避免循环依赖
*/
@EventListener(ApplicationReadyEvent.class)
public void onApplicationReady() {
try {
knowledgeIndex.stream()
.map(KnowledgeEntry::getCategory)
.filter(c -> c != null && !c.isBlank())
.distinct()
.forEach(category -> {
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
log.info("域 {} 无 knowledge_domain 记录,触发生成", category);
knowledgeDomainService.onDocumentChange(category);
}
});
} catch (Exception e) {
log.error("域级记录生成失败", e);
}
}
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
return null;
}
try {
Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
return KnowledgeEntry.builder()
.filePath(doc.getFilePath())
.title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
.keywords(frontmatter.getKeywords())
.summary(frontmatter.getSummary())
.category(frontmatter.getCategory())
.kbScope(frontmatter.getKbScope())
.covers(frontmatter.getCovers())
.whenToRetrieve(frontmatter.getWhenToRetrieve())
.build();
} catch (Exception e) {
log.warn("解析 metadata 失败: {}", doc.getDocId(), e);
return null;
}
}
/** 兼容旧调用:只返回命中的文档条目。 */
public List<KnowledgeEntry> exactMatch(String query) {
return analyzeQuery(query).matches();
}
/**
* 分析 query,产出 L0 hint。
* 遍历内存索引,收集匹配 keyword、domain、title;不做向量检索。
*/
public L0Hint analyzeQuery(String query) {
long startTime = System.currentTimeMillis();
if (query == null || query.trim().isEmpty()) {
log.debug("查询关键词为空,返回空结果");
return L0Hint.empty();
}
String queryLower = query.toLowerCase();
List<KnowledgeEntry> results = new ArrayList<>();
Set<String> matchedKeywords = new LinkedHashSet<>();
Set<String> domains = new LinkedHashSet<>();
Set<String> entities = new LinkedHashSet<>();
Set<String> titles = new LinkedHashSet<>();
for (KnowledgeEntry entry : knowledgeIndex) {
if (!matchesConfiguredScope(entry)) {
continue;
}
List<String> entryMatchedKeywords = matchedKeywords(entry, queryLower);
if (entryMatchedKeywords.isEmpty()) {
continue;
}
results.add(entry);
matchedKeywords.addAll(entryMatchedKeywords);
entities.addAll(entryMatchedKeywords);
if (entry.getCategory() != null && !entry.getCategory().isBlank()) {
domains.add(entry.getCategory());
}
if (entry.getTitle() != null && !entry.getTitle().isBlank()) {
titles.add(entry.getTitle());
}
}
long elapsedTime = System.currentTimeMillis() - startTime;
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
return new L0Hint(
List.copyOf(results),
List.copyOf(matchedKeywords),
List.copyOf(domains),
List.copyOf(entities),
List.copyOf(titles)
);
}
private boolean matchesKeywords(KnowledgeEntry entry, String query) {
return !matchedKeywords(entry, query).isEmpty();
}
private boolean matchesConfiguredScope(KnowledgeEntry entry) {
String scope = trimToNull(kbScope);
if (scope == null) {
return true;
}
return scope.equals(trimToNull(entry.getKbScope()));
}
private String trimToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
/**
* 关键词双向包含匹配。
* query 已在调用方 lower-case;keyword 在此 lower-case。
* 例:query="mysql timeout" 可命中 keyword="mysql";
* 反过来 keyword="mysql connection pool timeout" 也可能被短 query 命中。
*/
private List<String> matchedKeywords(KnowledgeEntry entry, String query) {
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
return List.of();
}
List<String> matches = new ArrayList<>();
for (String keyword : entry.getKeywords()) {
String keywordLower = keyword.toLowerCase();
if (query.contains(keywordLower) || keywordLower.contains(query)) {
matches.add(keyword);
}
}
return matches;
}
public String readDocument(String filePath, int maxChars) {
try {
Path fullPath = resolveDocumentPath(filePath);
if (!Files.exists(fullPath)) {
log.warn("读取文档失败,文件不存在: basePath={}, filePath={}, resolvedPath={}",
knowledgeBasePath, filePath, fullPath);
return null;
}
String content = Files.readString(fullPath);
if (content.length() > maxChars) {
return content.substring(0, maxChars) + "...";
}
return content;
} catch (IOException e) {
log.error("读取文档失败: basePath={}, filePath={}", knowledgeBasePath, filePath, e);
return null;
}
}
Path resolveDocumentPath(String filePath) {
if (filePath == null || filePath.isBlank()) {
throw new IllegalArgumentException("filePath cannot be blank");
}
Path path = Paths.get(filePath).normalize();
if (path.isAbsolute()) {
return path;
}
Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize();
Path baseName = basePath.getFileName();
if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) {
return basePath.getParent().resolve(path).normalize();
}
Path pathFromWorkingDir = path.toAbsolutePath().normalize();
if (pathFromWorkingDir.startsWith(basePath)) {
return pathFromWorkingDir;
}
return basePath.resolve(path).normalize();
}
public void addToIndex(KnowledgeEntry entry) {
knowledgeIndex.add(entry);
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
}
public void removeFromIndex(String filePath) {
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
log.debug("文档已从 L0 索引移除: {}", filePath);
}
/** Clear in-memory L0 entries (used by knowledge rebuild). */
public void clearIndex() {
knowledgeIndex.clear();
log.info("L0 knowledge index cleared");
}
public int getIndexSize() {
return knowledgeIndex.size();
}
/**
* 获取所有索引条目(供域聚合使用)
*/
public List<KnowledgeEntry> getAllEntries() {
return List.copyOf(knowledgeIndex);
}
/**
* L0 分析结果。
*
* @param matches 命中的文档条目(仅 hint,不是 evidence)
* @param matchedKeywords 命中的关键词
* @param domains 命中文档的 category 集合
* @param entities 当前实现等同 matchedKeywords,预留实体字段
* @param titles 命中文档标题
*/
public record L0Hint(
List<KnowledgeEntry> matches,
List<String> matchedKeywords,
List<String> domains,
List<String> entities,
List<String> titles
) {
public static L0Hint empty() {
return new L0Hint(List.of(), List.of(), List.of(), List.of(), List.of());
}
/** 仅当恰好一个 domain 时返回,用于安全地加 category filter。 */
public String singleDomainOrNull() {
return domains.size() == 1 ? domains.get(0) : null;
}
}
}
@@ -1,56 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.KnowledgeQuery;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* 检索前的 query 理解层(L0 出口)。
*
* <p>输入是 Agent 的原始检索句,输出 {@link KnowledgeQuery},供后续 L1 过滤与 rerank 使用。</p>
*
* <h3>当前能力边界</h3>
* <ul>
* <li>会做:关键词匹配、domain/entity/title hint、唯一 domain 时生成 categoryFilter</li>
* <li>不会做:真正的 query rewrite / 同义词扩展 / 多 query 改写
* ({@code rewrittenQuery} 目前等于 {@code originalQuery})</li>
* <li>L0 命中文档正文不会直接当作 evidence;证据只来自 L1 向量召回</li>
* </ul>
*/
@Service
public class KnowledgeQueryTransformer {
private final KnowledgeIndexService knowledgeIndexService;
public KnowledgeQueryTransformer(KnowledgeIndexService knowledgeIndexService) {
this.knowledgeIndexService = knowledgeIndexService;
}
/**
* 将原始 query 转为检索控制结构。
*
* <p>{@code categoryFilter} 仅在 L0 恰好命中一个 domain 时非空;
* 多 domain 或零 domain 时为 null,避免错误收窄召回。</p>
*/
public KnowledgeQuery transform(String rawQuery) {
String normalized = rawQuery == null ? "" : rawQuery.trim();
KnowledgeIndexService.L0Hint hint = knowledgeIndexService.analyzeQuery(normalized);
return KnowledgeQuery.builder()
.originalQuery(normalized)
// 预留改写字段;当前未实现 rewrite,保持与 original 一致
.rewrittenQuery(normalized)
.domainHints(safeList(hint.domains()))
.matchedKeywords(safeList(hint.matchedKeywords()))
.entities(safeList(hint.entities()))
// 只有唯一 domain 才作为向量 metadata 的 category 过滤条件
.categoryFilter(hint.singleDomainOrNull())
.l0Titles(safeList(hint.titles()))
.l0MatchCount(hint.matches() == null ? 0 : hint.matches().size())
.build();
}
private List<String> safeList(List<String> values) {
return values == null ? List.of() : values;
}
}
@@ -1,179 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.config.RagSidecarProperties;
import com.superbiz.agent.dto.ComparableRetrievalResult;
import com.superbiz.agent.dto.RetrievalComparisonCase;
import com.superbiz.agent.dto.RetrievalComparisonReport;
import com.superbiz.agent.dto.RetrievalComparisonResult;
import com.superbiz.agent.dto.SidecarRetrievalResponse;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@Service
public class RagRetrievalSidecarComparisonService {
private final VectorSearchService vectorSearchService;
private final SpringAiVectorStoreSidecarService sidecarService;
private final RetrievalResultNormalizer normalizer;
private final RagSidecarProperties properties;
private final ObjectMapper objectMapper;
public RagRetrievalSidecarComparisonService(VectorSearchService vectorSearchService,
SpringAiVectorStoreSidecarService sidecarService,
RetrievalResultNormalizer normalizer,
RagSidecarProperties properties,
ObjectMapper objectMapper) {
this.vectorSearchService = vectorSearchService;
this.sidecarService = sidecarService;
this.normalizer = normalizer;
this.properties = properties;
this.objectMapper = objectMapper;
}
public RetrievalComparisonReport compare(List<RetrievalComparisonCase> cases, int topK) {
List<RetrievalComparisonResult> results = new ArrayList<>();
String sidecarStatus = "not_run";
for (RetrievalComparisonCase comparisonCase : cases) {
List<ComparableRetrievalResult> currentResults = normalizeCurrentResults(
vectorSearchService.searchSimilarDocuments(
comparisonCase.getQuery(),
topK,
comparisonCase.getCategory()
)
);
SidecarRetrievalResponse sidecar = sidecarService.search(
comparisonCase.getQuery(),
topK,
comparisonCase.getCategory()
);
sidecarStatus = sidecar.getStatus();
results.add(RetrievalComparisonResult.builder()
.caseId(comparisonCase.getCaseId())
.scenario(comparisonCase.getScenario())
.query(comparisonCase.getQuery())
.category(comparisonCase.getCategory())
.currentResults(currentResults)
.sidecar(sidecar)
.differences(compareDifferences(currentResults, sidecar.getResults()))
.build());
}
return RetrievalComparisonReport.builder()
.generatedAt(OffsetDateTime.now(ZoneOffset.UTC).toString())
.caseCount(cases.size())
.topK(topK)
.sidecarStatus(sidecarStatus)
.results(results)
.build();
}
public RetrievalComparisonReport compareGoldenCases(Path caseFile) throws IOException {
var root = objectMapper.readTree(caseFile.toFile());
int topK = root.path("topK").asInt(5);
List<RetrievalComparisonCase> cases = new ArrayList<>();
for (var node : root.path("cases")) {
cases.add(RetrievalComparisonCase.builder()
.caseId(node.path("caseId").asText())
.scenario(node.path("scenario").asText())
.query(node.path("query").asText())
.build());
}
return compare(cases, topK);
}
public void writeReports(RetrievalComparisonReport report, Path jsonPath, Path markdownPath) throws IOException {
createParentDirectories(jsonPath);
createParentDirectories(markdownPath);
objectMapper.writerWithDefaultPrettyPrinter().writeValue(jsonPath.toFile(), report);
Files.writeString(markdownPath, renderMarkdown(report));
}
private void createParentDirectories(Path path) throws IOException {
Path parent = path.getParent();
if (parent != null) {
Files.createDirectories(parent);
}
}
private List<ComparableRetrievalResult> normalizeCurrentResults(List<VectorSearchService.SearchResult> rawResults) {
List<ComparableRetrievalResult> results = new ArrayList<>();
for (int i = 0; i < rawResults.size(); i++) {
results.add(normalizer.fromCurrent(rawResults.get(i), i + 1, properties.getContentPreviewLimit()));
}
return results;
}
private List<String> compareDifferences(List<ComparableRetrievalResult> currentResults,
List<ComparableRetrievalResult> sidecarResults) {
if (sidecarResults == null || sidecarResults.isEmpty()) {
return List.of("sidecar_unavailable_or_empty");
}
List<String> differences = new ArrayList<>();
String currentTopSource = currentResults.isEmpty() ? null : currentResults.get(0).getSource();
String sidecarTopSource = sidecarResults.get(0).getSource();
if (!Objects.equals(currentTopSource, sidecarTopSource)) {
differences.add("top_source_differs");
}
String currentTopBreadcrumb = currentResults.isEmpty() ? null : currentResults.get(0).getBreadcrumb();
String sidecarTopBreadcrumb = sidecarResults.get(0).getBreadcrumb();
if (!Objects.equals(currentTopBreadcrumb, sidecarTopBreadcrumb)) {
differences.add("top_breadcrumb_differs");
}
String currentScoreLabel = currentResults.isEmpty() ? null : currentResults.get(0).getScoreLabel();
String sidecarScoreLabel = sidecarResults.get(0).getScoreLabel();
if (!Objects.equals(currentScoreLabel, sidecarScoreLabel)) {
differences.add("score_label_differs");
}
return differences;
}
private String renderMarkdown(RetrievalComparisonReport report) {
StringBuilder builder = new StringBuilder();
builder.append("# RAG Sidecar Retrieval Comparison\n\n");
builder.append("Generated at: `").append(report.getGeneratedAt()).append("`\n\n");
builder.append("- Cases: ").append(report.getCaseCount()).append("\n");
builder.append("- Top K: ").append(report.getTopK()).append("\n");
builder.append("- Sidecar status: `").append(report.getSidecarStatus()).append("`\n\n");
builder.append("| Case | Query | Current Top | Sidecar Top | Differences |\n");
builder.append("|---|---|---|---|---|\n");
for (RetrievalComparisonResult result : report.getResults()) {
builder.append("| ")
.append(nullToBlank(result.getCaseId()))
.append(" | ")
.append(escapePipe(result.getQuery()))
.append(" | ")
.append(formatTop(result.getCurrentResults()))
.append(" | ")
.append(formatTop(result.getSidecar() != null ? result.getSidecar().getResults() : List.of()))
.append(" | ")
.append(String.join("<br>", result.getDifferences()))
.append(" |\n");
}
return builder.toString();
}
private String formatTop(List<ComparableRetrievalResult> results) {
if (results == null || results.isEmpty()) {
return "";
}
ComparableRetrievalResult top = results.get(0);
return escapePipe(nullToBlank(top.getSource())) + " (" + nullToBlank(top.getScoreLabel()) + ")";
}
private String escapePipe(String value) {
return nullToBlank(value).replace("|", "\\|");
}
private String nullToBlank(String value) {
return value == null ? "" : value;
}
}
@@ -1,190 +0,0 @@
package com.superbiz.agent.service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* RAG (Retrieval-Augmented Generation) 服务
* 结合向量检索和大语言模型生成答案
*/
@Service
public class RagService {
private static final Logger logger = LoggerFactory.getLogger(RagService.class);
@Autowired
private VectorSearchService vectorSearchService;
@Autowired
private ChatModel chatModel;
@Value("${rag.top-k:3}")
private int topK;
/**
* 流式处理用户问题(不带历史消息)
*
* @param question 用户问题
* @param callback 流式回调接口
*/
public void queryStream(String question, StreamCallback callback) {
queryStream(question, new ArrayList<>(), callback);
}
/**
* 流式处理用户问题(带历史消息)
*
* @param question 用户问题
* @param history 历史消息列表,格式:[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
* @param callback 流式回调接口
*/
public void queryStream(String question, List<Map<String, String>> history, StreamCallback callback) {
try {
logger.info("收到 RAG 流式查询: {}", question);
// 1. 从向量数据库检索相关文档
List<VectorSearchService.SearchResult> searchResults =
vectorSearchService.searchSimilarDocuments(question, topK);
// 发送检索结果
callback.onSearchResults(searchResults);
if (searchResults.isEmpty()) {
logger.warn("未找到相关文档");
callback.onComplete("抱歉,我在知识库中没有找到相关信息来回答您的问题。", "");
return;
}
// 2. 构建上下文和提示词
String context = buildContext(searchResults);
String prompt = buildPrompt(question, context);
// 3. 流式调用大语言模型(传入历史消息)
generateAnswerStream(prompt, history, callback);
} catch (Exception e) {
logger.error("RAG 流式查询失败", e);
callback.onError(e);
}
}
/**
* 构建上下文
*/
private String buildContext(List<VectorSearchService.SearchResult> searchResults) {
StringBuilder context = new StringBuilder();
for (int i = 0; i < searchResults.size(); i++) {
VectorSearchService.SearchResult result = searchResults.get(i);
context.append("【参考资料 ").append(i + 1).append("】\n");
context.append(result.getContent()).append("\n\n");
}
return context.toString();
}
/**
* 构建提示词
*/
private String buildPrompt(String question, String context) {
return String.format(
"你是一个专业的AI助手。请根据以下参考资料回答用户的问题。\n\n" +
"参考资料:\n%s\n" +
"用户问题:%s\n\n" +
"请基于上述参考资料给出准确、详细的回答。如果参考资料中没有相关信息,请明确说明。",
context, question
);
}
/**
* 生成答案(流式)
*
* @param prompt 当前问题的提示词
* @param history 历史消息列表
* @param callback 流式回调接口
*/
private void generateAnswerStream(String prompt, List<Map<String, String>> history, StreamCallback callback) {
// 构建消息列表:历史消息 + 当前问题
List<Message> messages = new ArrayList<>();
// 添加历史消息
for (Map<String, String> historyMsg : history) {
String role = historyMsg.get("role");
String content = historyMsg.get("content");
if ("user".equals(role)) {
messages.add(new UserMessage(content));
} else if ("assistant".equals(role)) {
messages.add(new AssistantMessage(content));
}
}
// 添加当前用户问题
messages.add(new UserMessage(prompt));
logger.debug("发送给AI模型的消息数量: {}(包含 {} 条历史消息)",
messages.size(), history.size());
logger.info("开始调用AI模型流式接口...");
StringBuilder reasoningContent = new StringBuilder();
StringBuilder finalContent = new StringBuilder();
Flux<ChatResponse> flux = chatModel.stream(new Prompt(messages));
logger.info("开始接收AI模型流式响应...");
flux.subscribe(
response -> {
if (response.getResults() != null && !response.getResults().isEmpty()) {
String content = response.getResults().get(0).getOutput().getText();
if (content != null && !content.isEmpty()) {
logger.debug("收到AI模型内容块: {}", content);
finalContent.append(content);
callback.onContentChunk(content);
logger.debug("已调用 onContentChunk 回调");
} else {
logger.debug("收到空内容块,跳过");
}
}
},
error -> {
logger.error("AI模型流式响应失败", error);
callback.onError(new Exception("AI模型流式响应失败: " + error.getMessage(), error));
},
() -> {
logger.info("AI模型流式响应完成,总内容长度: {}", finalContent.length());
callback.onComplete(finalContent.toString(), reasoningContent.toString());
logger.info("已调用 onComplete 回调");
}
);
}
/**
* 流式回调接口
*/
public interface StreamCallback {
void onSearchResults(List<VectorSearchService.SearchResult> results);
void onReasoningChunk(String chunk);
void onContentChunk(String chunk);
void onComplete(String fullContent, String fullReasoning);
void onError(Exception e);
}
}
@@ -1,96 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.ComparableRetrievalResult;
import org.springframework.ai.document.Document;
import org.springframework.stereotype.Component;
import java.util.LinkedHashMap;
import java.util.Map;
@Component
public class RetrievalResultNormalizer {
private final ObjectMapper objectMapper;
public RetrievalResultNormalizer(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
public ComparableRetrievalResult fromCurrent(VectorSearchService.SearchResult result, int rank, int previewLimit) {
Map<String, String> metadata = parseMetadata(result.getMetadata());
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), result.getMetadata(), result.getId());
return ComparableRetrievalResult.builder()
.path("current")
.rank(rank)
.id(result.getId())
.source(source)
.docId(metadata.get("docId"))
.title(metadata.get("title"))
.breadcrumb(metadata.get("breadcrumb"))
.category(metadata.get("category"))
.contentPreview(truncate(result.getContent(), previewLimit))
.scoreLabel("l2_distance")
.scoreValue((double) result.getScore())
.build();
}
public ComparableRetrievalResult fromSidecar(Document document, int rank, int previewLimit) {
Map<String, String> metadata = stringifyMetadata(document.getMetadata());
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), metadata.get("docId"), document.getId());
return ComparableRetrievalResult.builder()
.path("sidecar")
.rank(rank)
.id(document.getId())
.source(source)
.docId(metadata.get("docId"))
.title(metadata.get("title"))
.breadcrumb(metadata.get("breadcrumb"))
.category(metadata.get("category"))
.contentPreview(truncate(document.getText(), previewLimit))
.scoreLabel("similarity")
.scoreValue(document.getScore())
.build();
}
private Map<String, String> parseMetadata(String metadata) {
if (metadata == null || metadata.isBlank()) {
return Map.of();
}
try {
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
return stringifyMetadata(raw);
} catch (Exception e) {
return Map.of();
}
}
private Map<String, String> stringifyMetadata(Map<?, ?> raw) {
if (raw == null || raw.isEmpty()) {
return Map.of();
}
Map<String, String> result = new LinkedHashMap<>();
for (Map.Entry<?, ?> entry : raw.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
result.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
}
}
return result;
}
private String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
return value;
}
}
return null;
}
private String truncate(String text, int maxLength) {
if (text == null || text.length() <= maxLength) {
return text;
}
return text.substring(0, maxLength) + "...";
}
}
@@ -1,106 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.config.RagSidecarProperties;
import com.superbiz.agent.dto.ComparableRetrievalResult;
import com.superbiz.agent.dto.SidecarRetrievalResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.document.Document;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
@Slf4j
@Service
public class SpringAiVectorStoreSidecarService {
private final RagSidecarProperties properties;
private final ObjectProvider<VectorStore> vectorStoreProvider;
private final RetrievalResultNormalizer normalizer;
@Value("${retrieval.kb-scope:}")
private String kbScope = "";
public SpringAiVectorStoreSidecarService(RagSidecarProperties properties,
ObjectProvider<VectorStore> vectorStoreProvider,
RetrievalResultNormalizer normalizer) {
this.properties = properties;
this.vectorStoreProvider = vectorStoreProvider;
this.normalizer = normalizer;
}
public SidecarRetrievalResponse search(String query, int topK, String category) {
if (!properties.isEnabled()) {
return unavailable("disabled", null);
}
VectorStore vectorStore = vectorStoreProvider.getIfAvailable();
if (vectorStore == null) {
return unavailable("missing_vector_store", "No Spring AI VectorStore bean is available");
}
try {
SearchRequest.Builder builder = SearchRequest.builder()
.query(query)
.topK(topK)
.similarityThresholdAll();
String filterExpression = buildFilterExpression(category);
if (filterExpression != null) {
builder.filterExpression(filterExpression);
}
List<Document> documents = vectorStore.similaritySearch(builder.build());
List<ComparableRetrievalResult> results = new ArrayList<>();
for (int i = 0; i < documents.size(); i++) {
results.add(normalizer.fromSidecar(documents.get(i), i + 1, properties.getContentPreviewLimit()));
}
return SidecarRetrievalResponse.builder()
.enabled(true)
.available(true)
.status("available")
.results(results)
.build();
} catch (Exception e) {
log.warn("Spring AI sidecar retrieval failed: {}", e.getMessage());
return unavailable("query_failed", e.getMessage());
}
}
private SidecarRetrievalResponse unavailable(String status, String errorMessage) {
return SidecarRetrievalResponse.builder()
.enabled(properties.isEnabled())
.available(false)
.status(status)
.errorMessage(errorMessage)
.results(List.of())
.build();
}
private String escapeFilterValue(String value) {
return value.replace("'", "\\'");
}
String buildFilterExpression(String category) {
List<String> parts = new ArrayList<>();
String categoryFilter = trimToNull(category);
if (categoryFilter != null) {
parts.add("category == '" + escapeFilterValue(categoryFilter) + "'");
}
String scopeFilter = trimToNull(kbScope);
if (scopeFilter != null) {
parts.add("kb_scope == '" + escapeFilterValue(scopeFilter) + "'");
}
return parts.isEmpty() ? null : String.join(" && ", parts);
}
private String trimToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
}
@@ -1,89 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.exception.DocumentProcessException;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
/**
* 文本提取服务
* 仅支持 Markdown (.md) 和纯文本 (.txt) 格式
* 其他格式(.docx、.pdf 等)需要通过外部转换服务先转为 Markdown
*/
@Slf4j
@Service
public class TextExtractorService {
/**
* 从文件中提取文本
*
* @param file 上传的文件
* @param fileName 文件名
* @return 提取的文本内容
*/
public String extractText(MultipartFile file, String fileName) {
if (file == null || file.isEmpty()) {
throw new DocumentProcessException(fileName, "extract", "文件为空");
}
String extension = getFileExtension(fileName);
log.info("开始提取文本,文件名: {}, 格式: {}, 大小: {} bytes", fileName, extension, file.getSize());
if (!isSupportedFormat(fileName)) {
throw new DocumentProcessException(
fileName, "extract",
"不支持的文件格式: " + extension + ",仅支持 .md 和 .txt。其他格式请先通过转换服务转为 Markdown。"
);
}
try {
String text = extractPlainText(file);
log.info("文本提取成功,文件名: {}, 提取字符数: {}", fileName, text.length());
return text;
} catch (IOException e) {
log.error("文本提取失败,文件名: {}", fileName, e);
throw new DocumentProcessException(fileName, "extract", "文件读取失败: " + e.getMessage(), e);
}
}
/**
* 提取纯文本(.txt、.md)
*/
private String extractPlainText(MultipartFile file) throws IOException {
StringBuilder content = new StringBuilder();
try (InputStream is = file.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
content.append(line).append("\n");
}
}
return content.toString().trim();
}
/**
* 获取文件扩展名
*/
private String getFileExtension(String fileName) {
if (fileName == null || !fileName.contains(".")) {
return "";
}
return fileName.substring(fileName.lastIndexOf(".") + 1);
}
/**
* 验证文件格式是否支持
*/
public boolean isSupportedFormat(String fileName) {
String extension = getFileExtension(fileName).toLowerCase();
return extension.equals("md") || extension.equals("txt");
}
}
@@ -1,125 +0,0 @@
package com.superbiz.agent.service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* 向量嵌入服务
* 使用阿里云 DashScope Text Embedding API
*/
@Service
public class VectorEmbeddingService {
private static final Logger logger = LoggerFactory.getLogger(VectorEmbeddingService.class);
@Autowired
private EmbeddingModel embeddingModel;
/**
* 生成向量嵌入
* 调用阿里云 DashScope Text Embedding API
*
* @param content 文本内容
* @return 向量嵌入(浮点数列表)
*/
public List<Float> generateEmbedding(String content) {
try {
if (content == null || content.trim().isEmpty()) {
logger.warn("内容为空,无法生成向量");
throw new IllegalArgumentException("内容不能为空");
}
logger.debug("开始生成向量嵌入, 内容长度: {} 字符", content.length());
float[] embedding = embeddingModel.embed(content);
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
for (float v : embedding) {
floatEmbedding.add(v);
}
logger.info("成功生成向量嵌入, 内容长度: {} 字符, 向量维度: {}",
content.length(), floatEmbedding.size());
return floatEmbedding;
} catch (Exception e) {
logger.error("生成向量嵌入失败, 内容长度: {}", content != null ? content.length() : 0, e);
throw new RuntimeException("生成向量嵌入失败: " + e.getMessage(), e);
}
}
public List<List<Float>> generateEmbeddings(List<String> contents) {
try {
if (contents == null || contents.isEmpty()) {
logger.warn("内容列表为空,无法生成向量");
return Collections.emptyList();
}
logger.info("开始批量生成向量嵌入, 数量: {}", contents.size());
List<float[]> embeddings = embeddingModel.embed(contents);
List<List<Float>> result = new ArrayList<>();
for (float[] embedding : embeddings) {
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
for (float v : embedding) {
floatEmbedding.add(v);
}
result.add(floatEmbedding);
}
logger.info("成功批量生成向量嵌入, 数量: {}, 维度: {}",
result.size(),
result.isEmpty() ? 0 : result.get(0).size());
return result;
} catch (Exception e) {
logger.error("批量生成向量嵌入失败", e);
throw new RuntimeException("批量生成向量嵌入失败: " + e.getMessage(), e);
}
}
/**
* 生成查询向量
*
* @param query 查询文本
* @return 向量嵌入
*/
public List<Float> generateQueryVector(String query) {
return generateEmbedding(query);
}
/**
* 计算两个向量的余弦相似度
*
* @param vector1 向量1
* @param vector2 向量2
* @return 余弦相似度 [-1, 1]
*/
public float calculateCosineSimilarity(List<Float> vector1, List<Float> vector2) {
if (vector1.size() != vector2.size()) {
throw new IllegalArgumentException("向量维度不匹配");
}
float dotProduct = 0.0f;
float norm1 = 0.0f;
float norm2 = 0.0f;
for (int i = 0; i < vector1.size(); i++) {
dotProduct += vector1.get(i) * vector2.get(i);
norm1 += vector1.get(i) * vector1.get(i);
norm2 += vector2.get(i) * vector2.get(i);
}
return dotProduct / (float) (Math.sqrt(norm1) * Math.sqrt(norm2));
}
}
@@ -1,380 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.DocumentChunk;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
import lombok.Getter;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 向量索引写入服务(RAG 入库侧)。
*
* <p>唯一后端 {@link MilvusHybridKnowledgeStore}(Milvus SDK v2):</p>
* <ul>
* <li>dense:应用侧 embedding → 字段 {@code vector}</li>
* <li>BM25:{@link #buildSearchText} → 字段 {@code search_text};
* sparse 由 collection 上 BM25 Function 自动生成,本类不写 sparse</li>
* </ul>
* <p>不再使用 legacy {@code MilvusServiceClient} insert/delete,
* 也不走 Spring AI {@code VectorStore#add}(starter 无 hybrid schema/BM25 Function)。</p>
*/
@Service
public class VectorIndexService {
private static final Logger logger = LoggerFactory.getLogger(VectorIndexService.class);
@Autowired
private MilvusHybridKnowledgeStore knowledgeStore;
@Autowired
private VectorEmbeddingService embeddingService;
@Autowired
private DocumentChunkService chunkService;
@Value("${file.upload.path}")
private String uploadPath;
public IndexingResult indexDirectory(String directoryPath) {
IndexingResult result = new IndexingResult();
result.setStartTime(LocalDateTime.now());
try {
String targetPath = (directoryPath != null && !directoryPath.trim().isEmpty())
? directoryPath : uploadPath;
Path dirPath = Paths.get(targetPath).normalize();
File directory = dirPath.toFile();
if (!directory.exists() || !directory.isDirectory()) {
throw new IllegalArgumentException("目录不存在或不是有效目录: " + targetPath);
}
result.setDirectoryPath(directory.getAbsolutePath());
File[] files = directory.listFiles((dir, name) ->
name.endsWith(".txt") || name.endsWith(".md")
);
if (files == null || files.length == 0) {
logger.warn("目录中没有找到支持的文件: {}", targetPath);
result.setTotalFiles(0);
result.setSuccess(true);
result.setEndTime(LocalDateTime.now());
return result;
}
result.setTotalFiles(files.length);
logger.info("开始索引目录: {}, 找到 {} 个文件", targetPath, files.length);
for (File file : files) {
try {
indexSingleFile(file.getAbsolutePath());
result.incrementSuccessCount();
logger.info("文件索引成功: {}", file.getName());
} catch (Exception e) {
result.incrementFailCount();
result.addFailedFile(file.getAbsolutePath(), e.getMessage());
logger.error("文件索引失败: {}", file.getName(), e);
}
}
result.setSuccess(result.getFailCount() == 0);
result.setEndTime(LocalDateTime.now());
return result;
} catch (Exception e) {
logger.error("索引目录失败", e);
result.setSuccess(false);
result.setErrorMessage(e.getMessage());
result.setEndTime(LocalDateTime.now());
return result;
}
}
public void indexSingleFile(String filePath) throws Exception {
Path path = Paths.get(filePath).normalize();
File file = path.toFile();
if (!file.exists() || !file.isFile()) {
throw new IllegalArgumentException("文件不存在: " + filePath);
}
logger.info("开始索引文件: {}", path);
String content = Files.readString(path);
deleteExistingData(path.toString());
List<DocumentChunk> chunks = chunkService.chunkDocument(content, path.toString());
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
for (int i = 0; i < chunks.size(); i++) {
DocumentChunk chunk = chunks.get(i);
try {
// dense embedding 与 BM25 search_text 同源(title/path 增强)
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
Map<String, Object> metadata = buildMetadata(path.toString(), chunk, chunks.size());
knowledgeStore.upsertChunk(
chunk.getContent(), // 返回原文
buildSearchText(chunk), // BM25 语料;sparse 由 Milvus Function 生成
vector, // dense 向量
metadata,
chunk.getChunkIndex());
logger.info("分片 {}/{} 索引成功", i + 1, chunks.size());
} catch (Exception e) {
logger.error("分片 {}/{} 索引失败", i + 1, chunks.size(), e);
throw new RuntimeException("分片索引失败: " + e.getMessage(), e);
}
}
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
}
public void indexDocumentChunks(String docId, List<DocumentChunk> chunks, String category) throws Exception {
indexDocumentChunks(docId, chunks, category, null);
}
public void indexDocumentChunks(String docId,
List<DocumentChunk> chunks,
String category,
Frontmatter frontmatter) throws Exception {
if (chunks == null || chunks.isEmpty()) {
throw new IllegalArgumentException("文档分块列表为空");
}
logger.info("开始索引文档分块,docId: {}, 分块数: {}, 类别: {}", docId, chunks.size(), category);
deleteDocumentChunks(docId);
for (int i = 0; i < chunks.size(); i++) {
DocumentChunk chunk = chunks.get(i);
try {
// dense embedding 与 BM25 search_text 同源(title/path 增强)
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
Map<String, Object> metadata = buildDocumentMetadata(docId, chunk, chunks.size(), category, frontmatter);
knowledgeStore.upsertChunk(
chunk.getContent(), // 返回原文
buildSearchText(chunk), // BM25 语料;sparse 由 Milvus Function 生成
vector, // dense 向量
metadata,
chunk.getChunkIndex());
logger.info("文档分块 {}/{} 索引成功,docId: {}", i + 1, chunks.size(), docId);
} catch (Exception e) {
logger.error("文档分块 {}/{} 索引失败,docId: {}", i + 1, chunks.size(), docId, e);
throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
}
}
logger.info("文档索引完成,docId: {}, 共 {} 个分块,类别: {}", docId, chunks.size(), category);
}
public void deleteDocumentChunks(String docId) {
try {
knowledgeStore.deleteByDocId(docId);
logger.info("删除文档旧数据成功,docId: {}", docId);
} catch (Exception e) {
logger.warn("删除文档旧数据异常,docId: {}", docId, e);
}
}
static Map<String, Object> buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks, String category) {
return buildDocumentMetadata(docId, chunk, totalChunks, category, null);
}
static Map<String, Object> buildDocumentMetadata(String docId,
DocumentChunk chunk,
int totalChunks,
String category,
Frontmatter frontmatter) {
Map<String, Object> metadata = new HashMap<>();
String source = firstNonBlank(frontmatter != null ? frontmatter.getSource() : null, "upload:" + docId);
metadata.put("docId", docId);
metadata.put("_source", source);
metadata.put("source", source);
metadata.put("chunkIndex", chunk.getChunkIndex());
metadata.put("totalChunks", totalChunks);
String title = firstNonBlank(chunk.getTitle(), frontmatter != null ? frontmatter.getTitle() : null);
if (title != null) {
metadata.put("title", title);
}
String breadcrumb = firstNonBlank(frontmatter != null ? frontmatter.getBreadcrumb() : null, chunk.getBreadcrumb());
if (breadcrumb != null) {
metadata.put("breadcrumb", breadcrumb);
}
metadata.put("category", category != null && !category.isBlank() ? category : "upload");
String kbScope = trimToNull(frontmatter != null ? frontmatter.getKbScope() : null);
if (kbScope != null) {
metadata.put("kb_scope", kbScope);
}
return metadata;
}
/**
* Dense embedding 输入。与 {@link #buildSearchText} 同源,保证 dense/BM25 看到同一增强文本。
*/
static String buildEmbeddingText(DocumentChunk chunk) {
return buildSearchText(chunk);
}
/**
* 构造写入 Milvus 的检索文本(BM25 {@code search_text},并复用为 dense embedding 输入)。
*
* <p>在正文前拼接 title / breadcrumb,提高「按标题或路径关键词」的 BM25 命中率,
* 同时让 dense 向量也编码结构信息。无标题路径时退回纯 content。</p>
*/
static String buildSearchText(DocumentChunk chunk) {
String content = trimToEmpty(chunk.getContent());
String title = trimToEmpty(chunk.getTitle());
String breadcrumb = trimToEmpty(chunk.getBreadcrumb());
if (title.isEmpty() && breadcrumb.isEmpty()) {
return content;
}
StringBuilder text = new StringBuilder();
if (!title.isEmpty()) {
text.append("Title: ").append(title).append("\n");
}
if (!breadcrumb.isEmpty()) {
text.append("Path: ").append(breadcrumb).append("\n");
}
text.append("Content:\n").append(content);
return text.toString();
}
private static String trimToEmpty(String value) {
return value == null ? "" : value.trim();
}
private static String trimToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
private static String firstNonBlank(String... values) {
for (String value : values) {
String trimmed = trimToNull(value);
if (trimmed != null) {
return trimmed;
}
}
return null;
}
private void deleteExistingData(String filePath) {
try {
Path path = Paths.get(filePath).normalize();
String normalizedPath = path.toString().replace(File.separator, "/");
knowledgeStore.deleteBySource(normalizedPath);
logger.info("已删除文件的旧数据: {}", normalizedPath);
} catch (Exception e) {
logger.warn("删除旧数据失败(可能是首次索引): {}", e.getMessage());
}
}
private Map<String, Object> buildMetadata(String filePath, DocumentChunk chunk, int totalChunks) {
Map<String, Object> metadata = new HashMap<>();
Path path = Paths.get(filePath).normalize();
String normalizedPath = path.toString().replace(File.separator, "/");
Path fileName = path.getFileName();
String fileNameStr = fileName != null ? fileName.toString() : "";
String extension = "";
int dotIndex = fileNameStr.lastIndexOf('.');
if (dotIndex > 0) {
extension = fileNameStr.substring(dotIndex);
}
metadata.put("_source", normalizedPath);
metadata.put("source", normalizedPath);
metadata.put("_extension", extension);
metadata.put("_file_name", fileNameStr);
String category = extractCategory(normalizedPath);
if (category != null && !category.isEmpty()) {
metadata.put("category", category);
}
metadata.put("chunkIndex", chunk.getChunkIndex());
metadata.put("totalChunks", totalChunks);
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
metadata.put("title", chunk.getTitle());
}
if (chunk.getBreadcrumb() != null && !chunk.getBreadcrumb().isEmpty()) {
metadata.put("breadcrumb", chunk.getBreadcrumb());
}
return metadata;
}
private String extractCategory(String filePath) {
try {
String normalized = filePath.replace("\\", "/");
int docsIndex = normalized.indexOf("aiops-docs/");
if (docsIndex >= 0) {
String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
int slashIndex = afterDocs.indexOf("/");
if (slashIndex > 0) {
return afterDocs.substring(0, slashIndex);
}
}
int firstSlash = normalized.indexOf("/");
if (firstSlash > 0) {
return normalized.substring(0, firstSlash);
}
return null;
} catch (Exception e) {
logger.warn("提取类别失败,路径: {}", filePath, e);
return null;
}
}
@Getter
public static class IndexingResult {
@Setter
private boolean success;
@Setter
private String directoryPath;
@Setter
private int totalFiles;
private int successCount;
private int failCount;
@Setter
private LocalDateTime startTime;
@Setter
private LocalDateTime endTime;
@Setter
private String errorMessage;
private Map<String, String> failedFiles = new HashMap<>();
public void incrementSuccessCount() {
this.successCount++;
}
public void incrementFailCount() {
this.failCount++;
}
public long getDurationMs() {
if (startTime != null && endTime != null) {
return java.time.Duration.between(startTime, endTime).toMillis();
}
return 0;
}
public void addFailedFile(String filePath, String error) {
this.failedFiles.put(filePath, error);
}
}
}
@@ -1,94 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
import lombok.Getter;
import lombok.Setter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Locale;
/**
* 知识库向量检索门面(lookup_knowledge / RAG 召回入口)。
*
* <p><b>唯一后端:</b>{@link MilvusHybridKnowledgeStore}(Milvus Java SDK v2)。</p>
*
* <h3>模式切换</h3>
* <p>{@code retrieval.search.mode}(同库查询算法,非两套写入):</p>
* <ul>
* <li>{@code hybrid} —— 线上主路径:dense + 服务端 BM25 + RRF</li>
* <li>{@code dense} —— 对照/评测:仅 dense ANN</li>
* </ul>
* <p>命中 {@link SearchResult#scoreLabel} 仅为 {@link RetrievalScoreLabels#DENSE} /
* {@link RetrievalScoreLabels#HYBRID}。质量分由后处理 {@code RetrievalScoreNormalizer} 统一计算。</p>
*/
@Service
public class VectorSearchService {
private static final Logger logger = LoggerFactory.getLogger(VectorSearchService.class);
@Autowired
private MilvusHybridKnowledgeStore knowledgeStore;
@Autowired
private VectorEmbeddingService embeddingService;
/**
* 检索模式:{@code hybrid}(主路径)| {@code dense}(召回对照)。
*/
@Value("${retrieval.search.mode:dense}")
private String searchMode = "dense";
public List<SearchResult> searchSimilarDocuments(String query, int topK) {
return searchSimilarDocuments(query, topK, null);
}
public List<SearchResult> searchSimilarDocuments(String query, int topK, String category) {
String mode = searchMode == null ? "dense" : searchMode.trim().toLowerCase(Locale.ROOT);
List<Float> queryVector = embeddingService.generateQueryVector(query);
if ("hybrid".equals(mode)) {
logger.info("Hybrid dense+BM25 search topK={} category={} collection={}",
topK, category, knowledgeStore.collectionName());
return knowledgeStore.searchHybrid(query, queryVector, topK, category);
}
logger.info("Dense search topK={} category={} collection={}",
topK, category, knowledgeStore.collectionName());
return knowledgeStore.searchDense(query, queryVector, topK, category);
}
/**
* 单条召回结果。列表顺序即检索权威序(adapter 赋 originalRank=1..n)。
*
* <ul>
* <li>{@code scoreLabel=dense}:{@link #score} = L2 距离(越小越好)</li>
* <li>{@code scoreLabel=hybrid}:{@link #score}/{@link #rawScore} = 引擎融合分;
* 后处理 quality 主要按 rank 映射,不把 score 当 L2</li>
* </ul>
*/
@Setter
@Getter
public static class SearchResult {
private String id;
private String content;
/**
* 引擎主分:dense=L2;hybrid=融合分(量纲由 scoreLabel 解释)。
*/
private float score;
/** 引擎原始分(与 score 同源或更细,便于调试)。 */
private Double rawScore;
/** {@link RetrievalScoreLabels#DENSE} 或 {@link RetrievalScoreLabels#HYBRID}。 */
private String scoreLabel;
/**
* Optional dense L2 for the same id (hybrid path only).
* Used for absolute quality / low-quality gates; does <b>not</b> replace sort order.
*/
private Double denseDistance;
/** metadata JSON 字符串(docId、source、title…)。 */
private String metadata;
}
}
@@ -1,547 +0,0 @@
package com.superbiz.agent.service.milvus;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.superbiz.agent.config.MilvusProperties;
import com.superbiz.agent.constant.MilvusConstants;
import com.superbiz.agent.service.VectorSearchService;
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
import io.milvus.common.clientenum.FunctionType;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.common.DataType;
import io.milvus.v2.common.IndexParam;
import io.milvus.v2.service.collection.request.AddFieldReq;
import io.milvus.v2.service.collection.request.CreateCollectionReq;
import io.milvus.v2.service.collection.request.DropCollectionReq;
import io.milvus.v2.service.collection.request.HasCollectionReq;
import io.milvus.v2.service.collection.request.LoadCollectionReq;
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
import io.milvus.v2.service.index.request.CreateIndexReq;
import io.milvus.v2.service.vector.request.AnnSearchReq;
import io.milvus.v2.service.vector.request.DeleteReq;
import io.milvus.v2.service.vector.request.HybridSearchReq;
import io.milvus.v2.service.vector.request.InsertReq;
import io.milvus.v2.service.vector.request.SearchReq;
import io.milvus.v2.service.vector.request.data.BaseVector;
import io.milvus.v2.service.vector.request.data.EmbeddedText;
import io.milvus.v2.service.vector.request.data.FloatVec;
import io.milvus.v2.service.vector.request.ranker.RRFRanker;
import io.milvus.v2.service.vector.response.SearchResp;
import jakarta.annotation.PreDestroy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* 知识库向量后端(Milvus Java SDK v2)—— dense + BM25 混合检索的唯一实现。
*
* <h3>为什么不用 Spring AI {@code spring-ai-starter-vector-store-milvus}</h3>
* <ul>
* <li>Spring AI Milvus starter(截至 2.0.0 / 1.1.8)只封装 dense {@code similaritySearch}。</li>
* <li>底层仍是 V1 {@code MilvusServiceClient} + 单路 {@code SearchParam},无 {@code hybridSearch} /
* BM25 Function / {@link RRFRanker}。</li>
* <li>真混合检索(dense ANN + 服务端 BM25 sparse,再 RRF 融合)必须走 Milvus SDK v2,
* 见 {@link #searchHybrid}。</li>
* </ul>
*
* <h3>Collection schema(默认名 {@code biz})</h3>
* <pre>
* id VarChar PK
* content VarChar —— 原文,返回给上层
* search_text VarChar+analyzer —— BM25 输入文本(可含 title/path 增强)
* sparse_vector SparseFloatVector —— 由 BM25 Function 从 search_text 自动生成,写入时不必填
* vector FloatVector —— dense 向量(应用侧 embedding)
* metadata JSON —— docId / source / category / kb_scope 等
* </pre>
*
* <h3>检索模式</h3>
* <ul>
* <li>{@link #searchDense}:单路 L2 ANN;{@code scoreLabel=dense}。</li>
* <li>{@link #searchHybrid}:dense + BM25 + 服务端 {@link RRFRanker};{@code scoreLabel=hybrid};
* 返回序即 RRF 序,不再用 dense L2 覆盖主分。</li>
* </ul>
*
* <p>配置入口:{@code milvus.collection}、{@code retrieval.search.mode}、{@code retrieval.hybrid.rrf-k}。</p>
*/
@Service
public class MilvusHybridKnowledgeStore {
private static final Logger log = LoggerFactory.getLogger(MilvusHybridKnowledgeStore.class);
private static final Gson GSON = new Gson();
/** 主键(稳定 UUID,由 source + chunkIndex 派生,便于幂等重写)。 */
public static final String FIELD_ID = "id";
/** 返回给 LLM / 上层的原文 chunk。 */
public static final String FIELD_CONTENT = "content";
/**
* BM25 输入字段。写入明文;Milvus 侧 analyzer + BM25 Function 生成 {@link #FIELD_SPARSE}。
* 通常比 content 多带 title/path 等检索增强词。
*/
public static final String FIELD_SEARCH_TEXT = "search_text";
/** 稀疏向量字段;由 BM25 Function 自动产出,insert 时不要手动填。 */
public static final String FIELD_SPARSE = "sparse_vector";
/** Dense 向量字段(应用侧 EmbeddingModel 生成)。 */
public static final String FIELD_DENSE = "vector";
/** 业务元数据 JSON(过滤、证据身份、展示用)。 */
public static final String FIELD_METADATA = "metadata";
private final MilvusProperties milvusProperties;
@Value("${milvus.collection:biz}")
private String collectionName = "biz";
/**
* RRF 平滑参数 k:score(d) = Σ 1/(k + rank_i(d))。
* k 越大,各路排名差异被压得越平;默认 60 与常见 RRF 设定一致。
*/
@Value("${retrieval.hybrid.rrf-k:60}")
private int rrfK = 60;
/** 非空时追加 {@code metadata.kb_scope} 过滤,实现多知识域隔离。 */
@Value("${retrieval.kb-scope:}")
private String kbScope = "";
private volatile MilvusClientV2 client;
public MilvusHybridKnowledgeStore(MilvusProperties milvusProperties) {
this.milvusProperties = milvusProperties;
}
/**
* 懒连接:首次调用时建连、确保 collection schema 存在并 load。
* 线程安全;后续检索/写入复用同一 {@link MilvusClientV2}。
*/
public synchronized MilvusClientV2 client() {
if (client == null) {
client = connect();
ensureCollection(client);
loadCollection(client);
}
return client;
}
public String collectionName() {
return collectionName;
}
/**
* 写入单个 chunk(dense + BM25 所需明文)。
*
* <p>只插入 {@code content / search_text / vector / metadata};
* {@code sparse_vector} 由 collection 上的 BM25 Function 在服务端从 {@code search_text} 生成。</p>
*
* <p>id 由 {@code source|docId + chunkIndex} 的 nameUUID 派生,同一 chunk 重复写入会得到相同 id
*(配合先 delete 再 insert 的上层逻辑实现覆盖)。</p>
*
* @param content 原文(返回字段)
* @param searchText BM25 / 可与 dense embedding 同源的检索文本
* @param denseVector 应用侧 embedding
* @param metadata 须尽量带 {@code _source} 或 {@code docId},供 id 与过滤使用
* @param chunkIndex 分片序号
*/
public void upsertChunk(String content,
String searchText,
List<Float> denseVector,
Map<String, Object> metadata,
int chunkIndex) {
String source = metadata == null ? null : stringVal(metadata.get("_source"));
if (source == null) {
source = metadata == null ? null : stringVal(metadata.get("source"));
}
if (source == null) {
source = metadata == null ? null : stringVal(metadata.get("docId"));
}
String idSeed = (source == null ? "chunk" : source) + "_" + chunkIndex;
String id = UUID.nameUUIDFromBytes(idSeed.getBytes()).toString();
JsonObject row = new JsonObject();
row.addProperty(FIELD_ID, id);
row.addProperty(FIELD_CONTENT, content == null ? "" : content);
// 仅写明文;sparse 由 BM25 Function(search_text -> sparse_vector) 自动生成
row.addProperty(FIELD_SEARCH_TEXT, searchText == null ? "" : searchText);
row.add(FIELD_DENSE, GSON.toJsonTree(denseVector));
row.add(FIELD_METADATA, GSON.toJsonTree(metadata == null ? Map.of() : metadata));
client().insert(InsertReq.builder()
.collectionName(collectionName)
.data(List.of(row))
.build());
}
/** 按 metadata.docId 删除该文档全部 chunk(重建/覆盖前调用)。 */
public void deleteByDocId(String docId) {
if (docId == null || docId.isBlank()) {
return;
}
String filter = "metadata[\"docId\"] == \"" + escapeFilter(docId) + "\"";
client().delete(DeleteReq.builder()
.collectionName(collectionName)
.filter(filter)
.build());
}
/** 按 metadata._source(规范化路径)删除,用于按文件路径重索引。 */
public void deleteBySource(String sourcePath) {
if (sourcePath == null || sourcePath.isBlank()) {
return;
}
String normalized = sourcePath.replace('\\', '/');
String filter = "metadata[\"_source\"] == \"" + escapeFilter(normalized) + "\"";
client().delete(DeleteReq.builder()
.collectionName(collectionName)
.filter(filter)
.build());
}
/**
* 删除并重建当前知识 collection(空的 dense+BM25 schema)。
* 供 {@code /api/knowledge/rebuild-hybrid} 与重建脚本使用;会销毁该 collection 全部向量。
*/
public synchronized Map<String, Object> dropAndRecreateCollection() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("collection", collectionName);
MilvusClientV2 milvusClient = client();
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
.collectionName(collectionName)
.build());
result.put("existedBefore", Boolean.TRUE.equals(exists));
if (Boolean.TRUE.equals(exists)) {
try {
milvusClient.releaseCollection(ReleaseCollectionReq.builder()
.collectionName(collectionName)
.build());
} catch (Exception e) {
log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
}
milvusClient.dropCollection(DropCollectionReq.builder()
.collectionName(collectionName)
.build());
log.info("Dropped hybrid collection '{}'", collectionName);
result.put("dropped", true);
} else {
result.put("dropped", false);
}
ensureCollection(milvusClient);
loadCollection(milvusClient);
result.put("recreated", true);
result.put("loaded", true);
return result;
}
/**
* 单路 dense ANN(L2)。
* {@code score} = L2 距离(越小越好);{@code scoreLabel} = {@link RetrievalScoreLabels#DENSE}。
*/
public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
List<Float> queryVector,
int topK,
String category) {
String filter = buildFilter(category);
SearchReq.SearchReqBuilder builder = SearchReq.builder()
.collectionName(collectionName)
.annsField(FIELD_DENSE)
.data(List.of(new FloatVec(queryVector)))
.topK(topK)
.outputFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
.metricType(IndexParam.MetricType.L2);
if (filter != null) {
builder.filter(filter);
}
SearchResp resp = client().search(builder.build());
return toSearchResults(resp, RetrievalScoreLabels.DENSE);
}
/**
* Dense + BM25 真混合检索(Milvus 服务端融合)。
*
* <ol>
* <li>dense 子路:{@code vector},L2</li>
* <li>BM25 子路:{@code sparse_vector} + {@link EmbeddedText}</li>
* <li>{@link HybridSearchReq} + {@link RRFRanker} → 返回序即权威序</li>
* </ol>
*
* <p>{@code scoreLabel=hybrid};{@code score}/{@code rawScore} 保留引擎融合分,
* <b>不</b>用 dense L2 覆盖主分或改 label。可选并行 dense 探测仅填充
* {@link VectorSearchService.SearchResult#setDenseDistance},供后处理绝对质量闸门
* (如 L0 filter low-quality → unfiltered retry),排序仍以 RRF 返回序为准。</p>
*/
public List<VectorSearchService.SearchResult> searchHybrid(String queryText,
List<Float> queryVector,
int topK,
String category) {
String filter = buildFilter(category);
int pathTopK = Math.max(topK, 10);
AnnSearchReq.AnnSearchReqBuilder denseAnn = AnnSearchReq.builder()
.vectorFieldName(FIELD_DENSE)
.vectors(List.of((BaseVector) new FloatVec(queryVector)))
.topK(pathTopK)
.metricType(IndexParam.MetricType.L2)
.params("{\"nprobe\":10}");
if (filter != null) {
denseAnn.filter(filter);
}
AnnSearchReq.AnnSearchReqBuilder sparseAnn = AnnSearchReq.builder()
.vectorFieldName(FIELD_SPARSE)
.vectors(List.of((BaseVector) new EmbeddedText(queryText == null ? "" : queryText)))
.topK(pathTopK)
.metricType(IndexParam.MetricType.BM25);
if (filter != null) {
sparseAnn.filter(filter);
}
HybridSearchReq hybridReq = HybridSearchReq.builder()
.collectionName(collectionName)
.searchRequests(List.of(denseAnn.build(), sparseAnn.build()))
.ranker(new RRFRanker(Math.max(1, rrfK)))
.topK(topK)
.outFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
.build();
SearchResp hybridResp = client().hybridSearch(hybridReq);
List<VectorSearchService.SearchResult> fused = toSearchResults(hybridResp, RetrievalScoreLabels.HYBRID);
attachDenseDistances(fused, queryText, queryVector, pathTopK, category);
return fused;
}
/**
* Attach dense L2 by id for quality gates only — never overwrites hybrid score/label/order.
*/
private void attachDenseDistances(List<VectorSearchService.SearchResult> fused,
String queryText,
List<Float> queryVector,
int pathTopK,
String category) {
if (fused == null || fused.isEmpty()) {
return;
}
try {
Map<String, Float> denseById = new HashMap<>();
for (VectorSearchService.SearchResult denseHit :
searchDense(queryText, queryVector, pathTopK, category)) {
if (denseHit.getId() != null) {
denseById.put(denseHit.getId(), denseHit.getScore());
}
}
for (VectorSearchService.SearchResult hit : fused) {
Float l2 = denseById.get(hit.getId());
if (l2 != null) {
hit.setDenseDistance(l2.doubleValue());
}
}
} catch (Exception e) {
log.warn("Dense distance attach for hybrid quality gate failed: {}", e.getMessage());
}
}
/**
* 将 Milvus {@link SearchResp} 映射为上层结果;列表顺序即检索权威序(adapter 赋 originalRank)。
*/
private List<VectorSearchService.SearchResult> toSearchResults(SearchResp resp, String scoreLabel) {
List<VectorSearchService.SearchResult> out = new ArrayList<>();
if (resp == null || resp.getSearchResults() == null || resp.getSearchResults().isEmpty()) {
return out;
}
List<SearchResp.SearchResult> first = resp.getSearchResults().get(0);
if (first == null) {
return out;
}
for (SearchResp.SearchResult row : first) {
VectorSearchService.SearchResult mapped = new VectorSearchService.SearchResult();
Object id = row.getId();
mapped.setId(id == null ? null : String.valueOf(id));
Map<String, Object> entity = row.getEntity() == null ? Map.of() : row.getEntity();
Object content = entity.get(FIELD_CONTENT);
mapped.setContent(content == null ? null : String.valueOf(content));
Object metadata = entity.get(FIELD_METADATA);
if (metadata instanceof JsonObject jsonObject) {
mapped.setMetadata(jsonObject.toString());
} else if (metadata instanceof Map<?, ?> map) {
mapped.setMetadata(GSON.toJson(map));
} else if (metadata != null) {
mapped.setMetadata(String.valueOf(metadata));
}
Float score = row.getScore();
mapped.setRawScore(score == null ? null : score.doubleValue());
mapped.setScoreLabel(scoreLabel);
// dense: L2;hybrid: 引擎融合分(后处理 quality 主要看 rank,不依赖此量纲)
mapped.setScore(score == null ? 0f : score);
out.add(mapped);
}
return out;
}
/**
* 组装标量过滤表达式:category、kb_scope(配置级)可叠加,用 {@code &&} 连接。
*/
private String buildFilter(String category) {
List<String> parts = new ArrayList<>();
String categoryFilter = trimToNull(category);
if (categoryFilter != null) {
parts.add("metadata[\"category\"] == \"" + escapeFilter(categoryFilter) + "\"");
}
String scope = trimToNull(kbScope);
if (scope != null) {
parts.add("metadata[\"kb_scope\"] == \"" + escapeFilter(scope) + "\"");
}
return parts.isEmpty() ? null : String.join(" && ", parts);
}
private MilvusClientV2 connect() {
String uri;
if (milvusProperties.isSecure() || milvusProperties.getPort() == 443) {
uri = "https://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
} else {
uri = "http://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
}
ConnectConfig.ConnectConfigBuilder builder = ConnectConfig.builder()
.uri(uri)
.connectTimeoutMs(milvusProperties.getTimeout() == null ? 10000L : milvusProperties.getTimeout());
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isBlank()) {
builder.token(milvusProperties.getToken());
builder.secure(true);
} else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isBlank()) {
builder.username(milvusProperties.getUsername());
builder.password(milvusProperties.getPassword());
}
if (milvusProperties.getDatabase() != null && !milvusProperties.getDatabase().isBlank()) {
builder.dbName(milvusProperties.getDatabase());
}
log.info("Connecting MilvusClientV2 uri={} db={} collection={}",
uri, milvusProperties.getDatabase(), collectionName);
return new MilvusClientV2(builder.build());
}
/**
* 若不存在则创建 dense+BM25 hybrid collection。
*
* <p>关键点:</p>
* <ul>
* <li>{@code search_text} 开启 analyzer,作为 BM25 语料。</li>
* <li>{@link FunctionType#BM25}:input={@code search_text} → output={@code sparse_vector}。</li>
* <li>dense:IVF_FLAT + L2;sparse:SPARSE_INVERTED_INDEX + BM25。</li>
* </ul>
* <p>已存在的 collection 不会改 schema;schema 变更需走 {@link #dropAndRecreateCollection()}。</p>
*/
private void ensureCollection(MilvusClientV2 milvusClient) {
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
.collectionName(collectionName)
.build());
if (Boolean.TRUE.equals(exists)) {
log.info("Hybrid collection '{}' already exists", collectionName);
return;
}
log.info("Creating hybrid collection '{}'", collectionName);
CreateCollectionReq.CollectionSchema schema = milvusClient.createSchema();
schema.setEnableDynamicField(false);
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_ID)
.dataType(DataType.VarChar)
.maxLength(MilvusConstants.ID_MAX_LENGTH)
.isPrimaryKey(true)
.autoID(false)
.build());
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_CONTENT)
.dataType(DataType.VarChar)
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
.build());
// BM25 语料字段:必须 enableAnalyzer,Function 才能从文本生成 sparse
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_SEARCH_TEXT)
.dataType(DataType.VarChar)
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
.enableAnalyzer(true)
.build());
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_SPARSE)
.dataType(DataType.SparseFloatVector)
.build());
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_DENSE)
.dataType(DataType.FloatVector)
.dimension(milvusProperties.getVectorDim())
.build());
schema.addField(AddFieldReq.builder()
.fieldName(FIELD_METADATA)
.dataType(DataType.JSON)
.build());
// 写入 search_text 时,Milvus 自动维护 sparse_vector(应用层 insert 不填 sparse)
schema.addFunction(CreateCollectionReq.Function.builder()
.functionType(FunctionType.BM25)
.name("bm25_fn")
.inputFieldNames(List.of(FIELD_SEARCH_TEXT))
.outputFieldNames(List.of(FIELD_SPARSE))
.build());
milvusClient.createCollection(CreateCollectionReq.builder()
.collectionName(collectionName)
.collectionSchema(schema)
.description("Knowledge hybrid dense+BM25 collection")
.numShards(MilvusConstants.DEFAULT_SHARD_NUMBER)
.build());
List<IndexParam> indexes = List.of(
IndexParam.builder()
.fieldName(FIELD_DENSE)
.indexType(IndexParam.IndexType.IVF_FLAT)
.metricType(IndexParam.MetricType.L2)
.extraParams(Map.of("nlist", 128))
.build(),
IndexParam.builder()
.fieldName(FIELD_SPARSE)
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
.metricType(IndexParam.MetricType.BM25)
.build()
);
milvusClient.createIndex(CreateIndexReq.builder()
.collectionName(collectionName)
.indexParams(indexes)
.build());
log.info("Hybrid collection '{}' created with dense+BM25 indexes", collectionName);
}
private void loadCollection(MilvusClientV2 milvusClient) {
milvusClient.loadCollection(LoadCollectionReq.builder()
.collectionName(collectionName)
.build());
}
@PreDestroy
public void close() {
if (client != null) {
try {
client.close();
} catch (Exception e) {
log.warn("Error closing MilvusClientV2: {}", e.getMessage());
}
}
}
/** 过滤表达式字符串转义,防止引号打断 expr。 */
private static String escapeFilter(String value) {
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static String trimToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
private static String stringVal(Object value) {
return value == null ? null : String.valueOf(value);
}
}
@@ -5,9 +5,9 @@ import java.util.List;
/** /**
* 知识语义检索的应用边界端口。 * 知识语义检索的应用边界端口。
* *
* <p>实现可对接 dense / hybrid 等引擎,但不得向上层泄漏 SDK 类型。 * <p>实现可对接 dense / hybrid 等引擎,但不得向上层泄漏远端 API 类型。
* 当前实现:{@link VectorKnowledgeSearchAdapter} → {@code VectorSearchService} * 当前实现:{@link PyRagKnowledgeSearchAdapter}(py-rag 知识服务 /api/v1/search,
* → {@code MilvusHybridKnowledgeStore}(Milvus SDK v2 dense 或 dense+BM25 RRF)。</p> * 服务端负责 hybrid 融合、BM25、rerank 与判级)。RAG 模块抽离后为唯一实现。</p>
*/ */
public interface KnowledgeSearchPort { public interface KnowledgeSearchPort {
@@ -1,73 +0,0 @@
package com.superbiz.agent.service.retrieval;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
/**
* Sparse-lite lexical ranking over already recalled candidates.
* Not a substitute for inverted-index BM25; expands ordering signal only.
*/
public final class LexicalRanker {
private LexicalRanker() {
}
public static List<KnowledgeSearchHit> rank(String query, List<KnowledgeSearchHit> candidates) {
if (candidates == null || candidates.isEmpty()) {
return List.of();
}
Set<String> terms = tokenize(query);
if (terms.isEmpty()) {
return List.copyOf(candidates);
}
List<ScoredHit> scored = new ArrayList<>(candidates.size());
for (KnowledgeSearchHit hit : candidates) {
String haystack = (nullToEmpty(hit.title()) + " "
+ nullToEmpty(hit.breadcrumb()) + " "
+ nullToEmpty(hit.content())).toLowerCase(Locale.ROOT);
int hits = 0;
for (String term : terms) {
if (haystack.contains(term)) {
hits++;
}
}
double coverage = hits / (double) terms.size();
scored.add(new ScoredHit(hit, coverage, hits));
}
scored.sort(Comparator
.comparingDouble((ScoredHit s) -> s.coverage).reversed()
.thenComparingInt((ScoredHit s) -> s.hits).reversed()
.thenComparingInt(s -> s.hit.originalRank()));
return scored.stream().map(s -> s.hit).toList();
}
static Set<String> tokenize(String query) {
if (query == null || query.isBlank()) {
return Set.of();
}
String normalized = query.toLowerCase(Locale.ROOT);
String[] parts = normalized.split("[^\\p{IsAlphabetic}\\p{IsDigit}]+");
Set<String> terms = new LinkedHashSet<>();
for (String part : parts) {
if (part == null) {
continue;
}
String term = part.trim();
if (term.length() >= 2) {
terms.add(term);
}
}
return terms;
}
private static String nullToEmpty(String value) {
return value == null ? "" : value;
}
private record ScoredHit(KnowledgeSearchHit hit, double coverage, int hits) {
}
}
@@ -0,0 +1,118 @@
package com.superbiz.agent.service.retrieval;
import com.superbiz.agent.client.PyRagClient;
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* {@link KnowledgeSearchPort} 的 py-rag 远端实现(RAG 模块抽离后的唯一检索后端)。
*
* <p>原进程内链路(VectorKnowledgeSearchAdapter → VectorSearchService → MilvusHybridKnowledgeStore)
* 已由 py-rag 服务端接管:hybrid 融合、BM25、rerank、chunk 去重、判级阈值全部下沉。
* Java 侧只做请求映射与命中结构归一化,不碰检索算法。</p>
*
* <h3>映射约定</h3>
* <ul>
* <li>mode:{@link KnowledgeSearchMode#DENSE} → {@code semantic},{@link KnowledgeSearchMode#HYBRID} → {@code hybrid}</li>
* <li>retrieve_k = return_n = topK:返回 topK 条精排后命中,chunk 去重/截断仍由
* {@code KnowledgeEvidencePostProcessor} 统一负责,故 max_chunks_per_document 同步放大避免服务端预截断</li>
* <li>category:{@code categoryFilter} 透传;null = 不过滤;kb_scope 不传,由服务端部署配置决定</li>
* <li>score:py-rag rerank 绝对相关分([0,1],越大越好),scoreLabel =
* {@link RetrievalScoreLabels#RERANK}(quality 原样采用,不做 L2/rank 归一化)</li>
* <li>evidence_key:{@code docId#chunk-N},与 EvidenceGuard 验真约定一致;
* {@code evidence_status=no_evidence} 时服务端保证 hits=[],按"无知识"正常返回</li>
* </ul>
*/
@Component
public class PyRagKnowledgeSearchAdapter implements KnowledgeSearchPort {
private static final String CHUNK_MARK = "#chunk-";
private final PyRagClient pyRagClient;
public PyRagKnowledgeSearchAdapter(PyRagClient pyRagClient) {
this.pyRagClient = pyRagClient;
}
@Override
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
PyRagSearchResponse response = pyRagClient.search(toPyRagRequest(request));
if (response == null || response.hits() == null || response.hits().isEmpty()) {
return List.of();
}
List<KnowledgeSearchHit> hits = new ArrayList<>(response.hits().size());
for (int i = 0; i < response.hits().size(); i++) {
hits.add(toHit(response.hits().get(i), i + 1));
}
return hits;
}
/** 请求映射:topK 同时作为召回宽度与返回条数,服务端不预截断 chunk。 */
private PyRagSearchRequest toPyRagRequest(KnowledgeSearchRequest request) {
return new PyRagSearchRequest(
request.query(),
request.mode() == KnowledgeSearchMode.HYBRID ? "hybrid" : "semantic",
request.topK(),
request.topK(),
request.topK(),
blankToNull(request.categoryFilter()),
null);
}
private KnowledgeSearchHit toHit(PyRagSearchHit hit, int originalRank) {
String docId = blankToNull(hit.documentId());
Integer chunkIndex = parseChunkIndex(hit.evidenceKey());
String evidenceKey = EvidenceIdentity.firstNonBlank(
hit.evidenceKey(),
EvidenceIdentity.evidenceKey(docId, chunkIndex, null, originalRank));
Double score = hit.qualityScore() == null ? 0.0 : hit.qualityScore();
return new KnowledgeSearchHit(
firstNonBlank(evidenceKey, docId, "rank:" + originalRank),
hit.excerpt(),
score,
score,
RetrievalScoreLabels.RERANK,
null,
Map.of(),
docId,
chunkIndex,
evidenceKey,
hit.source(),
hit.title(),
hit.breadcrumb(),
originalRank,
null);
}
/** evidence_key 形如 {@code docId#chunk-N},解析末尾 chunk 序号;不符返回 null。 */
private Integer parseChunkIndex(String evidenceKey) {
String key = blankToNull(evidenceKey);
if (key == null) {
return null;
}
int mark = key.lastIndexOf(CHUNK_MARK);
if (mark < 0) {
return null;
}
try {
return Integer.valueOf(key.substring(mark + CHUNK_MARK.length()).trim());
} catch (NumberFormatException ignored) {
return null;
}
}
private String blankToNull(String value) {
return value == null || value.isBlank() ? null : value.trim();
}
private String firstNonBlank(String... values) {
return EvidenceIdentity.firstNonBlank(values);
}
}
@@ -3,8 +3,13 @@ package com.superbiz.agent.service.retrieval;
/** /**
* 检索结果一级 {@code scoreLabel} 约定。 * 检索结果一级 {@code scoreLabel} 约定。
* *
* <p>只区分两种检索形态(与 {@code retrieval.search.mode} 对齐), * <p>三种检索形态:</p>
* 不再使用 {@code bm25_only_*} 等作为正式一级 label。</p> * <ul>
* <li>{@link #DENSE} —— 单路向量 ANN(L2 距离,越小越好)</li>
* <li>{@link #HYBRID} —— dense + BM25 + RRF 融合(质量主要看 rank)</li>
* <li>{@link #RERANK} —— py-rag 服务端 rerank 绝对相关分([0,1],越大越好);
* RAG 模块抽离后的线上主路径</li>
* </ul>
*/ */
public final class RetrievalScoreLabels { public final class RetrievalScoreLabels {
@@ -14,11 +19,14 @@ public final class RetrievalScoreLabels {
/** hybrid(dense+BM25+RRF):{@code score}/raw 为融合侧信号;质量分主要看 rank。 */ /** hybrid(dense+BM25+RRF):{@code score}/raw 为融合侧信号;质量分主要看 rank。 */
public static final String HYBRID = "hybrid"; public static final String HYBRID = "hybrid";
/** py-rag 服务端 rerank 绝对分:{@code score} 即归一化质量([0,1],越大越好)。 */
public static final String RERANK = "rerank";
private RetrievalScoreLabels() { private RetrievalScoreLabels() {
} }
/** /**
* 将历史/别名 label 归一到 {@link #DENSE} 或 {@link #HYBRID}。 * 将历史/别名 label 归一到 {@link #DENSE}、{@link #HYBRID} 或 {@link #RERANK}。
* 未知或空 → dense(保守,按 L2 解释失败时 quality 偏低)。 * 未知或空 → dense(保守,按 L2 解释失败时 quality 偏低)。
*/ */
public static String canonicalize(String scoreLabel) { public static String canonicalize(String scoreLabel) {
@@ -29,12 +37,19 @@ public final class RetrievalScoreLabels {
return switch (label) { return switch (label) {
case DENSE, "l2_distance", "l2" -> DENSE; case DENSE, "l2_distance", "l2" -> DENSE;
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID; case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
default -> label.contains("hybrid") || label.contains("rrf") || label.contains("bm25") case RERANK, "rerank_score", "quality_score" -> RERANK;
? HYBRID default -> label.contains("rerank") || label.contains("quality")
: DENSE; ? RERANK
: label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
? HYBRID
: DENSE;
}; };
} }
public static boolean isRerank(String scoreLabel) {
return RERANK.equals(canonicalize(scoreLabel));
}
public static boolean isHybrid(String scoreLabel) { public static boolean isHybrid(String scoreLabel) {
return HYBRID.equals(canonicalize(scoreLabel)); return HYBRID.equals(canonicalize(scoreLabel));
} }
@@ -6,6 +6,8 @@ package com.superbiz.agent.service.retrieval;
* <p>后处理排序仍按 {@code originalRank};本类只负责质量闸门 / relevance 用分。</p> * <p>后处理排序仍按 {@code originalRank};本类只负责质量闸门 / relevance 用分。</p>
* *
* <ul> * <ul>
* <li>{@link RetrievalScoreLabels#RERANK}:py-rag 服务端 rerank 绝对分,
* {@code score} 已归一化,原样 clamp 到 [0,1](RAG 抽离后的主路径)</li>
* <li>{@link RetrievalScoreLabels#DENSE}:{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li> * <li>{@link RetrievalScoreLabels#DENSE}:{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li>
* <li>{@link RetrievalScoreLabels#HYBRID}:优先用可选 {@code denseDistance} 做绝对质量 * <li>{@link RetrievalScoreLabels#HYBRID}:优先用可选 {@code denseDistance} 做绝对质量
* (恢复 L0 filter low-quality 等闸门);无 dense 时回退 rank 映射</li> * (恢复 L0 filter low-quality 等闸门);无 dense 时回退 rank 映射</li>
@@ -17,8 +19,8 @@ public final class RetrievalScoreNormalizer {
} }
/** /**
* @param scoreLabel {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID} * @param scoreLabel {@link RetrievalScoreLabels#RERANK} / {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
* @param score 引擎主分:dense=L2;hybrid=融合分(hybrid 质量不依赖其量纲) * @param score 引擎主分:rerank=绝对相关分[0,1];dense=L2;hybrid=融合分(hybrid 质量不依赖其量纲)
* @param originalRank 检索名次(1-based) * @param originalRank 检索名次(1-based)
* @param batchSize 本轮候选数(rank 回退映射用) * @param batchSize 本轮候选数(rank 回退映射用)
* @param maxL2Distance L2 上界 * @param maxL2Distance L2 上界
@@ -31,6 +33,12 @@ public final class RetrievalScoreNormalizer {
double maxL2Distance, double maxL2Distance,
Double denseDistance) { Double denseDistance) {
String label = RetrievalScoreLabels.canonicalize(scoreLabel); String label = RetrievalScoreLabels.canonicalize(scoreLabel);
if (RetrievalScoreLabels.RERANK.equals(label)) {
if (score == null) {
return 0.0;
}
return Math.max(0.0, Math.min(1.0, score));
}
if (RetrievalScoreLabels.HYBRID.equals(label)) { if (RetrievalScoreLabels.HYBRID.equals(label)) {
if (denseDistance != null) { if (denseDistance != null) {
return l2ToQuality(denseDistance, maxL2Distance); return l2ToQuality(denseDistance, maxL2Distance);
@@ -1,100 +0,0 @@
package com.superbiz.agent.service.retrieval;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
/**
* Reciprocal Rank Fusion 工具:把多路检索的排名列表融合成一个分数排序。
*
* <pre>
* RRF_w(d) = Σ w_i / (k + rank_i(d))
* </pre>
*
* <p>只依赖排名不依赖原始分数——屏蔽跨路分数尺度不可比的问题;
* 每路可加权(w &lt;= 0 时按 1.0 等权),k 是平滑参数(默认 60,可配)。
*/
public final class RrfFusion {
private RrfFusion() {
}
/**
* 融合多路排名:对每路的每个 item 累加 w/(k+rank),按总分降序输出。
*
* @param paths 多路排名(每路带 name / items / weight)
* @param rrfK 平滑参数 k(至少 1)
* @param identityFn 跨路识别同一 item 的身份函数(如 evidenceKey)
* @return 融合后排序(含每路排名明细)
*/
public static <T> List<Scored<T>> fuse(List<RankedPath<T>> paths,
int rrfK,
Function<T, String> identityFn) {
if (paths == null || paths.isEmpty()) {
return List.of();
}
int k = Math.max(1, rrfK);
Map<String, Acc<T>> acc = new LinkedHashMap<>();
for (RankedPath<T> path : paths) {
if (path == null || path.items() == null || path.items().isEmpty()) {
continue;
}
double weight = path.weight() <= 0 ? 1.0 : path.weight();
List<T> items = path.items();
for (int i = 0; i < items.size(); i++) {
T item = items.get(i);
if (item == null) {
continue;
}
String id = identityFn.apply(item);
if (id == null || id.isBlank()) {
continue;
}
int rank = i + 1;
double contrib = weight / (k + rank); // 排名越前贡献越大
Acc<T> bucket = acc.computeIfAbsent(id, ignored -> new Acc<>(item));
bucket.score += contrib;
bucket.ranks.put(path.name(), rank);
// Prefer first-seen item payload; callers should put preferred path first if needed.
}
}
List<Scored<T>> scored = new ArrayList<>(acc.size());
for (Map.Entry<String, Acc<T>> entry : acc.entrySet()) {
Acc<T> value = entry.getValue();
scored.add(new Scored<>(entry.getKey(), value.item, value.score, Map.copyOf(value.ranks)));
}
// 总分降序(两路共识的靠前),同分按身份稳定排序
scored.sort(Comparator
.comparingDouble((Scored<T> s) -> s.rrfScore()).reversed()
.thenComparing(Scored::identity));
return scored;
}
/** 一路检索结果:name(路名)+ items(按排名顺序)+ weight(可选加权,≤0 视为等权)。 */
public record RankedPath<T>(String name, List<T> items, double weight) {
public RankedPath {
Objects.requireNonNull(name, "name");
items = items == null ? List.of() : List.copyOf(items);
}
}
/** 融合后的单个 item:identity + 原始 item + rrfScore + 每路排名明细。 */
public record Scored<T>(String identity, T item, double rrfScore, Map<String, Integer> ranks) {
}
/** 跨路累加器:同一 identity 的 item 累加 RRF 分并记录各路排名。 */
private static final class Acc<T> {
private final T item;
private double score;
private final Map<String, Integer> ranks = new HashMap<>();
private Acc(T item) {
this.item = item;
}
}
}
@@ -1,101 +0,0 @@
package com.superbiz.agent.service.retrieval;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.VectorSearchService;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* {@link KnowledgeSearchPort} 适配器:把向量检索结果映射为带 evidenceKey 的命中结构。
*
* <p>委托 {@link VectorSearchService}(背后仅 {@code MilvusHybridKnowledgeStore}):
* dense 或 dense+BM25 hybrid 由配置 {@code retrieval.search.mode} 选择。
* 本类负责 metadata 解析、docId/chunk 身份与 evidenceKey,不碰 SDK。</p>
*/
@Component
public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
private final VectorSearchService vectorSearchService;
private final ObjectMapper objectMapper;
public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
this.vectorSearchService = vectorSearchService;
this.objectMapper = objectMapper;
}
@Override
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
// Mode is global on VectorSearchService; request.mode is advisory for future per-call overrides.
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(
request.query(),
request.topK(),
request.categoryFilter());
return toHits(results);
}
private List<KnowledgeSearchHit> toHits(List<VectorSearchService.SearchResult> results) {
if (results == null || results.isEmpty()) {
return List.of();
}
List<KnowledgeSearchHit> hits = new ArrayList<>(results.size());
for (int i = 0; i < results.size(); i++) {
hits.add(toHit(results.get(i), i + 1));
}
return hits;
}
private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
Map<String, String> metadata = parseMetadata(result.getMetadata());
String docId = EvidenceIdentity.extractDocId(
metadata,
EvidenceIdentity.metadataValue(metadata, "_source"),
EvidenceIdentity.metadataValue(metadata, "source"));
Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
String source = EvidenceIdentity.firstNonBlank(
EvidenceIdentity.metadataValue(metadata, "_source"),
EvidenceIdentity.metadataValue(metadata, "source"),
EvidenceIdentity.metadataValue(metadata, "filePath"),
docId,
result.getId());
return new KnowledgeSearchHit(
result.getId(),
result.getContent(),
(double) result.getScore(),
result.getRawScore(),
result.getScoreLabel(),
result.getMetadata(),
metadata,
docId,
chunkIndex,
evidenceKey,
source,
EvidenceIdentity.metadataValue(metadata, "title"),
EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
originalRank,
result.getDenseDistance()
);
}
private Map<String, String> parseMetadata(String metadata) {
if (metadata == null || metadata.isBlank()) {
return Map.of();
}
try {
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
Map<String, String> parsed = new LinkedHashMap<>();
for (Map.Entry<?, ?> entry : raw.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
}
}
return parsed;
} catch (Exception ignored) {
return Map.of();
}
}
}
@@ -1,69 +0,0 @@
package com.superbiz.agent.tool;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.ConnectParam;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.DropCollectionParam;
import io.milvus.param.collection.HasCollectionParam;
/**
* 删除 Milvus Collection 的工具类
* 用于重建 Collection 时清理旧数据
*/
public class DropCollection {
public static void main(String[] args) {
MilvusServiceClient client = null;
try {
// 连接到 Milvus
System.out.println("正在连接到 Milvus localhost:19530...");
client = new MilvusServiceClient(
ConnectParam.newBuilder()
.withHost("localhost")
.withPort(19530)
.build()
);
System.out.println("✓ 连接成功");
String collectionName = "biz";
// 检查 Collection 是否存在
R<Boolean> hasResponse = client.hasCollection(
HasCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build()
);
if (hasResponse.getData()) {
System.out.println("发现 Collection: " + collectionName);
System.out.println("正在删除...");
// 删除 Collection
R<RpcStatus> dropResponse = client.dropCollection(
DropCollectionParam.newBuilder()
.withCollectionName(collectionName)
.build()
);
if (dropResponse.getStatus() == 0) {
System.out.println("✓ Collection 已成功删除");
System.out.println("\n请重启 Spring Boot 应用,它会自动创建新的 FloatVector Collection");
} else {
System.err.println("✗ 删除失败: " + dropResponse.getMessage());
}
} else {
System.out.println("Collection '" + collectionName + "' 不存在");
}
} catch (Exception e) {
System.err.println("错误: " + e.getMessage());
e.printStackTrace();
} finally {
if (client != null) {
client.close();
}
}
}
}
@@ -8,7 +8,6 @@ import com.superbiz.agent.dto.RetrievalTrace;
import com.superbiz.agent.service.KnowledgeContextPacker; import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever; import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor; import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler; import com.superbiz.agent.service.LookupResultAssembler;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -29,10 +28,9 @@ import java.util.Map;
* *
* <h3>主链路</h3> * <h3>主链路</h3>
* <pre> * <pre>
* query * query(原始句直传;L0 query 理解已下沉 py-rag 服务端)
* -> KnowledgeQueryTransformer * -> KnowledgeDocumentRetriever (via KnowledgeSearchPort → py-rag, retrieve-k)
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k) * -> KnowledgeEvidencePostProcessor (qualityScore / chunk dedup / caps / return-n)
* -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
* -> [optional] unfiltered retry * -> [optional] unfiltered retry
* -> KnowledgeContextPacker * -> KnowledgeContextPacker
* -> LookupResultAssembler * -> LookupResultAssembler
@@ -62,9 +60,6 @@ public class LookupKnowledgeTool {
private int retrieveK = 20; private int retrieveK = 20;
@Autowired
private KnowledgeQueryTransformer queryTransformer;
@Autowired @Autowired
private KnowledgeDocumentRetriever documentRetriever; private KnowledgeDocumentRetriever documentRetriever;
@@ -98,10 +93,10 @@ public class LookupKnowledgeTool {
* *
* <p>流程(模块化三段): * <p>流程(模块化三段):
* <ol> * <ol>
* <li>检索前:QueryTransformer.transform → KnowledgeQuery(分类过滤/域/关键词);</li> * <li>检索前:原始 query 直传(L0 domain 分析已下沉 py-rag,categoryFilter 恒为 null);</li>
* <li>检索:DocumentRetriever.retrieve(FILTERED 或 UNFILTERED,retrieveK 候选);</li> * <li>检索:DocumentRetriever.retrieve(UNFILTERED,retrieveK 候选,py-rag 服务端融合+精排);</li>
* <li>检索后:PostProcessor.process(qualityScore/去重/判级);</li> * <li>检索后:PostProcessor.process(qualityScore/去重/判级);</li>
* <li>低质量降级:带分类过滤结果低质 → 去掉过滤、用原始 query 重查;</li> * <li>低质量降级:带分类过滤结果低质 → 去掉过滤、用原始 query 重查(L0 移除后保留兜底语义);</li>
* <li>打包 + 组装:ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li> * <li>打包 + 组装:ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li>
* </ol> * </ol>
* *
@@ -113,12 +108,17 @@ public class LookupKnowledgeTool {
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK); log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
log.info("----------------------------------------"); log.info("----------------------------------------");
// ── 检索前:查询理解(L0)── // ── 检索前:原始 query 直传(L0 已下沉 py-rag,不做 Java 侧 category 收窄)──
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query); String normalized = query == null ? "" : query.trim();
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}", KnowledgeQuery knowledgeQuery = KnowledgeQuery.builder()
knowledgeQuery.getCategoryFilter(), .originalQuery(normalized)
knowledgeQuery.getDomainHints().size(), .rewrittenQuery(normalized)
knowledgeQuery.getMatchedKeywords().size()); .domainHints(List.of())
.matchedKeywords(List.of())
.entities(List.of())
.l0Titles(List.of())
.l0MatchCount(0)
.build();
List<RetrievalTrace.Attempt> attempts = new ArrayList<>(); List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
String fallbackReason = null; String fallbackReason = null;
+18 -70
View File
@@ -15,30 +15,28 @@ file:
knowledge: knowledge:
base-path: knowledge_base/ base-path: knowledge_base/
milvus: # =====================================================
host: in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com # py-rag 知识服务接入
port: 443 # =====================================================
username: "" # RAG 检索与文档入库均由 py-rag 服务承担(契约见 py-rag 仓库 docs/Java接入文档.md):
password: "" # 检索 /api/v1/search,入库 /api/v1/documents:ingest,全量重建 /api/v1/collections:rebuild。
database: db_4a578da0f27ce9d pyrag:
timeout: 10000 base-url: ${PYRAG_BASE_URL:http://localhost:8000}
token: ${MILVUS_TOKEN} connect-timeout-ms: 3000
secure: true search-read-timeout-ms: 5000 # 正常 300–800ms(含 rerank 外呼)
vector-dim: 1024 # BGE-M3 = 1024,换模型时同步改 ingest-read-timeout-ms: 30000 # 正常 1–5s
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema) default-read-timeout-ms: 10000
collection: biz
# ===================================================== # =====================================================
# 模型路由配置 # 模型路由配置
# ===================================================== # =====================================================
# 通过关键字匹配 Bean,切换模型只改这里 + 对应 api-key # 通过关键字匹配 Bean,切换模型只改这里 + 对应 api-key
# Chat: deepseek | openai | ollama | ... # Chat: deepseek | openai | ollama | ...
# Embedding: siliconflow | openai | ollama | dashscope | ... # (Embedding 已随 RAG 抽离至 py-rag 服务端)
# ===================================================== # =====================================================
model-routing: model-routing:
chat: deepseek chat: deepseek
embedding: siliconflow
spring: spring:
config: config:
@@ -102,30 +100,6 @@ spring:
retry: retry:
max-attempts: 1 max-attempts: 1
vectorstore:
type: milvus
milvus:
initialize-schema: false
database-name: ${milvus.database}
collection-name: biz
embedding-dimension: ${milvus.vector-dim}
index-type: IVF_FLAT
metric-type: L2
index-parameters: '{"nlist":128}'
id-field-name: id
auto-id: false
content-field-name: content
metadata-field-name: metadata
embedding-field-name: vector
client:
host: ${milvus.host}
port: ${milvus.port}
token: ${milvus.token}
username: ${milvus.username}
password: ${milvus.password}
secure: ${milvus.secure}
connect-timeout-ms: ${milvus.timeout}
# --- Chat: DeepSeek (原生) --- # --- Chat: DeepSeek (原生) ---
deepseek: deepseek:
api-key: ${DEEPSEEK_API_KEY} api-key: ${DEEPSEEK_API_KEY}
@@ -134,53 +108,27 @@ spring:
options: options:
model: deepseek-v4-flash model: deepseek-v4-flash
# --- OpenAI 模块供 SiliconFlow Embedding 复用 ---
openai:
api-key: unused
# Spring AI MCP 客户端配置 # Spring AI MCP 客户端配置
mcp: mcp:
client: client:
enabled: false enabled: false
# --- Embedding: SiliconFlow BGE-M3 ---
siliconflow:
api-key: ${SILICONFLOW_API_KEY}
base-url: https://api.siliconflow.cn
embedding:
model: BAAI/bge-m3
# 文档分片配置
document:
chunk:
max-size: 800
overlap: 100
# RAG 配置 # RAG 配置
rag: rag:
top-k: 3 # legacy fallback when retrieve-k/return-n absent top-k: 3 # legacy fallback when retrieve-k/return-n absent
retrieve-k: 20 retrieve-k: 20
return-n: 5 return-n: 5
max-chunks-per-document: 2 max-chunks-per-document: 2
sidecar:
spring-ai:
enabled: false
content-preview-limit: 300
# 检索配置 # 检索配置
# 知识主路径:Milvus Java SDK v2(MilvusHybridKnowledgeStore),非 Spring AI VectorStore starter。 # 知识主路径:py-rag 知识服务(PyRagKnowledgeSearchAdapter → /api/v1/search)。
# 原因:starter(含 2.0.0)仅 dense similarity,无 hybridSearch / BM25 Function / RRFRanker。 # 服务端负责 dense+BM25 融合、rerank(BGE-Reranker)与判级;Java 侧只做请求映射与后处理。
# 已移除 legacy sdk/spring/auto 多后端路由。
retrieval: retrieval:
kb-scope: "" # 非空则过滤 metadata.kb_scope;空=不过滤
search: search:
# hybrid=线上主路径;dense=同库对照/评测/排障(非第二套线上策略)。见 mvp/architecture/rag-knowledge-retrieval-architecture.md §6.0 # hybrid=线上主路径;dense 为对照/排障(映射 py-rag mode:hybrid→hybrid,dense→semantic)
mode: hybrid # dense=单路L2对照 | hybrid=dense+服务端BM25+RRF mode: hybrid
hybrid:
rrf-k: 60 # RRF 平滑参数 k,score=Σ 1/(k+rank)
normalization: normalization:
max-l2-distance: 2.0 # dense quality:L2 上界(单位向量 ≈ 2.0) highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE(与 py-rag 判级阈值一致)
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE(hybrid 为序数分,见架构 §6)
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE;低于则低质/可 unfiltered retry reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE;低于则低质/可 unfiltered retry
# Prometheus 配置 # Prometheus 配置
@@ -11,7 +11,6 @@ import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.service.KnowledgeContextPacker; import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever; import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor; import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler; import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.repository.AgentStepRepository; import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.AgentReasoningAuditRepository; import com.superbiz.agent.repository.AgentReasoningAuditRepository;
@@ -84,7 +83,6 @@ class HarnessChatConfigurationTest {
.withBean(ChatModel.class, () -> mock(ChatModel.class)) .withBean(ChatModel.class, () -> mock(ChatModel.class))
.withBean(RedisTemplate.class, () -> mock(RedisTemplate.class)) .withBean(RedisTemplate.class, () -> mock(RedisTemplate.class))
.withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class)) .withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class))
.withBean(KnowledgeQueryTransformer.class, () -> mock(KnowledgeQueryTransformer.class))
.withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class)) .withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class))
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class)) .withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class)) .withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
@@ -4,18 +4,13 @@ import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.test.context.TestPropertySource;
import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.api.Assertions.*;
/** /**
* 单独测试 Redis 连接 * 单独测试 Redis 连接(需要可达的 Redis 实例)。
* 禁用 Milvus 以避免启动失败
*/ */
@SpringBootTest @SpringBootTest
@TestPropertySource(properties = {
"spring.autoconfigure.exclude=org.example.config.MilvusConfig"
})
class RedisConnectionTest { class RedisConnectionTest {
@Autowired(required = false) @Autowired(required = false)
@@ -1,93 +0,0 @@
package com.superbiz.agent.eval;
import com.superbiz.agent.Main;
import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.dto.DocumentUploadRequest;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.service.DocumentManagementService;
import com.superbiz.agent.service.FrontmatterParser;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.mock.web.MockMultipartFile;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
/**
* Imports canonical RAG eval documents through the real document pipeline.
*
* <p>Disabled by default because it writes DB rows, local knowledge files, and
* vector index records in the configured runtime environment.</p>
*/
@SpringBootTest(
classes = Main.class,
webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = "spring.main.web-application-type=none"
)
@EnabledIfSystemProperty(named = "rag.seed.enabled", matches = "true")
class RagEvalSeedImporterTest {
private static final Path DEFAULT_SEED_DOCS = Path.of("eval/rag-retrieval/seed-docs");
@Autowired
private DocumentManagementService documentManagementService;
@Autowired
private FrontmatterParser frontmatterParser;
@Autowired
private ApiDocumentRepository apiDocumentRepository;
@Test
void importSeedDocuments() throws Exception {
Path seedDir = Path.of(System.getProperty("rag.seed.docs", DEFAULT_SEED_DOCS.toString()));
List<Path> docs;
try (var stream = Files.list(seedDir)) {
docs = stream
.filter(path -> path.getFileName().toString().endsWith(".md"))
.sorted()
.toList();
}
assertFalse(docs.isEmpty(), "seed docs directory must contain markdown files");
for (Path docPath : docs) {
String content = Files.readString(docPath, StandardCharsets.UTF_8);
Frontmatter frontmatter = frontmatterParser.parse(content);
if (frontmatter == null || frontmatter.getSource() == null || frontmatter.getSource().isBlank()) {
throw new IllegalArgumentException("seed doc must include frontmatter source: " + docPath);
}
apiDocumentRepository.findByDocId(frontmatter.getSource().trim())
.map(ApiDocument::getDocId)
.ifPresent(documentManagementService::deleteDocument);
String fileName = docPath.getFileName().toString();
MockMultipartFile file = new MockMultipartFile(
"file",
fileName,
"text/markdown",
content.getBytes(StandardCharsets.UTF_8)
);
DocumentUploadRequest request = DocumentUploadRequest.builder()
.file(file)
.category(resolveCategory(frontmatter))
.build();
documentManagementService.uploadDocument(request);
}
}
private String resolveCategory(Frontmatter frontmatter) {
if (frontmatter.getCategory() != null && !frontmatter.getCategory().isBlank()) {
return frontmatter.getCategory().trim();
}
return "rag-eval";
}
}
@@ -1,127 +0,0 @@
package com.superbiz.agent.eval;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.superbiz.agent.Main;
import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.tool.LookupKnowledgeTool;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Locale;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
*
* <p>Disabled by default: writes repository files and needs the live retrieval stack
* (embedding + Milvus hybrid collection + optional MySQL/L0).</p>
*
* <p>System properties (via Maven {@code -D}):</p>
* <ul>
* <li>{@code rag.snapshot.enabled=true} — required to run</li>
* <li>{@code retrieval.search.mode=hybrid|dense} — default hybrid</li>
* <li>{@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})</li>
* <li>{@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}</li>
* </ul>
*/
@SpringBootTest(
classes = Main.class,
webEnvironment = SpringBootTest.WebEnvironment.NONE,
properties = "spring.main.web-application-type=none"
)
@EnabledIfSystemProperty(named = "rag.snapshot.enabled", matches = "true")
class RagLookupSnapshotGeneratorTest {
private static final Path DEFAULT_CASES = Path.of("eval/rag-retrieval/cases/golden-cases.json");
private static final Path DEFAULT_FIXTURES = Path.of("eval/rag-retrieval/fixtures");
@Autowired
private LookupKnowledgeTool lookupKnowledgeTool;
@Autowired
private ObjectMapper objectMapper;
/**
* Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
*/
@DynamicPropertySource
static void retrievalProperties(DynamicPropertyRegistry registry) {
String mode = System.getProperty("retrieval.search.mode", "hybrid");
if (mode == null || mode.isBlank()) {
mode = "hybrid";
}
String normalized = mode.trim().toLowerCase(Locale.ROOT);
registry.add("retrieval.search.mode", () -> normalized);
String kbScope = System.getProperty("retrieval.kb-scope", "");
if (kbScope != null && !kbScope.isBlank()) {
registry.add("retrieval.kb-scope", kbScope::trim);
}
}
@Test
void generateLookupResultFixtures() throws Exception {
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
JsonNode root = objectMapper.readTree(casesPath.toFile());
JsonNode cases = root.path("cases");
assertTrue(cases.isArray(), "golden cases file must contain a cases array");
Files.createDirectories(fixturesDir);
for (JsonNode testCase : cases) {
String caseId = requiredText(testCase, "caseId");
String query = requiredText(testCase, "query");
LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
ObjectNode fixture = objectMapper.createObjectNode();
fixture.put("caseId", caseId);
fixture.put("query", query);
fixture.put("retrievedAt", retrievedAt);
fixture.put("searchMode", searchMode);
if (kbScope != null) {
fixture.put("kbScope", kbScope);
}
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
Path output = fixturesDir.resolve(caseId + ".json");
objectMapper.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), fixture);
}
}
private static String normalizeMode(String mode) {
if (mode == null || mode.isBlank()) {
return "hybrid";
}
return mode.trim().toLowerCase(Locale.ROOT);
}
private static String blankToNull(String value) {
if (value == null || value.isBlank()) {
return null;
}
return value.trim();
}
private String requiredText(JsonNode node, String fieldName) {
JsonNode value = node.get(fieldName);
if (value == null || value.asText().isBlank()) {
throw new IllegalArgumentException("golden case is missing required field: " + fieldName);
}
return value.asText();
}
}
@@ -1,539 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.config.DocumentChunkConfig;
import com.superbiz.agent.dto.DocumentChunk;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* 当前分片策略的单元测试 — 覆盖旧能力回归 + Phase 1 新增能力
*/
@DisplayName("DocumentChunkService 分片策略")
class DocumentChunkServiceTest {
private DocumentChunkService service;
private DocumentChunkConfig config;
@BeforeEach
void setUp() {
config = new DocumentChunkConfig();
config.setMaxSize(800);
config.setMaxTokens(500);
config.setMaxTokensHard(600);
config.setOverlap(100);
service = new DocumentChunkService();
try {
var field = DocumentChunkService.class.getDeclaredField("chunkConfig");
field.setAccessible(true);
field.set(service, config);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// ==================== 回归:边界条件 ====================
@Nested
@DisplayName("边界条件")
class BoundaryTests {
@Test
@DisplayName("null 内容 → 空列表")
void nullContent_returnsEmpty() {
List<DocumentChunk> chunks = service.chunkDocument(null, "/test/null.md");
assertTrue(chunks.isEmpty());
}
@Test
@DisplayName("空字符串 → 空列表")
void emptyContent_returnsEmpty() {
List<DocumentChunk> chunks = service.chunkDocument(" \n ", "/test/empty.md");
assertTrue(chunks.isEmpty());
}
@Test
@DisplayName("短文档(≤maxSize)→ 1个分块")
void shortDocument_singleChunk() {
String content = "这是一篇短文档,内容不超过800个字符。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/short.md");
assertEquals(1, chunks.size());
assertEquals(content, chunks.get(0).getContent());
assertEquals(0, chunks.get(0).getChunkIndex());
}
@Test
@DisplayName("恰好 maxSize 边界 → 1个分块")
void exactlyMaxSize_singleChunk() {
String content = "A".repeat(800);
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/boundary.md");
assertEquals(1, chunks.size());
}
}
// ==================== 回归:标题分割 ====================
@Nested
@DisplayName("Markdown 标题分割")
class HeadingSplitTests {
@Test
@DisplayName("单个 H1 标题 → section 继承标题")
void singleHeading_titlePropagates() {
String content = "# CPU高负载问题\n\n这是CPU高负载的描述内容。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/cpu.md");
assertEquals(1, chunks.size());
assertEquals("CPU高负载问题", chunks.get(0).getTitle());
}
@Test
@DisplayName("多个标题 → 按标题边界分割")
void multipleHeadings_splitAtHeadings() {
String content =
"# CPU高负载\n\nCPU问题的详细描述。\n\n" +
"# 内存高负载\n\n内存问题的详细描述。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/multi.md");
assertEquals(2, chunks.size());
assertEquals("CPU高负载", chunks.get(0).getTitle());
assertEquals("内存高负载", chunks.get(1).getTitle());
}
@Test
@DisplayName("多级标题(H1/H2/H3)→ 标题独立不冲突")
void multiLevelHeadings() {
String content =
"# 一级标题\n\n一级内容。\n\n" +
"## 二级标题\n\n二级内容。\n\n" +
"### 三级标题\n\n三级内容。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/levels.md");
assertEquals(3, chunks.size());
assertEquals("一级标题", chunks.get(0).getTitle());
assertEquals("二级标题", chunks.get(1).getTitle());
assertEquals("三级标题", chunks.get(2).getTitle());
}
@Test
@DisplayName("H1-H6 全部支持")
void allHeadingLevels() {
StringBuilder sb = new StringBuilder();
for (int i = 1; i <= 6; i++) {
sb.append("#".repeat(i)).append(" 标题").append(i).append("\n\n内容").append(i).append("。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/h1h6.md");
assertEquals(6, chunks.size());
}
@Test
@DisplayName("无标题文档 → 整个文档作为1个 section")
void noHeadings_entireAsOneSection() {
String content = "纯文本没有标题。\n\n第二段内容。\n\n第三段内容。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/nohead.md");
assertFalse(chunks.isEmpty());
assertNull(chunks.get(0).getTitle());
}
}
// ==================== 回归:段落边界切分 ====================
@Nested
@DisplayName("超长章节 — 段落边界切分")
class ParagraphSplitTests {
@Test
@DisplayName("短章节(≤maxSize)→ 不进入段落切割")
void shortSection_noParagraphSplit() {
StringBuilder sb = new StringBuilder();
sb.append("# 测试\n\n");
for (int i = 0; i < 5; i++) {
sb.append("段落").append(i).append(":这是一段短内容。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/short_sec.md");
assertEquals(1, chunks.size());
}
@Test
@DisplayName("超长章节 → 在段落边界切分")
void longSection_splitsAtParagraphBoundaries() {
config.setMaxSize(50);
config.setMaxTokens(30);
StringBuilder sb = new StringBuilder();
sb.append("# 长章节\n\n");
for (int i = 0; i < 10; i++) {
sb.append("段落").append(i).append(":ABCDEFGHIJKLMNOPQRSTUVWXYZ。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_sec.md");
assertTrue(chunks.size() >= 2, "超长章节应切分为多个分块,实际: " + chunks.size());
// 所有分块携带相同的 title
for (DocumentChunk c : chunks) {
assertEquals("长章节", c.getTitle());
}
}
}
// ==================== 回归:chunkIndex 元数据 ====================
@Nested
@DisplayName("分块元数据")
class ChunkMetadataTests {
@Test
@DisplayName("chunkIndex 自增且唯一")
void chunkIndexSequential() {
config.setMaxSize(50);
config.setMaxTokens(30);
StringBuilder sb = new StringBuilder("# Meta\n\n");
for (int i = 0; i < 10; i++) {
sb.append("段落").append(i).append(":填充内容以触发切分机制。ABCDE。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/meta.md");
assertTrue(chunks.size() >= 2);
for (int i = 0; i < chunks.size(); i++) {
assertEquals(i, chunks.get(i).getChunkIndex(),
"chunkIndex 应从0开始连续递增");
}
}
@Test
@DisplayName("startIndex/endIndex 范围合法 — 无漂移")
void indexRangeValid_noDrift() {
String content = "# 标题\n\n测试内容。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/index.md");
for (DocumentChunk c : chunks) {
assertTrue(c.getStartOffset() >= 0);
assertTrue(c.getEndOffset() > c.getStartOffset(),
"endIndex(" + c.getEndOffset() + ") 应 > startIndex(" + c.getStartOffset() + ")");
assertTrue(c.getEndOffset() <= content.length());
}
}
}
// ==================== 新增:Token 估算 ====================
@Nested
@DisplayName("Token 估算")
class TokenEstimationTests {
@Test
@DisplayName("纯中文 800 字符 ≈ 800 tokens → 短章节不切")
void pureChinese_fewerTokensThanMax() {
config.setMaxTokens(400);
StringBuilder sb = new StringBuilder();
sb.append("# 中文测试\n\n");
// 纯中文 ~300 字符 ≈ 300 tokens
for (int i = 0; i < 3; i++) {
sb.append("这是纯中文测试内容的第十").append(i).append("段落。");
sb.append("每个中文字符大约占用一个令牌的位置。");
sb.append("因此这段文本的令牌数大致等于字符数。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/cn_tokens.md");
// 300 字符 ≈ 300 tokens < 400 maxTokens → 1 个分块
assertEquals(1, chunks.size());
}
@Test
@DisplayName("纯英文 2000 字符 ≈ 500 tokens → 刚好不超过上限")
void pureEnglish_moreCharactersSameTokens() {
config.setMaxTokens(200);
config.setMaxTokensHard(250);
StringBuilder sb = new StringBuilder();
sb.append("# English Test\n\n");
for (int i = 0; i < 8; i++) {
sb.append("This is paragraph number ").append(i)
.append(" containing English text. ")
.append("English characters are much cheaper in tokens. ")
.append("More filler text here to reach the limit properly. ")
.append("Yet another sentence for good measure. ")
.append("Still more words needed to reach token limit here.\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/en_tokens.md");
// 大量英文才占少量 token → 分块数应少于用字符计数的版本
assertTrue(chunks.size() >= 2, "1200+ 字符英文应切分");
}
}
// ==================== 新增:列表结构感知 ====================
@Nested
@DisplayName("列表结构感知")
class ListStructureTests {
@Test
@DisplayName("有序列表项之间不切分 — 即使超过 maxTokens")
void orderedList_notSplitBetweenItems() {
config.setMaxTokens(80);
config.setMaxTokensHard(200);
config.setOverlap(30);
StringBuilder sb = new StringBuilder();
sb.append("# 排查步骤\n\n");
// 5个有序列表项,每项 ~40 字符 ≈ 40 tokens,总共 ~200 tokens
for (int i = 1; i <= 5; i++) {
sb.append(i).append(". 这是排查步骤第").append(i)
.append("项,包含具体的操作指引和注意事项说明。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/ordered_list.md");
// 5项应保持在一起(未触及 hard 上限)
assertEquals(1, chunks.size(),
"有序列表项不应被拆散,实际分块数: " + chunks.size());
String content = chunks.get(0).getContent();
assertTrue(content.contains("1. "), "应包含第1项");
assertTrue(content.contains("5. "), "应包含第5项");
}
@Test
@DisplayName("有序列表触及硬上限 → 在列表项边界强制切分")
void orderedList_hardLimitSplits() {
config.setMaxTokens(50);
config.setMaxTokensHard(100);
config.setOverlap(20);
StringBuilder sb = new StringBuilder();
sb.append("# 长列表\n\n");
// 每项 ~60 tokens,硬上限 100 → 最多装 1 项多
for (int i = 1; i <= 6; i++) {
sb.append(i).append(". 这是很长的排查步骤内容,包含详细的说明信息。")
.append("每个步骤都要执行多个检查操作。继续填充文本以增加令牌计数。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_list.md");
System.out.println(" 长列表硬上限测试 — 实际分块数: " + chunks.size());
for (DocumentChunk c : chunks) {
System.out.println(" Chunk #" + c.getChunkIndex() + ": " + c.getContent().length() + "字符 "
+ "| start=" + c.getStartOffset() + " end=" + c.getEndOffset()
+ " | preview=" + c.getContent().substring(0, Math.min(60, c.getContent().length())).replace("\n", "\\n"));
}
// 硬上限会强制切分,但每个分块内的列表项应保持连续
assertTrue(chunks.size() >= 2, "长列表应至少触发1次切分,实际: " + chunks.size());
// 验证:除了第一个分块(可能是标题),其余应包含列表项
for (int i = 1; i < chunks.size(); i++) {
DocumentChunk c = chunks.get(i);
assertFalse(c.getContent().isEmpty());
assertTrue(c.getContent().matches("(?s).*\\d+\\.\\s.*"),
"非标题分块应包含列表项,Chunk #" + c.getChunkIndex()
+ " preview: " + c.getContent().substring(0, Math.min(60, c.getContent().length())));
}
}
@Test
@DisplayName("无序列表项之间不切分")
void unorderedList_notSplitBetweenItems() {
config.setMaxTokens(80);
config.setMaxTokensHard(200);
StringBuilder sb = new StringBuilder();
sb.append("# 检查清单\n\n");
for (int i = 1; i <= 5; i++) {
sb.append("- 检查项").append(i).append(":确认服务运行状态正常并记录相关指标。\n\n");
}
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/unordered_list.md");
assertEquals(1, chunks.size(), "无序列表项不应被拆散");
}
@Test
@DisplayName("列表结束后普通段落应从下一段落开始新分块")
void listEnds_normalParagraphStartsNewChunk() {
config.setMaxTokens(150);
config.setMaxTokensHard(250);
StringBuilder sb = new StringBuilder();
sb.append("# 文档\n\n");
// 先一个普通段落
sb.append("这是介绍段落,描述系统的整体架构和设计思路。\n\n");
// 有序列表
for (int i = 1; i <= 3; i++) {
sb.append(i).append(". 列表项第").append(i).append("条,包含操作说明。\n\n");
}
// 普通段落
sb.append("这是总结段落,包含上述操作完成后需要关注的监控指标。\n\n");
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/list_mixed.md");
assertTrue(chunks.size() >= 1);
// 列表项应保持在一起
for (DocumentChunk c : chunks) {
String content = c.getContent();
// 分块中不应有孤立的单个列表项(除非只有一个)
if (content.contains("1. ") && content.contains("3. ")) {
// 这个分块包含了全部3个列表项 → 正确
}
}
}
}
// ==================== 新增:代码块结构感知 ====================
@Nested
@DisplayName("代码块结构感知")
class CodeBlockTests {
@Test
@DisplayName("代码块内部不切分")
void codeBlock_notSplitInside() {
config.setMaxTokens(60);
config.setMaxTokensHard(200);
config.setOverlap(20);
String content =
"# 代码示例\n\n" +
"以下是配置代码:\n\n" +
"```yaml\n" +
"server:\n" +
" port: 8080\n" +
" host: localhost\n" +
" timeout: 30s\n" +
"```\n\n" +
"配置说明结束。";
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/code.md");
// 代码块应保持完整(未触及硬上限)
// 验证:至少有一个分块包含完整的 ```...```
boolean foundCompleteBlock = false;
for (DocumentChunk c : chunks) {
String text = c.getContent();
if (text.contains("```yaml") && text.contains("```") &&
text.indexOf("```yaml") < text.lastIndexOf("```")) {
foundCompleteBlock = true;
}
}
// 可能整体在一个分块中
assertTrue(chunks.size() >= 1);
}
}
// ==================== 可视化 ====================
@Nested
@DisplayName("可视化 — 打印切分结果")
class VisualInspectionTests {
@Test
@DisplayName("模拟运维文档 — 展示新策略效果")
void realWorldAIOpsDoc() {
config.setMaxTokens(150);
config.setMaxTokensHard(200);
config.setOverlap(40);
String doc = """
# CPU高负载问题排查指南
## 问题现象
服务器CPU使用率持续超过90%,系统响应变慢,用户反馈页面加载超时。
监控告警系统连续发出多条CPU使用率告警。
## 排查步骤
1. 登录服务器,执行 top 命令查看当前CPU使用率最高的进程。记录进程ID和CPU占用百分比。
2. 使用 ps aux | grep {进程名} 确认相关服务的运行状态。检查是否有异常进程占用资源。
3. 查看应用日志,重点关注最近15分钟的ERROR级别日志。使用 tail -n 500 命令。
4. 检查数据库连接池状态,确认是否有慢查询或连接泄漏。查看慢查询日志。
5. 检查JVM内存使用情况和GC日志。使用 jstat -gcutil {pid} 1000 命令观察GC频率。
## 常见原因
1. 死循环或递归调用导致CPU满载。检查是否有未设置退出条件的循环逻辑。
2. 大量正则表达式匹配操作。检查是否有未编译的正则在循环中使用。
## 解决方案
根据排查结果采取对应措施:代码问题则回滚或热修复;资源不足则扩容。
处理完成后持续观察监控指标30分钟,确认CPU使用率恢复正常。
""";
List<DocumentChunk> chunks = service.chunkDocument(doc, "/kb/cpu_high_usage.md");
System.out.println("========================================");
System.out.println(" Phase 1 新策略效果 — 模拟运维文档");
System.out.println(" 配置: maxTokens=150, hard=200, overlap=40");
System.out.println(" 总字符数: " + doc.length());
System.out.println(" 总分块数: " + chunks.size());
System.out.println("========================================\n");
for (DocumentChunk c : chunks) {
System.out.println("┌─ Chunk #" + c.getChunkIndex());
System.out.println("│ Title: " + (c.getTitle() != null ? c.getTitle() : "(无)"));
System.out.println("│ Range: [" + c.getStartOffset() + "→" + c.getEndOffset() + "] (" + c.getContent().length() + "字符)");
// 显示前150字符
String preview = c.getContent().length() > 120
? c.getContent().substring(0, 120).replace("\n", "\\n") + "..."
: c.getContent().replace("\n", "\\n");
System.out.println("│ Preview: " + preview);
System.out.println("└──────────────────────\n");
}
assertTrue(chunks.size() >= 3, "应产生多个分块");
}
@Test
@DisplayName("中英混排对比 — token vs 字符计数差异")
void mixedContentComparison() {
config.setMaxTokens(100);
config.setMaxTokensHard(150);
config.setOverlap(30);
String chinese = "这是中文内容示范。中文每个字符在LLM中约占用1个token。" +
"因此这段文本在上下文窗口中占用的token数较多。" +
"继续填充文字以触发切分逻辑,验证中文token估算是否合理。" +
"更多中文文本来增加令牌计数。";
String english = "This is English content. Each word may take one or two tokens. " +
"A sentence like this one actually consumes relatively few tokens compared to " +
"Chinese characters. More English text to reach the same token count as above. " +
"Still need more words because English is very efficient in tokenization. " +
"Adding even more content to make this paragraph long enough to test properly.";
List<DocumentChunk> cnChunks = service.chunkDocument("# CN\n\n" + chinese + "\n\n" + chinese, "/test/cn.md");
List<DocumentChunk> enChunks = service.chunkDocument("# EN\n\n" + english + "\n\n" + english, "/test/en.md");
System.out.println("========================================");
System.out.println(" Token 计数对比");
System.out.println(" 配置: maxTokens=100, overlap=30");
System.out.println("========================================");
System.out.println(" 中文文档: " + (chinese.length() * 2) + "字符 → " + cnChunks.size() + "个分块");
System.out.println(" 英文文档: " + (english.length() * 2) + "字符 → " + enChunks.size() + "个分块");
for (DocumentChunk c : cnChunks) {
System.out.println(" 中文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
}
for (DocumentChunk c : enChunks) {
System.out.println(" 英文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
}
System.out.println(" ★ 现在中文和英文的分块数更接近(基于 token 而非字符)");
System.out.println("========================================");
}
}
}
@@ -1,6 +1,5 @@
package com.superbiz.agent.service; package com.superbiz.agent.service;
import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
import org.springframework.mock.web.MockMultipartFile; import org.springframework.mock.web.MockMultipartFile;
@@ -39,16 +38,4 @@ class DocumentManagementServiceTest {
assertEquals("payment/runbook.md", storedPath); assertEquals("payment/runbook.md", storedPath);
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md"))); assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
} }
@Test
void resolveDocumentIdUsesFrontmatterSourceWhenItFitsDatabaseColumn() {
DocumentManagementService service = new DocumentManagementService();
Frontmatter frontmatter = Frontmatter.builder()
.source("mysql-connection-pool")
.build();
String docId = ReflectionTestUtils.invokeMethod(service, "resolveDocumentId", frontmatter);
assertEquals("mysql-connection-pool", docId);
}
} }
@@ -1,193 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* FrontmatterParser 单元测试
*/
class FrontmatterParserTest {
private FrontmatterParser parser;
@BeforeEach
void setUp() {
parser = new FrontmatterParser();
}
@Test
void testHasFrontmatter_withValidFrontmatter() {
String content = "---\ntitle: Test\n---\nContent";
assertTrue(parser.hasFrontmatter(content));
}
@Test
void testHasFrontmatter_withoutFrontmatter() {
String content = "# Just a title\nContent";
assertFalse(parser.hasFrontmatter(content));
}
@Test
void testHasFrontmatter_nullContent() {
assertFalse(parser.hasFrontmatter(null));
}
@Test
void testHasFrontmatter_emptyContent() {
assertFalse(parser.hasFrontmatter(""));
}
@Test
void testParse_validFrontmatter() {
String content = """
---
title: 支付网关错误码
keywords: [ERR_TIMEOUT, 超时, 支付网关]
summary: 记录了支付网关所有核心错误码
category: api
---
# 正文内容
""";
Frontmatter result = parser.parse(content);
assertNotNull(result);
assertEquals("支付网关错误码", result.getTitle());
assertEquals(3, result.getKeywords().size());
assertTrue(result.getKeywords().contains("ERR_TIMEOUT"));
assertEquals("记录了支付网关所有核心错误码", result.getSummary());
assertEquals("api", result.getCategory());
}
@Test
void testParse_withoutFrontmatter() {
String content = "# Just content\nNo frontmatter here";
assertNull(parser.parse(content));
}
@Test
void testParse_missingRequiredFields() {
String content = """
---
title: Only Title
---
Content
""";
// 缺少 keywords 和 summary,应返回 null
Frontmatter result = parser.parse(content);
assertNull(result);
}
@Test
void testParse_malformedYaml() {
String content = """
---
title: Test
keywords: [unclosed array
---
Content
""";
// YAML 格式错误,应返回 null
Frontmatter result = parser.parse(content);
assertNull(result);
}
@Test
void testParse_noClosingDelimiter() {
String content = """
---
title: Test
keywords: [test]
summary: Test summary
Content without closing ---
""";
// 缺少结束标记,应返回 null
Frontmatter result = parser.parse(content);
assertNull(result);
}
@Test
void testParse_windowsLineEndings() {
String content = "---\r\ntitle: Test\r\nkeywords: [test]\r\nsummary: Summary\r\n---\r\nContent";
Frontmatter result = parser.parse(content);
assertNotNull(result);
assertEquals("Test", result.getTitle());
}
@Test
void testParse_withOptionalFields() {
String content = """
---
title: Test Document
keywords: [test, doc]
summary: A test document
version: 1.0.0
author: Test Author
---
Content
""";
Frontmatter result = parser.parse(content);
assertNotNull(result);
assertEquals("Test Document", result.getTitle());
assertEquals("1.0.0", result.getVersion());
assertEquals("Test Author", result.getAuthor());
}
@Test
void testParse_withRetrievalMetadata() {
String content = """
---
title: MySQL Connection Pool
keywords: [connection pool, HikariCP]
summary: Diagnose exhausted MySQL connection pools
category: database
source: mysql-connection-pool
breadcrumb: Database > MySQL > Connection Pool
kb_scope: rag-eval
---
Content
""";
Frontmatter result = parser.parse(content);
assertNotNull(result);
assertEquals("mysql-connection-pool", result.getSource());
assertEquals("Database > MySQL > Connection Pool", result.getBreadcrumb());
assertEquals("rag-eval", result.getKbScope());
}
@Test
void testStripFrontmatter_returnsMarkdownBodyOnly() {
String content = """
---
title: Test
keywords: [frontmatter-only]
summary: Summary
---
# Body
Body content
""";
String body = parser.stripFrontmatter(content);
assertFalse(body.contains("frontmatter-only"));
assertTrue(body.startsWith("# Body"));
assertTrue(body.contains("Body content"));
}
}
@@ -1,168 +0,0 @@
package com.superbiz.agent.service;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.context.TestConfiguration;
import org.springframework.context.annotation.Bean;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* 全链路验证:DeepSeek → BGE-M3 → Milvus
*/
@SpringBootTest
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@DisplayName("DeepSeek → BGE-M3 → Milvus 全链路")
class FullPipelineSmokeTest {
@Autowired
private ChatModel chatModel;
@Autowired
private EmbeddingModel embeddingModel;
@Autowired
private VectorEmbeddingService vectorEmbeddingService;
@Autowired
private VectorSearchService vectorSearchService;
@TestConfiguration
static class MockToolConfig {
@Bean
public ToolCallbackProvider toolCallbackProvider() {
return () -> new ToolCallback[0];
}
}
// ===== ① Chat: DeepSeek =====
@Test
@Order(1)
@DisplayName("Chat: DeepSeek 聊天验证")
void chatDeepSeekWorks() {
System.out.println("\n===== ① Chat: DeepSeek =====");
System.out.println("ChatModel: " + chatModel.getClass().getSimpleName());
System.out.println("ChatOptions: " + chatModel.toString());
// 直接调用 chat
var response = chatModel.call(new Prompt("请用一句话介绍你自己"));
String text = response.getResult().getOutput().getText();
assertNotNull(text);
assertFalse(text.isEmpty());
System.out.println("Response: " + text.substring(0, Math.min(200, text.length())) + "...");
System.out.println("Chat ✓");
}
// ===== ② Embedding: BGE-M3 via SiliconFlow =====
@Test
@Order(2)
@DisplayName("Embedding: BGE-M3 向量生成验证")
void embeddingBgeM3Works() {
System.out.println("\n===== ② Embedding: BGE-M3 (SiliconFlow) =====");
System.out.println("EmbeddingModel: " + embeddingModel.getClass().getSimpleName());
String text = "你好,这是一条测试文本";
List<Float> vector = vectorEmbeddingService.generateEmbedding(text);
assertNotNull(vector);
assertFalse(vector.isEmpty());
assertEquals(1024, vector.size(), "BGE-M3 应返回 1024 维向量");
// 非零校验
boolean hasNonZero = vector.stream().anyMatch(v -> Math.abs(v) > 1e-6);
assertTrue(hasNonZero, "向量不能全为零");
// L2 范数校验:BGE-M3 输出应为 L2 归一化的单位向量
double norm = Math.sqrt(vector.stream().mapToDouble(v -> (double) v * v).sum());
System.out.println("维度: " + vector.size());
System.out.println("前5维: " + vector.subList(0, Math.min(5, vector.size())));
System.out.println("L2 范数: " + String.format("%.10f", norm));
System.out.println("是否归一化 (|norm - 1.0| < 0.01): " + (Math.abs(norm - 1.0) < 0.01));
assertEquals(1.0, norm, 0.01, "BGE-M3 向量应为 L2 归一化单位向量,实际范数=" + norm);
System.out.println("Embedding ✓");
}
@Test
@Order(3)
@DisplayName("Embedding: BGE-M3 批量向量生成验证")
void embeddingBatchWorks() {
System.out.println("\n===== ③ Embedding 批量 =====");
List<String> texts = List.of("文本一", "文本二", "文本三");
List<List<Float>> results = vectorEmbeddingService.generateEmbeddings(texts);
assertEquals(3, results.size());
for (List<Float> r : results) {
assertEquals(1024, r.size());
}
System.out.println("批量生成: " + results.size() + " 个 向量,各 " + results.get(0).size() + " 维 ✓");
}
// ===== ③ Milvus: 向量搜索 =====
@Test
@Order(4)
@DisplayName("Milvus: 连接 + 搜索验证")
void milvusSearchWorks() {
System.out.println("\n===== ④ Milvus: 向量搜索 =====");
// 用 BGE-M3 生成查询向量
String query = "内部文档";
List<Float> queryVector = vectorEmbeddingService.generateQueryVector(query);
assertNotNull(queryVector);
assertEquals(1024, queryVector.size());
// 搜索
List<VectorSearchService.SearchResult> results =
vectorSearchService.searchSimilarDocuments(query, 3);
assertNotNull(results);
System.out.println("查询: " + query);
System.out.println("返回: " + results.size() + " 条");
if (!results.isEmpty()) {
// 至少有结果,验证结构
for (int i = 0; i < results.size(); i++) {
var r = results.get(i);
assertNotNull(r.getId());
assertNotNull(r.getContent());
System.out.println(" [" + (i + 1) + "] id=" + r.getId()
+ ", score=" + String.format("%.4f", r.getScore())
+ ", content=" + r.getContent().substring(0, Math.min(50, r.getContent().length())) + "...");
}
} else {
System.out.println("(Milvus 中暂无数据,但连接正常)");
}
System.out.println("Milvus ✓");
}
// ===== 汇总 =====
@Test
@Order(5)
@DisplayName("总结")
void summary() {
System.out.println("\n==========================================");
System.out.println("全链路验证完成:");
System.out.println(" ① Chat → DeepSeek ✓");
System.out.println(" ② Embedding → BGE-M3 ✓ (SiliconFlow, 1024维)");
System.out.println(" ③ 向量存储 → Milvus ✓ (Zilliz Cloud)");
System.out.println("==========================================");
}
}
@@ -1,316 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.KnowledgeEntry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.test.util.ReflectionTestUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
/**
* KnowledgeIndexService 单元测试
*/
class KnowledgeIndexServiceTest {
private KnowledgeIndexService service;
@TempDir
Path tempDir;
@BeforeEach
void setUp() {
service = new KnowledgeIndexService();
ReflectionTestUtils.setField(service, "knowledgeBasePath", tempDir.toString());
}
@Test
void testExactMatch_singleMatch() {
// 准备测试数据
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("test.md")
.title("Test")
.keywords(List.of("ERR_TIMEOUT", "超时"))
.summary("Test summary")
.category("api")
.build();
service.addToIndex(entry);
// 测试匹配
List<KnowledgeEntry> results = service.exactMatch("ERR_TIMEOUT");
assertEquals(1, results.size());
assertEquals("Test", results.get(0).getTitle());
}
@Test
void testExactMatch_caseInsensitive() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("test.md")
.keywords(List.of("ERR_TIMEOUT"))
.build();
service.addToIndex(entry);
// 小写查询应该匹配
List<KnowledgeEntry> results = service.exactMatch("err_timeout");
assertEquals(1, results.size());
}
@Test
void testExactMatch_partialMatch() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("test.md")
.keywords(List.of("支付网关"))
.build();
service.addToIndex(entry);
// 包含关键词的查询应该匹配
List<KnowledgeEntry> results = service.exactMatch("支付网关超时问题");
assertEquals(1, results.size());
}
@Test
void testExactMatch_multipleMatches() {
KnowledgeEntry entry1 = KnowledgeEntry.builder()
.filePath("doc1.md")
.title("Doc 1")
.keywords(List.of("超时"))
.build();
KnowledgeEntry entry2 = KnowledgeEntry.builder()
.filePath("doc2.md")
.title("Doc 2")
.keywords(List.of("超时", "错误"))
.build();
service.addToIndex(entry1);
service.addToIndex(entry2);
// 应该匹配两个文档
List<KnowledgeEntry> results = service.exactMatch("超时");
assertEquals(2, results.size());
}
@Test
void testAnalyzeQuery_returnsStructuredHint() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("mysql.md")
.title("MySQL Doc")
.keywords(List.of("mysql", "connection pool"))
.category("database")
.build();
service.addToIndex(entry);
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("mysql connection pool timeout");
assertEquals(1, hint.matches().size());
assertEquals(List.of("mysql", "connection pool"), hint.matchedKeywords());
assertEquals(List.of("database"), hint.domains());
assertEquals(List.of("mysql", "connection pool"), hint.entities());
assertEquals(List.of("MySQL Doc"), hint.titles());
assertEquals("database", hint.singleDomainOrNull());
}
@Test
void testAnalyzeQuery_multipleDomainsHasNoSingleDomain() {
service.addToIndex(KnowledgeEntry.builder()
.filePath("mysql.md")
.keywords(List.of("timeout"))
.category("database")
.build());
service.addToIndex(KnowledgeEntry.builder()
.filePath("api.md")
.keywords(List.of("timeout"))
.category("api")
.build());
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
assertEquals(2, hint.matches().size());
assertNull(hint.singleDomainOrNull());
}
@Test
void testAnalyzeQuery_filtersByConfiguredKbScope() {
ReflectionTestUtils.setField(service, "kbScope", "rag-eval");
service.addToIndex(KnowledgeEntry.builder()
.filePath("legacy.md")
.keywords(List.of("timeout"))
.category("legacy")
.build());
service.addToIndex(KnowledgeEntry.builder()
.filePath("eval.md")
.keywords(List.of("timeout"))
.category("eval")
.kbScope("rag-eval")
.build());
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
assertEquals(1, hint.matches().size());
assertEquals(List.of("eval"), hint.domains());
assertEquals("eval", hint.singleDomainOrNull());
}
@Test
void testAnalyzeQuery_keepsLegacyEntriesWhenNoScopeConfigured() {
ReflectionTestUtils.setField(service, "kbScope", "");
service.addToIndex(KnowledgeEntry.builder()
.filePath("legacy.md")
.keywords(List.of("timeout"))
.category("legacy")
.build());
service.addToIndex(KnowledgeEntry.builder()
.filePath("eval.md")
.keywords(List.of("timeout"))
.category("eval")
.kbScope("rag-eval")
.build());
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
assertEquals(2, hint.matches().size());
assertNull(hint.singleDomainOrNull());
}
@Test
void testExactMatch_noMatch() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("test.md")
.keywords(List.of("错误码"))
.build();
service.addToIndex(entry);
// 不匹配的查询
List<KnowledgeEntry> results = service.exactMatch("限流");
assertEquals(0, results.size());
}
@Test
void testExactMatch_emptyQuery() {
List<KnowledgeEntry> results = service.exactMatch("");
assertEquals(0, results.size());
}
@Test
void testExactMatch_nullQuery() {
List<KnowledgeEntry> results = service.exactMatch(null);
assertEquals(0, results.size());
}
@Test
void testReadDocument_success() throws Exception {
// 创建测试文件
Path testFile = tempDir.resolve("test.md");
String content = "Test content line 1\nTest content line 2\n";
Files.writeString(testFile, content);
// 读取文件
String result = service.readDocument(testFile.toString(), 100);
assertNotNull(result);
assertTrue(result.contains("Test content"));
}
@Test
void testReadDocument_relativePathUnderBasePath() throws Exception {
Path categoryDir = tempDir.resolve("payment");
Files.createDirectories(categoryDir);
Path testFile = categoryDir.resolve("relative.md");
Files.writeString(testFile, "Relative content");
String result = service.readDocument("payment/relative.md", 100);
assertEquals("Relative content", result);
}
@Test
void testReadDocument_legacyPathAlreadyContainsBasePath() throws Exception {
Path categoryDir = tempDir.resolve("payment");
Files.createDirectories(categoryDir);
Path testFile = categoryDir.resolve("legacy.md");
Files.writeString(testFile, "Legacy content");
String result = service.readDocument(tempDir.getFileName() + "/payment/legacy.md", 100);
assertEquals("Legacy content", result);
}
@Test
void testReadDocument_exceedsMaxChars() throws Exception {
// 创建超长内容
String longContent = "x".repeat(3000);
Path testFile = tempDir.resolve("long.md");
Files.writeString(testFile, longContent);
// 读取限制字符数
String result = service.readDocument(testFile.toString(), 2000);
assertNotNull(result);
assertEquals(2003, result.length()); // 2000 + "..."
assertTrue(result.endsWith("..."));
}
@Test
void testReadDocument_fileNotFound() {
String result = service.readDocument("nonexistent.md", 100);
assertNull(result);
}
@Test
void testAddToIndex() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("new.md")
.title("New Document")
.keywords(List.of("test"))
.build();
service.addToIndex(entry);
List<KnowledgeEntry> results = service.exactMatch("test");
assertEquals(1, results.size());
assertEquals("New Document", results.get(0).getTitle());
}
@Test
void testRemoveFromIndex() {
KnowledgeEntry entry = KnowledgeEntry.builder()
.filePath("remove.md")
.keywords(List.of("test"))
.build();
service.addToIndex(entry);
assertEquals(1, service.exactMatch("test").size());
service.removeFromIndex("remove.md");
assertEquals(0, service.exactMatch("test").size());
}
@Test
void testGetIndexSize() {
assertEquals(0, service.getIndexSize());
service.addToIndex(KnowledgeEntry.builder()
.filePath("doc1.md")
.keywords(List.of("test"))
.build());
assertEquals(1, service.getIndexSize());
service.addToIndex(KnowledgeEntry.builder()
.filePath("doc2.md")
.keywords(List.of("test"))
.build());
assertEquals(2, service.getIndexSize());
}
}
@@ -1,237 +0,0 @@
package com.superbiz.agent.service;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.DataType;
import io.milvus.grpc.FlushResponse;
import io.milvus.grpc.MutationResult;
import io.milvus.grpc.SearchResults;
import io.milvus.grpc.ShowCollectionsResponse;
import io.milvus.common.clientenum.ConsistencyLevelEnum;
import io.milvus.param.ConnectParam;
import io.milvus.param.IndexType;
import io.milvus.param.MetricType;
import io.milvus.param.R;
import io.milvus.param.RpcStatus;
import io.milvus.param.collection.*;
import io.milvus.param.dml.InsertParam;
import io.milvus.param.dml.SearchParam;
import io.milvus.param.index.CreateIndexParam;
import io.milvus.response.SearchResultsWrapper;
import org.junit.jupiter.api.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
@DisplayName("Milvus 连接验证")
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class MilvusConnectionTest {
private static final String COLLECTION = "conn_test";
private static final int DIM = 128;
private static MilvusServiceClient client;
@BeforeAll
static void connect() {
String host = envOrDefault("MILVUS_HOST",
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
int port = Integer.parseInt(envOrDefault("MILVUS_PORT", "443"));
String token = System.getenv("MILVUS_TOKEN");
assertNotNull(token, "环境变量 MILVUS_TOKEN 未设置");
ConnectParam connectParam = ConnectParam.newBuilder()
.withHost(host)
.withPort(port)
.withToken(token)
.withSecure(true)
.withDatabaseName("db_4a578da0f27ce9d")
.withConnectTimeout(30, TimeUnit.SECONDS)
.build();
client = new MilvusServiceClient(connectParam);
System.out.println("连接目标: " + host + ":" + port);
}
@AfterAll
static void disconnect() {
if (client != null) {
try {
client.dropCollection(DropCollectionParam.newBuilder()
.withCollectionName(COLLECTION).build());
} catch (Exception ignored) {}
client.close();
}
}
private static String safeMsg(R<?> resp) {
try {
return resp.getMessage();
} catch (Exception e) {
return "(no message)";
}
}
@Test
@Order(1)
@DisplayName("1. 连接成功 - 能列出 collection")
void listCollections() {
R<ShowCollectionsResponse> resp = client.showCollections(
ShowCollectionsParam.newBuilder().build());
System.out.println("listCollections status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
assertEquals(0, resp.getStatus(), "连接失败,status=" + resp.getStatus());
List<String> names = resp.getData().getCollectionNamesList();
System.out.println("现有 collections: " + names);
}
@Test
@Order(2)
@DisplayName("2. 创建测试 collection")
void createCollection() {
client.dropCollection(DropCollectionParam.newBuilder()
.withCollectionName(COLLECTION).build());
FieldType idField = FieldType.newBuilder()
.withName("id")
.withDataType(DataType.Int64)
.withPrimaryKey(true)
.withAutoID(true)
.build();
FieldType vectorField = FieldType.newBuilder()
.withName("vector")
.withDataType(DataType.FloatVector)
.withDimension(DIM)
.build();
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
.addFieldType(idField)
.addFieldType(vectorField)
.build();
R<RpcStatus> resp = client.createCollection(
CreateCollectionParam.newBuilder()
.withCollectionName(COLLECTION)
.withSchema(schema)
.build());
System.out.println("createCollection status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
assertEquals(0, resp.getStatus(), "创建 collection 失败");
}
@Test
@Order(3)
@DisplayName("3. 插入数据 + flush")
void insertAndFlush() {
List<Float> vec1 = makeVector(1.0f);
List<Float> vec2 = makeVector(2.0f);
List<Float> vec3 = makeVector(3.0f);
List<InsertParam.Field> fields = Collections.singletonList(
new InsertParam.Field("vector", Arrays.asList(vec1, vec2, vec3))
);
R<MutationResult> insertResp = client.insert(
InsertParam.newBuilder()
.withCollectionName(COLLECTION)
.withFields(fields)
.build());
System.out.println("insert status: " + insertResp.getStatus() + ", msg: " + safeMsg(insertResp));
assertEquals(0, insertResp.getStatus(), "插入失败");
// 官方示例要求:insert 后必须 flush,数据才对搜索可见
R<FlushResponse> flushResp = client.flush(FlushParam.newBuilder()
.withCollectionNames(Collections.singletonList(COLLECTION))
.withSyncFlush(true)
.withSyncFlushWaitingTimeout(30L)
.build());
System.out.println("flush status: " + flushResp.getStatus() + ", msg: " + safeMsg(flushResp));
assertEquals(0, flushResp.getStatus(), "flush 失败");
System.out.println("插入 3 条数据并 flush 完成");
}
@Test
@Order(4)
@DisplayName("4. 创建索引 + 加载")
void createIndexAndLoad() {
R<RpcStatus> indexResp = client.createIndex(
CreateIndexParam.newBuilder()
.withCollectionName(COLLECTION)
.withFieldName("vector")
.withIndexType(IndexType.AUTOINDEX)
.withMetricType(MetricType.L2)
.build());
System.out.println("createIndex status: " + indexResp.getStatus() + ", msg: " + safeMsg(indexResp));
assertEquals(0, indexResp.getStatus(), "创建索引失败");
R<RpcStatus> loadResp = client.loadCollection(
LoadCollectionParam.newBuilder()
.withCollectionName(COLLECTION)
.withSyncLoad(true)
.withSyncLoadWaitingTimeout(30L)
.build());
System.out.println("load status: " + loadResp.getStatus() + ", msg: " + safeMsg(loadResp));
assertEquals(0, loadResp.getStatus(), "加载失败");
System.out.println("索引创建 + 加载完成");
}
@Test
@Order(5)
@DisplayName("5. 向量搜索")
void search() throws InterruptedException {
Thread.sleep(3000);
List<Float> queryVec = makeVector(1.1f);
R<SearchResults> resp = null;
for (int retry = 0; retry < 10; retry++) {
resp = client.search(
SearchParam.newBuilder()
.withCollectionName(COLLECTION)
.withMetricType(MetricType.L2)
.withTopK(2)
.withVectors(Collections.singletonList(queryVec))
.withVectorFieldName("vector")
.withParams("{}")
.withConsistencyLevel(ConsistencyLevelEnum.STRONG)
.build());
if (resp.getStatus() == 0) break;
System.out.println("search retry " + (retry + 1) + ": status=" + resp.getStatus() + ", msg=" + safeMsg(resp));
Thread.sleep(5000);
}
System.out.println("search status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
assertEquals(0, resp.getStatus(), "搜索失败");
SearchResultsWrapper wrapper = new SearchResultsWrapper(resp.getData().getResults());
List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(0);
assertFalse(scores.isEmpty(), "搜索结果不应为空");
System.out.println("搜索结果 (top " + scores.size() + "):");
for (SearchResultsWrapper.IDScore idScore : scores) {
System.out.println(" score=" + idScore.getScore() + ", id=" + idScore.getLongID());
}
}
private static List<Float> makeVector(float val) {
Float[] arr = new Float[DIM];
Arrays.fill(arr, val);
return Arrays.asList(arr);
}
private static String envOrDefault(String key, String defaultVal) {
String val = System.getenv(key);
return (val != null && !val.isEmpty()) ? val : defaultVal;
}
}
@@ -1,117 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.config.RagSidecarProperties;
import com.superbiz.agent.dto.ComparableRetrievalResult;
import com.superbiz.agent.dto.RetrievalComparisonCase;
import com.superbiz.agent.dto.RetrievalComparisonReport;
import com.superbiz.agent.dto.SidecarRetrievalResponse;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class RagRetrievalSidecarComparisonServiceTest {
@TempDir
Path tempDir;
@Test
void compareWritesSeparateSidecarReports() throws Exception {
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
RagSidecarProperties properties = new RagSidecarProperties();
RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
vectorSearchService,
sidecarService,
normalizer,
properties,
new ObjectMapper()
);
VectorSearchService.SearchResult current = new VectorSearchService.SearchResult();
current.setId("current-1");
current.setMetadata("{\"_source\":\"current.md\",\"breadcrumb\":\"A\",\"category\":\"api\"}");
current.setContent("current content");
current.setScore(0.1f);
when(vectorSearchService.searchSimilarDocuments("timeout", 3, "api"))
.thenReturn(List.of(current));
when(sidecarService.search("timeout", 3, "api"))
.thenReturn(SidecarRetrievalResponse.builder()
.enabled(true)
.available(true)
.status("available")
.results(List.of(ComparableRetrievalResult.builder()
.path("sidecar")
.rank(1)
.source("sidecar.md")
.breadcrumb("B")
.scoreLabel("similarity")
.scoreValue(0.9)
.build()))
.build());
RetrievalComparisonReport report = comparisonService.compare(List.of(
RetrievalComparisonCase.builder()
.caseId("case-1")
.scenario("aiops")
.query("timeout")
.category("api")
.build()
), 3);
assertEquals(1, report.getCaseCount());
assertEquals("available", report.getSidecarStatus());
assertTrue(report.getResults().get(0).getDifferences().contains("top_source_differs"));
Path json = tempDir.resolve("sidecar.json");
Path markdown = tempDir.resolve("sidecar.md");
comparisonService.writeReports(report, json, markdown);
assertTrue(Files.readString(json).contains("\"sidecarStatus\""));
assertTrue(Files.readString(markdown).contains("RAG Sidecar Retrieval Comparison"));
}
@Test
void compareGoldenCasesLoadsExistingCaseShape() throws Exception {
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
vectorSearchService,
sidecarService,
new RetrievalResultNormalizer(new ObjectMapper()),
new RagSidecarProperties(),
new ObjectMapper()
);
when(vectorSearchService.searchSimilarDocuments("query", 2, null)).thenReturn(List.of());
when(sidecarService.search("query", 2, null))
.thenReturn(SidecarRetrievalResponse.builder()
.enabled(false)
.available(false)
.status("disabled")
.results(List.of())
.build());
Path cases = tempDir.resolve("cases.json");
Files.writeString(cases, """
{
"topK": 2,
"cases": [
{"caseId": "case-1", "scenario": "chat", "query": "query"}
]
}
""");
RetrievalComparisonReport report = comparisonService.compareGoldenCases(cases);
assertEquals(1, report.getCaseCount());
assertEquals(2, report.getTopK());
assertEquals("disabled", report.getSidecarStatus());
}
}
@@ -1,60 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.ComparableRetrievalResult;
import org.junit.jupiter.api.Test;
import org.springframework.ai.document.Document;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
class RetrievalResultNormalizerTest {
private final RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
@Test
void fromCurrentParsesMetadataAndLabelsDistanceScore() {
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
result.setId("vec-1");
result.setMetadata("{\"docId\":\"doc-1\",\"_source\":\"docs/api.md\",\"title\":\"API\",\"breadcrumb\":\"A > B\",\"category\":\"api\"}");
result.setContent("abcdef");
result.setScore(0.25f);
ComparableRetrievalResult comparable = normalizer.fromCurrent(result, 1, 3);
assertEquals("current", comparable.getPath());
assertEquals("docs/api.md", comparable.getSource());
assertEquals("doc-1", comparable.getDocId());
assertEquals("API", comparable.getTitle());
assertEquals("A > B", comparable.getBreadcrumb());
assertEquals("api", comparable.getCategory());
assertEquals("abc...", comparable.getContentPreview());
assertEquals("l2_distance", comparable.getScoreLabel());
assertEquals(0.25, comparable.getScoreValue(), 0.0001);
}
@Test
void fromSidecarNormalizesDocumentMetadataAndLabelsSimilarityScore() {
Document document = Document.builder()
.id("doc-vector")
.text("sidecar content")
.metadata(Map.of(
"docId", "doc-2",
"_source", "docs/sidecar.md",
"title", "Sidecar",
"breadcrumb", "Root > Sidecar",
"category", "rag"
))
.score(0.91)
.build();
ComparableRetrievalResult comparable = normalizer.fromSidecar(document, 2, 100);
assertEquals("sidecar", comparable.getPath());
assertEquals(2, comparable.getRank());
assertEquals("docs/sidecar.md", comparable.getSource());
assertEquals("similarity", comparable.getScoreLabel());
assertEquals(0.91, comparable.getScoreValue(), 0.0001);
}
}
@@ -1,66 +0,0 @@
package com.superbiz.agent.service;
import io.milvus.client.MilvusServiceClient;
import io.milvus.param.ConnectParam;
import io.milvus.param.R;
import io.milvus.param.collection.HasCollectionParam;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
/**
* 简单的 Milvus 连接测试
*/
public class SimpleMilvusTest {
@Test
public void testConnection() {
String host = System.getenv().getOrDefault(
"MILVUS_HOST",
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
int port = Integer.parseInt(System.getenv().getOrDefault("MILVUS_PORT", "443"));
String token = System.getenv("MILVUS_TOKEN");
Assumptions.assumeTrue(token != null && !token.isBlank(), "MILVUS_TOKEN is required");
System.out.println("尝试连接 Milvus...");
System.out.println("Host: " + host);
System.out.println("Port: " + port);
try {
MilvusServiceClient client = new MilvusServiceClient(
ConnectParam.newBuilder()
.withHost(host)
.withPort(port)
.withToken(token)
.withSecure(true)
.withConnectTimeout(10L, java.util.concurrent.TimeUnit.SECONDS)
.build()
);
System.out.println("✓ 客户端创建成功");
// 测试连接:查询是否存在某个 collection
R<Boolean> response = client.hasCollection(
HasCollectionParam.newBuilder()
.withCollectionName("test_collection")
.build()
);
System.out.println("✓ 连接成功!");
System.out.println("Status Code: " + response.getStatus());
if (response.getStatus() == 0 || response.getStatus() == io.milvus.param.R.Status.Success.getCode()) {
System.out.println("✓ Milvus 集群状态:正常运行");
} else {
System.out.println("✗ 响应状态异常: " + response.getStatus());
}
client.close();
System.out.println("✓ 连接已关闭");
} catch (Exception e) {
System.err.println("✗ 连接失败:" + e.getMessage());
e.printStackTrace();
throw new RuntimeException("Milvus 连接失败", e);
}
}
}
@@ -1,55 +0,0 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.config.RagSidecarProperties;
import com.superbiz.agent.dto.SidecarRetrievalResponse;
import org.junit.jupiter.api.Test;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.ObjectProvider;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class SpringAiVectorStoreSidecarServiceTest {
@Test
void disabledSidecarDoesNotRequestVectorStore() {
RagSidecarProperties properties = new RagSidecarProperties();
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
properties,
provider,
new RetrievalResultNormalizer(new ObjectMapper())
);
SidecarRetrievalResponse response = service.search("query", 3, null);
assertFalse(response.isEnabled());
assertFalse(response.isAvailable());
assertEquals("disabled", response.getStatus());
verify(provider, never()).getIfAvailable();
}
@Test
void enabledSidecarReportsMissingVectorStore() {
RagSidecarProperties properties = new RagSidecarProperties();
properties.setEnabled(true);
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
when(provider.getIfAvailable()).thenReturn(null);
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
properties,
provider,
new RetrievalResultNormalizer(new ObjectMapper())
);
SidecarRetrievalResponse response = service.search("query", 3, "api");
assertEquals("missing_vector_store", response.getStatus());
assertFalse(response.isAvailable());
assertEquals(0, response.getResults().size());
}
}
@@ -1,71 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.dto.DocumentChunk;
import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.Test;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
class VectorIndexServiceTest {
@Test
void buildEmbeddingTextIncludesTitleAndBreadcrumb() {
DocumentChunk chunk = DocumentChunk.builder()
.title("Connection Pool")
.breadcrumb("Database > MySQL > Connection Pool")
.content("Check active connections and leak detection.")
.build();
String embeddingText = VectorIndexService.buildEmbeddingText(chunk);
assertEquals("""
Title: Connection Pool
Path: Database > MySQL > Connection Pool
Content:
Check active connections and leak detection.""", embeddingText);
}
@Test
void buildEmbeddingTextKeepsPlainContentWhenNoStructureExists() {
DocumentChunk chunk = DocumentChunk.builder()
.title(" ")
.breadcrumb(null)
.content("Plain chunk content.")
.build();
assertEquals("Plain chunk content.", VectorIndexService.buildEmbeddingText(chunk));
}
@Test
void buildDocumentMetadataUsesFrontmatterRetrievalFields() {
DocumentChunk chunk = DocumentChunk.builder()
.chunkIndex(0)
.title("Chunk Title")
.breadcrumb("Chunk > Path")
.content("content")
.build();
Frontmatter frontmatter = Frontmatter.builder()
.title("Document Title")
.source("mysql-connection-pool")
.breadcrumb("Database > MySQL > Connection Pool")
.kbScope("rag-eval")
.build();
Map<String, Object> metadata = VectorIndexService.buildDocumentMetadata(
"mysql-connection-pool",
chunk,
2,
"database",
frontmatter
);
assertEquals("mysql-connection-pool", metadata.get("docId"));
assertEquals("mysql-connection-pool", metadata.get("_source"));
assertEquals("mysql-connection-pool", metadata.get("source"));
assertEquals("database", metadata.get("category"));
assertEquals("rag-eval", metadata.get("kb_scope"));
assertEquals("Database > MySQL > Connection Pool", metadata.get("breadcrumb"));
}
}
@@ -1,71 +0,0 @@
package com.superbiz.agent.service;
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class VectorSearchServiceTest {
@Test
void denseModeCallsHybridStoreDenseSearch() {
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
when(embeddingService.generateQueryVector("query")).thenReturn(List.of(0.1f, 0.2f));
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
.thenReturn(List.of(expected));
when(store.collectionName()).thenReturn("biz");
VectorSearchService service = new VectorSearchService();
ReflectionTestUtils.setField(service, "knowledgeStore", store);
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
ReflectionTestUtils.setField(service, "searchMode", "dense");
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("query", 3, null);
assertEquals(List.of(expected), results);
verify(store).searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull());
}
@Test
void hybridModeCallsHybridStoreHybridSearch() {
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
when(embeddingService.generateQueryVector("pool")).thenReturn(List.of(0.3f));
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
.thenReturn(List.of(expected));
when(store.collectionName()).thenReturn("biz");
VectorSearchService service = new VectorSearchService();
ReflectionTestUtils.setField(service, "knowledgeStore", store);
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
ReflectionTestUtils.setField(service, "searchMode", "hybrid");
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("pool", 5, "mysql");
assertEquals(1, results.size());
assertEquals("doc-h", results.get(0).getId());
verify(store).searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql"));
}
private static VectorSearchService.SearchResult result(String id, float score) {
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
result.setId(id);
result.setScore(score);
result.setRawScore((double) score);
result.setScoreLabel("dense");
result.setContent("content");
result.setMetadata("{}");
return result;
}
}
@@ -0,0 +1,136 @@
package com.superbiz.agent.service.retrieval;
import com.superbiz.agent.client.PyRagClient;
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* {@link PyRagKnowledgeSearchAdapter} 请求/响应映射契约测试。
*/
@ExtendWith(MockitoExtension.class)
class PyRagKnowledgeSearchAdapterTest {
@Mock
private PyRagClient pyRagClient;
@Test
void mapsRequestModeTopKAndCategory() {
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
"q", "hybrid", List.of(), "PRECISE", "supported", null));
adapter.search(new KnowledgeSearchRequest("网关超时怎么排查", 20, "gateway", KnowledgeSearchMode.HYBRID));
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
verify(pyRagClient).search(captor.capture());
PyRagSearchRequest request = captor.getValue();
assertEquals("网关超时怎么排查", request.query());
assertEquals("hybrid", request.mode());
assertEquals(20, request.retrieveK());
assertEquals(20, request.returnN());
assertEquals(20, request.maxChunksPerDocument());
assertEquals("gateway", request.category());
assertNull(request.kbScope());
}
@Test
void denseModeMapsToSemanticWithoutCategory() {
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
"q", "semantic", List.of(), null, "supported", null));
adapter.search(KnowledgeSearchRequest.dense("性能优化", 5, null));
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
verify(pyRagClient).search(captor.capture());
assertEquals("semantic", captor.getValue().mode());
assertNull(captor.getValue().category());
}
@Test
void mapsHitsWithRerankScoreAndEvidenceIdentity() {
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
"网关超时怎么排查",
"hybrid",
List.of(new PyRagSearchHit(
"e2e-gateway-b9c1fa12-md-34223174#chunk-1",
"e2e-gateway-b9c1fa12-md-34223174",
"e2e-gateway-b9c1fa12-md",
"网关超时排查",
"网关超时排查 > 处理步骤",
"网关超时先检查 upstream 配置…",
0.9147,
"PRECISE")),
"PRECISE",
"supported",
null));
List<KnowledgeSearchHit> hits = adapter.search(
new KnowledgeSearchRequest("网关超时怎么排查", 5, "gateway", KnowledgeSearchMode.HYBRID));
assertEquals(1, hits.size());
KnowledgeSearchHit hit = hits.get(0);
assertEquals("e2e-gateway-b9c1fa12-md-34223174#chunk-1", hit.evidenceKey());
assertEquals("e2e-gateway-b9c1fa12-md-34223174", hit.docId());
assertEquals(1, hit.chunkIndex());
assertEquals("网关超时先检查 upstream 配置…", hit.content());
assertEquals(0.9147, hit.score(), 1e-9);
assertEquals(0.9147, hit.rawScore(), 1e-9);
assertEquals(RetrievalScoreLabels.RERANK, hit.scoreLabel());
assertEquals("e2e-gateway-b9c1fa12-md", hit.source());
assertEquals("网关超时排查", hit.title());
assertEquals("网关超时排查 > 处理步骤", hit.breadcrumb());
assertEquals(1, hit.originalRank());
assertNull(hit.denseDistance());
}
@Test
void noEvidenceReturnsEmptyListAsNormalBusinessResult() {
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
"乱码查询", "hybrid", List.of(), null, "no_evidence", null));
List<KnowledgeSearchHit> hits = adapter.search(
KnowledgeSearchRequest.dense("乱码查询", 5, null));
assertTrue(hits.isEmpty());
}
@Test
void hitWithoutChunkMarkerStillGetsStableIdentity() {
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
"q",
"hybrid",
List.of(new PyRagSearchHit(
"legacy-doc", "legacy-doc", "legacy-doc", "Legacy",
null, "content", 0.42, "REFERENCE")),
"REFERENCE",
"supported",
null));
List<KnowledgeSearchHit> hits = adapter.search(
KnowledgeSearchRequest.dense("q", 3, null));
assertEquals(1, hits.size());
// 无 #chunk-N 标记:chunkIndex 为 null,evidenceKey 保留服务端原值
assertNull(hits.get(0).chunkIndex());
assertEquals("legacy-doc", hits.get(0).evidenceKey());
}
}
@@ -47,10 +47,25 @@ class RetrievalScoreNormalizerTest {
assertTrue(hybridRank2 < hybridRank); assertTrue(hybridRank2 < hybridRank);
} }
@Test
void rerankScorePassesThroughClamped() {
// py-rag rerank 绝对分:quality = score 原样(越大越好),不受 L2/rank 分支影响
assertEquals(0.9147, RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.RERANK, 0.9147, 1, 20, 2.0, null), 1e-9);
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.RERANK, null, 1, 20, 2.0, null), 1e-9);
assertEquals(1.0, RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.RERANK, 1.7, 1, 20, 2.0, null), 1e-9);
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
RetrievalScoreLabels.RERANK, -0.3, 1, 20, 2.0, null), 1e-9);
}
@Test @Test
void canonicalizeAliases() { void canonicalizeAliases() {
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance")); assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused")); assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense")); assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("rerank"));
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("quality_score"));
} }
} }
@@ -1,50 +0,0 @@
package com.superbiz.agent.service.retrieval;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class RrfFusionTest {
@Test
void multiPathAgreementOutranksSinglePathHead() {
List<String> dense = List.of("a", "b", "c");
List<String> lexical = List.of("c", "b", "d");
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
List.of(
new RrfFusion.RankedPath<>("dense", dense, 1.0),
new RrfFusion.RankedPath<>("lexical", lexical, 1.0)
),
60,
s -> s
);
// c: dense#3 + lexical#1 ; b: dense#2 + lexical#2 ; a: dense#1 only
// With k=60, c edges b slightly, and both beat single-path a.
assertEquals("c", fused.get(0).identity());
assertEquals("b", fused.get(1).identity());
assertEquals("a", fused.get(2).identity());
assertTrue(fused.get(0).rrfScore() > fused.get(2).rrfScore());
}
@Test
void pathWeightCanElevateSecondaryPath() {
List<String> dense = List.of("a", "b");
List<String> lexical = List.of("b", "a");
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
List.of(
new RrfFusion.RankedPath<>("dense", dense, 1.0),
new RrfFusion.RankedPath<>("lexical", lexical, 2.0)
),
60,
s -> s
);
assertEquals("b", fused.get(0).identity());
}
}
@@ -1,48 +0,0 @@
package com.superbiz.agent.service.retrieval;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.VectorSearchService;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class VectorKnowledgeSearchAdapterHybridTest {
@Test
void adapterMapsStoreHitsWithChunkIdentity() {
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
when(vectorSearchService.searchSimilarDocuments("pool timeout", 3, "mysql")).thenReturn(List.of(
result("id-1",
"{\"_source\":\"c.md\",\"docId\":\"c\",\"chunkIndex\":0,\"title\":\"mysql pool timeout\"}",
"mysql pool timeout runbook",
0.35f)
));
VectorKnowledgeSearchAdapter adapter =
new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper());
List<KnowledgeSearchHit> hits = adapter.search(
new KnowledgeSearchRequest("pool timeout", 3, "mysql", KnowledgeSearchMode.HYBRID));
assertEquals(1, hits.size());
assertEquals("c#chunk-0", hits.get(0).evidenceKey());
assertEquals("c.md", hits.get(0).source());
verify(vectorSearchService).searchSimilarDocuments("pool timeout", 3, "mysql");
}
private static VectorSearchService.SearchResult result(String id, String metadata, String content, float score) {
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
result.setId(id);
result.setMetadata(metadata);
result.setContent(content);
result.setScore(score);
result.setRawScore((double) score);
result.setScoreLabel("dense");
return result;
}
}
@@ -1,16 +1,14 @@
package com.superbiz.agent.tool; package com.superbiz.agent.tool;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.dto.LookupResult; import com.superbiz.agent.dto.LookupResult;
import com.superbiz.agent.service.KnowledgeContextPacker; import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever; import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor; import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
import com.superbiz.agent.service.KnowledgeIndexService;
import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler; import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.service.VectorSearchService; import com.superbiz.agent.service.retrieval.KnowledgeSearchHit;
import com.superbiz.agent.service.retrieval.VectorKnowledgeSearchAdapter; import com.superbiz.agent.service.retrieval.KnowledgeSearchPort;
import com.superbiz.agent.service.retrieval.KnowledgeSearchRequest;
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.mockito.Mock; import org.mockito.Mock;
@@ -19,25 +17,26 @@ import org.springframework.test.util.ReflectionTestUtils;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.never; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
/** /**
* LookupKnowledgeTool evidence-first contract tests. * LookupKnowledgeTool evidence-first contract tests.
*
* <p>检索后端为 py-rag(经 {@link KnowledgeSearchPort} mock);L0 query 理解已下沉服务端,
* categoryFilter 恒为 null,走 UNFILTERED_VECTOR 单 attempt 主路径。</p>
*/ */
class LookupKnowledgeToolTest { class LookupKnowledgeToolTest {
@Mock @Mock
private KnowledgeIndexService knowledgeIndexService; private KnowledgeSearchPort knowledgeSearchPort;
@Mock
private VectorSearchService vectorSearchService;
private LookupKnowledgeTool tool; private LookupKnowledgeTool tool;
@@ -50,9 +49,8 @@ class LookupKnowledgeToolTest {
ReflectionTestUtils.setField(postProcessor, "maxChunksPerDocument", 2); ReflectionTestUtils.setField(postProcessor, "maxChunksPerDocument", 2);
KnowledgeContextPacker contextPacker = new KnowledgeContextPacker(); KnowledgeContextPacker contextPacker = new KnowledgeContextPacker();
tool = new LookupKnowledgeTool(); tool = new LookupKnowledgeTool();
ReflectionTestUtils.setField(tool, "queryTransformer", new KnowledgeQueryTransformer(knowledgeIndexService));
ReflectionTestUtils.setField(tool, "documentRetriever", ReflectionTestUtils.setField(tool, "documentRetriever",
new KnowledgeDocumentRetriever(new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper()))); new KnowledgeDocumentRetriever(knowledgeSearchPort));
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor); ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker); ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler()); ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
@@ -60,17 +58,11 @@ class LookupKnowledgeToolTest {
} }
@Test @Test
void filteredL1SuccessDoesNotRetry() { void lookupSuccessSingleAttemptUnfiltered() {
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database"); KnowledgeSearchHit hit = hit("db#chunk-0", "db", 0, "db.md", "Database Doc",
VectorSearchService.SearchResult result = searchResult( "mysql timeout runbook", 0.9);
"vec-1",
"{\"_source\":\"db.md\",\"docId\":\"db\",\"chunkIndex\":0,\"title\":\"Database Doc\",\"category\":\"database\"}",
"mysql timeout runbook",
0.2f);
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry)); when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
.thenReturn(List.of(result));
LookupResult lookup = tool.lookupKnowledge("mysql timeout"); LookupResult lookup = tool.lookupKnowledge("mysql timeout");
@@ -80,76 +72,30 @@ class LookupKnowledgeToolTest {
assertEquals("db#chunk-0", lookup.getEvidenceBlocks().get(0).getEvidenceKey()); assertEquals("db#chunk-0", lookup.getEvidenceBlocks().get(0).getEvidenceKey());
assertNotNull(lookup.getContextPack()); assertNotNull(lookup.getContextPack());
assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook")); assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook"));
assertEquals("FILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt()); assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
assertEquals(1, lookup.getRetrievalTrace().getAttempts().size()); assertEquals(1, lookup.getRetrievalTrace().getAttempts().size());
assertEquals("PRECISE", lookup.getRelevanceLevel()); assertEquals("PRECISE", lookup.getRelevanceLevel());
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database"); // 原始 query 直传、无 category 收窄(L0 已下沉 py-rag)
verify(vectorSearchService, never()).searchSimilarDocuments("mysql timeout", 3, null); verify(knowledgeSearchPort).search(new KnowledgeSearchRequest("mysql timeout", 3, null, null));
} }
@Test @Test
void filteredLowQualityTriggersRawUnfilteredRetry() { void rerankScoreDrivesRelevanceLevel() {
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database"); // rerank 绝对分 0.6:quality 原样采用 → REFERENCE(>=0.5 且 <0.75)
VectorSearchService.SearchResult weak = searchResult( KnowledgeSearchHit hit = hit("ref#chunk-0", "ref", 0, "ref.md", "Reference Doc",
"weak", "reference level content", 0.6);
"{\"_source\":\"weak.md\",\"docId\":\"weak\",\"chunkIndex\":0,\"title\":\"Weak\"}",
"weak candidate",
1.4f);
VectorSearchService.SearchResult strong = searchResult(
"strong",
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
"mysql timeout strong runbook",
0.2f);
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry)); when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
.thenReturn(List.of(weak));
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
.thenReturn(List.of(strong));
LookupResult lookup = tool.lookupKnowledge("mysql timeout"); LookupResult lookup = tool.lookupKnowledge("reference query");
assertTrue(lookup.isFound()); assertTrue(lookup.isFound());
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt()); assertEquals("REFERENCE", lookup.getRelevanceLevel());
assertEquals("filtered_vector_low_quality", lookup.getRetrievalTrace().getFallbackReason());
assertEquals(2, lookup.getRetrievalTrace().getAttempts().size());
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, null);
} }
@Test @Test
void filteredNoEvidenceTriggersRawUnfilteredRetry() { void emptyHitsYieldNoEvidence() {
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database"); when(knowledgeSearchPort.search(any())).thenReturn(Collections.emptyList());
VectorSearchService.SearchResult strong = searchResult(
"strong",
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
"mysql timeout strong runbook",
0.2f);
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
.thenReturn(Collections.emptyList());
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
.thenReturn(List.of(strong));
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
assertTrue(lookup.isFound());
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
assertEquals("filtered_vector_no_evidence", lookup.getRetrievalTrace().getFallbackReason());
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
}
@Test
void l0HintsDoNotBecomeStandaloneEvidenceWhenL1Fails() {
KnowledgeEntry entry = entry("fallback.md", "Fallback Doc", "fallback", "database");
when(knowledgeIndexService.analyzeQuery("fallback")).thenReturn(hint(entry));
when(vectorSearchService.searchSimilarDocuments("fallback", 3, "database"))
.thenReturn(Collections.emptyList());
when(vectorSearchService.searchSimilarDocuments("fallback", 3, null))
.thenReturn(Collections.emptyList());
LookupResult lookup = tool.lookupKnowledge("fallback"); LookupResult lookup = tool.lookupKnowledge("fallback");
@@ -157,52 +103,26 @@ class LookupKnowledgeToolTest {
assertEquals(0, lookup.getEvidenceBlockCount()); assertEquals(0, lookup.getEvidenceBlockCount());
assertTrue(lookup.getEvidenceBlocks().isEmpty()); assertTrue(lookup.getEvidenceBlocks().isEmpty());
assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus()); assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus());
assertTrue(String.valueOf(lookup.getRetrievalTrace().getQueryHints()).contains("Fallback Doc"));
}
@Test
void noL0HintUsesUnfilteredVectorSearch() {
VectorSearchService.SearchResult result = searchResult(
"vec-1",
"{\"_source\":\"perf.md\",\"docId\":\"perf\",\"chunkIndex\":0,\"title\":\"Perf\"}",
"performance tuning guide",
0.3f);
when(knowledgeIndexService.analyzeQuery("性能优化"))
.thenReturn(KnowledgeIndexService.L0Hint.empty());
when(vectorSearchService.searchSimilarDocuments("性能优化", 3, null))
.thenReturn(List.of(result));
LookupResult lookup = tool.lookupKnowledge("性能优化");
assertTrue(lookup.isFound());
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
assertEquals("perf.md", lookup.getEvidenceBlocks().get(0).getSource());
verify(vectorSearchService).searchSimilarDocuments("性能优化", 3, null);
} }
@Test @Test
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() { void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment"); KnowledgeSearchHit first = hit("a#chunk-0", "a", 0, "a.md", "Generic",
VectorSearchService.SearchResult first = searchResult( "generic troubleshooting", 0.95);
"a", KnowledgeSearchHit second = hit("b#chunk-0", "b", 0, "b.md", "Payment ERR_TIMEOUT",
"{\"_source\":\"a.md\",\"docId\":\"a\",\"chunkIndex\":0,\"title\":\"Generic\",\"category\":\"other\"}", "payment ERR_TIMEOUT timeout diagnosis", 0.55);
"generic troubleshooting", second = new KnowledgeSearchHit(
0.4f); second.id(), second.content(), second.score(), second.rawScore(), second.scoreLabel(),
VectorSearchService.SearchResult second = searchResult( second.metadataJson(), second.metadata(), second.docId(), second.chunkIndex(),
"b", second.evidenceKey(), second.source(), second.title(), "Payment > Timeout",
"{\"_source\":\"b.md\",\"docId\":\"b\",\"chunkIndex\":0,\"title\":\"Payment ERR_TIMEOUT\",\"breadcrumb\":\"Payment > Timeout\",\"category\":\"payment\"}", 2, second.denseDistance());
"payment ERR_TIMEOUT timeout diagnosis",
0.45f);
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry)); when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
.thenReturn(List.of(first, second));
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT"); LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
assertTrue(lookup.isFound()); assertTrue(lookup.isFound());
// originalRank order wins; keyword/domain boost must not promote second over first // originalRank 顺序权威;服务端 rerank 分不得改变主序
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource()); assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource()); assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
assertTrue(lookup.getRerankTrace().getItems().stream() assertTrue(lookup.getRerankTrace().getItems().stream()
@@ -216,21 +136,12 @@ class LookupKnowledgeToolTest {
@Test @Test
void keepsDistinctChunksFromSameSource() { void keepsDistinctChunksFromSameSource() {
KnowledgeEntry entry = entry("shared.md", "Shared", "shared", "payment"); KnowledgeSearchHit first = hit("shared#chunk-0", "shared", 0, "shared.md", "Shared",
VectorSearchService.SearchResult first = searchResult( "shared content 1", 0.9);
"a", KnowledgeSearchHit second = hit("shared#chunk-1", "shared", 1, "shared.md", "Shared",
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":0,\"title\":\"Shared\"}", "shared content 2", 0.85);
"shared content 1",
0.2f);
VectorSearchService.SearchResult second = searchResult(
"b",
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":1,\"title\":\"Shared\"}",
"shared content 2",
0.25f);
when(knowledgeIndexService.analyzeQuery("shared")).thenReturn(hint(entry)); when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
when(vectorSearchService.searchSimilarDocuments("shared", 3, "payment"))
.thenReturn(List.of(first, second));
LookupResult lookup = tool.lookupKnowledge("shared"); LookupResult lookup = tool.lookupKnowledge("shared");
@@ -243,48 +154,29 @@ class LookupKnowledgeToolTest {
assertEquals("shared content 2", lookup.getEvidenceBlocks().get(1).getContent()); assertEquals("shared content 2", lookup.getEvidenceBlocks().get(1).getContent());
} }
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) { /** 构造 py-rag 形态的命中:rerank 绝对分 + evidenceKey(docId#chunk-N)。 */
return KnowledgeEntry.builder() private KnowledgeSearchHit hit(String evidenceKey,
.filePath(filePath) String docId,
.title(title) int chunkIndex,
.keywords(List.of(keyword)) String source,
.summary(title + " summary") String title,
.category(category) String content,
.build(); double score) {
} return new KnowledgeSearchHit(
evidenceKey,
private VectorSearchService.SearchResult searchResult(String id, content,
String metadata, score,
String content, score,
float score) { RetrievalScoreLabels.RERANK,
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult(); null,
result.setId(id); Map.of(),
result.setMetadata(metadata); docId,
result.setContent(content); chunkIndex,
result.setScore(score); evidenceKey,
result.setRawScore((double) score); source,
result.setScoreLabel("dense"); title,
return result; null,
} 1,
null);
private KnowledgeIndexService.L0Hint hint(KnowledgeEntry... entries) {
List<KnowledgeEntry> matches = List.of(entries);
List<String> keywords = matches.stream()
.flatMap(entry -> entry.getKeywords() == null
? java.util.stream.Stream.empty()
: entry.getKeywords().stream())
.distinct()
.toList();
List<String> domains = matches.stream()
.map(KnowledgeEntry::getCategory)
.filter(category -> category != null && !category.isBlank())
.distinct()
.toList();
List<String> titles = matches.stream()
.map(KnowledgeEntry::getTitle)
.filter(title -> title != null && !title.isBlank())
.distinct()
.toList();
return new KnowledgeIndexService.L0Hint(matches, keywords, domains, keywords, titles);
} }
} }
-73
View File
@@ -1,73 +0,0 @@
services:
etcd:
container_name: milvus-etcd
image: quay.io/coreos/etcd:v3.5.18
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
healthcheck:
test: ["CMD", "etcdctl", "endpoint", "health"]
interval: 30s
timeout: 20s
retries: 3
minio:
container_name: milvus-minio
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
environment:
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
ports:
- "9001:9001"
- "9000:9000"
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
command: minio server /minio_data --console-address ":9001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
standalone:
container_name: milvus-standalone
image: milvusdb/milvus:v2.5.10
command: ["milvus", "run", "standalone"]
security_opt:
- seccomp:unconfined
environment:
ETCD_ENDPOINTS: etcd:2379
MINIO_ADDRESS: minio:9000
volumes:
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
interval: 30s
start_period: 90s
timeout: 20s
retries: 3
ports:
- "19530:19530"
- "9091:9091"
depends_on:
- "etcd"
- "minio"
# 这是新增的 Attu 服务哦!
attu:
container_name: milvus-attu
image: zilliz/attu:v2.5
ports:
- "8000:3000" # 把本地的 8000 端口映射到容器的 3000 端口 (Attu 默认端口)
environment:
# MILVUS_URL 指向 Docker 网络里的 Milvus standalone 服务
MILVUS_URL: standalone:19530
depends_on:
- standalone # 确保 Milvus 启动后再启动 Attu
networks:
default:
name: milvus