Compare commits
2
Commits
83193bdf4a
...
473bb5d004
| Author | SHA1 | Date | |
|---|---|---|---|
 zhuyongxin
|
473bb5d004 | ||
|
zhuyongxin
|
9cf162482d |
@@ -5,9 +5,9 @@
|
||||
SERVER_URL = http://localhost:9900
|
||||
UPLOAD_API = $(SERVER_URL)/api/upload
|
||||
DOCS_DIR = aiops-docs
|
||||
HEALTH_CHECK_API = $(SERVER_URL)/milvus/health
|
||||
DOCKER_COMPOSE_FILE = vector-database.yml
|
||||
MILVUS_CONTAINER = milvus-standalone
|
||||
# 服务就绪探测：9900 端口有 HTTP 响应即视为就绪
|
||||
HEALTH_CHECK = curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL)
|
||||
DOCKER_COMPOSE_FILE = docker-compose.yml
|
||||
|
||||
# 颜色输出
|
||||
GREEN = \033[0;32m
|
||||
@@ -23,7 +23,7 @@ help:
|
||||
@echo ""
|
||||
@echo "可用命令："
|
||||
@echo " $(YELLOW)make init$(NC) - 🚀 一键初始化（启动Docker → 启动服务 → 上传文档）"
|
||||
@echo " $(YELLOW)make up$(NC) - 启动 Docker Compose（Milvus 向量数据库）"
|
||||
@echo " $(YELLOW)make up$(NC) - 启动 Docker Compose（MySQL/Redis）"
|
||||
@echo " $(YELLOW)make down$(NC) - 停止 Docker Compose"
|
||||
@echo " $(YELLOW)make status$(NC) - 查看 Docker 容器状态"
|
||||
@echo " $(YELLOW)make start$(NC) - 启动 Spring Boot 服务（后台运行）"
|
||||
@@ -42,7 +42,7 @@ help:
|
||||
init:
|
||||
@echo "$(GREEN)🚀 开始一键初始化 SuperBizAgent...$(NC)"
|
||||
@echo ""
|
||||
@echo "$(YELLOW)步骤 1/4: 启动 Docker Compose（Milvus 向量数据库）$(NC)"
|
||||
@echo "$(YELLOW)步骤 1/4: 启动 Docker Compose（MySQL/Redis）$(NC)"
|
||||
@$(MAKE) up
|
||||
@echo ""
|
||||
@echo "$(YELLOW)步骤 2/4: 启动 Spring Boot 服务$(NC)"
|
||||
@@ -51,23 +51,23 @@ init:
|
||||
@echo "$(YELLOW)步骤 3/4: 等待服务就绪$(NC)"
|
||||
@$(MAKE) wait
|
||||
@echo ""
|
||||
@echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档到向量数据库$(NC)"
|
||||
@echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档（经 py-rag 入库）$(NC)"
|
||||
@$(MAKE) upload
|
||||
@echo ""
|
||||
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
||||
@echo "$(GREEN)✅ 初始化完成！所有文档已成功向量化存储到数据库$(NC)"
|
||||
@echo "$(GREEN)✅ 初始化完成！所有文档已成功入库（py-rag）$(NC)"
|
||||
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
||||
@echo ""
|
||||
@echo "$(GREEN)🌐 服务访问地址:$(NC)"
|
||||
@echo " API 服务: $(SERVER_URL)"
|
||||
@echo " Attu (Web UI): http://localhost:8000"
|
||||
@echo "$(YELLOW)💡 提示: 知识检索/入库由 py-rag 服务承担，请在其仓库单独启动$(NC)"
|
||||
@echo ""
|
||||
@echo "$(YELLOW)💡 提示: 服务正在后台运行，查看日志: tail -f server.log$(NC)"
|
||||
|
||||
# 启动 Spring Boot 服务（后台运行）
|
||||
start:
|
||||
@echo "$(YELLOW)🚀 启动 Spring Boot 服务...$(NC)"
|
||||
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
||||
@if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||
echo "$(GREEN)✅ 服务已经在运行中 ($(SERVER_URL))$(NC)"; \
|
||||
else \
|
||||
echo "$(YELLOW)📦 正在启动服务（后台运行）...$(NC)"; \
|
||||
@@ -84,7 +84,7 @@ wait:
|
||||
@max_attempts=60; \
|
||||
attempt=0; \
|
||||
while [ $$attempt -lt $$max_attempts ]; do \
|
||||
if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
||||
if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||
echo "$(GREEN)✅ 服务器已就绪！($(SERVER_URL))$(NC)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
@@ -100,7 +100,7 @@ wait:
|
||||
# 检查服务器是否运行
|
||||
check:
|
||||
@echo "$(YELLOW)🔍 检查服务器状态...$(NC)"
|
||||
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
||||
@if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||
echo "$(GREEN)✅ 服务器运行正常 ($(SERVER_URL))$(NC)"; \
|
||||
else \
|
||||
echo "$(RED)❌ 服务器未运行或无法连接！$(NC)"; \
|
||||
@@ -205,38 +205,14 @@ test-upload:
|
||||
echo "$(RED)测试文件不存在$(NC)"; \
|
||||
fi
|
||||
|
||||
# 启动 Docker Compose（智能检测，避免重复启动）
|
||||
# 启动 Docker Compose（MySQL/Redis；py-rag 服务在其仓库单独启动）
|
||||
up:
|
||||
@echo "$(YELLOW)🐳 检查 Docker 容器状态...$(NC)"
|
||||
@echo "$(YELLOW)🐳 启动 Docker Compose（MySQL/Redis）...$(NC)"
|
||||
@if [ ! -f "$(DOCKER_COMPOSE_FILE)" ]; then \
|
||||
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \
|
||||
echo "$(GREEN)✅ Milvus 容器已经在运行中$(NC)"; \
|
||||
echo "$(YELLOW)📋 当前运行的容器:$(NC)"; \
|
||||
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
||||
else \
|
||||
echo "$(YELLOW)🚀 启动 Docker Compose...$(NC)"; \
|
||||
docker-compose -f $(DOCKER_COMPOSE_FILE) up -d; \
|
||||
echo ""; \
|
||||
echo "$(YELLOW)⏳ 等待容器启动...$(NC)"; \
|
||||
sleep 5; \
|
||||
if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \
|
||||
echo "$(GREEN)✅ Docker Compose 启动成功！$(NC)"; \
|
||||
echo ""; \
|
||||
echo "$(GREEN)📋 运行中的容器:$(NC)"; \
|
||||
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
||||
echo ""; \
|
||||
echo "$(GREEN)🌐 服务访问地址:$(NC)"; \
|
||||
echo " Milvus: localhost:19530"; \
|
||||
echo " Attu (Web UI): http://localhost:8000"; \
|
||||
echo " MinIO: http://localhost:9001 (admin/minioadmin)"; \
|
||||
else \
|
||||
echo "$(RED)❌ 容器启动失败，请检查日志: docker-compose -f $(DOCKER_COMPOSE_FILE) logs$(NC)"; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi
|
||||
@docker-compose -f $(DOCKER_COMPOSE_FILE) up -d && echo "$(GREEN)✅ Docker Compose 启动完成$(NC)"
|
||||
|
||||
# 停止 Docker Compose
|
||||
down:
|
||||
@@ -245,24 +221,10 @@ down:
|
||||
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@if docker ps --format '{{.Names}}' | grep -q "milvus"; then \
|
||||
docker-compose -f $(DOCKER_COMPOSE_FILE) down; \
|
||||
echo "$(GREEN)✅ Docker Compose 已停止$(NC)"; \
|
||||
else \
|
||||
echo "$(YELLOW)⚠️ 没有运行中的 Milvus 容器$(NC)"; \
|
||||
fi
|
||||
@docker-compose -f $(DOCKER_COMPOSE_FILE) down && echo "$(GREEN)✅ Docker Compose 已停止$(NC)"
|
||||
|
||||
# 查看 Docker 容器状态
|
||||
status:
|
||||
@echo "$(YELLOW)📊 Docker 容器状态:$(NC)"
|
||||
@echo ""
|
||||
@if docker ps -a --format '{{.Names}}' | grep -q "milvus"; then \
|
||||
docker ps -a --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
||||
echo ""; \
|
||||
running=$$(docker ps --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
|
||||
total=$$(docker ps -a --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
|
||||
echo "$(GREEN)运行中: $$running / $$total$(NC)"; \
|
||||
else \
|
||||
echo "$(YELLOW)⚠️ 没有找到 Milvus 相关容器$(NC)"; \
|
||||
echo "$(YELLOW)提示: 运行 'make docker-up' 启动容器$(NC)"; \
|
||||
fi
|
||||
@docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||
|
||||
+2
-58
@@ -40,68 +40,12 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# Milvus 向量数据库（Standalone 模式）
|
||||
# 注意：生产环境建议使用 Zilliz Cloud 或 Milvus 集群
|
||||
etcd:
|
||||
image: quay.io/coreos/etcd:v3.5.5
|
||||
container_name: superbiz-etcd
|
||||
environment:
|
||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
||||
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
||||
- ETCD_QUOTA_BACKEND_BYTES=4294967296
|
||||
- ETCD_SNAPSHOT_COUNT=50000
|
||||
volumes:
|
||||
- etcd-data:/etcd
|
||||
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
|
||||
healthcheck:
|
||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
||||
container_name: superbiz-minio
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
volumes:
|
||||
- minio-data:/minio_data
|
||||
command: minio server /minio_data --console-address ":9001"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
milvus:
|
||||
image: milvusdb/milvus:v2.3.3
|
||||
container_name: superbiz-milvus
|
||||
depends_on:
|
||||
- etcd
|
||||
- minio
|
||||
environment:
|
||||
ETCD_ENDPOINTS: etcd:2379
|
||||
MINIO_ADDRESS: minio:9000
|
||||
volumes:
|
||||
- milvus-data:/var/lib/milvus
|
||||
ports:
|
||||
- "19530:19530"
|
||||
- "9091:9091"
|
||||
command: ["milvus", "run", "standalone"]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
||||
interval: 30s
|
||||
start_period: 90s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
# 向量检索与知识入库由独立的 py-rag 服务承担（见 py-rag 仓库），
|
||||
# 其依赖的 Milvus/etcd/MinIO 随 py-rag 部署，不再由本 compose 管理。
|
||||
|
||||
volumes:
|
||||
mysql-data:
|
||||
redis-data:
|
||||
etcd-data:
|
||||
minio-data:
|
||||
milvus-data:
|
||||
|
||||
networks:
|
||||
default:
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@
|
||||
| [architecture/agent-orchestration.md](architecture/agent-orchestration.md) | Agent 编排架构 |
|
||||
| [architecture/harness-quality-gates.md](architecture/harness-quality-gates.md) | Harness 与质量门禁 |
|
||||
| [architecture/session-trace-lifecycle.md](architecture/session-trace-lifecycle.md) | 会话与 Trace 生命周期 |
|
||||
| [architecture/RAG知识检索架构.md](architecture/RAG知识检索架构.md) | 当前 hybrid 检索架构 |
|
||||
| [architecture/RAG知识检索架构.md](architecture/RAG知识检索架构.md) | 当前检索架构（py-rag 知识服务接入） |
|
||||
| [issues/README.md](issues/README.md) | MVP issue 索引 |
|
||||
| [tables/README.md](tables/README.md) | 当前 MySQL 表说明 |
|
||||
| [demo/README.md](demo/README.md) | Demo 运行和演示材料 |
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
# RAG 检索可观测性、审计与 Trace（现行）
|
||||
|
||||
**更新日期**：2026-07-28
|
||||
**更新日期**：2026-09-29
|
||||
**状态**：当前可运行
|
||||
**关联**：`lookup_knowledge`、Harness `ToolBoundary`、`tool_invocation`、`DiagnosisTraceService`、离线 eval
|
||||
**关联**：`lookup_knowledge`、Harness `ToolBoundary`、`tool_invocation`、`DiagnosisTraceService`、离线 eval
|
||||
|
||||
> **2026-09-29 RAG 抽离影响**：检索后端切换为 py-rag 服务（见 [RAG知识检索架构.md](./RAG知识检索架构.md)）。
|
||||
> Trace / 审计的三层边界与读写接口**不变**；变化仅在内容语义：
|
||||
> L0 已下沉（`queryHints` 恒为空结构、`categoryFilter` 恒为 null、attempt 只剩 `UNFILTERED_VECTOR`），
|
||||
> 质量分统一为 py-rag rerank 绝对分（scoreLabel=RERANK）。
|
||||
> 文中涉及 FILTERED/RETRY attempt 的示例为历史数据读法，保留供回放旧 Run。
|
||||
|
||||
---
|
||||
|
||||
@@ -119,21 +125,21 @@ flowchart TB
|
||||
| 字段 | 含义 |
|
||||
|------|------|
|
||||
| `originalQuery` | 原始查询 |
|
||||
| `rewrittenQuery` | L0/变换后用于检索的 query |
|
||||
| `categoryFilter` | 首次过滤的 category（可 null） |
|
||||
| `rewrittenQuery` | 用于检索的 query（L0 下沉后恒等于 originalQuery） |
|
||||
| `categoryFilter` | 首次过滤的 category（L0 下沉后恒为 null） |
|
||||
| `selectedAttempt` | 最终采用的 attempt 名 |
|
||||
| `fallbackReason` | 如 `filtered_vector_low_quality`；未降级为 null |
|
||||
| `evidenceStatus` | 内部：`supported` / `no_evidence` 等 |
|
||||
| `queryHints` | L0：domains、keywords、entities、l0_match_count… |
|
||||
| `queryHints` | L0 提示（下沉后恒为空 domains/keywords/entities 与 l0_match_count=0） |
|
||||
| `attempts[]` | 每次检索尝试快照 |
|
||||
|
||||
**常见 `selectedAttempt`：**
|
||||
**`selectedAttempt` 取值：**
|
||||
|
||||
| 值 | 含义 |
|
||||
|----|------|
|
||||
| `FILTERED_VECTOR` | 带 category 的首次检索即采用 |
|
||||
| `UNFILTERED_VECTOR` | 无 category，直接全库检索 |
|
||||
| `UNFILTERED_VECTOR_RETRY` | filtered 低质/无证据后去掉 category 重试 |
|
||||
| `UNFILTERED_VECTOR` | **当前唯一会出现**：无 category，直传 py-rag 检索 |
|
||||
| `FILTERED_VECTOR` | （历史）带 category 的首次检索即采用；L0 下沉后不再产生 |
|
||||
| `UNFILTERED_VECTOR_RETRY` | （历史）filtered 低质/无证据后去掉 category 重试；分支保留但不可达，仅见于旧 Run 回放 |
|
||||
|
||||
**单次 `attempts[]` 元素：**
|
||||
|
||||
@@ -148,21 +154,18 @@ flowchart TB
|
||||
| `durationMs` | 耗时 |
|
||||
| `errorMessage` | 失败时 |
|
||||
|
||||
### 3.3 一次典型路径（含 filter fallback）
|
||||
### 3.3 一次典型路径（当前：单 attempt 直传）
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Q[query] --> L0[L0 hint → 可选 categoryFilter]
|
||||
L0 --> A1[attempt FILTERED_VECTOR]
|
||||
A1 --> PQ{isLowQuality?}
|
||||
PQ -->|否| USE1[selectedAttempt = FILTERED_VECTOR]
|
||||
PQ -->|是| A2[attempt UNFILTERED_VECTOR_RETRY]
|
||||
A2 --> USE2[selectedAttempt = RETRY<br/>fallbackReason = low_quality / no_evidence]
|
||||
USE1 --> POST[PostProcess · evidenceBlocks · relevanceLevel]
|
||||
USE2 --> POST
|
||||
Q[query 原始句直传] --> A1[attempt UNFILTERED_VECTOR<br/>PyRagKnowledgeSearchAdapter → py-rag]
|
||||
A1 --> POST[PostProcess · evidenceBlocks · relevanceLevel]
|
||||
POST --> LR[LookupResult 完整 Trace]
|
||||
```
|
||||
|
||||
> 历史 filter fallback 路径（L0 → FILTERED_VECTOR → 低质 → UNFILTERED_VECTOR_RETRY）的流程图已随 L0 下沉移除；
|
||||
> 旧 Run 的 Trace 回放仍可见该结构，字段含义见 §3.2。
|
||||
|
||||
### 3.4 与 Agent 投影的关系
|
||||
|
||||
```mermaid
|
||||
@@ -208,35 +211,27 @@ flowchart LR
|
||||
| `output_preview` | level/attempt 摘要 | status=… |
|
||||
| `duration_ms` / `success` | 有 | 有 |
|
||||
|
||||
### 4.3 `retrieval_details`（rag_lookup_v1）示例
|
||||
### 4.3 `retrieval_details`（rag_lookup_v1）示例（当前形态）
|
||||
|
||||
```json
|
||||
{
|
||||
"audit_schema": "rag_lookup_v1",
|
||||
"search_mode": "hybrid",
|
||||
"selected_attempt": "UNFILTERED_VECTOR_RETRY",
|
||||
"fallback_reason": "filtered_vector_low_quality",
|
||||
"category_filter": "overfilter-decoy",
|
||||
"evidence_keys": ["doc#chunk-0"],
|
||||
"sources": ["doc"],
|
||||
"evidence_candidate_count": 8,
|
||||
"selected_attempt": "UNFILTERED_VECTOR",
|
||||
"fallback_reason": null,
|
||||
"category_filter": null,
|
||||
"evidence_keys": ["e2e-gateway-b9c1fa12-md-34223174#chunk-1"],
|
||||
"sources": ["e2e-gateway-b9c1fa12-md"],
|
||||
"evidence_candidate_count": 5,
|
||||
"evidence_block_count": 2,
|
||||
"l0_hints": { "domains": ["mysql"], "matched_keywords": ["pool"] },
|
||||
"l0_hints": { "domains": [], "matched_keywords": [] },
|
||||
"attempts": [
|
||||
{
|
||||
"name": "FILTERED_VECTOR",
|
||||
"category_filter": "overfilter-decoy",
|
||||
"candidate_count": 2,
|
||||
"usable": false,
|
||||
"top_similarity": 0.3,
|
||||
"duration_ms": 12
|
||||
},
|
||||
{
|
||||
"name": "UNFILTERED_VECTOR_RETRY",
|
||||
"name": "UNFILTERED_VECTOR",
|
||||
"candidate_count": 5,
|
||||
"usable": true,
|
||||
"top_similarity": 0.9,
|
||||
"duration_ms": 20
|
||||
"top_similarity": 0.91,
|
||||
"duration_ms": 640
|
||||
}
|
||||
],
|
||||
"truncated": false,
|
||||
@@ -247,7 +242,8 @@ flowchart LR
|
||||
}
|
||||
```
|
||||
|
||||
**默认不落库：** 原始 query 全文、chunk 正文 excerpt、完整 rerankTrace（体积与隐私）。
|
||||
**默认不落库：** 原始 query 全文、chunk 正文 excerpt、完整 rerankTrace（体积与隐私）。
|
||||
历史 Run 中 `selected_attempt=FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 与非空 `l0_hints` 为 L0 下沉前的旧数据形态。
|
||||
|
||||
### 4.4 Trace API：人怎么读 RAG
|
||||
|
||||
@@ -304,12 +300,12 @@ flowchart TB
|
||||
|
||||
| 现象 | 优先看 |
|
||||
|------|--------|
|
||||
| 为何走了 retry | `fallback_reason` + 两次 `attempts` |
|
||||
| 是否 hybrid | `search_mode` |
|
||||
| 滤错域 | `category_filter` + L0 domains |
|
||||
| 是否 hybrid | `search_mode`（hybrid/semantic 对应 py-rag 融合/纯向量） |
|
||||
| 滤错域 | （历史）`category_filter` + L0 domains；L0 下沉后恒为 null |
|
||||
| 相关度档 | 列 `relevanceLevel`（PRECISE/REFERENCE） |
|
||||
| 返回了哪些块 | `evidence_keys` / `sources`（无正文） |
|
||||
| Agent 是否被截断 | `truncated` / `returned_count` |
|
||||
| 为何走了 retry | （历史）`fallback_reason` + 两次 `attempts`；L0 下沉后单 attempt，不再产生 |
|
||||
|
||||
### 4.6 diagnosis_trace 事件 vs tool_invocation 行
|
||||
|
||||
@@ -352,10 +348,11 @@ flowchart LR
|
||||
|
||||
| 旧（archive `retrieval-observability`） | 现 |
|
||||
|----------------------------------------|-----|
|
||||
| `vector-store.mode` 多后端 | `search_mode` dense\|hybrid，单一 V2 store |
|
||||
| `vector-store.mode` 多后端 | `search_mode` dense\|hybrid（现映射 py-rag semantic\|hybrid） |
|
||||
| sink 理想化未落地 | `RagLookupAuditEnricher` + 列回填 |
|
||||
| `relevance_level` 混用 evidence_status | **列仅 RAG 等级**；契约状态在 details |
|
||||
| 未写清 Trace API 读法 | 本文 §4.4–4.5 |
|
||||
| L0 hint / FILTERED-RETRY attempt（2026-07-28 形态） | 2026-09-29 L0 下沉 py-rag：单 attempt、queryHints 恒空、质量分 RERANK 直传 |
|
||||
|
||||
---
|
||||
|
||||
|
||||
+131
-271
@@ -1,13 +1,16 @@
|
||||
# RAG 知识检索架构
|
||||
|
||||
**更新日期**：2026-07-28
|
||||
**更新日期**：2026-09-29
|
||||
**状态**：当前可运行架构
|
||||
**关联实现**：`lookup_knowledge`、`MilvusHybridKnowledgeStore`、`KnowledgeSearchPort`
|
||||
**关联运维**：`scripts/rebuild_hybrid_knowledge.py`、`POST /api/knowledge/rebuild-hybrid`
|
||||
**关联实现**：`lookup_knowledge`、`KnowledgeSearchPort`、`PyRagKnowledgeSearchAdapter`、`PyRagClient`
|
||||
**关联契约**：py-rag 仓库 `docs/Java接入文档.md`（API v1，冻结面）
|
||||
**关联运维**：py-rag `/api/v1/collections:rebuild`（全量重建）、py-rag `/api/v1/documents:ingest`（单文档入库）
|
||||
|
||||
## 1. 定位
|
||||
|
||||
知识检索是 Diagnosis Agent 的显式证据工具，不是隐式 Advisor。
|
||||
检索算法（dense+BM25 融合、rerank、判级）与文档入库（解析、frontmatter、分块、向量化）
|
||||
**全部由独立的 py-rag 知识服务承担**；Java 侧只保留 harness 消费面与 HTTP 客户端。
|
||||
|
||||
```text
|
||||
Diagnosis Agent
|
||||
@@ -19,7 +22,7 @@ Diagnosis Agent
|
||||
目标：
|
||||
|
||||
- 保留 Agent 可见的工具调用与证据边界
|
||||
- 用单一向量后端完成 dense + BM25 hybrid 检索
|
||||
- 检索/入库基础设施外置为独立服务，Java 侧不感知引擎细节
|
||||
- 用 chunk 级证据身份保证同文档多片段可同时进入上下文
|
||||
- 检索行为可配置、可重建、可审计
|
||||
|
||||
@@ -41,336 +44,193 @@ flowchart LR
|
||||
subgraph RetrievalBoundary["Retrieval boundary"]
|
||||
Backend["LookupKnowledgeTool"]
|
||||
Port["KnowledgeSearchPort"]
|
||||
Store["MilvusHybridKnowledgeStore"]
|
||||
Remote["PyRagKnowledgeSearchAdapter"]
|
||||
Service["py-rag 知识服务 (HTTP /api/v1)"]
|
||||
end
|
||||
|
||||
Agent --> Tool
|
||||
Tool --> Adapter
|
||||
Adapter --> Backend
|
||||
Backend --> Port
|
||||
Port --> Store
|
||||
Port --> Remote
|
||||
Remote -->|HTTP| Service
|
||||
Adapter --> Projector
|
||||
Adapter --> Canonical
|
||||
```
|
||||
|
||||
| 边界 | 职责 | 不负责 |
|
||||
|---|---|---|
|
||||
| Agent | 决定何时检索、如何用证据写报告 | 不直接访问 Milvus / MySQL 元数据表 |
|
||||
| Agent | 决定何时检索、如何用证据写报告 | 不直接访问 py-rag / MySQL 元数据表 |
|
||||
| Harness | Tool 校验、投影裁剪、canonical 存证 | 不改写检索排序算法 |
|
||||
| Retrieval | L0 hint、dense/BM25 召回、后处理、打包 | 不绕过 ACI 直接给 Agent 原始库响应 |
|
||||
| Retrieval | 请求映射、后处理、打包；py-rag 承担召回/融合/rerank/判级 | 不绕过 ACI 直接给 Agent 原始库响应 |
|
||||
|
||||
## 3. 当前主链路
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A["lookup_knowledge(query)"] --> B["KnowledgeQueryTransformer"]
|
||||
B --> C["L0 hint: domain / keywords / categoryFilter"]
|
||||
C --> D["KnowledgeDocumentRetriever"]
|
||||
D --> E["KnowledgeSearchPort"]
|
||||
E --> F["VectorSearchService"]
|
||||
F --> G{"retrieval.search.mode"}
|
||||
G -->|dense| H["MilvusHybridKnowledgeStore.searchDense"]
|
||||
G -->|hybrid| I["MilvusHybridKnowledgeStore.searchHybrid"]
|
||||
H --> J["candidates + chunk identity"]
|
||||
I --> J
|
||||
J --> K["KnowledgeEvidencePostProcessor"]
|
||||
K --> L["evidenceKey dedup / maxChunksPerDocument / return-n"]
|
||||
L --> M{"filtered low quality?"}
|
||||
M -->|yes and had categoryFilter| N["unfiltered retry"]
|
||||
N --> K
|
||||
M -->|no| O["KnowledgeContextPacker"]
|
||||
O --> P["LookupResultAssembler"]
|
||||
P --> Q["RagResultProjector"]
|
||||
Q --> R["Agent-facing RagToolResult"]
|
||||
A["lookup_knowledge(query)"] --> B["原始 query 直传（L0 已下沉 py-rag）"]
|
||||
B --> C["KnowledgeDocumentRetriever"]
|
||||
C --> D["KnowledgeSearchPort"]
|
||||
D --> E["PyRagKnowledgeSearchAdapter"]
|
||||
E -->|POST /api/v1/search| F["py-rag: dense+BM25 融合 / rerank / 判级"]
|
||||
F --> G["hits + evidenceKey（docId#chunk-N）"]
|
||||
G --> H["KnowledgeEvidencePostProcessor"]
|
||||
H --> I["evidenceKey dedup / maxChunksPerDocument / return-n"]
|
||||
I --> J["KnowledgeContextPacker"]
|
||||
J --> K["LookupResultAssembler"]
|
||||
K --> L["RagResultProjector"]
|
||||
L --> M["Agent-facing RagToolResult"]
|
||||
```
|
||||
|
||||
对应代码：
|
||||
|
||||
| 阶段 | 类 | 职责 |
|
||||
|---|---|---|
|
||||
| Tool 编排 | `LookupKnowledgeTool` | 串联 transform / retrieve / post / pack |
|
||||
| Query 理解 | `KnowledgeQueryTransformer` + `KnowledgeIndexService` | L0 只产 hint 与可选 category filter |
|
||||
| 检索端口 | `KnowledgeSearchPort` / `VectorKnowledgeSearchAdapter` | 屏蔽底层存储细节 |
|
||||
| 检索门面 | `VectorSearchService` | `dense` 或 `hybrid` 路由 |
|
||||
| 向量后端 | `MilvusHybridKnowledgeStore` | 唯一知识库读写后端（MilvusClientV2） |
|
||||
| 后处理 | `KnowledgeEvidencePostProcessor` | 归一化、规则 boost、chunk 去重、相关度等级 |
|
||||
| Tool 编排 | `LookupKnowledgeTool` | 串联 retrieve / post / pack；UNFILTERED 单 attempt 主路径 |
|
||||
| 检索端口 | `KnowledgeSearchPort` / `PyRagKnowledgeSearchAdapter` | 防腐层；请求映射 + 命中归一化 |
|
||||
| HTTP 客户端 | `PyRagClient` | API v1 调用、错误信封（`E_*`）、`X-Request-ID`、分端点超时 |
|
||||
| 后处理 | `KnowledgeEvidencePostProcessor` | qualityScore（RERANK 直传）、chunk 去重、相关度等级 |
|
||||
| 打包 | `KnowledgeContextPacker` | 有界 context pack |
|
||||
| 投影 | `RagResultProjector` | 只暴露 Agent 可见 evidence 字段 |
|
||||
|
||||
## 4. 唯一向量后端：MilvusClientV2
|
||||
2026-09-29 抽离时删除的 Java 侧组件：`MilvusHybridKnowledgeStore`、`VectorSearchService`、
|
||||
`VectorKnowledgeSearchAdapter`、`RrfFusion` / `LexicalRanker`（融合评分下沉）、
|
||||
`KnowledgeIndexService` / `KnowledgeDomainService`（L0 索引）、
|
||||
`DocumentChunkService` / `FrontmatterParser` / `TextExtractorService`（入库解析）、
|
||||
`KnowledgeQueryTransformer`（L0 query 理解）。
|
||||
|
||||
### 4.1 已废弃路径
|
||||
## 4. py-rag 检索契约映射
|
||||
|
||||
以下路径**不再**用于 `lookup_knowledge`：
|
||||
请求映射（`PyRagKnowledgeSearchAdapter`）：
|
||||
|
||||
- legacy `MilvusServiceClient` search / insert
|
||||
- `retrieval.vector-store.mode=sdk|spring|auto`
|
||||
- Spring AI `VectorStore` 作为知识检索主路径
|
||||
| Java（KnowledgeSearchRequest） | py-rag（/api/v1/search） | 说明 |
|
||||
|---|---|---|
|
||||
| `query` | `query` | 原始检索句直传，服务端自行处理边界与精排 |
|
||||
| `mode=DENSE` | `mode=semantic` | 纯向量，对照/排障用 |
|
||||
| `mode=HYBRID` | `mode=hybrid` | dense+BM25 融合，线上主路径（`retrieval.search.mode: hybrid`） |
|
||||
| `topK` | `retrieve_k` / `return_n` / `max_chunks_per_document` | 三者同置 topK：chunk 去重截断由 Java 后处理器统一负责，避免服务端预截断 |
|
||||
| `categoryFilter` | `category` | L0 下沉后恒为 null（不过滤） |
|
||||
| — | `kb_scope` | 不传，由 py-rag 部署配置决定 |
|
||||
|
||||
### 4.2 当前后端
|
||||
响应映射：
|
||||
|
||||
| py-rag | Java（KnowledgeSearchHit） | 说明 |
|
||||
|---|---|---|
|
||||
| `evidence_key` | `evidenceKey` / `docId` / `chunkIndex` | `docId#chunk-N`，与 EvidenceGuard 验真约定一致 |
|
||||
| `excerpt` | `content` | 进入 context pack 的正文 |
|
||||
| `quality_score` | `score` / `rawScore` | rerank 绝对相关分 [0,1]，越大越好 |
|
||||
| — | `scoreLabel=RERANK` | `RetrievalScoreNormalizer` 对 RERANK 分支 quality 原样 clamp，不走 L2/rank 归一化 |
|
||||
| `relevance_level` | （参考值） | Java 后处理按同阈值（0.75/0.5）独立判级，语义一致 |
|
||||
| `evidence_status=no_evidence` | 空列表 | 正常业务响应（服务端保证 hits=[]），Agent 侧按"无知识可用"处理 |
|
||||
|
||||
超时与重试矩阵见 py-rag 仓库 `docs/Java接入文档.md` 第 6 节；Java 侧由 `pyrag.*` 配置承载。
|
||||
|
||||
## 5. 入库与重建
|
||||
|
||||
### 5.1 日常写入
|
||||
|
||||
```text
|
||||
写入:
|
||||
VectorIndexService
|
||||
-> MilvusHybridKnowledgeStore.upsertChunk
|
||||
业务上传（DocumentController /api/documents/upload）
|
||||
-> MySQL api_document（业务元数据：faultSource 等）+ 本地原件保存
|
||||
-> PyRagClient.ingest（multipart 透传原件 + category）
|
||||
-> py-rag：解析 / frontmatter 校验 / 分块 / 向量化 / 索引
|
||||
|
||||
读取:
|
||||
VectorSearchService
|
||||
-> MilvusHybridKnowledgeStore.searchDense
|
||||
-> MilvusHybridKnowledgeStore.searchHybrid
|
||||
简单上传（FileUploadController /api/upload）
|
||||
-> 本地保存 + PyRagClient.ingest（category 可选参数，缺省 default；入库失败不影响上传成功语义）
|
||||
```
|
||||
|
||||
默认 collection：
|
||||
MySQL `api_document.docId` 取 py-rag 返回的 `doc_id`（路径 slug + 内容 SHA-256 前 8 位），
|
||||
与检索 `evidence_key` 的 docId 段对齐；`chunk_count` 取 ingest 响应。
|
||||
|
||||
### 5.2 全量重建
|
||||
|
||||
```text
|
||||
POST /api/v1/collections:rebuild?confirm=REBUILD （py-rag 服务端）
|
||||
GET /api/v1/tasks/{task_id} （任务状态）
|
||||
```
|
||||
|
||||
- rebuild 为 py-rag 异步任务；执行期间 ingest 返回 409（`E_REBUILD_IN_PROGRESS`），search 不受影响
|
||||
- 原料为 py-rag 服务端 `data/knowledge_base/` 下历次 ingest 落盘的 md
|
||||
- 旧 Java 侧 `POST /api/knowledge/rebuild-hybrid` 与 `scripts/rebuild_hybrid_knowledge.py` 已删除
|
||||
|
||||
### 5.3 单文档删除
|
||||
|
||||
py-rag API v1 没有单文档删除端点。`DELETE /api/documents/{docId}` 只删 MySQL 元数据与本地原件；
|
||||
py-rag 侧已入库内容需全量重建后才会消失（见 `DocumentManagementService.deleteDocument` 注释）。
|
||||
|
||||
## 6. 部署与配置
|
||||
|
||||
```yaml
|
||||
milvus:
|
||||
collection: biz
|
||||
pyrag:
|
||||
base-url: ${PYRAG_BASE_URL:http://localhost:8000}
|
||||
connect-timeout-ms: 3000
|
||||
search-read-timeout-ms: 5000 # 正常 300–800ms（含 rerank 外呼）
|
||||
ingest-read-timeout-ms: 30000
|
||||
default-read-timeout-ms: 10000
|
||||
```
|
||||
|
||||
重建时会 drop + recreate 该 collection，并按 dense + BM25 schema 重建。
|
||||
- Milvus / etcd / MinIO 随 py-rag 部署，不再由本仓库 `docker-compose.yml` 管理（compose 仅剩 MySQL/Redis）
|
||||
- `vector-database.yml` 已删除；Makefile 的 up/down/status 只管 MySQL/Redis
|
||||
- `retrieval.search.mode: hybrid` 语义保留：映射 py-rag 的 `hybrid` / `semantic`
|
||||
|
||||
## 5. Collection Schema
|
||||
|
||||
`biz`（可配置）逻辑字段：
|
||||
|
||||
| 字段 | 类型 | 用途 |
|
||||
|---|---|---|
|
||||
| `id` | VarChar PK | chunk 级主键 |
|
||||
| `content` | VarChar | 返回给 Agent 的原文片段 |
|
||||
| `search_text` | VarChar + analyzer | BM25 输入文本 |
|
||||
| `sparse_vector` | SparseFloatVector | BM25 Function 输出 |
|
||||
| `vector` | FloatVector | dense embedding |
|
||||
| `metadata` | JSON | docId / chunkIndex / category / kb_scope / title / breadcrumb 等 |
|
||||
|
||||
Function：
|
||||
## 7. 证据身份与去重（不变）
|
||||
|
||||
```text
|
||||
BM25(search_text -> sparse_vector)
|
||||
```
|
||||
|
||||
索引：
|
||||
|
||||
```text
|
||||
vector -> IVF_FLAT + L2
|
||||
sparse_vector -> SPARSE_INVERTED_INDEX + BM25
|
||||
```
|
||||
|
||||
写入时：
|
||||
|
||||
- `content` 保存原始 chunk 正文
|
||||
- `search_text` / dense embedding 使用 `Title + Path + Content` 拼装文本
|
||||
- metadata 必须带 `docId`、`chunkIndex`，供 chunk 级证据身份使用
|
||||
|
||||
## 6. 检索模式
|
||||
|
||||
配置：
|
||||
|
||||
```yaml
|
||||
retrieval:
|
||||
search:
|
||||
mode: hybrid # dense | hybrid（见 6.0 用途约定）
|
||||
hybrid:
|
||||
rrf-k: 60
|
||||
kb-scope: ""
|
||||
rag:
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
```
|
||||
|
||||
### 6.0 模式用途约定（保留双 mode 的原因）
|
||||
|
||||
知识库 **只维护一套** dense + BM25 schema 数据（默认 collection `biz`）。
|
||||
`retrieval.search.mode` 切换的是**同库上的查询算法**，不是两套互斥索引、也不是两套写入路径。
|
||||
|
||||
| 模式 | 定位 | 说明 |
|
||||
|---|---|---|
|
||||
| **hybrid** | **线上主路径 / 默认** | dense ANN + 服务端 BM25 + RRF；`lookup_knowledge` 正式召回只认此模式 |
|
||||
| **dense** | **对照 / 评测 / 排障** | 仅 dense ANN，用于和 hybrid 对比召回效果（命中文档/chunk、排名差异等） |
|
||||
|
||||
约定：
|
||||
|
||||
1. 生产配置保持 `mode: hybrid`；不要把 dense 当成第二套长期并行的线上策略。
|
||||
2. 需要看「去掉 BM25+RRF 后召回差在哪」时，临时切 `mode: dense`，其它参数（`retrieve-k`、`return-n`、category filter、query 集）尽量固定，再切回 hybrid。
|
||||
3. hybrid 入库的数据 **完全适用于** dense-only 查询：每条 chunk 都写了 `vector`；dense 模式只是不使用 `sparse_vector` / BM25 子路。
|
||||
4. 代码里 `@Value` 在配置缺失时的兜底仍可能是 `dense`（历史兼容）；**以 `application.yml` 的 hybrid 为准**。若做回归，确认运行配置而不是只看注解默认值。
|
||||
|
||||
不建议的用法：
|
||||
|
||||
- 按请求/按租户在 dense 与 hybrid 之间当产品功能随意切换（当前也无稳定的 per-call mode 覆盖）。
|
||||
- 把 dense 模式的相关度表现直接当成 hybrid 的最终质量结论（hybrid 排序信 RRF，后处理分数仍多 L2 兼容，见下节）。
|
||||
|
||||
### 6.1 dense（对照基线）
|
||||
|
||||
```text
|
||||
query
|
||||
-> embedding
|
||||
-> dense ANN on vector
|
||||
-> topK
|
||||
```
|
||||
|
||||
仅走 `vector` 字段的 L2 ANN。用于基线对比，不作为正式主路径。
|
||||
|
||||
### 6.2 hybrid（当前默认 / 主路径）
|
||||
|
||||
```text
|
||||
query
|
||||
-> path A: dense ANN(query embedding)
|
||||
-> path B: BM25 sparse ANN(raw query text)
|
||||
-> Milvus hybridSearch + RRFRanker(k)
|
||||
-> topK fused hits
|
||||
```
|
||||
|
||||
说明：hybrid **内部**的 dense 子路是融合的一部分，与配置项 mode=dense（整次检索只跑单路 ANN）不是同一概念。
|
||||
|
||||
分数与后处理（quality 统一，2026-07-28）：
|
||||
|
||||
- 一级 scoreLabel 仅 **dense | hybrid**（旧别名 canonicalize）。
|
||||
- **dense**：score = L2；qualityScore = 1 - clamp(L2)/maxL2Distance。
|
||||
- **hybrid**：返回序 = RRF 序；qualityScore 由 **本轮 rank 线性映射**（不把 RRF 原分当 L2；不做 dense L2 回填覆盖主分；无 m25_only_* 一级 label）。
|
||||
- 后处理：**统一**消费 qualityScore；排序主序 = originalRank；**不做** L0 关键词/domain contains 加分改序（重叠仅可写 hitReasons 解释）。
|
||||
-
|
||||
elevance_level / category 低质 unfiltered retry：只看 top qualityScore 与阈值。
|
||||
- 实现：RetrievalScoreNormalizer、KnowledgeEvidencePostProcessor；详见 OpenSpec
|
||||
ag-quality-score-unify。
|
||||
|
||||
### 6.3 category filter 与降级
|
||||
|
||||
```text
|
||||
if L0 给出唯一 domain:
|
||||
先 filtered 检索
|
||||
if 无证据或 topSimilarity < referenceThreshold:
|
||||
再 unfiltered retry
|
||||
else:
|
||||
直接 unfiltered
|
||||
```
|
||||
|
||||
这里的 filter 是 metadata category / kb_scope 约束，不是第二套向量库。
|
||||
|
||||
## 7. 证据身份与去重
|
||||
|
||||
Delivery 1 已落地：
|
||||
|
||||
```text
|
||||
evidenceKey =
|
||||
docId#chunk-{chunkIndex}
|
||||
evidenceKey = docId#chunk-{chunkIndex}
|
||||
fallback: vector:{id}
|
||||
fallback: rank:{n}
|
||||
```
|
||||
|
||||
规则：
|
||||
- 去重按 `evidenceKey`，同文档不同 chunk 可同时保留
|
||||
- `rag.max-chunks-per-document` / `rag.return-n` 在 Java 后处理器生效
|
||||
- Agent 投影中的 `document_id` 使用 chunk 级 evidenceKey，EvidenceGuard 据此验真
|
||||
|
||||
- 去重按 `evidenceKey`，不是按 source 文档路径
|
||||
- 同文档不同 chunk 可同时保留
|
||||
- `rag.max-chunks-per-document` 限制单文档最多进入结果的 chunk 数
|
||||
- `rag.return-n` 限制后处理后最多返回条数
|
||||
- Agent 投影中的 `document_id` 使用 chunk 级 evidenceKey
|
||||
|
||||
这保证 hybrid 召回的多片段不会在后处理/投影阶段被文档级折叠吞掉。
|
||||
|
||||
## 8. L0 / L1 职责
|
||||
|
||||
| 层 | 做什么 | 不做什么 |
|
||||
|---|---|---|
|
||||
| L0 | domain/keyword hint、可选 category filter、trace 解释、轻规则 boost | 不直接当事实 evidence |
|
||||
| L1 dense/BM25 | 事实证据召回 | 不依赖 frontmatter 关键词命中才返回正文 |
|
||||
|
||||
L0 命中文档正文不会在 L1 失败时兜底成 evidence。
|
||||
|
||||
## 9. Agent 可见契约
|
||||
## 8. Agent 可见契约（不变）
|
||||
|
||||
Agent 只看到有界 `RagToolResult`：
|
||||
|
||||
- `evidence_status`
|
||||
- `tool_call_id`
|
||||
- `query`
|
||||
- `evidence_status` / `tool_call_id` / `query`
|
||||
- `evidence[]`：`document_id` / `source` / `title` / `breadcrumb` / `excerpt`
|
||||
- `relevance_level`
|
||||
- `relevance_level`（PRECISE / REFERENCE；`RagRelevanceLevel.HIGHLY_RELEVANT` 为保留档）
|
||||
- `truncated` / `returned_count`
|
||||
|
||||
不暴露：
|
||||
不暴露：raw score、retrievalTrace / rerankTrace、contextPack 全文、py-rag 地址与凭据。
|
||||
完整内部结果仍在 `LookupResult` 中，供审计与调试使用。Trace / 审计读法见
|
||||
[RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)。
|
||||
|
||||
- raw score / fused score
|
||||
- retrievalTrace / rerankTrace
|
||||
- contextPack 全文
|
||||
- Milvus 内部字段与凭据
|
||||
## 9. L0 下沉
|
||||
|
||||
完整内部结果仍在 `LookupResult` 中，供审计与调试使用。
|
||||
L0 query 理解（domain/keyword hint → 可选 categoryFilter）随本次抽离**整体下沉 py-rag**：
|
||||
|
||||
### 9.1 Trace 与审计（现行入口）
|
||||
- `KnowledgeQueryTransformer` / `KnowledgeIndexService.analyzeQuery` 已删除
|
||||
- `lookup_knowledge` 直传原始 query；`categoryFilter` 恒为 null，走 UNFILTERED 单 attempt
|
||||
- `LookupKnowledgeTool` 中 filtered→unfiltered 降级分支保留但不可达（作为未来 Java 侧过滤策略的兜底骨架）
|
||||
- `RetrievalTrace.queryHints` 恒为空结构；`attempt` 命名只剩 `UNFILTERED_VECTOR`
|
||||
|
||||
请求内 `retrievalTrace` / 落库 `tool_invocation` / Trace API 读法见：
|
||||
## 10. 与旧文档的差异
|
||||
|
||||
**[RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)**
|
||||
|
||||
要点：
|
||||
|
||||
- Agent **看不到**完整 retrievalTrace；人通过 `GET /api/diagnosis/{sessionId}/trace` 的 `toolInvocations[].retrievalDetails` 回放。
|
||||
- `relevance_level` 列存 RAG 等级（PRECISE/REFERENCE）；`evidence_status` 在 details JSON。
|
||||
- 默认审计不落原始 query 全文与 excerpt 正文。
|
||||
|
||||
## 10. 写入与重建
|
||||
|
||||
### 10.1 日常写入
|
||||
|
||||
文档上传 / 知识库初始化：
|
||||
|
||||
```text
|
||||
markdown
|
||||
-> frontmatter + body
|
||||
-> DocumentChunkService
|
||||
-> dense embedding + search_text
|
||||
-> MilvusHybridKnowledgeStore.upsertChunk
|
||||
-> MySQL api_document + L0 memory index
|
||||
```
|
||||
|
||||
### 10.2 全量重建
|
||||
|
||||
危险操作，需显式确认：
|
||||
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
```
|
||||
|
||||
等价 API：
|
||||
|
||||
```text
|
||||
POST /api/knowledge/rebuild-hybrid?confirm=REBUILD
|
||||
```
|
||||
|
||||
服务端顺序：
|
||||
|
||||
1. drop + recreate `milvus.collection`（默认 `biz`）
|
||||
2. 清空 MySQL `api_document`
|
||||
3. 清空内存 L0
|
||||
4. 扫描 `knowledge_base/**/*.md`（跳过 `README.md`）force 导入
|
||||
|
||||
不会修改磁盘上的 `knowledge_base/` 源文件。
|
||||
|
||||
## 11. 与旧文档的差异
|
||||
|
||||
| 旧描述（已归档） | 当前实现 |
|
||||
| 旧描述（2026-07-28 版） | 当前实现（2026-09-29 抽离后） |
|
||||
|---|---|
|
||||
| Spring AI VectorStore 主路径 + SDK fallback | 单一 MilvusClientV2 后端 |
|
||||
| `retrieval.vector-store.mode=auto/sdk/spring` | 已移除；改为 `retrieval.search.mode=dense/hybrid` |
|
||||
| source 级 evidence 去重 | chunk 级 `evidenceKey` 去重 |
|
||||
| 应用层 sparse-lite lexical 伪 hybrid | 库内 dense ANN + BM25 + RRFRanker |
|
||||
| 新建 `biz_hybrid` 过渡 collection | 默认使用并重建 `biz` |
|
||||
| 进程内 MilvusClientV2，dense+BM25+RRF | py-rag 服务端承担；Java 经 `KnowledgeSearchPort` → HTTP |
|
||||
| `retrieval.search.mode` 切 Milvus 查询算法 | 同名配置映射 py-rag `hybrid` / `semantic` |
|
||||
| scoreLabel 仅 dense \| hybrid，L2/rank 归一化 | 新增 `RERANK`：py-rag rerank 绝对分直传 |
|
||||
| L0 hint + categoryFilter + filtered/unfiltered retry | L0 下沉；原始 query 直传，单 attempt |
|
||||
| Java 侧 frontmatter 解析 / 分块 / embedding 入库 | py-rag `documents:ingest`；Java 只做 MySQL 元数据 + 原件保存 |
|
||||
| `POST /api/knowledge/rebuild-hybrid` 重建 | py-rag `POST /api/v1/collections:rebuild` |
|
||||
| `GET /milvus/health` 健康检查 | py-rag `GET /api/v1/health`（含 milvus/embedding/rerank 探针） |
|
||||
| 删除文档同步删向量索引 | 仅删 MySQL+本地文件；py-rag 侧靠全量重建生效 |
|
||||
|
||||
历史材料见：
|
||||
|
||||
- `mvp/architecture/archive/2026-07-22-legacy/rag-architecture.md`
|
||||
- `mvp/architecture/archive/2026-07-22-legacy/modular-rag-pipeline.md`
|
||||
- `mvp/architecture/archive/2026-07-22-legacy/rag-architecture.md`（Milvus 前身）
|
||||
- `mvp/engineering/rag/Milvus-Hybrid接入清单.md`（进程内 Milvus hybrid 接入纪要，已过时）
|
||||
- 本仓库 git 历史：`refactor/extract-rag-module` 分支，71 文件 / 约 -8000 行
|
||||
|
||||
## 12. 当前已知边界
|
||||
## 11. 当前已知边界
|
||||
|
||||
- hybrid 依赖云端/实例支持 BM25 Function 与 sparse index
|
||||
- 全量重建受 embedding API 与 Milvus 写入延迟影响，可能较慢
|
||||
- L0 关键词匹配仍较粗，只作 hint，不作主召回
|
||||
- 尚未做邻块上下文自动扩展、cross-encoder rerank、真 query rewrite
|
||||
- `totalVectors` 统计接口仍可能返回 0，不代表 collection 为空；以 rebuild/init 结果与检索命中为准
|
||||
- `retrieval.search.mode=dense` 仅作召回对照，不是第二套主路径
|
||||
- 同一 hybrid schema 数据可被 dense / hybrid 两种查询复用；从纯旧 dense-only collection 升级必须 rebuild
|
||||
- hybrid 质量闸门优先用 `denseDistance` 绝对 L2；无 dense 时 rank 回退；排序仍跟 RRF
|
||||
- 后处理不再用 L0 关键词 boost 改序；词面信号以库内 BM25+RRF 为准
|
||||
- py-rag 判级阈值（0.75/0.5/0.3）未校准，`quality_score` 仅供排序与展示参考（契约已知边界）
|
||||
- frontmatter 的 keywords/summary/covers/when_to_retrieve 当前仅上传方提供；Java 侧 LLM 补全（原 `DocumentFieldEnricher`）已随抽离移除，待 py-rag 开放
|
||||
- `knowledge_base/` 历史原料需在 py-rag 侧完成一次性 ingest 迁移后方可检索
|
||||
- 无单文档删除；下线文档靠 py-rag 全量重建
|
||||
- eval/rag-retrieval 离线基线基于旧 L0/scoreLabel 语义构建，抽离后需重新校准（见 `mvp/engineering/rag/RAG离线评测-基线设计.md` 顶部说明）
|
||||
- Trace/审计细节与限制见 [RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# MVP 架构文档
|
||||
|
||||
**更新日期**：2026-07-29
|
||||
**更新日期**：2026-09-29
|
||||
**状态**：当前单 Diagnosis Agent + Harness 架构
|
||||
|
||||
当前文档入口：
|
||||
@@ -12,12 +12,19 @@
|
||||
| [harness-quality-gates.md](harness-quality-gates.md) | Run、Tool、Evidence、Semantic、Release 与 Trace Recorder 门禁 |
|
||||
| [session-trace-lifecycle.md](session-trace-lifecycle.md) | sessionId/runId、SSE、统一 Timeline 和 reasoning audit 生命周期 |
|
||||
| [diagnosis-information-gain-stop-architecture.md](diagnosis-information-gain-stop-architecture.md) | 已实施的信息增益评价、Harness 饱和检测、Draft 合同失败降级与证据不足停止设计 |
|
||||
| [RAG知识检索架构.md](RAG知识检索架构.md) | 当前 `lookup_knowledge` 检索：MilvusClientV2 dense+BM25 hybrid、chunk 证据身份、重建运维 |
|
||||
| [RAG知识检索架构.md](RAG知识检索架构.md) | 当前 `lookup_knowledge` 检索：py-rag 知识服务接入、契约映射、chunk 证据身份、入库与重建运维 |
|
||||
| [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) | RAG Trace / 审计：请求内 retrievalTrace、tool_invocation 富字段、Trace API 读法 |
|
||||
|
||||
**工程纪要**（问题 / 决策 / E2E，非架构规范正文）见 [../engineering/README.md](../engineering/README.md)。
|
||||
|
||||
2026-07-22 前的多角色编排、双入口和旧证据链文档已移动到 `archive/2026-07-22-legacy/`，仅用于历史决策追溯，不代表当前运行时。其中旧 RAG 描述（Spring AI VectorStore 主路径 + Milvus SDK fallback）已被当前 hybrid 实现取代，请以 [RAG知识检索架构.md](RAG知识检索架构.md) 为准。检索可观测与 Trace 以 [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) 为准（勿再依赖 archive 内旧 retrieval-observability）。
|
||||
2026-07-22 前的多角色编排、双入口和旧证据链文档已移动到 `archive/2026-07-22-legacy/`，仅用于历史决策追溯，不代表当前运行时。
|
||||
|
||||
RAG 架构经历两次更替，均以 [RAG知识检索架构.md](RAG知识检索架构.md) 为准：
|
||||
|
||||
1. 2026-07-28：进程内 MilvusClientV2 hybrid（取代更早的 Spring AI VectorStore 主路径 + Milvus SDK fallback）；
|
||||
2. **2026-09-29（当前）**：RAG 模块抽离为独立 py-rag 知识服务，Java 经 `KnowledgeSearchPort` → `PyRagKnowledgeSearchAdapter` → HTTP `/api/v1` 调用；进程内 Milvus/embedding/L0/分块全部移除。
|
||||
|
||||
检索可观测与 Trace 以 [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) 为准（勿再依赖 archive 内旧 retrieval-observability）。
|
||||
|
||||
当前普通 Trace 与 LLM 步骤审计（`agent_reasoning_audit`：`reasoning_content` + `assistant_text`）使用独立存储和独立接口。
|
||||
DeepSeek thinking 捕获路径与 V015–V017 字段已 live 验证（2026-07-28）。Reasoning 访问控制、保留期限、加密要求仍由 ISS-015 跟踪，不能把“数据已分表 + 能抓到 thinking”理解为“治理已经完成”。
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# 当前 MVP 架构
|
||||
|
||||
**更新日期**：2026-07-28
|
||||
**更新日期**：2026-09-29
|
||||
**状态**：当前可运行架构
|
||||
|
||||
## 1. 系统定位
|
||||
|
||||
SuperBizAgent 是面向故障诊断的可追踪 Agent 应用。当前系统只保留一个拥有 Tool loop 的 `Diagnosis Agent`；Harness 负责确定性的预算、取消、工具边界、证据验真、语义审查和安全发布。
|
||||
|
||||
知识检索当前为显式 `lookup_knowledge` 工具 + 单一 MilvusClientV2 后端（dense / dense+BM25 hybrid）。详细链路见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||
知识检索当前为显式 `lookup_knowledge` 工具 + 独立 py-rag 知识服务（HTTP `/api/v1`）；检索算法（dense+BM25 融合、rerank、判级）与文档入库全部在 py-rag 侧，Java 只保留 harness 消费面与 HTTP 客户端。详细链路见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||
|
||||
## 2. 分层
|
||||
|
||||
@@ -71,22 +71,22 @@ Agent 只看到三个固定 Tool：
|
||||
Agent
|
||||
-> RagToolAdapter / ToolBoundary
|
||||
-> LookupKnowledgeTool
|
||||
-> L0 hint（可选 category filter）
|
||||
-> 原始 query 直传（L0 已下沉 py-rag）
|
||||
-> KnowledgeSearchPort
|
||||
-> VectorSearchService
|
||||
-> MilvusHybridKnowledgeStore # 唯一知识向量后端
|
||||
-> RagResultProjector # 有界 Agent 投影
|
||||
-> PyRagKnowledgeSearchAdapter # HTTP 客户端（PyRagClient）
|
||||
-> py-rag 知识服务 /api/v1/search # 融合 / rerank / 判级
|
||||
-> KnowledgeEvidencePostProcessor # chunk 去重 / return-n / 判级（Java 侧）
|
||||
-> RagResultProjector # 有界 Agent 投影
|
||||
```
|
||||
|
||||
要点：
|
||||
|
||||
- 默认 `retrieval.search.mode=hybrid`：dense ANN + BM25 sparse ANN + RRFRanker。
|
||||
- 也可切 `dense`：仅 dense ANN。
|
||||
- 已移除知识路径上的 legacy `MilvusServiceClient` search 与 `vector-store.mode=sdk|spring|auto` 路由。
|
||||
- 默认 `retrieval.search.mode=hybrid`：映射 py-rag `hybrid`（dense+BM25 融合）；`dense` 映射 `semantic` 作对照。
|
||||
- 证据按 chunk 级 `evidenceKey` 去重；Agent 侧 `document_id` 为 chunk 级身份。
|
||||
- 知识库全量重建：`python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD`，默认操作 collection `biz`。
|
||||
- py-rag `evidence_status=no_evidence` 按正常"无知识可用"处理，不是错误。
|
||||
- 知识库全量重建：py-rag `POST /api/v1/collections:rebuild?confirm=REBUILD`（异步任务）。
|
||||
|
||||
完整 schema、模式、重建与历史差异见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||
完整契约映射、入库、重建与历史差异见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||
|
||||
## 5. Trace 与持久化
|
||||
|
||||
@@ -117,10 +117,12 @@ Reasoning endpoint 是敏感审计面，不属于普通业务 API。数据和查
|
||||
|
||||
与知识检索相关的独立 API：
|
||||
|
||||
- `POST /api/knowledge/init`：导入/增量初始化 `knowledge_base`
|
||||
- `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD`：清空并重建 dense+BM25 collection（默认 `biz`）
|
||||
- `GET /api/knowledge/stats`：文档元数据统计
|
||||
- `GET /milvus/health`：MilvusClientV2 健康检查与 knowledge collection 名
|
||||
- `POST /api/documents/upload`：文档上传（MySQL 业务元数据 + 本地原件 + py-rag ingest）
|
||||
- `POST /api/upload`：简单上传（本地保存 + py-rag ingest，可选 `category`）
|
||||
- `GET /api/documents/{docId}` / `GET /api/documents/status/{status}` / `GET /api/documents/faultSource/{faultSource}`：文档元数据查询
|
||||
- `DELETE /api/documents/{docId}`：删除 MySQL 元数据与本地原件（py-rag 侧索引需全量重建生效）
|
||||
|
||||
知识库检索/入库/重建的服务端健康与统计由 py-rag 提供：`GET /api/v1/health`、`GET /api/v1/stats`（`{pyrag.base-url}`）。
|
||||
|
||||
## 7. 安全边界
|
||||
|
||||
|
||||
@@ -20,14 +20,19 @@
|
||||
|
||||
## RAG
|
||||
|
||||
> 2026-09-29 RAG 模块已抽离为独立 py-rag 知识服务（架构见
|
||||
> [../architecture/RAG知识检索架构.md](../architecture/RAG知识检索架构.md)）。
|
||||
> 下列纪要保留决策过程价值，涉及进程内 Milvus / L0 的实现细节以各文顶部说明为准。
|
||||
|
||||
| 文档 | 内容 |
|
||||
|---|---|
|
||||
| [rag/RAG排序-多路召回与RRF.md](rag/RAG排序-多路召回与RRF.md) | K、多路融合、RRF、L0 边界 |
|
||||
| [rag/RAG-Hybrid质量分与后处理.md](rag/RAG-Hybrid质量分与后处理.md) | qualityScore 统一、L2 伪装废止、后处理 |
|
||||
| [rag/RAG-Agent如何读relevance_level.md](rag/RAG-Agent如何读relevance_level.md) | Agent 侧相关度标签含义与误读 |
|
||||
| [rag/RAG离线评测-基线设计.md](rag/RAG离线评测-基线设计.md) | Golden/Fixture、hybrid 评测与闸门 |
|
||||
| [rag/Milvus-Hybrid接入清单.md](rag/Milvus-Hybrid接入清单.md) | Hybrid 交付拆分与接入清单 |
|
||||
| [rag/RAG审计补丁-stepid-query-E2E验收.md](rag/RAG审计补丁-stepid-query-E2E验收.md) | step_id / query 审计 live 验收 |
|
||||
| [rag/RAG证据链探索笔记-从py-rag响应到引用验真.md](rag/RAG证据链探索笔记-从py-rag响应到引用验真.md) | **抽离后链路首发导读**：数据逐站形态、关键字段、设计哲学与化石清单 |
|
||||
| [rag/RAG排序-多路召回与RRF.md](rag/RAG排序-多路召回与RRF.md) | K、多路融合、RRF、L0 边界（实现已下沉 py-rag，判断框架仍有效） |
|
||||
| [rag/RAG-Hybrid质量分与后处理.md](rag/RAG-Hybrid质量分与后处理.md) | qualityScore 统一、后处理（分数语义现为 RERANK 直传） |
|
||||
| [rag/RAG-Agent如何读relevance_level.md](rag/RAG-Agent如何读relevance_level.md) | Agent 侧相关度标签含义与误读（现行） |
|
||||
| [rag/RAG离线评测-基线设计.md](rag/RAG离线评测-基线设计.md) | Golden/Fixture、hybrid 评测与闸门（需按新语义重新校准） |
|
||||
| [rag/Milvus-Hybrid接入清单.md](rag/Milvus-Hybrid接入清单.md) | Hybrid 交付拆分与接入清单（已过时，仅历史追溯） |
|
||||
| [rag/RAG审计补丁-stepid-query-E2E验收.md](rag/RAG审计补丁-stepid-query-E2E验收.md) | step_id / query 审计 live 验收（现行） |
|
||||
|
||||
架构对照：
|
||||
|
||||
@@ -36,7 +41,7 @@
|
||||
|
||||
相关 Issue：
|
||||
|
||||
- [ISS-017 L0 过滤收窄与 Fallback 加固](../issues/active/ISS-017-rag-l0-filter-fallback-hardening.md)（暂缓，保持现网）
|
||||
- [ISS-017 L0 过滤收窄与 Fallback 加固](../issues/active/ISS-017-rag-l0-filter-fallback-hardening.md)（已失效：L0 下沉 py-rag，问题前提不复存在）
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
**状态**：SUCCESS 完整诊断主文档；工具阶段按**现行**审计能力（`step_id` / `query`）说明
|
||||
**文档路径**：`mvp/engineering/diagnosis/一次诊断全流程-E2E导读.md`
|
||||
|
||||
> **现状说明（2026-09-29）**：本文基于 2026-07 的 live Run 编写，图中 `VectorSearchService` /
|
||||
> `MilvusHybridKnowledgeStore`（进程内 Milvus）已替换为 py-rag 服务调用；审计/Trace 结构不变。
|
||||
> 当前检索链路见 [../architecture/RAG知识检索架构.md](../architecture/RAG知识检索架构.md)。
|
||||
|
||||
### 主样本（正文数值与 timeline 来源）
|
||||
|
||||
| 项 | 值 |
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Harness RAG 检索体系学习笔记：从 query 到可验证证据
|
||||
|
||||
> **现状说明（2026-09-29）**：RAG 模块抽离后，本文「检索前 L0 导航」与
|
||||
> `MilvusHybridKnowledgeStore` / `KnowledgeQueryTransformer` 相关章节描述的类已删除
|
||||
> （L0 与向量检索下沉 py-rag 服务端）；检索后处理/打包/投影/审计部分仍然现行。
|
||||
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
|
||||
**更新日期**：2026-08-03
|
||||
**主题**：lookup_knowledge 完整后端链路——检索前/检索/检索后/打包/组装/降级/契约/验证
|
||||
**设计文档**：`mvp/engineering/rag/`（RAG 排序、Hybrid 质量分、relevance_level 等）
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Harness 整体架构学习笔记：从装配到入口到记忆到知识库写入
|
||||
|
||||
> **现状说明（2026-09-29）**：RAG 模块抽离后，本文「知识库写入链路」章节描述的
|
||||
> `KnowledgeIndexService` / Milvus 写入路径已删除（入库下沉 py-rag `documents:ingest`）；
|
||||
> 其余装配/入口/记忆章节仍现行。当前架构见
|
||||
> [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
|
||||
**更新日期**：2026-08-04
|
||||
**主题**：整体架构五块补充（了解层面）——配置装配中心 / HTTP 入口层 / 会话系统与记忆体系 / 知识库写入链路
|
||||
**配套**：九域主线笔记（core/retry/progress/tool/guard/release/agent/audit/contract 全 ✅）
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# Milvus Hybrid Search 接入对照清单
|
||||
|
||||
> **已过时（2026-09-29）**：RAG 模块已抽离为独立 py-rag 知识服务，进程内 Milvus
|
||||
> （`MilvusHybridKnowledgeStore` / `VectorSearchService`）与本文描述的接入路径已整体删除。
|
||||
> 当前检索架构与契约映射见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
> 本文仅作历史决策追溯。
|
||||
|
||||
**日期**：2026-07-27
|
||||
**前提**：旧 Milvus SDK 直连检索路径后续废弃，不作为长期实现基础
|
||||
**目标**：在现有 `lookup_knowledge` pipeline 上接入 dense + sparse/BM25 混合检索，融合优先走服务端 RRF
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
# 混合检索上线之后：为什么还要统一 qualityScore，以及上一代后处理错在哪
|
||||
|
||||
> **现状说明（2026-09-29）**：本文讨论的后处理排序/去重/判级仍在 Java 侧
|
||||
> （`KnowledgeEvidencePostProcessor` / `RetrievalScoreNormalizer`）；
|
||||
> 但分数语义已变：py-rag 服务端返回 rerank 绝对分（scoreLabel=`RERANK`，[0,1] 越大越好），
|
||||
> quality 直传，不再走本文所述 dense L2 / hybrid rank 归一化分支（分支保留作兼容）。
|
||||
> 判级阈值 0.75/0.5 与 py-rag 契约一致。当前架构见
|
||||
> [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
|
||||
**日期**：2026-07-28
|
||||
**范围**：hybrid 检索后的分数语义、后处理排序、相关度闸门、scoreLabel 约定
|
||||
**读者**：已经（或准备）上 dense+BM25+RRF，却发现「召回变了、质量判断还拧着」的工程同学
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# 诊断 Agent 场景下的 RAG 排序：从 K=3 规则加分，到多路召回与 RRF
|
||||
|
||||
> **现状说明（2026-09-29）**：本文的排序判断框架（多路召回、RRF、Rerank 选型）仍是理解
|
||||
> py-rag 服务端检索设计的背景材料；但 RRF 融合、BM25、rerank 的**实现**已下沉 py-rag 服务端，
|
||||
> Java 侧不再有 `RrfFusion` / `MilvusHybridKnowledgeStore`。
|
||||
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
|
||||
**日期**：2026-07-27
|
||||
**范围**：知识检索排序、多路召回、分数融合、Rerank 选型
|
||||
**读者**：需要在 Agent 系统里落地 RAG，而不是只做 Demo 问答的工程同学
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# RAG 离线评测：讨论、设计与落地
|
||||
|
||||
> **需重新校准（2026-09-29）**：RAG 模块抽离后，L0 hint / categoryFilter 已下沉 py-rag、
|
||||
> 质量分改为 RERANK 直传、attempt 只剩 `UNFILTERED_VECTOR`；本文设计的 fixture/baseline
|
||||
> 基于旧 L0/scoreLabel 语义构建，回归评测需按新语义重新生成 fixture 并校准闸门。
|
||||
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||
|
||||
**日期**：2026-07-28
|
||||
**范围**：`eval/rag-retrieval` 离线 baseline、fixture 生成、与 hybrid/quality 主路径对齐
|
||||
**读者**：要维护或扩展知识库回归评测的工程同学
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
# RAG 证据链探索笔记：从 py-rag 响应到引用验真
|
||||
|
||||
**日期**：2026-09-29
|
||||
**范围**：RAG 模块抽离后的完整数据链路——每个站点"数据长什么样、字段怎么来、为什么这样设计"
|
||||
**读者**：要理解或维护 `lookup_knowledge` 证据链的工程同学
|
||||
**关联文档**：
|
||||
|
||||
- 架构规范：[../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)（本次抽离后的权威口径）
|
||||
- Trace / 审计：[../../architecture/RAG检索可观测性与审计.md](../../architecture/RAG检索可观测性与审计.md)
|
||||
- 契约原文：py-rag 仓库 `docs/Java接入文档.md`（API v1 冻结面）
|
||||
- 前置知识：`RAG-Hybrid质量分与后处理.md`（分数语义演进史）
|
||||
|
||||
> 本文按一次真实代码探索的顺序组织：从 py-rag 返回的 JSON 出发，沿着数据走过的每一站，
|
||||
> 讲清关键字段、加工规则与设计取舍，最后到 Agent 引用验真收口。
|
||||
|
||||
---
|
||||
|
||||
## 0. 全景路线图
|
||||
|
||||
```text
|
||||
py-rag JSON ──► KnowledgeSearchHit 站点1-2：数据形态与防腐层映射
|
||||
│
|
||||
① 后处理 ──► EvidencePostprocessResult 站点3：去重/限流/判级
|
||||
│
|
||||
② 打包 ──► ContextPack 站点4：文本储备（非 Agent 口粮）
|
||||
│
|
||||
③ 组装 ──► LookupResult 站点5：内部真相全集
|
||||
│
|
||||
④ 投影 ──► RagToolResult ──► Agent 站点6-7：冻结契约与 Agent 视图
|
||||
│
|
||||
⑤ Agent 写报告引用 toolCallId
|
||||
│
|
||||
⑥ EvidenceGuard 对账验真 ──► 语义审查 ──► 发布 站点8：证据安全链
|
||||
（旁路：每站关键字段 ──► tool_invocation 审计表，供人排障）
|
||||
```
|
||||
|
||||
一句话定位：**py-rag 负责"把对的片段找出来"，Java 侧负责"把找到的证据管起来"**；
|
||||
检索质量可以整体外换，证据治理一寸不动——这次抽离（71 文件 / 约 -8000 行）本身就是证明。
|
||||
|
||||
---
|
||||
|
||||
## 1. 站点一：py-rag 返回的数据结构
|
||||
|
||||
响应 JSON 按职责分四块，四块数据两条去路（①②③喂后处理，④喂审计）：
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "网关超时怎么排查", // ① 入参回显
|
||||
"mode": "hybrid", // ① 查询模式（hybrid | semantic）
|
||||
"hits": [ // ② 命中数组（检索的基本单位是 chunk）
|
||||
{
|
||||
"evidence_key": "e2e-gateway-b9c1fa12-md-34223174#chunk-1", // chunk 级身份
|
||||
"document_id": "e2e-gateway-b9c1fa12-md-34223174", // 所属文档
|
||||
"source": "e2e-gateway-b9c1fa12-md", // 来源路径
|
||||
"title": "网关超时排查",
|
||||
"breadcrumb": "网关超时排查 > 处理步骤",
|
||||
"excerpt": "网关超时先检查 upstream 配置…", // 正文
|
||||
"quality_score": 0.9147, // rerank 绝对分
|
||||
"relevance_level": "PRECISE" // py-rag 自己的判级（Java 不用）
|
||||
}
|
||||
],
|
||||
"relevance_level": "PRECISE", // ③ 顶层判级（top1）
|
||||
"evidence_status": "supported", // ③ 业务状态
|
||||
"retrieval_trace": { // ④ 过程记录（不参与后处理，进审计）
|
||||
"mode": "hybrid", "filters": {"category": "gateway"},
|
||||
"recall_count": 20, "rerank_model": "BAAI/bge-reranker-v2-m3",
|
||||
"no_evidence_basis": null
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
关键语义：
|
||||
|
||||
- **`evidence_status` 只有两类**：`supported`（正常）/ `no_evidence`（查到了但都是垃圾或无候选，
|
||||
此时 `hits` 恒为空）。它是 **200 正常业务响应**，Java 侧直接走"无知识可用"分支，不重试不报错。
|
||||
- **`relevance_level` 是分数的档位化**：`quality_score ≥0.75 → PRECISE`，`≥0.5 → REFERENCE`，
|
||||
`<0.5` 不输出。阈值当前未校准（契约已知边界）。**它是给模型的，分数是给系统的**——
|
||||
同一信息两种表达，服务两种消费者。
|
||||
- **命中没有独立 metadata map**（旧 Milvus 方案遗留概念），文档归属信息就是那几个平铺字段。
|
||||
|
||||
---
|
||||
|
||||
## 2. 站点二：防腐层映射——`KnowledgeSearchHit`
|
||||
|
||||
`PyRagKnowledgeSearchAdapter` 把每个 hit 映射成可移植结构。从此全 Java 侧只认这个类型，
|
||||
py-rag 字段再怎么变只改 adapter 一处。后处理实际只消费其中 **7 个字段**：
|
||||
|
||||
```text
|
||||
evidenceKey → 去重键（docId#chunk-N，原样采纳）
|
||||
docId / chunkIndex → 文档分桶（单文档上限）；chunkIndex 从 "#chunk-N" 解析
|
||||
content ← excerpt，最终证据正文
|
||||
score + scoreLabel ← quality_score + 常量 "rerank"（quality 直传）
|
||||
originalRank ← 数组下标 +1，排序的权威
|
||||
source / title / breadcrumb → 透传展示
|
||||
```
|
||||
|
||||
`retrieval_trace` 不进这条链——它走审计旁路。**进入后处理时，每个 hit 被浓缩成
|
||||
"身份 + 分数 + 名次 + 正文"四组信息，四步加工全部围绕它们转。**
|
||||
|
||||
---
|
||||
|
||||
## 3. 站点三：后处理四步——`EvidencePostprocessResult`
|
||||
|
||||
```text
|
||||
① 打分 score + scoreLabel → qualityScore（RERANK 分支直传，clamp [0,1]）
|
||||
② 排序 只按 originalRank —— ★ 分数不参与排序，只用于判级
|
||||
③ 去重截断 evidenceKey 去重 → 单文档 chunk ≤2 → 总数 ≤5（return-n）
|
||||
④ 判级 顶分 ≥0.75 PRECISE / ≥0.5 REFERENCE / 否则无档
|
||||
```
|
||||
|
||||
输出结构逐字段（7 条命中进、5 块存出的例子）：
|
||||
|
||||
| 字段 | 规则 | 例值 |
|
||||
|---|---|---|
|
||||
| `candidateCount` | 输入候选数 | 7 |
|
||||
| `evidenceBlockCount` | 存活块数（差值 = 被治理掉的） | 5 |
|
||||
| `topSimilarity` | 排序后第一名的 qualityScore | 0.91 |
|
||||
| `relevanceLevel` | Java 按本地阈值独立判定（**不用 py-rag 返回的档位**，那个字段映射时已丢弃） | PRECISE |
|
||||
| `completenessHint` | 档位绑定的天花板提示文案 | "知识库中不存在比上述结果更精准的文档" |
|
||||
| `rerankTrace[]` | 每个**存活**块的最终序账本 | finalRank 1~5 |
|
||||
|
||||
### 3.1 去重辨析：三条规则别混淆
|
||||
|
||||
| 层 | 键 | 规则 | 目的 |
|
||||
|---|---|---|---|
|
||||
| 去重 | `evidenceKey = docId#chunk-N` | 同一块再现 → **合并**（hitReasons 取并集） | 同一片段只出现一次 |
|
||||
| 单文档上限 | `docId`（分桶） | 同文档**不同** chunk 可共存，最多 2 | 防单文档刷屏 |
|
||||
| 总条数 | — | 最多 5 | 上下文预算 |
|
||||
|
||||
**合并键是 evidenceKey 而不是 docId**——同文档的第 0 段和第 1 段是两份不同证据，按 docId
|
||||
合并会把多片段证据文档级折叠掉。单次检索正常不会返回同一个 chunk 两次（每个 chunk 是唯一
|
||||
索引条目），合并分支是给"索引脏数据 / 未来多路归并"准备的**防御性兜底**，成本一次 map 查询。
|
||||
|
||||
### 3.2 为什么会同文档多块命中——根源在分块
|
||||
|
||||
入库时文档切成多个 chunk，每块独立 embedding、独立索引；检索按 chunk 算相似度。
|
||||
这是刻意的颗粒度选择：整篇文档一个向量会稀释语义，且上下文也塞不下全文。
|
||||
**切块是为了检索得准、取得少；同文档多块命中是切块的自然结果；
|
||||
"chunk 级身份 + 单文档上限"就是为管理这个现象而生的。**
|
||||
|
||||
### 3.3 `rerankTrace`：最终序账本
|
||||
|
||||
```java
|
||||
Item { finalRank; source; baseScore; finalScore; boostReasons }
|
||||
```
|
||||
|
||||
- 条数 = 存活块数（finalRank 连续编号）；被合并/截断的块不产生条目；
|
||||
- `baseScore == finalScore` **恒相等**——双字段是规则加分时代的化石（当年关键词加分，
|
||||
现已废除防操纵）；`boostReasons` 同理，装的已是纯解释标签；
|
||||
- 只记到**文档级**（无 evidenceKey），chunk 身份要看 `evidenceBlocks[]`；
|
||||
- Agent 看不到它，审计默认不落库——活在内存 LookupResult 与评测快照里。
|
||||
|
||||
### 3.4 附带闸门
|
||||
|
||||
`isLowQuality()`：无可用块或顶分 <0.5 即低质。原触发 filtered→unfiltered 重试，
|
||||
L0 下沉后分支休眠、闸门保留——将来 Java 侧重引过滤策略可直接接上。
|
||||
|
||||
---
|
||||
|
||||
## 4. 站点四：打包——`ContextPack`（不是 Agent 口粮）
|
||||
|
||||
`KnowledgeContextPacker` 把证据块压成**一段有字符预算的文本**（默认 4000 字符）：
|
||||
|
||||
```text
|
||||
策略 ranked_evidence_char_budget：名次即优先级，先到先得
|
||||
[Evidence 1]
|
||||
source: gateway-timeout.md
|
||||
breadcrumb: 网关超时排查 > 处理步骤
|
||||
reasons: semantic_rank:1, attempt:UNFILTERED_VECTOR ← 召回溯源标签
|
||||
content:
|
||||
网关超时先检查 upstream 配置…
|
||||
预算见底：装得下 header → 正文截断加 "..."；连 header 都放不下 → 整条进 omittedSources
|
||||
```
|
||||
|
||||
```java
|
||||
ContextPack { packedText; strategy; charBudget; usedChars; includedSources; omittedSources }
|
||||
```
|
||||
|
||||
**必须澄清的定位**：主诊断链路里 **Agent 不消费 packedText**（主代码零调用）——Agent 拿的是
|
||||
站点六投影后的结构化列表。它的价值：人工回放可读、评测快照存证、以及将来任何
|
||||
"证据进 prompt"路径的现成格式化出口（字符预算是与后处理"块数预算"互补的物理闸门）。
|
||||
|
||||
`reasons:` 行是证据的"简历"：`semantic_rank:N`（本次检索第几名）+ `attempt:X`（哪次尝试产出，
|
||||
当前只剩 `UNFILTERED_VECTOR`；历史值 `FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 已随
|
||||
L0 下沉绝迹）。
|
||||
|
||||
> 小化石：`ContextPack` 的 javadoc 仍写 "Agent-facing"，与 packer 侧注释矛盾——
|
||||
> 它诞生时确实面向 Agent，投影路线成为主路径后退居内部，注释没跟上身份变化。
|
||||
|
||||
---
|
||||
|
||||
## 5. 站点五：组装——`LookupResult` 真相全集
|
||||
|
||||
`LookupResultAssembler` 把三样东西合体成内部契约出口：
|
||||
|
||||
```text
|
||||
EvidencePostprocessResult（证据集+档位+trace）┐
|
||||
ContextPack（打包文本） ├─► LookupResult
|
||||
RetrievalTrace（检索路径） ┘
|
||||
```
|
||||
|
||||
只有两个字段是"算"出来的：`found = hasUsableEvidence()`（false 时附固定兜底文案
|
||||
"知识库未检索到可用证据，请结合日志、指标、告警继续排查"）；两个 count 把"进多少/出多少"
|
||||
带给审计。其余字段一一搬运。
|
||||
|
||||
---
|
||||
|
||||
## 6. 站点六：投影——`RagResultProjector`（加工链最后一站）
|
||||
|
||||
**输入是 LookupResult 的 JSON 字符串而非对象**——内部结构随便演化，冻结契约纹丝不动，
|
||||
解耦的关键就是这层"字符串边界"（字段名还带防御性别名对：`evidenceBlocks`/`evidence_blocks`）。
|
||||
|
||||
```text
|
||||
① query 截断（≤500 字） 动了 → truncated
|
||||
② 逐块过三道闸：
|
||||
条数 ≤8（超出 truncated+停止）
|
||||
身份去重（evidenceKey → document_id → docId#chunk-N → 序号兜底 的回退链）
|
||||
摘录 ≤1200 字（截断 → truncated）
|
||||
③ evidence_status 客观判定：数组空不空（EVIDENCE_FOUND / NO_EVIDENCE）
|
||||
④ relevance_level：有证据才读，解析不出 → null
|
||||
⑤ fitBudget 总字节兜底：整个 JSON 超 16KB → 从尾部逐条裁
|
||||
裁到空 → 诚实降级 NO_EVIDENCE；还超 → 抛异常（fail closed）
|
||||
```
|
||||
|
||||
**三道递进预算闸**（条数管语义 / 片段管局部 / 字节管整体）集中在 `ToolProjectionLimits`
|
||||
一个 record（8 条 / 1200 字 / 16KB，与日志、MySQL 工具共用）。**`truncated` 只要任何一处
|
||||
动过就置位——Agent 永远知道"看到的可能不全"**，不会把截断结果当全量。
|
||||
|
||||
---
|
||||
|
||||
## 7. 站点七：Agent 视图——模型实际看到的 JSON
|
||||
|
||||
```json
|
||||
{
|
||||
"evidence_status": "EVIDENCE_FOUND",
|
||||
"tool_call_id": "call_x1",
|
||||
"query": "网关超时怎么排查",
|
||||
"evidence": [
|
||||
{ "document_id": "e2e-gateway-…-34223174#chunk-1",
|
||||
"source": "e2e-gateway-b9c1fa12-md",
|
||||
"title": "网关超时排查",
|
||||
"breadcrumb": "网关超时排查 > 处理步骤",
|
||||
"excerpt": "网关超时先检查 upstream 配置…" }
|
||||
],
|
||||
"returned_count": 5,
|
||||
"relevance_level": "PRECISE",
|
||||
"truncated": false
|
||||
}
|
||||
```
|
||||
|
||||
| 字段 | 模型的正确用法 |
|
||||
|---|---|
|
||||
| `evidence_status` | `NO_EVIDENCE` → 老实换工具（日志/指标/MySQL），不许编 |
|
||||
| `evidence[].excerpt` | 结论唯一的内容依据 |
|
||||
| `evidence[].document_id` | **引用坐标**——报告里逐字引用它，EvidenceGuard 只认这个 |
|
||||
| `relevance_level` | PRECISE 可放心下结论；REFERENCE 结合上下文判断，必要时说清还缺什么维度 |
|
||||
| `truncated` | true → 看到的可能不全，可收窄 query 重搜 |
|
||||
|
||||
**三个"看不到"**：分数（防未校准数字诱导过度自信）、过程（attempt/trace 留给人）、
|
||||
其他站的内部字段。一句话：**留坐标、留内容、留诚实，剥掉一切会误导或撑爆上下文的东西。**
|
||||
|
||||
---
|
||||
|
||||
## 8. 站点八：验真——`EvidenceGuard`（证据安全链第一道闸）
|
||||
|
||||
Agent 写完诊断产出结构化草稿 `DiagnosisDraft`（analysis 带引用的 toolCallIds，
|
||||
conclusion/action_plan 带 basedOnAnalysisIds，limitations 必填）。EvidenceGuard 纯规则验真：
|
||||
|
||||
```text
|
||||
A 结构校验 id 唯一、正文非空、报告引用必须指向已登记分析、limitations 必填
|
||||
B 引用验真 runId+toolCallId → Redis 账本可查 → READY 状态 → 同 Run（防跨 Run 挪用）
|
||||
→ kind 语义匹配：NORMAL↔EVIDENCE_FOUND / NEGATIVE_OBSERVATION↔NO_EVIDENCE
|
||||
C 重读重建 从账本严格反序列化投影（多一个字段都违规）→ 用账本内容重建证据快照
|
||||
```
|
||||
|
||||
四个设计点：
|
||||
|
||||
1. **证据内容从账本重读，不信模型复述**——模型转述的"检索结果"进不了快照；
|
||||
2. **kind 匹配堵两头撒谎**——"查到了装没查到"与"没查到装查到了"都过不去；
|
||||
3. **runId 绑定 + READY**——别的 Run 的证据、失败/过期的调用不可引用；
|
||||
4. **纯规则、20 个违规码全枚举**——便宜、确定、可审计；"结论是否夸大"留给下一道
|
||||
SemanticGuard，这里只回答"引用是否真实、结构是否合法"。
|
||||
|
||||
---
|
||||
|
||||
## 9. 附：模型怎么知道要输出那份 JSON
|
||||
|
||||
四层合力，没有一刻依赖模型"自觉"：
|
||||
|
||||
| 层 | 机制 |
|
||||
|---|---|
|
||||
| 格式 | `DiagnosisDraftOutputSchema`（BeanOutputConverter）从 Java 类自动生成 JSON Schema 注入 prompt；解析失败抛异常 |
|
||||
| 语义 | `diagnosis-agent-prompt.md`：证据充分 → 全填并建立完整引用链；证据不足 → `conclusion=null` 是合法成功（schema 层把 conclusion 类型改成 `object\|null`）；limitations 无条件必填 |
|
||||
| 时机 | 模型自主停止（无有效查询范围即停）+ Harness 强制（连续 NO_GAIN / 预算耗尽 → STOP_REQUIRED 后必须直接交卷） |
|
||||
| 兜底 | 格式错/引用违规 → evidence-repair-prompt 修复重试 → 仍败 → 固定降级 FALLBACK |
|
||||
|
||||
---
|
||||
|
||||
## 10. 设计思路总结（五条哲学）
|
||||
|
||||
1. **防腐层：换引擎不换证据链**——`KnowledgeSearchPort` 是接缝，抽离时消费面零改动、
|
||||
250 个测试原样通过，这是接口设计价值的最硬证明。
|
||||
2. **分数给系统，档位给模型**——未校准的连续分数会诱导过度自信；`quality_score` 在
|
||||
Java 侧做闸门和审计，Agent 只见 PRECISE/REFERENCE。
|
||||
3. **chunk 级身份贯穿始终**——入库分块、检索按块、去重按 `docId#chunk-N`、引用坐标到块、
|
||||
验真对块。颗粒度统一，才有多片段证据共存与伪引用无处遁形。
|
||||
4. **诚实标记**——`truncated`、`no_evidence`、`unchanged`、`no_evidence_basis`：
|
||||
系统从不假装"看到的即全部"，每个不完整/为空都有显式信号与原因。
|
||||
5. **所见即所证**——投影结果是"模型看到的"与"Redis 存证的"同一份；EvidenceGuard 对账
|
||||
没有翻译损耗，伪引用无处遁形。
|
||||
|
||||
### 化石清单（读代码时的辨认指南）
|
||||
|
||||
| 化石 | 现状 |
|
||||
|---|---|
|
||||
| `RerankTrace.baseScore/finalScore` 双字段 | 恒相等（规则加分已废），保留兼容旧审计格式 |
|
||||
| `boostReasons` 字段名 | 装的是纯解释标签，不再加分 |
|
||||
| `ContextPack` javadoc "Agent-facing" | 身份已变（内部/储备），注释未跟上 |
|
||||
| `LookupResultAssembler.deduped()` | 会话级去重回包的历史占位，主链路不再调用 |
|
||||
| `FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` attempt | L0 下沉后不可达，仅存于旧 Run 回放 |
|
||||
| `KnowledgeQuery` 的 L0 hint 字段 | 恒空结构，供后处理与 trace 兼容保留 |
|
||||
|
||||
---
|
||||
|
||||
## 11. 关键字段速查
|
||||
|
||||
| 字段 | 哪一站 | 一句话 |
|
||||
|---|---|---|
|
||||
| `evidence_key` | py-rag → 全程 | chunk 级身份 `docId#chunk-N`，去重与验真的锚 |
|
||||
| `quality_score` | py-rag → 后处理 | rerank 绝对分 [0,1]，quality 直传 |
|
||||
| `evidence_status` | py-rag / 投影 | 两类：supported / no_evidence（正常业务响应） |
|
||||
| `relevance_level` | 后处理 → Agent | PRECISE/REFERENCE 档位，Java 按本地阈值独立判定 |
|
||||
| `originalRank` | adapter → 后处理 | 排序唯一权威，分数不动序 |
|
||||
| `candidateCount / evidenceBlockCount` | 后处理 | 进多少 / 出多少，差值即治理幅度 |
|
||||
| `truncated` | 投影 | 任何截断都置位，诚实标记 |
|
||||
| `tool_call_ids` | Draft → EvidenceGuard | 引用验真的入口，runId 绑定 + READY 校验 |
|
||||
@@ -1,11 +1,18 @@
|
||||
# ISS-017 RAG L0 过滤收窄与 Fallback 加固
|
||||
|
||||
**状态**：开放，暂缓实施（保持现网行为）
|
||||
**状态**：已失效（2026-09-29 RAG 抽离，L0 整体下沉 py-rag，问题前提不复存在）
|
||||
**严重程度**：中
|
||||
**发现时间**：2026-07-28
|
||||
**更新日期**：2026-07-28
|
||||
**更新日期**：2026-09-29
|
||||
**来源**：hybrid + qualityScore 收口后的评测/设计讨论；`chat-l0-filter-fallback` golden case
|
||||
**关联**：`LookupKnowledgeTool`、`KnowledgeQueryTransformer`、`KnowledgeEvidencePostProcessor`、`eval/rag-retrieval`、历史 `rag/rag-l0-domain-entity-hint.md`
|
||||
**关联**：`LookupKnowledgeTool`、`KnowledgeQueryTransformer`（已删除）、`KnowledgeEvidencePostProcessor`、`eval/rag-retrieval`、历史 `rag/rag-l0-domain-entity-hint.md`
|
||||
|
||||
> **处理记录（2026-09-29）**：RAG 模块抽离为 py-rag 知识服务时，L0 query 理解
|
||||
> （`KnowledgeQueryTransformer` / `KnowledgeIndexService.analyzeQuery`）整体下沉服务端，
|
||||
> `categoryFilter` 恒为 null，`FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 分支不再可达。
|
||||
> 本 issue 讨论的「硬过滤赌一把 + 失败整页替换」路径已不存在，无需加固，关闭。
|
||||
> 若未来在 Java 侧重新引入检索过滤策略，filtered→unfiltered 的降级骨架仍在
|
||||
> `LookupKnowledgeTool` 中保留，届时可参考本文的改进方向与回归动机。
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
# 知识域表：knowledge_domain
|
||||
|
||||
**状态**：当前表
|
||||
**状态**：孤儿表（2026-09-29 RAG 抽离后无写入方）
|
||||
**来源**：`V009__add_knowledge_domain.sql`、`KnowledgeDomain`
|
||||
|
||||
> **2026-09-29 变更**：本表的写入方 `KnowledgeDomainService` 已随 RAG 模块抽离删除
|
||||
> （L0 domain 分析下沉 py-rag）。表结构由 Flyway 保留（`ddl-auto: validate`），当前无读写方，
|
||||
> 待后续迁移清理。历史数据仅供追溯。
|
||||
|
||||
## 定位
|
||||
|
||||
`knowledge_domain` 保存知识库领域级元数据，为 RAG backend 的 domain hint、检索选择和可观测性提供基础信息；它不是 Agent-facing Tool contract。
|
||||
|
||||
@@ -76,12 +76,6 @@
|
||||
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Embedding: SiliconFlow BGE-M3 (需要 OpenAI 模块的 OpenAiEmbeddingModel) -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.alibaba.cloud.ai</groupId>
|
||||
<artifactId>spring-ai-alibaba-agent-framework</artifactId>
|
||||
@@ -91,19 +85,14 @@
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<!--
|
||||
spring-boot-devtools removed on purpose.
|
||||
Classpath restart (restartedMain) recreates beans without reliably closing
|
||||
MilvusClientV2 gRPC channels, causing orphan channels and long hybrid RPC retries.
|
||||
Prefer full process restart: stop then `mvn spring-boot:run`.
|
||||
spring-boot-devtools removed on purpose (bean recreation on classpath restart
|
||||
is unreliable). Prefer full process restart: stop then `mvn spring-boot:run`.
|
||||
-->
|
||||
<!-- okhttp：DashScopeConfig 的 RestClient.Builder 使用（此前由 milvus-sdk 传递引入） -->
|
||||
<dependency>
|
||||
<groupId>io.milvus</groupId>
|
||||
<artifactId>milvus-sdk-java</artifactId>
|
||||
<version>2.6.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-starter-vector-store-milvus</artifactId>
|
||||
<groupId>com.squareup.okhttp3</groupId>
|
||||
<artifactId>okhttp</artifactId>
|
||||
<version>4.12.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# 重建 hybrid 知识库（dense + BM25）
|
||||
|
||||
面向当前 `knowledge_base/` 目录文档，**清空并重建**配置中的 Milvus collection（默认 **`biz`**）。
|
||||
|
||||
## 前提
|
||||
|
||||
1. 应用已启动（默认 `http://localhost:9900`）
|
||||
2. `MILVUS_TOKEN` 等连接配置可用
|
||||
3. `application.yml` 已配置：
|
||||
|
||||
```yaml
|
||||
milvus:
|
||||
collection: biz
|
||||
retrieval:
|
||||
search:
|
||||
mode: hybrid
|
||||
knowledge:
|
||||
base-path: knowledge_base/
|
||||
```
|
||||
|
||||
## 一键脚本（Python）
|
||||
|
||||
在项目根目录执行：
|
||||
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
```
|
||||
|
||||
指定服务地址：
|
||||
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
||||
```
|
||||
|
||||
跳过前后 stats：
|
||||
|
||||
```bash
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD --skip-stats
|
||||
```
|
||||
|
||||
依赖：Python 3.9+ 标准库即可（无需 pip 包）。
|
||||
|
||||
## 脚本会做什么
|
||||
|
||||
| 步骤 | 动作 |
|
||||
|---|---|
|
||||
| 1 | 检查 `/milvus/health` |
|
||||
| 2 | 打印重建前 `/api/knowledge/stats` |
|
||||
| 3 | `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD` |
|
||||
| 4 | 打印重建后 stats |
|
||||
|
||||
服务端 `rebuild-hybrid` 内部顺序：
|
||||
|
||||
1. **Drop + recreate** Milvus collection（`milvus.collection`，默认 `biz`）
|
||||
- 原有向量数据会被删除
|
||||
- 按 dense + BM25 schema 重建
|
||||
2. **清空** MySQL `api_document`
|
||||
3. **清空** 内存 L0 索引
|
||||
4. **扫描** `knowledge_base/**/*.md`（跳过 `README.md`）并 force 全量导入
|
||||
- 写 MySQL 元数据
|
||||
- 切片
|
||||
- 写 dense 向量 + BM25 `search_text`
|
||||
- 更新 L0
|
||||
|
||||
## 不会做什么
|
||||
|
||||
- **不会**动 `knowledge_base/` 源文件
|
||||
- **不会**在未传 `--confirm REBUILD` 时执行
|
||||
|
||||
## 手动 curl 等价命令
|
||||
|
||||
```bash
|
||||
# 重建（危险：会清空 biz collection + api_document）
|
||||
curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD"
|
||||
|
||||
# 仅强制导入（不 drop collection）
|
||||
curl -X POST "http://localhost:9900/api/knowledge/init?force=true"
|
||||
|
||||
# 统计
|
||||
curl "http://localhost:9900/api/knowledge/stats"
|
||||
```
|
||||
|
||||
## 成功判据
|
||||
|
||||
响应中大致应有：
|
||||
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"collection": "biz",
|
||||
"inserted": 15,
|
||||
"failed": 0,
|
||||
"milvus": { "recreated": true, "loaded": true }
|
||||
}
|
||||
```
|
||||
|
||||
然后用一条知识库里真实存在的术语/故障词走 `lookup_knowledge` 或 chat 验证 hybrid 命中。
|
||||
|
||||
## 失败排查
|
||||
|
||||
| 现象 | 可能原因 |
|
||||
|---|---|
|
||||
| connect / token 错误 | `MILVUS_TOKEN`、host、database |
|
||||
| BM25 / analyzer 相关报错 | 云端 Milvus/Zilliz 版本不支持 BM25 Function |
|
||||
| inserted=0 | `knowledge_base` 路径不对，或 md 缺 frontmatter/title |
|
||||
| failed>0 | 看响应 `details` 与应用日志 |
|
||||
@@ -1,292 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live acceptance runner for post-reindex RAG retrieval checks.
|
||||
|
||||
This script calls the running Spring Boot retrieval endpoint. It is intentionally
|
||||
separate from the offline fixture baseline because it depends on live service and
|
||||
Milvus/Zilliz state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://127.0.0.1:9900"
|
||||
DEFAULT_JSON_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.json")
|
||||
DEFAULT_MD_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.md")
|
||||
|
||||
|
||||
DEFAULT_CASES: list[dict[str, Any]] = [
|
||||
{
|
||||
"caseId": "breadcrumb-rag-chunk-context",
|
||||
"query": "If a long RAG section is split into multiple chunks, how do we keep retrieval context?",
|
||||
"topK": 5,
|
||||
"purpose": "Breadcrumb-sensitive RAG chunk context retrieval.",
|
||||
},
|
||||
{
|
||||
"caseId": "breadcrumb-diagnosis-flow",
|
||||
"query": "What is the standard troubleshooting flow for an application incident?",
|
||||
"topK": 5,
|
||||
"purpose": "Process-style retrieval where section path matters.",
|
||||
},
|
||||
{
|
||||
"caseId": "core-err-timeout",
|
||||
"query": "ERR_TIMEOUT",
|
||||
"topK": 3,
|
||||
"purpose": "Exact error-code retrieval should remain stable.",
|
||||
},
|
||||
{
|
||||
"caseId": "core-mysql-connection-pool",
|
||||
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
|
||||
"topK": 3,
|
||||
"purpose": "Core infrastructure troubleshooting retrieval.",
|
||||
},
|
||||
{
|
||||
"caseId": "aiops-payment-latency",
|
||||
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
|
||||
"topK": 3,
|
||||
"purpose": "AIOps alert-style retrieval.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LiveCase:
|
||||
case_id: str
|
||||
query: str
|
||||
top_k: int
|
||||
purpose: str
|
||||
category: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, raw: dict[str, Any]) -> "LiveCase":
|
||||
return cls(
|
||||
case_id=str(raw["caseId"]),
|
||||
query=str(raw["query"]),
|
||||
top_k=int(raw.get("topK") or 3),
|
||||
purpose=str(raw.get("purpose") or raw.get("notes") or ""),
|
||||
category=(
|
||||
str(raw.get("category"))
|
||||
if raw.get("category") not in (None, "")
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def load_cases(path: Path | None) -> list[LiveCase]:
|
||||
if path is None:
|
||||
return [LiveCase.from_json(item) for item in DEFAULT_CASES]
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
raw_cases = payload.get("cases", payload)
|
||||
return [LiveCase.from_json(item) for item in raw_cases]
|
||||
|
||||
|
||||
def write_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def write_text(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(content)
|
||||
|
||||
|
||||
def request_case(base_url: str, case: LiveCase, timeout_seconds: float) -> dict[str, Any]:
|
||||
endpoint = base_url.rstrip("/") + "/api/search/similar"
|
||||
params: dict[str, str] = {
|
||||
"query": case.query,
|
||||
"topK": str(case.top_k),
|
||||
}
|
||||
if case.category:
|
||||
params["category"] = case.category
|
||||
url = endpoint + "?" + urllib.parse.urlencode(params)
|
||||
|
||||
started_at = datetime.now(timezone.utc)
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=timeout_seconds) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
payload = json.loads(body)
|
||||
status = int(getattr(response, "status", 200))
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
return {
|
||||
"caseId": case.case_id,
|
||||
"query": case.query,
|
||||
"topK": case.top_k,
|
||||
"category": case.category,
|
||||
"purpose": case.purpose,
|
||||
"url": url,
|
||||
"ok": False,
|
||||
"error": str(exc),
|
||||
"resultCount": 0,
|
||||
"topCandidates": [],
|
||||
"rawResponse": None,
|
||||
"startedAt": started_at.isoformat(),
|
||||
}
|
||||
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
if not isinstance(data, list):
|
||||
data = []
|
||||
|
||||
ok = status == 200 and payload.get("code") == 200
|
||||
return {
|
||||
"caseId": case.case_id,
|
||||
"query": case.query,
|
||||
"topK": case.top_k,
|
||||
"category": case.category,
|
||||
"purpose": case.purpose,
|
||||
"url": url,
|
||||
"ok": ok,
|
||||
"httpStatus": status,
|
||||
"responseCode": payload.get("code"),
|
||||
"responseMessage": payload.get("message"),
|
||||
"resultCount": len(data),
|
||||
"topCandidates": [summarize_candidate(item, index + 1) for index, item in enumerate(data)],
|
||||
"rawResponse": payload,
|
||||
"startedAt": started_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def summarize_candidate(raw: dict[str, Any], rank: int) -> dict[str, Any]:
|
||||
metadata = parse_metadata(raw.get("metadata"))
|
||||
return {
|
||||
"rank": rank,
|
||||
"id": raw.get("id"),
|
||||
"title": metadata.get("title"),
|
||||
"breadcrumb": metadata.get("breadcrumb"),
|
||||
"category": metadata.get("category"),
|
||||
"source": metadata.get("_source") or metadata.get("source"),
|
||||
"score": raw.get("score"),
|
||||
"rawScore": raw.get("rawScore"),
|
||||
"scoreLabel": raw.get("scoreLabel"),
|
||||
"contentPreview": preview(raw.get("content")),
|
||||
}
|
||||
|
||||
|
||||
def parse_metadata(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
return parsed if isinstance(parsed, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def preview(value: Any, limit: int = 180) -> str:
|
||||
text = " ".join(str(value or "").split())
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 3] + "..."
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# RAG Live Post-Reindex Acceptance",
|
||||
"",
|
||||
f"Generated at: `{report['generatedAt']}`",
|
||||
f"Base URL: `{report['baseUrl']}`",
|
||||
"",
|
||||
"> Reindex prerequisite: this report only reflects breadcrumb-aware embedding if the knowledge base was reindexed after the embedding-text change.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
"| Metric | Value |",
|
||||
"|---|---:|",
|
||||
f"| Cases | {report['caseCount']} |",
|
||||
f"| Successful calls | {report['successfulCalls']} |",
|
||||
f"| Empty result cases | {report['emptyResultCases']} |",
|
||||
"",
|
||||
"## Cases",
|
||||
"",
|
||||
"| Case | Purpose | Results | Top Candidates |",
|
||||
"|---|---|---:|---|",
|
||||
]
|
||||
for item in report["results"]:
|
||||
top = "<br>".join(format_candidate(candidate) for candidate in item["topCandidates"])
|
||||
if not top and item.get("error"):
|
||||
top = "ERROR: " + str(item["error"])
|
||||
lines.append(
|
||||
"| {case} | {purpose} | {count} | {top} |".format(
|
||||
case=item["caseId"],
|
||||
purpose=item.get("purpose") or "",
|
||||
count=item["resultCount"],
|
||||
top=top,
|
||||
)
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_candidate(candidate: dict[str, Any]) -> str:
|
||||
label = candidate.get("title") or candidate.get("source") or candidate.get("id") or ""
|
||||
breadcrumb = candidate.get("breadcrumb") or ""
|
||||
score_label = candidate.get("scoreLabel") or ""
|
||||
score = candidate.get("score")
|
||||
raw_score = candidate.get("rawScore")
|
||||
details = f"score={score}"
|
||||
if raw_score is not None:
|
||||
details += f", raw={raw_score}"
|
||||
if score_label:
|
||||
details += f", label={score_label}"
|
||||
if breadcrumb:
|
||||
return f"{candidate['rank']}. {label} ({breadcrumb}; {details})"
|
||||
return f"{candidate['rank']}. {label} ({details})"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
||||
parser.add_argument("--cases", type=Path, default=None)
|
||||
parser.add_argument("--json-report", type=Path, default=DEFAULT_JSON_REPORT)
|
||||
parser.add_argument("--markdown-report", type=Path, default=DEFAULT_MD_REPORT)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=10.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
cases = load_cases(args.cases)
|
||||
results = [
|
||||
request_case(args.base_url, case, args.timeout_seconds)
|
||||
for case in cases
|
||||
]
|
||||
successful = [item for item in results if item["ok"]]
|
||||
empty = [item for item in results if item["ok"] and item["resultCount"] == 0]
|
||||
report = {
|
||||
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"baseUrl": args.base_url,
|
||||
"caseCount": len(results),
|
||||
"successfulCalls": len(successful),
|
||||
"emptyResultCases": len(empty),
|
||||
"reindexPrerequisite": "Run or trigger knowledge-base reindex before treating this as breadcrumb-aware embedding evidence.",
|
||||
"results": results,
|
||||
}
|
||||
write_json(args.json_report, report)
|
||||
write_text(args.markdown_report, render_markdown(report))
|
||||
print(
|
||||
"Ran {total} live cases: successful={successful}, empty={empty}".format(
|
||||
total=len(results),
|
||||
successful=len(successful),
|
||||
empty=len(empty),
|
||||
)
|
||||
)
|
||||
return 1 if len(successful) != len(results) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,176 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild knowledge into the configured Milvus collection (default: biz).
|
||||
|
||||
Clears:
|
||||
- milvus.collection (drop + recreate dense+BM25 schema)
|
||||
- MySQL api_document
|
||||
- in-memory L0 index
|
||||
|
||||
Then force-imports all markdown under server-side knowledge.base-path
|
||||
(default: knowledge_base/).
|
||||
|
||||
Usage:
|
||||
# start Spring Boot first, then:
|
||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
||||
|
||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_BASE_URL = "http://localhost:9900"
|
||||
|
||||
|
||||
def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]:
|
||||
req = urllib.request.Request(url=url, method=method.upper())
|
||||
req.add_header("Accept", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="replace")
|
||||
status = getattr(resp, "status", 200)
|
||||
if not raw.strip():
|
||||
return status, None
|
||||
return status, json.loads(raw)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read().decode("utf-8", errors="replace")
|
||||
body: Any
|
||||
try:
|
||||
body = json.loads(raw) if raw.strip() else None
|
||||
except json.JSONDecodeError:
|
||||
body = raw
|
||||
raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc
|
||||
|
||||
|
||||
def pretty(obj: Any) -> str:
|
||||
return json.dumps(obj, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def step(title: str) -> None:
|
||||
print()
|
||||
print(f"==> {title}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, "
|
||||
"and reimport knowledge_base markdown into dense+BM25."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
default=DEFAULT_BASE_URL,
|
||||
help=f"Service base URL (default: {DEFAULT_BASE_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confirm",
|
||||
required=True,
|
||||
choices=["REBUILD"],
|
||||
help="Must be REBUILD to execute destructive rebuild",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-stats",
|
||||
action="store_true",
|
||||
help="Skip before/after /api/knowledge/stats",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=7200.0,
|
||||
help="Rebuild request timeout seconds (default: 7200)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
base_url = args.base_url.rstrip("/")
|
||||
|
||||
print("Hybrid knowledge rebuild")
|
||||
print(f" BaseUrl : {base_url}")
|
||||
print(f" Confirm : {args.confirm}")
|
||||
print(" Source : knowledge_base/ (server-side knowledge.base-path)")
|
||||
print()
|
||||
print("This will DESTROY data in:")
|
||||
print(" - Milvus collection milvus.collection (default: biz)")
|
||||
print(" - MySQL table api_document")
|
||||
print(" - In-memory L0 knowledge index")
|
||||
print("Then re-import all markdown under knowledge_base.")
|
||||
print()
|
||||
|
||||
# 1) health
|
||||
step("Check service health")
|
||||
try:
|
||||
status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30)
|
||||
print(f" milvus health status={status}")
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure
|
||||
print(f" WARN: /milvus/health failed: {exc}")
|
||||
print(" Continue if app is up but milvus health endpoint has issues.")
|
||||
|
||||
# 2) stats before
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (before)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats before failed: {exc}")
|
||||
|
||||
# 3) rebuild
|
||||
step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD")
|
||||
rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}"
|
||||
try:
|
||||
status, body = http_json("POST", rebuild_url, timeout=args.timeout)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc))
|
||||
return 1
|
||||
|
||||
print(f" HTTP {status}")
|
||||
print(pretty(body))
|
||||
|
||||
if not isinstance(body, dict):
|
||||
print("Unexpected rebuild response type", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
inserted = int(body.get("inserted") or 0)
|
||||
failed = int(body.get("failed") or 0)
|
||||
success = bool(body.get("success"))
|
||||
|
||||
if not success:
|
||||
if inserted <= 0:
|
||||
print()
|
||||
print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr)
|
||||
return 2
|
||||
print()
|
||||
print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.")
|
||||
else:
|
||||
print()
|
||||
print(f"Rebuild OK: inserted={inserted}, failed={failed}")
|
||||
|
||||
# 4) stats after
|
||||
if not args.skip_stats:
|
||||
step("Knowledge stats (after)")
|
||||
try:
|
||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
||||
print(pretty(body))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f" WARN: stats after failed: {exc}")
|
||||
|
||||
print()
|
||||
print("Done.")
|
||||
print("Next:")
|
||||
print(" 1) Ensure application.yml has:")
|
||||
print(" milvus.collection: biz")
|
||||
print(" retrieval.search.mode: hybrid")
|
||||
print(" 2) Smoke test lookup_knowledge / chat with a known doc query")
|
||||
return 0 if success or inserted > 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,196 +0,0 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.grpc.DataType;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.IndexType;
|
||||
import io.milvus.param.MetricType;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.RpcStatus;
|
||||
import io.milvus.param.collection.*;
|
||||
import io.milvus.param.index.CreateIndexParam;
|
||||
import com.superbiz.agent.config.MilvusProperties;
|
||||
import com.superbiz.agent.constant.MilvusConstants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Milvus 客户端工厂类
|
||||
* 负责创建和初始化 Milvus 客户端连接
|
||||
*/
|
||||
@Component
|
||||
public class MilvusClientFactory {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MilvusClientFactory.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusProperties milvusProperties;
|
||||
|
||||
/**
|
||||
* 创建并初始化 Milvus 客户端
|
||||
*
|
||||
* 简化版本：直接连接并创建 collection
|
||||
*
|
||||
* @return MilvusServiceClient 实例
|
||||
* @throws RuntimeException 如果连接或初始化失败
|
||||
*/
|
||||
public MilvusServiceClient createClient() {
|
||||
MilvusServiceClient client = null;
|
||||
|
||||
try {
|
||||
// 1. 连接到 Milvus
|
||||
logger.info("正在连接到 Milvus: {}:{}", milvusProperties.getHost(), milvusProperties.getPort());
|
||||
client = connectToMilvus();
|
||||
logger.info("成功连接到 Milvus");
|
||||
|
||||
// 2. 检查并创建 biz collection（如果不存在）
|
||||
if (!collectionExists(client, MilvusConstants.MILVUS_COLLECTION_NAME)) {
|
||||
logger.info("collection '{}' 不存在，正在创建...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
createBizCollection(client);
|
||||
logger.info("成功创建 collection '{}'", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
|
||||
// 创建索引
|
||||
createIndexes(client);
|
||||
logger.info("成功创建索引");
|
||||
} else {
|
||||
logger.info("collection '{}' 已存在", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
}
|
||||
|
||||
// 3. 加载 collection 到内存（搜索必须）
|
||||
logger.info("正在加载 collection '{}' 到内存...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
R<RpcStatus> loadResp = client.loadCollection(LoadCollectionParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.build());
|
||||
if (loadResp.getStatus() == 0) {
|
||||
logger.info("collection '{}' 已加载", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
} else {
|
||||
logger.warn("collection '{}' 加载失败: {}", MilvusConstants.MILVUS_COLLECTION_NAME, loadResp.getMessage());
|
||||
}
|
||||
|
||||
return client;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("创建 Milvus 客户端失败", e);
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
throw new RuntimeException("创建 Milvus 客户端失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接到 Milvus
|
||||
*/
|
||||
private MilvusServiceClient connectToMilvus() {
|
||||
ConnectParam.Builder builder = ConnectParam.newBuilder()
|
||||
.withHost(milvusProperties.getHost())
|
||||
.withPort(milvusProperties.getPort())
|
||||
.withDatabaseName(milvusProperties.getDatabase())
|
||||
.withConnectTimeout(milvusProperties.getTimeout(), TimeUnit.MILLISECONDS);
|
||||
|
||||
// Zilliz Cloud: token + SSL
|
||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isEmpty()) {
|
||||
builder.withToken(milvusProperties.getToken());
|
||||
builder.withSecure(true);
|
||||
}
|
||||
// 本地 Milvus: username + password
|
||||
else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isEmpty()) {
|
||||
builder.withAuthorization(milvusProperties.getUsername(), milvusProperties.getPassword());
|
||||
}
|
||||
|
||||
return new MilvusServiceClient(builder.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 collection 是否存在
|
||||
*/
|
||||
private boolean collectionExists(MilvusServiceClient client, String collectionName) {
|
||||
R<Boolean> response = client.hasCollection(HasCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build());
|
||||
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("检查 collection 失败: " + response.getMessage());
|
||||
}
|
||||
|
||||
return response.getData();
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 biz collection
|
||||
*/
|
||||
private void createBizCollection(MilvusServiceClient client) {
|
||||
// 定义字段
|
||||
FieldType idField = FieldType.newBuilder()
|
||||
.withName("id")
|
||||
.withDataType(DataType.VarChar)
|
||||
.withMaxLength(MilvusConstants.ID_MAX_LENGTH)
|
||||
.withPrimaryKey(true)
|
||||
.build();
|
||||
|
||||
FieldType vectorField = FieldType.newBuilder()
|
||||
.withName("vector")
|
||||
.withDataType(DataType.FloatVector) // 改为 FloatVector
|
||||
.withDimension(milvusProperties.getVectorDim())
|
||||
.build();
|
||||
|
||||
FieldType contentField = FieldType.newBuilder()
|
||||
.withName("content")
|
||||
.withDataType(DataType.VarChar)
|
||||
.withMaxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.build();
|
||||
|
||||
FieldType metadataField = FieldType.newBuilder()
|
||||
.withName("metadata")
|
||||
.withDataType(DataType.JSON)
|
||||
.build();
|
||||
|
||||
// 创建 collection schema
|
||||
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
|
||||
.withEnableDynamicField(false)
|
||||
.addFieldType(idField)
|
||||
.addFieldType(vectorField)
|
||||
.addFieldType(contentField)
|
||||
.addFieldType(metadataField)
|
||||
.build();
|
||||
|
||||
// 创建 collection
|
||||
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.withDescription("Business knowledge collection")
|
||||
.withSchema(schema)
|
||||
.withShardsNum(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
||||
.build();
|
||||
|
||||
R<RpcStatus> response = client.createCollection(createParam);
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("创建 collection 失败: " + response.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 collection 创建索引
|
||||
*/
|
||||
private void createIndexes(MilvusServiceClient client) {
|
||||
// 为 vector 字段创建索引（FloatVector 使用 IVF_FLAT 和 L2 距离）
|
||||
CreateIndexParam vectorIndexParam = CreateIndexParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.withFieldName("vector")
|
||||
.withIndexType(IndexType.IVF_FLAT)
|
||||
.withMetricType(MetricType.L2) // L2 距离（欧氏距离）
|
||||
.withExtraParam("{\"nlist\":128}")
|
||||
.withSyncMode(Boolean.FALSE)
|
||||
.build();
|
||||
|
||||
R<RpcStatus> response = client.createIndex(vectorIndexParam);
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("创建 vector 索引失败: " + response.getMessage());
|
||||
}
|
||||
|
||||
logger.info("成功为 vector 字段创建索引");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import com.superbiz.agent.config.PyRagProperties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* py-rag 知识服务 HTTP 客户端（API v1，契约见 py-rag 仓库 docs/Java接入文档.md）。
|
||||
*
|
||||
* <p>错误信封：4xx/5xx 一律 {@code {"error":{"code":"E_XXX","message":…,"details":[…]}}}，
|
||||
* 统一抛出 {@link PyRagClientException}；网络异常包装为 {@code E_NETWORK}。
|
||||
* {@code evidence_status=no_evidence} 是 200 正常业务响应，不作为错误。</p>
|
||||
*
|
||||
* <p>每个请求携带 {@code X-Request-ID}（UUID）用于跨服务日志关联；
|
||||
* 超时按接入文档矩阵分端点配置（见 {@link PyRagProperties}）。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PyRagClient {
|
||||
|
||||
private static final String REQUEST_ID_HEADER = "X-Request-ID";
|
||||
|
||||
private final PyRagProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RestClient searchClient;
|
||||
private final RestClient ingestClient;
|
||||
private final RestClient defaultClient;
|
||||
|
||||
public PyRagClient(PyRagProperties properties, ObjectMapper objectMapper) {
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
this.searchClient = buildRestClient(properties.getSearchReadTimeoutMs());
|
||||
this.ingestClient = buildRestClient(properties.getIngestReadTimeoutMs());
|
||||
this.defaultClient = buildRestClient(properties.getDefaultReadTimeoutMs());
|
||||
}
|
||||
|
||||
private RestClient buildRestClient(int readTimeoutMs) {
|
||||
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
|
||||
factory.setConnectTimeout(properties.getConnectTimeoutMs());
|
||||
factory.setReadTimeout(readTimeoutMs);
|
||||
return RestClient.builder()
|
||||
.baseUrl(properties.getBaseUrl())
|
||||
.requestFactory(factory)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ── 检索 ────────────────────────────────────────────────
|
||||
|
||||
public PyRagSearchResponse search(PyRagSearchRequest request) {
|
||||
return exchange(() -> searchClient.post()
|
||||
.uri("/api/v1/search")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(request), PyRagSearchResponse.class);
|
||||
}
|
||||
|
||||
// ── 文档入库 ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* multipart 文档入库。category 为契约必填；title/breadcrumb/kbScope 可选（null 不传）。
|
||||
* 同内容重复上传返回 unchanged（幂等），网络超时可安全重试。
|
||||
*/
|
||||
public PyRagIngestResponse ingest(String filename,
|
||||
byte[] content,
|
||||
String contentType,
|
||||
String category,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
String kbScope) {
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("file", new ByteArrayResource(content) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
});
|
||||
body.add("category", category);
|
||||
if (title != null && !title.isBlank()) {
|
||||
body.add("title", title);
|
||||
}
|
||||
if (breadcrumb != null && !breadcrumb.isBlank()) {
|
||||
body.add("breadcrumb", breadcrumb);
|
||||
}
|
||||
if (kbScope != null && !kbScope.isBlank()) {
|
||||
body.add("kb_scope", kbScope);
|
||||
}
|
||||
return exchange(() -> ingestClient.post()
|
||||
.uri("/api/v1/documents:ingest")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.MULTIPART_FORM_DATA)
|
||||
.body(body), PyRagIngestResponse.class);
|
||||
}
|
||||
|
||||
// ── 全量重建（异步任务） ────────────────────────────────
|
||||
|
||||
/** 202 返回任务号；已有 rebuild 执行中抛 E_REBUILD_IN_PROGRESS。 */
|
||||
public PyRagTaskAccepted rebuild() {
|
||||
return exchange(() -> defaultClient.post()
|
||||
.uri("/api/v1/collections:rebuild?confirm=REBUILD")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(Map.of()), PyRagTaskAccepted.class);
|
||||
}
|
||||
|
||||
public PyRagTaskStatus task(String taskId) {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/tasks/{id}", taskId)
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagTaskStatus.class);
|
||||
}
|
||||
|
||||
// ── 统计与健康 ──────────────────────────────────────────
|
||||
|
||||
public PyRagStats stats() {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/stats")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagStats.class);
|
||||
}
|
||||
|
||||
public PyRagHealth health() {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/health")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagHealth.class);
|
||||
}
|
||||
|
||||
// ── 内部 ────────────────────────────────────────────────
|
||||
|
||||
private <T> T exchange(Supplier<RestClient.RequestHeadersSpec<?>> spec, Class<T> type) {
|
||||
try {
|
||||
return spec.get().exchange((request, response) -> {
|
||||
if (response.getStatusCode().isError()) {
|
||||
throw toClientException(response.getStatusCode().value(), response.getBody());
|
||||
}
|
||||
return objectMapper.readValue(response.getBody(), type);
|
||||
});
|
||||
} catch (PyRagClientException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.error("py-rag 调用失败: {}", e.getMessage(), e);
|
||||
throw new PyRagClientException("E_NETWORK",
|
||||
"py-rag 调用失败: " + e.getMessage(), null, e);
|
||||
}
|
||||
}
|
||||
|
||||
private PyRagClientException toClientException(int httpStatus, java.io.InputStream body) {
|
||||
String code = "E_HTTP_" + httpStatus;
|
||||
String message = "HTTP " + httpStatus;
|
||||
try {
|
||||
PyRagErrorEnvelope envelope = objectMapper.readValue(body, PyRagErrorEnvelope.class);
|
||||
if (envelope != null && envelope.error() != null) {
|
||||
code = envelope.error().code() == null ? code : envelope.error().code();
|
||||
message = envelope.error().message() == null ? message : envelope.error().message();
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// 错误响应体不是契约信封（如网关 502 页面），保留 HTTP 默认语义
|
||||
}
|
||||
return new PyRagClientException(code, message, httpStatus, null);
|
||||
}
|
||||
|
||||
// ── 契约 DTO（snake_case 对齐 py-rag API） ──────────────
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
public record PyRagSearchRequest(
|
||||
String query,
|
||||
/** hybrid=dense+BM25 融合；semantic=纯向量 */
|
||||
String mode,
|
||||
Integer retrieveK,
|
||||
Integer returnN,
|
||||
Integer maxChunksPerDocument,
|
||||
String category,
|
||||
String kbScope
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagSearchHit(
|
||||
String evidenceKey,
|
||||
String documentId,
|
||||
String source,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
String excerpt,
|
||||
Double qualityScore,
|
||||
String relevanceLevel
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagRetrievalTrace(
|
||||
String mode,
|
||||
Map<String, Object> filters,
|
||||
Integer recallCount,
|
||||
String rerankModel,
|
||||
String noEvidenceBasis
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagSearchResponse(
|
||||
String query,
|
||||
String mode,
|
||||
List<PyRagSearchHit> hits,
|
||||
String relevanceLevel,
|
||||
/** supported | no_evidence（no_evidence 时 hits=[]，属正常业务响应） */
|
||||
String evidenceStatus,
|
||||
PyRagRetrievalTrace retrievalTrace
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagIngestResponse(
|
||||
String docId,
|
||||
String source,
|
||||
/** created | updated | unchanged */
|
||||
String status,
|
||||
Integer chunkCount,
|
||||
List<String> warnings,
|
||||
Map<String, Object> frontmatter
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagTaskAccepted(
|
||||
String taskId,
|
||||
String status
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagTaskStatus(
|
||||
String taskId,
|
||||
String status,
|
||||
Integer documents,
|
||||
String detail,
|
||||
String createdAt,
|
||||
String finishedAt
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagStats(
|
||||
String collection,
|
||||
Integer rowCount
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagHealth(
|
||||
String status,
|
||||
String version,
|
||||
Map<String, String> checks
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagErrorEnvelope(ErrorBody error) {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record ErrorBody(String code, String message, List<Map<String, Object>> details) {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* py-rag 调用异常：携带契约错误码（E_*）与 HTTP 状态。
|
||||
*
|
||||
* <p>调用方按错误码分支：422 参数/数据问题不重试；
|
||||
* 409 E_REBUILD_IN_PROGRESS 延迟重试；网络异常（E_NETWORK）可安全重试。</p>
|
||||
*/
|
||||
@Getter
|
||||
public class PyRagClientException extends RuntimeException {
|
||||
|
||||
/** 契约错误码：E_INVALID_REQUEST / E_FRONTMATTER_INVALID / E_REBUILD_IN_PROGRESS / E_NETWORK 等 */
|
||||
private final String code;
|
||||
|
||||
/** HTTP 状态码；网络层异常（未拿到响应）为 null */
|
||||
private final Integer httpStatus;
|
||||
|
||||
public PyRagClientException(String code, String message, Integer httpStatus, Throwable cause) {
|
||||
super("[" + code + "] " + message, cause);
|
||||
this.code = code;
|
||||
this.httpStatus = httpStatus;
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* 文档分片配置
|
||||
*/
|
||||
@Getter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "document.chunk")
|
||||
public class DocumentChunkConfig {
|
||||
|
||||
/**
|
||||
* 每个分片的最大字符数（保留向后兼容）
|
||||
*/
|
||||
private int maxSize = 800;
|
||||
|
||||
/**
|
||||
* 分片之间的重叠字符数
|
||||
*/
|
||||
private int overlap = 100;
|
||||
|
||||
/**
|
||||
* 每个分片的最大 token 数（中文~1:1，英文~0.25:1）
|
||||
* 替代 maxSize 作为切割触发器
|
||||
*/
|
||||
private int maxTokens = 500;
|
||||
|
||||
/**
|
||||
* 硬上限 token 数 = maxTokens × 1.2
|
||||
* 仅在不可中断上下文（列表、代码块）内触发
|
||||
*/
|
||||
private int maxTokensHard = 600;
|
||||
|
||||
public void setMaxSize(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
public void setOverlap(int overlap) {
|
||||
this.overlap = overlap;
|
||||
}
|
||||
|
||||
public void setMaxTokens(int maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokensHard(int maxTokensHard) {
|
||||
this.maxTokensHard = maxTokensHard;
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* Milvus 知识路径配置说明（无额外 Bean 装配）。
|
||||
*
|
||||
* <p>知识库 RAG 唯一实现：{@link MilvusHybridKnowledgeStore}（{@code MilvusClientV2}）。</p>
|
||||
* <ul>
|
||||
* <li>支持 dense 与 dense+BM25 {@code hybridSearch}+RRF。</li>
|
||||
* <li>不再为知识路径创建 legacy {@code MilvusServiceClient} Bean。</li>
|
||||
* <li>Spring AI {@code VectorStore} starter 仍可存在于 classpath，但只作 sidecar，
|
||||
* 不作 lookup_knowledge 主路径（starter 无 BM25 hybrid API）。</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Configuration
|
||||
public class MilvusConfig {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MilvusConfig.class);
|
||||
|
||||
public MilvusConfig() {
|
||||
logger.info("Milvus knowledge path: MilvusClientV2 hybrid store only (legacy SDK search disabled)");
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "milvus")
|
||||
public class MilvusProperties {
|
||||
|
||||
private String host = "localhost";
|
||||
private Integer port = 19530;
|
||||
private String username = "";
|
||||
private String password = "";
|
||||
private String database = "default";
|
||||
private Long timeout = 10000L;
|
||||
private String token = "";
|
||||
private boolean secure = false;
|
||||
private int vectorDim = 1024;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public Integer getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public void setPort(Integer port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public Long getTimeout() {
|
||||
return timeout;
|
||||
}
|
||||
|
||||
public void setTimeout(Long timeout) {
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public String getToken() {
|
||||
return token;
|
||||
}
|
||||
|
||||
public void setToken(String token) {
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
public boolean isSecure() {
|
||||
return secure;
|
||||
}
|
||||
|
||||
public void setSecure(boolean secure) {
|
||||
this.secure = secure;
|
||||
}
|
||||
|
||||
public int getVectorDim() {
|
||||
return vectorDim;
|
||||
}
|
||||
|
||||
public void setVectorDim(int vectorDim) {
|
||||
this.vectorDim = vectorDim;
|
||||
}
|
||||
|
||||
public String getAddress() {
|
||||
return host + ":" + port;
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -19,11 +17,11 @@ import org.springframework.context.annotation.Primary;
|
||||
* <pre>{@code
|
||||
* model-routing:
|
||||
* chat: deepseek
|
||||
* embedding: siliconflow
|
||||
* }</pre>
|
||||
* <p>
|
||||
* 匹配优先级：Bean 名 > 类名（均不区分大小写）。
|
||||
* 切换模型只改 yml + pom + 对应 api-key，Java 代码不动。
|
||||
* （Embedding 路由已随 RAG 模块抽离至 py-rag 服务端，此处仅路由 Chat。）
|
||||
*/
|
||||
@Configuration
|
||||
public class ModelRoutingConfig {
|
||||
@@ -33,9 +31,6 @@ public class ModelRoutingConfig {
|
||||
@Value("${model-routing.chat:deepseek}")
|
||||
private String chatKeyword;
|
||||
|
||||
@Value("${model-routing.embedding:siliconflow}")
|
||||
private String embeddingKeyword;
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public ChatModel chatModel(List<ChatModel> chatModels) {
|
||||
@@ -53,33 +48,6 @@ public class ModelRoutingConfig {
|
||||
return chatModels.get(0);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public EmbeddingModel embeddingModel(Map<String, EmbeddingModel> embeddingBeans) {
|
||||
log.info("Embedding 路由: keyword='{}', 可用: {}", embeddingKeyword, embeddingBeans.keySet());
|
||||
|
||||
// 先按 Bean 名匹配
|
||||
for (Map.Entry<String, EmbeddingModel> entry : embeddingBeans.entrySet()) {
|
||||
if (containsIgnoreCase(entry.getKey(), embeddingKeyword)) {
|
||||
log.info(" → Bean 名匹配: {} → {}", entry.getKey(),
|
||||
entry.getValue().getClass().getSimpleName());
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
// 再按类名匹配
|
||||
for (EmbeddingModel em : embeddingBeans.values()) {
|
||||
if (matches(em.getClass(), embeddingKeyword)) {
|
||||
log.info(" → 类名匹配: {}", em.getClass().getSimpleName());
|
||||
return em;
|
||||
}
|
||||
}
|
||||
|
||||
var first = embeddingBeans.values().iterator().next();
|
||||
log.warn(" → 未匹配, 回退到 {}", first.getClass().getSimpleName());
|
||||
return first;
|
||||
}
|
||||
|
||||
private boolean matches(Class<?> clazz, String keyword) {
|
||||
return containsIgnoreCase(clazz.getName(), keyword)
|
||||
|| containsIgnoreCase(clazz.getSimpleName(), keyword);
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* py-rag 知识服务接入配置。
|
||||
*
|
||||
* <p>超时矩阵来自《py-rag 知识服务 · Java 接入文档》第 6 节：
|
||||
* 服务含 embedding/rerank 外呼，search 正常 300–800ms、ingest 正常 1–5s。</p>
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "pyrag")
|
||||
public class PyRagProperties {
|
||||
|
||||
/** py-rag 服务根地址，如 http://py-rag:8000 */
|
||||
private String baseUrl = "http://localhost:8000";
|
||||
|
||||
/** 连接超时（毫秒），全端点统一 */
|
||||
private int connectTimeoutMs = 3000;
|
||||
|
||||
/** /api/v1/search 读取超时（毫秒） */
|
||||
private int searchReadTimeoutMs = 5000;
|
||||
|
||||
/** /api/v1/documents:ingest 读取超时（毫秒） */
|
||||
private int ingestReadTimeoutMs = 30000;
|
||||
|
||||
/** rebuild/tasks/stats/health 读取超时（毫秒） */
|
||||
private int defaultReadTimeoutMs = 10000;
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Getter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "rag.sidecar.spring-ai")
|
||||
public class RagSidecarProperties {
|
||||
|
||||
private boolean enabled = false;
|
||||
|
||||
private int contentPreviewLimit = 300;
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public void setContentPreviewLimit(int contentPreviewLimit) {
|
||||
this.contentPreviewLimit = contentPreviewLimit;
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
/**
|
||||
* SiliconFlow Embedding 配置（BGE-M3, OpenAI 兼容协议, 1024维）
|
||||
* <p>
|
||||
* Chat 走 DeepSeek、Embedding 走 SiliconFlow，两者都是 OpenAI 兼容但地址不同，
|
||||
* 因此单独为 SiliconFlow 创建 OpenAiApi + EmbeddingModel Bean。
|
||||
*/
|
||||
@Configuration
|
||||
public class SiliconFlowEmbeddingConfig {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SiliconFlowEmbeddingConfig.class);
|
||||
|
||||
@Value("${siliconflow.api-key}")
|
||||
private String apiKey;
|
||||
|
||||
@Value("${siliconflow.base-url}")
|
||||
private String baseUrl;
|
||||
|
||||
@Value("${siliconflow.embedding.model}")
|
||||
private String model;
|
||||
|
||||
@Bean
|
||||
public OpenAiApi siliconFlowApi(RestClient.Builder restClientBuilder, WebClient.Builder webClientBuilder) {
|
||||
log.info("创建 SiliconFlow OpenAiApi: {}", baseUrl);
|
||||
return OpenAiApi.builder()
|
||||
.baseUrl(baseUrl)
|
||||
.apiKey(apiKey)
|
||||
.restClientBuilder(restClientBuilder)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EmbeddingModel siliconFlowEmbeddingModel(OpenAiApi siliconFlowApi) {
|
||||
log.info("创建 SiliconFlow EmbeddingModel, model: {}", model);
|
||||
return new OpenAiEmbeddingModel(siliconFlowApi, MetadataMode.EMBED,
|
||||
OpenAiEmbeddingOptions.builder()
|
||||
.model(model)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package com.superbiz.agent.constant;
|
||||
|
||||
public class MilvusConstants {
|
||||
|
||||
/**
|
||||
* Milvus 数据库名称
|
||||
*/
|
||||
public static final String MILVUS_DB_NAME = "default";
|
||||
|
||||
/**
|
||||
* Default knowledge collection name (dense + BM25).
|
||||
* Overridable via {@code milvus.collection}.
|
||||
*/
|
||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
||||
|
||||
/**
|
||||
* Alias kept for readability in hybrid-related code.
|
||||
*/
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
||||
|
||||
/**
|
||||
* 向量维度（豆包 embedding 模型的维度）
|
||||
*/
|
||||
public static final int VECTOR_DIM = 1024; // 豆包模型返回1024维向量
|
||||
|
||||
/**
|
||||
* ID字段最大长度
|
||||
*/
|
||||
public static final int ID_MAX_LENGTH = 256;
|
||||
|
||||
/**
|
||||
* Content字段最大长度
|
||||
*/
|
||||
public static final int CONTENT_MAX_LENGTH = 8192;
|
||||
|
||||
/**
|
||||
* 默认分片数
|
||||
*/
|
||||
public static final int DEFAULT_SHARD_NUMBER = 2;
|
||||
|
||||
private MilvusConstants() {
|
||||
// 工具类，禁止实例化
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.config.FileUploadConfig;
|
||||
import com.superbiz.agent.dto.FileUploadRes;
|
||||
import com.superbiz.agent.service.VectorIndexService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -29,10 +29,11 @@ public class FileUploadController {
|
||||
private FileUploadConfig fileUploadConfig;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
private PyRagClient pyRagClient;
|
||||
|
||||
@PostMapping(value = "/api/upload", consumes = "multipart/form-data")
|
||||
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file) {
|
||||
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file,
|
||||
@RequestParam(value = "category", required = false) String category) {
|
||||
if (file.isEmpty()) {
|
||||
return ResponseEntity.badRequest().body("文件不能为空");
|
||||
}
|
||||
@@ -68,15 +69,17 @@ public class FileUploadController {
|
||||
|
||||
logger.info("文件上传成功: {}", filePath);
|
||||
|
||||
// 文件上传成功后，自动调用向量索引服务
|
||||
// 转发 py-rag 入库（同内容重传返回 unchanged）。入库失败不影响上传成功语义。
|
||||
try {
|
||||
logger.info("开始为上传文件创建向量索引: {}", filePath);
|
||||
vectorIndexService.indexSingleFile(filePath.toString());
|
||||
logger.info("向量索引创建成功: {}", filePath);
|
||||
String ingestCategory = (category == null || category.isBlank()) ? "default" : category;
|
||||
logger.info("开始 py-rag 入库: {}, category={}", filePath, ingestCategory);
|
||||
var ingest = pyRagClient.ingest(originalFilename, file.getBytes(), file.getContentType(),
|
||||
ingestCategory, null, null, null);
|
||||
logger.info("py-rag 入库完成: docId={}, status={}, chunks={}",
|
||||
ingest.docId(), ingest.status(), ingest.chunkCount());
|
||||
} catch (Exception e) {
|
||||
logger.error("向量索引创建失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
||||
// 注意：即使索引失败，文件上传仍然成功，只是记录错误日志
|
||||
// 可以根据业务需求决定是否要删除文件或返回错误
|
||||
logger.error("py-rag 入库失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
||||
// 注意：即使入库失败，文件上传仍然成功，只是记录错误日志
|
||||
}
|
||||
|
||||
FileUploadRes response = new FileUploadRes(
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.service.KnowledgeBaseInitService;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 知识库管理控制器
|
||||
* 提供知识库初始化、查询等接口
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/knowledge")
|
||||
public class KnowledgeBaseController {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
|
||||
|
||||
@Autowired
|
||||
private KnowledgeBaseInitService initService;
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
* 扫描 knowledge_base 目录下的所有文档，去重后批量导入到数据库和 Milvus
|
||||
*
|
||||
* @param force 是否强制重新导入（跳过去重检查）
|
||||
* @return 初始化结果
|
||||
*/
|
||||
@PostMapping("/init")
|
||||
public ResponseEntity<?> initKnowledgeBase(@RequestParam(defaultValue = "false") boolean force) {
|
||||
logger.info("收到知识库初始化请求, force={}", force);
|
||||
|
||||
try {
|
||||
KnowledgeBaseInitService.InitResult result = initService.initializeKnowledgeBase(force);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", true);
|
||||
response.put("message", "知识库初始化完成");
|
||||
response.put("scanned", result.getScanned());
|
||||
response.put("skipped", result.getSkipped());
|
||||
response.put("inserted", result.getInserted());
|
||||
response.put("failed", result.getFailed());
|
||||
response.put("details", result.getDetails());
|
||||
|
||||
logger.info("知识库初始化成功: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("知识库初始化失败", e);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "初始化失败: " + e.getMessage());
|
||||
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引，
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作：会删除 {@code milvus.collection}（默认 {@code biz}）与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
public ResponseEntity<?> rebuildHybrid(
|
||||
@RequestParam(defaultValue = "") String confirm) {
|
||||
if (!"REBUILD".equals(confirm)) {
|
||||
Map<String, Object> rejected = new HashMap<>();
|
||||
rejected.put("success", false);
|
||||
rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建");
|
||||
rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
|
||||
return ResponseEntity.badRequest().body(rejected);
|
||||
}
|
||||
|
||||
logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
|
||||
try {
|
||||
KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", result.isSuccess());
|
||||
response.put("message", result.isSuccess()
|
||||
? "hybrid 知识库重建完成"
|
||||
: "hybrid 知识库重建结束，但存在失败项");
|
||||
response.put("collection", result.getCollection());
|
||||
response.put("basePath", result.getBasePath());
|
||||
response.put("milvus", result.getMilvus());
|
||||
response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
|
||||
response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
|
||||
response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
|
||||
response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
|
||||
response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
|
||||
if (result.getInit() != null) {
|
||||
response.put("scanned", result.getInit().getScanned());
|
||||
response.put("skipped", result.getInit().getSkipped());
|
||||
response.put("inserted", result.getInit().getInserted());
|
||||
response.put("failed", result.getInit().getFailed());
|
||||
response.put("details", result.getInit().getDetails());
|
||||
}
|
||||
return result.isSuccess()
|
||||
? ResponseEntity.ok(response)
|
||||
: ResponseEntity.status(500).body(response);
|
||||
} catch (Exception e) {
|
||||
logger.error("hybrid 知识库重建失败", e);
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "重建失败: " + e.getMessage());
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询知识库统计信息
|
||||
*
|
||||
* @return 统计信息
|
||||
*/
|
||||
@GetMapping("/stats")
|
||||
public ResponseEntity<?> getStats() {
|
||||
try {
|
||||
KnowledgeBaseInitService.Stats stats = initService.getStats();
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", true);
|
||||
response.put("totalDocuments", stats.getTotalDocuments());
|
||||
response.put("totalVectors", stats.getTotalVectors());
|
||||
response.put("categories", stats.getCategoryCount());
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("查询统计信息失败", e);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "查询失败: " + e.getMessage());
|
||||
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import io.milvus.v2.service.collection.response.ListCollectionsResp;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Milvus health check using the single V2 knowledge backend.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/milvus")
|
||||
public class MilvusCheckController {
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@GetMapping("/health")
|
||||
public ResponseEntity<Map<String, Object>> simpleHealth() {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
try {
|
||||
ListCollectionsResp response = knowledgeStore.client().listCollections();
|
||||
result.put("message", "ok");
|
||||
result.put("backend", "milvus-client-v2");
|
||||
result.put("knowledgeCollection", knowledgeStore.collectionName());
|
||||
result.put("collections", response == null ? null : response.getCollectionNames());
|
||||
return ResponseEntity.ok(result);
|
||||
} catch (Exception e) {
|
||||
result.put("error", e.getMessage());
|
||||
return ResponseEntity.status(503).body(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.dto.Result;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档检索控制器（测试用）
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/search")
|
||||
public class SearchController {
|
||||
|
||||
@Autowired
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
/**
|
||||
* 搜索相似文档
|
||||
*/
|
||||
@GetMapping("/similar")
|
||||
public Result<List<VectorSearchService.SearchResult>> searchSimilar(
|
||||
@RequestParam("query") String query,
|
||||
@RequestParam(value = "topK", defaultValue = "5") int topK,
|
||||
@RequestParam(value = "category", required = false) String category
|
||||
) {
|
||||
try {
|
||||
log.info("收到检索请求，query: {}, topK: {}, category: {}", query, topK, category);
|
||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(query, topK, category);
|
||||
log.info("检索完成，返回 {} 条结果", results.size());
|
||||
return Result.success(results);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("检索失败", e);
|
||||
return Result.error(500, "检索失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* 文档分片
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DocumentChunk {
|
||||
|
||||
/**
|
||||
* 分片内容
|
||||
*/
|
||||
private String content;
|
||||
|
||||
/**
|
||||
* 分片在原文档中的起始位置
|
||||
*/
|
||||
private int startOffset;
|
||||
|
||||
/**
|
||||
* 分片在原文档中的结束位置
|
||||
*/
|
||||
private int endOffset;
|
||||
|
||||
/**
|
||||
* 分片序号（从0开始）
|
||||
*/
|
||||
private int chunkIndex;
|
||||
|
||||
/**
|
||||
* 分片标题或上下文信息
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 面包屑导航（完整标题层级路径）
|
||||
* 例如: "故障诊断流程规范 > 应急响应流程 > 1. 初步评估"
|
||||
*/
|
||||
private String breadcrumb;
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Frontmatter 数据模型
|
||||
* 用于解析 Markdown 文件头的 YAML frontmatter
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Frontmatter {
|
||||
|
||||
/**
|
||||
* 文档标题（必填）
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 关键词列表（必填，用于 L0 精确匹配）
|
||||
*/
|
||||
private List<String> keywords;
|
||||
|
||||
/**
|
||||
* 文档摘要（必填）
|
||||
*/
|
||||
private String summary;
|
||||
|
||||
/**
|
||||
* 文档类别（可选）
|
||||
*/
|
||||
private String category;
|
||||
|
||||
private String source;
|
||||
|
||||
private String breadcrumb;
|
||||
|
||||
@JsonProperty("kb_scope")
|
||||
private String kbScope;
|
||||
|
||||
/**
|
||||
* 章节锚点（预留字段，MVP 不使用）
|
||||
* Key: 章节标题，Value: 章节 Markdown 标题
|
||||
*/
|
||||
private Map<String, String> sections;
|
||||
|
||||
/**
|
||||
* 版本号（预留字段）
|
||||
*/
|
||||
private String version;
|
||||
|
||||
/**
|
||||
* 作者（预留字段）
|
||||
*/
|
||||
private String author;
|
||||
|
||||
/**
|
||||
* 最后更新日期（预留字段）
|
||||
*/
|
||||
private LocalDate lastUpdated;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用（LLM 上传时自动生成）
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机（LLM 上传时自动生成）
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 知识库索引条目
|
||||
* L0 内存索引使用的数据结构
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class KnowledgeEntry {
|
||||
|
||||
/**
|
||||
* 文件路径（如：knowledge_base/api/payment-errors.md）
|
||||
*/
|
||||
private String filePath;
|
||||
|
||||
/**
|
||||
* 文档标题
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 关键词列表（用于精确匹配）
|
||||
*/
|
||||
private List<String> keywords;
|
||||
|
||||
/**
|
||||
* 文档摘要
|
||||
*/
|
||||
private String summary;
|
||||
|
||||
/**
|
||||
* 文档类别（如：api、domain、troubleshooting）
|
||||
*/
|
||||
private String category;
|
||||
|
||||
private String kbScope;
|
||||
|
||||
/**
|
||||
* 章节锚点（预留字段，MVP 不使用）
|
||||
*/
|
||||
private Map<String, String> sections;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
@@ -6,9 +6,10 @@ import lombok.Data;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 检索前 query understanding 的输出（L0 -> pipeline 控制面）。
|
||||
* 检索 pipeline 控制面参数。
|
||||
*
|
||||
* <p>由 {@code KnowledgeQueryTransformer} 生成，供 L1 过滤、规则 rerank 与 trace 使用。
|
||||
* <p>L0 query 理解已下沉 py-rag 服务端；当前 {@code originalQuery} = {@code rewrittenQuery}、
|
||||
* hint 字段恒为空、{@code categoryFilter} 恒为 null，结构保留供后处理与 trace 使用。
|
||||
* 不是 Agent 可见契约。</p>
|
||||
*/
|
||||
@Data
|
||||
|
||||
@@ -1,446 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.config.DocumentChunkConfig;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文档切片服务（RAG 入库前处理）。
|
||||
*
|
||||
* <p>把长 Markdown/文本切成带 title/breadcrumb 的 {@link com.superbiz.agent.dto.DocumentChunk}，
|
||||
* 供 {@link VectorIndexService} 向量化。</p>
|
||||
*
|
||||
* <h3>策略摘要</h3>
|
||||
* <ol>
|
||||
* <li>先按 Markdown 标题分 section，并维护 breadcrumb 层级</li>
|
||||
* <li>section 过长再按段落累积；用 token 估算做软边界 / 硬上限</li>
|
||||
* <li>尽量不在有序/无序列表或未闭合代码块中间切断</li>
|
||||
* <li>相邻 chunk 保留 overlap，减轻边界语义断裂</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>检索命中单个 chunk 后，当前主链路不会自动回补同章节相邻 chunk
|
||||
* （上下文重建仍是后续增强点）。</p>
|
||||
*/
|
||||
@Service
|
||||
public class DocumentChunkService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DocumentChunkService.class);
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkConfig chunkConfig;
|
||||
|
||||
/**
|
||||
* 智能分片文档
|
||||
* 优先按照标题、段落边界进行分割，保持语义完整性
|
||||
*
|
||||
* @param content 文档内容
|
||||
* @param filePath 文件路径（用于日志）
|
||||
* @return 文档分片列表
|
||||
*/
|
||||
public List<DocumentChunk> chunkDocument(String content, String filePath) {
|
||||
List<DocumentChunk> chunks = new ArrayList<>();
|
||||
|
||||
if (content == null || content.trim().isEmpty()) {
|
||||
logger.warn("文档内容为空: {}", filePath);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 1. 首先尝试按标题分割（Markdown格式）
|
||||
List<Section> sections = splitByHeadings(content);
|
||||
|
||||
// 2. 对每个章节进行进一步分片
|
||||
int globalChunkIndex = 0;
|
||||
for (Section section : sections) {
|
||||
List<DocumentChunk> sectionChunks = chunkSection(section, globalChunkIndex);
|
||||
chunks.addAll(sectionChunks);
|
||||
globalChunkIndex += sectionChunks.size();
|
||||
}
|
||||
|
||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按照 Markdown 标题分割文档，同时构建面包屑层级路径
|
||||
*/
|
||||
private List<Section> splitByHeadings(String content) {
|
||||
List<Section> sections = new ArrayList<>();
|
||||
|
||||
// 匹配 Markdown 标题：# 标题, ## 标题, ### 标题等
|
||||
Pattern headingPattern = Pattern.compile("^(#{1,6})\\s+(.+)$", Pattern.MULTILINE);
|
||||
Matcher matcher = headingPattern.matcher(content);
|
||||
|
||||
// 标题层级栈：维护当前标题的完整路径
|
||||
List<String> headingStack = new ArrayList<>();
|
||||
int lastEnd = 0;
|
||||
String currentBreadcrumb = null;
|
||||
|
||||
while (matcher.find()) {
|
||||
int level = matcher.group(1).length(); // #→1, ##→2, ###→3 ...
|
||||
String title = matcher.group(2).trim();
|
||||
|
||||
// 保存上一个章节
|
||||
if (lastEnd < matcher.start()) {
|
||||
String sectionContent = content.substring(lastEnd, matcher.start()).trim();
|
||||
if (!sectionContent.isEmpty()) {
|
||||
sections.add(new Section(
|
||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
||||
level,
|
||||
currentBreadcrumb,
|
||||
sectionContent,
|
||||
lastEnd));
|
||||
}
|
||||
}
|
||||
|
||||
// 维护层级栈：同级别或更高级别 → 弹出，低级 → 追加
|
||||
while (!headingStack.isEmpty() && headingStack.size() >= level) {
|
||||
headingStack.remove(headingStack.size() - 1);
|
||||
}
|
||||
headingStack.add(title);
|
||||
currentBreadcrumb = String.join(" > ", headingStack);
|
||||
lastEnd = matcher.start();
|
||||
}
|
||||
|
||||
// 添加最后一个章节
|
||||
if (lastEnd < content.length()) {
|
||||
String sectionContent = content.substring(lastEnd).trim();
|
||||
if (!sectionContent.isEmpty()) {
|
||||
sections.add(new Section(
|
||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
||||
headingStack.size(),
|
||||
currentBreadcrumb,
|
||||
sectionContent,
|
||||
lastEnd));
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有找到任何标题，将整个文档作为一个章节
|
||||
if (sections.isEmpty()) {
|
||||
sections.add(new Section(null, 0, null, content, 0));
|
||||
}
|
||||
|
||||
return sections;
|
||||
}
|
||||
|
||||
/**
|
||||
* 对单个章节进行分片
|
||||
* <p>
|
||||
* 核心改造（Phase 1）：
|
||||
* - Token 估算替代字符计数
|
||||
* - 感知有序/无序列表结构，不在列表中间切断
|
||||
* - 软边界（maxTokens）+ 硬上限（maxTokensHard）双重控制
|
||||
* - 修复 currentStartIndex 漂移：用段落原始位置而非手工推算
|
||||
*/
|
||||
private List<DocumentChunk> chunkSection(Section section, int startChunkIndex) {
|
||||
List<DocumentChunk> chunks = new ArrayList<>();
|
||||
String content = section.content;
|
||||
String title = section.title;
|
||||
String breadcrumb = section.breadcrumb;
|
||||
|
||||
// 短章节直接作为一个分片（用 token 估算替代字符数做短路判断）
|
||||
if (content.length() <= chunkConfig.getMaxSize()
|
||||
&& estimateTokens(content) <= chunkConfig.getMaxTokens()) {
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(content)
|
||||
.startOffset(section.startIndex)
|
||||
.endOffset(section.startIndex + content.length())
|
||||
.chunkIndex(startChunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 章节内容较长，需要进一步分片
|
||||
List<String> paragraphs = splitByParagraphs(content);
|
||||
if (paragraphs.isEmpty()) {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 定位每个段落在 section.content 中的位置（修复 index 漂移）
|
||||
List<ParagraphPos> paraPositions = locateParagraphPositions(paragraphs, content);
|
||||
|
||||
// 当前分片的段落范围
|
||||
int chunkParaStart = 0; // 当前分片第一个段落的索引（在 paragraphs 中）
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
int tokenCount = 0;
|
||||
int chunkIndex = startChunkIndex;
|
||||
|
||||
for (int i = 0; i < paragraphs.size(); i++) {
|
||||
String paragraph = paragraphs.get(i);
|
||||
int paraTokens = estimateTokens(paragraph);
|
||||
|
||||
// 判断是否需要切分
|
||||
if (buffer.length() > 0 && tokenCount + paraTokens > chunkConfig.getMaxTokens()) {
|
||||
|
||||
// 检查是否处于不可中断的上下文中
|
||||
if (isInUnbreakableContext(buffer.toString(), paragraph)) {
|
||||
// 硬上限保护：即使不可中断也不能无限膨胀
|
||||
if (tokenCount + paraTokens > chunkConfig.getMaxTokensHard()) {
|
||||
logger.debug(" 触及硬上限 ({} tokens)，强制切分", tokenCount + paraTokens);
|
||||
chunkParaStart = saveChunkAndGetNextStart(
|
||||
chunks, section, paraPositions,
|
||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
||||
chunkIndex++;
|
||||
|
||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
||||
String overlap = getOverlapText(prevChunkContent);
|
||||
buffer = new StringBuilder(overlap);
|
||||
tokenCount = estimateTokens(overlap);
|
||||
}
|
||||
// 否则：容忍超出（软边界）
|
||||
} else {
|
||||
// 安全切点：段落边界
|
||||
chunkParaStart = saveChunkAndGetNextStart(
|
||||
chunks, section, paraPositions,
|
||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
||||
chunkIndex++;
|
||||
|
||||
// 新分片以重叠文本开头
|
||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
||||
String overlap = getOverlapText(prevChunkContent);
|
||||
buffer = new StringBuilder(overlap);
|
||||
tokenCount = estimateTokens(overlap);
|
||||
}
|
||||
}
|
||||
|
||||
buffer.append(paragraph).append("\n\n");
|
||||
tokenCount += paraTokens;
|
||||
}
|
||||
|
||||
// 保存最后一个分片
|
||||
if (buffer.length() > 0 && chunkParaStart < paragraphs.size()) {
|
||||
String chunkContent = buffer.toString().trim();
|
||||
int actualStart = paraPositions.get(chunkParaStart).start;
|
||||
int actualEnd = paraPositions.get(paragraphs.size() - 1).end;
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(chunkContent)
|
||||
.startOffset(section.startIndex + actualStart)
|
||||
.endOffset(section.startIndex + actualEnd)
|
||||
.chunkIndex(chunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存当前分块，返回下一个分块的起始段落索引
|
||||
* <p>
|
||||
* 从 section.content 中提取原始文本（而非手工拼装），修复 index 漂移问题
|
||||
*/
|
||||
private int saveChunkAndGetNextStart(
|
||||
List<DocumentChunk> chunks,
|
||||
Section section,
|
||||
List<ParagraphPos> paraPositions,
|
||||
int fromPara,
|
||||
int toPara,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
int chunkIndex) {
|
||||
|
||||
int actualStart = paraPositions.get(fromPara).start;
|
||||
int actualEnd = paraPositions.get(toPara - 1).end;
|
||||
String originalText = section.content.substring(actualStart, actualEnd);
|
||||
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(originalText)
|
||||
.startOffset(section.startIndex + actualStart)
|
||||
.endOffset(section.startIndex + actualEnd)
|
||||
.chunkIndex(chunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
|
||||
return toPara; // 下一个分块的起始段落索引
|
||||
}
|
||||
|
||||
/**
|
||||
* 按段落分割文本
|
||||
*/
|
||||
private List<String> splitByParagraphs(String content) {
|
||||
List<String> paragraphs = new ArrayList<>();
|
||||
|
||||
// 按双换行符分割段落
|
||||
String[] parts = content.split("\n\n+");
|
||||
for (String part : parts) {
|
||||
String trimmed = part.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
paragraphs.add(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
return paragraphs;
|
||||
}
|
||||
|
||||
/**
|
||||
* 定位每个段落在原始文本中的字符偏移
|
||||
*/
|
||||
private List<ParagraphPos> locateParagraphPositions(List<String> paragraphs, String sectionContent) {
|
||||
List<ParagraphPos> positions = new ArrayList<>();
|
||||
int searchFrom = 0;
|
||||
for (String p : paragraphs) {
|
||||
int idx = sectionContent.indexOf(p, searchFrom);
|
||||
if (idx >= 0) {
|
||||
positions.add(new ParagraphPos(idx, idx + p.length()));
|
||||
searchFrom = idx + p.length();
|
||||
} else {
|
||||
// fallback: 段落在原文中找不到（不应该发生）
|
||||
positions.add(new ParagraphPos(searchFrom, searchFrom + p.length()));
|
||||
searchFrom += p.length();
|
||||
}
|
||||
}
|
||||
return positions;
|
||||
}
|
||||
|
||||
/**
|
||||
* 启发式 token 估算（无需外部依赖）
|
||||
* <p>
|
||||
* 中文(BMP): ~1 字符/token
|
||||
* 英文/数字/标点: ~4 字符/token
|
||||
* 空白字符忽略
|
||||
*/
|
||||
private int estimateTokens(String text) {
|
||||
int nonCjkCount = 0;
|
||||
int cjkCount = 0;
|
||||
for (char c : text.toCharArray()) {
|
||||
if (Character.isWhitespace(c)) {
|
||||
continue;
|
||||
}
|
||||
Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
|
||||
if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
|
||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
|
||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
|
||||
|| block == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS) {
|
||||
cjkCount++;
|
||||
} else {
|
||||
nonCjkCount++;
|
||||
}
|
||||
}
|
||||
return cjkCount + (nonCjkCount + 3) / 4; // 非中文每 4 字符算 1 token，向上取整
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断当前段落是否属于不可中断的结构
|
||||
* <p>
|
||||
* 不可中断结构包括：
|
||||
* - 有序列表项（"1. ", "2. " 格式）
|
||||
* - 无序列表项（"- " 或 "* " 格式）
|
||||
* - 未闭合的代码块（``` 内）
|
||||
*/
|
||||
private boolean isInUnbreakableContext(String buffer, String nextParagraph) {
|
||||
// 有序列表：判断 buffer 末尾和下一段是否都是列表项
|
||||
if (nextParagraph.matches("^\\d{1,2}\\.\\s.*")) {
|
||||
String lastLine = getLastNonEmptyLine(buffer);
|
||||
if (lastLine != null && lastLine.matches("^\\d{1,2}\\.\\s.*")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// 无序列表："- " 或 "* " 格式
|
||||
if (nextParagraph.matches("^[-*]\\s.*")) {
|
||||
String lastLine = getLastNonEmptyLine(buffer);
|
||||
if (lastLine != null && lastLine.matches("^[-*]\\s.*")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// 代码块：``` 未闭合
|
||||
if (buffer.contains("```")) {
|
||||
int count = 0;
|
||||
for (int i = 0; i <= buffer.length() - 3; i++) {
|
||||
if (buffer.substring(i).startsWith("```")) {
|
||||
count++;
|
||||
i += 2;
|
||||
}
|
||||
}
|
||||
if (count % 2 == 1) {
|
||||
return true; // 奇数个 ``` → 在代码块内部
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 buffer 中最后一行非空白文本
|
||||
*/
|
||||
private String getLastNonEmptyLine(String buffer) {
|
||||
String[] lines = buffer.split("\n");
|
||||
for (int i = lines.length - 1; i >= 0; i--) {
|
||||
String line = lines[i].trim();
|
||||
if (!line.isEmpty()) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取重叠文本
|
||||
* 从文本末尾提取指定长度的内容作为下一个分片的开头
|
||||
*/
|
||||
private String getOverlapText(String text) {
|
||||
int overlapSize = Math.min(chunkConfig.getOverlap(), text.length());
|
||||
if (overlapSize <= 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// 从末尾提取重叠内容
|
||||
String overlap = text.substring(text.length() - overlapSize);
|
||||
|
||||
// 尝试在句子边界截断（查找最后一个句号、问号、感叹号）
|
||||
int lastSentenceEnd = Math.max(
|
||||
overlap.lastIndexOf('。'),
|
||||
Math.max(overlap.lastIndexOf('？'), overlap.lastIndexOf('！'))
|
||||
);
|
||||
|
||||
if (lastSentenceEnd > overlapSize / 2) {
|
||||
return overlap.substring(lastSentenceEnd + 1).trim();
|
||||
}
|
||||
|
||||
return overlap.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 段落在原文中的位置
|
||||
*/
|
||||
private static class ParagraphPos {
|
||||
final int start;
|
||||
final int end;
|
||||
|
||||
ParagraphPos(int start, int end) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 章节数据类
|
||||
*/
|
||||
private static class Section {
|
||||
String title; // 最近一级标题名称
|
||||
int level; // 标题级别（1-6），0=无标题
|
||||
String breadcrumb; // 完整面包屑路径
|
||||
String content; // 章节内容
|
||||
int startIndex; // 在原文中的起始偏移
|
||||
|
||||
Section(String title, int level, String breadcrumb, String content, int startIndex) {
|
||||
this.title = title;
|
||||
this.level = level;
|
||||
this.breadcrumb = breadcrumb;
|
||||
this.content = content;
|
||||
this.startIndex = startIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 文档字段补全服务
|
||||
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DocumentFieldEnricher {
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
private String promptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
promptTemplate = new String(
|
||||
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("DocumentFieldEnricher prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 doc-field-enricher-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void enrich(Frontmatter frontmatter, String bodyText) {
|
||||
enrich(frontmatter, bodyText, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 Frontmatter 补全 covers 和 whenToRetrieve
|
||||
* 若已有值则跳过；LLM 失败时降级，不阻断主流程
|
||||
*
|
||||
* @param frontmatter 待补全的 frontmatter
|
||||
* @param bodyText 文档正文
|
||||
* @param category 文档所属域（用于查找同域其他文档）
|
||||
*/
|
||||
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
|
||||
if (frontmatter == null) return;
|
||||
|
||||
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
|
||||
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
|
||||
|
||||
if (!needsCovers && !needsWhen) {
|
||||
log.debug("covers 和 whenToRetrieve 已存在，跳过 LLM 生成");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String snippet = bodyText != null && bodyText.length() > 1000
|
||||
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
|
||||
|
||||
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
|
||||
|
||||
String promptText = String.format(promptTemplate,
|
||||
frontmatter.getTitle(),
|
||||
frontmatter.getSummary(),
|
||||
sameDomainDocs,
|
||||
snippet);
|
||||
|
||||
String response = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
|
||||
// 提取 JSON 部分（防止模型输出多余文本）
|
||||
String json = extractJson(response);
|
||||
JsonNode node = objectMapper.readTree(json);
|
||||
|
||||
if (needsCovers && node.has("covers")) {
|
||||
List<String> covers = new ArrayList<>();
|
||||
node.get("covers").forEach(n -> covers.add(n.asText()));
|
||||
frontmatter.setCovers(covers);
|
||||
log.debug("LLM 生成 covers: {}", covers);
|
||||
}
|
||||
|
||||
if (needsWhen && node.has("whenToRetrieve")) {
|
||||
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
|
||||
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成文档字段失败，降级处理: title={}", frontmatter.getTitle(), e);
|
||||
if (needsCovers) frontmatter.setCovers(List.of());
|
||||
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
|
||||
}
|
||||
}
|
||||
|
||||
private String extractJson(String text) {
|
||||
if (text == null) return "{}";
|
||||
int start = text.indexOf('{');
|
||||
int end = text.lastIndexOf('}');
|
||||
if (start == -1 || end == -1 || end <= start) return "{}";
|
||||
return text.substring(start, end + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建同域其他文档标题列表（供 LLM 做排除判断）
|
||||
*/
|
||||
private String buildSameDomainDocs(String currentTitle, String category) {
|
||||
if (category == null || category.isBlank()) return "（无同域文档信息）";
|
||||
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.map(KnowledgeEntry::getTitle)
|
||||
.filter(t -> t != null && !t.equals(currentTitle))
|
||||
.collect(Collectors.toList());
|
||||
if (otherTitles.isEmpty()) return "（无同域其他文档）";
|
||||
return String.join("、", otherTitles);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagIngestResponse;
|
||||
import com.superbiz.agent.client.PyRagClientException;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import com.superbiz.agent.dto.DocumentQueryResponse;
|
||||
import com.superbiz.agent.dto.DocumentUploadRequest;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.exception.DocumentProcessException;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -27,11 +27,13 @@ import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 文档管理服务
|
||||
* 文档管理服务。
|
||||
*
|
||||
* <p>负责 MySQL 元数据（{@link ApiDocument}）、本地原件保存与业务查询/删除；
|
||||
* 文档解析、frontmatter 校验、分块与向量索引全部由 py-rag 服务端 ingest 完成。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -41,34 +43,19 @@ public class DocumentManagementService {
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Autowired
|
||||
private TextExtractorService textExtractorService;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService documentChunkService;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
private PyRagClient pyRagClient;
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private DocumentFieldEnricher documentFieldEnricher;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* 上传文档
|
||||
* 上传文档。
|
||||
*
|
||||
* <p>流程：hash 去重 → 本地原件保存 → py-rag ingest（同步，服务端解析/分块/索引）→
|
||||
* MySQL 元数据落库。docId 取 py-rag 返回的 doc_id，与检索 evidence_key 的 docId 段对齐。</p>
|
||||
*
|
||||
* @param request 上传请求
|
||||
* @return 文档ID
|
||||
@@ -83,15 +70,7 @@ public class DocumentManagementService {
|
||||
log.info("开始上传文档，文件名: {}, 大小: {} bytes", fileName, file.getSize());
|
||||
|
||||
try {
|
||||
// 1. 验证文件格式
|
||||
if (!textExtractorService.isSupportedFormat(fileName)) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "upload",
|
||||
"不支持的文件格式，仅支持 .md 和 .txt"
|
||||
);
|
||||
}
|
||||
|
||||
// 2. 计算文件 hash（去重）
|
||||
// 1. 计算文件 hash（去重）
|
||||
long hashStart = System.currentTimeMillis();
|
||||
String fileHash = calculateFileHash(file);
|
||||
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
|
||||
@@ -105,16 +84,7 @@ public class DocumentManagementService {
|
||||
);
|
||||
}
|
||||
|
||||
// 3. 提取文本
|
||||
long extractStart = System.currentTimeMillis();
|
||||
String text = textExtractorService.extractText(file, fileName);
|
||||
log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart);
|
||||
|
||||
if (text == null || text.isBlank()) {
|
||||
throw new DocumentProcessException(fileName, "upload", "文档内容为空");
|
||||
}
|
||||
|
||||
// 4. 保存原始文件到本地
|
||||
// 2. category 缺省处理 + 保存原始文件到本地
|
||||
String category = request.getCategory();
|
||||
if (category == null || category.isBlank()) {
|
||||
category = "default";
|
||||
@@ -123,47 +93,33 @@ public class DocumentManagementService {
|
||||
localPath = saveToLocal(file, fileName, category);
|
||||
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
|
||||
|
||||
// 5. 解析 frontmatter
|
||||
long frontmatterStart = System.currentTimeMillis();
|
||||
Frontmatter frontmatter = null;
|
||||
String bodyText = text;
|
||||
if (frontmatterParser.hasFrontmatter(text)) {
|
||||
frontmatter = frontmatterParser.parse(text);
|
||||
if (frontmatter != null) {
|
||||
// LLM 补全 covers / whenToRetrieve（已有值则跳过）
|
||||
bodyText = frontmatterParser.stripFrontmatter(text);
|
||||
documentFieldEnricher.enrich(frontmatter, bodyText, category);
|
||||
log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
|
||||
} else {
|
||||
log.warn("frontmatter解析失败，文件名: {}", fileName);
|
||||
}
|
||||
} else {
|
||||
log.debug("文件不包含frontmatter: {}", fileName);
|
||||
// 3. py-rag 入库（格式校验/frontmatter/分块/向量索引都在服务端；同内容重传返回 unchanged）
|
||||
long ingestStart = System.currentTimeMillis();
|
||||
PyRagIngestResponse ingest;
|
||||
try {
|
||||
ingest = pyRagClient.ingest(fileName, file.getBytes(), file.getContentType(),
|
||||
category, null, null, null);
|
||||
} catch (PyRagClientException | IOException e) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "ingest", "py-rag 入库失败: " + e.getMessage(), e
|
||||
);
|
||||
}
|
||||
log.info("py-rag 入库完成: docId={}, status={}, chunks={}, time={}ms",
|
||||
ingest.docId(), ingest.status(), ingest.chunkCount(),
|
||||
System.currentTimeMillis() - ingestStart);
|
||||
|
||||
// 6. 分块
|
||||
long chunkStart = System.currentTimeMillis();
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(bodyText, fileName);
|
||||
if (chunks.isEmpty()) {
|
||||
throw new DocumentProcessException(fileName, "upload", "文档分块失败");
|
||||
}
|
||||
log.info("文档分块完成: fileName={}, chunks={}, time={}ms",
|
||||
fileName, chunks.size(), System.currentTimeMillis() - chunkStart);
|
||||
|
||||
// 7. 创建文档元数据
|
||||
String docId = resolveDocumentId(frontmatter);
|
||||
// 4. 保存文档元数据
|
||||
String metadataJson = null;
|
||||
if (frontmatter != null) {
|
||||
if (ingest.frontmatter() != null) {
|
||||
try {
|
||||
metadataJson = objectMapper.writeValueAsString(frontmatter);
|
||||
metadataJson = objectMapper.writeValueAsString(ingest.frontmatter());
|
||||
} catch (Exception e) {
|
||||
log.warn("Frontmatter序列化失败", e);
|
||||
log.warn("frontmatter 序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
ApiDocument document = ApiDocument.builder()
|
||||
.docId(docId)
|
||||
.docId(ingest.docId())
|
||||
.fileName(fileName)
|
||||
.filePath(localPath)
|
||||
.metadata(metadataJson)
|
||||
@@ -173,56 +129,17 @@ public class DocumentManagementService {
|
||||
.version(request.getVersion())
|
||||
.fileSize(file.getSize())
|
||||
.fileHash(fileHash)
|
||||
.status("PROCESSING")
|
||||
.chunkCount(chunks.size())
|
||||
.status("INDEXED")
|
||||
.chunkCount(ingest.chunkCount())
|
||||
.build();
|
||||
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
log.info("文档元数据已保存: docId={}", docId);
|
||||
log.info("文档元数据已保存: docId={}", document.getDocId());
|
||||
|
||||
// 8. 向量化并索引
|
||||
try {
|
||||
long vectorStart = System.currentTimeMillis();
|
||||
vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter);
|
||||
document.setStatus("INDEXED");
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
log.info("文档向量索引完成: docId={}, category={}, time={}ms",
|
||||
docId, category, System.currentTimeMillis() - vectorStart);
|
||||
log.info("文档上传完成: docId={}, fileName={}, ingestStatus={}, totalTime={}ms",
|
||||
document.getDocId(), fileName, ingest.status(), System.currentTimeMillis() - startTime);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("文档索引失败: docId={}", docId, e);
|
||||
document.setStatus("FAILED");
|
||||
apiDocumentRepository.save(document);
|
||||
throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
// 9. 更新 L0 索引
|
||||
if (frontmatter != null) {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath(localPath)
|
||||
.title(frontmatter.getTitle())
|
||||
.keywords(frontmatter.getKeywords())
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.sections(frontmatter.getSections())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
|
||||
}
|
||||
|
||||
// 触发域级聚合重算
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
|
||||
docId, fileName, frontmatter != null, totalTime);
|
||||
|
||||
return docId;
|
||||
return document.getDocId();
|
||||
|
||||
} catch (Exception e) {
|
||||
// 失败时清理本地文件
|
||||
@@ -314,16 +231,6 @@ public class DocumentManagementService {
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveDocumentId(Frontmatter frontmatter) {
|
||||
if (frontmatter != null && frontmatter.getSource() != null) {
|
||||
String source = frontmatter.getSource().trim();
|
||||
if (!source.isEmpty() && source.length() <= 64) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 docId 查询文档
|
||||
*/
|
||||
@@ -358,7 +265,10 @@ public class DocumentManagementService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文档
|
||||
* 删除文档（仅 MySQL 元数据与本地原件）。
|
||||
*
|
||||
* <p>py-rag v1 契约没有单文档删除端点：已入库内容需在其服务端
|
||||
* 全量重建（{@code /api/v1/collections:rebuild}）后才会从知识库消失。</p>
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteDocument(String docId) {
|
||||
@@ -379,47 +289,9 @@ public class DocumentManagementService {
|
||||
}
|
||||
}
|
||||
|
||||
// 删除 L0 索引
|
||||
if (doc.getFilePath() != null) {
|
||||
knowledgeIndexService.removeFromIndex(doc.getFilePath());
|
||||
}
|
||||
|
||||
// 删除向量索引
|
||||
try {
|
||||
vectorIndexService.deleteDocumentChunks(docId);
|
||||
log.info("文档向量索引已删除，docId: {}", docId);
|
||||
} catch (Exception e) {
|
||||
log.warn("删除向量索引失败，docId: {}", docId, e);
|
||||
}
|
||||
|
||||
// 删除元数据
|
||||
// 删除元数据（py-rag 侧索引留存，重建后失效）
|
||||
apiDocumentRepository.delete(doc);
|
||||
log.info("文档已删除，docId: {}", docId);
|
||||
|
||||
// 触发域级聚合重算
|
||||
String category = doc.getFilePath() != null
|
||||
? resolveCategory(doc.getFilePath()) : null;
|
||||
if (category != null) {
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为响应 DTO
|
||||
*/
|
||||
/**
|
||||
* 从 filePath 解析 category（取 knowledge_base/{category}/... 中的 category 段）
|
||||
*/
|
||||
private String resolveCategory(String filePath) {
|
||||
try {
|
||||
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
|
||||
// filePath 形如 knowledge_base/payment/xxx.md，取倒数第二段
|
||||
int nameCount = p.getNameCount();
|
||||
if (nameCount >= 2) {
|
||||
return p.getName(nameCount - 2).toString();
|
||||
}
|
||||
} catch (Exception ignored) {}
|
||||
return null;
|
||||
log.info("文档已删除，docId={}（py-rag 侧需全量重建后生效）", docId);
|
||||
}
|
||||
|
||||
private Path resolveLocalPath(String filePath) {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Frontmatter 解析器
|
||||
* 解析 Markdown 文件头的 YAML frontmatter
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class FrontmatterParser {
|
||||
|
||||
private final Yaml yaml = new Yaml();
|
||||
|
||||
/**
|
||||
* 检查文件是否包含 frontmatter
|
||||
*
|
||||
* @param content 文件内容
|
||||
* @return true 如果包含 frontmatter
|
||||
*/
|
||||
public boolean hasFrontmatter(String content) {
|
||||
if (content == null || content.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return content.trim().startsWith("---");
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析 Markdown frontmatter
|
||||
*
|
||||
* @param content 完整文件内容
|
||||
* @return Frontmatter 对象，如果不存在或解析失败返回 null
|
||||
*/
|
||||
public Frontmatter parse(String content) {
|
||||
if (!hasFrontmatter(content)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. 提取 frontmatter 部分（两个 --- 之间）
|
||||
String frontmatterText = extractFrontmatter(content);
|
||||
if (frontmatterText == null) {
|
||||
log.warn("未找到有效的 frontmatter 结束标记");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 2. 使用 SnakeYAML 解析
|
||||
Map<String, Object> map = yaml.load(frontmatterText);
|
||||
if (map == null || map.isEmpty()) {
|
||||
log.warn("Frontmatter 解析结果为空");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 3. 映射到 Frontmatter 对象
|
||||
Frontmatter frontmatter = Frontmatter.builder()
|
||||
.title((String) map.get("title"))
|
||||
.keywords((java.util.List<String>) map.get("keywords"))
|
||||
.summary((String) map.get("summary"))
|
||||
.category((String) map.get("category"))
|
||||
.source((String) map.get("source"))
|
||||
.breadcrumb((String) map.get("breadcrumb"))
|
||||
.kbScope(firstString(map, "kb_scope", "kbScope"))
|
||||
.sections((Map<String, String>) map.get("sections"))
|
||||
.version((String) map.get("version"))
|
||||
.author((String) map.get("author"))
|
||||
.covers((java.util.List<String>) map.get("covers"))
|
||||
.whenToRetrieve((String) map.get("when_to_retrieve"))
|
||||
.build();
|
||||
|
||||
// 4. 验证必填字段
|
||||
if (frontmatter.getTitle() == null || frontmatter.getKeywords() == null ||
|
||||
frontmatter.getSummary() == null) {
|
||||
log.warn("Frontmatter 缺少必填字段: title={}, keywords={}, summary={}",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords(), frontmatter.getSummary());
|
||||
return null;
|
||||
}
|
||||
|
||||
log.debug("Frontmatter 解析成功: title={}, keywords=",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords());
|
||||
return frontmatter;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Frontmatter 解析失败", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String stripFrontmatter(String content) {
|
||||
if (!hasFrontmatter(content)) {
|
||||
return content;
|
||||
}
|
||||
|
||||
String trimmed = content.trim();
|
||||
int secondDelimiter = trimmed.indexOf("\n---", 3);
|
||||
int delimiterLength = 4;
|
||||
if (secondDelimiter == -1) {
|
||||
secondDelimiter = trimmed.indexOf("\r\n---", 3);
|
||||
delimiterLength = 5;
|
||||
}
|
||||
if (secondDelimiter == -1) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int bodyStart = secondDelimiter + delimiterLength;
|
||||
if (bodyStart < trimmed.length()) {
|
||||
char next = trimmed.charAt(bodyStart);
|
||||
if (next == '\r') {
|
||||
bodyStart++;
|
||||
}
|
||||
if (bodyStart < trimmed.length() && trimmed.charAt(bodyStart) == '\n') {
|
||||
bodyStart++;
|
||||
}
|
||||
}
|
||||
return trimmed.substring(Math.min(bodyStart, trimmed.length())).stripLeading();
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取 frontmatter 文本（两个 --- 之间的内容）
|
||||
*
|
||||
* @param content 完整文件内容
|
||||
* @return frontmatter 文本，如果格式错误返回 null
|
||||
*/
|
||||
private String extractFrontmatter(String content) {
|
||||
// 去除开头的空白
|
||||
content = content.trim();
|
||||
|
||||
// 检查是否以 --- 开头
|
||||
if (!content.startsWith("---")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// 查找第二个 ---（结束标记）
|
||||
int secondDelimiter = content.indexOf("\n---", 3);
|
||||
if (secondDelimiter == -1) {
|
||||
// 尝试查找 Windows 风格换行
|
||||
secondDelimiter = content.indexOf("\r\n---", 3);
|
||||
if (secondDelimiter == -1) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 提取 frontmatter（不包含 --- 标记）
|
||||
return content.substring(3, secondDelimiter).trim();
|
||||
}
|
||||
|
||||
private String firstString(Map<String, Object> map, String... keys) {
|
||||
for (String key : keys) {
|
||||
Object value = map.get(key);
|
||||
if (value instanceof String text && !text.isBlank()) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,407 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识库初始化服务
|
||||
* 负责批量导入 knowledge_base 目录下的文档到数据库和 Milvus
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseInitService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseInitService.class);
|
||||
|
||||
@Value("${knowledge.base-path:knowledge_base}")
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService documentChunkService;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService vectorEmbeddingService;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
|
||||
RebuildResult rebuild = new RebuildResult();
|
||||
rebuild.setBasePath(knowledgeBasePath);
|
||||
rebuild.setCollection(hybridKnowledgeStore.collectionName());
|
||||
|
||||
long mysqlBefore = apiDocumentRepository.count();
|
||||
rebuild.setMysqlDocumentsBefore(mysqlBefore);
|
||||
|
||||
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
|
||||
rebuild.setMilvus(milvus);
|
||||
|
||||
apiDocumentRepository.deleteAll();
|
||||
apiDocumentRepository.flush();
|
||||
knowledgeIndexService.clearIndex();
|
||||
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
|
||||
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
|
||||
|
||||
InitResult init = initializeKnowledgeBase(true);
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
// Success when at least one doc indexed and no hard failures.
|
||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
*
|
||||
* @param force 是否强制重新导入（跳过去重检查）
|
||||
* @return 初始化结果
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public InitResult initializeKnowledgeBase(boolean force) {
|
||||
logger.info("开始初始化知识库: basePath={}, force={}", knowledgeBasePath, force);
|
||||
|
||||
InitResult result = new InitResult();
|
||||
Path baseDir = Paths.get(knowledgeBasePath);
|
||||
|
||||
if (!Files.exists(baseDir)) {
|
||||
logger.error("知识库目录不存在: {}", knowledgeBasePath);
|
||||
throw new RuntimeException("知识库目录不存在: " + knowledgeBasePath);
|
||||
}
|
||||
|
||||
// 1. 扫描所有 Markdown 文件
|
||||
List<Path> markdownFiles = scanMarkdownFiles(baseDir);
|
||||
result.setScanned(markdownFiles.size());
|
||||
logger.info("扫描到 {} 个 Markdown 文件", markdownFiles.size());
|
||||
|
||||
// 2. 如果非强制模式，获取已存在的文档（用于去重）
|
||||
Set<String> existingFilePaths = new HashSet<>();
|
||||
if (!force) {
|
||||
existingFilePaths = apiDocumentRepository.findAll().stream()
|
||||
.map(ApiDocument::getFilePath)
|
||||
.collect(Collectors.toSet());
|
||||
logger.info("已存在 个文档记录", existingFilePaths.size());
|
||||
}
|
||||
|
||||
// 3. 逐个处理文档
|
||||
for (Path file : markdownFiles) {
|
||||
String relativePath = baseDir.relativize(file).toString().replace("\\", "/");
|
||||
|
||||
try {
|
||||
// 去重检查
|
||||
if (!force && existingFilePaths.contains(relativePath)) {
|
||||
logger.debug("跳过已存在的文档: {}", relativePath);
|
||||
result.incrementSkipped();
|
||||
result.addDetail(relativePath, "已存在，跳过");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 解析文档
|
||||
String content = Files.readString(file);
|
||||
Frontmatter frontmatter = frontmatterParser.parse(content);
|
||||
|
||||
if (frontmatter == null) {
|
||||
logger.warn("文档格式无效: {}, frontmatter 解析失败", relativePath);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "格式无效: frontmatter 解析失败");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 提取字段
|
||||
String title = frontmatter.getTitle();
|
||||
String summary = frontmatter.getSummary();
|
||||
String category = frontmatter.getCategory() != null ? frontmatter.getCategory() : "general";
|
||||
List<String> keywords = frontmatter.getKeywords();
|
||||
|
||||
if (title == null || title.isBlank()) {
|
||||
logger.warn("文档缺少标题: {}", relativePath);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "缺少标题");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 保存到数据库
|
||||
ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
|
||||
|
||||
// 提取文档正文（去除 frontmatter）
|
||||
String body = extractBody(content);
|
||||
|
||||
// 文档分块
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
||||
|
||||
// 上传到 Milvus hybrid collection（dense + BM25 search_text）
|
||||
try {
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
|
||||
|
||||
document.setStatus("INDEXED");
|
||||
document.setChunkCount(chunks.size());
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
|
||||
title, document.getDocId(), chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
||||
|
||||
document.setStatus("FAILED");
|
||||
document.setErrorMessage(e.getMessage());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
|
||||
continue; // 跳过该文档，继续处理下一个
|
||||
}
|
||||
|
||||
// 添加到 L0 内存索引
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath(relativePath)
|
||||
.title(title)
|
||||
.keywords(keywords)
|
||||
.summary(summary)
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.build();
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
|
||||
result.incrementInserted();
|
||||
result.addDetail(relativePath, "导入成功（L0+L1）");
|
||||
logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("处理文档失败: {}", relativePath, e);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "处理失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("知识库初始化完成: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库统计信息
|
||||
*/
|
||||
public Stats getStats() {
|
||||
Stats stats = new Stats();
|
||||
|
||||
// 数据库中的文档数量
|
||||
long totalDocuments = apiDocumentRepository.count();
|
||||
stats.setTotalDocuments(totalDocuments);
|
||||
|
||||
// L0 索引中的文档数量
|
||||
int indexSize = knowledgeIndexService.getIndexSize();
|
||||
logger.debug("L0 索引大小: {}", indexSize);
|
||||
|
||||
// 按分类统计（从 fault_category 字段读取）
|
||||
Map<String, Long> categoryCount = apiDocumentRepository.findAll().stream()
|
||||
.collect(Collectors.groupingBy(
|
||||
doc -> doc.getFaultCategory() != null ? doc.getFaultCategory().name() : "GENERAL",
|
||||
Collectors.counting()
|
||||
));
|
||||
stats.setCategoryCount(categoryCount);
|
||||
|
||||
// Milvus 中的向量数量（需要实现）
|
||||
// TODO: 查询 Milvus collection 的实体数量
|
||||
stats.setTotalVectors(0L);
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* 扫描目录下所有 Markdown 文件
|
||||
*/
|
||||
private List<Path> scanMarkdownFiles(Path baseDir) {
|
||||
List<Path> files = new ArrayList<>();
|
||||
|
||||
try {
|
||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
||||
// Import content docs only; skip README/index markdown without frontmatter.
|
||||
if (name.endsWith(".md")
|
||||
&& !name.equalsIgnoreCase("README.md")
|
||||
&& !name.equalsIgnoreCase("readme.md")) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException exc) {
|
||||
logger.warn("访问文件失败: {}", file, exc);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
} catch (IOException e) {
|
||||
logger.error("扫描目录失败: {}", baseDir, e);
|
||||
throw new RuntimeException("扫描目录失败", e);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存文档到数据库
|
||||
*/
|
||||
private ApiDocument saveToDatabase(String filePath, String title, String summary,
|
||||
String category, String content, List<String> keywords) {
|
||||
ApiDocument document = new ApiDocument();
|
||||
document.setDocId(UUID.randomUUID().toString());
|
||||
document.setFileName(Paths.get(filePath).getFileName().toString());
|
||||
document.setFilePath(filePath);
|
||||
document.setApiName(title); // 使用 title 作为 apiName
|
||||
document.setStatus("PENDING"); // 初始状态为 PENDING，索引成功后更新为 INDEXED
|
||||
|
||||
// 映射 category 到 FaultCategory 枚举
|
||||
FaultCategory faultCategory = FaultCategory.fromString(category);
|
||||
document.setFaultCategory(faultCategory);
|
||||
|
||||
// 将 frontmatter 信息保存到 metadata（JSON 格式）
|
||||
String metadataJson = String.format(
|
||||
"{\"title\":\"%s\",\"summary\":\"%s\",\"category\":\"%s\",\"keywords\":%s}",
|
||||
escapeJson(title),
|
||||
escapeJson(summary),
|
||||
escapeJson(category),
|
||||
"[\"" + String.join("\",\"", keywords.stream().map(this::escapeJson).toArray(String[]::new)) + "\"]"
|
||||
);
|
||||
document.setMetadata(metadataJson);
|
||||
|
||||
document.setFileSize((long) content.length());
|
||||
|
||||
return apiDocumentRepository.save(document);
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON 转义
|
||||
*/
|
||||
private String escapeJson(String str) {
|
||||
if (str == null) {
|
||||
return "";
|
||||
}
|
||||
return str.replace("\\", "\\\\")
|
||||
.replace("\"", "\\\"")
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r");
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取文档正文（去除 frontmatter）
|
||||
*/
|
||||
private String extractBody(String content) {
|
||||
if (!content.trim().startsWith("---")) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int firstEnd = content.indexOf("---", 3);
|
||||
if (firstEnd == -1) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int secondEnd = content.indexOf("---", firstEnd + 3);
|
||||
if (secondEnd == -1) {
|
||||
return content.substring(firstEnd + 3).trim();
|
||||
}
|
||||
|
||||
return content.substring(secondEnd + 3).trim();
|
||||
}
|
||||
|
||||
// ==================== 数据模型 ====================
|
||||
|
||||
/**
|
||||
* 初始化结果
|
||||
*/
|
||||
@Data
|
||||
public static class InitResult {
|
||||
private int scanned; // 扫描到的文件数量
|
||||
private int skipped; // 跳过的文件数量（已存在）
|
||||
private int inserted; // 成功导入的文件数量
|
||||
private int failed; // 失败的文件数量
|
||||
private Map<String, String> details = new LinkedHashMap<>(); // 详细信息
|
||||
|
||||
public void incrementSkipped() {
|
||||
this.skipped++;
|
||||
}
|
||||
|
||||
public void incrementInserted() {
|
||||
this.inserted++;
|
||||
}
|
||||
|
||||
public void incrementFailed() {
|
||||
this.failed++;
|
||||
}
|
||||
|
||||
public void addDetail(String filePath, String message) {
|
||||
this.details.put(filePath, message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计信息
|
||||
*/
|
||||
@Data
|
||||
public static class Stats {
|
||||
private long totalDocuments; // 数据库中的文档总数
|
||||
private long totalVectors; // Milvus 中的向量总数
|
||||
private Map<String, Long> categoryCount; // 按分类统计
|
||||
}
|
||||
|
||||
/**
|
||||
* Full hybrid rebuild result.
|
||||
*/
|
||||
@Data
|
||||
public static class RebuildResult {
|
||||
private boolean success;
|
||||
private String basePath;
|
||||
private String collection;
|
||||
private long mysqlDocumentsBefore;
|
||||
private long mysqlDocumentsAfterClear;
|
||||
private long mysqlDocumentsAfterInit;
|
||||
private int l0IndexSizeAfterClear;
|
||||
private int l0IndexSizeAfterInit;
|
||||
private Map<String, Object> milvus;
|
||||
private InitResult init;
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.KnowledgeDomain;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识域服务
|
||||
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KnowledgeDomainService {
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
private String domainPromptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
domainPromptTemplate = new String(
|
||||
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("KnowledgeDomainService prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 domain-summary-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load domain-summary prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文档变更后重算指定域的 when_to_retrieve
|
||||
*/
|
||||
public void onDocumentChange(String category) {
|
||||
if (category == null || category.isBlank()) return;
|
||||
|
||||
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
buildDomainSummary(category, entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合同域文档，调用 LLM 生成域级摘要，写入 DB
|
||||
*/
|
||||
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
|
||||
if (entries.isEmpty()) {
|
||||
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
|
||||
d.setDocumentCount(0);
|
||||
knowledgeDomainRepository.save(d);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 构建文档列表描述
|
||||
StringBuilder docList = new StringBuilder();
|
||||
for (KnowledgeEntry entry : entries) {
|
||||
docList.append("- 文档：").append(entry.getTitle()).append("\n");
|
||||
if (entry.getWhenToRetrieve() != null) {
|
||||
docList.append(" 适用场景：").append(entry.getWhenToRetrieve()).append("\n");
|
||||
}
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
docList.append(" 覆盖：").append(String.join("、", entry.getCovers())).append("\n");
|
||||
}
|
||||
}
|
||||
|
||||
String description = entries.stream()
|
||||
.map(KnowledgeEntry::getSummary)
|
||||
.filter(s -> s != null && !s.isBlank())
|
||||
.findFirst().orElse(category);
|
||||
|
||||
String whenToRetrieve = null;
|
||||
try {
|
||||
String otherDomainsInfo = buildOtherDomainsInfo(category);
|
||||
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
|
||||
whenToRetrieve = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成域级 when_to_retrieve 失败，保留旧值: domain={}", category, e);
|
||||
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
|
||||
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
|
||||
}
|
||||
|
||||
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
|
||||
.orElse(KnowledgeDomain.builder().domainId(category).build());
|
||||
|
||||
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
|
||||
domain.setWhenToRetrieve(whenToRetrieve);
|
||||
domain.setDocumentCount(entries.size());
|
||||
knowledgeDomainRepository.save(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建注入 Planner 的 knowledge map YAML 文本
|
||||
*/
|
||||
public String buildKnowledgeMap() {
|
||||
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
|
||||
if (domains.isEmpty()) return "";
|
||||
|
||||
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
|
||||
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
|
||||
.filter(e -> e.getCategory() != null)
|
||||
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
|
||||
|
||||
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
|
||||
|
||||
for (KnowledgeDomain domain : domains) {
|
||||
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
|
||||
if (domain.getDescription() != null) {
|
||||
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
|
||||
}
|
||||
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
|
||||
yaml.append(" when_to_retrieve: \"")
|
||||
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
|
||||
}
|
||||
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
|
||||
|
||||
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
|
||||
if (!domainEntries.isEmpty()) {
|
||||
yaml.append(" documents:\n");
|
||||
for (KnowledgeEntry entry : domainEntries) {
|
||||
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建其他域的摘要信息（用于 LLM 域级 prompt 的边界判断）
|
||||
* 优先使用其他域的 when_to_retrieve（边界信号），而非 description
|
||||
*/
|
||||
private String buildOtherDomainsInfo(String currentCategory) {
|
||||
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (KnowledgeDomain d : allDomains) {
|
||||
if (d.getDomainId().equals(currentCategory)) continue;
|
||||
sb.append("- ").append(d.getDomainId());
|
||||
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
|
||||
sb.append("：").append(d.getWhenToRetrieve());
|
||||
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
|
||||
sb.append("（").append(d.getDescription()).append("）");
|
||||
}
|
||||
sb.append("\n");
|
||||
}
|
||||
// 如果 DB 里还没有其他域的记录（首次启动），从 L0 索引补充
|
||||
if (sb.isEmpty()) {
|
||||
knowledgeIndexService.getAllEntries().stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
|
||||
.distinct()
|
||||
.forEach(c -> sb.append("- ").append(c).append("\n"));
|
||||
}
|
||||
return sb.isEmpty() ? "（无其他域信息）" : sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,341 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
/**
|
||||
* L0 知识索引服务（关键词 / domain hint，不是向量库）。
|
||||
*
|
||||
* <h3>定位</h3>
|
||||
* 从 MySQL {@code api_document.metadata}（frontmatter）加载文档级关键词与 category，
|
||||
* 供检索前 query understanding 使用。L0 输出只作为：
|
||||
* <ul>
|
||||
* <li>可选 category filter（唯一 domain 时）</li>
|
||||
* <li>rerank 的 domain/keyword/entity boost 信号</li>
|
||||
* <li>trace 可解释信息</li>
|
||||
* </ul>
|
||||
* <b>L0 命中文档不会直接当作事实 evidence</b>；证据正文只来自 L1 向量召回。
|
||||
*
|
||||
* <h3>匹配方式（当前较粗）</h3>
|
||||
* {@code query.contains(keyword) || keyword.contains(query)}，大小写不敏感。
|
||||
* 没有分词、别名归一或停用词；短词/泛词可能误命中。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KnowledgeIndexService {
|
||||
|
||||
@Value("${knowledge.base-path:knowledge_base}")
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Lazy
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
|
||||
|
||||
@PostConstruct
|
||||
public void loadIndex() {
|
||||
log.info("开始从数据库加载知识库索引");
|
||||
|
||||
try {
|
||||
List<ApiDocument> documents = apiDocumentRepository.findAll();
|
||||
|
||||
int loaded = 0;
|
||||
for (ApiDocument doc : documents) {
|
||||
try {
|
||||
KnowledgeEntry entry = parseDocumentToEntry(doc);
|
||||
if (entry != null) {
|
||||
knowledgeIndex.add(entry);
|
||||
loaded++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("解析文档失败: docId={}, error={}", doc.getDocId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.info("知识库索引加载完成，共 {} 个文档", loaded);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("知识库索引加载失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用就绪后，检查各域是否有 knowledge_domain 记录，无则触发生成
|
||||
* 使用 ApplicationReadyEvent 而非 PostConstruct，避免循环依赖
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void onApplicationReady() {
|
||||
try {
|
||||
knowledgeIndex.stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank())
|
||||
.distinct()
|
||||
.forEach(category -> {
|
||||
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
|
||||
log.info("域 {} 无 knowledge_domain 记录，触发生成", category);
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("域级记录生成失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
|
||||
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
|
||||
|
||||
return KnowledgeEntry.builder()
|
||||
.filePath(doc.getFilePath())
|
||||
.title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
|
||||
.keywords(frontmatter.getKeywords())
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(frontmatter.getCategory())
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("解析 metadata 失败: {}", doc.getDocId(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 兼容旧调用：只返回命中的文档条目。 */
|
||||
public List<KnowledgeEntry> exactMatch(String query) {
|
||||
return analyzeQuery(query).matches();
|
||||
}
|
||||
|
||||
/**
|
||||
* 分析 query，产出 L0 hint。
|
||||
* 遍历内存索引，收集匹配 keyword、domain、title；不做向量检索。
|
||||
*/
|
||||
public L0Hint analyzeQuery(String query) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
if (query == null || query.trim().isEmpty()) {
|
||||
log.debug("查询关键词为空，返回空结果");
|
||||
return L0Hint.empty();
|
||||
}
|
||||
|
||||
String queryLower = query.toLowerCase();
|
||||
List<KnowledgeEntry> results = new ArrayList<>();
|
||||
Set<String> matchedKeywords = new LinkedHashSet<>();
|
||||
Set<String> domains = new LinkedHashSet<>();
|
||||
Set<String> entities = new LinkedHashSet<>();
|
||||
Set<String> titles = new LinkedHashSet<>();
|
||||
|
||||
for (KnowledgeEntry entry : knowledgeIndex) {
|
||||
if (!matchesConfiguredScope(entry)) {
|
||||
continue;
|
||||
}
|
||||
List<String> entryMatchedKeywords = matchedKeywords(entry, queryLower);
|
||||
if (entryMatchedKeywords.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
results.add(entry);
|
||||
matchedKeywords.addAll(entryMatchedKeywords);
|
||||
entities.addAll(entryMatchedKeywords);
|
||||
|
||||
if (entry.getCategory() != null && !entry.getCategory().isBlank()) {
|
||||
domains.add(entry.getCategory());
|
||||
}
|
||||
if (entry.getTitle() != null && !entry.getTitle().isBlank()) {
|
||||
titles.add(entry.getTitle());
|
||||
}
|
||||
}
|
||||
|
||||
long elapsedTime = System.currentTimeMillis() - startTime;
|
||||
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
|
||||
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
|
||||
|
||||
return new L0Hint(
|
||||
List.copyOf(results),
|
||||
List.copyOf(matchedKeywords),
|
||||
List.copyOf(domains),
|
||||
List.copyOf(entities),
|
||||
List.copyOf(titles)
|
||||
);
|
||||
}
|
||||
|
||||
private boolean matchesKeywords(KnowledgeEntry entry, String query) {
|
||||
return !matchedKeywords(entry, query).isEmpty();
|
||||
}
|
||||
|
||||
private boolean matchesConfiguredScope(KnowledgeEntry entry) {
|
||||
String scope = trimToNull(kbScope);
|
||||
if (scope == null) {
|
||||
return true;
|
||||
}
|
||||
return scope.equals(trimToNull(entry.getKbScope()));
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关键词双向包含匹配。
|
||||
* query 已在调用方 lower-case；keyword 在此 lower-case。
|
||||
* 例：query="mysql timeout" 可命中 keyword="mysql"；
|
||||
* 反过来 keyword="mysql connection pool timeout" 也可能被短 query 命中。
|
||||
*/
|
||||
private List<String> matchedKeywords(KnowledgeEntry entry, String query) {
|
||||
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<String> matches = new ArrayList<>();
|
||||
for (String keyword : entry.getKeywords()) {
|
||||
String keywordLower = keyword.toLowerCase();
|
||||
if (query.contains(keywordLower) || keywordLower.contains(query)) {
|
||||
matches.add(keyword);
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
public String readDocument(String filePath, int maxChars) {
|
||||
try {
|
||||
Path fullPath = resolveDocumentPath(filePath);
|
||||
if (!Files.exists(fullPath)) {
|
||||
log.warn("读取文档失败，文件不存在: basePath={}, filePath={}, resolvedPath={}",
|
||||
knowledgeBasePath, filePath, fullPath);
|
||||
return null;
|
||||
}
|
||||
String content = Files.readString(fullPath);
|
||||
|
||||
if (content.length() > maxChars) {
|
||||
return content.substring(0, maxChars) + "...";
|
||||
}
|
||||
|
||||
return content;
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("读取文档失败: basePath={}, filePath={}", knowledgeBasePath, filePath, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Path resolveDocumentPath(String filePath) {
|
||||
if (filePath == null || filePath.isBlank()) {
|
||||
throw new IllegalArgumentException("filePath cannot be blank");
|
||||
}
|
||||
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
if (path.isAbsolute()) {
|
||||
return path;
|
||||
}
|
||||
|
||||
Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize();
|
||||
Path baseName = basePath.getFileName();
|
||||
if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) {
|
||||
return basePath.getParent().resolve(path).normalize();
|
||||
}
|
||||
|
||||
Path pathFromWorkingDir = path.toAbsolutePath().normalize();
|
||||
if (pathFromWorkingDir.startsWith(basePath)) {
|
||||
return pathFromWorkingDir;
|
||||
}
|
||||
|
||||
return basePath.resolve(path).normalize();
|
||||
}
|
||||
|
||||
public void addToIndex(KnowledgeEntry entry) {
|
||||
knowledgeIndex.add(entry);
|
||||
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
|
||||
}
|
||||
|
||||
public void removeFromIndex(String filePath) {
|
||||
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
|
||||
log.debug("文档已从 L0 索引移除: {}", filePath);
|
||||
}
|
||||
|
||||
/** Clear in-memory L0 entries (used by knowledge rebuild). */
|
||||
public void clearIndex() {
|
||||
knowledgeIndex.clear();
|
||||
log.info("L0 knowledge index cleared");
|
||||
}
|
||||
|
||||
public int getIndexSize() {
|
||||
return knowledgeIndex.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有索引条目（供域聚合使用）
|
||||
*/
|
||||
public List<KnowledgeEntry> getAllEntries() {
|
||||
return List.copyOf(knowledgeIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* L0 分析结果。
|
||||
*
|
||||
* @param matches 命中的文档条目（仅 hint，不是 evidence）
|
||||
* @param matchedKeywords 命中的关键词
|
||||
* @param domains 命中文档的 category 集合
|
||||
* @param entities 当前实现等同 matchedKeywords，预留实体字段
|
||||
* @param titles 命中文档标题
|
||||
*/
|
||||
public record L0Hint(
|
||||
List<KnowledgeEntry> matches,
|
||||
List<String> matchedKeywords,
|
||||
List<String> domains,
|
||||
List<String> entities,
|
||||
List<String> titles
|
||||
) {
|
||||
public static L0Hint empty() {
|
||||
return new L0Hint(List.of(), List.of(), List.of(), List.of(), List.of());
|
||||
}
|
||||
|
||||
/** 仅当恰好一个 domain 时返回，用于安全地加 category filter。 */
|
||||
public String singleDomainOrNull() {
|
||||
return domains.size() == 1 ? domains.get(0) : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 检索前的 query 理解层（L0 出口）。
|
||||
*
|
||||
* <p>输入是 Agent 的原始检索句，输出 {@link KnowledgeQuery}，供后续 L1 过滤与 rerank 使用。</p>
|
||||
*
|
||||
* <h3>当前能力边界</h3>
|
||||
* <ul>
|
||||
* <li>会做：关键词匹配、domain/entity/title hint、唯一 domain 时生成 categoryFilter</li>
|
||||
* <li>不会做：真正的 query rewrite / 同义词扩展 / 多 query 改写
|
||||
* （{@code rewrittenQuery} 目前等于 {@code originalQuery}）</li>
|
||||
* <li>L0 命中文档正文不会直接当作 evidence；证据只来自 L1 向量召回</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeQueryTransformer {
|
||||
|
||||
private final KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
public KnowledgeQueryTransformer(KnowledgeIndexService knowledgeIndexService) {
|
||||
this.knowledgeIndexService = knowledgeIndexService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将原始 query 转为检索控制结构。
|
||||
*
|
||||
* <p>{@code categoryFilter} 仅在 L0 恰好命中一个 domain 时非空；
|
||||
* 多 domain 或零 domain 时为 null，避免错误收窄召回。</p>
|
||||
*/
|
||||
public KnowledgeQuery transform(String rawQuery) {
|
||||
String normalized = rawQuery == null ? "" : rawQuery.trim();
|
||||
KnowledgeIndexService.L0Hint hint = knowledgeIndexService.analyzeQuery(normalized);
|
||||
return KnowledgeQuery.builder()
|
||||
.originalQuery(normalized)
|
||||
// 预留改写字段；当前未实现 rewrite，保持与 original 一致
|
||||
.rewrittenQuery(normalized)
|
||||
.domainHints(safeList(hint.domains()))
|
||||
.matchedKeywords(safeList(hint.matchedKeywords()))
|
||||
.entities(safeList(hint.entities()))
|
||||
// 只有唯一 domain 才作为向量 metadata 的 category 过滤条件
|
||||
.categoryFilter(hint.singleDomainOrNull())
|
||||
.l0Titles(safeList(hint.titles()))
|
||||
.l0MatchCount(hint.matches() == null ? 0 : hint.matches().size())
|
||||
.build();
|
||||
}
|
||||
|
||||
private List<String> safeList(List<String> values) {
|
||||
return values == null ? List.of() : values;
|
||||
}
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonCase;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonReport;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonResult;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Service
|
||||
public class RagRetrievalSidecarComparisonService {
|
||||
|
||||
private final VectorSearchService vectorSearchService;
|
||||
private final SpringAiVectorStoreSidecarService sidecarService;
|
||||
private final RetrievalResultNormalizer normalizer;
|
||||
private final RagSidecarProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public RagRetrievalSidecarComparisonService(VectorSearchService vectorSearchService,
|
||||
SpringAiVectorStoreSidecarService sidecarService,
|
||||
RetrievalResultNormalizer normalizer,
|
||||
RagSidecarProperties properties,
|
||||
ObjectMapper objectMapper) {
|
||||
this.vectorSearchService = vectorSearchService;
|
||||
this.sidecarService = sidecarService;
|
||||
this.normalizer = normalizer;
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public RetrievalComparisonReport compare(List<RetrievalComparisonCase> cases, int topK) {
|
||||
List<RetrievalComparisonResult> results = new ArrayList<>();
|
||||
String sidecarStatus = "not_run";
|
||||
for (RetrievalComparisonCase comparisonCase : cases) {
|
||||
List<ComparableRetrievalResult> currentResults = normalizeCurrentResults(
|
||||
vectorSearchService.searchSimilarDocuments(
|
||||
comparisonCase.getQuery(),
|
||||
topK,
|
||||
comparisonCase.getCategory()
|
||||
)
|
||||
);
|
||||
SidecarRetrievalResponse sidecar = sidecarService.search(
|
||||
comparisonCase.getQuery(),
|
||||
topK,
|
||||
comparisonCase.getCategory()
|
||||
);
|
||||
sidecarStatus = sidecar.getStatus();
|
||||
results.add(RetrievalComparisonResult.builder()
|
||||
.caseId(comparisonCase.getCaseId())
|
||||
.scenario(comparisonCase.getScenario())
|
||||
.query(comparisonCase.getQuery())
|
||||
.category(comparisonCase.getCategory())
|
||||
.currentResults(currentResults)
|
||||
.sidecar(sidecar)
|
||||
.differences(compareDifferences(currentResults, sidecar.getResults()))
|
||||
.build());
|
||||
}
|
||||
|
||||
return RetrievalComparisonReport.builder()
|
||||
.generatedAt(OffsetDateTime.now(ZoneOffset.UTC).toString())
|
||||
.caseCount(cases.size())
|
||||
.topK(topK)
|
||||
.sidecarStatus(sidecarStatus)
|
||||
.results(results)
|
||||
.build();
|
||||
}
|
||||
|
||||
public RetrievalComparisonReport compareGoldenCases(Path caseFile) throws IOException {
|
||||
var root = objectMapper.readTree(caseFile.toFile());
|
||||
int topK = root.path("topK").asInt(5);
|
||||
List<RetrievalComparisonCase> cases = new ArrayList<>();
|
||||
for (var node : root.path("cases")) {
|
||||
cases.add(RetrievalComparisonCase.builder()
|
||||
.caseId(node.path("caseId").asText())
|
||||
.scenario(node.path("scenario").asText())
|
||||
.query(node.path("query").asText())
|
||||
.build());
|
||||
}
|
||||
return compare(cases, topK);
|
||||
}
|
||||
|
||||
public void writeReports(RetrievalComparisonReport report, Path jsonPath, Path markdownPath) throws IOException {
|
||||
createParentDirectories(jsonPath);
|
||||
createParentDirectories(markdownPath);
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(jsonPath.toFile(), report);
|
||||
Files.writeString(markdownPath, renderMarkdown(report));
|
||||
}
|
||||
|
||||
private void createParentDirectories(Path path) throws IOException {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
}
|
||||
|
||||
private List<ComparableRetrievalResult> normalizeCurrentResults(List<VectorSearchService.SearchResult> rawResults) {
|
||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
||||
for (int i = 0; i < rawResults.size(); i++) {
|
||||
results.add(normalizer.fromCurrent(rawResults.get(i), i + 1, properties.getContentPreviewLimit()));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private List<String> compareDifferences(List<ComparableRetrievalResult> currentResults,
|
||||
List<ComparableRetrievalResult> sidecarResults) {
|
||||
if (sidecarResults == null || sidecarResults.isEmpty()) {
|
||||
return List.of("sidecar_unavailable_or_empty");
|
||||
}
|
||||
List<String> differences = new ArrayList<>();
|
||||
String currentTopSource = currentResults.isEmpty() ? null : currentResults.get(0).getSource();
|
||||
String sidecarTopSource = sidecarResults.get(0).getSource();
|
||||
if (!Objects.equals(currentTopSource, sidecarTopSource)) {
|
||||
differences.add("top_source_differs");
|
||||
}
|
||||
String currentTopBreadcrumb = currentResults.isEmpty() ? null : currentResults.get(0).getBreadcrumb();
|
||||
String sidecarTopBreadcrumb = sidecarResults.get(0).getBreadcrumb();
|
||||
if (!Objects.equals(currentTopBreadcrumb, sidecarTopBreadcrumb)) {
|
||||
differences.add("top_breadcrumb_differs");
|
||||
}
|
||||
String currentScoreLabel = currentResults.isEmpty() ? null : currentResults.get(0).getScoreLabel();
|
||||
String sidecarScoreLabel = sidecarResults.get(0).getScoreLabel();
|
||||
if (!Objects.equals(currentScoreLabel, sidecarScoreLabel)) {
|
||||
differences.add("score_label_differs");
|
||||
}
|
||||
return differences;
|
||||
}
|
||||
|
||||
private String renderMarkdown(RetrievalComparisonReport report) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("# RAG Sidecar Retrieval Comparison\n\n");
|
||||
builder.append("Generated at: `").append(report.getGeneratedAt()).append("`\n\n");
|
||||
builder.append("- Cases: ").append(report.getCaseCount()).append("\n");
|
||||
builder.append("- Top K: ").append(report.getTopK()).append("\n");
|
||||
builder.append("- Sidecar status: `").append(report.getSidecarStatus()).append("`\n\n");
|
||||
builder.append("| Case | Query | Current Top | Sidecar Top | Differences |\n");
|
||||
builder.append("|---|---|---|---|---|\n");
|
||||
for (RetrievalComparisonResult result : report.getResults()) {
|
||||
builder.append("| ")
|
||||
.append(nullToBlank(result.getCaseId()))
|
||||
.append(" | ")
|
||||
.append(escapePipe(result.getQuery()))
|
||||
.append(" | ")
|
||||
.append(formatTop(result.getCurrentResults()))
|
||||
.append(" | ")
|
||||
.append(formatTop(result.getSidecar() != null ? result.getSidecar().getResults() : List.of()))
|
||||
.append(" | ")
|
||||
.append(String.join("<br>", result.getDifferences()))
|
||||
.append(" |\n");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private String formatTop(List<ComparableRetrievalResult> results) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
ComparableRetrievalResult top = results.get(0);
|
||||
return escapePipe(nullToBlank(top.getSource())) + " (" + nullToBlank(top.getScoreLabel()) + ")";
|
||||
}
|
||||
|
||||
private String escapePipe(String value) {
|
||||
return nullToBlank(value).replace("|", "\\|");
|
||||
}
|
||||
|
||||
private String nullToBlank(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* RAG (Retrieval-Augmented Generation) 服务
|
||||
* 结合向量检索和大语言模型生成答案
|
||||
*/
|
||||
@Service
|
||||
public class RagService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RagService.class);
|
||||
|
||||
@Autowired
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Value("${rag.top-k:3}")
|
||||
private int topK;
|
||||
|
||||
/**
|
||||
* 流式处理用户问题（不带历史消息）
|
||||
*
|
||||
* @param question 用户问题
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
public void queryStream(String question, StreamCallback callback) {
|
||||
queryStream(question, new ArrayList<>(), callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式处理用户问题（带历史消息）
|
||||
*
|
||||
* @param question 用户问题
|
||||
* @param history 历史消息列表，格式：[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
public void queryStream(String question, List<Map<String, String>> history, StreamCallback callback) {
|
||||
try {
|
||||
logger.info("收到 RAG 流式查询: {}", question);
|
||||
|
||||
// 1. 从向量数据库检索相关文档
|
||||
List<VectorSearchService.SearchResult> searchResults =
|
||||
vectorSearchService.searchSimilarDocuments(question, topK);
|
||||
|
||||
// 发送检索结果
|
||||
callback.onSearchResults(searchResults);
|
||||
|
||||
if (searchResults.isEmpty()) {
|
||||
logger.warn("未找到相关文档");
|
||||
callback.onComplete("抱歉，我在知识库中没有找到相关信息来回答您的问题。", "");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 构建上下文和提示词
|
||||
String context = buildContext(searchResults);
|
||||
String prompt = buildPrompt(question, context);
|
||||
|
||||
// 3. 流式调用大语言模型（传入历史消息）
|
||||
generateAnswerStream(prompt, history, callback);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("RAG 流式查询失败", e);
|
||||
callback.onError(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建上下文
|
||||
*/
|
||||
private String buildContext(List<VectorSearchService.SearchResult> searchResults) {
|
||||
StringBuilder context = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < searchResults.size(); i++) {
|
||||
VectorSearchService.SearchResult result = searchResults.get(i);
|
||||
context.append("【参考资料 ").append(i + 1).append("】\n");
|
||||
context.append(result.getContent()).append("\n\n");
|
||||
}
|
||||
|
||||
return context.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建提示词
|
||||
*/
|
||||
private String buildPrompt(String question, String context) {
|
||||
return String.format(
|
||||
"你是一个专业的AI助手。请根据以下参考资料回答用户的问题。\n\n" +
|
||||
"参考资料：\n%s\n" +
|
||||
"用户问题：%s\n\n" +
|
||||
"请基于上述参考资料给出准确、详细的回答。如果参考资料中没有相关信息，请明确说明。",
|
||||
context, question
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成答案（流式）
|
||||
*
|
||||
* @param prompt 当前问题的提示词
|
||||
* @param history 历史消息列表
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
private void generateAnswerStream(String prompt, List<Map<String, String>> history, StreamCallback callback) {
|
||||
// 构建消息列表：历史消息 + 当前问题
|
||||
List<Message> messages = new ArrayList<>();
|
||||
|
||||
// 添加历史消息
|
||||
for (Map<String, String> historyMsg : history) {
|
||||
String role = historyMsg.get("role");
|
||||
String content = historyMsg.get("content");
|
||||
|
||||
if ("user".equals(role)) {
|
||||
messages.add(new UserMessage(content));
|
||||
} else if ("assistant".equals(role)) {
|
||||
messages.add(new AssistantMessage(content));
|
||||
}
|
||||
}
|
||||
|
||||
// 添加当前用户问题
|
||||
messages.add(new UserMessage(prompt));
|
||||
|
||||
logger.debug("发送给AI模型的消息数量: {}（包含 {} 条历史消息）",
|
||||
messages.size(), history.size());
|
||||
|
||||
logger.info("开始调用AI模型流式接口...");
|
||||
|
||||
StringBuilder reasoningContent = new StringBuilder();
|
||||
StringBuilder finalContent = new StringBuilder();
|
||||
|
||||
Flux<ChatResponse> flux = chatModel.stream(new Prompt(messages));
|
||||
|
||||
logger.info("开始接收AI模型流式响应...");
|
||||
|
||||
flux.subscribe(
|
||||
response -> {
|
||||
if (response.getResults() != null && !response.getResults().isEmpty()) {
|
||||
String content = response.getResults().get(0).getOutput().getText();
|
||||
|
||||
if (content != null && !content.isEmpty()) {
|
||||
logger.debug("收到AI模型内容块: {}", content);
|
||||
|
||||
finalContent.append(content);
|
||||
callback.onContentChunk(content);
|
||||
|
||||
logger.debug("已调用 onContentChunk 回调");
|
||||
} else {
|
||||
logger.debug("收到空内容块，跳过");
|
||||
}
|
||||
}
|
||||
},
|
||||
error -> {
|
||||
logger.error("AI模型流式响应失败", error);
|
||||
callback.onError(new Exception("AI模型流式响应失败: " + error.getMessage(), error));
|
||||
},
|
||||
() -> {
|
||||
logger.info("AI模型流式响应完成，总内容长度: {}", finalContent.length());
|
||||
callback.onComplete(finalContent.toString(), reasoningContent.toString());
|
||||
logger.info("已调用 onComplete 回调");
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式回调接口
|
||||
*/
|
||||
public interface StreamCallback {
|
||||
void onSearchResults(List<VectorSearchService.SearchResult> results);
|
||||
void onReasoningChunk(String chunk);
|
||||
void onContentChunk(String chunk);
|
||||
void onComplete(String fullContent, String fullReasoning);
|
||||
void onError(Exception e);
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class RetrievalResultNormalizer {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public RetrievalResultNormalizer(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public ComparableRetrievalResult fromCurrent(VectorSearchService.SearchResult result, int rank, int previewLimit) {
|
||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), result.getMetadata(), result.getId());
|
||||
return ComparableRetrievalResult.builder()
|
||||
.path("current")
|
||||
.rank(rank)
|
||||
.id(result.getId())
|
||||
.source(source)
|
||||
.docId(metadata.get("docId"))
|
||||
.title(metadata.get("title"))
|
||||
.breadcrumb(metadata.get("breadcrumb"))
|
||||
.category(metadata.get("category"))
|
||||
.contentPreview(truncate(result.getContent(), previewLimit))
|
||||
.scoreLabel("l2_distance")
|
||||
.scoreValue((double) result.getScore())
|
||||
.build();
|
||||
}
|
||||
|
||||
public ComparableRetrievalResult fromSidecar(Document document, int rank, int previewLimit) {
|
||||
Map<String, String> metadata = stringifyMetadata(document.getMetadata());
|
||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), metadata.get("docId"), document.getId());
|
||||
return ComparableRetrievalResult.builder()
|
||||
.path("sidecar")
|
||||
.rank(rank)
|
||||
.id(document.getId())
|
||||
.source(source)
|
||||
.docId(metadata.get("docId"))
|
||||
.title(metadata.get("title"))
|
||||
.breadcrumb(metadata.get("breadcrumb"))
|
||||
.category(metadata.get("category"))
|
||||
.contentPreview(truncate(document.getText(), previewLimit))
|
||||
.scoreLabel("similarity")
|
||||
.scoreValue(document.getScore())
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, String> parseMetadata(String metadata) {
|
||||
if (metadata == null || metadata.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
||||
return stringifyMetadata(raw);
|
||||
} catch (Exception e) {
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> stringifyMetadata(Map<?, ?> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
result.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String truncate(String text, int maxLength) {
|
||||
if (text == null || text.length() <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
return text.substring(0, maxLength) + "...";
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SpringAiVectorStoreSidecarService {
|
||||
|
||||
private final RagSidecarProperties properties;
|
||||
private final ObjectProvider<VectorStore> vectorStoreProvider;
|
||||
private final RetrievalResultNormalizer normalizer;
|
||||
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
public SpringAiVectorStoreSidecarService(RagSidecarProperties properties,
|
||||
ObjectProvider<VectorStore> vectorStoreProvider,
|
||||
RetrievalResultNormalizer normalizer) {
|
||||
this.properties = properties;
|
||||
this.vectorStoreProvider = vectorStoreProvider;
|
||||
this.normalizer = normalizer;
|
||||
}
|
||||
|
||||
public SidecarRetrievalResponse search(String query, int topK, String category) {
|
||||
if (!properties.isEnabled()) {
|
||||
return unavailable("disabled", null);
|
||||
}
|
||||
|
||||
VectorStore vectorStore = vectorStoreProvider.getIfAvailable();
|
||||
if (vectorStore == null) {
|
||||
return unavailable("missing_vector_store", "No Spring AI VectorStore bean is available");
|
||||
}
|
||||
|
||||
try {
|
||||
SearchRequest.Builder builder = SearchRequest.builder()
|
||||
.query(query)
|
||||
.topK(topK)
|
||||
.similarityThresholdAll();
|
||||
String filterExpression = buildFilterExpression(category);
|
||||
if (filterExpression != null) {
|
||||
builder.filterExpression(filterExpression);
|
||||
}
|
||||
|
||||
List<Document> documents = vectorStore.similaritySearch(builder.build());
|
||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
||||
for (int i = 0; i < documents.size(); i++) {
|
||||
results.add(normalizer.fromSidecar(documents.get(i), i + 1, properties.getContentPreviewLimit()));
|
||||
}
|
||||
return SidecarRetrievalResponse.builder()
|
||||
.enabled(true)
|
||||
.available(true)
|
||||
.status("available")
|
||||
.results(results)
|
||||
.build();
|
||||
} catch (Exception e) {
|
||||
log.warn("Spring AI sidecar retrieval failed: {}", e.getMessage());
|
||||
return unavailable("query_failed", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private SidecarRetrievalResponse unavailable(String status, String errorMessage) {
|
||||
return SidecarRetrievalResponse.builder()
|
||||
.enabled(properties.isEnabled())
|
||||
.available(false)
|
||||
.status(status)
|
||||
.errorMessage(errorMessage)
|
||||
.results(List.of())
|
||||
.build();
|
||||
}
|
||||
|
||||
private String escapeFilterValue(String value) {
|
||||
return value.replace("'", "\\'");
|
||||
}
|
||||
|
||||
String buildFilterExpression(String category) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
String categoryFilter = trimToNull(category);
|
||||
if (categoryFilter != null) {
|
||||
parts.add("category == '" + escapeFilterValue(categoryFilter) + "'");
|
||||
}
|
||||
String scopeFilter = trimToNull(kbScope);
|
||||
if (scopeFilter != null) {
|
||||
parts.add("kb_scope == '" + escapeFilterValue(scopeFilter) + "'");
|
||||
}
|
||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.exception.DocumentProcessException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 文本提取服务
|
||||
* 仅支持 Markdown (.md) 和纯文本 (.txt) 格式
|
||||
* 其他格式（.docx、.pdf 等）需要通过外部转换服务先转为 Markdown
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TextExtractorService {
|
||||
|
||||
/**
|
||||
* 从文件中提取文本
|
||||
*
|
||||
* @param file 上传的文件
|
||||
* @param fileName 文件名
|
||||
* @return 提取的文本内容
|
||||
*/
|
||||
public String extractText(MultipartFile file, String fileName) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new DocumentProcessException(fileName, "extract", "文件为空");
|
||||
}
|
||||
|
||||
String extension = getFileExtension(fileName);
|
||||
log.info("开始提取文本，文件名: {}, 格式: {}, 大小: {} bytes", fileName, extension, file.getSize());
|
||||
|
||||
if (!isSupportedFormat(fileName)) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "extract",
|
||||
"不支持的文件格式: " + extension + "，仅支持 .md 和 .txt。其他格式请先通过转换服务转为 Markdown。"
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
String text = extractPlainText(file);
|
||||
log.info("文本提取成功，文件名: {}, 提取字符数: {}", fileName, text.length());
|
||||
return text;
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("文本提取失败，文件名: {}", fileName, e);
|
||||
throw new DocumentProcessException(fileName, "extract", "文件读取失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取纯文本（.txt、.md）
|
||||
*/
|
||||
private String extractPlainText(MultipartFile file) throws IOException {
|
||||
StringBuilder content = new StringBuilder();
|
||||
try (InputStream is = file.getInputStream();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
|
||||
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
content.append(line).append("\n");
|
||||
}
|
||||
}
|
||||
return content.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件扩展名
|
||||
*/
|
||||
private String getFileExtension(String fileName) {
|
||||
if (fileName == null || !fileName.contains(".")) {
|
||||
return "";
|
||||
}
|
||||
return fileName.substring(fileName.lastIndexOf(".") + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文件格式是否支持
|
||||
*/
|
||||
public boolean isSupportedFormat(String fileName) {
|
||||
String extension = getFileExtension(fileName).toLowerCase();
|
||||
return extension.equals("md") || extension.equals("txt");
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 向量嵌入服务
|
||||
* 使用阿里云 DashScope Text Embedding API
|
||||
*/
|
||||
@Service
|
||||
public class VectorEmbeddingService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorEmbeddingService.class);
|
||||
|
||||
@Autowired
|
||||
private EmbeddingModel embeddingModel;
|
||||
|
||||
/**
|
||||
* 生成向量嵌入
|
||||
* 调用阿里云 DashScope Text Embedding API
|
||||
*
|
||||
* @param content 文本内容
|
||||
* @return 向量嵌入（浮点数列表）
|
||||
*/
|
||||
public List<Float> generateEmbedding(String content) {
|
||||
try {
|
||||
if (content == null || content.trim().isEmpty()) {
|
||||
logger.warn("内容为空，无法生成向量");
|
||||
throw new IllegalArgumentException("内容不能为空");
|
||||
}
|
||||
|
||||
logger.debug("开始生成向量嵌入, 内容长度: {} 字符", content.length());
|
||||
|
||||
float[] embedding = embeddingModel.embed(content);
|
||||
|
||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
||||
for (float v : embedding) {
|
||||
floatEmbedding.add(v);
|
||||
}
|
||||
|
||||
logger.info("成功生成向量嵌入, 内容长度: {} 字符, 向量维度: {}",
|
||||
content.length(), floatEmbedding.size());
|
||||
|
||||
return floatEmbedding;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("生成向量嵌入失败, 内容长度: {}", content != null ? content.length() : 0, e);
|
||||
throw new RuntimeException("生成向量嵌入失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public List<List<Float>> generateEmbeddings(List<String> contents) {
|
||||
try {
|
||||
if (contents == null || contents.isEmpty()) {
|
||||
logger.warn("内容列表为空，无法生成向量");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
logger.info("开始批量生成向量嵌入, 数量: {}", contents.size());
|
||||
|
||||
List<float[]> embeddings = embeddingModel.embed(contents);
|
||||
|
||||
List<List<Float>> result = new ArrayList<>();
|
||||
for (float[] embedding : embeddings) {
|
||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
||||
for (float v : embedding) {
|
||||
floatEmbedding.add(v);
|
||||
}
|
||||
result.add(floatEmbedding);
|
||||
}
|
||||
|
||||
logger.info("成功批量生成向量嵌入, 数量: {}, 维度: {}",
|
||||
result.size(),
|
||||
result.isEmpty() ? 0 : result.get(0).size());
|
||||
|
||||
return result;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("批量生成向量嵌入失败", e);
|
||||
throw new RuntimeException("批量生成向量嵌入失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成查询向量
|
||||
*
|
||||
* @param query 查询文本
|
||||
* @return 向量嵌入
|
||||
*/
|
||||
public List<Float> generateQueryVector(String query) {
|
||||
return generateEmbedding(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算两个向量的余弦相似度
|
||||
*
|
||||
* @param vector1 向量1
|
||||
* @param vector2 向量2
|
||||
* @return 余弦相似度 [-1, 1]
|
||||
*/
|
||||
public float calculateCosineSimilarity(List<Float> vector1, List<Float> vector2) {
|
||||
if (vector1.size() != vector2.size()) {
|
||||
throw new IllegalArgumentException("向量维度不匹配");
|
||||
}
|
||||
|
||||
float dotProduct = 0.0f;
|
||||
float norm1 = 0.0f;
|
||||
float norm2 = 0.0f;
|
||||
|
||||
for (int i = 0; i < vector1.size(); i++) {
|
||||
dotProduct += vector1.get(i) * vector2.get(i);
|
||||
norm1 += vector1.get(i) * vector1.get(i);
|
||||
norm2 += vector2.get(i) * vector2.get(i);
|
||||
}
|
||||
|
||||
return dotProduct / (float) (Math.sqrt(norm1) * Math.sqrt(norm2));
|
||||
}
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 向量索引写入服务（RAG 入库侧）。
|
||||
*
|
||||
* <p>唯一后端 {@link MilvusHybridKnowledgeStore}（Milvus SDK v2）：</p>
|
||||
* <ul>
|
||||
* <li>dense：应用侧 embedding → 字段 {@code vector}</li>
|
||||
* <li>BM25：{@link #buildSearchText} → 字段 {@code search_text}；
|
||||
* sparse 由 collection 上 BM25 Function 自动生成，本类不写 sparse</li>
|
||||
* </ul>
|
||||
* <p>不再使用 legacy {@code MilvusServiceClient} insert/delete，
|
||||
* 也不走 Spring AI {@code VectorStore#add}（starter 无 hybrid schema/BM25 Function）。</p>
|
||||
*/
|
||||
@Service
|
||||
public class VectorIndexService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorIndexService.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService embeddingService;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService chunkService;
|
||||
|
||||
@Value("${file.upload.path}")
|
||||
private String uploadPath;
|
||||
|
||||
public IndexingResult indexDirectory(String directoryPath) {
|
||||
IndexingResult result = new IndexingResult();
|
||||
result.setStartTime(LocalDateTime.now());
|
||||
|
||||
try {
|
||||
String targetPath = (directoryPath != null && !directoryPath.trim().isEmpty())
|
||||
? directoryPath : uploadPath;
|
||||
|
||||
Path dirPath = Paths.get(targetPath).normalize();
|
||||
File directory = dirPath.toFile();
|
||||
|
||||
if (!directory.exists() || !directory.isDirectory()) {
|
||||
throw new IllegalArgumentException("目录不存在或不是有效目录: " + targetPath);
|
||||
}
|
||||
|
||||
result.setDirectoryPath(directory.getAbsolutePath());
|
||||
|
||||
File[] files = directory.listFiles((dir, name) ->
|
||||
name.endsWith(".txt") || name.endsWith(".md")
|
||||
);
|
||||
|
||||
if (files == null || files.length == 0) {
|
||||
logger.warn("目录中没有找到支持的文件: {}", targetPath);
|
||||
result.setTotalFiles(0);
|
||||
result.setSuccess(true);
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
}
|
||||
|
||||
result.setTotalFiles(files.length);
|
||||
logger.info("开始索引目录: {}, 找到 {} 个文件", targetPath, files.length);
|
||||
|
||||
for (File file : files) {
|
||||
try {
|
||||
indexSingleFile(file.getAbsolutePath());
|
||||
result.incrementSuccessCount();
|
||||
logger.info("文件索引成功: {}", file.getName());
|
||||
} catch (Exception e) {
|
||||
result.incrementFailCount();
|
||||
result.addFailedFile(file.getAbsolutePath(), e.getMessage());
|
||||
logger.error("文件索引失败: {}", file.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
result.setSuccess(result.getFailCount() == 0);
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("索引目录失败", e);
|
||||
result.setSuccess(false);
|
||||
result.setErrorMessage(e.getMessage());
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public void indexSingleFile(String filePath) throws Exception {
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
File file = path.toFile();
|
||||
|
||||
if (!file.exists() || !file.isFile()) {
|
||||
throw new IllegalArgumentException("文件不存在: " + filePath);
|
||||
}
|
||||
|
||||
logger.info("开始索引文件: {}", path);
|
||||
String content = Files.readString(path);
|
||||
deleteExistingData(path.toString());
|
||||
|
||||
List<DocumentChunk> chunks = chunkService.chunkDocument(content, path.toString());
|
||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
DocumentChunk chunk = chunks.get(i);
|
||||
try {
|
||||
// dense embedding 与 BM25 search_text 同源（title/path 增强）
|
||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
||||
Map<String, Object> metadata = buildMetadata(path.toString(), chunk, chunks.size());
|
||||
knowledgeStore.upsertChunk(
|
||||
chunk.getContent(), // 返回原文
|
||||
buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
|
||||
vector, // dense 向量
|
||||
metadata,
|
||||
chunk.getChunkIndex());
|
||||
logger.info("分片 {}/{} 索引成功", i + 1, chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("分片 {}/{} 索引失败", i + 1, chunks.size(), e);
|
||||
throw new RuntimeException("分片索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
|
||||
}
|
||||
|
||||
public void indexDocumentChunks(String docId, List<DocumentChunk> chunks, String category) throws Exception {
|
||||
indexDocumentChunks(docId, chunks, category, null);
|
||||
}
|
||||
|
||||
public void indexDocumentChunks(String docId,
|
||||
List<DocumentChunk> chunks,
|
||||
String category,
|
||||
Frontmatter frontmatter) throws Exception {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
throw new IllegalArgumentException("文档分块列表为空");
|
||||
}
|
||||
|
||||
logger.info("开始索引文档分块，docId: {}, 分块数: {}, 类别: {}", docId, chunks.size(), category);
|
||||
deleteDocumentChunks(docId);
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
DocumentChunk chunk = chunks.get(i);
|
||||
try {
|
||||
// dense embedding 与 BM25 search_text 同源（title/path 增强）
|
||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
||||
Map<String, Object> metadata = buildDocumentMetadata(docId, chunk, chunks.size(), category, frontmatter);
|
||||
knowledgeStore.upsertChunk(
|
||||
chunk.getContent(), // 返回原文
|
||||
buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
|
||||
vector, // dense 向量
|
||||
metadata,
|
||||
chunk.getChunkIndex());
|
||||
logger.info("文档分块 {}/{} 索引成功，docId: {}", i + 1, chunks.size(), docId);
|
||||
} catch (Exception e) {
|
||||
logger.error("文档分块 {}/{} 索引失败，docId: {}", i + 1, chunks.size(), docId, e);
|
||||
throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
logger.info("文档索引完成，docId: {}, 共 {} 个分块，类别: {}", docId, chunks.size(), category);
|
||||
}
|
||||
|
||||
public void deleteDocumentChunks(String docId) {
|
||||
try {
|
||||
knowledgeStore.deleteByDocId(docId);
|
||||
logger.info("删除文档旧数据成功，docId: {}", docId);
|
||||
} catch (Exception e) {
|
||||
logger.warn("删除文档旧数据异常，docId: {}", docId, e);
|
||||
}
|
||||
}
|
||||
|
||||
static Map<String, Object> buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks, String category) {
|
||||
return buildDocumentMetadata(docId, chunk, totalChunks, category, null);
|
||||
}
|
||||
|
||||
static Map<String, Object> buildDocumentMetadata(String docId,
|
||||
DocumentChunk chunk,
|
||||
int totalChunks,
|
||||
String category,
|
||||
Frontmatter frontmatter) {
|
||||
Map<String, Object> metadata = new HashMap<>();
|
||||
String source = firstNonBlank(frontmatter != null ? frontmatter.getSource() : null, "upload:" + docId);
|
||||
metadata.put("docId", docId);
|
||||
metadata.put("_source", source);
|
||||
metadata.put("source", source);
|
||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||
metadata.put("totalChunks", totalChunks);
|
||||
|
||||
String title = firstNonBlank(chunk.getTitle(), frontmatter != null ? frontmatter.getTitle() : null);
|
||||
if (title != null) {
|
||||
metadata.put("title", title);
|
||||
}
|
||||
String breadcrumb = firstNonBlank(frontmatter != null ? frontmatter.getBreadcrumb() : null, chunk.getBreadcrumb());
|
||||
if (breadcrumb != null) {
|
||||
metadata.put("breadcrumb", breadcrumb);
|
||||
}
|
||||
metadata.put("category", category != null && !category.isBlank() ? category : "upload");
|
||||
String kbScope = trimToNull(frontmatter != null ? frontmatter.getKbScope() : null);
|
||||
if (kbScope != null) {
|
||||
metadata.put("kb_scope", kbScope);
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dense embedding 输入。与 {@link #buildSearchText} 同源，保证 dense/BM25 看到同一增强文本。
|
||||
*/
|
||||
static String buildEmbeddingText(DocumentChunk chunk) {
|
||||
return buildSearchText(chunk);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造写入 Milvus 的检索文本（BM25 {@code search_text}，并复用为 dense embedding 输入）。
|
||||
*
|
||||
* <p>在正文前拼接 title / breadcrumb，提高「按标题或路径关键词」的 BM25 命中率，
|
||||
* 同时让 dense 向量也编码结构信息。无标题路径时退回纯 content。</p>
|
||||
*/
|
||||
static String buildSearchText(DocumentChunk chunk) {
|
||||
String content = trimToEmpty(chunk.getContent());
|
||||
String title = trimToEmpty(chunk.getTitle());
|
||||
String breadcrumb = trimToEmpty(chunk.getBreadcrumb());
|
||||
|
||||
if (title.isEmpty() && breadcrumb.isEmpty()) {
|
||||
return content;
|
||||
}
|
||||
|
||||
StringBuilder text = new StringBuilder();
|
||||
if (!title.isEmpty()) {
|
||||
text.append("Title: ").append(title).append("\n");
|
||||
}
|
||||
if (!breadcrumb.isEmpty()) {
|
||||
text.append("Path: ").append(breadcrumb).append("\n");
|
||||
}
|
||||
text.append("Content:\n").append(content);
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
private static String trimToEmpty(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
private static String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
String trimmed = trimToNull(value);
|
||||
if (trimmed != null) {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void deleteExistingData(String filePath) {
|
||||
try {
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
||||
knowledgeStore.deleteBySource(normalizedPath);
|
||||
logger.info("已删除文件的旧数据: {}", normalizedPath);
|
||||
} catch (Exception e) {
|
||||
logger.warn("删除旧数据失败（可能是首次索引）: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> buildMetadata(String filePath, DocumentChunk chunk, int totalChunks) {
|
||||
Map<String, Object> metadata = new HashMap<>();
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
||||
|
||||
Path fileName = path.getFileName();
|
||||
String fileNameStr = fileName != null ? fileName.toString() : "";
|
||||
String extension = "";
|
||||
int dotIndex = fileNameStr.lastIndexOf('.');
|
||||
if (dotIndex > 0) {
|
||||
extension = fileNameStr.substring(dotIndex);
|
||||
}
|
||||
|
||||
metadata.put("_source", normalizedPath);
|
||||
metadata.put("source", normalizedPath);
|
||||
metadata.put("_extension", extension);
|
||||
metadata.put("_file_name", fileNameStr);
|
||||
|
||||
String category = extractCategory(normalizedPath);
|
||||
if (category != null && !category.isEmpty()) {
|
||||
metadata.put("category", category);
|
||||
}
|
||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||
metadata.put("totalChunks", totalChunks);
|
||||
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
|
||||
metadata.put("title", chunk.getTitle());
|
||||
}
|
||||
if (chunk.getBreadcrumb() != null && !chunk.getBreadcrumb().isEmpty()) {
|
||||
metadata.put("breadcrumb", chunk.getBreadcrumb());
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private String extractCategory(String filePath) {
|
||||
try {
|
||||
String normalized = filePath.replace("\\", "/");
|
||||
int docsIndex = normalized.indexOf("aiops-docs/");
|
||||
if (docsIndex >= 0) {
|
||||
String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
|
||||
int slashIndex = afterDocs.indexOf("/");
|
||||
if (slashIndex > 0) {
|
||||
return afterDocs.substring(0, slashIndex);
|
||||
}
|
||||
}
|
||||
int firstSlash = normalized.indexOf("/");
|
||||
if (firstSlash > 0) {
|
||||
return normalized.substring(0, firstSlash);
|
||||
}
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
logger.warn("提取类别失败，路径: {}", filePath, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class IndexingResult {
|
||||
@Setter
|
||||
private boolean success;
|
||||
@Setter
|
||||
private String directoryPath;
|
||||
@Setter
|
||||
private int totalFiles;
|
||||
private int successCount;
|
||||
private int failCount;
|
||||
@Setter
|
||||
private LocalDateTime startTime;
|
||||
@Setter
|
||||
private LocalDateTime endTime;
|
||||
@Setter
|
||||
private String errorMessage;
|
||||
private Map<String, String> failedFiles = new HashMap<>();
|
||||
|
||||
public void incrementSuccessCount() {
|
||||
this.successCount++;
|
||||
}
|
||||
|
||||
public void incrementFailCount() {
|
||||
this.failCount++;
|
||||
}
|
||||
|
||||
public long getDurationMs() {
|
||||
if (startTime != null && endTime != null) {
|
||||
return java.time.Duration.between(startTime, endTime).toMillis();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public void addFailedFile(String filePath, String error) {
|
||||
this.failedFiles.put(filePath, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* 知识库向量检索门面（lookup_knowledge / RAG 召回入口）。
|
||||
*
|
||||
* <p><b>唯一后端：</b>{@link MilvusHybridKnowledgeStore}（Milvus Java SDK v2）。</p>
|
||||
*
|
||||
* <h3>模式切换</h3>
|
||||
* <p>{@code retrieval.search.mode}（同库查询算法，非两套写入）：</p>
|
||||
* <ul>
|
||||
* <li>{@code hybrid} —— 线上主路径：dense + 服务端 BM25 + RRF</li>
|
||||
* <li>{@code dense} —— 对照/评测：仅 dense ANN</li>
|
||||
* </ul>
|
||||
* <p>命中 {@link SearchResult#scoreLabel} 仅为 {@link RetrievalScoreLabels#DENSE} /
|
||||
* {@link RetrievalScoreLabels#HYBRID}。质量分由后处理 {@code RetrievalScoreNormalizer} 统一计算。</p>
|
||||
*/
|
||||
@Service
|
||||
public class VectorSearchService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorSearchService.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService embeddingService;
|
||||
|
||||
/**
|
||||
* 检索模式：{@code hybrid}（主路径）| {@code dense}（召回对照）。
|
||||
*/
|
||||
@Value("${retrieval.search.mode:dense}")
|
||||
private String searchMode = "dense";
|
||||
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK) {
|
||||
return searchSimilarDocuments(query, topK, null);
|
||||
}
|
||||
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK, String category) {
|
||||
String mode = searchMode == null ? "dense" : searchMode.trim().toLowerCase(Locale.ROOT);
|
||||
List<Float> queryVector = embeddingService.generateQueryVector(query);
|
||||
if ("hybrid".equals(mode)) {
|
||||
logger.info("Hybrid dense+BM25 search topK={} category={} collection={}",
|
||||
topK, category, knowledgeStore.collectionName());
|
||||
return knowledgeStore.searchHybrid(query, queryVector, topK, category);
|
||||
}
|
||||
logger.info("Dense search topK={} category={} collection={}",
|
||||
topK, category, knowledgeStore.collectionName());
|
||||
return knowledgeStore.searchDense(query, queryVector, topK, category);
|
||||
}
|
||||
|
||||
/**
|
||||
* 单条召回结果。列表顺序即检索权威序（adapter 赋 originalRank=1..n）。
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code scoreLabel=dense}：{@link #score} = L2 距离（越小越好）</li>
|
||||
* <li>{@code scoreLabel=hybrid}：{@link #score}/{@link #rawScore} = 引擎融合分；
|
||||
* 后处理 quality 主要按 rank 映射，不把 score 当 L2</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Setter
|
||||
@Getter
|
||||
public static class SearchResult {
|
||||
private String id;
|
||||
private String content;
|
||||
/**
|
||||
* 引擎主分：dense=L2；hybrid=融合分（量纲由 scoreLabel 解释）。
|
||||
*/
|
||||
private float score;
|
||||
/** 引擎原始分（与 score 同源或更细，便于调试）。 */
|
||||
private Double rawScore;
|
||||
/** {@link RetrievalScoreLabels#DENSE} 或 {@link RetrievalScoreLabels#HYBRID}。 */
|
||||
private String scoreLabel;
|
||||
/**
|
||||
* Optional dense L2 for the same id (hybrid path only).
|
||||
* Used for absolute quality / low-quality gates; does <b>not</b> replace sort order.
|
||||
*/
|
||||
private Double denseDistance;
|
||||
/** metadata JSON 字符串（docId、source、title…）。 */
|
||||
private String metadata;
|
||||
}
|
||||
}
|
||||
@@ -1,547 +0,0 @@
|
||||
package com.superbiz.agent.service.milvus;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.superbiz.agent.config.MilvusProperties;
|
||||
import com.superbiz.agent.constant.MilvusConstants;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import io.milvus.common.clientenum.FunctionType;
|
||||
import io.milvus.v2.client.ConnectConfig;
|
||||
import io.milvus.v2.client.MilvusClientV2;
|
||||
import io.milvus.v2.common.DataType;
|
||||
import io.milvus.v2.common.IndexParam;
|
||||
import io.milvus.v2.service.collection.request.AddFieldReq;
|
||||
import io.milvus.v2.service.collection.request.CreateCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.DropCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.HasCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.LoadCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
|
||||
import io.milvus.v2.service.index.request.CreateIndexReq;
|
||||
import io.milvus.v2.service.vector.request.AnnSearchReq;
|
||||
import io.milvus.v2.service.vector.request.DeleteReq;
|
||||
import io.milvus.v2.service.vector.request.HybridSearchReq;
|
||||
import io.milvus.v2.service.vector.request.InsertReq;
|
||||
import io.milvus.v2.service.vector.request.SearchReq;
|
||||
import io.milvus.v2.service.vector.request.data.BaseVector;
|
||||
import io.milvus.v2.service.vector.request.data.EmbeddedText;
|
||||
import io.milvus.v2.service.vector.request.data.FloatVec;
|
||||
import io.milvus.v2.service.vector.request.ranker.RRFRanker;
|
||||
import io.milvus.v2.service.vector.response.SearchResp;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* 知识库向量后端（Milvus Java SDK v2）—— dense + BM25 混合检索的唯一实现。
|
||||
*
|
||||
* <h3>为什么不用 Spring AI {@code spring-ai-starter-vector-store-milvus}</h3>
|
||||
* <ul>
|
||||
* <li>Spring AI Milvus starter（截至 2.0.0 / 1.1.8）只封装 dense {@code similaritySearch}。</li>
|
||||
* <li>底层仍是 V1 {@code MilvusServiceClient} + 单路 {@code SearchParam}，无 {@code hybridSearch} /
|
||||
* BM25 Function / {@link RRFRanker}。</li>
|
||||
* <li>真混合检索（dense ANN + 服务端 BM25 sparse，再 RRF 融合）必须走 Milvus SDK v2，
|
||||
* 见 {@link #searchHybrid}。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <h3>Collection schema（默认名 {@code biz}）</h3>
|
||||
* <pre>
|
||||
* id VarChar PK
|
||||
* content VarChar —— 原文，返回给上层
|
||||
* search_text VarChar+analyzer —— BM25 输入文本（可含 title/path 增强）
|
||||
* sparse_vector SparseFloatVector —— 由 BM25 Function 从 search_text 自动生成，写入时不必填
|
||||
* vector FloatVector —— dense 向量（应用侧 embedding）
|
||||
* metadata JSON —— docId / source / category / kb_scope 等
|
||||
* </pre>
|
||||
*
|
||||
* <h3>检索模式</h3>
|
||||
* <ul>
|
||||
* <li>{@link #searchDense}：单路 L2 ANN；{@code scoreLabel=dense}。</li>
|
||||
* <li>{@link #searchHybrid}：dense + BM25 + 服务端 {@link RRFRanker}；{@code scoreLabel=hybrid}；
|
||||
* 返回序即 RRF 序，不再用 dense L2 覆盖主分。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>配置入口：{@code milvus.collection}、{@code retrieval.search.mode}、{@code retrieval.hybrid.rrf-k}。</p>
|
||||
*/
|
||||
@Service
|
||||
public class MilvusHybridKnowledgeStore {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MilvusHybridKnowledgeStore.class);
|
||||
private static final Gson GSON = new Gson();
|
||||
|
||||
/** 主键（稳定 UUID，由 source + chunkIndex 派生，便于幂等重写）。 */
|
||||
public static final String FIELD_ID = "id";
|
||||
/** 返回给 LLM / 上层的原文 chunk。 */
|
||||
public static final String FIELD_CONTENT = "content";
|
||||
/**
|
||||
* BM25 输入字段。写入明文；Milvus 侧 analyzer + BM25 Function 生成 {@link #FIELD_SPARSE}。
|
||||
* 通常比 content 多带 title/path 等检索增强词。
|
||||
*/
|
||||
public static final String FIELD_SEARCH_TEXT = "search_text";
|
||||
/** 稀疏向量字段；由 BM25 Function 自动产出，insert 时不要手动填。 */
|
||||
public static final String FIELD_SPARSE = "sparse_vector";
|
||||
/** Dense 向量字段（应用侧 EmbeddingModel 生成）。 */
|
||||
public static final String FIELD_DENSE = "vector";
|
||||
/** 业务元数据 JSON（过滤、证据身份、展示用）。 */
|
||||
public static final String FIELD_METADATA = "metadata";
|
||||
|
||||
private final MilvusProperties milvusProperties;
|
||||
|
||||
@Value("${milvus.collection:biz}")
|
||||
private String collectionName = "biz";
|
||||
|
||||
/**
|
||||
* RRF 平滑参数 k：score(d) = Σ 1/(k + rank_i(d))。
|
||||
* k 越大，各路排名差异被压得越平；默认 60 与常见 RRF 设定一致。
|
||||
*/
|
||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
||||
private int rrfK = 60;
|
||||
|
||||
/** 非空时追加 {@code metadata.kb_scope} 过滤，实现多知识域隔离。 */
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
private volatile MilvusClientV2 client;
|
||||
|
||||
public MilvusHybridKnowledgeStore(MilvusProperties milvusProperties) {
|
||||
this.milvusProperties = milvusProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 懒连接：首次调用时建连、确保 collection schema 存在并 load。
|
||||
* 线程安全；后续检索/写入复用同一 {@link MilvusClientV2}。
|
||||
*/
|
||||
public synchronized MilvusClientV2 client() {
|
||||
if (client == null) {
|
||||
client = connect();
|
||||
ensureCollection(client);
|
||||
loadCollection(client);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
public String collectionName() {
|
||||
return collectionName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入单个 chunk（dense + BM25 所需明文）。
|
||||
*
|
||||
* <p>只插入 {@code content / search_text / vector / metadata}；
|
||||
* {@code sparse_vector} 由 collection 上的 BM25 Function 在服务端从 {@code search_text} 生成。</p>
|
||||
*
|
||||
* <p>id 由 {@code source|docId + chunkIndex} 的 nameUUID 派生，同一 chunk 重复写入会得到相同 id
|
||||
*（配合先 delete 再 insert 的上层逻辑实现覆盖）。</p>
|
||||
*
|
||||
* @param content 原文（返回字段）
|
||||
* @param searchText BM25 / 可与 dense embedding 同源的检索文本
|
||||
* @param denseVector 应用侧 embedding
|
||||
* @param metadata 须尽量带 {@code _source} 或 {@code docId}，供 id 与过滤使用
|
||||
* @param chunkIndex 分片序号
|
||||
*/
|
||||
public void upsertChunk(String content,
|
||||
String searchText,
|
||||
List<Float> denseVector,
|
||||
Map<String, Object> metadata,
|
||||
int chunkIndex) {
|
||||
String source = metadata == null ? null : stringVal(metadata.get("_source"));
|
||||
if (source == null) {
|
||||
source = metadata == null ? null : stringVal(metadata.get("source"));
|
||||
}
|
||||
if (source == null) {
|
||||
source = metadata == null ? null : stringVal(metadata.get("docId"));
|
||||
}
|
||||
String idSeed = (source == null ? "chunk" : source) + "_" + chunkIndex;
|
||||
String id = UUID.nameUUIDFromBytes(idSeed.getBytes()).toString();
|
||||
|
||||
JsonObject row = new JsonObject();
|
||||
row.addProperty(FIELD_ID, id);
|
||||
row.addProperty(FIELD_CONTENT, content == null ? "" : content);
|
||||
// 仅写明文；sparse 由 BM25 Function(search_text -> sparse_vector) 自动生成
|
||||
row.addProperty(FIELD_SEARCH_TEXT, searchText == null ? "" : searchText);
|
||||
row.add(FIELD_DENSE, GSON.toJsonTree(denseVector));
|
||||
row.add(FIELD_METADATA, GSON.toJsonTree(metadata == null ? Map.of() : metadata));
|
||||
|
||||
client().insert(InsertReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.data(List.of(row))
|
||||
.build());
|
||||
}
|
||||
|
||||
/** 按 metadata.docId 删除该文档全部 chunk（重建/覆盖前调用）。 */
|
||||
public void deleteByDocId(String docId) {
|
||||
if (docId == null || docId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String filter = "metadata[\"docId\"] == \"" + escapeFilter(docId) + "\"";
|
||||
client().delete(DeleteReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.filter(filter)
|
||||
.build());
|
||||
}
|
||||
|
||||
/** 按 metadata._source（规范化路径）删除，用于按文件路径重索引。 */
|
||||
public void deleteBySource(String sourcePath) {
|
||||
if (sourcePath == null || sourcePath.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String normalized = sourcePath.replace('\\', '/');
|
||||
String filter = "metadata[\"_source\"] == \"" + escapeFilter(normalized) + "\"";
|
||||
client().delete(DeleteReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.filter(filter)
|
||||
.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除并重建当前知识 collection（空的 dense+BM25 schema）。
|
||||
* 供 {@code /api/knowledge/rebuild-hybrid} 与重建脚本使用；会销毁该 collection 全部向量。
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("collection", collectionName);
|
||||
MilvusClientV2 milvusClient = client();
|
||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
result.put("existedBefore", Boolean.TRUE.equals(exists));
|
||||
if (Boolean.TRUE.equals(exists)) {
|
||||
try {
|
||||
milvusClient.releaseCollection(ReleaseCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
} catch (Exception e) {
|
||||
log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
|
||||
}
|
||||
milvusClient.dropCollection(DropCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
log.info("Dropped hybrid collection '{}'", collectionName);
|
||||
result.put("dropped", true);
|
||||
} else {
|
||||
result.put("dropped", false);
|
||||
}
|
||||
ensureCollection(milvusClient);
|
||||
loadCollection(milvusClient);
|
||||
result.put("recreated", true);
|
||||
result.put("loaded", true);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单路 dense ANN（L2）。
|
||||
* {@code score} = L2 距离（越小越好）；{@code scoreLabel} = {@link RetrievalScoreLabels#DENSE}。
|
||||
*/
|
||||
public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
|
||||
List<Float> queryVector,
|
||||
int topK,
|
||||
String category) {
|
||||
String filter = buildFilter(category);
|
||||
SearchReq.SearchReqBuilder builder = SearchReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.annsField(FIELD_DENSE)
|
||||
.data(List.of(new FloatVec(queryVector)))
|
||||
.topK(topK)
|
||||
.outputFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
||||
.metricType(IndexParam.MetricType.L2);
|
||||
if (filter != null) {
|
||||
builder.filter(filter);
|
||||
}
|
||||
SearchResp resp = client().search(builder.build());
|
||||
return toSearchResults(resp, RetrievalScoreLabels.DENSE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dense + BM25 真混合检索（Milvus 服务端融合）。
|
||||
*
|
||||
* <ol>
|
||||
* <li>dense 子路：{@code vector}，L2</li>
|
||||
* <li>BM25 子路：{@code sparse_vector} + {@link EmbeddedText}</li>
|
||||
* <li>{@link HybridSearchReq} + {@link RRFRanker} → 返回序即权威序</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>{@code scoreLabel=hybrid}；{@code score}/{@code rawScore} 保留引擎融合分，
|
||||
* <b>不</b>用 dense L2 覆盖主分或改 label。可选并行 dense 探测仅填充
|
||||
* {@link VectorSearchService.SearchResult#setDenseDistance}，供后处理绝对质量闸门
|
||||
* （如 L0 filter low-quality → unfiltered retry），排序仍以 RRF 返回序为准。</p>
|
||||
*/
|
||||
public List<VectorSearchService.SearchResult> searchHybrid(String queryText,
|
||||
List<Float> queryVector,
|
||||
int topK,
|
||||
String category) {
|
||||
String filter = buildFilter(category);
|
||||
int pathTopK = Math.max(topK, 10);
|
||||
|
||||
AnnSearchReq.AnnSearchReqBuilder denseAnn = AnnSearchReq.builder()
|
||||
.vectorFieldName(FIELD_DENSE)
|
||||
.vectors(List.of((BaseVector) new FloatVec(queryVector)))
|
||||
.topK(pathTopK)
|
||||
.metricType(IndexParam.MetricType.L2)
|
||||
.params("{\"nprobe\":10}");
|
||||
if (filter != null) {
|
||||
denseAnn.filter(filter);
|
||||
}
|
||||
|
||||
AnnSearchReq.AnnSearchReqBuilder sparseAnn = AnnSearchReq.builder()
|
||||
.vectorFieldName(FIELD_SPARSE)
|
||||
.vectors(List.of((BaseVector) new EmbeddedText(queryText == null ? "" : queryText)))
|
||||
.topK(pathTopK)
|
||||
.metricType(IndexParam.MetricType.BM25);
|
||||
if (filter != null) {
|
||||
sparseAnn.filter(filter);
|
||||
}
|
||||
|
||||
HybridSearchReq hybridReq = HybridSearchReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.searchRequests(List.of(denseAnn.build(), sparseAnn.build()))
|
||||
.ranker(new RRFRanker(Math.max(1, rrfK)))
|
||||
.topK(topK)
|
||||
.outFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
||||
.build();
|
||||
|
||||
SearchResp hybridResp = client().hybridSearch(hybridReq);
|
||||
List<VectorSearchService.SearchResult> fused = toSearchResults(hybridResp, RetrievalScoreLabels.HYBRID);
|
||||
attachDenseDistances(fused, queryText, queryVector, pathTopK, category);
|
||||
return fused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach dense L2 by id for quality gates only — never overwrites hybrid score/label/order.
|
||||
*/
|
||||
private void attachDenseDistances(List<VectorSearchService.SearchResult> fused,
|
||||
String queryText,
|
||||
List<Float> queryVector,
|
||||
int pathTopK,
|
||||
String category) {
|
||||
if (fused == null || fused.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Map<String, Float> denseById = new HashMap<>();
|
||||
for (VectorSearchService.SearchResult denseHit :
|
||||
searchDense(queryText, queryVector, pathTopK, category)) {
|
||||
if (denseHit.getId() != null) {
|
||||
denseById.put(denseHit.getId(), denseHit.getScore());
|
||||
}
|
||||
}
|
||||
for (VectorSearchService.SearchResult hit : fused) {
|
||||
Float l2 = denseById.get(hit.getId());
|
||||
if (l2 != null) {
|
||||
hit.setDenseDistance(l2.doubleValue());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Dense distance attach for hybrid quality gate failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Milvus {@link SearchResp} 映射为上层结果；列表顺序即检索权威序（adapter 赋 originalRank）。
|
||||
*/
|
||||
private List<VectorSearchService.SearchResult> toSearchResults(SearchResp resp, String scoreLabel) {
|
||||
List<VectorSearchService.SearchResult> out = new ArrayList<>();
|
||||
if (resp == null || resp.getSearchResults() == null || resp.getSearchResults().isEmpty()) {
|
||||
return out;
|
||||
}
|
||||
List<SearchResp.SearchResult> first = resp.getSearchResults().get(0);
|
||||
if (first == null) {
|
||||
return out;
|
||||
}
|
||||
for (SearchResp.SearchResult row : first) {
|
||||
VectorSearchService.SearchResult mapped = new VectorSearchService.SearchResult();
|
||||
Object id = row.getId();
|
||||
mapped.setId(id == null ? null : String.valueOf(id));
|
||||
Map<String, Object> entity = row.getEntity() == null ? Map.of() : row.getEntity();
|
||||
Object content = entity.get(FIELD_CONTENT);
|
||||
mapped.setContent(content == null ? null : String.valueOf(content));
|
||||
Object metadata = entity.get(FIELD_METADATA);
|
||||
if (metadata instanceof JsonObject jsonObject) {
|
||||
mapped.setMetadata(jsonObject.toString());
|
||||
} else if (metadata instanceof Map<?, ?> map) {
|
||||
mapped.setMetadata(GSON.toJson(map));
|
||||
} else if (metadata != null) {
|
||||
mapped.setMetadata(String.valueOf(metadata));
|
||||
}
|
||||
Float score = row.getScore();
|
||||
mapped.setRawScore(score == null ? null : score.doubleValue());
|
||||
mapped.setScoreLabel(scoreLabel);
|
||||
// dense: L2；hybrid: 引擎融合分（后处理 quality 主要看 rank，不依赖此量纲）
|
||||
mapped.setScore(score == null ? 0f : score);
|
||||
out.add(mapped);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 组装标量过滤表达式：category、kb_scope（配置级）可叠加，用 {@code &&} 连接。
|
||||
*/
|
||||
private String buildFilter(String category) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
String categoryFilter = trimToNull(category);
|
||||
if (categoryFilter != null) {
|
||||
parts.add("metadata[\"category\"] == \"" + escapeFilter(categoryFilter) + "\"");
|
||||
}
|
||||
String scope = trimToNull(kbScope);
|
||||
if (scope != null) {
|
||||
parts.add("metadata[\"kb_scope\"] == \"" + escapeFilter(scope) + "\"");
|
||||
}
|
||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
||||
}
|
||||
|
||||
private MilvusClientV2 connect() {
|
||||
String uri;
|
||||
if (milvusProperties.isSecure() || milvusProperties.getPort() == 443) {
|
||||
uri = "https://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
||||
} else {
|
||||
uri = "http://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
||||
}
|
||||
ConnectConfig.ConnectConfigBuilder builder = ConnectConfig.builder()
|
||||
.uri(uri)
|
||||
.connectTimeoutMs(milvusProperties.getTimeout() == null ? 10000L : milvusProperties.getTimeout());
|
||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isBlank()) {
|
||||
builder.token(milvusProperties.getToken());
|
||||
builder.secure(true);
|
||||
} else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isBlank()) {
|
||||
builder.username(milvusProperties.getUsername());
|
||||
builder.password(milvusProperties.getPassword());
|
||||
}
|
||||
if (milvusProperties.getDatabase() != null && !milvusProperties.getDatabase().isBlank()) {
|
||||
builder.dbName(milvusProperties.getDatabase());
|
||||
}
|
||||
log.info("Connecting MilvusClientV2 uri={} db={} collection={}",
|
||||
uri, milvusProperties.getDatabase(), collectionName);
|
||||
return new MilvusClientV2(builder.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 若不存在则创建 dense+BM25 hybrid collection。
|
||||
*
|
||||
* <p>关键点：</p>
|
||||
* <ul>
|
||||
* <li>{@code search_text} 开启 analyzer，作为 BM25 语料。</li>
|
||||
* <li>{@link FunctionType#BM25}：input={@code search_text} → output={@code sparse_vector}。</li>
|
||||
* <li>dense：IVF_FLAT + L2；sparse：SPARSE_INVERTED_INDEX + BM25。</li>
|
||||
* </ul>
|
||||
* <p>已存在的 collection 不会改 schema；schema 变更需走 {@link #dropAndRecreateCollection()}。</p>
|
||||
*/
|
||||
private void ensureCollection(MilvusClientV2 milvusClient) {
|
||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
if (Boolean.TRUE.equals(exists)) {
|
||||
log.info("Hybrid collection '{}' already exists", collectionName);
|
||||
return;
|
||||
}
|
||||
log.info("Creating hybrid collection '{}'", collectionName);
|
||||
|
||||
CreateCollectionReq.CollectionSchema schema = milvusClient.createSchema();
|
||||
schema.setEnableDynamicField(false);
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_ID)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.ID_MAX_LENGTH)
|
||||
.isPrimaryKey(true)
|
||||
.autoID(false)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_CONTENT)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.build());
|
||||
// BM25 语料字段：必须 enableAnalyzer，Function 才能从文本生成 sparse
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_SEARCH_TEXT)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.enableAnalyzer(true)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_SPARSE)
|
||||
.dataType(DataType.SparseFloatVector)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_DENSE)
|
||||
.dataType(DataType.FloatVector)
|
||||
.dimension(milvusProperties.getVectorDim())
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_METADATA)
|
||||
.dataType(DataType.JSON)
|
||||
.build());
|
||||
// 写入 search_text 时，Milvus 自动维护 sparse_vector（应用层 insert 不填 sparse）
|
||||
schema.addFunction(CreateCollectionReq.Function.builder()
|
||||
.functionType(FunctionType.BM25)
|
||||
.name("bm25_fn")
|
||||
.inputFieldNames(List.of(FIELD_SEARCH_TEXT))
|
||||
.outputFieldNames(List.of(FIELD_SPARSE))
|
||||
.build());
|
||||
|
||||
milvusClient.createCollection(CreateCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.collectionSchema(schema)
|
||||
.description("Knowledge hybrid dense+BM25 collection")
|
||||
.numShards(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
||||
.build());
|
||||
|
||||
List<IndexParam> indexes = List.of(
|
||||
IndexParam.builder()
|
||||
.fieldName(FIELD_DENSE)
|
||||
.indexType(IndexParam.IndexType.IVF_FLAT)
|
||||
.metricType(IndexParam.MetricType.L2)
|
||||
.extraParams(Map.of("nlist", 128))
|
||||
.build(),
|
||||
IndexParam.builder()
|
||||
.fieldName(FIELD_SPARSE)
|
||||
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
|
||||
.metricType(IndexParam.MetricType.BM25)
|
||||
.build()
|
||||
);
|
||||
milvusClient.createIndex(CreateIndexReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.indexParams(indexes)
|
||||
.build());
|
||||
log.info("Hybrid collection '{}' created with dense+BM25 indexes", collectionName);
|
||||
}
|
||||
|
||||
private void loadCollection(MilvusClientV2 milvusClient) {
|
||||
milvusClient.loadCollection(LoadCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
if (client != null) {
|
||||
try {
|
||||
client.close();
|
||||
} catch (Exception e) {
|
||||
log.warn("Error closing MilvusClientV2: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 过滤表达式字符串转义，防止引号打断 expr。 */
|
||||
private static String escapeFilter(String value) {
|
||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
}
|
||||
|
||||
private static String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String stringVal(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,9 @@ import java.util.List;
|
||||
/**
|
||||
* 知识语义检索的应用边界端口。
|
||||
*
|
||||
* <p>实现可对接 dense / hybrid 等引擎，但不得向上层泄漏 SDK 类型。
|
||||
* 当前实现：{@link VectorKnowledgeSearchAdapter} → {@code VectorSearchService}
|
||||
* → {@code MilvusHybridKnowledgeStore}（Milvus SDK v2 dense 或 dense+BM25 RRF）。</p>
|
||||
* <p>实现可对接 dense / hybrid 等引擎，但不得向上层泄漏远端 API 类型。
|
||||
* 当前实现：{@link PyRagKnowledgeSearchAdapter}（py-rag 知识服务 /api/v1/search，
|
||||
* 服务端负责 hybrid 融合、BM25、rerank 与判级）。RAG 模块抽离后为唯一实现。</p>
|
||||
*/
|
||||
public interface KnowledgeSearchPort {
|
||||
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Sparse-lite lexical ranking over already recalled candidates.
|
||||
* Not a substitute for inverted-index BM25; expands ordering signal only.
|
||||
*/
|
||||
public final class LexicalRanker {
|
||||
|
||||
private LexicalRanker() {
|
||||
}
|
||||
|
||||
public static List<KnowledgeSearchHit> rank(String query, List<KnowledgeSearchHit> candidates) {
|
||||
if (candidates == null || candidates.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
Set<String> terms = tokenize(query);
|
||||
if (terms.isEmpty()) {
|
||||
return List.copyOf(candidates);
|
||||
}
|
||||
List<ScoredHit> scored = new ArrayList<>(candidates.size());
|
||||
for (KnowledgeSearchHit hit : candidates) {
|
||||
String haystack = (nullToEmpty(hit.title()) + " "
|
||||
+ nullToEmpty(hit.breadcrumb()) + " "
|
||||
+ nullToEmpty(hit.content())).toLowerCase(Locale.ROOT);
|
||||
int hits = 0;
|
||||
for (String term : terms) {
|
||||
if (haystack.contains(term)) {
|
||||
hits++;
|
||||
}
|
||||
}
|
||||
double coverage = hits / (double) terms.size();
|
||||
scored.add(new ScoredHit(hit, coverage, hits));
|
||||
}
|
||||
scored.sort(Comparator
|
||||
.comparingDouble((ScoredHit s) -> s.coverage).reversed()
|
||||
.thenComparingInt((ScoredHit s) -> s.hits).reversed()
|
||||
.thenComparingInt(s -> s.hit.originalRank()));
|
||||
return scored.stream().map(s -> s.hit).toList();
|
||||
}
|
||||
|
||||
static Set<String> tokenize(String query) {
|
||||
if (query == null || query.isBlank()) {
|
||||
return Set.of();
|
||||
}
|
||||
String normalized = query.toLowerCase(Locale.ROOT);
|
||||
String[] parts = normalized.split("[^\\p{IsAlphabetic}\\p{IsDigit}]+");
|
||||
Set<String> terms = new LinkedHashSet<>();
|
||||
for (String part : parts) {
|
||||
if (part == null) {
|
||||
continue;
|
||||
}
|
||||
String term = part.trim();
|
||||
if (term.length() >= 2) {
|
||||
terms.add(term);
|
||||
}
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
|
||||
private static String nullToEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
private record ScoredHit(KnowledgeSearchHit hit, double coverage, int hits) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link KnowledgeSearchPort} 的 py-rag 远端实现（RAG 模块抽离后的唯一检索后端）。
|
||||
*
|
||||
* <p>原进程内链路（VectorKnowledgeSearchAdapter → VectorSearchService → MilvusHybridKnowledgeStore）
|
||||
* 已由 py-rag 服务端接管：hybrid 融合、BM25、rerank、chunk 去重、判级阈值全部下沉。
|
||||
* Java 侧只做请求映射与命中结构归一化，不碰检索算法。</p>
|
||||
*
|
||||
* <h3>映射约定</h3>
|
||||
* <ul>
|
||||
* <li>mode：{@link KnowledgeSearchMode#DENSE} → {@code semantic}，{@link KnowledgeSearchMode#HYBRID} → {@code hybrid}</li>
|
||||
* <li>retrieve_k = return_n = topK：返回 topK 条精排后命中，chunk 去重/截断仍由
|
||||
* {@code KnowledgeEvidencePostProcessor} 统一负责，故 max_chunks_per_document 同步放大避免服务端预截断</li>
|
||||
* <li>category：{@code categoryFilter} 透传；null = 不过滤；kb_scope 不传，由服务端部署配置决定</li>
|
||||
* <li>score：py-rag rerank 绝对相关分（[0,1]，越大越好），scoreLabel =
|
||||
* {@link RetrievalScoreLabels#RERANK}（quality 原样采用，不做 L2/rank 归一化）</li>
|
||||
* <li>evidence_key：{@code docId#chunk-N}，与 EvidenceGuard 验真约定一致；
|
||||
* {@code evidence_status=no_evidence} 时服务端保证 hits=[]，按"无知识"正常返回</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Component
|
||||
public class PyRagKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
||||
|
||||
private static final String CHUNK_MARK = "#chunk-";
|
||||
|
||||
private final PyRagClient pyRagClient;
|
||||
|
||||
public PyRagKnowledgeSearchAdapter(PyRagClient pyRagClient) {
|
||||
this.pyRagClient = pyRagClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
||||
PyRagSearchResponse response = pyRagClient.search(toPyRagRequest(request));
|
||||
if (response == null || response.hits() == null || response.hits().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<KnowledgeSearchHit> hits = new ArrayList<>(response.hits().size());
|
||||
for (int i = 0; i < response.hits().size(); i++) {
|
||||
hits.add(toHit(response.hits().get(i), i + 1));
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
/** 请求映射：topK 同时作为召回宽度与返回条数，服务端不预截断 chunk。 */
|
||||
private PyRagSearchRequest toPyRagRequest(KnowledgeSearchRequest request) {
|
||||
return new PyRagSearchRequest(
|
||||
request.query(),
|
||||
request.mode() == KnowledgeSearchMode.HYBRID ? "hybrid" : "semantic",
|
||||
request.topK(),
|
||||
request.topK(),
|
||||
request.topK(),
|
||||
blankToNull(request.categoryFilter()),
|
||||
null);
|
||||
}
|
||||
|
||||
private KnowledgeSearchHit toHit(PyRagSearchHit hit, int originalRank) {
|
||||
String docId = blankToNull(hit.documentId());
|
||||
Integer chunkIndex = parseChunkIndex(hit.evidenceKey());
|
||||
String evidenceKey = EvidenceIdentity.firstNonBlank(
|
||||
hit.evidenceKey(),
|
||||
EvidenceIdentity.evidenceKey(docId, chunkIndex, null, originalRank));
|
||||
Double score = hit.qualityScore() == null ? 0.0 : hit.qualityScore();
|
||||
return new KnowledgeSearchHit(
|
||||
firstNonBlank(evidenceKey, docId, "rank:" + originalRank),
|
||||
hit.excerpt(),
|
||||
score,
|
||||
score,
|
||||
RetrievalScoreLabels.RERANK,
|
||||
null,
|
||||
Map.of(),
|
||||
docId,
|
||||
chunkIndex,
|
||||
evidenceKey,
|
||||
hit.source(),
|
||||
hit.title(),
|
||||
hit.breadcrumb(),
|
||||
originalRank,
|
||||
null);
|
||||
}
|
||||
|
||||
/** evidence_key 形如 {@code docId#chunk-N}，解析末尾 chunk 序号；不符返回 null。 */
|
||||
private Integer parseChunkIndex(String evidenceKey) {
|
||||
String key = blankToNull(evidenceKey);
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
int mark = key.lastIndexOf(CHUNK_MARK);
|
||||
if (mark < 0) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Integer.valueOf(key.substring(mark + CHUNK_MARK.length()).trim());
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String blankToNull(String value) {
|
||||
return value == null || value.isBlank() ? null : value.trim();
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
return EvidenceIdentity.firstNonBlank(values);
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,13 @@ package com.superbiz.agent.service.retrieval;
|
||||
/**
|
||||
* 检索结果一级 {@code scoreLabel} 约定。
|
||||
*
|
||||
* <p>只区分两种检索形态（与 {@code retrieval.search.mode} 对齐），
|
||||
* 不再使用 {@code bm25_only_*} 等作为正式一级 label。</p>
|
||||
* <p>三种检索形态：</p>
|
||||
* <ul>
|
||||
* <li>{@link #DENSE} —— 单路向量 ANN（L2 距离，越小越好）</li>
|
||||
* <li>{@link #HYBRID} —— dense + BM25 + RRF 融合（质量主要看 rank）</li>
|
||||
* <li>{@link #RERANK} —— py-rag 服务端 rerank 绝对相关分（[0,1]，越大越好）；
|
||||
* RAG 模块抽离后的线上主路径</li>
|
||||
* </ul>
|
||||
*/
|
||||
public final class RetrievalScoreLabels {
|
||||
|
||||
@@ -14,11 +19,14 @@ public final class RetrievalScoreLabels {
|
||||
/** hybrid（dense+BM25+RRF）：{@code score}/raw 为融合侧信号；质量分主要看 rank。 */
|
||||
public static final String HYBRID = "hybrid";
|
||||
|
||||
/** py-rag 服务端 rerank 绝对分：{@code score} 即归一化质量（[0,1]，越大越好）。 */
|
||||
public static final String RERANK = "rerank";
|
||||
|
||||
private RetrievalScoreLabels() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 将历史/别名 label 归一到 {@link #DENSE} 或 {@link #HYBRID}。
|
||||
* 将历史/别名 label 归一到 {@link #DENSE}、{@link #HYBRID} 或 {@link #RERANK}。
|
||||
* 未知或空 → dense（保守，按 L2 解释失败时 quality 偏低）。
|
||||
*/
|
||||
public static String canonicalize(String scoreLabel) {
|
||||
@@ -29,12 +37,19 @@ public final class RetrievalScoreLabels {
|
||||
return switch (label) {
|
||||
case DENSE, "l2_distance", "l2" -> DENSE;
|
||||
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
|
||||
default -> label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
||||
? HYBRID
|
||||
: DENSE;
|
||||
case RERANK, "rerank_score", "quality_score" -> RERANK;
|
||||
default -> label.contains("rerank") || label.contains("quality")
|
||||
? RERANK
|
||||
: label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
||||
? HYBRID
|
||||
: DENSE;
|
||||
};
|
||||
}
|
||||
|
||||
public static boolean isRerank(String scoreLabel) {
|
||||
return RERANK.equals(canonicalize(scoreLabel));
|
||||
}
|
||||
|
||||
public static boolean isHybrid(String scoreLabel) {
|
||||
return HYBRID.equals(canonicalize(scoreLabel));
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ package com.superbiz.agent.service.retrieval;
|
||||
* <p>后处理排序仍按 {@code originalRank}；本类只负责质量闸门 / relevance 用分。</p>
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link RetrievalScoreLabels#RERANK}：py-rag 服务端 rerank 绝对分，
|
||||
* {@code score} 已归一化，原样 clamp 到 [0,1]（RAG 抽离后的主路径）</li>
|
||||
* <li>{@link RetrievalScoreLabels#DENSE}：{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li>
|
||||
* <li>{@link RetrievalScoreLabels#HYBRID}：优先用可选 {@code denseDistance} 做绝对质量
|
||||
* （恢复 L0 filter low-quality 等闸门）；无 dense 时回退 rank 映射</li>
|
||||
@@ -17,8 +19,8 @@ public final class RetrievalScoreNormalizer {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param scoreLabel {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
||||
* @param score 引擎主分：dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
|
||||
* @param scoreLabel {@link RetrievalScoreLabels#RERANK} / {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
||||
* @param score 引擎主分：rerank=绝对相关分[0,1]；dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
|
||||
* @param originalRank 检索名次（1-based）
|
||||
* @param batchSize 本轮候选数（rank 回退映射用）
|
||||
* @param maxL2Distance L2 上界
|
||||
@@ -31,6 +33,12 @@ public final class RetrievalScoreNormalizer {
|
||||
double maxL2Distance,
|
||||
Double denseDistance) {
|
||||
String label = RetrievalScoreLabels.canonicalize(scoreLabel);
|
||||
if (RetrievalScoreLabels.RERANK.equals(label)) {
|
||||
if (score == null) {
|
||||
return 0.0;
|
||||
}
|
||||
return Math.max(0.0, Math.min(1.0, score));
|
||||
}
|
||||
if (RetrievalScoreLabels.HYBRID.equals(label)) {
|
||||
if (denseDistance != null) {
|
||||
return l2ToQuality(denseDistance, maxL2Distance);
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* Reciprocal Rank Fusion 工具：把多路检索的排名列表融合成一个分数排序。
|
||||
*
|
||||
* <pre>
|
||||
* RRF_w(d) = Σ w_i / (k + rank_i(d))
|
||||
* </pre>
|
||||
*
|
||||
* <p>只依赖排名不依赖原始分数——屏蔽跨路分数尺度不可比的问题；
|
||||
* 每路可加权（w <= 0 时按 1.0 等权），k 是平滑参数（默认 60，可配）。
|
||||
*/
|
||||
public final class RrfFusion {
|
||||
|
||||
private RrfFusion() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 融合多路排名：对每路的每个 item 累加 w/(k+rank)，按总分降序输出。
|
||||
*
|
||||
* @param paths 多路排名（每路带 name / items / weight）
|
||||
* @param rrfK 平滑参数 k（至少 1）
|
||||
* @param identityFn 跨路识别同一 item 的身份函数（如 evidenceKey）
|
||||
* @return 融合后排序（含每路排名明细）
|
||||
*/
|
||||
public static <T> List<Scored<T>> fuse(List<RankedPath<T>> paths,
|
||||
int rrfK,
|
||||
Function<T, String> identityFn) {
|
||||
if (paths == null || paths.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
int k = Math.max(1, rrfK);
|
||||
Map<String, Acc<T>> acc = new LinkedHashMap<>();
|
||||
for (RankedPath<T> path : paths) {
|
||||
if (path == null || path.items() == null || path.items().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
double weight = path.weight() <= 0 ? 1.0 : path.weight();
|
||||
List<T> items = path.items();
|
||||
for (int i = 0; i < items.size(); i++) {
|
||||
T item = items.get(i);
|
||||
if (item == null) {
|
||||
continue;
|
||||
}
|
||||
String id = identityFn.apply(item);
|
||||
if (id == null || id.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
int rank = i + 1;
|
||||
double contrib = weight / (k + rank); // 排名越前贡献越大
|
||||
Acc<T> bucket = acc.computeIfAbsent(id, ignored -> new Acc<>(item));
|
||||
bucket.score += contrib;
|
||||
bucket.ranks.put(path.name(), rank);
|
||||
// Prefer first-seen item payload; callers should put preferred path first if needed.
|
||||
}
|
||||
}
|
||||
List<Scored<T>> scored = new ArrayList<>(acc.size());
|
||||
for (Map.Entry<String, Acc<T>> entry : acc.entrySet()) {
|
||||
Acc<T> value = entry.getValue();
|
||||
scored.add(new Scored<>(entry.getKey(), value.item, value.score, Map.copyOf(value.ranks)));
|
||||
}
|
||||
// 总分降序（两路共识的靠前），同分按身份稳定排序
|
||||
scored.sort(Comparator
|
||||
.comparingDouble((Scored<T> s) -> s.rrfScore()).reversed()
|
||||
.thenComparing(Scored::identity));
|
||||
return scored;
|
||||
}
|
||||
|
||||
/** 一路检索结果：name（路名）+ items（按排名顺序）+ weight（可选加权，≤0 视为等权）。 */
|
||||
public record RankedPath<T>(String name, List<T> items, double weight) {
|
||||
public RankedPath {
|
||||
Objects.requireNonNull(name, "name");
|
||||
items = items == null ? List.of() : List.copyOf(items);
|
||||
}
|
||||
}
|
||||
|
||||
/** 融合后的单个 item：identity + 原始 item + rrfScore + 每路排名明细。 */
|
||||
public record Scored<T>(String identity, T item, double rrfScore, Map<String, Integer> ranks) {
|
||||
}
|
||||
|
||||
/** 跨路累加器：同一 identity 的 item 累加 RRF 分并记录各路排名。 */
|
||||
private static final class Acc<T> {
|
||||
private final T item;
|
||||
private double score;
|
||||
private final Map<String, Integer> ranks = new HashMap<>();
|
||||
|
||||
private Acc(T item) {
|
||||
this.item = item;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link KnowledgeSearchPort} 适配器：把向量检索结果映射为带 evidenceKey 的命中结构。
|
||||
*
|
||||
* <p>委托 {@link VectorSearchService}（背后仅 {@code MilvusHybridKnowledgeStore}）：
|
||||
* dense 或 dense+BM25 hybrid 由配置 {@code retrieval.search.mode} 选择。
|
||||
* 本类负责 metadata 解析、docId/chunk 身份与 evidenceKey，不碰 SDK。</p>
|
||||
*/
|
||||
@Component
|
||||
public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
||||
|
||||
private final VectorSearchService vectorSearchService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
|
||||
this.vectorSearchService = vectorSearchService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
||||
// Mode is global on VectorSearchService; request.mode is advisory for future per-call overrides.
|
||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(
|
||||
request.query(),
|
||||
request.topK(),
|
||||
request.categoryFilter());
|
||||
return toHits(results);
|
||||
}
|
||||
|
||||
private List<KnowledgeSearchHit> toHits(List<VectorSearchService.SearchResult> results) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<KnowledgeSearchHit> hits = new ArrayList<>(results.size());
|
||||
for (int i = 0; i < results.size(); i++) {
|
||||
hits.add(toHit(results.get(i), i + 1));
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
|
||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
||||
String docId = EvidenceIdentity.extractDocId(
|
||||
metadata,
|
||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "source"));
|
||||
Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
|
||||
String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
|
||||
String source = EvidenceIdentity.firstNonBlank(
|
||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "filePath"),
|
||||
docId,
|
||||
result.getId());
|
||||
return new KnowledgeSearchHit(
|
||||
result.getId(),
|
||||
result.getContent(),
|
||||
(double) result.getScore(),
|
||||
result.getRawScore(),
|
||||
result.getScoreLabel(),
|
||||
result.getMetadata(),
|
||||
metadata,
|
||||
docId,
|
||||
chunkIndex,
|
||||
evidenceKey,
|
||||
source,
|
||||
EvidenceIdentity.metadataValue(metadata, "title"),
|
||||
EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
|
||||
originalRank,
|
||||
result.getDenseDistance()
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, String> parseMetadata(String metadata) {
|
||||
if (metadata == null || metadata.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
||||
Map<String, String> parsed = new LinkedHashMap<>();
|
||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
} catch (Exception ignored) {
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.RpcStatus;
|
||||
import io.milvus.param.collection.DropCollectionParam;
|
||||
import io.milvus.param.collection.HasCollectionParam;
|
||||
|
||||
/**
|
||||
* 删除 Milvus Collection 的工具类
|
||||
* 用于重建 Collection 时清理旧数据
|
||||
*/
|
||||
public class DropCollection {
|
||||
|
||||
public static void main(String[] args) {
|
||||
MilvusServiceClient client = null;
|
||||
|
||||
try {
|
||||
// 连接到 Milvus
|
||||
System.out.println("正在连接到 Milvus localhost:19530...");
|
||||
client = new MilvusServiceClient(
|
||||
ConnectParam.newBuilder()
|
||||
.withHost("localhost")
|
||||
.withPort(19530)
|
||||
.build()
|
||||
);
|
||||
System.out.println("✓ 连接成功");
|
||||
|
||||
String collectionName = "biz";
|
||||
|
||||
// 检查 Collection 是否存在
|
||||
R<Boolean> hasResponse = client.hasCollection(
|
||||
HasCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build()
|
||||
);
|
||||
|
||||
if (hasResponse.getData()) {
|
||||
System.out.println("发现 Collection: " + collectionName);
|
||||
System.out.println("正在删除...");
|
||||
|
||||
// 删除 Collection
|
||||
R<RpcStatus> dropResponse = client.dropCollection(
|
||||
DropCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build()
|
||||
);
|
||||
|
||||
if (dropResponse.getStatus() == 0) {
|
||||
System.out.println("✓ Collection 已成功删除");
|
||||
System.out.println("\n请重启 Spring Boot 应用，它会自动创建新的 FloatVector Collection");
|
||||
} else {
|
||||
System.err.println("✗ 删除失败: " + dropResponse.getMessage());
|
||||
}
|
||||
} else {
|
||||
System.out.println("Collection '" + collectionName + "' 不存在");
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
System.err.println("错误: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import com.superbiz.agent.dto.RetrievalTrace;
|
||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -29,10 +28,9 @@ import java.util.Map;
|
||||
*
|
||||
* <h3>主链路</h3>
|
||||
* <pre>
|
||||
* query
|
||||
* -> KnowledgeQueryTransformer
|
||||
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k)
|
||||
* -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
|
||||
* query（原始句直传；L0 query 理解已下沉 py-rag 服务端）
|
||||
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort → py-rag, retrieve-k)
|
||||
* -> KnowledgeEvidencePostProcessor (qualityScore / chunk dedup / caps / return-n)
|
||||
* -> [optional] unfiltered retry
|
||||
* -> KnowledgeContextPacker
|
||||
* -> LookupResultAssembler
|
||||
@@ -62,9 +60,6 @@ public class LookupKnowledgeTool {
|
||||
|
||||
private int retrieveK = 20;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeQueryTransformer queryTransformer;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDocumentRetriever documentRetriever;
|
||||
|
||||
@@ -98,10 +93,10 @@ public class LookupKnowledgeTool {
|
||||
*
|
||||
* <p>流程（模块化三段）：
|
||||
* <ol>
|
||||
* <li>检索前：QueryTransformer.transform → KnowledgeQuery（分类过滤/域/关键词）；</li>
|
||||
* <li>检索：DocumentRetriever.retrieve（FILTERED 或 UNFILTERED，retrieveK 候选）；</li>
|
||||
* <li>检索前：原始 query 直传（L0 domain 分析已下沉 py-rag，categoryFilter 恒为 null）；</li>
|
||||
* <li>检索：DocumentRetriever.retrieve（UNFILTERED，retrieveK 候选，py-rag 服务端融合+精排）；</li>
|
||||
* <li>检索后：PostProcessor.process（qualityScore/去重/判级）；</li>
|
||||
* <li>低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查；</li>
|
||||
* <li>低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查（L0 移除后保留兜底语义）；</li>
|
||||
* <li>打包 + 组装：ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li>
|
||||
* </ol>
|
||||
*
|
||||
@@ -113,12 +108,17 @@ public class LookupKnowledgeTool {
|
||||
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
|
||||
log.info("----------------------------------------");
|
||||
|
||||
// ── 检索前：查询理解（L0）──
|
||||
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
|
||||
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
|
||||
knowledgeQuery.getCategoryFilter(),
|
||||
knowledgeQuery.getDomainHints().size(),
|
||||
knowledgeQuery.getMatchedKeywords().size());
|
||||
// ── 检索前：原始 query 直传（L0 已下沉 py-rag，不做 Java 侧 category 收窄）──
|
||||
String normalized = query == null ? "" : query.trim();
|
||||
KnowledgeQuery knowledgeQuery = KnowledgeQuery.builder()
|
||||
.originalQuery(normalized)
|
||||
.rewrittenQuery(normalized)
|
||||
.domainHints(List.of())
|
||||
.matchedKeywords(List.of())
|
||||
.entities(List.of())
|
||||
.l0Titles(List.of())
|
||||
.l0MatchCount(0)
|
||||
.build();
|
||||
|
||||
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
|
||||
String fallbackReason = null;
|
||||
|
||||
@@ -15,30 +15,28 @@ file:
|
||||
knowledge:
|
||||
base-path: knowledge_base/
|
||||
|
||||
milvus:
|
||||
host: in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com
|
||||
port: 443
|
||||
username: ""
|
||||
password: ""
|
||||
database: db_4a578da0f27ce9d
|
||||
timeout: 10000
|
||||
token: ${MILVUS_TOKEN}
|
||||
secure: true
|
||||
vector-dim: 1024 # BGE-M3 = 1024，换模型时同步改
|
||||
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
|
||||
collection: biz
|
||||
# =====================================================
|
||||
# py-rag 知识服务接入
|
||||
# =====================================================
|
||||
# RAG 检索与文档入库均由 py-rag 服务承担（契约见 py-rag 仓库 docs/Java接入文档.md）：
|
||||
# 检索 /api/v1/search，入库 /api/v1/documents:ingest，全量重建 /api/v1/collections:rebuild。
|
||||
pyrag:
|
||||
base-url: ${PYRAG_BASE_URL:http://localhost:8000}
|
||||
connect-timeout-ms: 3000
|
||||
search-read-timeout-ms: 5000 # 正常 300–800ms（含 rerank 外呼）
|
||||
ingest-read-timeout-ms: 30000 # 正常 1–5s
|
||||
default-read-timeout-ms: 10000
|
||||
|
||||
# =====================================================
|
||||
# 模型路由配置
|
||||
# =====================================================
|
||||
# 通过关键字匹配 Bean，切换模型只改这里 + 对应 api-key
|
||||
# Chat: deepseek | openai | ollama | ...
|
||||
# Embedding: siliconflow | openai | ollama | dashscope | ...
|
||||
# Chat: deepseek | openai | ollama | ...
|
||||
# （Embedding 已随 RAG 抽离至 py-rag 服务端）
|
||||
# =====================================================
|
||||
|
||||
model-routing:
|
||||
chat: deepseek
|
||||
embedding: siliconflow
|
||||
|
||||
spring:
|
||||
config:
|
||||
@@ -102,30 +100,6 @@ spring:
|
||||
retry:
|
||||
max-attempts: 1
|
||||
|
||||
vectorstore:
|
||||
type: milvus
|
||||
milvus:
|
||||
initialize-schema: false
|
||||
database-name: ${milvus.database}
|
||||
collection-name: biz
|
||||
embedding-dimension: ${milvus.vector-dim}
|
||||
index-type: IVF_FLAT
|
||||
metric-type: L2
|
||||
index-parameters: '{"nlist":128}'
|
||||
id-field-name: id
|
||||
auto-id: false
|
||||
content-field-name: content
|
||||
metadata-field-name: metadata
|
||||
embedding-field-name: vector
|
||||
client:
|
||||
host: ${milvus.host}
|
||||
port: ${milvus.port}
|
||||
token: ${milvus.token}
|
||||
username: ${milvus.username}
|
||||
password: ${milvus.password}
|
||||
secure: ${milvus.secure}
|
||||
connect-timeout-ms: ${milvus.timeout}
|
||||
|
||||
# --- Chat: DeepSeek (原生) ---
|
||||
deepseek:
|
||||
api-key: ${DEEPSEEK_API_KEY}
|
||||
@@ -134,53 +108,27 @@ spring:
|
||||
options:
|
||||
model: deepseek-v4-flash
|
||||
|
||||
# --- OpenAI 模块供 SiliconFlow Embedding 复用 ---
|
||||
openai:
|
||||
api-key: unused
|
||||
|
||||
# Spring AI MCP 客户端配置
|
||||
mcp:
|
||||
client:
|
||||
enabled: false
|
||||
|
||||
# --- Embedding: SiliconFlow BGE-M3 ---
|
||||
siliconflow:
|
||||
api-key: ${SILICONFLOW_API_KEY}
|
||||
base-url: https://api.siliconflow.cn
|
||||
embedding:
|
||||
model: BAAI/bge-m3
|
||||
|
||||
# 文档分片配置
|
||||
document:
|
||||
chunk:
|
||||
max-size: 800
|
||||
overlap: 100
|
||||
|
||||
# RAG 配置
|
||||
rag:
|
||||
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
sidecar:
|
||||
spring-ai:
|
||||
enabled: false
|
||||
content-preview-limit: 300
|
||||
|
||||
# 检索配置
|
||||
# 知识主路径：Milvus Java SDK v2（MilvusHybridKnowledgeStore），非 Spring AI VectorStore starter。
|
||||
# 原因：starter（含 2.0.0）仅 dense similarity，无 hybridSearch / BM25 Function / RRFRanker。
|
||||
# 已移除 legacy sdk/spring/auto 多后端路由。
|
||||
# 知识主路径：py-rag 知识服务（PyRagKnowledgeSearchAdapter → /api/v1/search）。
|
||||
# 服务端负责 dense+BM25 融合、rerank（BGE-Reranker）与判级；Java 侧只做请求映射与后处理。
|
||||
retrieval:
|
||||
kb-scope: "" # 非空则过滤 metadata.kb_scope；空=不过滤
|
||||
search:
|
||||
# hybrid=线上主路径；dense=同库对照/评测/排障（非第二套线上策略）。见 mvp/architecture/rag-knowledge-retrieval-architecture.md §6.0
|
||||
mode: hybrid # dense=单路L2对照 | hybrid=dense+服务端BM25+RRF
|
||||
hybrid:
|
||||
rrf-k: 60 # RRF 平滑参数 k，score=Σ 1/(k+rank)
|
||||
# hybrid=线上主路径；dense 为对照/排障（映射 py-rag mode：hybrid→hybrid，dense→semantic）
|
||||
mode: hybrid
|
||||
normalization:
|
||||
max-l2-distance: 2.0 # dense quality：L2 上界（单位向量 ≈ 2.0）
|
||||
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（hybrid 为序数分，见架构 §6）
|
||||
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（与 py-rag 判级阈值一致）
|
||||
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE；低于则低质/可 unfiltered retry
|
||||
|
||||
# Prometheus 配置
|
||||
|
||||
@@ -11,7 +11,6 @@ import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||
@@ -84,7 +83,6 @@ class HarnessChatConfigurationTest {
|
||||
.withBean(ChatModel.class, () -> mock(ChatModel.class))
|
||||
.withBean(RedisTemplate.class, () -> mock(RedisTemplate.class))
|
||||
.withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class))
|
||||
.withBean(KnowledgeQueryTransformer.class, () -> mock(KnowledgeQueryTransformer.class))
|
||||
.withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class))
|
||||
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
|
||||
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
|
||||
|
||||
@@ -4,18 +4,13 @@ import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.data.redis.core.RedisTemplate;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* 单独测试 Redis 连接
|
||||
* 禁用 Milvus 以避免启动失败
|
||||
* 单独测试 Redis 连接（需要可达的 Redis 实例）。
|
||||
*/
|
||||
@SpringBootTest
|
||||
@TestPropertySource(properties = {
|
||||
"spring.autoconfigure.exclude=org.example.config.MilvusConfig"
|
||||
})
|
||||
class RedisConnectionTest {
|
||||
|
||||
@Autowired(required = false)
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.superbiz.agent.Main;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.dto.DocumentUploadRequest;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.service.DocumentManagementService;
|
||||
import com.superbiz.agent.service.FrontmatterParser;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
|
||||
/**
|
||||
* Imports canonical RAG eval documents through the real document pipeline.
|
||||
*
|
||||
* <p>Disabled by default because it writes DB rows, local knowledge files, and
|
||||
* vector index records in the configured runtime environment.</p>
|
||||
*/
|
||||
@SpringBootTest(
|
||||
classes = Main.class,
|
||||
webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
||||
properties = "spring.main.web-application-type=none"
|
||||
)
|
||||
@EnabledIfSystemProperty(named = "rag.seed.enabled", matches = "true")
|
||||
class RagEvalSeedImporterTest {
|
||||
|
||||
private static final Path DEFAULT_SEED_DOCS = Path.of("eval/rag-retrieval/seed-docs");
|
||||
|
||||
@Autowired
|
||||
private DocumentManagementService documentManagementService;
|
||||
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Test
|
||||
void importSeedDocuments() throws Exception {
|
||||
Path seedDir = Path.of(System.getProperty("rag.seed.docs", DEFAULT_SEED_DOCS.toString()));
|
||||
List<Path> docs;
|
||||
try (var stream = Files.list(seedDir)) {
|
||||
docs = stream
|
||||
.filter(path -> path.getFileName().toString().endsWith(".md"))
|
||||
.sorted()
|
||||
.toList();
|
||||
}
|
||||
assertFalse(docs.isEmpty(), "seed docs directory must contain markdown files");
|
||||
|
||||
for (Path docPath : docs) {
|
||||
String content = Files.readString(docPath, StandardCharsets.UTF_8);
|
||||
Frontmatter frontmatter = frontmatterParser.parse(content);
|
||||
if (frontmatter == null || frontmatter.getSource() == null || frontmatter.getSource().isBlank()) {
|
||||
throw new IllegalArgumentException("seed doc must include frontmatter source: " + docPath);
|
||||
}
|
||||
|
||||
apiDocumentRepository.findByDocId(frontmatter.getSource().trim())
|
||||
.map(ApiDocument::getDocId)
|
||||
.ifPresent(documentManagementService::deleteDocument);
|
||||
|
||||
String fileName = docPath.getFileName().toString();
|
||||
MockMultipartFile file = new MockMultipartFile(
|
||||
"file",
|
||||
fileName,
|
||||
"text/markdown",
|
||||
content.getBytes(StandardCharsets.UTF_8)
|
||||
);
|
||||
DocumentUploadRequest request = DocumentUploadRequest.builder()
|
||||
.file(file)
|
||||
.category(resolveCategory(frontmatter))
|
||||
.build();
|
||||
|
||||
documentManagementService.uploadDocument(request);
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveCategory(Frontmatter frontmatter) {
|
||||
if (frontmatter.getCategory() != null && !frontmatter.getCategory().isBlank()) {
|
||||
return frontmatter.getCategory().trim();
|
||||
}
|
||||
return "rag-eval";
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.superbiz.agent.Main;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.Locale;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
|
||||
*
|
||||
* <p>Disabled by default: writes repository files and needs the live retrieval stack
|
||||
* (embedding + Milvus hybrid collection + optional MySQL/L0).</p>
|
||||
*
|
||||
* <p>System properties (via Maven {@code -D}):</p>
|
||||
* <ul>
|
||||
* <li>{@code rag.snapshot.enabled=true} — required to run</li>
|
||||
* <li>{@code retrieval.search.mode=hybrid|dense} — default hybrid</li>
|
||||
* <li>{@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})</li>
|
||||
* <li>{@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}</li>
|
||||
* </ul>
|
||||
*/
|
||||
@SpringBootTest(
|
||||
classes = Main.class,
|
||||
webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
||||
properties = "spring.main.web-application-type=none"
|
||||
)
|
||||
@EnabledIfSystemProperty(named = "rag.snapshot.enabled", matches = "true")
|
||||
class RagLookupSnapshotGeneratorTest {
|
||||
|
||||
private static final Path DEFAULT_CASES = Path.of("eval/rag-retrieval/cases/golden-cases.json");
|
||||
private static final Path DEFAULT_FIXTURES = Path.of("eval/rag-retrieval/fixtures");
|
||||
|
||||
@Autowired
|
||||
private LookupKnowledgeTool lookupKnowledgeTool;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
|
||||
*/
|
||||
@DynamicPropertySource
|
||||
static void retrievalProperties(DynamicPropertyRegistry registry) {
|
||||
String mode = System.getProperty("retrieval.search.mode", "hybrid");
|
||||
if (mode == null || mode.isBlank()) {
|
||||
mode = "hybrid";
|
||||
}
|
||||
String normalized = mode.trim().toLowerCase(Locale.ROOT);
|
||||
registry.add("retrieval.search.mode", () -> normalized);
|
||||
|
||||
String kbScope = System.getProperty("retrieval.kb-scope", "");
|
||||
if (kbScope != null && !kbScope.isBlank()) {
|
||||
registry.add("retrieval.kb-scope", kbScope::trim);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void generateLookupResultFixtures() throws Exception {
|
||||
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
|
||||
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
|
||||
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
|
||||
String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
|
||||
String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
|
||||
|
||||
JsonNode root = objectMapper.readTree(casesPath.toFile());
|
||||
JsonNode cases = root.path("cases");
|
||||
assertTrue(cases.isArray(), "golden cases file must contain a cases array");
|
||||
|
||||
Files.createDirectories(fixturesDir);
|
||||
for (JsonNode testCase : cases) {
|
||||
String caseId = requiredText(testCase, "caseId");
|
||||
String query = requiredText(testCase, "query");
|
||||
|
||||
LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
|
||||
|
||||
ObjectNode fixture = objectMapper.createObjectNode();
|
||||
fixture.put("caseId", caseId);
|
||||
fixture.put("query", query);
|
||||
fixture.put("retrievedAt", retrievedAt);
|
||||
fixture.put("searchMode", searchMode);
|
||||
if (kbScope != null) {
|
||||
fixture.put("kbScope", kbScope);
|
||||
}
|
||||
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
|
||||
|
||||
Path output = fixturesDir.resolve(caseId + ".json");
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), fixture);
|
||||
}
|
||||
}
|
||||
|
||||
private static String normalizeMode(String mode) {
|
||||
if (mode == null || mode.isBlank()) {
|
||||
return "hybrid";
|
||||
}
|
||||
return mode.trim().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
private static String blankToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private String requiredText(JsonNode node, String fieldName) {
|
||||
JsonNode value = node.get(fieldName);
|
||||
if (value == null || value.asText().isBlank()) {
|
||||
throw new IllegalArgumentException("golden case is missing required field: " + fieldName);
|
||||
}
|
||||
return value.asText();
|
||||
}
|
||||
}
|
||||
@@ -1,539 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.config.DocumentChunkConfig;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* 当前分片策略的单元测试 — 覆盖旧能力回归 + Phase 1 新增能力
|
||||
*/
|
||||
@DisplayName("DocumentChunkService 分片策略")
|
||||
class DocumentChunkServiceTest {
|
||||
|
||||
private DocumentChunkService service;
|
||||
private DocumentChunkConfig config;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
config = new DocumentChunkConfig();
|
||||
config.setMaxSize(800);
|
||||
config.setMaxTokens(500);
|
||||
config.setMaxTokensHard(600);
|
||||
config.setOverlap(100);
|
||||
service = new DocumentChunkService();
|
||||
try {
|
||||
var field = DocumentChunkService.class.getDeclaredField("chunkConfig");
|
||||
field.setAccessible(true);
|
||||
field.set(service, config);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 回归：边界条件 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("边界条件")
|
||||
class BoundaryTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("null 内容 → 空列表")
|
||||
void nullContent_returnsEmpty() {
|
||||
List<DocumentChunk> chunks = service.chunkDocument(null, "/test/null.md");
|
||||
assertTrue(chunks.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("空字符串 → 空列表")
|
||||
void emptyContent_returnsEmpty() {
|
||||
List<DocumentChunk> chunks = service.chunkDocument(" \n ", "/test/empty.md");
|
||||
assertTrue(chunks.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("短文档（≤maxSize）→ 1个分块")
|
||||
void shortDocument_singleChunk() {
|
||||
String content = "这是一篇短文档，内容不超过800个字符。";
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/short.md");
|
||||
|
||||
assertEquals(1, chunks.size());
|
||||
assertEquals(content, chunks.get(0).getContent());
|
||||
assertEquals(0, chunks.get(0).getChunkIndex());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("恰好 maxSize 边界 → 1个分块")
|
||||
void exactlyMaxSize_singleChunk() {
|
||||
String content = "A".repeat(800);
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/boundary.md");
|
||||
assertEquals(1, chunks.size());
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 回归：标题分割 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("Markdown 标题分割")
|
||||
class HeadingSplitTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("单个 H1 标题 → section 继承标题")
|
||||
void singleHeading_titlePropagates() {
|
||||
String content = "# CPU高负载问题\n\n这是CPU高负载的描述内容。";
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/cpu.md");
|
||||
|
||||
assertEquals(1, chunks.size());
|
||||
assertEquals("CPU高负载问题", chunks.get(0).getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("多个标题 → 按标题边界分割")
|
||||
void multipleHeadings_splitAtHeadings() {
|
||||
String content =
|
||||
"# CPU高负载\n\nCPU问题的详细描述。\n\n" +
|
||||
"# 内存高负载\n\n内存问题的详细描述。";
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/multi.md");
|
||||
|
||||
assertEquals(2, chunks.size());
|
||||
assertEquals("CPU高负载", chunks.get(0).getTitle());
|
||||
assertEquals("内存高负载", chunks.get(1).getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("多级标题（H1/H2/H3）→ 标题独立不冲突")
|
||||
void multiLevelHeadings() {
|
||||
String content =
|
||||
"# 一级标题\n\n一级内容。\n\n" +
|
||||
"## 二级标题\n\n二级内容。\n\n" +
|
||||
"### 三级标题\n\n三级内容。";
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/levels.md");
|
||||
assertEquals(3, chunks.size());
|
||||
assertEquals("一级标题", chunks.get(0).getTitle());
|
||||
assertEquals("二级标题", chunks.get(1).getTitle());
|
||||
assertEquals("三级标题", chunks.get(2).getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("H1-H6 全部支持")
|
||||
void allHeadingLevels() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 1; i <= 6; i++) {
|
||||
sb.append("#".repeat(i)).append(" 标题").append(i).append("\n\n内容").append(i).append("。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/h1h6.md");
|
||||
assertEquals(6, chunks.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("无标题文档 → 整个文档作为1个 section")
|
||||
void noHeadings_entireAsOneSection() {
|
||||
String content = "纯文本没有标题。\n\n第二段内容。\n\n第三段内容。";
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/nohead.md");
|
||||
assertFalse(chunks.isEmpty());
|
||||
assertNull(chunks.get(0).getTitle());
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 回归：段落边界切分 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("超长章节 — 段落边界切分")
|
||||
class ParagraphSplitTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("短章节（≤maxSize）→ 不进入段落切割")
|
||||
void shortSection_noParagraphSplit() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 测试\n\n");
|
||||
for (int i = 0; i < 5; i++) {
|
||||
sb.append("段落").append(i).append("：这是一段短内容。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/short_sec.md");
|
||||
assertEquals(1, chunks.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("超长章节 → 在段落边界切分")
|
||||
void longSection_splitsAtParagraphBoundaries() {
|
||||
config.setMaxSize(50);
|
||||
config.setMaxTokens(30);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 长章节\n\n");
|
||||
for (int i = 0; i < 10; i++) {
|
||||
sb.append("段落").append(i).append("：ABCDEFGHIJKLMNOPQRSTUVWXYZ。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_sec.md");
|
||||
assertTrue(chunks.size() >= 2, "超长章节应切分为多个分块，实际: " + chunks.size());
|
||||
|
||||
// 所有分块携带相同的 title
|
||||
for (DocumentChunk c : chunks) {
|
||||
assertEquals("长章节", c.getTitle());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 回归：chunkIndex 元数据 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("分块元数据")
|
||||
class ChunkMetadataTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("chunkIndex 自增且唯一")
|
||||
void chunkIndexSequential() {
|
||||
config.setMaxSize(50);
|
||||
config.setMaxTokens(30);
|
||||
|
||||
StringBuilder sb = new StringBuilder("# Meta\n\n");
|
||||
for (int i = 0; i < 10; i++) {
|
||||
sb.append("段落").append(i).append("：填充内容以触发切分机制。ABCDE。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/meta.md");
|
||||
assertTrue(chunks.size() >= 2);
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
assertEquals(i, chunks.get(i).getChunkIndex(),
|
||||
"chunkIndex 应从0开始连续递增");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("startIndex/endIndex 范围合法 — 无漂移")
|
||||
void indexRangeValid_noDrift() {
|
||||
String content = "# 标题\n\n测试内容。";
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/index.md");
|
||||
|
||||
for (DocumentChunk c : chunks) {
|
||||
assertTrue(c.getStartOffset() >= 0);
|
||||
assertTrue(c.getEndOffset() > c.getStartOffset(),
|
||||
"endIndex(" + c.getEndOffset() + ") 应 > startIndex(" + c.getStartOffset() + ")");
|
||||
assertTrue(c.getEndOffset() <= content.length());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 新增：Token 估算 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("Token 估算")
|
||||
class TokenEstimationTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("纯中文 800 字符 ≈ 800 tokens → 短章节不切")
|
||||
void pureChinese_fewerTokensThanMax() {
|
||||
config.setMaxTokens(400);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 中文测试\n\n");
|
||||
// 纯中文 ~300 字符 ≈ 300 tokens
|
||||
for (int i = 0; i < 3; i++) {
|
||||
sb.append("这是纯中文测试内容的第十").append(i).append("段落。");
|
||||
sb.append("每个中文字符大约占用一个令牌的位置。");
|
||||
sb.append("因此这段文本的令牌数大致等于字符数。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/cn_tokens.md");
|
||||
// 300 字符 ≈ 300 tokens < 400 maxTokens → 1 个分块
|
||||
assertEquals(1, chunks.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("纯英文 2000 字符 ≈ 500 tokens → 刚好不超过上限")
|
||||
void pureEnglish_moreCharactersSameTokens() {
|
||||
config.setMaxTokens(200);
|
||||
config.setMaxTokensHard(250);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# English Test\n\n");
|
||||
for (int i = 0; i < 8; i++) {
|
||||
sb.append("This is paragraph number ").append(i)
|
||||
.append(" containing English text. ")
|
||||
.append("English characters are much cheaper in tokens. ")
|
||||
.append("More filler text here to reach the limit properly. ")
|
||||
.append("Yet another sentence for good measure. ")
|
||||
.append("Still more words needed to reach token limit here.\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/en_tokens.md");
|
||||
// 大量英文才占少量 token → 分块数应少于用字符计数的版本
|
||||
assertTrue(chunks.size() >= 2, "1200+ 字符英文应切分");
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 新增：列表结构感知 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("列表结构感知")
|
||||
class ListStructureTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("有序列表项之间不切分 — 即使超过 maxTokens")
|
||||
void orderedList_notSplitBetweenItems() {
|
||||
config.setMaxTokens(80);
|
||||
config.setMaxTokensHard(200);
|
||||
config.setOverlap(30);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 排查步骤\n\n");
|
||||
// 5个有序列表项，每项 ~40 字符 ≈ 40 tokens，总共 ~200 tokens
|
||||
for (int i = 1; i <= 5; i++) {
|
||||
sb.append(i).append(". 这是排查步骤第").append(i)
|
||||
.append("项，包含具体的操作指引和注意事项说明。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/ordered_list.md");
|
||||
|
||||
// 5项应保持在一起（未触及 hard 上限）
|
||||
assertEquals(1, chunks.size(),
|
||||
"有序列表项不应被拆散，实际分块数: " + chunks.size());
|
||||
|
||||
String content = chunks.get(0).getContent();
|
||||
assertTrue(content.contains("1. "), "应包含第1项");
|
||||
assertTrue(content.contains("5. "), "应包含第5项");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("有序列表触及硬上限 → 在列表项边界强制切分")
|
||||
void orderedList_hardLimitSplits() {
|
||||
config.setMaxTokens(50);
|
||||
config.setMaxTokensHard(100);
|
||||
config.setOverlap(20);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 长列表\n\n");
|
||||
// 每项 ~60 tokens，硬上限 100 → 最多装 1 项多
|
||||
for (int i = 1; i <= 6; i++) {
|
||||
sb.append(i).append(". 这是很长的排查步骤内容，包含详细的说明信息。")
|
||||
.append("每个步骤都要执行多个检查操作。继续填充文本以增加令牌计数。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_list.md");
|
||||
|
||||
System.out.println(" 长列表硬上限测试 — 实际分块数: " + chunks.size());
|
||||
for (DocumentChunk c : chunks) {
|
||||
System.out.println(" Chunk #" + c.getChunkIndex() + ": " + c.getContent().length() + "字符 "
|
||||
+ "| start=" + c.getStartOffset() + " end=" + c.getEndOffset()
|
||||
+ " | preview=" + c.getContent().substring(0, Math.min(60, c.getContent().length())).replace("\n", "\\n"));
|
||||
}
|
||||
|
||||
// 硬上限会强制切分，但每个分块内的列表项应保持连续
|
||||
assertTrue(chunks.size() >= 2, "长列表应至少触发1次切分，实际: " + chunks.size());
|
||||
|
||||
// 验证：除了第一个分块（可能是标题），其余应包含列表项
|
||||
for (int i = 1; i < chunks.size(); i++) {
|
||||
DocumentChunk c = chunks.get(i);
|
||||
assertFalse(c.getContent().isEmpty());
|
||||
assertTrue(c.getContent().matches("(?s).*\\d+\\.\\s.*"),
|
||||
"非标题分块应包含列表项，Chunk #" + c.getChunkIndex()
|
||||
+ " preview: " + c.getContent().substring(0, Math.min(60, c.getContent().length())));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("无序列表项之间不切分")
|
||||
void unorderedList_notSplitBetweenItems() {
|
||||
config.setMaxTokens(80);
|
||||
config.setMaxTokensHard(200);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 检查清单\n\n");
|
||||
for (int i = 1; i <= 5; i++) {
|
||||
sb.append("- 检查项").append(i).append("：确认服务运行状态正常并记录相关指标。\n\n");
|
||||
}
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/unordered_list.md");
|
||||
assertEquals(1, chunks.size(), "无序列表项不应被拆散");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("列表结束后普通段落应从下一段落开始新分块")
|
||||
void listEnds_normalParagraphStartsNewChunk() {
|
||||
config.setMaxTokens(150);
|
||||
config.setMaxTokensHard(250);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("# 文档\n\n");
|
||||
// 先一个普通段落
|
||||
sb.append("这是介绍段落，描述系统的整体架构和设计思路。\n\n");
|
||||
// 有序列表
|
||||
for (int i = 1; i <= 3; i++) {
|
||||
sb.append(i).append(". 列表项第").append(i).append("条，包含操作说明。\n\n");
|
||||
}
|
||||
// 普通段落
|
||||
sb.append("这是总结段落，包含上述操作完成后需要关注的监控指标。\n\n");
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/list_mixed.md");
|
||||
assertTrue(chunks.size() >= 1);
|
||||
// 列表项应保持在一起
|
||||
for (DocumentChunk c : chunks) {
|
||||
String content = c.getContent();
|
||||
// 分块中不应有孤立的单个列表项（除非只有一个）
|
||||
if (content.contains("1. ") && content.contains("3. ")) {
|
||||
// 这个分块包含了全部3个列表项 → 正确
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 新增：代码块结构感知 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("代码块结构感知")
|
||||
class CodeBlockTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("代码块内部不切分")
|
||||
void codeBlock_notSplitInside() {
|
||||
config.setMaxTokens(60);
|
||||
config.setMaxTokensHard(200);
|
||||
config.setOverlap(20);
|
||||
|
||||
String content =
|
||||
"# 代码示例\n\n" +
|
||||
"以下是配置代码：\n\n" +
|
||||
"```yaml\n" +
|
||||
"server:\n" +
|
||||
" port: 8080\n" +
|
||||
" host: localhost\n" +
|
||||
" timeout: 30s\n" +
|
||||
"```\n\n" +
|
||||
"配置说明结束。";
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/code.md");
|
||||
|
||||
// 代码块应保持完整（未触及硬上限）
|
||||
// 验证：至少有一个分块包含完整的 ```...```
|
||||
boolean foundCompleteBlock = false;
|
||||
for (DocumentChunk c : chunks) {
|
||||
String text = c.getContent();
|
||||
if (text.contains("```yaml") && text.contains("```") &&
|
||||
text.indexOf("```yaml") < text.lastIndexOf("```")) {
|
||||
foundCompleteBlock = true;
|
||||
}
|
||||
}
|
||||
// 可能整体在一个分块中
|
||||
assertTrue(chunks.size() >= 1);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 可视化 ====================
|
||||
|
||||
@Nested
|
||||
@DisplayName("可视化 — 打印切分结果")
|
||||
class VisualInspectionTests {
|
||||
|
||||
@Test
|
||||
@DisplayName("模拟运维文档 — 展示新策略效果")
|
||||
void realWorldAIOpsDoc() {
|
||||
config.setMaxTokens(150);
|
||||
config.setMaxTokensHard(200);
|
||||
config.setOverlap(40);
|
||||
|
||||
String doc = """
|
||||
# CPU高负载问题排查指南
|
||||
|
||||
## 问题现象
|
||||
|
||||
服务器CPU使用率持续超过90%，系统响应变慢，用户反馈页面加载超时。
|
||||
监控告警系统连续发出多条CPU使用率告警。
|
||||
|
||||
## 排查步骤
|
||||
|
||||
1. 登录服务器，执行 top 命令查看当前CPU使用率最高的进程。记录进程ID和CPU占用百分比。
|
||||
|
||||
2. 使用 ps aux | grep {进程名} 确认相关服务的运行状态。检查是否有异常进程占用资源。
|
||||
|
||||
3. 查看应用日志，重点关注最近15分钟的ERROR级别日志。使用 tail -n 500 命令。
|
||||
|
||||
4. 检查数据库连接池状态，确认是否有慢查询或连接泄漏。查看慢查询日志。
|
||||
|
||||
5. 检查JVM内存使用情况和GC日志。使用 jstat -gcutil {pid} 1000 命令观察GC频率。
|
||||
|
||||
## 常见原因
|
||||
|
||||
1. 死循环或递归调用导致CPU满载。检查是否有未设置退出条件的循环逻辑。
|
||||
2. 大量正则表达式匹配操作。检查是否有未编译的正则在循环中使用。
|
||||
|
||||
## 解决方案
|
||||
|
||||
根据排查结果采取对应措施：代码问题则回滚或热修复；资源不足则扩容。
|
||||
处理完成后持续观察监控指标30分钟，确认CPU使用率恢复正常。
|
||||
""";
|
||||
|
||||
List<DocumentChunk> chunks = service.chunkDocument(doc, "/kb/cpu_high_usage.md");
|
||||
|
||||
System.out.println("========================================");
|
||||
System.out.println(" Phase 1 新策略效果 — 模拟运维文档");
|
||||
System.out.println(" 配置: maxTokens=150, hard=200, overlap=40");
|
||||
System.out.println(" 总字符数: " + doc.length());
|
||||
System.out.println(" 总分块数: " + chunks.size());
|
||||
System.out.println("========================================\n");
|
||||
|
||||
for (DocumentChunk c : chunks) {
|
||||
System.out.println("┌─ Chunk #" + c.getChunkIndex());
|
||||
System.out.println("│ Title: " + (c.getTitle() != null ? c.getTitle() : "(无)"));
|
||||
System.out.println("│ Range: [" + c.getStartOffset() + "→" + c.getEndOffset() + "] (" + c.getContent().length() + "字符)");
|
||||
// 显示前150字符
|
||||
String preview = c.getContent().length() > 120
|
||||
? c.getContent().substring(0, 120).replace("\n", "\\n") + "..."
|
||||
: c.getContent().replace("\n", "\\n");
|
||||
System.out.println("│ Preview: " + preview);
|
||||
System.out.println("└──────────────────────\n");
|
||||
}
|
||||
|
||||
assertTrue(chunks.size() >= 3, "应产生多个分块");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("中英混排对比 — token vs 字符计数差异")
|
||||
void mixedContentComparison() {
|
||||
config.setMaxTokens(100);
|
||||
config.setMaxTokensHard(150);
|
||||
config.setOverlap(30);
|
||||
|
||||
String chinese = "这是中文内容示范。中文每个字符在LLM中约占用1个token。" +
|
||||
"因此这段文本在上下文窗口中占用的token数较多。" +
|
||||
"继续填充文字以触发切分逻辑，验证中文token估算是否合理。" +
|
||||
"更多中文文本来增加令牌计数。";
|
||||
|
||||
String english = "This is English content. Each word may take one or two tokens. " +
|
||||
"A sentence like this one actually consumes relatively few tokens compared to " +
|
||||
"Chinese characters. More English text to reach the same token count as above. " +
|
||||
"Still need more words because English is very efficient in tokenization. " +
|
||||
"Adding even more content to make this paragraph long enough to test properly.";
|
||||
|
||||
List<DocumentChunk> cnChunks = service.chunkDocument("# CN\n\n" + chinese + "\n\n" + chinese, "/test/cn.md");
|
||||
List<DocumentChunk> enChunks = service.chunkDocument("# EN\n\n" + english + "\n\n" + english, "/test/en.md");
|
||||
|
||||
System.out.println("========================================");
|
||||
System.out.println(" Token 计数对比");
|
||||
System.out.println(" 配置: maxTokens=100, overlap=30");
|
||||
System.out.println("========================================");
|
||||
System.out.println(" 中文文档: " + (chinese.length() * 2) + "字符 → " + cnChunks.size() + "个分块");
|
||||
System.out.println(" 英文文档: " + (english.length() * 2) + "字符 → " + enChunks.size() + "个分块");
|
||||
|
||||
for (DocumentChunk c : cnChunks) {
|
||||
System.out.println(" 中文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
|
||||
}
|
||||
for (DocumentChunk c : enChunks) {
|
||||
System.out.println(" 英文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
|
||||
}
|
||||
System.out.println(" ★ 现在中文和英文的分块数更接近（基于 token 而非字符）");
|
||||
System.out.println("========================================");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.mock.web.MockMultipartFile;
|
||||
@@ -39,16 +38,4 @@ class DocumentManagementServiceTest {
|
||||
assertEquals("payment/runbook.md", storedPath);
|
||||
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolveDocumentIdUsesFrontmatterSourceWhenItFitsDatabaseColumn() {
|
||||
DocumentManagementService service = new DocumentManagementService();
|
||||
Frontmatter frontmatter = Frontmatter.builder()
|
||||
.source("mysql-connection-pool")
|
||||
.build();
|
||||
|
||||
String docId = ReflectionTestUtils.invokeMethod(service, "resolveDocumentId", frontmatter);
|
||||
|
||||
assertEquals("mysql-connection-pool", docId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* FrontmatterParser 单元测试
|
||||
*/
|
||||
class FrontmatterParserTest {
|
||||
|
||||
private FrontmatterParser parser;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
parser = new FrontmatterParser();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHasFrontmatter_withValidFrontmatter() {
|
||||
String content = "---\ntitle: Test\n---\nContent";
|
||||
assertTrue(parser.hasFrontmatter(content));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHasFrontmatter_withoutFrontmatter() {
|
||||
String content = "# Just a title\nContent";
|
||||
assertFalse(parser.hasFrontmatter(content));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHasFrontmatter_nullContent() {
|
||||
assertFalse(parser.hasFrontmatter(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testHasFrontmatter_emptyContent() {
|
||||
assertFalse(parser.hasFrontmatter(""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_validFrontmatter() {
|
||||
String content = """
|
||||
---
|
||||
title: 支付网关错误码
|
||||
keywords: [ERR_TIMEOUT, 超时, 支付网关]
|
||||
summary: 记录了支付网关所有核心错误码
|
||||
category: api
|
||||
---
|
||||
|
||||
# 正文内容
|
||||
""";
|
||||
|
||||
Frontmatter result = parser.parse(content);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals("支付网关错误码", result.getTitle());
|
||||
assertEquals(3, result.getKeywords().size());
|
||||
assertTrue(result.getKeywords().contains("ERR_TIMEOUT"));
|
||||
assertEquals("记录了支付网关所有核心错误码", result.getSummary());
|
||||
assertEquals("api", result.getCategory());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_withoutFrontmatter() {
|
||||
String content = "# Just content\nNo frontmatter here";
|
||||
assertNull(parser.parse(content));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_missingRequiredFields() {
|
||||
String content = """
|
||||
---
|
||||
title: Only Title
|
||||
---
|
||||
Content
|
||||
""";
|
||||
|
||||
// 缺少 keywords 和 summary，应返回 null
|
||||
Frontmatter result = parser.parse(content);
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_malformedYaml() {
|
||||
String content = """
|
||||
---
|
||||
title: Test
|
||||
keywords: [unclosed array
|
||||
---
|
||||
Content
|
||||
""";
|
||||
|
||||
// YAML 格式错误，应返回 null
|
||||
Frontmatter result = parser.parse(content);
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_noClosingDelimiter() {
|
||||
String content = """
|
||||
---
|
||||
title: Test
|
||||
keywords: [test]
|
||||
summary: Test summary
|
||||
|
||||
Content without closing ---
|
||||
""";
|
||||
|
||||
// 缺少结束标记，应返回 null
|
||||
Frontmatter result = parser.parse(content);
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_windowsLineEndings() {
|
||||
String content = "---\r\ntitle: Test\r\nkeywords: [test]\r\nsummary: Summary\r\n---\r\nContent";
|
||||
|
||||
Frontmatter result = parser.parse(content);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals("Test", result.getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_withOptionalFields() {
|
||||
String content = """
|
||||
---
|
||||
title: Test Document
|
||||
keywords: [test, doc]
|
||||
summary: A test document
|
||||
version: 1.0.0
|
||||
author: Test Author
|
||||
---
|
||||
Content
|
||||
""";
|
||||
|
||||
Frontmatter result = parser.parse(content);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals("Test Document", result.getTitle());
|
||||
assertEquals("1.0.0", result.getVersion());
|
||||
assertEquals("Test Author", result.getAuthor());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParse_withRetrievalMetadata() {
|
||||
String content = """
|
||||
---
|
||||
title: MySQL Connection Pool
|
||||
keywords: [connection pool, HikariCP]
|
||||
summary: Diagnose exhausted MySQL connection pools
|
||||
category: database
|
||||
source: mysql-connection-pool
|
||||
breadcrumb: Database > MySQL > Connection Pool
|
||||
kb_scope: rag-eval
|
||||
---
|
||||
Content
|
||||
""";
|
||||
|
||||
Frontmatter result = parser.parse(content);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals("mysql-connection-pool", result.getSource());
|
||||
assertEquals("Database > MySQL > Connection Pool", result.getBreadcrumb());
|
||||
assertEquals("rag-eval", result.getKbScope());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStripFrontmatter_returnsMarkdownBodyOnly() {
|
||||
String content = """
|
||||
---
|
||||
title: Test
|
||||
keywords: [frontmatter-only]
|
||||
summary: Summary
|
||||
---
|
||||
|
||||
# Body
|
||||
|
||||
Body content
|
||||
""";
|
||||
|
||||
String body = parser.stripFrontmatter(content);
|
||||
|
||||
assertFalse(body.contains("frontmatter-only"));
|
||||
assertTrue(body.startsWith("# Body"));
|
||||
assertTrue(body.contains("Body content"));
|
||||
}
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestMethodOrder;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.ToolCallbackProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* 全链路验证：DeepSeek → BGE-M3 → Milvus
|
||||
*/
|
||||
@SpringBootTest
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
@DisplayName("DeepSeek → BGE-M3 → Milvus 全链路")
|
||||
class FullPipelineSmokeTest {
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private EmbeddingModel embeddingModel;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService vectorEmbeddingService;
|
||||
|
||||
@Autowired
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
@TestConfiguration
|
||||
static class MockToolConfig {
|
||||
@Bean
|
||||
public ToolCallbackProvider toolCallbackProvider() {
|
||||
return () -> new ToolCallback[0];
|
||||
}
|
||||
}
|
||||
|
||||
// ===== ① Chat: DeepSeek =====
|
||||
|
||||
@Test
|
||||
@Order(1)
|
||||
@DisplayName("Chat: DeepSeek 聊天验证")
|
||||
void chatDeepSeekWorks() {
|
||||
System.out.println("\n===== ① Chat: DeepSeek =====");
|
||||
System.out.println("ChatModel: " + chatModel.getClass().getSimpleName());
|
||||
System.out.println("ChatOptions: " + chatModel.toString());
|
||||
|
||||
// 直接调用 chat
|
||||
var response = chatModel.call(new Prompt("请用一句话介绍你自己"));
|
||||
String text = response.getResult().getOutput().getText();
|
||||
assertNotNull(text);
|
||||
assertFalse(text.isEmpty());
|
||||
System.out.println("Response: " + text.substring(0, Math.min(200, text.length())) + "...");
|
||||
System.out.println("Chat ✓");
|
||||
}
|
||||
|
||||
// ===== ② Embedding: BGE-M3 via SiliconFlow =====
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
@DisplayName("Embedding: BGE-M3 向量生成验证")
|
||||
void embeddingBgeM3Works() {
|
||||
System.out.println("\n===== ② Embedding: BGE-M3 (SiliconFlow) =====");
|
||||
System.out.println("EmbeddingModel: " + embeddingModel.getClass().getSimpleName());
|
||||
|
||||
String text = "你好，这是一条测试文本";
|
||||
List<Float> vector = vectorEmbeddingService.generateEmbedding(text);
|
||||
|
||||
assertNotNull(vector);
|
||||
assertFalse(vector.isEmpty());
|
||||
assertEquals(1024, vector.size(), "BGE-M3 应返回 1024 维向量");
|
||||
|
||||
// 非零校验
|
||||
boolean hasNonZero = vector.stream().anyMatch(v -> Math.abs(v) > 1e-6);
|
||||
assertTrue(hasNonZero, "向量不能全为零");
|
||||
|
||||
// L2 范数校验：BGE-M3 输出应为 L2 归一化的单位向量
|
||||
double norm = Math.sqrt(vector.stream().mapToDouble(v -> (double) v * v).sum());
|
||||
|
||||
System.out.println("维度: " + vector.size());
|
||||
System.out.println("前5维: " + vector.subList(0, Math.min(5, vector.size())));
|
||||
System.out.println("L2 范数: " + String.format("%.10f", norm));
|
||||
System.out.println("是否归一化 (|norm - 1.0| < 0.01): " + (Math.abs(norm - 1.0) < 0.01));
|
||||
|
||||
assertEquals(1.0, norm, 0.01, "BGE-M3 向量应为 L2 归一化单位向量，实际范数=" + norm);
|
||||
System.out.println("Embedding ✓");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(3)
|
||||
@DisplayName("Embedding: BGE-M3 批量向量生成验证")
|
||||
void embeddingBatchWorks() {
|
||||
System.out.println("\n===== ③ Embedding 批量 =====");
|
||||
List<String> texts = List.of("文本一", "文本二", "文本三");
|
||||
List<List<Float>> results = vectorEmbeddingService.generateEmbeddings(texts);
|
||||
|
||||
assertEquals(3, results.size());
|
||||
for (List<Float> r : results) {
|
||||
assertEquals(1024, r.size());
|
||||
}
|
||||
System.out.println("批量生成: " + results.size() + " 个 向量，各 " + results.get(0).size() + " 维 ✓");
|
||||
}
|
||||
|
||||
// ===== ③ Milvus: 向量搜索 =====
|
||||
|
||||
@Test
|
||||
@Order(4)
|
||||
@DisplayName("Milvus: 连接 + 搜索验证")
|
||||
void milvusSearchWorks() {
|
||||
System.out.println("\n===== ④ Milvus: 向量搜索 =====");
|
||||
|
||||
// 用 BGE-M3 生成查询向量
|
||||
String query = "内部文档";
|
||||
List<Float> queryVector = vectorEmbeddingService.generateQueryVector(query);
|
||||
assertNotNull(queryVector);
|
||||
assertEquals(1024, queryVector.size());
|
||||
|
||||
// 搜索
|
||||
List<VectorSearchService.SearchResult> results =
|
||||
vectorSearchService.searchSimilarDocuments(query, 3);
|
||||
|
||||
assertNotNull(results);
|
||||
System.out.println("查询: " + query);
|
||||
System.out.println("返回: " + results.size() + " 条");
|
||||
|
||||
if (!results.isEmpty()) {
|
||||
// 至少有结果，验证结构
|
||||
for (int i = 0; i < results.size(); i++) {
|
||||
var r = results.get(i);
|
||||
assertNotNull(r.getId());
|
||||
assertNotNull(r.getContent());
|
||||
System.out.println(" [" + (i + 1) + "] id=" + r.getId()
|
||||
+ ", score=" + String.format("%.4f", r.getScore())
|
||||
+ ", content=" + r.getContent().substring(0, Math.min(50, r.getContent().length())) + "...");
|
||||
}
|
||||
} else {
|
||||
System.out.println("(Milvus 中暂无数据，但连接正常)");
|
||||
}
|
||||
|
||||
System.out.println("Milvus ✓");
|
||||
}
|
||||
|
||||
// ===== 汇总 =====
|
||||
|
||||
@Test
|
||||
@Order(5)
|
||||
@DisplayName("总结")
|
||||
void summary() {
|
||||
System.out.println("\n==========================================");
|
||||
System.out.println("全链路验证完成：");
|
||||
System.out.println(" ① Chat → DeepSeek ✓");
|
||||
System.out.println(" ② Embedding → BGE-M3 ✓ (SiliconFlow, 1024维)");
|
||||
System.out.println(" ③ 向量存储 → Milvus ✓ (Zilliz Cloud)");
|
||||
System.out.println("==========================================");
|
||||
}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* KnowledgeIndexService 单元测试
|
||||
*/
|
||||
class KnowledgeIndexServiceTest {
|
||||
|
||||
private KnowledgeIndexService service;
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
service = new KnowledgeIndexService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeBasePath", tempDir.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_singleMatch() {
|
||||
// 准备测试数据
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("test.md")
|
||||
.title("Test")
|
||||
.keywords(List.of("ERR_TIMEOUT", "超时"))
|
||||
.summary("Test summary")
|
||||
.category("api")
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
// 测试匹配
|
||||
List<KnowledgeEntry> results = service.exactMatch("ERR_TIMEOUT");
|
||||
|
||||
assertEquals(1, results.size());
|
||||
assertEquals("Test", results.get(0).getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_caseInsensitive() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("test.md")
|
||||
.keywords(List.of("ERR_TIMEOUT"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
// 小写查询应该匹配
|
||||
List<KnowledgeEntry> results = service.exactMatch("err_timeout");
|
||||
assertEquals(1, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_partialMatch() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("test.md")
|
||||
.keywords(List.of("支付网关"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
// 包含关键词的查询应该匹配
|
||||
List<KnowledgeEntry> results = service.exactMatch("支付网关超时问题");
|
||||
assertEquals(1, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_multipleMatches() {
|
||||
KnowledgeEntry entry1 = KnowledgeEntry.builder()
|
||||
.filePath("doc1.md")
|
||||
.title("Doc 1")
|
||||
.keywords(List.of("超时"))
|
||||
.build();
|
||||
|
||||
KnowledgeEntry entry2 = KnowledgeEntry.builder()
|
||||
.filePath("doc2.md")
|
||||
.title("Doc 2")
|
||||
.keywords(List.of("超时", "错误"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry1);
|
||||
service.addToIndex(entry2);
|
||||
|
||||
// 应该匹配两个文档
|
||||
List<KnowledgeEntry> results = service.exactMatch("超时");
|
||||
assertEquals(2, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAnalyzeQuery_returnsStructuredHint() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("mysql.md")
|
||||
.title("MySQL Doc")
|
||||
.keywords(List.of("mysql", "connection pool"))
|
||||
.category("database")
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("mysql connection pool timeout");
|
||||
|
||||
assertEquals(1, hint.matches().size());
|
||||
assertEquals(List.of("mysql", "connection pool"), hint.matchedKeywords());
|
||||
assertEquals(List.of("database"), hint.domains());
|
||||
assertEquals(List.of("mysql", "connection pool"), hint.entities());
|
||||
assertEquals(List.of("MySQL Doc"), hint.titles());
|
||||
assertEquals("database", hint.singleDomainOrNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAnalyzeQuery_multipleDomainsHasNoSingleDomain() {
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("mysql.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("database")
|
||||
.build());
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("api.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("api")
|
||||
.build());
|
||||
|
||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
||||
|
||||
assertEquals(2, hint.matches().size());
|
||||
assertNull(hint.singleDomainOrNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAnalyzeQuery_filtersByConfiguredKbScope() {
|
||||
ReflectionTestUtils.setField(service, "kbScope", "rag-eval");
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("legacy.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("legacy")
|
||||
.build());
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("eval.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("eval")
|
||||
.kbScope("rag-eval")
|
||||
.build());
|
||||
|
||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
||||
|
||||
assertEquals(1, hint.matches().size());
|
||||
assertEquals(List.of("eval"), hint.domains());
|
||||
assertEquals("eval", hint.singleDomainOrNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAnalyzeQuery_keepsLegacyEntriesWhenNoScopeConfigured() {
|
||||
ReflectionTestUtils.setField(service, "kbScope", "");
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("legacy.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("legacy")
|
||||
.build());
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("eval.md")
|
||||
.keywords(List.of("timeout"))
|
||||
.category("eval")
|
||||
.kbScope("rag-eval")
|
||||
.build());
|
||||
|
||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
||||
|
||||
assertEquals(2, hint.matches().size());
|
||||
assertNull(hint.singleDomainOrNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_noMatch() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("test.md")
|
||||
.keywords(List.of("错误码"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
// 不匹配的查询
|
||||
List<KnowledgeEntry> results = service.exactMatch("限流");
|
||||
assertEquals(0, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_emptyQuery() {
|
||||
List<KnowledgeEntry> results = service.exactMatch("");
|
||||
assertEquals(0, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactMatch_nullQuery() {
|
||||
List<KnowledgeEntry> results = service.exactMatch(null);
|
||||
assertEquals(0, results.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReadDocument_success() throws Exception {
|
||||
// 创建测试文件
|
||||
Path testFile = tempDir.resolve("test.md");
|
||||
String content = "Test content line 1\nTest content line 2\n";
|
||||
Files.writeString(testFile, content);
|
||||
|
||||
// 读取文件
|
||||
String result = service.readDocument(testFile.toString(), 100);
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.contains("Test content"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReadDocument_relativePathUnderBasePath() throws Exception {
|
||||
Path categoryDir = tempDir.resolve("payment");
|
||||
Files.createDirectories(categoryDir);
|
||||
Path testFile = categoryDir.resolve("relative.md");
|
||||
Files.writeString(testFile, "Relative content");
|
||||
|
||||
String result = service.readDocument("payment/relative.md", 100);
|
||||
|
||||
assertEquals("Relative content", result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReadDocument_legacyPathAlreadyContainsBasePath() throws Exception {
|
||||
Path categoryDir = tempDir.resolve("payment");
|
||||
Files.createDirectories(categoryDir);
|
||||
Path testFile = categoryDir.resolve("legacy.md");
|
||||
Files.writeString(testFile, "Legacy content");
|
||||
|
||||
String result = service.readDocument(tempDir.getFileName() + "/payment/legacy.md", 100);
|
||||
|
||||
assertEquals("Legacy content", result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReadDocument_exceedsMaxChars() throws Exception {
|
||||
// 创建超长内容
|
||||
String longContent = "x".repeat(3000);
|
||||
Path testFile = tempDir.resolve("long.md");
|
||||
Files.writeString(testFile, longContent);
|
||||
|
||||
// 读取限制字符数
|
||||
String result = service.readDocument(testFile.toString(), 2000);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals(2003, result.length()); // 2000 + "..."
|
||||
assertTrue(result.endsWith("..."));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testReadDocument_fileNotFound() {
|
||||
String result = service.readDocument("nonexistent.md", 100);
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAddToIndex() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("new.md")
|
||||
.title("New Document")
|
||||
.keywords(List.of("test"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
|
||||
List<KnowledgeEntry> results = service.exactMatch("test");
|
||||
assertEquals(1, results.size());
|
||||
assertEquals("New Document", results.get(0).getTitle());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRemoveFromIndex() {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath("remove.md")
|
||||
.keywords(List.of("test"))
|
||||
.build();
|
||||
|
||||
service.addToIndex(entry);
|
||||
assertEquals(1, service.exactMatch("test").size());
|
||||
|
||||
service.removeFromIndex("remove.md");
|
||||
assertEquals(0, service.exactMatch("test").size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetIndexSize() {
|
||||
assertEquals(0, service.getIndexSize());
|
||||
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("doc1.md")
|
||||
.keywords(List.of("test"))
|
||||
.build());
|
||||
|
||||
assertEquals(1, service.getIndexSize());
|
||||
|
||||
service.addToIndex(KnowledgeEntry.builder()
|
||||
.filePath("doc2.md")
|
||||
.keywords(List.of("test"))
|
||||
.build());
|
||||
|
||||
assertEquals(2, service.getIndexSize());
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.grpc.DataType;
|
||||
import io.milvus.grpc.FlushResponse;
|
||||
import io.milvus.grpc.MutationResult;
|
||||
import io.milvus.grpc.SearchResults;
|
||||
import io.milvus.grpc.ShowCollectionsResponse;
|
||||
import io.milvus.common.clientenum.ConsistencyLevelEnum;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.IndexType;
|
||||
import io.milvus.param.MetricType;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.RpcStatus;
|
||||
import io.milvus.param.collection.*;
|
||||
import io.milvus.param.dml.InsertParam;
|
||||
import io.milvus.param.dml.SearchParam;
|
||||
import io.milvus.param.index.CreateIndexParam;
|
||||
import io.milvus.response.SearchResultsWrapper;
|
||||
import org.junit.jupiter.api.*;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@DisplayName("Milvus 连接验证")
|
||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
||||
class MilvusConnectionTest {
|
||||
|
||||
private static final String COLLECTION = "conn_test";
|
||||
private static final int DIM = 128;
|
||||
|
||||
private static MilvusServiceClient client;
|
||||
|
||||
@BeforeAll
|
||||
static void connect() {
|
||||
String host = envOrDefault("MILVUS_HOST",
|
||||
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
|
||||
int port = Integer.parseInt(envOrDefault("MILVUS_PORT", "443"));
|
||||
String token = System.getenv("MILVUS_TOKEN");
|
||||
|
||||
assertNotNull(token, "环境变量 MILVUS_TOKEN 未设置");
|
||||
|
||||
ConnectParam connectParam = ConnectParam.newBuilder()
|
||||
.withHost(host)
|
||||
.withPort(port)
|
||||
.withToken(token)
|
||||
.withSecure(true)
|
||||
.withDatabaseName("db_4a578da0f27ce9d")
|
||||
.withConnectTimeout(30, TimeUnit.SECONDS)
|
||||
.build();
|
||||
|
||||
client = new MilvusServiceClient(connectParam);
|
||||
System.out.println("连接目标: " + host + ":" + port);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
static void disconnect() {
|
||||
if (client != null) {
|
||||
try {
|
||||
client.dropCollection(DropCollectionParam.newBuilder()
|
||||
.withCollectionName(COLLECTION).build());
|
||||
} catch (Exception ignored) {}
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static String safeMsg(R<?> resp) {
|
||||
try {
|
||||
return resp.getMessage();
|
||||
} catch (Exception e) {
|
||||
return "(no message)";
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(1)
|
||||
@DisplayName("1. 连接成功 - 能列出 collection")
|
||||
void listCollections() {
|
||||
R<ShowCollectionsResponse> resp = client.showCollections(
|
||||
ShowCollectionsParam.newBuilder().build());
|
||||
|
||||
System.out.println("listCollections status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
||||
assertEquals(0, resp.getStatus(), "连接失败，status=" + resp.getStatus());
|
||||
|
||||
List<String> names = resp.getData().getCollectionNamesList();
|
||||
System.out.println("现有 collections: " + names);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(2)
|
||||
@DisplayName("2. 创建测试 collection")
|
||||
void createCollection() {
|
||||
client.dropCollection(DropCollectionParam.newBuilder()
|
||||
.withCollectionName(COLLECTION).build());
|
||||
|
||||
FieldType idField = FieldType.newBuilder()
|
||||
.withName("id")
|
||||
.withDataType(DataType.Int64)
|
||||
.withPrimaryKey(true)
|
||||
.withAutoID(true)
|
||||
.build();
|
||||
|
||||
FieldType vectorField = FieldType.newBuilder()
|
||||
.withName("vector")
|
||||
.withDataType(DataType.FloatVector)
|
||||
.withDimension(DIM)
|
||||
.build();
|
||||
|
||||
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
|
||||
.addFieldType(idField)
|
||||
.addFieldType(vectorField)
|
||||
.build();
|
||||
|
||||
R<RpcStatus> resp = client.createCollection(
|
||||
CreateCollectionParam.newBuilder()
|
||||
.withCollectionName(COLLECTION)
|
||||
.withSchema(schema)
|
||||
.build());
|
||||
|
||||
System.out.println("createCollection status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
||||
assertEquals(0, resp.getStatus(), "创建 collection 失败");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(3)
|
||||
@DisplayName("3. 插入数据 + flush")
|
||||
void insertAndFlush() {
|
||||
List<Float> vec1 = makeVector(1.0f);
|
||||
List<Float> vec2 = makeVector(2.0f);
|
||||
List<Float> vec3 = makeVector(3.0f);
|
||||
|
||||
List<InsertParam.Field> fields = Collections.singletonList(
|
||||
new InsertParam.Field("vector", Arrays.asList(vec1, vec2, vec3))
|
||||
);
|
||||
|
||||
R<MutationResult> insertResp = client.insert(
|
||||
InsertParam.newBuilder()
|
||||
.withCollectionName(COLLECTION)
|
||||
.withFields(fields)
|
||||
.build());
|
||||
|
||||
System.out.println("insert status: " + insertResp.getStatus() + ", msg: " + safeMsg(insertResp));
|
||||
assertEquals(0, insertResp.getStatus(), "插入失败");
|
||||
|
||||
// 官方示例要求：insert 后必须 flush，数据才对搜索可见
|
||||
R<FlushResponse> flushResp = client.flush(FlushParam.newBuilder()
|
||||
.withCollectionNames(Collections.singletonList(COLLECTION))
|
||||
.withSyncFlush(true)
|
||||
.withSyncFlushWaitingTimeout(30L)
|
||||
.build());
|
||||
|
||||
System.out.println("flush status: " + flushResp.getStatus() + ", msg: " + safeMsg(flushResp));
|
||||
assertEquals(0, flushResp.getStatus(), "flush 失败");
|
||||
System.out.println("插入 3 条数据并 flush 完成");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(4)
|
||||
@DisplayName("4. 创建索引 + 加载")
|
||||
void createIndexAndLoad() {
|
||||
R<RpcStatus> indexResp = client.createIndex(
|
||||
CreateIndexParam.newBuilder()
|
||||
.withCollectionName(COLLECTION)
|
||||
.withFieldName("vector")
|
||||
.withIndexType(IndexType.AUTOINDEX)
|
||||
.withMetricType(MetricType.L2)
|
||||
.build());
|
||||
|
||||
System.out.println("createIndex status: " + indexResp.getStatus() + ", msg: " + safeMsg(indexResp));
|
||||
assertEquals(0, indexResp.getStatus(), "创建索引失败");
|
||||
|
||||
R<RpcStatus> loadResp = client.loadCollection(
|
||||
LoadCollectionParam.newBuilder()
|
||||
.withCollectionName(COLLECTION)
|
||||
.withSyncLoad(true)
|
||||
.withSyncLoadWaitingTimeout(30L)
|
||||
.build());
|
||||
|
||||
System.out.println("load status: " + loadResp.getStatus() + ", msg: " + safeMsg(loadResp));
|
||||
assertEquals(0, loadResp.getStatus(), "加载失败");
|
||||
System.out.println("索引创建 + 加载完成");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Order(5)
|
||||
@DisplayName("5. 向量搜索")
|
||||
void search() throws InterruptedException {
|
||||
Thread.sleep(3000);
|
||||
|
||||
List<Float> queryVec = makeVector(1.1f);
|
||||
|
||||
R<SearchResults> resp = null;
|
||||
for (int retry = 0; retry < 10; retry++) {
|
||||
resp = client.search(
|
||||
SearchParam.newBuilder()
|
||||
.withCollectionName(COLLECTION)
|
||||
.withMetricType(MetricType.L2)
|
||||
.withTopK(2)
|
||||
.withVectors(Collections.singletonList(queryVec))
|
||||
.withVectorFieldName("vector")
|
||||
.withParams("{}")
|
||||
.withConsistencyLevel(ConsistencyLevelEnum.STRONG)
|
||||
.build());
|
||||
|
||||
if (resp.getStatus() == 0) break;
|
||||
System.out.println("search retry " + (retry + 1) + ": status=" + resp.getStatus() + ", msg=" + safeMsg(resp));
|
||||
Thread.sleep(5000);
|
||||
}
|
||||
|
||||
System.out.println("search status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
||||
assertEquals(0, resp.getStatus(), "搜索失败");
|
||||
|
||||
SearchResultsWrapper wrapper = new SearchResultsWrapper(resp.getData().getResults());
|
||||
List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(0);
|
||||
|
||||
assertFalse(scores.isEmpty(), "搜索结果不应为空");
|
||||
System.out.println("搜索结果 (top " + scores.size() + "):");
|
||||
for (SearchResultsWrapper.IDScore idScore : scores) {
|
||||
System.out.println(" score=" + idScore.getScore() + ", id=" + idScore.getLongID());
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Float> makeVector(float val) {
|
||||
Float[] arr = new Float[DIM];
|
||||
Arrays.fill(arr, val);
|
||||
return Arrays.asList(arr);
|
||||
}
|
||||
|
||||
private static String envOrDefault(String key, String defaultVal) {
|
||||
String val = System.getenv(key);
|
||||
return (val != null && !val.isEmpty()) ? val : defaultVal;
|
||||
}
|
||||
}
|
||||
-117
@@ -1,117 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonCase;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonReport;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class RagRetrievalSidecarComparisonServiceTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void compareWritesSeparateSidecarReports() throws Exception {
|
||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
||||
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
|
||||
RagSidecarProperties properties = new RagSidecarProperties();
|
||||
RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
|
||||
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
|
||||
vectorSearchService,
|
||||
sidecarService,
|
||||
normalizer,
|
||||
properties,
|
||||
new ObjectMapper()
|
||||
);
|
||||
|
||||
VectorSearchService.SearchResult current = new VectorSearchService.SearchResult();
|
||||
current.setId("current-1");
|
||||
current.setMetadata("{\"_source\":\"current.md\",\"breadcrumb\":\"A\",\"category\":\"api\"}");
|
||||
current.setContent("current content");
|
||||
current.setScore(0.1f);
|
||||
when(vectorSearchService.searchSimilarDocuments("timeout", 3, "api"))
|
||||
.thenReturn(List.of(current));
|
||||
when(sidecarService.search("timeout", 3, "api"))
|
||||
.thenReturn(SidecarRetrievalResponse.builder()
|
||||
.enabled(true)
|
||||
.available(true)
|
||||
.status("available")
|
||||
.results(List.of(ComparableRetrievalResult.builder()
|
||||
.path("sidecar")
|
||||
.rank(1)
|
||||
.source("sidecar.md")
|
||||
.breadcrumb("B")
|
||||
.scoreLabel("similarity")
|
||||
.scoreValue(0.9)
|
||||
.build()))
|
||||
.build());
|
||||
|
||||
RetrievalComparisonReport report = comparisonService.compare(List.of(
|
||||
RetrievalComparisonCase.builder()
|
||||
.caseId("case-1")
|
||||
.scenario("aiops")
|
||||
.query("timeout")
|
||||
.category("api")
|
||||
.build()
|
||||
), 3);
|
||||
|
||||
assertEquals(1, report.getCaseCount());
|
||||
assertEquals("available", report.getSidecarStatus());
|
||||
assertTrue(report.getResults().get(0).getDifferences().contains("top_source_differs"));
|
||||
Path json = tempDir.resolve("sidecar.json");
|
||||
Path markdown = tempDir.resolve("sidecar.md");
|
||||
comparisonService.writeReports(report, json, markdown);
|
||||
|
||||
assertTrue(Files.readString(json).contains("\"sidecarStatus\""));
|
||||
assertTrue(Files.readString(markdown).contains("RAG Sidecar Retrieval Comparison"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void compareGoldenCasesLoadsExistingCaseShape() throws Exception {
|
||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
||||
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
|
||||
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
|
||||
vectorSearchService,
|
||||
sidecarService,
|
||||
new RetrievalResultNormalizer(new ObjectMapper()),
|
||||
new RagSidecarProperties(),
|
||||
new ObjectMapper()
|
||||
);
|
||||
when(vectorSearchService.searchSimilarDocuments("query", 2, null)).thenReturn(List.of());
|
||||
when(sidecarService.search("query", 2, null))
|
||||
.thenReturn(SidecarRetrievalResponse.builder()
|
||||
.enabled(false)
|
||||
.available(false)
|
||||
.status("disabled")
|
||||
.results(List.of())
|
||||
.build());
|
||||
Path cases = tempDir.resolve("cases.json");
|
||||
Files.writeString(cases, """
|
||||
{
|
||||
"topK": 2,
|
||||
"cases": [
|
||||
{"caseId": "case-1", "scenario": "chat", "query": "query"}
|
||||
]
|
||||
}
|
||||
""");
|
||||
|
||||
RetrievalComparisonReport report = comparisonService.compareGoldenCases(cases);
|
||||
|
||||
assertEquals(1, report.getCaseCount());
|
||||
assertEquals(2, report.getTopK());
|
||||
assertEquals("disabled", report.getSidecarStatus());
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.document.Document;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class RetrievalResultNormalizerTest {
|
||||
|
||||
private final RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
|
||||
|
||||
@Test
|
||||
void fromCurrentParsesMetadataAndLabelsDistanceScore() {
|
||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
||||
result.setId("vec-1");
|
||||
result.setMetadata("{\"docId\":\"doc-1\",\"_source\":\"docs/api.md\",\"title\":\"API\",\"breadcrumb\":\"A > B\",\"category\":\"api\"}");
|
||||
result.setContent("abcdef");
|
||||
result.setScore(0.25f);
|
||||
|
||||
ComparableRetrievalResult comparable = normalizer.fromCurrent(result, 1, 3);
|
||||
|
||||
assertEquals("current", comparable.getPath());
|
||||
assertEquals("docs/api.md", comparable.getSource());
|
||||
assertEquals("doc-1", comparable.getDocId());
|
||||
assertEquals("API", comparable.getTitle());
|
||||
assertEquals("A > B", comparable.getBreadcrumb());
|
||||
assertEquals("api", comparable.getCategory());
|
||||
assertEquals("abc...", comparable.getContentPreview());
|
||||
assertEquals("l2_distance", comparable.getScoreLabel());
|
||||
assertEquals(0.25, comparable.getScoreValue(), 0.0001);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fromSidecarNormalizesDocumentMetadataAndLabelsSimilarityScore() {
|
||||
Document document = Document.builder()
|
||||
.id("doc-vector")
|
||||
.text("sidecar content")
|
||||
.metadata(Map.of(
|
||||
"docId", "doc-2",
|
||||
"_source", "docs/sidecar.md",
|
||||
"title", "Sidecar",
|
||||
"breadcrumb", "Root > Sidecar",
|
||||
"category", "rag"
|
||||
))
|
||||
.score(0.91)
|
||||
.build();
|
||||
|
||||
ComparableRetrievalResult comparable = normalizer.fromSidecar(document, 2, 100);
|
||||
|
||||
assertEquals("sidecar", comparable.getPath());
|
||||
assertEquals(2, comparable.getRank());
|
||||
assertEquals("docs/sidecar.md", comparable.getSource());
|
||||
assertEquals("similarity", comparable.getScoreLabel());
|
||||
assertEquals(0.91, comparable.getScoreValue(), 0.0001);
|
||||
}
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.collection.HasCollectionParam;
|
||||
import org.junit.jupiter.api.Assumptions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* 简单的 Milvus 连接测试
|
||||
*/
|
||||
public class SimpleMilvusTest {
|
||||
|
||||
@Test
|
||||
public void testConnection() {
|
||||
String host = System.getenv().getOrDefault(
|
||||
"MILVUS_HOST",
|
||||
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
|
||||
int port = Integer.parseInt(System.getenv().getOrDefault("MILVUS_PORT", "443"));
|
||||
String token = System.getenv("MILVUS_TOKEN");
|
||||
Assumptions.assumeTrue(token != null && !token.isBlank(), "MILVUS_TOKEN is required");
|
||||
|
||||
System.out.println("尝试连接 Milvus...");
|
||||
System.out.println("Host: " + host);
|
||||
System.out.println("Port: " + port);
|
||||
|
||||
try {
|
||||
MilvusServiceClient client = new MilvusServiceClient(
|
||||
ConnectParam.newBuilder()
|
||||
.withHost(host)
|
||||
.withPort(port)
|
||||
.withToken(token)
|
||||
.withSecure(true)
|
||||
.withConnectTimeout(10L, java.util.concurrent.TimeUnit.SECONDS)
|
||||
.build()
|
||||
);
|
||||
|
||||
System.out.println("✓ 客户端创建成功");
|
||||
|
||||
// 测试连接：查询是否存在某个 collection
|
||||
R<Boolean> response = client.hasCollection(
|
||||
HasCollectionParam.newBuilder()
|
||||
.withCollectionName("test_collection")
|
||||
.build()
|
||||
);
|
||||
|
||||
System.out.println("✓ 连接成功！");
|
||||
System.out.println("Status Code: " + response.getStatus());
|
||||
|
||||
if (response.getStatus() == 0 || response.getStatus() == io.milvus.param.R.Status.Success.getCode()) {
|
||||
System.out.println("✓ Milvus 集群状态：正常运行");
|
||||
} else {
|
||||
System.out.println("✗ 响应状态异常: " + response.getStatus());
|
||||
}
|
||||
|
||||
client.close();
|
||||
System.out.println("✓ 连接已关闭");
|
||||
|
||||
} catch (Exception e) {
|
||||
System.err.println("✗ 连接失败：" + e.getMessage());
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException("Milvus 连接失败", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class SpringAiVectorStoreSidecarServiceTest {
|
||||
|
||||
@Test
|
||||
void disabledSidecarDoesNotRequestVectorStore() {
|
||||
RagSidecarProperties properties = new RagSidecarProperties();
|
||||
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
|
||||
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
|
||||
properties,
|
||||
provider,
|
||||
new RetrievalResultNormalizer(new ObjectMapper())
|
||||
);
|
||||
|
||||
SidecarRetrievalResponse response = service.search("query", 3, null);
|
||||
|
||||
assertFalse(response.isEnabled());
|
||||
assertFalse(response.isAvailable());
|
||||
assertEquals("disabled", response.getStatus());
|
||||
verify(provider, never()).getIfAvailable();
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledSidecarReportsMissingVectorStore() {
|
||||
RagSidecarProperties properties = new RagSidecarProperties();
|
||||
properties.setEnabled(true);
|
||||
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
|
||||
when(provider.getIfAvailable()).thenReturn(null);
|
||||
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
|
||||
properties,
|
||||
provider,
|
||||
new RetrievalResultNormalizer(new ObjectMapper())
|
||||
);
|
||||
|
||||
SidecarRetrievalResponse response = service.search("query", 3, "api");
|
||||
|
||||
assertEquals("missing_vector_store", response.getStatus());
|
||||
assertFalse(response.isAvailable());
|
||||
assertEquals(0, response.getResults().size());
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class VectorIndexServiceTest {
|
||||
|
||||
@Test
|
||||
void buildEmbeddingTextIncludesTitleAndBreadcrumb() {
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.title("Connection Pool")
|
||||
.breadcrumb("Database > MySQL > Connection Pool")
|
||||
.content("Check active connections and leak detection.")
|
||||
.build();
|
||||
|
||||
String embeddingText = VectorIndexService.buildEmbeddingText(chunk);
|
||||
|
||||
assertEquals("""
|
||||
Title: Connection Pool
|
||||
Path: Database > MySQL > Connection Pool
|
||||
Content:
|
||||
Check active connections and leak detection.""", embeddingText);
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildEmbeddingTextKeepsPlainContentWhenNoStructureExists() {
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.title(" ")
|
||||
.breadcrumb(null)
|
||||
.content("Plain chunk content.")
|
||||
.build();
|
||||
|
||||
assertEquals("Plain chunk content.", VectorIndexService.buildEmbeddingText(chunk));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildDocumentMetadataUsesFrontmatterRetrievalFields() {
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.chunkIndex(0)
|
||||
.title("Chunk Title")
|
||||
.breadcrumb("Chunk > Path")
|
||||
.content("content")
|
||||
.build();
|
||||
Frontmatter frontmatter = Frontmatter.builder()
|
||||
.title("Document Title")
|
||||
.source("mysql-connection-pool")
|
||||
.breadcrumb("Database > MySQL > Connection Pool")
|
||||
.kbScope("rag-eval")
|
||||
.build();
|
||||
|
||||
Map<String, Object> metadata = VectorIndexService.buildDocumentMetadata(
|
||||
"mysql-connection-pool",
|
||||
chunk,
|
||||
2,
|
||||
"database",
|
||||
frontmatter
|
||||
);
|
||||
|
||||
assertEquals("mysql-connection-pool", metadata.get("docId"));
|
||||
assertEquals("mysql-connection-pool", metadata.get("_source"));
|
||||
assertEquals("mysql-connection-pool", metadata.get("source"));
|
||||
assertEquals("database", metadata.get("category"));
|
||||
assertEquals("rag-eval", metadata.get("kb_scope"));
|
||||
assertEquals("Database > MySQL > Connection Pool", metadata.get("breadcrumb"));
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class VectorSearchServiceTest {
|
||||
|
||||
@Test
|
||||
void denseModeCallsHybridStoreDenseSearch() {
|
||||
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
|
||||
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
|
||||
when(embeddingService.generateQueryVector("query")).thenReturn(List.of(0.1f, 0.2f));
|
||||
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
|
||||
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
|
||||
ReflectionTestUtils.setField(service, "searchMode", "dense");
|
||||
|
||||
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("query", 3, null);
|
||||
|
||||
assertEquals(List.of(expected), results);
|
||||
verify(store).searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull());
|
||||
}
|
||||
|
||||
@Test
|
||||
void hybridModeCallsHybridStoreHybridSearch() {
|
||||
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
|
||||
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
|
||||
when(embeddingService.generateQueryVector("pool")).thenReturn(List.of(0.3f));
|
||||
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
|
||||
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
|
||||
ReflectionTestUtils.setField(service, "searchMode", "hybrid");
|
||||
|
||||
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("pool", 5, "mysql");
|
||||
|
||||
assertEquals(1, results.size());
|
||||
assertEquals("doc-h", results.get(0).getId());
|
||||
verify(store).searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql"));
|
||||
}
|
||||
|
||||
private static VectorSearchService.SearchResult result(String id, float score) {
|
||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
||||
result.setId(id);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("dense");
|
||||
result.setContent("content");
|
||||
result.setMetadata("{}");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* {@link PyRagKnowledgeSearchAdapter} 请求/响应映射契约测试。
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PyRagKnowledgeSearchAdapterTest {
|
||||
|
||||
@Mock
|
||||
private PyRagClient pyRagClient;
|
||||
|
||||
@Test
|
||||
void mapsRequestModeTopKAndCategory() {
|
||||
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||
"q", "hybrid", List.of(), "PRECISE", "supported", null));
|
||||
|
||||
adapter.search(new KnowledgeSearchRequest("网关超时怎么排查", 20, "gateway", KnowledgeSearchMode.HYBRID));
|
||||
|
||||
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
|
||||
verify(pyRagClient).search(captor.capture());
|
||||
PyRagSearchRequest request = captor.getValue();
|
||||
assertEquals("网关超时怎么排查", request.query());
|
||||
assertEquals("hybrid", request.mode());
|
||||
assertEquals(20, request.retrieveK());
|
||||
assertEquals(20, request.returnN());
|
||||
assertEquals(20, request.maxChunksPerDocument());
|
||||
assertEquals("gateway", request.category());
|
||||
assertNull(request.kbScope());
|
||||
}
|
||||
|
||||
@Test
|
||||
void denseModeMapsToSemanticWithoutCategory() {
|
||||
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||
"q", "semantic", List.of(), null, "supported", null));
|
||||
|
||||
adapter.search(KnowledgeSearchRequest.dense("性能优化", 5, null));
|
||||
|
||||
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
|
||||
verify(pyRagClient).search(captor.capture());
|
||||
assertEquals("semantic", captor.getValue().mode());
|
||||
assertNull(captor.getValue().category());
|
||||
}
|
||||
|
||||
@Test
|
||||
void mapsHitsWithRerankScoreAndEvidenceIdentity() {
|
||||
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||
"网关超时怎么排查",
|
||||
"hybrid",
|
||||
List.of(new PyRagSearchHit(
|
||||
"e2e-gateway-b9c1fa12-md-34223174#chunk-1",
|
||||
"e2e-gateway-b9c1fa12-md-34223174",
|
||||
"e2e-gateway-b9c1fa12-md",
|
||||
"网关超时排查",
|
||||
"网关超时排查 > 处理步骤",
|
||||
"网关超时先检查 upstream 配置…",
|
||||
0.9147,
|
||||
"PRECISE")),
|
||||
"PRECISE",
|
||||
"supported",
|
||||
null));
|
||||
|
||||
List<KnowledgeSearchHit> hits = adapter.search(
|
||||
new KnowledgeSearchRequest("网关超时怎么排查", 5, "gateway", KnowledgeSearchMode.HYBRID));
|
||||
|
||||
assertEquals(1, hits.size());
|
||||
KnowledgeSearchHit hit = hits.get(0);
|
||||
assertEquals("e2e-gateway-b9c1fa12-md-34223174#chunk-1", hit.evidenceKey());
|
||||
assertEquals("e2e-gateway-b9c1fa12-md-34223174", hit.docId());
|
||||
assertEquals(1, hit.chunkIndex());
|
||||
assertEquals("网关超时先检查 upstream 配置…", hit.content());
|
||||
assertEquals(0.9147, hit.score(), 1e-9);
|
||||
assertEquals(0.9147, hit.rawScore(), 1e-9);
|
||||
assertEquals(RetrievalScoreLabels.RERANK, hit.scoreLabel());
|
||||
assertEquals("e2e-gateway-b9c1fa12-md", hit.source());
|
||||
assertEquals("网关超时排查", hit.title());
|
||||
assertEquals("网关超时排查 > 处理步骤", hit.breadcrumb());
|
||||
assertEquals(1, hit.originalRank());
|
||||
assertNull(hit.denseDistance());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noEvidenceReturnsEmptyListAsNormalBusinessResult() {
|
||||
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||
"乱码查询", "hybrid", List.of(), null, "no_evidence", null));
|
||||
|
||||
List<KnowledgeSearchHit> hits = adapter.search(
|
||||
KnowledgeSearchRequest.dense("乱码查询", 5, null));
|
||||
|
||||
assertTrue(hits.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void hitWithoutChunkMarkerStillGetsStableIdentity() {
|
||||
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||
"q",
|
||||
"hybrid",
|
||||
List.of(new PyRagSearchHit(
|
||||
"legacy-doc", "legacy-doc", "legacy-doc", "Legacy",
|
||||
null, "content", 0.42, "REFERENCE")),
|
||||
"REFERENCE",
|
||||
"supported",
|
||||
null));
|
||||
|
||||
List<KnowledgeSearchHit> hits = adapter.search(
|
||||
KnowledgeSearchRequest.dense("q", 3, null));
|
||||
|
||||
assertEquals(1, hits.size());
|
||||
// 无 #chunk-N 标记：chunkIndex 为 null，evidenceKey 保留服务端原值
|
||||
assertNull(hits.get(0).chunkIndex());
|
||||
assertEquals("legacy-doc", hits.get(0).evidenceKey());
|
||||
}
|
||||
}
|
||||
@@ -47,10 +47,25 @@ class RetrievalScoreNormalizerTest {
|
||||
assertTrue(hybridRank2 < hybridRank);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rerankScorePassesThroughClamped() {
|
||||
// py-rag rerank 绝对分：quality = score 原样（越大越好），不受 L2/rank 分支影响
|
||||
assertEquals(0.9147, RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.RERANK, 0.9147, 1, 20, 2.0, null), 1e-9);
|
||||
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.RERANK, null, 1, 20, 2.0, null), 1e-9);
|
||||
assertEquals(1.0, RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.RERANK, 1.7, 1, 20, 2.0, null), 1e-9);
|
||||
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
|
||||
RetrievalScoreLabels.RERANK, -0.3, 1, 20, 2.0, null), 1e-9);
|
||||
}
|
||||
|
||||
@Test
|
||||
void canonicalizeAliases() {
|
||||
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
|
||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
|
||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
|
||||
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("rerank"));
|
||||
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("quality_score"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class RrfFusionTest {
|
||||
|
||||
@Test
|
||||
void multiPathAgreementOutranksSinglePathHead() {
|
||||
List<String> dense = List.of("a", "b", "c");
|
||||
List<String> lexical = List.of("c", "b", "d");
|
||||
|
||||
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
|
||||
List.of(
|
||||
new RrfFusion.RankedPath<>("dense", dense, 1.0),
|
||||
new RrfFusion.RankedPath<>("lexical", lexical, 1.0)
|
||||
),
|
||||
60,
|
||||
s -> s
|
||||
);
|
||||
|
||||
// c: dense#3 + lexical#1 ; b: dense#2 + lexical#2 ; a: dense#1 only
|
||||
// With k=60, c edges b slightly, and both beat single-path a.
|
||||
assertEquals("c", fused.get(0).identity());
|
||||
assertEquals("b", fused.get(1).identity());
|
||||
assertEquals("a", fused.get(2).identity());
|
||||
assertTrue(fused.get(0).rrfScore() > fused.get(2).rrfScore());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pathWeightCanElevateSecondaryPath() {
|
||||
List<String> dense = List.of("a", "b");
|
||||
List<String> lexical = List.of("b", "a");
|
||||
|
||||
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
|
||||
List.of(
|
||||
new RrfFusion.RankedPath<>("dense", dense, 1.0),
|
||||
new RrfFusion.RankedPath<>("lexical", lexical, 2.0)
|
||||
),
|
||||
60,
|
||||
s -> s
|
||||
);
|
||||
|
||||
assertEquals("b", fused.get(0).identity());
|
||||
}
|
||||
}
|
||||
-48
@@ -1,48 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class VectorKnowledgeSearchAdapterHybridTest {
|
||||
|
||||
@Test
|
||||
void adapterMapsStoreHitsWithChunkIdentity() {
|
||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
||||
when(vectorSearchService.searchSimilarDocuments("pool timeout", 3, "mysql")).thenReturn(List.of(
|
||||
result("id-1",
|
||||
"{\"_source\":\"c.md\",\"docId\":\"c\",\"chunkIndex\":0,\"title\":\"mysql pool timeout\"}",
|
||||
"mysql pool timeout runbook",
|
||||
0.35f)
|
||||
));
|
||||
|
||||
VectorKnowledgeSearchAdapter adapter =
|
||||
new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper());
|
||||
|
||||
List<KnowledgeSearchHit> hits = adapter.search(
|
||||
new KnowledgeSearchRequest("pool timeout", 3, "mysql", KnowledgeSearchMode.HYBRID));
|
||||
|
||||
assertEquals(1, hits.size());
|
||||
assertEquals("c#chunk-0", hits.get(0).evidenceKey());
|
||||
assertEquals("c.md", hits.get(0).source());
|
||||
verify(vectorSearchService).searchSimilarDocuments("pool timeout", 3, "mysql");
|
||||
}
|
||||
|
||||
private static VectorSearchService.SearchResult result(String id, String metadata, String content, float score) {
|
||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
||||
result.setId(id);
|
||||
result.setMetadata(metadata);
|
||||
result.setContent(content);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("dense");
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,14 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeIndexService;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import com.superbiz.agent.service.retrieval.VectorKnowledgeSearchAdapter;
|
||||
import com.superbiz.agent.service.retrieval.KnowledgeSearchHit;
|
||||
import com.superbiz.agent.service.retrieval.KnowledgeSearchPort;
|
||||
import com.superbiz.agent.service.retrieval.KnowledgeSearchRequest;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
@@ -19,25 +17,26 @@ import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* LookupKnowledgeTool evidence-first contract tests.
|
||||
*
|
||||
* <p>检索后端为 py-rag（经 {@link KnowledgeSearchPort} mock）；L0 query 理解已下沉服务端，
|
||||
* categoryFilter 恒为 null，走 UNFILTERED_VECTOR 单 attempt 主路径。</p>
|
||||
*/
|
||||
class LookupKnowledgeToolTest {
|
||||
|
||||
@Mock
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Mock
|
||||
private VectorSearchService vectorSearchService;
|
||||
private KnowledgeSearchPort knowledgeSearchPort;
|
||||
|
||||
private LookupKnowledgeTool tool;
|
||||
|
||||
@@ -50,9 +49,8 @@ class LookupKnowledgeToolTest {
|
||||
ReflectionTestUtils.setField(postProcessor, "maxChunksPerDocument", 2);
|
||||
KnowledgeContextPacker contextPacker = new KnowledgeContextPacker();
|
||||
tool = new LookupKnowledgeTool();
|
||||
ReflectionTestUtils.setField(tool, "queryTransformer", new KnowledgeQueryTransformer(knowledgeIndexService));
|
||||
ReflectionTestUtils.setField(tool, "documentRetriever",
|
||||
new KnowledgeDocumentRetriever(new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper())));
|
||||
new KnowledgeDocumentRetriever(knowledgeSearchPort));
|
||||
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
|
||||
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
|
||||
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
|
||||
@@ -60,17 +58,11 @@ class LookupKnowledgeToolTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void filteredL1SuccessDoesNotRetry() {
|
||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
||||
VectorSearchService.SearchResult result = searchResult(
|
||||
"vec-1",
|
||||
"{\"_source\":\"db.md\",\"docId\":\"db\",\"chunkIndex\":0,\"title\":\"Database Doc\",\"category\":\"database\"}",
|
||||
"mysql timeout runbook",
|
||||
0.2f);
|
||||
void lookupSuccessSingleAttemptUnfiltered() {
|
||||
KnowledgeSearchHit hit = hit("db#chunk-0", "db", 0, "db.md", "Database Doc",
|
||||
"mysql timeout runbook", 0.9);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
||||
.thenReturn(List.of(result));
|
||||
when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
||||
|
||||
@@ -80,76 +72,30 @@ class LookupKnowledgeToolTest {
|
||||
assertEquals("db#chunk-0", lookup.getEvidenceBlocks().get(0).getEvidenceKey());
|
||||
assertNotNull(lookup.getContextPack());
|
||||
assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook"));
|
||||
assertEquals("FILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||
assertEquals(1, lookup.getRetrievalTrace().getAttempts().size());
|
||||
assertEquals("PRECISE", lookup.getRelevanceLevel());
|
||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
||||
verify(vectorSearchService, never()).searchSimilarDocuments("mysql timeout", 3, null);
|
||||
// 原始 query 直传、无 category 收窄（L0 已下沉 py-rag）
|
||||
verify(knowledgeSearchPort).search(new KnowledgeSearchRequest("mysql timeout", 3, null, null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void filteredLowQualityTriggersRawUnfilteredRetry() {
|
||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
||||
VectorSearchService.SearchResult weak = searchResult(
|
||||
"weak",
|
||||
"{\"_source\":\"weak.md\",\"docId\":\"weak\",\"chunkIndex\":0,\"title\":\"Weak\"}",
|
||||
"weak candidate",
|
||||
1.4f);
|
||||
VectorSearchService.SearchResult strong = searchResult(
|
||||
"strong",
|
||||
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
|
||||
"mysql timeout strong runbook",
|
||||
0.2f);
|
||||
void rerankScoreDrivesRelevanceLevel() {
|
||||
// rerank 绝对分 0.6：quality 原样采用 → REFERENCE（>=0.5 且 <0.75）
|
||||
KnowledgeSearchHit hit = hit("ref#chunk-0", "ref", 0, "ref.md", "Reference Doc",
|
||||
"reference level content", 0.6);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
||||
.thenReturn(List.of(weak));
|
||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
||||
.thenReturn(List.of(strong));
|
||||
when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
||||
LookupResult lookup = tool.lookupKnowledge("reference query");
|
||||
|
||||
assertTrue(lookup.isFound());
|
||||
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||
assertEquals("filtered_vector_low_quality", lookup.getRetrievalTrace().getFallbackReason());
|
||||
assertEquals(2, lookup.getRetrievalTrace().getAttempts().size());
|
||||
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, null);
|
||||
assertEquals("REFERENCE", lookup.getRelevanceLevel());
|
||||
}
|
||||
|
||||
@Test
|
||||
void filteredNoEvidenceTriggersRawUnfilteredRetry() {
|
||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
||||
VectorSearchService.SearchResult strong = searchResult(
|
||||
"strong",
|
||||
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
|
||||
"mysql timeout strong runbook",
|
||||
0.2f);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
||||
.thenReturn(Collections.emptyList());
|
||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
||||
.thenReturn(List.of(strong));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
||||
|
||||
assertTrue(lookup.isFound());
|
||||
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||
assertEquals("filtered_vector_no_evidence", lookup.getRetrievalTrace().getFallbackReason());
|
||||
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
}
|
||||
|
||||
@Test
|
||||
void l0HintsDoNotBecomeStandaloneEvidenceWhenL1Fails() {
|
||||
KnowledgeEntry entry = entry("fallback.md", "Fallback Doc", "fallback", "database");
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("fallback")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("fallback", 3, "database"))
|
||||
.thenReturn(Collections.emptyList());
|
||||
when(vectorSearchService.searchSimilarDocuments("fallback", 3, null))
|
||||
.thenReturn(Collections.emptyList());
|
||||
void emptyHitsYieldNoEvidence() {
|
||||
when(knowledgeSearchPort.search(any())).thenReturn(Collections.emptyList());
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("fallback");
|
||||
|
||||
@@ -157,52 +103,26 @@ class LookupKnowledgeToolTest {
|
||||
assertEquals(0, lookup.getEvidenceBlockCount());
|
||||
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
||||
assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus());
|
||||
assertTrue(String.valueOf(lookup.getRetrievalTrace().getQueryHints()).contains("Fallback Doc"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void noL0HintUsesUnfilteredVectorSearch() {
|
||||
VectorSearchService.SearchResult result = searchResult(
|
||||
"vec-1",
|
||||
"{\"_source\":\"perf.md\",\"docId\":\"perf\",\"chunkIndex\":0,\"title\":\"Perf\"}",
|
||||
"performance tuning guide",
|
||||
0.3f);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("性能优化"))
|
||||
.thenReturn(KnowledgeIndexService.L0Hint.empty());
|
||||
when(vectorSearchService.searchSimilarDocuments("性能优化", 3, null))
|
||||
.thenReturn(List.of(result));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("性能优化");
|
||||
|
||||
assertTrue(lookup.isFound());
|
||||
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||
assertEquals("perf.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
verify(vectorSearchService).searchSimilarDocuments("性能优化", 3, null);
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
|
||||
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
||||
VectorSearchService.SearchResult first = searchResult(
|
||||
"a",
|
||||
"{\"_source\":\"a.md\",\"docId\":\"a\",\"chunkIndex\":0,\"title\":\"Generic\",\"category\":\"other\"}",
|
||||
"generic troubleshooting",
|
||||
0.4f);
|
||||
VectorSearchService.SearchResult second = searchResult(
|
||||
"b",
|
||||
"{\"_source\":\"b.md\",\"docId\":\"b\",\"chunkIndex\":0,\"title\":\"Payment ERR_TIMEOUT\",\"breadcrumb\":\"Payment > Timeout\",\"category\":\"payment\"}",
|
||||
"payment ERR_TIMEOUT timeout diagnosis",
|
||||
0.45f);
|
||||
KnowledgeSearchHit first = hit("a#chunk-0", "a", 0, "a.md", "Generic",
|
||||
"generic troubleshooting", 0.95);
|
||||
KnowledgeSearchHit second = hit("b#chunk-0", "b", 0, "b.md", "Payment ERR_TIMEOUT",
|
||||
"payment ERR_TIMEOUT timeout diagnosis", 0.55);
|
||||
second = new KnowledgeSearchHit(
|
||||
second.id(), second.content(), second.score(), second.rawScore(), second.scoreLabel(),
|
||||
second.metadataJson(), second.metadata(), second.docId(), second.chunkIndex(),
|
||||
second.evidenceKey(), second.source(), second.title(), "Payment > Timeout",
|
||||
2, second.denseDistance());
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
|
||||
.thenReturn(List.of(first, second));
|
||||
when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
||||
|
||||
assertTrue(lookup.isFound());
|
||||
// originalRank order wins; keyword/domain boost must not promote second over first
|
||||
// originalRank 顺序权威；服务端 rerank 分不得改变主序
|
||||
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
|
||||
assertTrue(lookup.getRerankTrace().getItems().stream()
|
||||
@@ -216,21 +136,12 @@ class LookupKnowledgeToolTest {
|
||||
|
||||
@Test
|
||||
void keepsDistinctChunksFromSameSource() {
|
||||
KnowledgeEntry entry = entry("shared.md", "Shared", "shared", "payment");
|
||||
VectorSearchService.SearchResult first = searchResult(
|
||||
"a",
|
||||
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":0,\"title\":\"Shared\"}",
|
||||
"shared content 1",
|
||||
0.2f);
|
||||
VectorSearchService.SearchResult second = searchResult(
|
||||
"b",
|
||||
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":1,\"title\":\"Shared\"}",
|
||||
"shared content 2",
|
||||
0.25f);
|
||||
KnowledgeSearchHit first = hit("shared#chunk-0", "shared", 0, "shared.md", "Shared",
|
||||
"shared content 1", 0.9);
|
||||
KnowledgeSearchHit second = hit("shared#chunk-1", "shared", 1, "shared.md", "Shared",
|
||||
"shared content 2", 0.85);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("shared")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("shared", 3, "payment"))
|
||||
.thenReturn(List.of(first, second));
|
||||
when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
|
||||
|
||||
LookupResult lookup = tool.lookupKnowledge("shared");
|
||||
|
||||
@@ -243,48 +154,29 @@ class LookupKnowledgeToolTest {
|
||||
assertEquals("shared content 2", lookup.getEvidenceBlocks().get(1).getContent());
|
||||
}
|
||||
|
||||
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) {
|
||||
return KnowledgeEntry.builder()
|
||||
.filePath(filePath)
|
||||
.title(title)
|
||||
.keywords(List.of(keyword))
|
||||
.summary(title + " summary")
|
||||
.category(category)
|
||||
.build();
|
||||
}
|
||||
|
||||
private VectorSearchService.SearchResult searchResult(String id,
|
||||
String metadata,
|
||||
String content,
|
||||
float score) {
|
||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
||||
result.setId(id);
|
||||
result.setMetadata(metadata);
|
||||
result.setContent(content);
|
||||
result.setScore(score);
|
||||
result.setRawScore((double) score);
|
||||
result.setScoreLabel("dense");
|
||||
return result;
|
||||
}
|
||||
|
||||
private KnowledgeIndexService.L0Hint hint(KnowledgeEntry... entries) {
|
||||
List<KnowledgeEntry> matches = List.of(entries);
|
||||
List<String> keywords = matches.stream()
|
||||
.flatMap(entry -> entry.getKeywords() == null
|
||||
? java.util.stream.Stream.empty()
|
||||
: entry.getKeywords().stream())
|
||||
.distinct()
|
||||
.toList();
|
||||
List<String> domains = matches.stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(category -> category != null && !category.isBlank())
|
||||
.distinct()
|
||||
.toList();
|
||||
List<String> titles = matches.stream()
|
||||
.map(KnowledgeEntry::getTitle)
|
||||
.filter(title -> title != null && !title.isBlank())
|
||||
.distinct()
|
||||
.toList();
|
||||
return new KnowledgeIndexService.L0Hint(matches, keywords, domains, keywords, titles);
|
||||
/** 构造 py-rag 形态的命中：rerank 绝对分 + evidenceKey（docId#chunk-N）。 */
|
||||
private KnowledgeSearchHit hit(String evidenceKey,
|
||||
String docId,
|
||||
int chunkIndex,
|
||||
String source,
|
||||
String title,
|
||||
String content,
|
||||
double score) {
|
||||
return new KnowledgeSearchHit(
|
||||
evidenceKey,
|
||||
content,
|
||||
score,
|
||||
score,
|
||||
RetrievalScoreLabels.RERANK,
|
||||
null,
|
||||
Map.of(),
|
||||
docId,
|
||||
chunkIndex,
|
||||
evidenceKey,
|
||||
source,
|
||||
title,
|
||||
null,
|
||||
1,
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
services:
|
||||
etcd:
|
||||
container_name: milvus-etcd
|
||||
image: quay.io/coreos/etcd:v3.5.18
|
||||
environment:
|
||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
||||
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
||||
- ETCD_QUOTA_BACKEND_BYTES=4294967296
|
||||
- ETCD_SNAPSHOT_COUNT=50000
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
|
||||
command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
|
||||
healthcheck:
|
||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
container_name: milvus-minio
|
||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
ports:
|
||||
- "9001:9001"
|
||||
- "9000:9000"
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
|
||||
command: minio server /minio_data --console-address ":9001"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
standalone:
|
||||
container_name: milvus-standalone
|
||||
image: milvusdb/milvus:v2.5.10
|
||||
command: ["milvus", "run", "standalone"]
|
||||
security_opt:
|
||||
- seccomp:unconfined
|
||||
environment:
|
||||
ETCD_ENDPOINTS: etcd:2379
|
||||
MINIO_ADDRESS: minio:9000
|
||||
volumes:
|
||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
||||
interval: 30s
|
||||
start_period: 90s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
ports:
|
||||
- "19530:19530"
|
||||
- "9091:9091"
|
||||
depends_on:
|
||||
- "etcd"
|
||||
- "minio"
|
||||
# 这是新增的 Attu 服务哦！
|
||||
attu:
|
||||
container_name: milvus-attu
|
||||
image: zilliz/attu:v2.5
|
||||
ports:
|
||||
- "8000:3000" # 把本地的 8000 端口映射到容器的 3000 端口 (Attu 默认端口)
|
||||
environment:
|
||||
# MILVUS_URL 指向 Docker 网络里的 Milvus standalone 服务
|
||||
MILVUS_URL: standalone:19530
|
||||
depends_on:
|
||||
- standalone # 确保 Milvus 启动后再启动 Attu
|
||||
networks:
|
||||
default:
|
||||
name: milvus
|
||||
Reference in New Issue
Block a user