Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
473bb5d004 | ||
|
|
9cf162482d | ||
|
|
83193bdf4a | ||
|
|
de56551fea | ||
|
|
da18fdf4e1 |
@@ -5,9 +5,9 @@
|
|||||||
SERVER_URL = http://localhost:9900
|
SERVER_URL = http://localhost:9900
|
||||||
UPLOAD_API = $(SERVER_URL)/api/upload
|
UPLOAD_API = $(SERVER_URL)/api/upload
|
||||||
DOCS_DIR = aiops-docs
|
DOCS_DIR = aiops-docs
|
||||||
HEALTH_CHECK_API = $(SERVER_URL)/milvus/health
|
# 服务就绪探测:9900 端口有 HTTP 响应即视为就绪
|
||||||
DOCKER_COMPOSE_FILE = vector-database.yml
|
HEALTH_CHECK = curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL)
|
||||||
MILVUS_CONTAINER = milvus-standalone
|
DOCKER_COMPOSE_FILE = docker-compose.yml
|
||||||
|
|
||||||
# 颜色输出
|
# 颜色输出
|
||||||
GREEN = \033[0;32m
|
GREEN = \033[0;32m
|
||||||
@@ -23,7 +23,7 @@ help:
|
|||||||
@echo ""
|
@echo ""
|
||||||
@echo "可用命令:"
|
@echo "可用命令:"
|
||||||
@echo " $(YELLOW)make init$(NC) - 🚀 一键初始化(启动Docker → 启动服务 → 上传文档)"
|
@echo " $(YELLOW)make init$(NC) - 🚀 一键初始化(启动Docker → 启动服务 → 上传文档)"
|
||||||
@echo " $(YELLOW)make up$(NC) - 启动 Docker Compose(Milvus 向量数据库)"
|
@echo " $(YELLOW)make up$(NC) - 启动 Docker Compose(MySQL/Redis)"
|
||||||
@echo " $(YELLOW)make down$(NC) - 停止 Docker Compose"
|
@echo " $(YELLOW)make down$(NC) - 停止 Docker Compose"
|
||||||
@echo " $(YELLOW)make status$(NC) - 查看 Docker 容器状态"
|
@echo " $(YELLOW)make status$(NC) - 查看 Docker 容器状态"
|
||||||
@echo " $(YELLOW)make start$(NC) - 启动 Spring Boot 服务(后台运行)"
|
@echo " $(YELLOW)make start$(NC) - 启动 Spring Boot 服务(后台运行)"
|
||||||
@@ -42,7 +42,7 @@ help:
|
|||||||
init:
|
init:
|
||||||
@echo "$(GREEN)🚀 开始一键初始化 SuperBizAgent...$(NC)"
|
@echo "$(GREEN)🚀 开始一键初始化 SuperBizAgent...$(NC)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(YELLOW)步骤 1/4: 启动 Docker Compose(Milvus 向量数据库)$(NC)"
|
@echo "$(YELLOW)步骤 1/4: 启动 Docker Compose(MySQL/Redis)$(NC)"
|
||||||
@$(MAKE) up
|
@$(MAKE) up
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(YELLOW)步骤 2/4: 启动 Spring Boot 服务$(NC)"
|
@echo "$(YELLOW)步骤 2/4: 启动 Spring Boot 服务$(NC)"
|
||||||
@@ -51,23 +51,23 @@ init:
|
|||||||
@echo "$(YELLOW)步骤 3/4: 等待服务就绪$(NC)"
|
@echo "$(YELLOW)步骤 3/4: 等待服务就绪$(NC)"
|
||||||
@$(MAKE) wait
|
@$(MAKE) wait
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档到向量数据库$(NC)"
|
@echo "$(YELLOW)步骤 4/4: 上传 AIOps 文档(经 py-rag 入库)$(NC)"
|
||||||
@$(MAKE) upload
|
@$(MAKE) upload
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
||||||
@echo "$(GREEN)✅ 初始化完成!所有文档已成功向量化存储到数据库$(NC)"
|
@echo "$(GREEN)✅ 初始化完成!所有文档已成功入库(py-rag)$(NC)"
|
||||||
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
@echo "$(GREEN)═══════════════════════════════════════════════════════$(NC)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(GREEN)🌐 服务访问地址:$(NC)"
|
@echo "$(GREEN)🌐 服务访问地址:$(NC)"
|
||||||
@echo " API 服务: $(SERVER_URL)"
|
@echo " API 服务: $(SERVER_URL)"
|
||||||
@echo " Attu (Web UI): http://localhost:8000"
|
@echo "$(YELLOW)💡 提示: 知识检索/入库由 py-rag 服务承担,请在其仓库单独启动$(NC)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "$(YELLOW)💡 提示: 服务正在后台运行,查看日志: tail -f server.log$(NC)"
|
@echo "$(YELLOW)💡 提示: 服务正在后台运行,查看日志: tail -f server.log$(NC)"
|
||||||
|
|
||||||
# 启动 Spring Boot 服务(后台运行)
|
# 启动 Spring Boot 服务(后台运行)
|
||||||
start:
|
start:
|
||||||
@echo "$(YELLOW)🚀 启动 Spring Boot 服务...$(NC)"
|
@echo "$(YELLOW)🚀 启动 Spring Boot 服务...$(NC)"
|
||||||
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
@if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||||
echo "$(GREEN)✅ 服务已经在运行中 ($(SERVER_URL))$(NC)"; \
|
echo "$(GREEN)✅ 服务已经在运行中 ($(SERVER_URL))$(NC)"; \
|
||||||
else \
|
else \
|
||||||
echo "$(YELLOW)📦 正在启动服务(后台运行)...$(NC)"; \
|
echo "$(YELLOW)📦 正在启动服务(后台运行)...$(NC)"; \
|
||||||
@@ -84,7 +84,7 @@ wait:
|
|||||||
@max_attempts=60; \
|
@max_attempts=60; \
|
||||||
attempt=0; \
|
attempt=0; \
|
||||||
while [ $$attempt -lt $$max_attempts ]; do \
|
while [ $$attempt -lt $$max_attempts ]; do \
|
||||||
if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||||
echo "$(GREEN)✅ 服务器已就绪!($(SERVER_URL))$(NC)"; \
|
echo "$(GREEN)✅ 服务器已就绪!($(SERVER_URL))$(NC)"; \
|
||||||
exit 0; \
|
exit 0; \
|
||||||
fi; \
|
fi; \
|
||||||
@@ -100,7 +100,7 @@ wait:
|
|||||||
# 检查服务器是否运行
|
# 检查服务器是否运行
|
||||||
check:
|
check:
|
||||||
@echo "$(YELLOW)🔍 检查服务器状态...$(NC)"
|
@echo "$(YELLOW)🔍 检查服务器状态...$(NC)"
|
||||||
@if curl -s -f $(HEALTH_CHECK_API) > /dev/null 2>&1; then \
|
@if curl -s -o /dev/null --connect-timeout 2 $(SERVER_URL); then \
|
||||||
echo "$(GREEN)✅ 服务器运行正常 ($(SERVER_URL))$(NC)"; \
|
echo "$(GREEN)✅ 服务器运行正常 ($(SERVER_URL))$(NC)"; \
|
||||||
else \
|
else \
|
||||||
echo "$(RED)❌ 服务器未运行或无法连接!$(NC)"; \
|
echo "$(RED)❌ 服务器未运行或无法连接!$(NC)"; \
|
||||||
@@ -205,38 +205,14 @@ test-upload:
|
|||||||
echo "$(RED)测试文件不存在$(NC)"; \
|
echo "$(RED)测试文件不存在$(NC)"; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# 启动 Docker Compose(智能检测,避免重复启动)
|
# 启动 Docker Compose(MySQL/Redis;py-rag 服务在其仓库单独启动)
|
||||||
up:
|
up:
|
||||||
@echo "$(YELLOW)🐳 检查 Docker 容器状态...$(NC)"
|
@echo "$(YELLOW)🐳 启动 Docker Compose(MySQL/Redis)...$(NC)"
|
||||||
@if [ ! -f "$(DOCKER_COMPOSE_FILE)" ]; then \
|
@if [ ! -f "$(DOCKER_COMPOSE_FILE)" ]; then \
|
||||||
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
||||||
exit 1; \
|
exit 1; \
|
||||||
fi
|
fi
|
||||||
@if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \
|
@docker-compose -f $(DOCKER_COMPOSE_FILE) up -d && echo "$(GREEN)✅ Docker Compose 启动完成$(NC)"
|
||||||
echo "$(GREEN)✅ Milvus 容器已经在运行中$(NC)"; \
|
|
||||||
echo "$(YELLOW)📋 当前运行的容器:$(NC)"; \
|
|
||||||
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
|
||||||
else \
|
|
||||||
echo "$(YELLOW)🚀 启动 Docker Compose...$(NC)"; \
|
|
||||||
docker-compose -f $(DOCKER_COMPOSE_FILE) up -d; \
|
|
||||||
echo ""; \
|
|
||||||
echo "$(YELLOW)⏳ 等待容器启动...$(NC)"; \
|
|
||||||
sleep 5; \
|
|
||||||
if docker ps --format '{{.Names}}' | grep -q "^$(MILVUS_CONTAINER)$$"; then \
|
|
||||||
echo "$(GREEN)✅ Docker Compose 启动成功!$(NC)"; \
|
|
||||||
echo ""; \
|
|
||||||
echo "$(GREEN)📋 运行中的容器:$(NC)"; \
|
|
||||||
docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
|
||||||
echo ""; \
|
|
||||||
echo "$(GREEN)🌐 服务访问地址:$(NC)"; \
|
|
||||||
echo " Milvus: localhost:19530"; \
|
|
||||||
echo " Attu (Web UI): http://localhost:8000"; \
|
|
||||||
echo " MinIO: http://localhost:9001 (admin/minioadmin)"; \
|
|
||||||
else \
|
|
||||||
echo "$(RED)❌ 容器启动失败,请检查日志: docker-compose -f $(DOCKER_COMPOSE_FILE) logs$(NC)"; \
|
|
||||||
exit 1; \
|
|
||||||
fi; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 停止 Docker Compose
|
# 停止 Docker Compose
|
||||||
down:
|
down:
|
||||||
@@ -245,24 +221,10 @@ down:
|
|||||||
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
echo "$(RED)❌ Docker Compose 文件不存在: $(DOCKER_COMPOSE_FILE)$(NC)"; \
|
||||||
exit 1; \
|
exit 1; \
|
||||||
fi
|
fi
|
||||||
@if docker ps --format '{{.Names}}' | grep -q "milvus"; then \
|
@docker-compose -f $(DOCKER_COMPOSE_FILE) down && echo "$(GREEN)✅ Docker Compose 已停止$(NC)"
|
||||||
docker-compose -f $(DOCKER_COMPOSE_FILE) down; \
|
|
||||||
echo "$(GREEN)✅ Docker Compose 已停止$(NC)"; \
|
|
||||||
else \
|
|
||||||
echo "$(YELLOW)⚠️ 没有运行中的 Milvus 容器$(NC)"; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
# 查看 Docker 容器状态
|
# 查看 Docker 容器状态
|
||||||
status:
|
status:
|
||||||
@echo "$(YELLOW)📊 Docker 容器状态:$(NC)"
|
@echo "$(YELLOW)📊 Docker 容器状态:$(NC)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@if docker ps -a --format '{{.Names}}' | grep -q "milvus"; then \
|
@docker ps -a --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||||||
docker ps -a --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"; \
|
|
||||||
echo ""; \
|
|
||||||
running=$$(docker ps --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
|
|
||||||
total=$$(docker ps -a --filter "name=milvus" --format '{{.Names}}' | wc -l | tr -d ' '); \
|
|
||||||
echo "$(GREEN)运行中: $$running / $$total$(NC)"; \
|
|
||||||
else \
|
|
||||||
echo "$(YELLOW)⚠️ 没有找到 Milvus 相关容器$(NC)"; \
|
|
||||||
echo "$(YELLOW)提示: 运行 'make docker-up' 启动容器$(NC)"; \
|
|
||||||
fi
|
|
||||||
|
|||||||
+2
-58
@@ -40,68 +40,12 @@ services:
|
|||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
# Milvus 向量数据库(Standalone 模式)
|
# 向量检索与知识入库由独立的 py-rag 服务承担(见 py-rag 仓库),
|
||||||
# 注意:生产环境建议使用 Zilliz Cloud 或 Milvus 集群
|
# 其依赖的 Milvus/etcd/MinIO 随 py-rag 部署,不再由本 compose 管理。
|
||||||
etcd:
|
|
||||||
image: quay.io/coreos/etcd:v3.5.5
|
|
||||||
container_name: superbiz-etcd
|
|
||||||
environment:
|
|
||||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
|
||||||
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
|
||||||
- ETCD_QUOTA_BACKEND_BYTES=4294967296
|
|
||||||
- ETCD_SNAPSHOT_COUNT=50000
|
|
||||||
volumes:
|
|
||||||
- etcd-data:/etcd
|
|
||||||
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
minio:
|
|
||||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
|
||||||
container_name: superbiz-minio
|
|
||||||
environment:
|
|
||||||
MINIO_ACCESS_KEY: minioadmin
|
|
||||||
MINIO_SECRET_KEY: minioadmin
|
|
||||||
volumes:
|
|
||||||
- minio-data:/minio_data
|
|
||||||
command: minio server /minio_data --console-address ":9001"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
milvus:
|
|
||||||
image: milvusdb/milvus:v2.3.3
|
|
||||||
container_name: superbiz-milvus
|
|
||||||
depends_on:
|
|
||||||
- etcd
|
|
||||||
- minio
|
|
||||||
environment:
|
|
||||||
ETCD_ENDPOINTS: etcd:2379
|
|
||||||
MINIO_ADDRESS: minio:9000
|
|
||||||
volumes:
|
|
||||||
- milvus-data:/var/lib/milvus
|
|
||||||
ports:
|
|
||||||
- "19530:19530"
|
|
||||||
- "9091:9091"
|
|
||||||
command: ["milvus", "run", "standalone"]
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
|
||||||
interval: 30s
|
|
||||||
start_period: 90s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
mysql-data:
|
mysql-data:
|
||||||
redis-data:
|
redis-data:
|
||||||
etcd-data:
|
|
||||||
minio-data:
|
|
||||||
milvus-data:
|
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@
|
|||||||
| [architecture/agent-orchestration.md](architecture/agent-orchestration.md) | Agent 编排架构 |
|
| [architecture/agent-orchestration.md](architecture/agent-orchestration.md) | Agent 编排架构 |
|
||||||
| [architecture/harness-quality-gates.md](architecture/harness-quality-gates.md) | Harness 与质量门禁 |
|
| [architecture/harness-quality-gates.md](architecture/harness-quality-gates.md) | Harness 与质量门禁 |
|
||||||
| [architecture/session-trace-lifecycle.md](architecture/session-trace-lifecycle.md) | 会话与 Trace 生命周期 |
|
| [architecture/session-trace-lifecycle.md](architecture/session-trace-lifecycle.md) | 会话与 Trace 生命周期 |
|
||||||
| [architecture/RAG知识检索架构.md](architecture/RAG知识检索架构.md) | 当前 hybrid 检索架构 |
|
| [architecture/RAG知识检索架构.md](architecture/RAG知识检索架构.md) | 当前检索架构(py-rag 知识服务接入) |
|
||||||
| [issues/README.md](issues/README.md) | MVP issue 索引 |
|
| [issues/README.md](issues/README.md) | MVP issue 索引 |
|
||||||
| [tables/README.md](tables/README.md) | 当前 MySQL 表说明 |
|
| [tables/README.md](tables/README.md) | 当前 MySQL 表说明 |
|
||||||
| [demo/README.md](demo/README.md) | Demo 运行和演示材料 |
|
| [demo/README.md](demo/README.md) | Demo 运行和演示材料 |
|
||||||
|
|||||||
@@ -1,8 +1,14 @@
|
|||||||
# RAG 检索可观测性、审计与 Trace(现行)
|
# RAG 检索可观测性、审计与 Trace(现行)
|
||||||
|
|
||||||
**更新日期**:2026-07-28
|
**更新日期**:2026-09-29
|
||||||
**状态**:当前可运行
|
**状态**:当前可运行
|
||||||
**关联**:`lookup_knowledge`、Harness `ToolBoundary`、`tool_invocation`、`DiagnosisTraceService`、离线 eval
|
**关联**:`lookup_knowledge`、Harness `ToolBoundary`、`tool_invocation`、`DiagnosisTraceService`、离线 eval
|
||||||
|
|
||||||
|
> **2026-09-29 RAG 抽离影响**:检索后端切换为 py-rag 服务(见 [RAG知识检索架构.md](./RAG知识检索架构.md))。
|
||||||
|
> Trace / 审计的三层边界与读写接口**不变**;变化仅在内容语义:
|
||||||
|
> L0 已下沉(`queryHints` 恒为空结构、`categoryFilter` 恒为 null、attempt 只剩 `UNFILTERED_VECTOR`),
|
||||||
|
> 质量分统一为 py-rag rerank 绝对分(scoreLabel=RERANK)。
|
||||||
|
> 文中涉及 FILTERED/RETRY attempt 的示例为历史数据读法,保留供回放旧 Run。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -119,21 +125,21 @@ flowchart TB
|
|||||||
| 字段 | 含义 |
|
| 字段 | 含义 |
|
||||||
|------|------|
|
|------|------|
|
||||||
| `originalQuery` | 原始查询 |
|
| `originalQuery` | 原始查询 |
|
||||||
| `rewrittenQuery` | L0/变换后用于检索的 query |
|
| `rewrittenQuery` | 用于检索的 query(L0 下沉后恒等于 originalQuery) |
|
||||||
| `categoryFilter` | 首次过滤的 category(可 null) |
|
| `categoryFilter` | 首次过滤的 category(L0 下沉后恒为 null) |
|
||||||
| `selectedAttempt` | 最终采用的 attempt 名 |
|
| `selectedAttempt` | 最终采用的 attempt 名 |
|
||||||
| `fallbackReason` | 如 `filtered_vector_low_quality`;未降级为 null |
|
| `fallbackReason` | 如 `filtered_vector_low_quality`;未降级为 null |
|
||||||
| `evidenceStatus` | 内部:`supported` / `no_evidence` 等 |
|
| `evidenceStatus` | 内部:`supported` / `no_evidence` 等 |
|
||||||
| `queryHints` | L0:domains、keywords、entities、l0_match_count… |
|
| `queryHints` | L0 提示(下沉后恒为空 domains/keywords/entities 与 l0_match_count=0) |
|
||||||
| `attempts[]` | 每次检索尝试快照 |
|
| `attempts[]` | 每次检索尝试快照 |
|
||||||
|
|
||||||
**常见 `selectedAttempt`:**
|
**`selectedAttempt` 取值:**
|
||||||
|
|
||||||
| 值 | 含义 |
|
| 值 | 含义 |
|
||||||
|----|------|
|
|----|------|
|
||||||
| `FILTERED_VECTOR` | 带 category 的首次检索即采用 |
|
| `UNFILTERED_VECTOR` | **当前唯一会出现**:无 category,直传 py-rag 检索 |
|
||||||
| `UNFILTERED_VECTOR` | 无 category,直接全库检索 |
|
| `FILTERED_VECTOR` | (历史)带 category 的首次检索即采用;L0 下沉后不再产生 |
|
||||||
| `UNFILTERED_VECTOR_RETRY` | filtered 低质/无证据后去掉 category 重试 |
|
| `UNFILTERED_VECTOR_RETRY` | (历史)filtered 低质/无证据后去掉 category 重试;分支保留但不可达,仅见于旧 Run 回放 |
|
||||||
|
|
||||||
**单次 `attempts[]` 元素:**
|
**单次 `attempts[]` 元素:**
|
||||||
|
|
||||||
@@ -148,21 +154,18 @@ flowchart TB
|
|||||||
| `durationMs` | 耗时 |
|
| `durationMs` | 耗时 |
|
||||||
| `errorMessage` | 失败时 |
|
| `errorMessage` | 失败时 |
|
||||||
|
|
||||||
### 3.3 一次典型路径(含 filter fallback)
|
### 3.3 一次典型路径(当前:单 attempt 直传)
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
flowchart TB
|
flowchart TB
|
||||||
Q[query] --> L0[L0 hint → 可选 categoryFilter]
|
Q[query 原始句直传] --> A1[attempt UNFILTERED_VECTOR<br/>PyRagKnowledgeSearchAdapter → py-rag]
|
||||||
L0 --> A1[attempt FILTERED_VECTOR]
|
A1 --> POST[PostProcess · evidenceBlocks · relevanceLevel]
|
||||||
A1 --> PQ{isLowQuality?}
|
|
||||||
PQ -->|否| USE1[selectedAttempt = FILTERED_VECTOR]
|
|
||||||
PQ -->|是| A2[attempt UNFILTERED_VECTOR_RETRY]
|
|
||||||
A2 --> USE2[selectedAttempt = RETRY<br/>fallbackReason = low_quality / no_evidence]
|
|
||||||
USE1 --> POST[PostProcess · evidenceBlocks · relevanceLevel]
|
|
||||||
USE2 --> POST
|
|
||||||
POST --> LR[LookupResult 完整 Trace]
|
POST --> LR[LookupResult 完整 Trace]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
> 历史 filter fallback 路径(L0 → FILTERED_VECTOR → 低质 → UNFILTERED_VECTOR_RETRY)的流程图已随 L0 下沉移除;
|
||||||
|
> 旧 Run 的 Trace 回放仍可见该结构,字段含义见 §3.2。
|
||||||
|
|
||||||
### 3.4 与 Agent 投影的关系
|
### 3.4 与 Agent 投影的关系
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
@@ -208,35 +211,27 @@ flowchart LR
|
|||||||
| `output_preview` | level/attempt 摘要 | status=… |
|
| `output_preview` | level/attempt 摘要 | status=… |
|
||||||
| `duration_ms` / `success` | 有 | 有 |
|
| `duration_ms` / `success` | 有 | 有 |
|
||||||
|
|
||||||
### 4.3 `retrieval_details`(rag_lookup_v1)示例
|
### 4.3 `retrieval_details`(rag_lookup_v1)示例(当前形态)
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"audit_schema": "rag_lookup_v1",
|
"audit_schema": "rag_lookup_v1",
|
||||||
"search_mode": "hybrid",
|
"search_mode": "hybrid",
|
||||||
"selected_attempt": "UNFILTERED_VECTOR_RETRY",
|
"selected_attempt": "UNFILTERED_VECTOR",
|
||||||
"fallback_reason": "filtered_vector_low_quality",
|
"fallback_reason": null,
|
||||||
"category_filter": "overfilter-decoy",
|
"category_filter": null,
|
||||||
"evidence_keys": ["doc#chunk-0"],
|
"evidence_keys": ["e2e-gateway-b9c1fa12-md-34223174#chunk-1"],
|
||||||
"sources": ["doc"],
|
"sources": ["e2e-gateway-b9c1fa12-md"],
|
||||||
"evidence_candidate_count": 8,
|
"evidence_candidate_count": 5,
|
||||||
"evidence_block_count": 2,
|
"evidence_block_count": 2,
|
||||||
"l0_hints": { "domains": ["mysql"], "matched_keywords": ["pool"] },
|
"l0_hints": { "domains": [], "matched_keywords": [] },
|
||||||
"attempts": [
|
"attempts": [
|
||||||
{
|
{
|
||||||
"name": "FILTERED_VECTOR",
|
"name": "UNFILTERED_VECTOR",
|
||||||
"category_filter": "overfilter-decoy",
|
|
||||||
"candidate_count": 2,
|
|
||||||
"usable": false,
|
|
||||||
"top_similarity": 0.3,
|
|
||||||
"duration_ms": 12
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "UNFILTERED_VECTOR_RETRY",
|
|
||||||
"candidate_count": 5,
|
"candidate_count": 5,
|
||||||
"usable": true,
|
"usable": true,
|
||||||
"top_similarity": 0.9,
|
"top_similarity": 0.91,
|
||||||
"duration_ms": 20
|
"duration_ms": 640
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"truncated": false,
|
"truncated": false,
|
||||||
@@ -247,7 +242,8 @@ flowchart LR
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
**默认不落库:** 原始 query 全文、chunk 正文 excerpt、完整 rerankTrace(体积与隐私)。
|
**默认不落库:** 原始 query 全文、chunk 正文 excerpt、完整 rerankTrace(体积与隐私)。
|
||||||
|
历史 Run 中 `selected_attempt=FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 与非空 `l0_hints` 为 L0 下沉前的旧数据形态。
|
||||||
|
|
||||||
### 4.4 Trace API:人怎么读 RAG
|
### 4.4 Trace API:人怎么读 RAG
|
||||||
|
|
||||||
@@ -304,12 +300,12 @@ flowchart TB
|
|||||||
|
|
||||||
| 现象 | 优先看 |
|
| 现象 | 优先看 |
|
||||||
|------|--------|
|
|------|--------|
|
||||||
| 为何走了 retry | `fallback_reason` + 两次 `attempts` |
|
| 是否 hybrid | `search_mode`(hybrid/semantic 对应 py-rag 融合/纯向量) |
|
||||||
| 是否 hybrid | `search_mode` |
|
| 滤错域 | (历史)`category_filter` + L0 domains;L0 下沉后恒为 null |
|
||||||
| 滤错域 | `category_filter` + L0 domains |
|
|
||||||
| 相关度档 | 列 `relevanceLevel`(PRECISE/REFERENCE) |
|
| 相关度档 | 列 `relevanceLevel`(PRECISE/REFERENCE) |
|
||||||
| 返回了哪些块 | `evidence_keys` / `sources`(无正文) |
|
| 返回了哪些块 | `evidence_keys` / `sources`(无正文) |
|
||||||
| Agent 是否被截断 | `truncated` / `returned_count` |
|
| Agent 是否被截断 | `truncated` / `returned_count` |
|
||||||
|
| 为何走了 retry | (历史)`fallback_reason` + 两次 `attempts`;L0 下沉后单 attempt,不再产生 |
|
||||||
|
|
||||||
### 4.6 diagnosis_trace 事件 vs tool_invocation 行
|
### 4.6 diagnosis_trace 事件 vs tool_invocation 行
|
||||||
|
|
||||||
@@ -352,10 +348,11 @@ flowchart LR
|
|||||||
|
|
||||||
| 旧(archive `retrieval-observability`) | 现 |
|
| 旧(archive `retrieval-observability`) | 现 |
|
||||||
|----------------------------------------|-----|
|
|----------------------------------------|-----|
|
||||||
| `vector-store.mode` 多后端 | `search_mode` dense\|hybrid,单一 V2 store |
|
| `vector-store.mode` 多后端 | `search_mode` dense\|hybrid(现映射 py-rag semantic\|hybrid) |
|
||||||
| sink 理想化未落地 | `RagLookupAuditEnricher` + 列回填 |
|
| sink 理想化未落地 | `RagLookupAuditEnricher` + 列回填 |
|
||||||
| `relevance_level` 混用 evidence_status | **列仅 RAG 等级**;契约状态在 details |
|
| `relevance_level` 混用 evidence_status | **列仅 RAG 等级**;契约状态在 details |
|
||||||
| 未写清 Trace API 读法 | 本文 §4.4–4.5 |
|
| 未写清 Trace API 读法 | 本文 §4.4–4.5 |
|
||||||
|
| L0 hint / FILTERED-RETRY attempt(2026-07-28 形态) | 2026-09-29 L0 下沉 py-rag:单 attempt、queryHints 恒空、质量分 RERANK 直传 |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+131
-271
@@ -1,13 +1,16 @@
|
|||||||
# RAG 知识检索架构
|
# RAG 知识检索架构
|
||||||
|
|
||||||
**更新日期**:2026-07-28
|
**更新日期**:2026-09-29
|
||||||
**状态**:当前可运行架构
|
**状态**:当前可运行架构
|
||||||
**关联实现**:`lookup_knowledge`、`MilvusHybridKnowledgeStore`、`KnowledgeSearchPort`
|
**关联实现**:`lookup_knowledge`、`KnowledgeSearchPort`、`PyRagKnowledgeSearchAdapter`、`PyRagClient`
|
||||||
**关联运维**:`scripts/rebuild_hybrid_knowledge.py`、`POST /api/knowledge/rebuild-hybrid`
|
**关联契约**:py-rag 仓库 `docs/Java接入文档.md`(API v1,冻结面)
|
||||||
|
**关联运维**:py-rag `/api/v1/collections:rebuild`(全量重建)、py-rag `/api/v1/documents:ingest`(单文档入库)
|
||||||
|
|
||||||
## 1. 定位
|
## 1. 定位
|
||||||
|
|
||||||
知识检索是 Diagnosis Agent 的显式证据工具,不是隐式 Advisor。
|
知识检索是 Diagnosis Agent 的显式证据工具,不是隐式 Advisor。
|
||||||
|
检索算法(dense+BM25 融合、rerank、判级)与文档入库(解析、frontmatter、分块、向量化)
|
||||||
|
**全部由独立的 py-rag 知识服务承担**;Java 侧只保留 harness 消费面与 HTTP 客户端。
|
||||||
|
|
||||||
```text
|
```text
|
||||||
Diagnosis Agent
|
Diagnosis Agent
|
||||||
@@ -19,7 +22,7 @@ Diagnosis Agent
|
|||||||
目标:
|
目标:
|
||||||
|
|
||||||
- 保留 Agent 可见的工具调用与证据边界
|
- 保留 Agent 可见的工具调用与证据边界
|
||||||
- 用单一向量后端完成 dense + BM25 hybrid 检索
|
- 检索/入库基础设施外置为独立服务,Java 侧不感知引擎细节
|
||||||
- 用 chunk 级证据身份保证同文档多片段可同时进入上下文
|
- 用 chunk 级证据身份保证同文档多片段可同时进入上下文
|
||||||
- 检索行为可配置、可重建、可审计
|
- 检索行为可配置、可重建、可审计
|
||||||
|
|
||||||
@@ -41,336 +44,193 @@ flowchart LR
|
|||||||
subgraph RetrievalBoundary["Retrieval boundary"]
|
subgraph RetrievalBoundary["Retrieval boundary"]
|
||||||
Backend["LookupKnowledgeTool"]
|
Backend["LookupKnowledgeTool"]
|
||||||
Port["KnowledgeSearchPort"]
|
Port["KnowledgeSearchPort"]
|
||||||
Store["MilvusHybridKnowledgeStore"]
|
Remote["PyRagKnowledgeSearchAdapter"]
|
||||||
|
Service["py-rag 知识服务 (HTTP /api/v1)"]
|
||||||
end
|
end
|
||||||
|
|
||||||
Agent --> Tool
|
Agent --> Tool
|
||||||
Tool --> Adapter
|
Tool --> Adapter
|
||||||
Adapter --> Backend
|
Adapter --> Backend
|
||||||
Backend --> Port
|
Backend --> Port
|
||||||
Port --> Store
|
Port --> Remote
|
||||||
|
Remote -->|HTTP| Service
|
||||||
Adapter --> Projector
|
Adapter --> Projector
|
||||||
Adapter --> Canonical
|
Adapter --> Canonical
|
||||||
```
|
```
|
||||||
|
|
||||||
| 边界 | 职责 | 不负责 |
|
| 边界 | 职责 | 不负责 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Agent | 决定何时检索、如何用证据写报告 | 不直接访问 Milvus / MySQL 元数据表 |
|
| Agent | 决定何时检索、如何用证据写报告 | 不直接访问 py-rag / MySQL 元数据表 |
|
||||||
| Harness | Tool 校验、投影裁剪、canonical 存证 | 不改写检索排序算法 |
|
| Harness | Tool 校验、投影裁剪、canonical 存证 | 不改写检索排序算法 |
|
||||||
| Retrieval | L0 hint、dense/BM25 召回、后处理、打包 | 不绕过 ACI 直接给 Agent 原始库响应 |
|
| Retrieval | 请求映射、后处理、打包;py-rag 承担召回/融合/rerank/判级 | 不绕过 ACI 直接给 Agent 原始库响应 |
|
||||||
|
|
||||||
## 3. 当前主链路
|
## 3. 当前主链路
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
flowchart TD
|
flowchart TD
|
||||||
A["lookup_knowledge(query)"] --> B["KnowledgeQueryTransformer"]
|
A["lookup_knowledge(query)"] --> B["原始 query 直传(L0 已下沉 py-rag)"]
|
||||||
B --> C["L0 hint: domain / keywords / categoryFilter"]
|
B --> C["KnowledgeDocumentRetriever"]
|
||||||
C --> D["KnowledgeDocumentRetriever"]
|
C --> D["KnowledgeSearchPort"]
|
||||||
D --> E["KnowledgeSearchPort"]
|
D --> E["PyRagKnowledgeSearchAdapter"]
|
||||||
E --> F["VectorSearchService"]
|
E -->|POST /api/v1/search| F["py-rag: dense+BM25 融合 / rerank / 判级"]
|
||||||
F --> G{"retrieval.search.mode"}
|
F --> G["hits + evidenceKey(docId#chunk-N)"]
|
||||||
G -->|dense| H["MilvusHybridKnowledgeStore.searchDense"]
|
G --> H["KnowledgeEvidencePostProcessor"]
|
||||||
G -->|hybrid| I["MilvusHybridKnowledgeStore.searchHybrid"]
|
H --> I["evidenceKey dedup / maxChunksPerDocument / return-n"]
|
||||||
H --> J["candidates + chunk identity"]
|
I --> J["KnowledgeContextPacker"]
|
||||||
I --> J
|
J --> K["LookupResultAssembler"]
|
||||||
J --> K["KnowledgeEvidencePostProcessor"]
|
K --> L["RagResultProjector"]
|
||||||
K --> L["evidenceKey dedup / maxChunksPerDocument / return-n"]
|
L --> M["Agent-facing RagToolResult"]
|
||||||
L --> M{"filtered low quality?"}
|
|
||||||
M -->|yes and had categoryFilter| N["unfiltered retry"]
|
|
||||||
N --> K
|
|
||||||
M -->|no| O["KnowledgeContextPacker"]
|
|
||||||
O --> P["LookupResultAssembler"]
|
|
||||||
P --> Q["RagResultProjector"]
|
|
||||||
Q --> R["Agent-facing RagToolResult"]
|
|
||||||
```
|
```
|
||||||
|
|
||||||
对应代码:
|
对应代码:
|
||||||
|
|
||||||
| 阶段 | 类 | 职责 |
|
| 阶段 | 类 | 职责 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Tool 编排 | `LookupKnowledgeTool` | 串联 transform / retrieve / post / pack |
|
| Tool 编排 | `LookupKnowledgeTool` | 串联 retrieve / post / pack;UNFILTERED 单 attempt 主路径 |
|
||||||
| Query 理解 | `KnowledgeQueryTransformer` + `KnowledgeIndexService` | L0 只产 hint 与可选 category filter |
|
| 检索端口 | `KnowledgeSearchPort` / `PyRagKnowledgeSearchAdapter` | 防腐层;请求映射 + 命中归一化 |
|
||||||
| 检索端口 | `KnowledgeSearchPort` / `VectorKnowledgeSearchAdapter` | 屏蔽底层存储细节 |
|
| HTTP 客户端 | `PyRagClient` | API v1 调用、错误信封(`E_*`)、`X-Request-ID`、分端点超时 |
|
||||||
| 检索门面 | `VectorSearchService` | `dense` 或 `hybrid` 路由 |
|
| 后处理 | `KnowledgeEvidencePostProcessor` | qualityScore(RERANK 直传)、chunk 去重、相关度等级 |
|
||||||
| 向量后端 | `MilvusHybridKnowledgeStore` | 唯一知识库读写后端(MilvusClientV2) |
|
|
||||||
| 后处理 | `KnowledgeEvidencePostProcessor` | 归一化、规则 boost、chunk 去重、相关度等级 |
|
|
||||||
| 打包 | `KnowledgeContextPacker` | 有界 context pack |
|
| 打包 | `KnowledgeContextPacker` | 有界 context pack |
|
||||||
| 投影 | `RagResultProjector` | 只暴露 Agent 可见 evidence 字段 |
|
| 投影 | `RagResultProjector` | 只暴露 Agent 可见 evidence 字段 |
|
||||||
|
|
||||||
## 4. 唯一向量后端:MilvusClientV2
|
2026-09-29 抽离时删除的 Java 侧组件:`MilvusHybridKnowledgeStore`、`VectorSearchService`、
|
||||||
|
`VectorKnowledgeSearchAdapter`、`RrfFusion` / `LexicalRanker`(融合评分下沉)、
|
||||||
|
`KnowledgeIndexService` / `KnowledgeDomainService`(L0 索引)、
|
||||||
|
`DocumentChunkService` / `FrontmatterParser` / `TextExtractorService`(入库解析)、
|
||||||
|
`KnowledgeQueryTransformer`(L0 query 理解)。
|
||||||
|
|
||||||
### 4.1 已废弃路径
|
## 4. py-rag 检索契约映射
|
||||||
|
|
||||||
以下路径**不再**用于 `lookup_knowledge`:
|
请求映射(`PyRagKnowledgeSearchAdapter`):
|
||||||
|
|
||||||
- legacy `MilvusServiceClient` search / insert
|
| Java(KnowledgeSearchRequest) | py-rag(/api/v1/search) | 说明 |
|
||||||
- `retrieval.vector-store.mode=sdk|spring|auto`
|
|---|---|---|
|
||||||
- Spring AI `VectorStore` 作为知识检索主路径
|
| `query` | `query` | 原始检索句直传,服务端自行处理边界与精排 |
|
||||||
|
| `mode=DENSE` | `mode=semantic` | 纯向量,对照/排障用 |
|
||||||
|
| `mode=HYBRID` | `mode=hybrid` | dense+BM25 融合,线上主路径(`retrieval.search.mode: hybrid`) |
|
||||||
|
| `topK` | `retrieve_k` / `return_n` / `max_chunks_per_document` | 三者同置 topK:chunk 去重截断由 Java 后处理器统一负责,避免服务端预截断 |
|
||||||
|
| `categoryFilter` | `category` | L0 下沉后恒为 null(不过滤) |
|
||||||
|
| — | `kb_scope` | 不传,由 py-rag 部署配置决定 |
|
||||||
|
|
||||||
### 4.2 当前后端
|
响应映射:
|
||||||
|
|
||||||
|
| py-rag | Java(KnowledgeSearchHit) | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `evidence_key` | `evidenceKey` / `docId` / `chunkIndex` | `docId#chunk-N`,与 EvidenceGuard 验真约定一致 |
|
||||||
|
| `excerpt` | `content` | 进入 context pack 的正文 |
|
||||||
|
| `quality_score` | `score` / `rawScore` | rerank 绝对相关分 [0,1],越大越好 |
|
||||||
|
| — | `scoreLabel=RERANK` | `RetrievalScoreNormalizer` 对 RERANK 分支 quality 原样 clamp,不走 L2/rank 归一化 |
|
||||||
|
| `relevance_level` | (参考值) | Java 后处理按同阈值(0.75/0.5)独立判级,语义一致 |
|
||||||
|
| `evidence_status=no_evidence` | 空列表 | 正常业务响应(服务端保证 hits=[]),Agent 侧按"无知识可用"处理 |
|
||||||
|
|
||||||
|
超时与重试矩阵见 py-rag 仓库 `docs/Java接入文档.md` 第 6 节;Java 侧由 `pyrag.*` 配置承载。
|
||||||
|
|
||||||
|
## 5. 入库与重建
|
||||||
|
|
||||||
|
### 5.1 日常写入
|
||||||
|
|
||||||
```text
|
```text
|
||||||
写入:
|
业务上传(DocumentController /api/documents/upload)
|
||||||
VectorIndexService
|
-> MySQL api_document(业务元数据:faultSource 等)+ 本地原件保存
|
||||||
-> MilvusHybridKnowledgeStore.upsertChunk
|
-> PyRagClient.ingest(multipart 透传原件 + category)
|
||||||
|
-> py-rag:解析 / frontmatter 校验 / 分块 / 向量化 / 索引
|
||||||
|
|
||||||
读取:
|
简单上传(FileUploadController /api/upload)
|
||||||
VectorSearchService
|
-> 本地保存 + PyRagClient.ingest(category 可选参数,缺省 default;入库失败不影响上传成功语义)
|
||||||
-> MilvusHybridKnowledgeStore.searchDense
|
|
||||||
-> MilvusHybridKnowledgeStore.searchHybrid
|
|
||||||
```
|
```
|
||||||
|
|
||||||
默认 collection:
|
MySQL `api_document.docId` 取 py-rag 返回的 `doc_id`(路径 slug + 内容 SHA-256 前 8 位),
|
||||||
|
与检索 `evidence_key` 的 docId 段对齐;`chunk_count` 取 ingest 响应。
|
||||||
|
|
||||||
|
### 5.2 全量重建
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/v1/collections:rebuild?confirm=REBUILD (py-rag 服务端)
|
||||||
|
GET /api/v1/tasks/{task_id} (任务状态)
|
||||||
|
```
|
||||||
|
|
||||||
|
- rebuild 为 py-rag 异步任务;执行期间 ingest 返回 409(`E_REBUILD_IN_PROGRESS`),search 不受影响
|
||||||
|
- 原料为 py-rag 服务端 `data/knowledge_base/` 下历次 ingest 落盘的 md
|
||||||
|
- 旧 Java 侧 `POST /api/knowledge/rebuild-hybrid` 与 `scripts/rebuild_hybrid_knowledge.py` 已删除
|
||||||
|
|
||||||
|
### 5.3 单文档删除
|
||||||
|
|
||||||
|
py-rag API v1 没有单文档删除端点。`DELETE /api/documents/{docId}` 只删 MySQL 元数据与本地原件;
|
||||||
|
py-rag 侧已入库内容需全量重建后才会消失(见 `DocumentManagementService.deleteDocument` 注释)。
|
||||||
|
|
||||||
|
## 6. 部署与配置
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
milvus:
|
pyrag:
|
||||||
collection: biz
|
base-url: ${PYRAG_BASE_URL:http://localhost:8000}
|
||||||
|
connect-timeout-ms: 3000
|
||||||
|
search-read-timeout-ms: 5000 # 正常 300–800ms(含 rerank 外呼)
|
||||||
|
ingest-read-timeout-ms: 30000
|
||||||
|
default-read-timeout-ms: 10000
|
||||||
```
|
```
|
||||||
|
|
||||||
重建时会 drop + recreate 该 collection,并按 dense + BM25 schema 重建。
|
- Milvus / etcd / MinIO 随 py-rag 部署,不再由本仓库 `docker-compose.yml` 管理(compose 仅剩 MySQL/Redis)
|
||||||
|
- `vector-database.yml` 已删除;Makefile 的 up/down/status 只管 MySQL/Redis
|
||||||
|
- `retrieval.search.mode: hybrid` 语义保留:映射 py-rag 的 `hybrid` / `semantic`
|
||||||
|
|
||||||
## 5. Collection Schema
|
## 7. 证据身份与去重(不变)
|
||||||
|
|
||||||
`biz`(可配置)逻辑字段:
|
|
||||||
|
|
||||||
| 字段 | 类型 | 用途 |
|
|
||||||
|---|---|---|
|
|
||||||
| `id` | VarChar PK | chunk 级主键 |
|
|
||||||
| `content` | VarChar | 返回给 Agent 的原文片段 |
|
|
||||||
| `search_text` | VarChar + analyzer | BM25 输入文本 |
|
|
||||||
| `sparse_vector` | SparseFloatVector | BM25 Function 输出 |
|
|
||||||
| `vector` | FloatVector | dense embedding |
|
|
||||||
| `metadata` | JSON | docId / chunkIndex / category / kb_scope / title / breadcrumb 等 |
|
|
||||||
|
|
||||||
Function:
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
BM25(search_text -> sparse_vector)
|
evidenceKey = docId#chunk-{chunkIndex}
|
||||||
```
|
|
||||||
|
|
||||||
索引:
|
|
||||||
|
|
||||||
```text
|
|
||||||
vector -> IVF_FLAT + L2
|
|
||||||
sparse_vector -> SPARSE_INVERTED_INDEX + BM25
|
|
||||||
```
|
|
||||||
|
|
||||||
写入时:
|
|
||||||
|
|
||||||
- `content` 保存原始 chunk 正文
|
|
||||||
- `search_text` / dense embedding 使用 `Title + Path + Content` 拼装文本
|
|
||||||
- metadata 必须带 `docId`、`chunkIndex`,供 chunk 级证据身份使用
|
|
||||||
|
|
||||||
## 6. 检索模式
|
|
||||||
|
|
||||||
配置:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
retrieval:
|
|
||||||
search:
|
|
||||||
mode: hybrid # dense | hybrid(见 6.0 用途约定)
|
|
||||||
hybrid:
|
|
||||||
rrf-k: 60
|
|
||||||
kb-scope: ""
|
|
||||||
rag:
|
|
||||||
retrieve-k: 20
|
|
||||||
return-n: 5
|
|
||||||
max-chunks-per-document: 2
|
|
||||||
```
|
|
||||||
|
|
||||||
### 6.0 模式用途约定(保留双 mode 的原因)
|
|
||||||
|
|
||||||
知识库 **只维护一套** dense + BM25 schema 数据(默认 collection `biz`)。
|
|
||||||
`retrieval.search.mode` 切换的是**同库上的查询算法**,不是两套互斥索引、也不是两套写入路径。
|
|
||||||
|
|
||||||
| 模式 | 定位 | 说明 |
|
|
||||||
|---|---|---|
|
|
||||||
| **hybrid** | **线上主路径 / 默认** | dense ANN + 服务端 BM25 + RRF;`lookup_knowledge` 正式召回只认此模式 |
|
|
||||||
| **dense** | **对照 / 评测 / 排障** | 仅 dense ANN,用于和 hybrid 对比召回效果(命中文档/chunk、排名差异等) |
|
|
||||||
|
|
||||||
约定:
|
|
||||||
|
|
||||||
1. 生产配置保持 `mode: hybrid`;不要把 dense 当成第二套长期并行的线上策略。
|
|
||||||
2. 需要看「去掉 BM25+RRF 后召回差在哪」时,临时切 `mode: dense`,其它参数(`retrieve-k`、`return-n`、category filter、query 集)尽量固定,再切回 hybrid。
|
|
||||||
3. hybrid 入库的数据 **完全适用于** dense-only 查询:每条 chunk 都写了 `vector`;dense 模式只是不使用 `sparse_vector` / BM25 子路。
|
|
||||||
4. 代码里 `@Value` 在配置缺失时的兜底仍可能是 `dense`(历史兼容);**以 `application.yml` 的 hybrid 为准**。若做回归,确认运行配置而不是只看注解默认值。
|
|
||||||
|
|
||||||
不建议的用法:
|
|
||||||
|
|
||||||
- 按请求/按租户在 dense 与 hybrid 之间当产品功能随意切换(当前也无稳定的 per-call mode 覆盖)。
|
|
||||||
- 把 dense 模式的相关度表现直接当成 hybrid 的最终质量结论(hybrid 排序信 RRF,后处理分数仍多 L2 兼容,见下节)。
|
|
||||||
|
|
||||||
### 6.1 dense(对照基线)
|
|
||||||
|
|
||||||
```text
|
|
||||||
query
|
|
||||||
-> embedding
|
|
||||||
-> dense ANN on vector
|
|
||||||
-> topK
|
|
||||||
```
|
|
||||||
|
|
||||||
仅走 `vector` 字段的 L2 ANN。用于基线对比,不作为正式主路径。
|
|
||||||
|
|
||||||
### 6.2 hybrid(当前默认 / 主路径)
|
|
||||||
|
|
||||||
```text
|
|
||||||
query
|
|
||||||
-> path A: dense ANN(query embedding)
|
|
||||||
-> path B: BM25 sparse ANN(raw query text)
|
|
||||||
-> Milvus hybridSearch + RRFRanker(k)
|
|
||||||
-> topK fused hits
|
|
||||||
```
|
|
||||||
|
|
||||||
说明:hybrid **内部**的 dense 子路是融合的一部分,与配置项 mode=dense(整次检索只跑单路 ANN)不是同一概念。
|
|
||||||
|
|
||||||
分数与后处理(quality 统一,2026-07-28):
|
|
||||||
|
|
||||||
- 一级 scoreLabel 仅 **dense | hybrid**(旧别名 canonicalize)。
|
|
||||||
- **dense**:score = L2;qualityScore = 1 - clamp(L2)/maxL2Distance。
|
|
||||||
- **hybrid**:返回序 = RRF 序;qualityScore 由 **本轮 rank 线性映射**(不把 RRF 原分当 L2;不做 dense L2 回填覆盖主分;无 m25_only_* 一级 label)。
|
|
||||||
- 后处理:**统一**消费 qualityScore;排序主序 = originalRank;**不做** L0 关键词/domain contains 加分改序(重叠仅可写 hitReasons 解释)。
|
|
||||||
-
|
|
||||||
elevance_level / category 低质 unfiltered retry:只看 top qualityScore 与阈值。
|
|
||||||
- 实现:RetrievalScoreNormalizer、KnowledgeEvidencePostProcessor;详见 OpenSpec
|
|
||||||
ag-quality-score-unify。
|
|
||||||
|
|
||||||
### 6.3 category filter 与降级
|
|
||||||
|
|
||||||
```text
|
|
||||||
if L0 给出唯一 domain:
|
|
||||||
先 filtered 检索
|
|
||||||
if 无证据或 topSimilarity < referenceThreshold:
|
|
||||||
再 unfiltered retry
|
|
||||||
else:
|
|
||||||
直接 unfiltered
|
|
||||||
```
|
|
||||||
|
|
||||||
这里的 filter 是 metadata category / kb_scope 约束,不是第二套向量库。
|
|
||||||
|
|
||||||
## 7. 证据身份与去重
|
|
||||||
|
|
||||||
Delivery 1 已落地:
|
|
||||||
|
|
||||||
```text
|
|
||||||
evidenceKey =
|
|
||||||
docId#chunk-{chunkIndex}
|
|
||||||
fallback: vector:{id}
|
fallback: vector:{id}
|
||||||
fallback: rank:{n}
|
fallback: rank:{n}
|
||||||
```
|
```
|
||||||
|
|
||||||
规则:
|
- 去重按 `evidenceKey`,同文档不同 chunk 可同时保留
|
||||||
|
- `rag.max-chunks-per-document` / `rag.return-n` 在 Java 后处理器生效
|
||||||
|
- Agent 投影中的 `document_id` 使用 chunk 级 evidenceKey,EvidenceGuard 据此验真
|
||||||
|
|
||||||
- 去重按 `evidenceKey`,不是按 source 文档路径
|
## 8. Agent 可见契约(不变)
|
||||||
- 同文档不同 chunk 可同时保留
|
|
||||||
- `rag.max-chunks-per-document` 限制单文档最多进入结果的 chunk 数
|
|
||||||
- `rag.return-n` 限制后处理后最多返回条数
|
|
||||||
- Agent 投影中的 `document_id` 使用 chunk 级 evidenceKey
|
|
||||||
|
|
||||||
这保证 hybrid 召回的多片段不会在后处理/投影阶段被文档级折叠吞掉。
|
|
||||||
|
|
||||||
## 8. L0 / L1 职责
|
|
||||||
|
|
||||||
| 层 | 做什么 | 不做什么 |
|
|
||||||
|---|---|---|
|
|
||||||
| L0 | domain/keyword hint、可选 category filter、trace 解释、轻规则 boost | 不直接当事实 evidence |
|
|
||||||
| L1 dense/BM25 | 事实证据召回 | 不依赖 frontmatter 关键词命中才返回正文 |
|
|
||||||
|
|
||||||
L0 命中文档正文不会在 L1 失败时兜底成 evidence。
|
|
||||||
|
|
||||||
## 9. Agent 可见契约
|
|
||||||
|
|
||||||
Agent 只看到有界 `RagToolResult`:
|
Agent 只看到有界 `RagToolResult`:
|
||||||
|
|
||||||
- `evidence_status`
|
- `evidence_status` / `tool_call_id` / `query`
|
||||||
- `tool_call_id`
|
|
||||||
- `query`
|
|
||||||
- `evidence[]`:`document_id` / `source` / `title` / `breadcrumb` / `excerpt`
|
- `evidence[]`:`document_id` / `source` / `title` / `breadcrumb` / `excerpt`
|
||||||
- `relevance_level`
|
- `relevance_level`(PRECISE / REFERENCE;`RagRelevanceLevel.HIGHLY_RELEVANT` 为保留档)
|
||||||
- `truncated` / `returned_count`
|
- `truncated` / `returned_count`
|
||||||
|
|
||||||
不暴露:
|
不暴露:raw score、retrievalTrace / rerankTrace、contextPack 全文、py-rag 地址与凭据。
|
||||||
|
完整内部结果仍在 `LookupResult` 中,供审计与调试使用。Trace / 审计读法见
|
||||||
|
[RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)。
|
||||||
|
|
||||||
- raw score / fused score
|
## 9. L0 下沉
|
||||||
- retrievalTrace / rerankTrace
|
|
||||||
- contextPack 全文
|
|
||||||
- Milvus 内部字段与凭据
|
|
||||||
|
|
||||||
完整内部结果仍在 `LookupResult` 中,供审计与调试使用。
|
L0 query 理解(domain/keyword hint → 可选 categoryFilter)随本次抽离**整体下沉 py-rag**:
|
||||||
|
|
||||||
### 9.1 Trace 与审计(现行入口)
|
- `KnowledgeQueryTransformer` / `KnowledgeIndexService.analyzeQuery` 已删除
|
||||||
|
- `lookup_knowledge` 直传原始 query;`categoryFilter` 恒为 null,走 UNFILTERED 单 attempt
|
||||||
|
- `LookupKnowledgeTool` 中 filtered→unfiltered 降级分支保留但不可达(作为未来 Java 侧过滤策略的兜底骨架)
|
||||||
|
- `RetrievalTrace.queryHints` 恒为空结构;`attempt` 命名只剩 `UNFILTERED_VECTOR`
|
||||||
|
|
||||||
请求内 `retrievalTrace` / 落库 `tool_invocation` / Trace API 读法见:
|
## 10. 与旧文档的差异
|
||||||
|
|
||||||
**[RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)**
|
| 旧描述(2026-07-28 版) | 当前实现(2026-09-29 抽离后) |
|
||||||
|
|
||||||
要点:
|
|
||||||
|
|
||||||
- Agent **看不到**完整 retrievalTrace;人通过 `GET /api/diagnosis/{sessionId}/trace` 的 `toolInvocations[].retrievalDetails` 回放。
|
|
||||||
- `relevance_level` 列存 RAG 等级(PRECISE/REFERENCE);`evidence_status` 在 details JSON。
|
|
||||||
- 默认审计不落原始 query 全文与 excerpt 正文。
|
|
||||||
|
|
||||||
## 10. 写入与重建
|
|
||||||
|
|
||||||
### 10.1 日常写入
|
|
||||||
|
|
||||||
文档上传 / 知识库初始化:
|
|
||||||
|
|
||||||
```text
|
|
||||||
markdown
|
|
||||||
-> frontmatter + body
|
|
||||||
-> DocumentChunkService
|
|
||||||
-> dense embedding + search_text
|
|
||||||
-> MilvusHybridKnowledgeStore.upsertChunk
|
|
||||||
-> MySQL api_document + L0 memory index
|
|
||||||
```
|
|
||||||
|
|
||||||
### 10.2 全量重建
|
|
||||||
|
|
||||||
危险操作,需显式确认:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
|
||||||
```
|
|
||||||
|
|
||||||
等价 API:
|
|
||||||
|
|
||||||
```text
|
|
||||||
POST /api/knowledge/rebuild-hybrid?confirm=REBUILD
|
|
||||||
```
|
|
||||||
|
|
||||||
服务端顺序:
|
|
||||||
|
|
||||||
1. drop + recreate `milvus.collection`(默认 `biz`)
|
|
||||||
2. 清空 MySQL `api_document`
|
|
||||||
3. 清空内存 L0
|
|
||||||
4. 扫描 `knowledge_base/**/*.md`(跳过 `README.md`)force 导入
|
|
||||||
|
|
||||||
不会修改磁盘上的 `knowledge_base/` 源文件。
|
|
||||||
|
|
||||||
## 11. 与旧文档的差异
|
|
||||||
|
|
||||||
| 旧描述(已归档) | 当前实现 |
|
|
||||||
|---|---|
|
|---|---|
|
||||||
| Spring AI VectorStore 主路径 + SDK fallback | 单一 MilvusClientV2 后端 |
|
| 进程内 MilvusClientV2,dense+BM25+RRF | py-rag 服务端承担;Java 经 `KnowledgeSearchPort` → HTTP |
|
||||||
| `retrieval.vector-store.mode=auto/sdk/spring` | 已移除;改为 `retrieval.search.mode=dense/hybrid` |
|
| `retrieval.search.mode` 切 Milvus 查询算法 | 同名配置映射 py-rag `hybrid` / `semantic` |
|
||||||
| source 级 evidence 去重 | chunk 级 `evidenceKey` 去重 |
|
| scoreLabel 仅 dense \| hybrid,L2/rank 归一化 | 新增 `RERANK`:py-rag rerank 绝对分直传 |
|
||||||
| 应用层 sparse-lite lexical 伪 hybrid | 库内 dense ANN + BM25 + RRFRanker |
|
| L0 hint + categoryFilter + filtered/unfiltered retry | L0 下沉;原始 query 直传,单 attempt |
|
||||||
| 新建 `biz_hybrid` 过渡 collection | 默认使用并重建 `biz` |
|
| Java 侧 frontmatter 解析 / 分块 / embedding 入库 | py-rag `documents:ingest`;Java 只做 MySQL 元数据 + 原件保存 |
|
||||||
|
| `POST /api/knowledge/rebuild-hybrid` 重建 | py-rag `POST /api/v1/collections:rebuild` |
|
||||||
|
| `GET /milvus/health` 健康检查 | py-rag `GET /api/v1/health`(含 milvus/embedding/rerank 探针) |
|
||||||
|
| 删除文档同步删向量索引 | 仅删 MySQL+本地文件;py-rag 侧靠全量重建生效 |
|
||||||
|
|
||||||
历史材料见:
|
历史材料见:
|
||||||
|
|
||||||
- `mvp/architecture/archive/2026-07-22-legacy/rag-architecture.md`
|
- `mvp/architecture/archive/2026-07-22-legacy/rag-architecture.md`(Milvus 前身)
|
||||||
- `mvp/architecture/archive/2026-07-22-legacy/modular-rag-pipeline.md`
|
- `mvp/engineering/rag/Milvus-Hybrid接入清单.md`(进程内 Milvus hybrid 接入纪要,已过时)
|
||||||
|
- 本仓库 git 历史:`refactor/extract-rag-module` 分支,71 文件 / 约 -8000 行
|
||||||
|
|
||||||
## 12. 当前已知边界
|
## 11. 当前已知边界
|
||||||
|
|
||||||
- hybrid 依赖云端/实例支持 BM25 Function 与 sparse index
|
- py-rag 判级阈值(0.75/0.5/0.3)未校准,`quality_score` 仅供排序与展示参考(契约已知边界)
|
||||||
- 全量重建受 embedding API 与 Milvus 写入延迟影响,可能较慢
|
- frontmatter 的 keywords/summary/covers/when_to_retrieve 当前仅上传方提供;Java 侧 LLM 补全(原 `DocumentFieldEnricher`)已随抽离移除,待 py-rag 开放
|
||||||
- L0 关键词匹配仍较粗,只作 hint,不作主召回
|
- `knowledge_base/` 历史原料需在 py-rag 侧完成一次性 ingest 迁移后方可检索
|
||||||
- 尚未做邻块上下文自动扩展、cross-encoder rerank、真 query rewrite
|
- 无单文档删除;下线文档靠 py-rag 全量重建
|
||||||
- `totalVectors` 统计接口仍可能返回 0,不代表 collection 为空;以 rebuild/init 结果与检索命中为准
|
- eval/rag-retrieval 离线基线基于旧 L0/scoreLabel 语义构建,抽离后需重新校准(见 `mvp/engineering/rag/RAG离线评测-基线设计.md` 顶部说明)
|
||||||
- `retrieval.search.mode=dense` 仅作召回对照,不是第二套主路径
|
|
||||||
- 同一 hybrid schema 数据可被 dense / hybrid 两种查询复用;从纯旧 dense-only collection 升级必须 rebuild
|
|
||||||
- hybrid 质量闸门优先用 `denseDistance` 绝对 L2;无 dense 时 rank 回退;排序仍跟 RRF
|
|
||||||
- 后处理不再用 L0 关键词 boost 改序;词面信号以库内 BM25+RRF 为准
|
|
||||||
- Trace/审计细节与限制见 [RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)
|
- Trace/审计细节与限制见 [RAG检索可观测性与审计.md](./RAG检索可观测性与审计.md)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# MVP 架构文档
|
# MVP 架构文档
|
||||||
|
|
||||||
**更新日期**:2026-07-29
|
**更新日期**:2026-09-29
|
||||||
**状态**:当前单 Diagnosis Agent + Harness 架构
|
**状态**:当前单 Diagnosis Agent + Harness 架构
|
||||||
|
|
||||||
当前文档入口:
|
当前文档入口:
|
||||||
@@ -12,12 +12,19 @@
|
|||||||
| [harness-quality-gates.md](harness-quality-gates.md) | Run、Tool、Evidence、Semantic、Release 与 Trace Recorder 门禁 |
|
| [harness-quality-gates.md](harness-quality-gates.md) | Run、Tool、Evidence、Semantic、Release 与 Trace Recorder 门禁 |
|
||||||
| [session-trace-lifecycle.md](session-trace-lifecycle.md) | sessionId/runId、SSE、统一 Timeline 和 reasoning audit 生命周期 |
|
| [session-trace-lifecycle.md](session-trace-lifecycle.md) | sessionId/runId、SSE、统一 Timeline 和 reasoning audit 生命周期 |
|
||||||
| [diagnosis-information-gain-stop-architecture.md](diagnosis-information-gain-stop-architecture.md) | 已实施的信息增益评价、Harness 饱和检测、Draft 合同失败降级与证据不足停止设计 |
|
| [diagnosis-information-gain-stop-architecture.md](diagnosis-information-gain-stop-architecture.md) | 已实施的信息增益评价、Harness 饱和检测、Draft 合同失败降级与证据不足停止设计 |
|
||||||
| [RAG知识检索架构.md](RAG知识检索架构.md) | 当前 `lookup_knowledge` 检索:MilvusClientV2 dense+BM25 hybrid、chunk 证据身份、重建运维 |
|
| [RAG知识检索架构.md](RAG知识检索架构.md) | 当前 `lookup_knowledge` 检索:py-rag 知识服务接入、契约映射、chunk 证据身份、入库与重建运维 |
|
||||||
| [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) | RAG Trace / 审计:请求内 retrievalTrace、tool_invocation 富字段、Trace API 读法 |
|
| [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) | RAG Trace / 审计:请求内 retrievalTrace、tool_invocation 富字段、Trace API 读法 |
|
||||||
|
|
||||||
**工程纪要**(问题 / 决策 / E2E,非架构规范正文)见 [../engineering/README.md](../engineering/README.md)。
|
**工程纪要**(问题 / 决策 / E2E,非架构规范正文)见 [../engineering/README.md](../engineering/README.md)。
|
||||||
|
|
||||||
2026-07-22 前的多角色编排、双入口和旧证据链文档已移动到 `archive/2026-07-22-legacy/`,仅用于历史决策追溯,不代表当前运行时。其中旧 RAG 描述(Spring AI VectorStore 主路径 + Milvus SDK fallback)已被当前 hybrid 实现取代,请以 [RAG知识检索架构.md](RAG知识检索架构.md) 为准。检索可观测与 Trace 以 [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) 为准(勿再依赖 archive 内旧 retrieval-observability)。
|
2026-07-22 前的多角色编排、双入口和旧证据链文档已移动到 `archive/2026-07-22-legacy/`,仅用于历史决策追溯,不代表当前运行时。
|
||||||
|
|
||||||
|
RAG 架构经历两次更替,均以 [RAG知识检索架构.md](RAG知识检索架构.md) 为准:
|
||||||
|
|
||||||
|
1. 2026-07-28:进程内 MilvusClientV2 hybrid(取代更早的 Spring AI VectorStore 主路径 + Milvus SDK fallback);
|
||||||
|
2. **2026-09-29(当前)**:RAG 模块抽离为独立 py-rag 知识服务,Java 经 `KnowledgeSearchPort` → `PyRagKnowledgeSearchAdapter` → HTTP `/api/v1` 调用;进程内 Milvus/embedding/L0/分块全部移除。
|
||||||
|
|
||||||
|
检索可观测与 Trace 以 [RAG检索可观测性与审计.md](RAG检索可观测性与审计.md) 为准(勿再依赖 archive 内旧 retrieval-observability)。
|
||||||
|
|
||||||
当前普通 Trace 与 LLM 步骤审计(`agent_reasoning_audit`:`reasoning_content` + `assistant_text`)使用独立存储和独立接口。
|
当前普通 Trace 与 LLM 步骤审计(`agent_reasoning_audit`:`reasoning_content` + `assistant_text`)使用独立存储和独立接口。
|
||||||
DeepSeek thinking 捕获路径与 V015–V017 字段已 live 验证(2026-07-28)。Reasoning 访问控制、保留期限、加密要求仍由 ISS-015 跟踪,不能把“数据已分表 + 能抓到 thinking”理解为“治理已经完成”。
|
DeepSeek thinking 捕获路径与 V015–V017 字段已 live 验证(2026-07-28)。Reasoning 访问控制、保留期限、加密要求仍由 ISS-015 跟踪,不能把“数据已分表 + 能抓到 thinking”理解为“治理已经完成”。
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
# 当前 MVP 架构
|
# 当前 MVP 架构
|
||||||
|
|
||||||
**更新日期**:2026-07-28
|
**更新日期**:2026-09-29
|
||||||
**状态**:当前可运行架构
|
**状态**:当前可运行架构
|
||||||
|
|
||||||
## 1. 系统定位
|
## 1. 系统定位
|
||||||
|
|
||||||
SuperBizAgent 是面向故障诊断的可追踪 Agent 应用。当前系统只保留一个拥有 Tool loop 的 `Diagnosis Agent`;Harness 负责确定性的预算、取消、工具边界、证据验真、语义审查和安全发布。
|
SuperBizAgent 是面向故障诊断的可追踪 Agent 应用。当前系统只保留一个拥有 Tool loop 的 `Diagnosis Agent`;Harness 负责确定性的预算、取消、工具边界、证据验真、语义审查和安全发布。
|
||||||
|
|
||||||
知识检索当前为显式 `lookup_knowledge` 工具 + 单一 MilvusClientV2 后端(dense / dense+BM25 hybrid)。详细链路见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
知识检索当前为显式 `lookup_knowledge` 工具 + 独立 py-rag 知识服务(HTTP `/api/v1`);检索算法(dense+BM25 融合、rerank、判级)与文档入库全部在 py-rag 侧,Java 只保留 harness 消费面与 HTTP 客户端。详细链路见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||||
|
|
||||||
## 2. 分层
|
## 2. 分层
|
||||||
|
|
||||||
@@ -71,22 +71,22 @@ Agent 只看到三个固定 Tool:
|
|||||||
Agent
|
Agent
|
||||||
-> RagToolAdapter / ToolBoundary
|
-> RagToolAdapter / ToolBoundary
|
||||||
-> LookupKnowledgeTool
|
-> LookupKnowledgeTool
|
||||||
-> L0 hint(可选 category filter)
|
-> 原始 query 直传(L0 已下沉 py-rag)
|
||||||
-> KnowledgeSearchPort
|
-> KnowledgeSearchPort
|
||||||
-> VectorSearchService
|
-> PyRagKnowledgeSearchAdapter # HTTP 客户端(PyRagClient)
|
||||||
-> MilvusHybridKnowledgeStore # 唯一知识向量后端
|
-> py-rag 知识服务 /api/v1/search # 融合 / rerank / 判级
|
||||||
-> RagResultProjector # 有界 Agent 投影
|
-> KnowledgeEvidencePostProcessor # chunk 去重 / return-n / 判级(Java 侧)
|
||||||
|
-> RagResultProjector # 有界 Agent 投影
|
||||||
```
|
```
|
||||||
|
|
||||||
要点:
|
要点:
|
||||||
|
|
||||||
- 默认 `retrieval.search.mode=hybrid`:dense ANN + BM25 sparse ANN + RRFRanker。
|
- 默认 `retrieval.search.mode=hybrid`:映射 py-rag `hybrid`(dense+BM25 融合);`dense` 映射 `semantic` 作对照。
|
||||||
- 也可切 `dense`:仅 dense ANN。
|
|
||||||
- 已移除知识路径上的 legacy `MilvusServiceClient` search 与 `vector-store.mode=sdk|spring|auto` 路由。
|
|
||||||
- 证据按 chunk 级 `evidenceKey` 去重;Agent 侧 `document_id` 为 chunk 级身份。
|
- 证据按 chunk 级 `evidenceKey` 去重;Agent 侧 `document_id` 为 chunk 级身份。
|
||||||
- 知识库全量重建:`python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD`,默认操作 collection `biz`。
|
- py-rag `evidence_status=no_evidence` 按正常"无知识可用"处理,不是错误。
|
||||||
|
- 知识库全量重建:py-rag `POST /api/v1/collections:rebuild?confirm=REBUILD`(异步任务)。
|
||||||
|
|
||||||
完整 schema、模式、重建与历史差异见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
完整契约映射、入库、重建与历史差异见 [RAG知识检索架构.md](RAG知识检索架构.md)。
|
||||||
|
|
||||||
## 5. Trace 与持久化
|
## 5. Trace 与持久化
|
||||||
|
|
||||||
@@ -117,10 +117,12 @@ Reasoning endpoint 是敏感审计面,不属于普通业务 API。数据和查
|
|||||||
|
|
||||||
与知识检索相关的独立 API:
|
与知识检索相关的独立 API:
|
||||||
|
|
||||||
- `POST /api/knowledge/init`:导入/增量初始化 `knowledge_base`
|
- `POST /api/documents/upload`:文档上传(MySQL 业务元数据 + 本地原件 + py-rag ingest)
|
||||||
- `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD`:清空并重建 dense+BM25 collection(默认 `biz`)
|
- `POST /api/upload`:简单上传(本地保存 + py-rag ingest,可选 `category`)
|
||||||
- `GET /api/knowledge/stats`:文档元数据统计
|
- `GET /api/documents/{docId}` / `GET /api/documents/status/{status}` / `GET /api/documents/faultSource/{faultSource}`:文档元数据查询
|
||||||
- `GET /milvus/health`:MilvusClientV2 健康检查与 knowledge collection 名
|
- `DELETE /api/documents/{docId}`:删除 MySQL 元数据与本地原件(py-rag 侧索引需全量重建生效)
|
||||||
|
|
||||||
|
知识库检索/入库/重建的服务端健康与统计由 py-rag 提供:`GET /api/v1/health`、`GET /api/v1/stats`(`{pyrag.base-url}`)。
|
||||||
|
|
||||||
## 7. 安全边界
|
## 7. 安全边界
|
||||||
|
|
||||||
|
|||||||
@@ -20,14 +20,19 @@
|
|||||||
|
|
||||||
## RAG
|
## RAG
|
||||||
|
|
||||||
|
> 2026-09-29 RAG 模块已抽离为独立 py-rag 知识服务(架构见
|
||||||
|
> [../architecture/RAG知识检索架构.md](../architecture/RAG知识检索架构.md))。
|
||||||
|
> 下列纪要保留决策过程价值,涉及进程内 Milvus / L0 的实现细节以各文顶部说明为准。
|
||||||
|
|
||||||
| 文档 | 内容 |
|
| 文档 | 内容 |
|
||||||
|---|---|
|
|---|---|
|
||||||
| [rag/RAG排序-多路召回与RRF.md](rag/RAG排序-多路召回与RRF.md) | K、多路融合、RRF、L0 边界 |
|
| [rag/RAG证据链探索笔记-从py-rag响应到引用验真.md](rag/RAG证据链探索笔记-从py-rag响应到引用验真.md) | **抽离后链路首发导读**:数据逐站形态、关键字段、设计哲学与化石清单 |
|
||||||
| [rag/RAG-Hybrid质量分与后处理.md](rag/RAG-Hybrid质量分与后处理.md) | qualityScore 统一、L2 伪装废止、后处理 |
|
| [rag/RAG排序-多路召回与RRF.md](rag/RAG排序-多路召回与RRF.md) | K、多路融合、RRF、L0 边界(实现已下沉 py-rag,判断框架仍有效) |
|
||||||
| [rag/RAG-Agent如何读relevance_level.md](rag/RAG-Agent如何读relevance_level.md) | Agent 侧相关度标签含义与误读 |
|
| [rag/RAG-Hybrid质量分与后处理.md](rag/RAG-Hybrid质量分与后处理.md) | qualityScore 统一、后处理(分数语义现为 RERANK 直传) |
|
||||||
| [rag/RAG离线评测-基线设计.md](rag/RAG离线评测-基线设计.md) | Golden/Fixture、hybrid 评测与闸门 |
|
| [rag/RAG-Agent如何读relevance_level.md](rag/RAG-Agent如何读relevance_level.md) | Agent 侧相关度标签含义与误读(现行) |
|
||||||
| [rag/Milvus-Hybrid接入清单.md](rag/Milvus-Hybrid接入清单.md) | Hybrid 交付拆分与接入清单 |
|
| [rag/RAG离线评测-基线设计.md](rag/RAG离线评测-基线设计.md) | Golden/Fixture、hybrid 评测与闸门(需按新语义重新校准) |
|
||||||
| [rag/RAG审计补丁-stepid-query-E2E验收.md](rag/RAG审计补丁-stepid-query-E2E验收.md) | step_id / query 审计 live 验收 |
|
| [rag/Milvus-Hybrid接入清单.md](rag/Milvus-Hybrid接入清单.md) | Hybrid 交付拆分与接入清单(已过时,仅历史追溯) |
|
||||||
|
| [rag/RAG审计补丁-stepid-query-E2E验收.md](rag/RAG审计补丁-stepid-query-E2E验收.md) | step_id / query 审计 live 验收(现行) |
|
||||||
|
|
||||||
架构对照:
|
架构对照:
|
||||||
|
|
||||||
@@ -36,7 +41,7 @@
|
|||||||
|
|
||||||
相关 Issue:
|
相关 Issue:
|
||||||
|
|
||||||
- [ISS-017 L0 过滤收窄与 Fallback 加固](../issues/active/ISS-017-rag-l0-filter-fallback-hardening.md)(暂缓,保持现网)
|
- [ISS-017 L0 过滤收窄与 Fallback 加固](../issues/active/ISS-017-rag-l0-filter-fallback-hardening.md)(已失效:L0 下沉 py-rag,问题前提不复存在)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
**状态**:SUCCESS 完整诊断主文档;工具阶段按**现行**审计能力(`step_id` / `query`)说明
|
**状态**:SUCCESS 完整诊断主文档;工具阶段按**现行**审计能力(`step_id` / `query`)说明
|
||||||
**文档路径**:`mvp/engineering/diagnosis/一次诊断全流程-E2E导读.md`
|
**文档路径**:`mvp/engineering/diagnosis/一次诊断全流程-E2E导读.md`
|
||||||
|
|
||||||
|
> **现状说明(2026-09-29)**:本文基于 2026-07 的 live Run 编写,图中 `VectorSearchService` /
|
||||||
|
> `MilvusHybridKnowledgeStore`(进程内 Milvus)已替换为 py-rag 服务调用;审计/Trace 结构不变。
|
||||||
|
> 当前检索链路见 [../architecture/RAG知识检索架构.md](../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
### 主样本(正文数值与 timeline 来源)
|
### 主样本(正文数值与 timeline 来源)
|
||||||
|
|
||||||
| 项 | 值 |
|
| 项 | 值 |
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
# Harness LLM Judge 设计笔记:从不可信判定到可信裁决
|
||||||
|
|
||||||
|
**更新日期**:2026-08-04
|
||||||
|
**主题**:SemanticGuard + EvidenceRepair + GuardModelCall = LLM-as-a-judge 模式在证据安全链的完整落地(面试问答版)
|
||||||
|
**代码位置**:`src/main/java/com/superbiz/agent/harness/guard/semantic/` + `src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java`
|
||||||
|
|
||||||
|
## 1. 定位:三个角色
|
||||||
|
|
||||||
|
```text
|
||||||
|
SemanticGuard → 典型 LLM judge:判「结论是否被已验证证据支持」,输出 verdict + reason
|
||||||
|
EvidenceRepair → judge 的修复延伸(rewriter):验真失败后「只修引用、不修结论」
|
||||||
|
GuardModelCall → 受控 LLM 调用底座:judge 类调用的基础设施(共用)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. 面试五段式回答稿(完整叙事)
|
||||||
|
|
||||||
|
### ① 动机(先讲问题,不报组件名)
|
||||||
|
|
||||||
|
> 我们的证据安全链里有一道「机械验真」——检查模型引用的每条证据是不是真实来自工具结果,这个用规则就能做。但光验真不够:模型可能引用真实的证据,结论却是「站不住」的——比如证据只支持 A 场景,它却拿去支撑 B 结论。这个「结论被没被证据支持」是**语义判断**,规则引擎做不了,必须靠模型。所以我们需要一个「裁判模型」来判——但裁判模型本身是不可信的,它可能乱判、可能输出奇怪的形状、可能跑很久。所以核心问题是:**怎么让一个不可信的模型做可信的判定**。
|
||||||
|
|
||||||
|
### ② 决策(方案 + 放弃了什么)
|
||||||
|
|
||||||
|
> 我的方案是:用**隔离的轻量判定模型**——单轮、无工具、输出被强约束,跟主 Agent 的循环完全分离。这里放弃了两条路:第一,让主 Agent 自己判——不行,它已经写了自己的结论,有偏向;第二,纯规则判——语义判断规则做不到。同时有个关键决策:**判读的输入是「视图」不是原始内容**——裁判只看到用户将看到的内容和已验证证据,看不到内部 id 这些实现细节,防止信息污染影响裁判的客观性。
|
||||||
|
|
||||||
|
### ③ 实现(关键机制)
|
||||||
|
|
||||||
|
> 三个关键机制:
|
||||||
|
> **输入视图化**:把 draft 投影成「用户可见视图」再交给裁判,剥离内部引用 id;
|
||||||
|
> **输出硬校验**:裁判的输出必须是恰好两个字段——verdict 和 reason,verdict 必须是合法枚举,reason 不能为空。多一个字段都不接受——我们不信任模型输出的形状,只信它在一个极小的空间里做选择;
|
||||||
|
> **受控调用**:裁判跑在独立线程、有硬超时、Run 取消能强杀它、它的输入输出都计入预算和 Token 账本——裁判的花费不是无底洞,它也是 Run 的一部分。
|
||||||
|
|
||||||
|
### ④ 边界(诚实说不做什么)
|
||||||
|
|
||||||
|
> 裁判不判「内容对不对」——那是事实问题,由证据链负责;裁判不自己调工具,单轮无工具;裁判有硬截止线,超时就放弃判定;裁判失败走降级,**不阻塞主结论的发布路径**——我们宁可没有裁决,也不让裁决失败卡死整个流程。
|
||||||
|
|
||||||
|
### ⑤ 30 秒话术
|
||||||
|
|
||||||
|
> "LLM judge 的完整设计:**动机**是结论的支持度是语义判断、规则做不了,但裁判模型不可信,所以核心是让不可信的模型做可信的判定。**方案**是隔离的轻量判定模型——单轮、无工具、强约束输出。三个关键机制:输入视图化(裁判只看用户可见内容,防信息污染)、输出硬校验(恰好 {verdict, reason} 两字段,多一个都不接受)、受控调用(独立线程、硬超时、取消强杀、计预算记账)。**边界**:裁判不判事实、不调工具、超时即放弃、失败走降级不阻塞主路径。总结一句话——judge 不是追加一个模型调用,而是把『不可信判定』关进笼子里:限定输入、锁死输出、受控运行、失败降级。"
|
||||||
|
|
||||||
|
## 3. 追问应对大全
|
||||||
|
|
||||||
|
### Q1:为什么 judge 不判事实?(最容易混的边界)
|
||||||
|
|
||||||
|
```text
|
||||||
|
分工:事实由证据链保证,judge 只判「支持关系」
|
||||||
|
事实真伪 → 证据来自真实工具结果 + EvidenceGuard 验引用真实(根在数据源)
|
||||||
|
支持关系 → SemanticGuard 判结论与证据的逻辑/相关性
|
||||||
|
|
||||||
|
judge 判不了事实的三个原因:
|
||||||
|
① 没有事实源——它只看「视图 + 已验证证据」,不能查库,判事实只能猜
|
||||||
|
② 事实真伪需要权威源复核(真实值在哪),judge 拿不到
|
||||||
|
③ 如果 judge 判事实,它成了第二个事实来源——两个来源可能打架
|
||||||
|
|
||||||
|
例子 1(judge 能判的——判支持不是判真伪):
|
||||||
|
证据:mysql 返回 count(*)=1000;结论:「user 表有 2000 条」
|
||||||
|
EvidenceGuard 验引用真实 → 通过;SemanticGuard → UNSUPPORTED(数字不一致)
|
||||||
|
注意:judge 不知道真实值是多少,它只发现「结论与证据不一致」
|
||||||
|
|
||||||
|
例子 2(支持关系成立,但事实未必对):
|
||||||
|
证据:慢查询日志显示 DB 全表扫描;结论:「延迟由 DB 全表扫描导致」
|
||||||
|
SemanticGuard → SUPPORTED(逻辑上站得住)
|
||||||
|
但真实原因可能是网络抖动——judge 判不了(没有网络数据源)
|
||||||
|
→ judge 只能保证「在现有证据下结论站得住」,不能保证「事实就是如此」
|
||||||
|
|
||||||
|
一句话:EvidenceGuard 保证「引用的证据是真的」,SemanticGuard 保证
|
||||||
|
「基于这些证据结论说得通」——事实的真伪从来不是 judge 的职责。
|
||||||
|
```
|
||||||
|
|
||||||
|
### Q2:为什么重试 2 次(semanticGuard 策略)?
|
||||||
|
|
||||||
|
```text
|
||||||
|
可重试性分析:语义审查单轮、无副作用(幂等)——多试几次不会造成破坏
|
||||||
|
但也不能无限重试:判定有硬截止线(总超时耗尽即放弃)
|
||||||
|
→ 2 次 = 一次失败的成本 × 收益的平衡点;judge 失败走 Fallback,不影响主路径
|
||||||
|
```
|
||||||
|
|
||||||
|
### Q3:语义不变性怎么保证(EvidenceRepair)?
|
||||||
|
|
||||||
|
```text
|
||||||
|
双重锁死:prompt(只能改 analysis_id / tool_call_ids / based_on_analysis_ids 三字段)
|
||||||
|
+ SemanticDraftView.hasSameUserVisibleSemantics(修复前后逐字段比对)
|
||||||
|
|
||||||
|
关键:比的是「用户可读的内容」不是内部引用 id——
|
||||||
|
Conclusion 只比 text,不比 basedOnAnalysisIds
|
||||||
|
Action 只比 action + requiresHumanConfirmation,不比 basedOnAnalysisIds
|
||||||
|
→ 引用 id 允许变(这正是修复目标),用户看到的文字不许动(碰了判 SCHEMA_INVALID 重试)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Q4:judge 判错了怎么办?
|
||||||
|
|
||||||
|
```text
|
||||||
|
judge 不是最终真相源,是「安全链的一道闸」:
|
||||||
|
① judge 判 UNSUPPORTED → 不发布,走 Fallback(宁可保守)
|
||||||
|
② judge 判 SUPPORTED 但事实错 → 那是事实问题,不在 judge 职责(见 Q1)
|
||||||
|
③ judge 自身失败 → 降级(不阻塞主路径)
|
||||||
|
→ 设计哲学:judge 的角色是「挡住明显不成立的结论」,不是「证明结论正确」
|
||||||
|
```
|
||||||
|
|
||||||
|
### Q5:为什么独立线程 + 单独超时?
|
||||||
|
|
||||||
|
```text
|
||||||
|
judge 调用不能阻塞主流程(主 Agent 循环)
|
||||||
|
独立线程 + future.get(timeout) = 硬超时截断
|
||||||
|
Run 取消 → future.cancel(true) 强杀在途判定(judge 也是 Run 的一部分)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Q6:输入为什么视图化?
|
||||||
|
|
||||||
|
```text
|
||||||
|
judge 只看该看的:用户可见内容 + 已验证证据
|
||||||
|
剥离内部 id(tool_call_id 等)——防止 judge 用内部信息做「看起来合理」的裁决
|
||||||
|
(信息污染:judge 看到内部 id 可能产生不当关联,或泄露内部结构到裁决)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 通用 LLM Judge 设计要素(可迁移)
|
||||||
|
|
||||||
|
| 通用要素 | 本项目实现 |
|
||||||
|
|---|---|
|
||||||
|
| 判什么(judgment task) | 结论是否被已验证证据支持 |
|
||||||
|
| 输入视图(该看什么) | SemanticDraftView(剥离内部 id)+ 已验证证据 |
|
||||||
|
| 输出约束(schema) | 恰好 {verdict, reason} + 枚举合法 + reason 非空 |
|
||||||
|
| 硬校验 | 字段集 equals({verdict, reason})——不多不少 |
|
||||||
|
| 隔离 | 单轮、无工具、独立线程——judge 不能自己调工具 |
|
||||||
|
| 硬超时 | 每次 attempt 剩余超时递减,总超时耗尽即放弃 |
|
||||||
|
| 重试策略 | 可重试性分析:单轮无副作用 → 2 次 |
|
||||||
|
| 失败降级 | judge 失败 → Fallback(不卡死主路径) |
|
||||||
|
| 可审计 | ModelCallLedger 记账 + semanticAttempt/evidenceRepairAttempt trace |
|
||||||
|
| 成本控制 | 输入/输出字节限制 + reserveRunBytes 计入 Run 预算 |
|
||||||
|
|
||||||
|
## 5. 代码位置索引
|
||||||
|
|
||||||
|
| 类 | 文件 |
|
||||||
|
|---|---|
|
||||||
|
| `GuardModelCall` | `src/main/java/com/superbiz/agent/harness/guard/semantic/GuardModelCall.java` |
|
||||||
|
| `SemanticGuard` | `src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticGuard.java` |
|
||||||
|
| `SemanticDraftView` | `src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticDraftView.java` |
|
||||||
|
| `SemanticGuardInput` / `SemanticGuardLimits` | `src/main/java/com/superbiz/agent/harness/guard/semantic/` |
|
||||||
|
| `EvidenceRepair` | `src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java` |
|
||||||
|
| `EvidenceRepairLimits` / `EvidenceRepairPrompt` | `src/main/java/com/superbiz/agent/harness/release/` |
|
||||||
|
| 重试策略(semanticGuard/evidenceRepair) | `src/main/java/com/superbiz/agent/harness/retry/HarnessRetryPolicies.java` |
|
||||||
@@ -1,5 +1,10 @@
|
|||||||
# Harness RAG 检索体系学习笔记:从 query 到可验证证据
|
# Harness RAG 检索体系学习笔记:从 query 到可验证证据
|
||||||
|
|
||||||
|
> **现状说明(2026-09-29)**:RAG 模块抽离后,本文「检索前 L0 导航」与
|
||||||
|
> `MilvusHybridKnowledgeStore` / `KnowledgeQueryTransformer` 相关章节描述的类已删除
|
||||||
|
> (L0 与向量检索下沉 py-rag 服务端);检索后处理/打包/投影/审计部分仍然现行。
|
||||||
|
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
**更新日期**:2026-08-03
|
**更新日期**:2026-08-03
|
||||||
**主题**:lookup_knowledge 完整后端链路——检索前/检索/检索后/打包/组装/降级/契约/验证
|
**主题**:lookup_knowledge 完整后端链路——检索前/检索/检索后/打包/组装/降级/契约/验证
|
||||||
**设计文档**:`mvp/engineering/rag/`(RAG 排序、Hybrid 质量分、relevance_level 等)
|
**设计文档**:`mvp/engineering/rag/`(RAG 排序、Hybrid 质量分、relevance_level 等)
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
# Harness agent 域学习笔记:从框架 ReAct 接入到受控停止
|
||||||
|
|
||||||
|
**更新日期**:2026-08-04
|
||||||
|
**主题**:agent 域完整链路——装配(Factory)/ 双拦截器(Model/Tool)/ 循环外壳(UseCase)/ 受控停止 / 双视图投影
|
||||||
|
**配套**:[tool 域代码学习笔记](Harness%20tool%20域代码学习笔记-工具的注册调用与执行链路.md)(工具链)、[progress 代码学习笔记](Harness%20progress%20代码学习笔记-从拦截器五道门到唯一发布点.md)(Tool 拦截器五道门)
|
||||||
|
|
||||||
|
## 1. 定位:框架 ReAct 接入层(粘合点)
|
||||||
|
|
||||||
|
```text
|
||||||
|
框架(spring-ai-alibaba ReactAgent):负责 ReAct 多轮(模型 ↔ tool_call)
|
||||||
|
Harness:不复制 loop,只通过 Interceptor 卡住【每次消耗】
|
||||||
|
→ Model Interceptor:每次模型调用(预算/审计/Token)
|
||||||
|
→ Tool Interceptor:每次工具调用(五道门)
|
||||||
|
|
||||||
|
原则:拦截器是挂点,不是 loop 实现——Harness 不需要知道框架内部怎么循环
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. 装配图(DiagnosisAgentFactory——粘合点)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
subgraph 框架能力
|
||||||
|
M["ChatModel"]
|
||||||
|
T["tools<br/>evidenceTools.callbacks()"]
|
||||||
|
L["ReactAgent 循环"]
|
||||||
|
end
|
||||||
|
subgraph Harness 控制面
|
||||||
|
I1["HarnessModelInterceptor<br/>预算+Token 审计"]
|
||||||
|
I2["HarnessToolInterceptor<br/>五道门+投影"]
|
||||||
|
H["Hooks<br/>agent_step 落库"]
|
||||||
|
O["outputSchema<br/>DiagnosisDraft(conclusion 可 null)"]
|
||||||
|
end
|
||||||
|
M --> L
|
||||||
|
T --> L
|
||||||
|
L --> I1
|
||||||
|
L --> I2
|
||||||
|
L --> H
|
||||||
|
L --> O
|
||||||
|
```
|
||||||
|
|
||||||
|
**关键装配决策**:
|
||||||
|
|
||||||
|
```text
|
||||||
|
.parallelToolExecution(false) ← 串行工具:预算与 step 绑定可解释
|
||||||
|
.returnReasoningContents(true) ← 推理内容返回
|
||||||
|
.releaseThread(true)
|
||||||
|
每次 run 新建 Agent(create(context))——拦截器持有 RunContext,不可跨 run 复用
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. HarnessModelInterceptor(模型拦截器)
|
||||||
|
|
||||||
|
```text
|
||||||
|
interceptModel(request, handler):
|
||||||
|
core.beforeModelCall(context) ← 预算门(模型调用前扣预算)
|
||||||
|
call = auditor.begin(...) ← 审计开始(Token 记账)
|
||||||
|
response = handler.call(request) ← 框架实际调用
|
||||||
|
recordUsage(call, response) ← 记 prompt/completion tokens
|
||||||
|
core.checkActive(context) ← 终态检查(预算耗尽在此打断)
|
||||||
|
return response
|
||||||
|
异常:记 0 token + 上抛(不吞)
|
||||||
|
```
|
||||||
|
|
||||||
|
**三个动作**:预算(beforeModelCall)→ 记账(auditor.begin/recordUsage)→ 终态(checkActive)——每次模型调用都被 Harness 卡住一次。Usage 字段 null/负数安全兜底(nonNegative)。
|
||||||
|
|
||||||
|
## 4. HarnessToolInterceptor(工具拦截器,已深学)
|
||||||
|
|
||||||
|
五道门(progress 会话已沉淀):证据工具必炸 handler → 拦截器唯一执行路径 → 边界投影 → 审计落库 → 返回。本会话只补装配视角:`interceptors` 列表里第二个,构造时注入 context + evidenceTools + objectMapper + traceRecorder。
|
||||||
|
|
||||||
|
## 5. DiagnosisAgentUseCase(循环外壳)
|
||||||
|
|
||||||
|
### 5.1 执行流程
|
||||||
|
|
||||||
|
```text
|
||||||
|
execute(context, input):
|
||||||
|
checkActive → 输入限制(query/previous_turn/input 字节)
|
||||||
|
reserveRunBytes(input) ← 输入也占 Run 预算
|
||||||
|
RunnableConfig.metadata 挂 RunContext ← 显式传递(避免隐式 ThreadLocal)
|
||||||
|
agent = factory.create(context) ← 每次 run 新建
|
||||||
|
response = agent.call(inputJson, config) ← ★ 框架跑完整个 ReAct 循环
|
||||||
|
output → 字节限制 → reserveRunBytes(draft) → parse DiagnosisDraft
|
||||||
|
→ completed(draft) 或 受控停止
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.2 受控停止(controlledExecution)——从异常栈捞回可控信号转正常返回值
|
||||||
|
|
||||||
|
```text
|
||||||
|
① DiagnosisCollectionStoppedException(信息饱和后仍强 tool)
|
||||||
|
→ stopped(stopReason) ← 收集该停(draft=null)
|
||||||
|
|
||||||
|
② RunAbortedException + BUDGET_EXHAUSTED
|
||||||
|
→ markBudgetLimitReached + stopped(BUDGET_LIMIT_REACHED)
|
||||||
|
|
||||||
|
③ 其他 RunAborted(取消/超时/内部失败终态)→ 原样再抛
|
||||||
|
← 留给 Application 写 CANCELLED/FAILED(不降级为正常停止)
|
||||||
|
|
||||||
|
④ BudgetExceededException 或 lifecycle 已 BUDGET_EXHAUSTED → 预算 stopped
|
||||||
|
|
||||||
|
⑤ 都识别不了 → 包装 DiagnosisAgentOutputException(Agent 执行失败)
|
||||||
|
```
|
||||||
|
|
||||||
|
**关键**:不是笼统「业务异常 → 正常」——**只识别 Harness 约定的可控信号**(沿 cause 链找,因框架可能再包一层);取消/超时必须上抛(诚实终态)。
|
||||||
|
|
||||||
|
### 5.3 与 recoverInvalidDraft 的分工
|
||||||
|
|
||||||
|
```text
|
||||||
|
controlledExecution:loop 被预算/收敛打断(往往还没有合法 draft)→ stopped
|
||||||
|
recoverInvalidDraft:loop 跑完了,但输出不是合法 DiagnosisDraft → 恢复/重试
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.4 输出解析(严格)
|
||||||
|
|
||||||
|
```text
|
||||||
|
draftReader = FAIL_ON_UNKNOWN_PROPERTIES + FAIL_ON_TRAILING_TOKENS(严格模式)
|
||||||
|
JsonParseException → INVALID_JSON / SchemaInvalid → SCHEMA_INVALID(分类错误码)
|
||||||
|
空输出 → EMPTY_DRAFT(分类错误码)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 6. 双视图投影(ToolResultViewProjector)
|
||||||
|
|
||||||
|
```text
|
||||||
|
modelObservation() → 模型观察:只含该工具的内容字段
|
||||||
|
lookup_knowledge → scope.query + evidence + relevance_level
|
||||||
|
query_logs → source_kind + scope + patterns + events
|
||||||
|
query_mysql → scope + columns + rows
|
||||||
|
+ stop_required/reason(需要停止时附加)
|
||||||
|
|
||||||
|
controlView() → 控制视图:evidence_status / relevance_level / returned_count / truncated
|
||||||
|
(Harness 控制面读,模型看不到)
|
||||||
|
|
||||||
|
分工:模型看到「内容」,Harness 看到「控制信息」(判级/截断/进度消费)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 7. 定义层小件
|
||||||
|
|
||||||
|
| 类 | 作用 |
|
||||||
|
|---|---|
|
||||||
|
| `DiagnosisAgentInput` | query + previous_turn(query 必填) |
|
||||||
|
| `DiagnosisAgentLimits` | maxQuery/PreviousTurn/Input/DraftBytes 四类字节上限 |
|
||||||
|
| `DiagnosisAgentPrompt` | classpath 加载系统提示词(prompts/diagnosis-agent-prompt.md) |
|
||||||
|
| `DiagnosisAgentExecution` | completed(draft, progress) / stopped(progress, stopReason) 双形态 |
|
||||||
|
| `DiagnosisDraftOutputSchema` | BeanOutputConverter postProcess:conclusion 允许 object/null(无结论合法) |
|
||||||
|
| `EvidenceToolInvoker` | 函数式:RunContext + toolCallId + arguments → ToolBoundaryResult |
|
||||||
|
| `ParsedAgentToolCall` | 解析出的工具调用(previousObservation + businessInput + arguments) |
|
||||||
|
|
||||||
|
## 8. 关键设计点(面试)
|
||||||
|
|
||||||
|
| 设计 | 为什么 |
|
||||||
|
|---|---|
|
||||||
|
| **不复制 loop** | 框架 ReAct 是标准能力;Harness 用拦截器挂在每次消耗点,不需要知道框架内部怎么循环 |
|
||||||
|
| 每次 run 新建 Agent | 拦截器持有 RunContext——Agent 与 run 绑定,防跨 run 串状态 |
|
||||||
|
| RunContext 显式传(metadata) | 避免隐式 ThreadLocal(框架线程池/异步下 ThreadLocal 不可靠) |
|
||||||
|
| 串行工具(parallel=false) | 预算与 step 绑定可解释(并行会让「哪一步花多少钱」不可审计) |
|
||||||
|
| 受控停止只认约定信号 | 取消/超时绝不降级为正常停止(诚实终态) |
|
||||||
|
| conclusion 允许 null | 无结论也是合法 Draft(FALLBACK 路径) |
|
||||||
|
| 双视图 | 模型观察 vs 控制视图分离——控制信息(判级/截断)不进模型上下文 |
|
||||||
|
| 输出严格解析 | FAIL_ON_UNKNOWN/TRAILING——防止模型输出混入意外字段 |
|
||||||
|
|
||||||
|
## 9. 易错点
|
||||||
|
|
||||||
|
| 易错 | 正确 |
|
||||||
|
|---|---|
|
||||||
|
| Harness 自己实现 Agent loop | 框架跑 loop,拦截器挂消耗点(不复制 loop) |
|
||||||
|
| 任何异常都转 stopped | 只认约定信号(CollectionStopped/预算);取消/超时原样上抛 |
|
||||||
|
| Agent 复用 | 每次 run 新建(拦截器绑定 RunContext) |
|
||||||
|
| ThreadLocal 传 context | RunnableConfig metadata 显式传 |
|
||||||
|
| 并行工具省时间 | 串行(预算与 step 绑定可解释) |
|
||||||
|
| 模型看到控制信息 | 双视图:模型看内容,Harness 看控制 |
|
||||||
|
| 输出宽容解析 | FAIL_ON_UNKNOWN + FAIL_ON_TRAILING(严格模式) |
|
||||||
|
|
||||||
|
## 10. 面试话术(30 秒)
|
||||||
|
|
||||||
|
> "agent 域是框架 ReAct 的接入层:不复制 loop——spring-ai-alibaba 的 ReactAgent 负责多轮循环,Harness 通过两个拦截器卡住每次消耗:Model Interceptor(每次模型调用前 checkActive + 预算,调用后记 Token 审计)、Tool Interceptor(工具调用五道门)。装配在 DiagnosisAgentFactory,每次 run 新建 Agent(拦截器绑定 RunContext,RunnableConfig metadata 显式传递避免 ThreadLocal)。循环外壳 DiagnosisAgentUseCase 做输入/输出字节限制 + Run 预算预留,并实现受控停止——只把 Harness 约定的可控信号(信息饱和、预算耗尽)从异常栈捞回转成 stopped,取消/超时原样上抛留给 Application 写 CANCELLED/FAILED。串行工具保证预算与 step 绑定可解释。"
|
||||||
|
|
||||||
|
## 11. 代码位置索引
|
||||||
|
|
||||||
|
| 类 | 文件 |
|
||||||
|
|---|---|
|
||||||
|
| `DiagnosisAgentFactory` | `src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentFactory.java` |
|
||||||
|
| `HarnessModelInterceptor` | `src/main/java/com/superbiz/agent/harness/agent/HarnessModelInterceptor.java` |
|
||||||
|
| `HarnessToolInterceptor` | `src/main/java/com/superbiz/agent/harness/agent/HarnessToolInterceptor.java` |
|
||||||
|
| `DiagnosisAgentUseCase` | `src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentUseCase.java` |
|
||||||
|
| `ToolResultViewProjector` | `src/main/java/com/superbiz/agent/harness/agent/ToolResultViewProjector.java` |
|
||||||
|
| `HarnessEvidenceTools` | `src/main/java/com/superbiz/agent/harness/agent/HarnessEvidenceTools.java` |
|
||||||
|
| `DiagnosisAgentExecution` | `src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentExecution.java` |
|
||||||
|
| `DiagnosisAgentOutputException` | `src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentOutputException.java` |
|
||||||
|
| 契约(DiagnosisDraft/PreviousTurn) | `src/main/java/com/superbiz/agent/harness/contract/` |
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
# Harness 整体架构学习笔记:从装配到入口到记忆到知识库写入
|
||||||
|
|
||||||
|
> **现状说明(2026-09-29)**:RAG 模块抽离后,本文「知识库写入链路」章节描述的
|
||||||
|
> `KnowledgeIndexService` / Milvus 写入路径已删除(入库下沉 py-rag `documents:ingest`);
|
||||||
|
> 其余装配/入口/记忆章节仍现行。当前架构见
|
||||||
|
> [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
|
**更新日期**:2026-08-04
|
||||||
|
**主题**:整体架构五块补充(了解层面)——配置装配中心 / HTTP 入口层 / 会话系统与记忆体系 / 知识库写入链路
|
||||||
|
**配套**:九域主线笔记(core/retry/progress/tool/guard/release/agent/audit/contract 全 ✅)
|
||||||
|
|
||||||
|
## 1. 配置装配中心(整体怎么搭起来)
|
||||||
|
|
||||||
|
### 1.1 装配全景(Bean 拓扑)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart LR
|
||||||
|
C["ChatHarnessProperties<br/>配置集中(yml)"] --> K["DiagnosisHarnessCore<br/>总闸门:超时/预算/重试/收敛"]
|
||||||
|
K --> B["ToolBoundary<br/>工具底座:canonical/门禁/投影"]
|
||||||
|
B --> A1["RagToolAdapter"]
|
||||||
|
B --> A2["QueryLogsToolAdapter"]
|
||||||
|
B --> A3["MysqlToolAdapter<br/>(可选装配)"]
|
||||||
|
A1 --> E["HarnessEvidenceTools<br/>组装注册"]
|
||||||
|
A2 --> E
|
||||||
|
A3 --> E
|
||||||
|
K --> G["GuardModelCall<br/>(守卫/修复/路由共用底座)"]
|
||||||
|
G --> SG["SemanticGuard"]
|
||||||
|
G --> ER["EvidenceRepair"]
|
||||||
|
G --> R["IntentRouter"]
|
||||||
|
E --> F["DiagnosisAgentFactory"]
|
||||||
|
F --> UC["DiagnosisAgentUseCase"]
|
||||||
|
UC --> D["DiagnosisChatExecutor"]
|
||||||
|
R --> D
|
||||||
|
R --> S["SystemChatExecutor"]
|
||||||
|
R --> KQ["KnowledgeQueryExecutor"]
|
||||||
|
D --> A["ChatApplicationUseCase<br/>应用入口"]
|
||||||
|
S --> A
|
||||||
|
KQ --> A
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.2 三层组织(话术版)
|
||||||
|
|
||||||
|
```text
|
||||||
|
① 总闸门(core):能花多少钱/跑多久/怎么重试/何时停——配置集中,改一处全局生效
|
||||||
|
② 工具底座:所有工具统一留痕(canonical)/拦截(ToolBoundary)/裁剪(投影)——行为整齐划一
|
||||||
|
③ 具体工具:RAG/Logs/MySQL 按需装配(没配数据源不装死工具)→ 组装注册给 Agent
|
||||||
|
|
||||||
|
串联:先判意图(路由)→ 走对应分支 → 全程在总闸门管辖下
|
||||||
|
一句话:边界集中、执行统一、工具可插拔
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1.3 关键设计点
|
||||||
|
|
||||||
|
| 设计 | 为什么 |
|
||||||
|
|---|---|
|
||||||
|
| 单一装配入口(HarnessChatConfiguration) | 读 Bean 签名 = 读架构拓扑 |
|
||||||
|
| 一切围绕 core | 所有链路共享同一套门禁(超时/预算/重试/收敛) |
|
||||||
|
| 配置属性集中(@EnableConfigurationProperties) | 一处改全局生效,不会有的环节漏管 |
|
||||||
|
| 工具可选装配(mysqlEnabled 判断) | 没配置不装死工具;ObjectProvider 可选后端 |
|
||||||
|
| 守卫/修复/路由共用 GuardModelCall | LLM judge 模式:同一轻量模型底座 |
|
||||||
|
| Redis 存 canonical | 跨实例共享 + TTL 过期 |
|
||||||
|
| 线程池 AbortPolicy | 队列满直接拒绝(fail fast) |
|
||||||
|
|
||||||
|
## 2. HTTP 入口层(薄协议适配)
|
||||||
|
|
||||||
|
### 2.1 请求流时序
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /api/chat {Id, Question}
|
||||||
|
→ 校验 → new SseEmitter + ChatSseSession(= ChatApplicationObserver)
|
||||||
|
→ chatWorkerExecutor.execute(...) ← 异步:HTTP 线程不跑模型
|
||||||
|
→ 立即返回 200 + TEXT_EVENT_STREAM
|
||||||
|
→ worker 线程执行编排,经 session 推事件
|
||||||
|
→ 队列满 → 503(RejectedExecutionException)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 SSE 状态机 + 五类事件
|
||||||
|
|
||||||
|
```text
|
||||||
|
状态机:NEW → OPEN → TERMINAL(收尾)/ DISCONNECTED(断连)
|
||||||
|
每个方法 requireState 校验顺序——防乱序推送
|
||||||
|
|
||||||
|
事件协议:
|
||||||
|
metadata → {session_id, run_id}(首推)
|
||||||
|
status → 编排进度(ROUTING / DIAGNOSIS_RUNNING / SAFETY_VALIDATING…)
|
||||||
|
content → 最终内容(content_type + payload)
|
||||||
|
failure → 失败码 + 消息
|
||||||
|
done → 终态(SUCCESS/FALLBACK/FAILED)★ CANCELLED 对外不可见
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 断连取消链路(贯穿到 Harness)
|
||||||
|
|
||||||
|
```text
|
||||||
|
客户端断开 → emitter.onTimeout/onError/onCompletion → session.disconnect()
|
||||||
|
→ 状态 DISCONNECTED → runControl.cancelClientDisconnect()
|
||||||
|
→ Harness 取消机制接管(checkActive / 拦截器 / 线程池 cancel)
|
||||||
|
onStarted 时若已断连:直接取消——不白跑
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.4 失败两层出口
|
||||||
|
|
||||||
|
```text
|
||||||
|
SSE 通道:ChatApplicationException → session.fail(failure + done(FAILED))
|
||||||
|
其他 RuntimeException → INTERNAL_FAILURE 通用信息(不泄露细节)
|
||||||
|
REST 通道:GlobalExceptionHandler → 404(SessionNotFound)/ 400(参数/文档/文件超限)/ 500(兜底)
|
||||||
|
|
||||||
|
→ 编排异常走 SSE failure,REST 异常走 HTTP 状态码——都不暴露内部细节
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. 会话系统与记忆体系(术语精确校准)
|
||||||
|
|
||||||
|
### 3.1 会话存储:不存历史,存「可重放的发布结果」
|
||||||
|
|
||||||
|
```text
|
||||||
|
ChatSession(chat_session 表):只存元数据(status/messagePairCount/时间戳)
|
||||||
|
——「message history is not persisted here」
|
||||||
|
DiagnosisSession:诊断快照(query/answer/selfEvaluation/feedback)
|
||||||
|
真正的历史:DiagnosisRun(每次运行一行)+ PublishedResult(JSON 落库)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.2 PreviousTurn 注入链路(短期记忆)
|
||||||
|
|
||||||
|
```text
|
||||||
|
ChatApplicationUseCase 开头读 findPreviousTurn(sessionId)
|
||||||
|
→ 查最近 SUCCESS+DIAGNOSIS+publishedResult 非空的 Run
|
||||||
|
→ 反序列化 PublishedResult → PublishedResultPolicy 生成【有界】摘要
|
||||||
|
(limitations 10 条×500 字 / 源文档 10 个 / 字段限长——有界在生成时)
|
||||||
|
→ 传 executePath → DiagnosisChatExecutor
|
||||||
|
→ new DiagnosisAgentInput(query, previous_turn)
|
||||||
|
→ 序列化成输入 JSON → agent.call(inputJson) → 模型从输入读到
|
||||||
|
|
||||||
|
设计三决策(话术版):
|
||||||
|
诊断短流程 → 只取上一轮(更早记忆靠多轮逐层传递)
|
||||||
|
非 SUCCESS 误导 → SUCCESS 才准入(且非 SUCCESS 轮次根本没写 PublishedResult)
|
||||||
|
token 爆炸 → 有界摘要(PreviousTurnLimits)
|
||||||
|
补充:可回放——模型看有界摘要,审计看全量 JSON(两层分离)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 记忆术语校准(重要认知)
|
||||||
|
|
||||||
|
```text
|
||||||
|
判定标准:记忆 = 会被【注入 prompt/上下文】的东西(不是存了什么)
|
||||||
|
|
||||||
|
PreviousTurn → 注入输入 JSON(user message)→ ✅ 短期记忆(当前会话)
|
||||||
|
lookup_knowledge → 工具调用动态获取(tool result)→ ❌ 记忆,是检索增强(RAG)
|
||||||
|
知识库 → 检索源,规模太大无法全量注入 → 归检索侧(工具检索是正确形态)
|
||||||
|
案例库 → 有结构有写入、缺检索注入 → 长期记忆的【候选原料】
|
||||||
|
运行档案 → 元数据层永不注入 → 审计数据
|
||||||
|
|
||||||
|
项目真实情况:只有短期记忆(PreviousTurn)+ 检索增强(RAG),【没有】长期记忆层
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.4 长期记忆设计路径(如果要做)
|
||||||
|
|
||||||
|
```text
|
||||||
|
筛选标准:规模可控 + 跨会话价值 + 可注入形态
|
||||||
|
|
||||||
|
案例库最符合:root_cause+solution 结构化摘要、注入 top 2-3 条、相似故障复用解法
|
||||||
|
PreviousTurn 扩展:最近 N 轮结论摘要(短期 → 中期记忆)
|
||||||
|
Feedback 偏好:用户偏好摘要
|
||||||
|
知识库不符合:全量太大 → 保持工具检索
|
||||||
|
|
||||||
|
案例 → skill 提炼(项目已实现):
|
||||||
|
6 个 SKILL.md(diagnose-mysql-connection-pool 等)
|
||||||
|
结构:Workflow / Required Evidence / Stop Conditions / Report Rules / Eval Anchor
|
||||||
|
注入:ClasspathSkillRegistry → SystemPromptTemplate → system prompt(程序性长期记忆)
|
||||||
|
情景记忆(案例)→ 程序记忆(skill)→ 常驻注入 ✅ 长期记忆的正确形态
|
||||||
|
现有缺口:单技能激活(只放行 1 个)/ 静态加载(无按 query 自动匹配)/ skill 与案例库断开
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 知识库写入链路(RAG 写半边)
|
||||||
|
|
||||||
|
```text
|
||||||
|
上传(/api/documents/upload)
|
||||||
|
→ TextExtractorService(文本提取)
|
||||||
|
→ DocumentChunkService.chunkDocument(分块)★
|
||||||
|
→ VectorEmbeddingService(dense embedding)
|
||||||
|
→ VectorIndexService.indexDocumentChunks(写 Milvus)★
|
||||||
|
→ 检索侧(lookup_knowledge)读同一份索引
|
||||||
|
```
|
||||||
|
|
||||||
|
### 关键设计
|
||||||
|
|
||||||
|
```text
|
||||||
|
① 分块:按章节分块(非定长硬切)+ 相邻 chunk 保留 overlap(减轻边界断裂)
|
||||||
|
双条件限制:maxSize(字符)+ maxTokens(token)
|
||||||
|
② hybrid 写入:dense(应用侧 embedding → vector 字段)
|
||||||
|
+ BM25(buildSearchText → search_text 字段)
|
||||||
|
★ 关键:dense embedding 输入 与 BM25 search_text 【同源】——
|
||||||
|
同一个「增强文本」既喂 embedding 又写 BM25 字段
|
||||||
|
→ 两路召回看到完全一致的文档内容,混合检索才公平
|
||||||
|
③ 弃用:legacy MilvusServiceClient(旧 SDK)/ Spring AI VectorStore#add(无 hybrid schema)
|
||||||
|
唯一后端:MilvusHybridKnowledgeStore(Milvus SDK v2)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. 易错点
|
||||||
|
|
||||||
|
| 易错 | 正确 |
|
||||||
|
|---|---|
|
||||||
|
| Controller 做业务编排 | 薄适配层:校验+开 SSE+异步+写回,编排在 Application |
|
||||||
|
| SSE 顺序不重要 | requireState 状态机校验——防乱序推送 |
|
||||||
|
| 取消对外可见 | Done 拒绝 CANCELLED——客户端只看到 SUCCESS/FALLBACK/FAILED |
|
||||||
|
| 知识库 = 长期记忆 | 是检索源(工具动态获取);记忆 = prompt 注入——项目无长期记忆层 |
|
||||||
|
| 案例 = 长期记忆 | 是候选原料——缺检索注入;skill 才是程序性长期记忆(已实现) |
|
||||||
|
| 工具预算在工具层 | maxToolCalls/收敛参数都在 core(总闸门) |
|
||||||
|
| 分块定长硬切 | 按章节 + overlap + 双条件限制 |
|
||||||
|
|
||||||
|
## 6. 面试话术(30 秒)
|
||||||
|
|
||||||
|
### 6.1 整体架构怎么组织
|
||||||
|
|
||||||
|
> "整个系统从下往上三层:最底层一套全局规则(超时/预算/重试/收敛,配置集中改一处全局生效);中间一层工具共用的底座(调用留痕、统一拦截、结果裁剪);上层按需装配具体工具(知识库/日志/数据库,配了才装)。最后串成应用入口——先判意图再走分支,全程在总闸门管辖下。一句话:边界集中、执行统一、工具可插拔。"
|
||||||
|
|
||||||
|
### 6.2 记忆体系
|
||||||
|
|
||||||
|
> "记忆的判定标准是会不会被注入上下文:PreviousTurn 是短期记忆(注入输入 JSON,上一轮 SUCCESS 的有界摘要);知识库是检索增强不是记忆(工具动态获取);项目没有长期记忆层——skill(案例提炼的诊断方法)注入 system prompt 是程序性长期记忆的正确形态;案例库是候选原料,缺检索注入。"
|
||||||
|
|
||||||
|
## 7. 代码位置索引
|
||||||
|
|
||||||
|
| 块 | 文件 |
|
||||||
|
|---|---|
|
||||||
|
| 装配中心 | `config/HarnessChatConfiguration.java`(+ `config/ChatHarnessProperties.java`) |
|
||||||
|
| HTTP 入口 | `controller/ChatController.java` + `controller/sse/ChatSseSession.java` / `ChatSseEvent.java` / `SseEmitterChatSink.java` |
|
||||||
|
| 异常映射 | `exception/GlobalExceptionHandler.java` |
|
||||||
|
| 会话存储 | `harness/application/persistence/JpaChatRunStore.java` / `PreviousTurnLimits.java` / `PublishedResultPolicy.java` |
|
||||||
|
| 记忆注入 | `harness/application/ChatApplicationUseCase.java` + `harness/application/executor/DiagnosisChatExecutor.java` + `harness/agent/DiagnosisAgentUseCase.java` |
|
||||||
|
| skill 机制 | `config/SkillConfig.java` + `src/main/resources/skills/*/SKILL.md` |
|
||||||
|
| 知识库写入 | `service/DocumentChunkService.java` / `VectorIndexService.java` / `VectorEmbeddingService.java` / `KnowledgeBaseInitService.java` + `controller/DocumentController.java` / `KnowledgeBaseController.java` |
|
||||||
@@ -83,7 +83,7 @@
|
|||||||
| `core` | **执行控制**:身份 / deadline / 预算 / 取消 / 唯一终态,checkActive 三道闸 | ✅ 深入 | RunContext、budget、cancel、lifecycle、checkActive、termination | [执行控制笔记](Harness执行控制笔记-终态检查与取消广播.md)、[RunBudget 时序图](RunBudget预算流程-一次Run的资源门禁时序图.md) |
|
| `core` | **执行控制**:身份 / deadline / 预算 / 取消 / 唯一终态,checkActive 三道闸 | ✅ 深入 | RunContext、budget、cancel、lifecycle、checkActive、termination | [执行控制笔记](Harness执行控制笔记-终态检查与取消广播.md)、[RunBudget 时序图](RunBudget预算流程-一次Run的资源门禁时序图.md) |
|
||||||
| `retry` | **显式可计量重试**:分类裁决(技术/业务)、次数/时间/成本三重封顶、attempt 可审计 | ✅ 深入 | 设计动机、分类裁决、剩余超时、幂等性、SDK 关闭 | [Retry 重试机制](Retry重试机制-显式可计量的attempt循环.md) |
|
| `retry` | **显式可计量重试**:分类裁决(技术/业务)、次数/时间/成本三重封顶、attempt 可审计 | ✅ 深入 | 设计动机、分类裁决、剩余超时、幂等性、SDK 关闭 | [Retry 重试机制](Retry重试机制-显式可计量的attempt循环.md) |
|
||||||
| `contract` | **跨层类型化语言**:Draft / PublishedResult / SafeFallback / 状态枚举,防字符串漂移 | ✅ 深入 | 11 个状态枚举五层全景、四个正交轴(RunState⊥ReleaseOutcome、InvocationStatus⊥EvidenceStatus)、纵向映射链、SseOutcome 未接线发现 | [状态流笔记](Harness%20contract%20状态流学习笔记-11个状态枚举的正交全景.md) |
|
| `contract` | **跨层类型化语言**:Draft / PublishedResult / SafeFallback / 状态枚举,防字符串漂移 | ✅ 深入 | 11 个状态枚举五层全景、四个正交轴(RunState⊥ReleaseOutcome、InvocationStatus⊥EvidenceStatus)、纵向映射链、SseOutcome 未接线发现 | [状态流笔记](Harness%20contract%20状态流学习笔记-11个状态枚举的正交全景.md) |
|
||||||
| `agent` | **框架 ReAct 接入**:拦截器把预算/审计/停止协议挂到框架循环上,不复制 loop | ⬜ 部分 | HarnessModelInterceptor(预算/Token 记账) | — |
|
| `agent` | **框架 ReAct 接入**:拦截器把预算/审计/停止协议挂到框架循环上,不复制 loop | ✅ 深入 | 装配(Factory 粘合点)、双拦截器(Model:预算+Token 审计;Tool:五道门)、UseCase 循环外壳(字节/预算限制)、受控停止(异常栈捞回可控信号)、双视图投影(模型观察 vs 控制视图)、串行工具 | [agent 域学习笔记](Harness%20agent%20域学习笔记-从框架%20ReAct%20接入到受控停止.md) |
|
||||||
| `audit` | **可观测账本**:Trace 事件回放、Token 对账、metadata-only(不存敏感正文) | ✅ 深入 | trace 时序线(15 帧真实数据)、Ledger 分账、模型步审计 hook、RunConclusionExtractor、DiagnosisTraceService 三级回放 | [application+audit 笔记](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) |
|
| `audit` | **可观测账本**:Trace 事件回放、Token 对账、metadata-only(不存敏感正文) | ✅ 深入 | trace 时序线(15 帧真实数据)、Ledger 分账、模型步审计 hook、RunConclusionExtractor、DiagnosisTraceService 三级回放 | [application+audit 笔记](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) |
|
||||||
| `application` | **Run 应用所有者**:创建 Run / 路由意图 / 执行分支 / 持久化 / SSE 输出 | ✅ 深入 | 六步编排、取消句柄(CoreRunControl)、统一失败出口、多轮记忆有界化、PublishedResultPolicy 落库 | [application+audit 笔记](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) |
|
| `application` | **Run 应用所有者**:创建 Run / 路由意图 / 执行分支 / 持久化 / SSE 输出 | ✅ 深入 | 六步编排、取消句柄(CoreRunControl)、统一失败出口、多轮记忆有界化、PublishedResultPolicy 落库 | [application+audit 笔记](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) |
|
||||||
| `guard` | **验证分离**:EvidenceGuard 机械验引用真实性 + SemanticGuard 隔离判结论支持度 | ✅ 深入 | 20 个违规码、三层校验(结构/验真/重读投影)、语义不变性、守卫模型受控调用、全栈衔接 | [证据安全链笔记](Harness%20证据安全链学习笔记-从收敛控制到唯一发布点.md) |
|
| `guard` | **验证分离**:EvidenceGuard 机械验引用真实性 + SemanticGuard 隔离判结论支持度 | ✅ 深入 | 20 个违规码、三层校验(结构/验真/重读投影)、语义不变性、守卫模型受控调用、全栈衔接 | [证据安全链笔记](Harness%20证据安全链学习笔记-从收敛控制到唯一发布点.md) |
|
||||||
@@ -110,6 +110,9 @@
|
|||||||
| [Harness application+audit 学习笔记-从 Run 编排到可回放审计](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) | application 六步编排/取消句柄/持久化策略;audit 域层次(trace 子体系/Ledger/审计表);**audit vs trace 区别**(真实数据对照)/三级回放 | ✅ 已沉淀 |
|
| [Harness application+audit 学习笔记-从 Run 编排到可回放审计](Harness%20application+audit%20学习笔记-从%20Run%20编排到可回放审计.md) | application 六步编排/取消句柄/持久化策略;audit 域层次(trace 子体系/Ledger/审计表);**audit vs trace 区别**(真实数据对照)/三级回放 | ✅ 已沉淀 |
|
||||||
| [Harness contract 状态流学习笔记-11个状态枚举的正交全景](Harness%20contract%20状态流学习笔记-11个状态枚举的正交全景.md) | 五层状态/四正交轴/纵向映射链/真实数据案例/面试叙事模板与追问应对 | ✅ 已沉淀 |
|
| [Harness contract 状态流学习笔记-11个状态枚举的正交全景](Harness%20contract%20状态流学习笔记-11个状态枚举的正交全景.md) | 五层状态/四正交轴/纵向映射链/真实数据案例/面试叙事模板与追问应对 | ✅ 已沉淀 |
|
||||||
| [Harness 面试复习笔记-五步复习与白板图沉淀](Harness%20面试复习笔记-五步复习与白板图沉淀.md) | **详细版**:30 秒陈述展开/三张白板图/九域五段式讲法(动机→决策→实现→边界→话术)/六易错点带原因/追问应对大全/支付超时案例/纠正认知清单/面试 Checklist | ✅ 已沉淀 |
|
| [Harness 面试复习笔记-五步复习与白板图沉淀](Harness%20面试复习笔记-五步复习与白板图沉淀.md) | **详细版**:30 秒陈述展开/三张白板图/九域五段式讲法(动机→决策→实现→边界→话术)/六易错点带原因/追问应对大全/支付超时案例/纠正认知清单/面试 Checklist | ✅ 已沉淀 |
|
||||||
|
| [Harness agent 域学习笔记-从框架 ReAct 接入到受控停止](Harness%20agent%20域学习笔记-从框架%20ReAct%20接入到受控停止.md) | agent 域:装配(Factory 粘合点)/双拦截器(Model 预算+Token 审计、Tool 五道门)/UseCase 循环外壳/受控停止/双视图投影/串行工具 | ✅ 已沉淀 |
|
||||||
|
| [Harness LLM Judge 设计笔记-从不可信判定到可信裁决](Harness%20LLM%20Judge%20设计笔记-从不可信判定到可信裁决.md) | LLM-as-a-judge 模式:SemanticGuard(判支持度)+ EvidenceRepair(修引用)+ GuardModelCall(受控底座);面试五段式回答稿 + 六追问应对 + 通用要素 | ✅ 已沉淀 |
|
||||||
|
| [Harness 整体架构学习笔记-从装配到入口到记忆到知识库写入](Harness%20整体架构学习笔记-从装配到入口到记忆到知识库写入.md) | 整体架构补充:配置装配中心(三层组织)/ HTTP 入口层(薄 Controller + SSE 状态机 + 断连取消)/ 会话与记忆体系(PreviousTurn 注入 + 术语校准 + skill 长期记忆)/ 知识库写入链路(分块 + hybrid 同源) | ✅ 已沉淀 |
|
||||||
|
|
||||||
## 3. 一次请求的完整学习主线
|
## 3. 一次请求的完整学习主线
|
||||||
|
|
||||||
@@ -127,7 +130,7 @@ flowchart LR
|
|||||||
## 4. 下一步规划
|
## 4. 下一步规划
|
||||||
|
|
||||||
```text
|
```text
|
||||||
主线九域全部 ✅ + 面试五步复习 ✅(共沉淀 13 篇笔记)
|
主线九域全部 ✅(含 agent 域收尾)+ 面试五步复习 ✅(共沉淀 14 篇笔记)
|
||||||
|
|
||||||
面试前一天建议:
|
面试前一天建议:
|
||||||
1. 重读「面试速查」§1-2 + §8(30 秒陈述 / 一张图 / 六易错点)
|
1. 重读「面试速查」§1-2 + §8(30 秒陈述 / 一张图 / 六易错点)
|
||||||
@@ -137,8 +140,9 @@ flowchart LR
|
|||||||
5. 2 分钟支付超时案例(复习笔记 §7)
|
5. 2 分钟支付超时案例(复习笔记 §7)
|
||||||
|
|
||||||
可选深化(不阻塞面试):
|
可选深化(不阻塞面试):
|
||||||
1. agent 域收尾:HarnessModelInterceptor / HarnessToolInterceptor 装配细节
|
1. audit 域深化:RagLookupAuditEnricher 检索审计明细(已覆盖大半)
|
||||||
2. audit 域深化:RagLookupAuditEnricher 检索审计明细
|
2. LLM Judge 设计(已沉淀:面试问答 + 追问应对)
|
||||||
|
3. 整体架构补充(已沉淀:装配/入口/记忆体系/知识库写入)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 5. 建议每次学完一个域后更新
|
## 5. 建议每次学完一个域后更新
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# Milvus Hybrid Search 接入对照清单
|
# Milvus Hybrid Search 接入对照清单
|
||||||
|
|
||||||
|
> **已过时(2026-09-29)**:RAG 模块已抽离为独立 py-rag 知识服务,进程内 Milvus
|
||||||
|
> (`MilvusHybridKnowledgeStore` / `VectorSearchService`)与本文描述的接入路径已整体删除。
|
||||||
|
> 当前检索架构与契约映射见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
> 本文仅作历史决策追溯。
|
||||||
|
|
||||||
**日期**:2026-07-27
|
**日期**:2026-07-27
|
||||||
**前提**:旧 Milvus SDK 直连检索路径后续废弃,不作为长期实现基础
|
**前提**:旧 Milvus SDK 直连检索路径后续废弃,不作为长期实现基础
|
||||||
**目标**:在现有 `lookup_knowledge` pipeline 上接入 dense + sparse/BM25 混合检索,融合优先走服务端 RRF
|
**目标**:在现有 `lookup_knowledge` pipeline 上接入 dense + sparse/BM25 混合检索,融合优先走服务端 RRF
|
||||||
|
|||||||
@@ -1,5 +1,12 @@
|
|||||||
# 混合检索上线之后:为什么还要统一 qualityScore,以及上一代后处理错在哪
|
# 混合检索上线之后:为什么还要统一 qualityScore,以及上一代后处理错在哪
|
||||||
|
|
||||||
|
> **现状说明(2026-09-29)**:本文讨论的后处理排序/去重/判级仍在 Java 侧
|
||||||
|
> (`KnowledgeEvidencePostProcessor` / `RetrievalScoreNormalizer`);
|
||||||
|
> 但分数语义已变:py-rag 服务端返回 rerank 绝对分(scoreLabel=`RERANK`,[0,1] 越大越好),
|
||||||
|
> quality 直传,不再走本文所述 dense L2 / hybrid rank 归一化分支(分支保留作兼容)。
|
||||||
|
> 判级阈值 0.75/0.5 与 py-rag 契约一致。当前架构见
|
||||||
|
> [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
**日期**:2026-07-28
|
**日期**:2026-07-28
|
||||||
**范围**:hybrid 检索后的分数语义、后处理排序、相关度闸门、scoreLabel 约定
|
**范围**:hybrid 检索后的分数语义、后处理排序、相关度闸门、scoreLabel 约定
|
||||||
**读者**:已经(或准备)上 dense+BM25+RRF,却发现「召回变了、质量判断还拧着」的工程同学
|
**读者**:已经(或准备)上 dense+BM25+RRF,却发现「召回变了、质量判断还拧着」的工程同学
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# 诊断 Agent 场景下的 RAG 排序:从 K=3 规则加分,到多路召回与 RRF
|
# 诊断 Agent 场景下的 RAG 排序:从 K=3 规则加分,到多路召回与 RRF
|
||||||
|
|
||||||
|
> **现状说明(2026-09-29)**:本文的排序判断框架(多路召回、RRF、Rerank 选型)仍是理解
|
||||||
|
> py-rag 服务端检索设计的背景材料;但 RRF 融合、BM25、rerank 的**实现**已下沉 py-rag 服务端,
|
||||||
|
> Java 侧不再有 `RrfFusion` / `MilvusHybridKnowledgeStore`。
|
||||||
|
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
**日期**:2026-07-27
|
**日期**:2026-07-27
|
||||||
**范围**:知识检索排序、多路召回、分数融合、Rerank 选型
|
**范围**:知识检索排序、多路召回、分数融合、Rerank 选型
|
||||||
**读者**:需要在 Agent 系统里落地 RAG,而不是只做 Demo 问答的工程同学
|
**读者**:需要在 Agent 系统里落地 RAG,而不是只做 Demo 问答的工程同学
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
# RAG 离线评测:讨论、设计与落地
|
# RAG 离线评测:讨论、设计与落地
|
||||||
|
|
||||||
|
> **需重新校准(2026-09-29)**:RAG 模块抽离后,L0 hint / categoryFilter 已下沉 py-rag、
|
||||||
|
> 质量分改为 RERANK 直传、attempt 只剩 `UNFILTERED_VECTOR`;本文设计的 fixture/baseline
|
||||||
|
> 基于旧 L0/scoreLabel 语义构建,回归评测需按新语义重新生成 fixture 并校准闸门。
|
||||||
|
> 当前架构见 [../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)。
|
||||||
|
|
||||||
**日期**:2026-07-28
|
**日期**:2026-07-28
|
||||||
**范围**:`eval/rag-retrieval` 离线 baseline、fixture 生成、与 hybrid/quality 主路径对齐
|
**范围**:`eval/rag-retrieval` 离线 baseline、fixture 生成、与 hybrid/quality 主路径对齐
|
||||||
**读者**:要维护或扩展知识库回归评测的工程同学
|
**读者**:要维护或扩展知识库回归评测的工程同学
|
||||||
|
|||||||
@@ -0,0 +1,336 @@
|
|||||||
|
# RAG 证据链探索笔记:从 py-rag 响应到引用验真
|
||||||
|
|
||||||
|
**日期**:2026-09-29
|
||||||
|
**范围**:RAG 模块抽离后的完整数据链路——每个站点"数据长什么样、字段怎么来、为什么这样设计"
|
||||||
|
**读者**:要理解或维护 `lookup_knowledge` 证据链的工程同学
|
||||||
|
**关联文档**:
|
||||||
|
|
||||||
|
- 架构规范:[../../architecture/RAG知识检索架构.md](../../architecture/RAG知识检索架构.md)(本次抽离后的权威口径)
|
||||||
|
- Trace / 审计:[../../architecture/RAG检索可观测性与审计.md](../../architecture/RAG检索可观测性与审计.md)
|
||||||
|
- 契约原文:py-rag 仓库 `docs/Java接入文档.md`(API v1 冻结面)
|
||||||
|
- 前置知识:`RAG-Hybrid质量分与后处理.md`(分数语义演进史)
|
||||||
|
|
||||||
|
> 本文按一次真实代码探索的顺序组织:从 py-rag 返回的 JSON 出发,沿着数据走过的每一站,
|
||||||
|
> 讲清关键字段、加工规则与设计取舍,最后到 Agent 引用验真收口。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. 全景路线图
|
||||||
|
|
||||||
|
```text
|
||||||
|
py-rag JSON ──► KnowledgeSearchHit 站点1-2:数据形态与防腐层映射
|
||||||
|
│
|
||||||
|
① 后处理 ──► EvidencePostprocessResult 站点3:去重/限流/判级
|
||||||
|
│
|
||||||
|
② 打包 ──► ContextPack 站点4:文本储备(非 Agent 口粮)
|
||||||
|
│
|
||||||
|
③ 组装 ──► LookupResult 站点5:内部真相全集
|
||||||
|
│
|
||||||
|
④ 投影 ──► RagToolResult ──► Agent 站点6-7:冻结契约与 Agent 视图
|
||||||
|
│
|
||||||
|
⑤ Agent 写报告引用 toolCallId
|
||||||
|
│
|
||||||
|
⑥ EvidenceGuard 对账验真 ──► 语义审查 ──► 发布 站点8:证据安全链
|
||||||
|
(旁路:每站关键字段 ──► tool_invocation 审计表,供人排障)
|
||||||
|
```
|
||||||
|
|
||||||
|
一句话定位:**py-rag 负责"把对的片段找出来",Java 侧负责"把找到的证据管起来"**;
|
||||||
|
检索质量可以整体外换,证据治理一寸不动——这次抽离(71 文件 / 约 -8000 行)本身就是证明。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 站点一:py-rag 返回的数据结构
|
||||||
|
|
||||||
|
响应 JSON 按职责分四块,四块数据两条去路(①②③喂后处理,④喂审计):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"query": "网关超时怎么排查", // ① 入参回显
|
||||||
|
"mode": "hybrid", // ① 查询模式(hybrid | semantic)
|
||||||
|
"hits": [ // ② 命中数组(检索的基本单位是 chunk)
|
||||||
|
{
|
||||||
|
"evidence_key": "e2e-gateway-b9c1fa12-md-34223174#chunk-1", // chunk 级身份
|
||||||
|
"document_id": "e2e-gateway-b9c1fa12-md-34223174", // 所属文档
|
||||||
|
"source": "e2e-gateway-b9c1fa12-md", // 来源路径
|
||||||
|
"title": "网关超时排查",
|
||||||
|
"breadcrumb": "网关超时排查 > 处理步骤",
|
||||||
|
"excerpt": "网关超时先检查 upstream 配置…", // 正文
|
||||||
|
"quality_score": 0.9147, // rerank 绝对分
|
||||||
|
"relevance_level": "PRECISE" // py-rag 自己的判级(Java 不用)
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"relevance_level": "PRECISE", // ③ 顶层判级(top1)
|
||||||
|
"evidence_status": "supported", // ③ 业务状态
|
||||||
|
"retrieval_trace": { // ④ 过程记录(不参与后处理,进审计)
|
||||||
|
"mode": "hybrid", "filters": {"category": "gateway"},
|
||||||
|
"recall_count": 20, "rerank_model": "BAAI/bge-reranker-v2-m3",
|
||||||
|
"no_evidence_basis": null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
关键语义:
|
||||||
|
|
||||||
|
- **`evidence_status` 只有两类**:`supported`(正常)/ `no_evidence`(查到了但都是垃圾或无候选,
|
||||||
|
此时 `hits` 恒为空)。它是 **200 正常业务响应**,Java 侧直接走"无知识可用"分支,不重试不报错。
|
||||||
|
- **`relevance_level` 是分数的档位化**:`quality_score ≥0.75 → PRECISE`,`≥0.5 → REFERENCE`,
|
||||||
|
`<0.5` 不输出。阈值当前未校准(契约已知边界)。**它是给模型的,分数是给系统的**——
|
||||||
|
同一信息两种表达,服务两种消费者。
|
||||||
|
- **命中没有独立 metadata map**(旧 Milvus 方案遗留概念),文档归属信息就是那几个平铺字段。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 站点二:防腐层映射——`KnowledgeSearchHit`
|
||||||
|
|
||||||
|
`PyRagKnowledgeSearchAdapter` 把每个 hit 映射成可移植结构。从此全 Java 侧只认这个类型,
|
||||||
|
py-rag 字段再怎么变只改 adapter 一处。后处理实际只消费其中 **7 个字段**:
|
||||||
|
|
||||||
|
```text
|
||||||
|
evidenceKey → 去重键(docId#chunk-N,原样采纳)
|
||||||
|
docId / chunkIndex → 文档分桶(单文档上限);chunkIndex 从 "#chunk-N" 解析
|
||||||
|
content ← excerpt,最终证据正文
|
||||||
|
score + scoreLabel ← quality_score + 常量 "rerank"(quality 直传)
|
||||||
|
originalRank ← 数组下标 +1,排序的权威
|
||||||
|
source / title / breadcrumb → 透传展示
|
||||||
|
```
|
||||||
|
|
||||||
|
`retrieval_trace` 不进这条链——它走审计旁路。**进入后处理时,每个 hit 被浓缩成
|
||||||
|
"身份 + 分数 + 名次 + 正文"四组信息,四步加工全部围绕它们转。**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 站点三:后处理四步——`EvidencePostprocessResult`
|
||||||
|
|
||||||
|
```text
|
||||||
|
① 打分 score + scoreLabel → qualityScore(RERANK 分支直传,clamp [0,1])
|
||||||
|
② 排序 只按 originalRank —— ★ 分数不参与排序,只用于判级
|
||||||
|
③ 去重截断 evidenceKey 去重 → 单文档 chunk ≤2 → 总数 ≤5(return-n)
|
||||||
|
④ 判级 顶分 ≥0.75 PRECISE / ≥0.5 REFERENCE / 否则无档
|
||||||
|
```
|
||||||
|
|
||||||
|
输出结构逐字段(7 条命中进、5 块存出的例子):
|
||||||
|
|
||||||
|
| 字段 | 规则 | 例值 |
|
||||||
|
|---|---|---|
|
||||||
|
| `candidateCount` | 输入候选数 | 7 |
|
||||||
|
| `evidenceBlockCount` | 存活块数(差值 = 被治理掉的) | 5 |
|
||||||
|
| `topSimilarity` | 排序后第一名的 qualityScore | 0.91 |
|
||||||
|
| `relevanceLevel` | Java 按本地阈值独立判定(**不用 py-rag 返回的档位**,那个字段映射时已丢弃) | PRECISE |
|
||||||
|
| `completenessHint` | 档位绑定的天花板提示文案 | "知识库中不存在比上述结果更精准的文档" |
|
||||||
|
| `rerankTrace[]` | 每个**存活**块的最终序账本 | finalRank 1~5 |
|
||||||
|
|
||||||
|
### 3.1 去重辨析:三条规则别混淆
|
||||||
|
|
||||||
|
| 层 | 键 | 规则 | 目的 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 去重 | `evidenceKey = docId#chunk-N` | 同一块再现 → **合并**(hitReasons 取并集) | 同一片段只出现一次 |
|
||||||
|
| 单文档上限 | `docId`(分桶) | 同文档**不同** chunk 可共存,最多 2 | 防单文档刷屏 |
|
||||||
|
| 总条数 | — | 最多 5 | 上下文预算 |
|
||||||
|
|
||||||
|
**合并键是 evidenceKey 而不是 docId**——同文档的第 0 段和第 1 段是两份不同证据,按 docId
|
||||||
|
合并会把多片段证据文档级折叠掉。单次检索正常不会返回同一个 chunk 两次(每个 chunk 是唯一
|
||||||
|
索引条目),合并分支是给"索引脏数据 / 未来多路归并"准备的**防御性兜底**,成本一次 map 查询。
|
||||||
|
|
||||||
|
### 3.2 为什么会同文档多块命中——根源在分块
|
||||||
|
|
||||||
|
入库时文档切成多个 chunk,每块独立 embedding、独立索引;检索按 chunk 算相似度。
|
||||||
|
这是刻意的颗粒度选择:整篇文档一个向量会稀释语义,且上下文也塞不下全文。
|
||||||
|
**切块是为了检索得准、取得少;同文档多块命中是切块的自然结果;
|
||||||
|
"chunk 级身份 + 单文档上限"就是为管理这个现象而生的。**
|
||||||
|
|
||||||
|
### 3.3 `rerankTrace`:最终序账本
|
||||||
|
|
||||||
|
```java
|
||||||
|
Item { finalRank; source; baseScore; finalScore; boostReasons }
|
||||||
|
```
|
||||||
|
|
||||||
|
- 条数 = 存活块数(finalRank 连续编号);被合并/截断的块不产生条目;
|
||||||
|
- `baseScore == finalScore` **恒相等**——双字段是规则加分时代的化石(当年关键词加分,
|
||||||
|
现已废除防操纵);`boostReasons` 同理,装的已是纯解释标签;
|
||||||
|
- 只记到**文档级**(无 evidenceKey),chunk 身份要看 `evidenceBlocks[]`;
|
||||||
|
- Agent 看不到它,审计默认不落库——活在内存 LookupResult 与评测快照里。
|
||||||
|
|
||||||
|
### 3.4 附带闸门
|
||||||
|
|
||||||
|
`isLowQuality()`:无可用块或顶分 <0.5 即低质。原触发 filtered→unfiltered 重试,
|
||||||
|
L0 下沉后分支休眠、闸门保留——将来 Java 侧重引过滤策略可直接接上。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 站点四:打包——`ContextPack`(不是 Agent 口粮)
|
||||||
|
|
||||||
|
`KnowledgeContextPacker` 把证据块压成**一段有字符预算的文本**(默认 4000 字符):
|
||||||
|
|
||||||
|
```text
|
||||||
|
策略 ranked_evidence_char_budget:名次即优先级,先到先得
|
||||||
|
[Evidence 1]
|
||||||
|
source: gateway-timeout.md
|
||||||
|
breadcrumb: 网关超时排查 > 处理步骤
|
||||||
|
reasons: semantic_rank:1, attempt:UNFILTERED_VECTOR ← 召回溯源标签
|
||||||
|
content:
|
||||||
|
网关超时先检查 upstream 配置…
|
||||||
|
预算见底:装得下 header → 正文截断加 "...";连 header 都放不下 → 整条进 omittedSources
|
||||||
|
```
|
||||||
|
|
||||||
|
```java
|
||||||
|
ContextPack { packedText; strategy; charBudget; usedChars; includedSources; omittedSources }
|
||||||
|
```
|
||||||
|
|
||||||
|
**必须澄清的定位**:主诊断链路里 **Agent 不消费 packedText**(主代码零调用)——Agent 拿的是
|
||||||
|
站点六投影后的结构化列表。它的价值:人工回放可读、评测快照存证、以及将来任何
|
||||||
|
"证据进 prompt"路径的现成格式化出口(字符预算是与后处理"块数预算"互补的物理闸门)。
|
||||||
|
|
||||||
|
`reasons:` 行是证据的"简历":`semantic_rank:N`(本次检索第几名)+ `attempt:X`(哪次尝试产出,
|
||||||
|
当前只剩 `UNFILTERED_VECTOR`;历史值 `FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 已随
|
||||||
|
L0 下沉绝迹)。
|
||||||
|
|
||||||
|
> 小化石:`ContextPack` 的 javadoc 仍写 "Agent-facing",与 packer 侧注释矛盾——
|
||||||
|
> 它诞生时确实面向 Agent,投影路线成为主路径后退居内部,注释没跟上身份变化。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 站点五:组装——`LookupResult` 真相全集
|
||||||
|
|
||||||
|
`LookupResultAssembler` 把三样东西合体成内部契约出口:
|
||||||
|
|
||||||
|
```text
|
||||||
|
EvidencePostprocessResult(证据集+档位+trace)┐
|
||||||
|
ContextPack(打包文本) ├─► LookupResult
|
||||||
|
RetrievalTrace(检索路径) ┘
|
||||||
|
```
|
||||||
|
|
||||||
|
只有两个字段是"算"出来的:`found = hasUsableEvidence()`(false 时附固定兜底文案
|
||||||
|
"知识库未检索到可用证据,请结合日志、指标、告警继续排查");两个 count 把"进多少/出多少"
|
||||||
|
带给审计。其余字段一一搬运。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 站点六:投影——`RagResultProjector`(加工链最后一站)
|
||||||
|
|
||||||
|
**输入是 LookupResult 的 JSON 字符串而非对象**——内部结构随便演化,冻结契约纹丝不动,
|
||||||
|
解耦的关键就是这层"字符串边界"(字段名还带防御性别名对:`evidenceBlocks`/`evidence_blocks`)。
|
||||||
|
|
||||||
|
```text
|
||||||
|
① query 截断(≤500 字) 动了 → truncated
|
||||||
|
② 逐块过三道闸:
|
||||||
|
条数 ≤8(超出 truncated+停止)
|
||||||
|
身份去重(evidenceKey → document_id → docId#chunk-N → 序号兜底 的回退链)
|
||||||
|
摘录 ≤1200 字(截断 → truncated)
|
||||||
|
③ evidence_status 客观判定:数组空不空(EVIDENCE_FOUND / NO_EVIDENCE)
|
||||||
|
④ relevance_level:有证据才读,解析不出 → null
|
||||||
|
⑤ fitBudget 总字节兜底:整个 JSON 超 16KB → 从尾部逐条裁
|
||||||
|
裁到空 → 诚实降级 NO_EVIDENCE;还超 → 抛异常(fail closed)
|
||||||
|
```
|
||||||
|
|
||||||
|
**三道递进预算闸**(条数管语义 / 片段管局部 / 字节管整体)集中在 `ToolProjectionLimits`
|
||||||
|
一个 record(8 条 / 1200 字 / 16KB,与日志、MySQL 工具共用)。**`truncated` 只要任何一处
|
||||||
|
动过就置位——Agent 永远知道"看到的可能不全"**,不会把截断结果当全量。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 站点七:Agent 视图——模型实际看到的 JSON
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"evidence_status": "EVIDENCE_FOUND",
|
||||||
|
"tool_call_id": "call_x1",
|
||||||
|
"query": "网关超时怎么排查",
|
||||||
|
"evidence": [
|
||||||
|
{ "document_id": "e2e-gateway-…-34223174#chunk-1",
|
||||||
|
"source": "e2e-gateway-b9c1fa12-md",
|
||||||
|
"title": "网关超时排查",
|
||||||
|
"breadcrumb": "网关超时排查 > 处理步骤",
|
||||||
|
"excerpt": "网关超时先检查 upstream 配置…" }
|
||||||
|
],
|
||||||
|
"returned_count": 5,
|
||||||
|
"relevance_level": "PRECISE",
|
||||||
|
"truncated": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| 字段 | 模型的正确用法 |
|
||||||
|
|---|---|
|
||||||
|
| `evidence_status` | `NO_EVIDENCE` → 老实换工具(日志/指标/MySQL),不许编 |
|
||||||
|
| `evidence[].excerpt` | 结论唯一的内容依据 |
|
||||||
|
| `evidence[].document_id` | **引用坐标**——报告里逐字引用它,EvidenceGuard 只认这个 |
|
||||||
|
| `relevance_level` | PRECISE 可放心下结论;REFERENCE 结合上下文判断,必要时说清还缺什么维度 |
|
||||||
|
| `truncated` | true → 看到的可能不全,可收窄 query 重搜 |
|
||||||
|
|
||||||
|
**三个"看不到"**:分数(防未校准数字诱导过度自信)、过程(attempt/trace 留给人)、
|
||||||
|
其他站的内部字段。一句话:**留坐标、留内容、留诚实,剥掉一切会误导或撑爆上下文的东西。**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 站点八:验真——`EvidenceGuard`(证据安全链第一道闸)
|
||||||
|
|
||||||
|
Agent 写完诊断产出结构化草稿 `DiagnosisDraft`(analysis 带引用的 toolCallIds,
|
||||||
|
conclusion/action_plan 带 basedOnAnalysisIds,limitations 必填)。EvidenceGuard 纯规则验真:
|
||||||
|
|
||||||
|
```text
|
||||||
|
A 结构校验 id 唯一、正文非空、报告引用必须指向已登记分析、limitations 必填
|
||||||
|
B 引用验真 runId+toolCallId → Redis 账本可查 → READY 状态 → 同 Run(防跨 Run 挪用)
|
||||||
|
→ kind 语义匹配:NORMAL↔EVIDENCE_FOUND / NEGATIVE_OBSERVATION↔NO_EVIDENCE
|
||||||
|
C 重读重建 从账本严格反序列化投影(多一个字段都违规)→ 用账本内容重建证据快照
|
||||||
|
```
|
||||||
|
|
||||||
|
四个设计点:
|
||||||
|
|
||||||
|
1. **证据内容从账本重读,不信模型复述**——模型转述的"检索结果"进不了快照;
|
||||||
|
2. **kind 匹配堵两头撒谎**——"查到了装没查到"与"没查到装查到了"都过不去;
|
||||||
|
3. **runId 绑定 + READY**——别的 Run 的证据、失败/过期的调用不可引用;
|
||||||
|
4. **纯规则、20 个违规码全枚举**——便宜、确定、可审计;"结论是否夸大"留给下一道
|
||||||
|
SemanticGuard,这里只回答"引用是否真实、结构是否合法"。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 附:模型怎么知道要输出那份 JSON
|
||||||
|
|
||||||
|
四层合力,没有一刻依赖模型"自觉":
|
||||||
|
|
||||||
|
| 层 | 机制 |
|
||||||
|
|---|---|
|
||||||
|
| 格式 | `DiagnosisDraftOutputSchema`(BeanOutputConverter)从 Java 类自动生成 JSON Schema 注入 prompt;解析失败抛异常 |
|
||||||
|
| 语义 | `diagnosis-agent-prompt.md`:证据充分 → 全填并建立完整引用链;证据不足 → `conclusion=null` 是合法成功(schema 层把 conclusion 类型改成 `object\|null`);limitations 无条件必填 |
|
||||||
|
| 时机 | 模型自主停止(无有效查询范围即停)+ Harness 强制(连续 NO_GAIN / 预算耗尽 → STOP_REQUIRED 后必须直接交卷) |
|
||||||
|
| 兜底 | 格式错/引用违规 → evidence-repair-prompt 修复重试 → 仍败 → 固定降级 FALLBACK |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. 设计思路总结(五条哲学)
|
||||||
|
|
||||||
|
1. **防腐层:换引擎不换证据链**——`KnowledgeSearchPort` 是接缝,抽离时消费面零改动、
|
||||||
|
250 个测试原样通过,这是接口设计价值的最硬证明。
|
||||||
|
2. **分数给系统,档位给模型**——未校准的连续分数会诱导过度自信;`quality_score` 在
|
||||||
|
Java 侧做闸门和审计,Agent 只见 PRECISE/REFERENCE。
|
||||||
|
3. **chunk 级身份贯穿始终**——入库分块、检索按块、去重按 `docId#chunk-N`、引用坐标到块、
|
||||||
|
验真对块。颗粒度统一,才有多片段证据共存与伪引用无处遁形。
|
||||||
|
4. **诚实标记**——`truncated`、`no_evidence`、`unchanged`、`no_evidence_basis`:
|
||||||
|
系统从不假装"看到的即全部",每个不完整/为空都有显式信号与原因。
|
||||||
|
5. **所见即所证**——投影结果是"模型看到的"与"Redis 存证的"同一份;EvidenceGuard 对账
|
||||||
|
没有翻译损耗,伪引用无处遁形。
|
||||||
|
|
||||||
|
### 化石清单(读代码时的辨认指南)
|
||||||
|
|
||||||
|
| 化石 | 现状 |
|
||||||
|
|---|---|
|
||||||
|
| `RerankTrace.baseScore/finalScore` 双字段 | 恒相等(规则加分已废),保留兼容旧审计格式 |
|
||||||
|
| `boostReasons` 字段名 | 装的是纯解释标签,不再加分 |
|
||||||
|
| `ContextPack` javadoc "Agent-facing" | 身份已变(内部/储备),注释未跟上 |
|
||||||
|
| `LookupResultAssembler.deduped()` | 会话级去重回包的历史占位,主链路不再调用 |
|
||||||
|
| `FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` attempt | L0 下沉后不可达,仅存于旧 Run 回放 |
|
||||||
|
| `KnowledgeQuery` 的 L0 hint 字段 | 恒空结构,供后处理与 trace 兼容保留 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. 关键字段速查
|
||||||
|
|
||||||
|
| 字段 | 哪一站 | 一句话 |
|
||||||
|
|---|---|---|
|
||||||
|
| `evidence_key` | py-rag → 全程 | chunk 级身份 `docId#chunk-N`,去重与验真的锚 |
|
||||||
|
| `quality_score` | py-rag → 后处理 | rerank 绝对分 [0,1],quality 直传 |
|
||||||
|
| `evidence_status` | py-rag / 投影 | 两类:supported / no_evidence(正常业务响应) |
|
||||||
|
| `relevance_level` | 后处理 → Agent | PRECISE/REFERENCE 档位,Java 按本地阈值独立判定 |
|
||||||
|
| `originalRank` | adapter → 后处理 | 排序唯一权威,分数不动序 |
|
||||||
|
| `candidateCount / evidenceBlockCount` | 后处理 | 进多少 / 出多少,差值即治理幅度 |
|
||||||
|
| `truncated` | 投影 | 任何截断都置位,诚实标记 |
|
||||||
|
| `tool_call_ids` | Draft → EvidenceGuard | 引用验真的入口,runId 绑定 + READY 校验 |
|
||||||
@@ -1,11 +1,18 @@
|
|||||||
# ISS-017 RAG L0 过滤收窄与 Fallback 加固
|
# ISS-017 RAG L0 过滤收窄与 Fallback 加固
|
||||||
|
|
||||||
**状态**:开放,暂缓实施(保持现网行为)
|
**状态**:已失效(2026-09-29 RAG 抽离,L0 整体下沉 py-rag,问题前提不复存在)
|
||||||
**严重程度**:中
|
**严重程度**:中
|
||||||
**发现时间**:2026-07-28
|
**发现时间**:2026-07-28
|
||||||
**更新日期**:2026-07-28
|
**更新日期**:2026-09-29
|
||||||
**来源**:hybrid + qualityScore 收口后的评测/设计讨论;`chat-l0-filter-fallback` golden case
|
**来源**:hybrid + qualityScore 收口后的评测/设计讨论;`chat-l0-filter-fallback` golden case
|
||||||
**关联**:`LookupKnowledgeTool`、`KnowledgeQueryTransformer`、`KnowledgeEvidencePostProcessor`、`eval/rag-retrieval`、历史 `rag/rag-l0-domain-entity-hint.md`
|
**关联**:`LookupKnowledgeTool`、`KnowledgeQueryTransformer`(已删除)、`KnowledgeEvidencePostProcessor`、`eval/rag-retrieval`、历史 `rag/rag-l0-domain-entity-hint.md`
|
||||||
|
|
||||||
|
> **处理记录(2026-09-29)**:RAG 模块抽离为 py-rag 知识服务时,L0 query 理解
|
||||||
|
> (`KnowledgeQueryTransformer` / `KnowledgeIndexService.analyzeQuery`)整体下沉服务端,
|
||||||
|
> `categoryFilter` 恒为 null,`FILTERED_VECTOR` / `UNFILTERED_VECTOR_RETRY` 分支不再可达。
|
||||||
|
> 本 issue 讨论的「硬过滤赌一把 + 失败整页替换」路径已不存在,无需加固,关闭。
|
||||||
|
> 若未来在 Java 侧重新引入检索过滤策略,filtered→unfiltered 的降级骨架仍在
|
||||||
|
> `LookupKnowledgeTool` 中保留,届时可参考本文的改进方向与回归动机。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
# 知识域表:knowledge_domain
|
# 知识域表:knowledge_domain
|
||||||
|
|
||||||
**状态**:当前表
|
**状态**:孤儿表(2026-09-29 RAG 抽离后无写入方)
|
||||||
**来源**:`V009__add_knowledge_domain.sql`、`KnowledgeDomain`
|
**来源**:`V009__add_knowledge_domain.sql`、`KnowledgeDomain`
|
||||||
|
|
||||||
|
> **2026-09-29 变更**:本表的写入方 `KnowledgeDomainService` 已随 RAG 模块抽离删除
|
||||||
|
> (L0 domain 分析下沉 py-rag)。表结构由 Flyway 保留(`ddl-auto: validate`),当前无读写方,
|
||||||
|
> 待后续迁移清理。历史数据仅供追溯。
|
||||||
|
|
||||||
## 定位
|
## 定位
|
||||||
|
|
||||||
`knowledge_domain` 保存知识库领域级元数据,为 RAG backend 的 domain hint、检索选择和可观测性提供基础信息;它不是 Agent-facing Tool contract。
|
`knowledge_domain` 保存知识库领域级元数据,为 RAG backend 的 domain hint、检索选择和可观测性提供基础信息;它不是 Agent-facing Tool contract。
|
||||||
|
|||||||
@@ -76,12 +76,6 @@
|
|||||||
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
<artifactId>spring-ai-starter-model-deepseek</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- Embedding: SiliconFlow BGE-M3 (需要 OpenAI 模块的 OpenAiEmbeddingModel) -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework.ai</groupId>
|
|
||||||
<artifactId>spring-ai-starter-model-openai</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.alibaba.cloud.ai</groupId>
|
<groupId>com.alibaba.cloud.ai</groupId>
|
||||||
<artifactId>spring-ai-alibaba-agent-framework</artifactId>
|
<artifactId>spring-ai-alibaba-agent-framework</artifactId>
|
||||||
@@ -91,19 +85,14 @@
|
|||||||
<artifactId>spring-boot-starter-web</artifactId>
|
<artifactId>spring-boot-starter-web</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<!--
|
<!--
|
||||||
spring-boot-devtools removed on purpose.
|
spring-boot-devtools removed on purpose (bean recreation on classpath restart
|
||||||
Classpath restart (restartedMain) recreates beans without reliably closing
|
is unreliable). Prefer full process restart: stop then `mvn spring-boot:run`.
|
||||||
MilvusClientV2 gRPC channels, causing orphan channels and long hybrid RPC retries.
|
|
||||||
Prefer full process restart: stop then `mvn spring-boot:run`.
|
|
||||||
-->
|
-->
|
||||||
|
<!-- okhttp:DashScopeConfig 的 RestClient.Builder 使用(此前由 milvus-sdk 传递引入) -->
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>io.milvus</groupId>
|
<groupId>com.squareup.okhttp3</groupId>
|
||||||
<artifactId>milvus-sdk-java</artifactId>
|
<artifactId>okhttp</artifactId>
|
||||||
<version>2.6.10</version>
|
<version>4.12.0</version>
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.springframework.ai</groupId>
|
|
||||||
<artifactId>spring-ai-starter-vector-store-milvus</artifactId>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.springframework.boot</groupId>
|
<groupId>org.springframework.boot</groupId>
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
# 重建 hybrid 知识库(dense + BM25)
|
|
||||||
|
|
||||||
面向当前 `knowledge_base/` 目录文档,**清空并重建**配置中的 Milvus collection(默认 **`biz`**)。
|
|
||||||
|
|
||||||
## 前提
|
|
||||||
|
|
||||||
1. 应用已启动(默认 `http://localhost:9900`)
|
|
||||||
2. `MILVUS_TOKEN` 等连接配置可用
|
|
||||||
3. `application.yml` 已配置:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
milvus:
|
|
||||||
collection: biz
|
|
||||||
retrieval:
|
|
||||||
search:
|
|
||||||
mode: hybrid
|
|
||||||
knowledge:
|
|
||||||
base-path: knowledge_base/
|
|
||||||
```
|
|
||||||
|
|
||||||
## 一键脚本(Python)
|
|
||||||
|
|
||||||
在项目根目录执行:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
|
||||||
```
|
|
||||||
|
|
||||||
指定服务地址:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
|
||||||
```
|
|
||||||
|
|
||||||
跳过前后 stats:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD --skip-stats
|
|
||||||
```
|
|
||||||
|
|
||||||
依赖:Python 3.9+ 标准库即可(无需 pip 包)。
|
|
||||||
|
|
||||||
## 脚本会做什么
|
|
||||||
|
|
||||||
| 步骤 | 动作 |
|
|
||||||
|---|---|
|
|
||||||
| 1 | 检查 `/milvus/health` |
|
|
||||||
| 2 | 打印重建前 `/api/knowledge/stats` |
|
|
||||||
| 3 | `POST /api/knowledge/rebuild-hybrid?confirm=REBUILD` |
|
|
||||||
| 4 | 打印重建后 stats |
|
|
||||||
|
|
||||||
服务端 `rebuild-hybrid` 内部顺序:
|
|
||||||
|
|
||||||
1. **Drop + recreate** Milvus collection(`milvus.collection`,默认 `biz`)
|
|
||||||
- 原有向量数据会被删除
|
|
||||||
- 按 dense + BM25 schema 重建
|
|
||||||
2. **清空** MySQL `api_document`
|
|
||||||
3. **清空** 内存 L0 索引
|
|
||||||
4. **扫描** `knowledge_base/**/*.md`(跳过 `README.md`)并 force 全量导入
|
|
||||||
- 写 MySQL 元数据
|
|
||||||
- 切片
|
|
||||||
- 写 dense 向量 + BM25 `search_text`
|
|
||||||
- 更新 L0
|
|
||||||
|
|
||||||
## 不会做什么
|
|
||||||
|
|
||||||
- **不会**动 `knowledge_base/` 源文件
|
|
||||||
- **不会**在未传 `--confirm REBUILD` 时执行
|
|
||||||
|
|
||||||
## 手动 curl 等价命令
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 重建(危险:会清空 biz collection + api_document)
|
|
||||||
curl -X POST "http://localhost:9900/api/knowledge/rebuild-hybrid?confirm=REBUILD"
|
|
||||||
|
|
||||||
# 仅强制导入(不 drop collection)
|
|
||||||
curl -X POST "http://localhost:9900/api/knowledge/init?force=true"
|
|
||||||
|
|
||||||
# 统计
|
|
||||||
curl "http://localhost:9900/api/knowledge/stats"
|
|
||||||
```
|
|
||||||
|
|
||||||
## 成功判据
|
|
||||||
|
|
||||||
响应中大致应有:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"collection": "biz",
|
|
||||||
"inserted": 15,
|
|
||||||
"failed": 0,
|
|
||||||
"milvus": { "recreated": true, "loaded": true }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
然后用一条知识库里真实存在的术语/故障词走 `lookup_knowledge` 或 chat 验证 hybrid 命中。
|
|
||||||
|
|
||||||
## 失败排查
|
|
||||||
|
|
||||||
| 现象 | 可能原因 |
|
|
||||||
|---|---|
|
|
||||||
| connect / token 错误 | `MILVUS_TOKEN`、host、database |
|
|
||||||
| BM25 / analyzer 相关报错 | 云端 Milvus/Zilliz 版本不支持 BM25 Function |
|
|
||||||
| inserted=0 | `knowledge_base` 路径不对,或 md 缺 frontmatter/title |
|
|
||||||
| failed>0 | 看响应 `details` 与应用日志 |
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Live acceptance runner for post-reindex RAG retrieval checks.
|
|
||||||
|
|
||||||
This script calls the running Spring Boot retrieval endpoint. It is intentionally
|
|
||||||
separate from the offline fixture baseline because it depends on live service and
|
|
||||||
Milvus/Zilliz state.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
import urllib.error
|
|
||||||
import urllib.parse
|
|
||||||
import urllib.request
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_BASE_URL = "http://127.0.0.1:9900"
|
|
||||||
DEFAULT_JSON_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.json")
|
|
||||||
DEFAULT_MD_REPORT = Path("eval/rag-retrieval/reports/live-post-reindex.md")
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_CASES: list[dict[str, Any]] = [
|
|
||||||
{
|
|
||||||
"caseId": "breadcrumb-rag-chunk-context",
|
|
||||||
"query": "If a long RAG section is split into multiple chunks, how do we keep retrieval context?",
|
|
||||||
"topK": 5,
|
|
||||||
"purpose": "Breadcrumb-sensitive RAG chunk context retrieval.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"caseId": "breadcrumb-diagnosis-flow",
|
|
||||||
"query": "What is the standard troubleshooting flow for an application incident?",
|
|
||||||
"topK": 5,
|
|
||||||
"purpose": "Process-style retrieval where section path matters.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"caseId": "core-err-timeout",
|
|
||||||
"query": "ERR_TIMEOUT",
|
|
||||||
"topK": 3,
|
|
||||||
"purpose": "Exact error-code retrieval should remain stable.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"caseId": "core-mysql-connection-pool",
|
|
||||||
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
|
|
||||||
"topK": 3,
|
|
||||||
"purpose": "Core infrastructure troubleshooting retrieval.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"caseId": "aiops-payment-latency",
|
|
||||||
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
|
|
||||||
"topK": 3,
|
|
||||||
"purpose": "AIOps alert-style retrieval.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class LiveCase:
|
|
||||||
case_id: str
|
|
||||||
query: str
|
|
||||||
top_k: int
|
|
||||||
purpose: str
|
|
||||||
category: str | None = None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_json(cls, raw: dict[str, Any]) -> "LiveCase":
|
|
||||||
return cls(
|
|
||||||
case_id=str(raw["caseId"]),
|
|
||||||
query=str(raw["query"]),
|
|
||||||
top_k=int(raw.get("topK") or 3),
|
|
||||||
purpose=str(raw.get("purpose") or raw.get("notes") or ""),
|
|
||||||
category=(
|
|
||||||
str(raw.get("category"))
|
|
||||||
if raw.get("category") not in (None, "")
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def load_cases(path: Path | None) -> list[LiveCase]:
|
|
||||||
if path is None:
|
|
||||||
return [LiveCase.from_json(item) for item in DEFAULT_CASES]
|
|
||||||
with path.open("r", encoding="utf-8") as handle:
|
|
||||||
payload = json.load(handle)
|
|
||||||
raw_cases = payload.get("cases", payload)
|
|
||||||
return [LiveCase.from_json(item) for item in raw_cases]
|
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: Path, payload: Any) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
|
||||||
json.dump(payload, handle, ensure_ascii=False, indent=2)
|
|
||||||
handle.write("\n")
|
|
||||||
|
|
||||||
|
|
||||||
def write_text(path: Path, content: str) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
|
||||||
handle.write(content)
|
|
||||||
|
|
||||||
|
|
||||||
def request_case(base_url: str, case: LiveCase, timeout_seconds: float) -> dict[str, Any]:
|
|
||||||
endpoint = base_url.rstrip("/") + "/api/search/similar"
|
|
||||||
params: dict[str, str] = {
|
|
||||||
"query": case.query,
|
|
||||||
"topK": str(case.top_k),
|
|
||||||
}
|
|
||||||
if case.category:
|
|
||||||
params["category"] = case.category
|
|
||||||
url = endpoint + "?" + urllib.parse.urlencode(params)
|
|
||||||
|
|
||||||
started_at = datetime.now(timezone.utc)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(url, timeout=timeout_seconds) as response:
|
|
||||||
body = response.read().decode("utf-8")
|
|
||||||
payload = json.loads(body)
|
|
||||||
status = int(getattr(response, "status", 200))
|
|
||||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
|
||||||
return {
|
|
||||||
"caseId": case.case_id,
|
|
||||||
"query": case.query,
|
|
||||||
"topK": case.top_k,
|
|
||||||
"category": case.category,
|
|
||||||
"purpose": case.purpose,
|
|
||||||
"url": url,
|
|
||||||
"ok": False,
|
|
||||||
"error": str(exc),
|
|
||||||
"resultCount": 0,
|
|
||||||
"topCandidates": [],
|
|
||||||
"rawResponse": None,
|
|
||||||
"startedAt": started_at.isoformat(),
|
|
||||||
}
|
|
||||||
|
|
||||||
data = payload.get("data") if isinstance(payload, dict) else None
|
|
||||||
if not isinstance(data, list):
|
|
||||||
data = []
|
|
||||||
|
|
||||||
ok = status == 200 and payload.get("code") == 200
|
|
||||||
return {
|
|
||||||
"caseId": case.case_id,
|
|
||||||
"query": case.query,
|
|
||||||
"topK": case.top_k,
|
|
||||||
"category": case.category,
|
|
||||||
"purpose": case.purpose,
|
|
||||||
"url": url,
|
|
||||||
"ok": ok,
|
|
||||||
"httpStatus": status,
|
|
||||||
"responseCode": payload.get("code"),
|
|
||||||
"responseMessage": payload.get("message"),
|
|
||||||
"resultCount": len(data),
|
|
||||||
"topCandidates": [summarize_candidate(item, index + 1) for index, item in enumerate(data)],
|
|
||||||
"rawResponse": payload,
|
|
||||||
"startedAt": started_at.isoformat(),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def summarize_candidate(raw: dict[str, Any], rank: int) -> dict[str, Any]:
|
|
||||||
metadata = parse_metadata(raw.get("metadata"))
|
|
||||||
return {
|
|
||||||
"rank": rank,
|
|
||||||
"id": raw.get("id"),
|
|
||||||
"title": metadata.get("title"),
|
|
||||||
"breadcrumb": metadata.get("breadcrumb"),
|
|
||||||
"category": metadata.get("category"),
|
|
||||||
"source": metadata.get("_source") or metadata.get("source"),
|
|
||||||
"score": raw.get("score"),
|
|
||||||
"rawScore": raw.get("rawScore"),
|
|
||||||
"scoreLabel": raw.get("scoreLabel"),
|
|
||||||
"contentPreview": preview(raw.get("content")),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_metadata(value: Any) -> dict[str, Any]:
|
|
||||||
if isinstance(value, dict):
|
|
||||||
return value
|
|
||||||
if isinstance(value, str) and value.strip():
|
|
||||||
try:
|
|
||||||
parsed = json.loads(value)
|
|
||||||
return parsed if isinstance(parsed, dict) else {}
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
return {}
|
|
||||||
return {}
|
|
||||||
|
|
||||||
|
|
||||||
def preview(value: Any, limit: int = 180) -> str:
|
|
||||||
text = " ".join(str(value or "").split())
|
|
||||||
if len(text) <= limit:
|
|
||||||
return text
|
|
||||||
return text[: limit - 3] + "..."
|
|
||||||
|
|
||||||
|
|
||||||
def render_markdown(report: dict[str, Any]) -> str:
|
|
||||||
lines = [
|
|
||||||
"# RAG Live Post-Reindex Acceptance",
|
|
||||||
"",
|
|
||||||
f"Generated at: `{report['generatedAt']}`",
|
|
||||||
f"Base URL: `{report['baseUrl']}`",
|
|
||||||
"",
|
|
||||||
"> Reindex prerequisite: this report only reflects breadcrumb-aware embedding if the knowledge base was reindexed after the embedding-text change.",
|
|
||||||
"",
|
|
||||||
"## Summary",
|
|
||||||
"",
|
|
||||||
"| Metric | Value |",
|
|
||||||
"|---|---:|",
|
|
||||||
f"| Cases | {report['caseCount']} |",
|
|
||||||
f"| Successful calls | {report['successfulCalls']} |",
|
|
||||||
f"| Empty result cases | {report['emptyResultCases']} |",
|
|
||||||
"",
|
|
||||||
"## Cases",
|
|
||||||
"",
|
|
||||||
"| Case | Purpose | Results | Top Candidates |",
|
|
||||||
"|---|---|---:|---|",
|
|
||||||
]
|
|
||||||
for item in report["results"]:
|
|
||||||
top = "<br>".join(format_candidate(candidate) for candidate in item["topCandidates"])
|
|
||||||
if not top and item.get("error"):
|
|
||||||
top = "ERROR: " + str(item["error"])
|
|
||||||
lines.append(
|
|
||||||
"| {case} | {purpose} | {count} | {top} |".format(
|
|
||||||
case=item["caseId"],
|
|
||||||
purpose=item.get("purpose") or "",
|
|
||||||
count=item["resultCount"],
|
|
||||||
top=top,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
lines.append("")
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def format_candidate(candidate: dict[str, Any]) -> str:
|
|
||||||
label = candidate.get("title") or candidate.get("source") or candidate.get("id") or ""
|
|
||||||
breadcrumb = candidate.get("breadcrumb") or ""
|
|
||||||
score_label = candidate.get("scoreLabel") or ""
|
|
||||||
score = candidate.get("score")
|
|
||||||
raw_score = candidate.get("rawScore")
|
|
||||||
details = f"score={score}"
|
|
||||||
if raw_score is not None:
|
|
||||||
details += f", raw={raw_score}"
|
|
||||||
if score_label:
|
|
||||||
details += f", label={score_label}"
|
|
||||||
if breadcrumb:
|
|
||||||
return f"{candidate['rank']}. {label} ({breadcrumb}; {details})"
|
|
||||||
return f"{candidate['rank']}. {label} ({details})"
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
|
||||||
parser.add_argument("--cases", type=Path, default=None)
|
|
||||||
parser.add_argument("--json-report", type=Path, default=DEFAULT_JSON_REPORT)
|
|
||||||
parser.add_argument("--markdown-report", type=Path, default=DEFAULT_MD_REPORT)
|
|
||||||
parser.add_argument("--timeout-seconds", type=float, default=10.0)
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
args = parse_args()
|
|
||||||
cases = load_cases(args.cases)
|
|
||||||
results = [
|
|
||||||
request_case(args.base_url, case, args.timeout_seconds)
|
|
||||||
for case in cases
|
|
||||||
]
|
|
||||||
successful = [item for item in results if item["ok"]]
|
|
||||||
empty = [item for item in results if item["ok"] and item["resultCount"] == 0]
|
|
||||||
report = {
|
|
||||||
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
|
||||||
"baseUrl": args.base_url,
|
|
||||||
"caseCount": len(results),
|
|
||||||
"successfulCalls": len(successful),
|
|
||||||
"emptyResultCases": len(empty),
|
|
||||||
"reindexPrerequisite": "Run or trigger knowledge-base reindex before treating this as breadcrumb-aware embedding evidence.",
|
|
||||||
"results": results,
|
|
||||||
}
|
|
||||||
write_json(args.json_report, report)
|
|
||||||
write_text(args.markdown_report, render_markdown(report))
|
|
||||||
print(
|
|
||||||
"Ran {total} live cases: successful={successful}, empty={empty}".format(
|
|
||||||
total=len(results),
|
|
||||||
successful=len(successful),
|
|
||||||
empty=len(empty),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return 1 if len(successful) != len(results) else 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,176 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Rebuild knowledge into the configured Milvus collection (default: biz).
|
|
||||||
|
|
||||||
Clears:
|
|
||||||
- milvus.collection (drop + recreate dense+BM25 schema)
|
|
||||||
- MySQL api_document
|
|
||||||
- in-memory L0 index
|
|
||||||
|
|
||||||
Then force-imports all markdown under server-side knowledge.base-path
|
|
||||||
(default: knowledge_base/).
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# start Spring Boot first, then:
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --confirm REBUILD
|
|
||||||
|
|
||||||
python scripts/rebuild_hybrid_knowledge.py --base-url http://127.0.0.1:9900 --confirm REBUILD
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
import urllib.error
|
|
||||||
import urllib.request
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_BASE_URL = "http://localhost:9900"
|
|
||||||
|
|
||||||
|
|
||||||
def http_json(method: str, url: str, timeout: float = 3600.0) -> tuple[int, Any]:
|
|
||||||
req = urllib.request.Request(url=url, method=method.upper())
|
|
||||||
req.add_header("Accept", "application/json")
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
||||||
raw = resp.read().decode("utf-8", errors="replace")
|
|
||||||
status = getattr(resp, "status", 200)
|
|
||||||
if not raw.strip():
|
|
||||||
return status, None
|
|
||||||
return status, json.loads(raw)
|
|
||||||
except urllib.error.HTTPError as exc:
|
|
||||||
raw = exc.read().decode("utf-8", errors="replace")
|
|
||||||
body: Any
|
|
||||||
try:
|
|
||||||
body = json.loads(raw) if raw.strip() else None
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
body = raw
|
|
||||||
raise RuntimeError(f"HTTP {method} {url} failed status={exc.code}: {body}") from exc
|
|
||||||
except urllib.error.URLError as exc:
|
|
||||||
raise RuntimeError(f"HTTP {method} {url} failed: {exc}") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def pretty(obj: Any) -> str:
|
|
||||||
return json.dumps(obj, ensure_ascii=False, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def step(title: str) -> None:
|
|
||||||
print()
|
|
||||||
print(f"==> {title}")
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="Drop/recreate milvus.collection (default biz), clear MySQL api_document + L0, "
|
|
||||||
"and reimport knowledge_base markdown into dense+BM25."
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--base-url",
|
|
||||||
default=DEFAULT_BASE_URL,
|
|
||||||
help=f"Service base URL (default: {DEFAULT_BASE_URL})",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--confirm",
|
|
||||||
required=True,
|
|
||||||
choices=["REBUILD"],
|
|
||||||
help="Must be REBUILD to execute destructive rebuild",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--skip-stats",
|
|
||||||
action="store_true",
|
|
||||||
help="Skip before/after /api/knowledge/stats",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--timeout",
|
|
||||||
type=float,
|
|
||||||
default=7200.0,
|
|
||||||
help="Rebuild request timeout seconds (default: 7200)",
|
|
||||||
)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
|
|
||||||
base_url = args.base_url.rstrip("/")
|
|
||||||
|
|
||||||
print("Hybrid knowledge rebuild")
|
|
||||||
print(f" BaseUrl : {base_url}")
|
|
||||||
print(f" Confirm : {args.confirm}")
|
|
||||||
print(" Source : knowledge_base/ (server-side knowledge.base-path)")
|
|
||||||
print()
|
|
||||||
print("This will DESTROY data in:")
|
|
||||||
print(" - Milvus collection milvus.collection (default: biz)")
|
|
||||||
print(" - MySQL table api_document")
|
|
||||||
print(" - In-memory L0 knowledge index")
|
|
||||||
print("Then re-import all markdown under knowledge_base.")
|
|
||||||
print()
|
|
||||||
|
|
||||||
# 1) health
|
|
||||||
step("Check service health")
|
|
||||||
try:
|
|
||||||
status, body = http_json("GET", f"{base_url}/milvus/health", timeout=30)
|
|
||||||
print(f" milvus health status={status}")
|
|
||||||
print(pretty(body))
|
|
||||||
except Exception as exc: # noqa: BLE001 - ops script should continue on soft health failure
|
|
||||||
print(f" WARN: /milvus/health failed: {exc}")
|
|
||||||
print(" Continue if app is up but milvus health endpoint has issues.")
|
|
||||||
|
|
||||||
# 2) stats before
|
|
||||||
if not args.skip_stats:
|
|
||||||
step("Knowledge stats (before)")
|
|
||||||
try:
|
|
||||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
|
||||||
print(pretty(body))
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
print(f" WARN: stats before failed: {exc}")
|
|
||||||
|
|
||||||
# 3) rebuild
|
|
||||||
step("POST /api/knowledge/rebuild-hybrid?confirm=REBUILD")
|
|
||||||
rebuild_url = f"{base_url}/api/knowledge/rebuild-hybrid?confirm={args.confirm}"
|
|
||||||
try:
|
|
||||||
status, body = http_json("POST", rebuild_url, timeout=args.timeout)
|
|
||||||
except RuntimeError as exc:
|
|
||||||
print(str(exc))
|
|
||||||
return 1
|
|
||||||
|
|
||||||
print(f" HTTP {status}")
|
|
||||||
print(pretty(body))
|
|
||||||
|
|
||||||
if not isinstance(body, dict):
|
|
||||||
print("Unexpected rebuild response type", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
inserted = int(body.get("inserted") or 0)
|
|
||||||
failed = int(body.get("failed") or 0)
|
|
||||||
success = bool(body.get("success"))
|
|
||||||
|
|
||||||
if not success:
|
|
||||||
if inserted <= 0:
|
|
||||||
print()
|
|
||||||
print("Rebuild reported failure and inserted=0. Inspect details above.", file=sys.stderr)
|
|
||||||
return 2
|
|
||||||
print()
|
|
||||||
print(f"Rebuild finished with failed={failed} inserted={inserted}. Review details.")
|
|
||||||
else:
|
|
||||||
print()
|
|
||||||
print(f"Rebuild OK: inserted={inserted}, failed={failed}")
|
|
||||||
|
|
||||||
# 4) stats after
|
|
||||||
if not args.skip_stats:
|
|
||||||
step("Knowledge stats (after)")
|
|
||||||
try:
|
|
||||||
_, body = http_json("GET", f"{base_url}/api/knowledge/stats", timeout=30)
|
|
||||||
print(pretty(body))
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
print(f" WARN: stats after failed: {exc}")
|
|
||||||
|
|
||||||
print()
|
|
||||||
print("Done.")
|
|
||||||
print("Next:")
|
|
||||||
print(" 1) Ensure application.yml has:")
|
|
||||||
print(" milvus.collection: biz")
|
|
||||||
print(" retrieval.search.mode: hybrid")
|
|
||||||
print(" 2) Smoke test lookup_knowledge / chat with a known doc query")
|
|
||||||
return 0 if success or inserted > 0 else 2
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,196 +0,0 @@
|
|||||||
package com.superbiz.agent.client;
|
|
||||||
|
|
||||||
import io.milvus.client.MilvusServiceClient;
|
|
||||||
import io.milvus.grpc.DataType;
|
|
||||||
import io.milvus.param.ConnectParam;
|
|
||||||
import io.milvus.param.IndexType;
|
|
||||||
import io.milvus.param.MetricType;
|
|
||||||
import io.milvus.param.R;
|
|
||||||
import io.milvus.param.RpcStatus;
|
|
||||||
import io.milvus.param.collection.*;
|
|
||||||
import io.milvus.param.index.CreateIndexParam;
|
|
||||||
import com.superbiz.agent.config.MilvusProperties;
|
|
||||||
import com.superbiz.agent.constant.MilvusConstants;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Milvus 客户端工厂类
|
|
||||||
* 负责创建和初始化 Milvus 客户端连接
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class MilvusClientFactory {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(MilvusClientFactory.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MilvusProperties milvusProperties;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建并初始化 Milvus 客户端
|
|
||||||
*
|
|
||||||
* 简化版本:直接连接并创建 collection
|
|
||||||
*
|
|
||||||
* @return MilvusServiceClient 实例
|
|
||||||
* @throws RuntimeException 如果连接或初始化失败
|
|
||||||
*/
|
|
||||||
public MilvusServiceClient createClient() {
|
|
||||||
MilvusServiceClient client = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 1. 连接到 Milvus
|
|
||||||
logger.info("正在连接到 Milvus: {}:{}", milvusProperties.getHost(), milvusProperties.getPort());
|
|
||||||
client = connectToMilvus();
|
|
||||||
logger.info("成功连接到 Milvus");
|
|
||||||
|
|
||||||
// 2. 检查并创建 biz collection(如果不存在)
|
|
||||||
if (!collectionExists(client, MilvusConstants.MILVUS_COLLECTION_NAME)) {
|
|
||||||
logger.info("collection '{}' 不存在,正在创建...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
|
||||||
createBizCollection(client);
|
|
||||||
logger.info("成功创建 collection '{}'", MilvusConstants.MILVUS_COLLECTION_NAME);
|
|
||||||
|
|
||||||
// 创建索引
|
|
||||||
createIndexes(client);
|
|
||||||
logger.info("成功创建索引");
|
|
||||||
} else {
|
|
||||||
logger.info("collection '{}' 已存在", MilvusConstants.MILVUS_COLLECTION_NAME);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 加载 collection 到内存(搜索必须)
|
|
||||||
logger.info("正在加载 collection '{}' 到内存...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
|
||||||
R<RpcStatus> loadResp = client.loadCollection(LoadCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
|
||||||
.build());
|
|
||||||
if (loadResp.getStatus() == 0) {
|
|
||||||
logger.info("collection '{}' 已加载", MilvusConstants.MILVUS_COLLECTION_NAME);
|
|
||||||
} else {
|
|
||||||
logger.warn("collection '{}' 加载失败: {}", MilvusConstants.MILVUS_COLLECTION_NAME, loadResp.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
return client;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("创建 Milvus 客户端失败", e);
|
|
||||||
if (client != null) {
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
throw new RuntimeException("创建 Milvus 客户端失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 连接到 Milvus
|
|
||||||
*/
|
|
||||||
private MilvusServiceClient connectToMilvus() {
|
|
||||||
ConnectParam.Builder builder = ConnectParam.newBuilder()
|
|
||||||
.withHost(milvusProperties.getHost())
|
|
||||||
.withPort(milvusProperties.getPort())
|
|
||||||
.withDatabaseName(milvusProperties.getDatabase())
|
|
||||||
.withConnectTimeout(milvusProperties.getTimeout(), TimeUnit.MILLISECONDS);
|
|
||||||
|
|
||||||
// Zilliz Cloud: token + SSL
|
|
||||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isEmpty()) {
|
|
||||||
builder.withToken(milvusProperties.getToken());
|
|
||||||
builder.withSecure(true);
|
|
||||||
}
|
|
||||||
// 本地 Milvus: username + password
|
|
||||||
else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isEmpty()) {
|
|
||||||
builder.withAuthorization(milvusProperties.getUsername(), milvusProperties.getPassword());
|
|
||||||
}
|
|
||||||
|
|
||||||
return new MilvusServiceClient(builder.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 检查 collection 是否存在
|
|
||||||
*/
|
|
||||||
private boolean collectionExists(MilvusServiceClient client, String collectionName) {
|
|
||||||
R<Boolean> response = client.hasCollection(HasCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
if (response.getStatus() != 0) {
|
|
||||||
throw new RuntimeException("检查 collection 失败: " + response.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.getData();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 创建 biz collection
|
|
||||||
*/
|
|
||||||
private void createBizCollection(MilvusServiceClient client) {
|
|
||||||
// 定义字段
|
|
||||||
FieldType idField = FieldType.newBuilder()
|
|
||||||
.withName("id")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(MilvusConstants.ID_MAX_LENGTH)
|
|
||||||
.withPrimaryKey(true)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
FieldType vectorField = FieldType.newBuilder()
|
|
||||||
.withName("vector")
|
|
||||||
.withDataType(DataType.FloatVector) // 改为 FloatVector
|
|
||||||
.withDimension(milvusProperties.getVectorDim())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
FieldType contentField = FieldType.newBuilder()
|
|
||||||
.withName("content")
|
|
||||||
.withDataType(DataType.VarChar)
|
|
||||||
.withMaxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
FieldType metadataField = FieldType.newBuilder()
|
|
||||||
.withName("metadata")
|
|
||||||
.withDataType(DataType.JSON)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
// 创建 collection schema
|
|
||||||
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
|
|
||||||
.withEnableDynamicField(false)
|
|
||||||
.addFieldType(idField)
|
|
||||||
.addFieldType(vectorField)
|
|
||||||
.addFieldType(contentField)
|
|
||||||
.addFieldType(metadataField)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
// 创建 collection
|
|
||||||
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
|
||||||
.withDescription("Business knowledge collection")
|
|
||||||
.withSchema(schema)
|
|
||||||
.withShardsNum(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> response = client.createCollection(createParam);
|
|
||||||
if (response.getStatus() != 0) {
|
|
||||||
throw new RuntimeException("创建 collection 失败: " + response.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 为 collection 创建索引
|
|
||||||
*/
|
|
||||||
private void createIndexes(MilvusServiceClient client) {
|
|
||||||
// 为 vector 字段创建索引(FloatVector 使用 IVF_FLAT 和 L2 距离)
|
|
||||||
CreateIndexParam vectorIndexParam = CreateIndexParam.newBuilder()
|
|
||||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
|
||||||
.withFieldName("vector")
|
|
||||||
.withIndexType(IndexType.IVF_FLAT)
|
|
||||||
.withMetricType(MetricType.L2) // L2 距离(欧氏距离)
|
|
||||||
.withExtraParam("{\"nlist\":128}")
|
|
||||||
.withSyncMode(Boolean.FALSE)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> response = client.createIndex(vectorIndexParam);
|
|
||||||
if (response.getStatus() != 0) {
|
|
||||||
throw new RuntimeException("创建 vector 索引失败: " + response.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("成功为 vector 字段创建索引");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
package com.superbiz.agent.client;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||||
|
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||||
|
import com.superbiz.agent.config.PyRagProperties;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.core.io.ByteArrayResource;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.util.LinkedMultiValueMap;
|
||||||
|
import org.springframework.util.MultiValueMap;
|
||||||
|
import org.springframework.web.client.RestClient;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import java.util.function.Supplier;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* py-rag 知识服务 HTTP 客户端(API v1,契约见 py-rag 仓库 docs/Java接入文档.md)。
|
||||||
|
*
|
||||||
|
* <p>错误信封:4xx/5xx 一律 {@code {"error":{"code":"E_XXX","message":…,"details":[…]}}},
|
||||||
|
* 统一抛出 {@link PyRagClientException};网络异常包装为 {@code E_NETWORK}。
|
||||||
|
* {@code evidence_status=no_evidence} 是 200 正常业务响应,不作为错误。</p>
|
||||||
|
*
|
||||||
|
* <p>每个请求携带 {@code X-Request-ID}(UUID)用于跨服务日志关联;
|
||||||
|
* 超时按接入文档矩阵分端点配置(见 {@link PyRagProperties})。</p>
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class PyRagClient {
|
||||||
|
|
||||||
|
private static final String REQUEST_ID_HEADER = "X-Request-ID";
|
||||||
|
|
||||||
|
private final PyRagProperties properties;
|
||||||
|
private final ObjectMapper objectMapper;
|
||||||
|
private final RestClient searchClient;
|
||||||
|
private final RestClient ingestClient;
|
||||||
|
private final RestClient defaultClient;
|
||||||
|
|
||||||
|
public PyRagClient(PyRagProperties properties, ObjectMapper objectMapper) {
|
||||||
|
this.properties = properties;
|
||||||
|
this.objectMapper = objectMapper;
|
||||||
|
this.searchClient = buildRestClient(properties.getSearchReadTimeoutMs());
|
||||||
|
this.ingestClient = buildRestClient(properties.getIngestReadTimeoutMs());
|
||||||
|
this.defaultClient = buildRestClient(properties.getDefaultReadTimeoutMs());
|
||||||
|
}
|
||||||
|
|
||||||
|
private RestClient buildRestClient(int readTimeoutMs) {
|
||||||
|
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
|
||||||
|
factory.setConnectTimeout(properties.getConnectTimeoutMs());
|
||||||
|
factory.setReadTimeout(readTimeoutMs);
|
||||||
|
return RestClient.builder()
|
||||||
|
.baseUrl(properties.getBaseUrl())
|
||||||
|
.requestFactory(factory)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 检索 ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
public PyRagSearchResponse search(PyRagSearchRequest request) {
|
||||||
|
return exchange(() -> searchClient.post()
|
||||||
|
.uri("/api/v1/search")
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||||
|
.contentType(MediaType.APPLICATION_JSON)
|
||||||
|
.body(request), PyRagSearchResponse.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 文档入库 ────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* multipart 文档入库。category 为契约必填;title/breadcrumb/kbScope 可选(null 不传)。
|
||||||
|
* 同内容重复上传返回 unchanged(幂等),网络超时可安全重试。
|
||||||
|
*/
|
||||||
|
public PyRagIngestResponse ingest(String filename,
|
||||||
|
byte[] content,
|
||||||
|
String contentType,
|
||||||
|
String category,
|
||||||
|
String title,
|
||||||
|
String breadcrumb,
|
||||||
|
String kbScope) {
|
||||||
|
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||||
|
body.add("file", new ByteArrayResource(content) {
|
||||||
|
@Override
|
||||||
|
public String getFilename() {
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
body.add("category", category);
|
||||||
|
if (title != null && !title.isBlank()) {
|
||||||
|
body.add("title", title);
|
||||||
|
}
|
||||||
|
if (breadcrumb != null && !breadcrumb.isBlank()) {
|
||||||
|
body.add("breadcrumb", breadcrumb);
|
||||||
|
}
|
||||||
|
if (kbScope != null && !kbScope.isBlank()) {
|
||||||
|
body.add("kb_scope", kbScope);
|
||||||
|
}
|
||||||
|
return exchange(() -> ingestClient.post()
|
||||||
|
.uri("/api/v1/documents:ingest")
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||||
|
.contentType(MediaType.MULTIPART_FORM_DATA)
|
||||||
|
.body(body), PyRagIngestResponse.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 全量重建(异步任务) ────────────────────────────────
|
||||||
|
|
||||||
|
/** 202 返回任务号;已有 rebuild 执行中抛 E_REBUILD_IN_PROGRESS。 */
|
||||||
|
public PyRagTaskAccepted rebuild() {
|
||||||
|
return exchange(() -> defaultClient.post()
|
||||||
|
.uri("/api/v1/collections:rebuild?confirm=REBUILD")
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||||
|
.contentType(MediaType.APPLICATION_JSON)
|
||||||
|
.body(Map.of()), PyRagTaskAccepted.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PyRagTaskStatus task(String taskId) {
|
||||||
|
return exchange(() -> defaultClient.get()
|
||||||
|
.uri("/api/v1/tasks/{id}", taskId)
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagTaskStatus.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 统计与健康 ──────────────────────────────────────────
|
||||||
|
|
||||||
|
public PyRagStats stats() {
|
||||||
|
return exchange(() -> defaultClient.get()
|
||||||
|
.uri("/api/v1/stats")
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagStats.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
public PyRagHealth health() {
|
||||||
|
return exchange(() -> defaultClient.get()
|
||||||
|
.uri("/api/v1/health")
|
||||||
|
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagHealth.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 内部 ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
private <T> T exchange(Supplier<RestClient.RequestHeadersSpec<?>> spec, Class<T> type) {
|
||||||
|
try {
|
||||||
|
return spec.get().exchange((request, response) -> {
|
||||||
|
if (response.getStatusCode().isError()) {
|
||||||
|
throw toClientException(response.getStatusCode().value(), response.getBody());
|
||||||
|
}
|
||||||
|
return objectMapper.readValue(response.getBody(), type);
|
||||||
|
});
|
||||||
|
} catch (PyRagClientException e) {
|
||||||
|
throw e;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.error("py-rag 调用失败: {}", e.getMessage(), e);
|
||||||
|
throw new PyRagClientException("E_NETWORK",
|
||||||
|
"py-rag 调用失败: " + e.getMessage(), null, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private PyRagClientException toClientException(int httpStatus, java.io.InputStream body) {
|
||||||
|
String code = "E_HTTP_" + httpStatus;
|
||||||
|
String message = "HTTP " + httpStatus;
|
||||||
|
try {
|
||||||
|
PyRagErrorEnvelope envelope = objectMapper.readValue(body, PyRagErrorEnvelope.class);
|
||||||
|
if (envelope != null && envelope.error() != null) {
|
||||||
|
code = envelope.error().code() == null ? code : envelope.error().code();
|
||||||
|
message = envelope.error().message() == null ? message : envelope.error().message();
|
||||||
|
}
|
||||||
|
} catch (Exception ignored) {
|
||||||
|
// 错误响应体不是契约信封(如网关 502 页面),保留 HTTP 默认语义
|
||||||
|
}
|
||||||
|
return new PyRagClientException(code, message, httpStatus, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 契约 DTO(snake_case 对齐 py-rag API) ──────────────
|
||||||
|
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
public record PyRagSearchRequest(
|
||||||
|
String query,
|
||||||
|
/** hybrid=dense+BM25 融合;semantic=纯向量 */
|
||||||
|
String mode,
|
||||||
|
Integer retrieveK,
|
||||||
|
Integer returnN,
|
||||||
|
Integer maxChunksPerDocument,
|
||||||
|
String category,
|
||||||
|
String kbScope
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagSearchHit(
|
||||||
|
String evidenceKey,
|
||||||
|
String documentId,
|
||||||
|
String source,
|
||||||
|
String title,
|
||||||
|
String breadcrumb,
|
||||||
|
String excerpt,
|
||||||
|
Double qualityScore,
|
||||||
|
String relevanceLevel
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagRetrievalTrace(
|
||||||
|
String mode,
|
||||||
|
Map<String, Object> filters,
|
||||||
|
Integer recallCount,
|
||||||
|
String rerankModel,
|
||||||
|
String noEvidenceBasis
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagSearchResponse(
|
||||||
|
String query,
|
||||||
|
String mode,
|
||||||
|
List<PyRagSearchHit> hits,
|
||||||
|
String relevanceLevel,
|
||||||
|
/** supported | no_evidence(no_evidence 时 hits=[],属正常业务响应) */
|
||||||
|
String evidenceStatus,
|
||||||
|
PyRagRetrievalTrace retrievalTrace
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagIngestResponse(
|
||||||
|
String docId,
|
||||||
|
String source,
|
||||||
|
/** created | updated | unchanged */
|
||||||
|
String status,
|
||||||
|
Integer chunkCount,
|
||||||
|
List<String> warnings,
|
||||||
|
Map<String, Object> frontmatter
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagTaskAccepted(
|
||||||
|
String taskId,
|
||||||
|
String status
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagTaskStatus(
|
||||||
|
String taskId,
|
||||||
|
String status,
|
||||||
|
Integer documents,
|
||||||
|
String detail,
|
||||||
|
String createdAt,
|
||||||
|
String finishedAt
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagStats(
|
||||||
|
String collection,
|
||||||
|
Integer rowCount
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagHealth(
|
||||||
|
String status,
|
||||||
|
String version,
|
||||||
|
Map<String, String> checks
|
||||||
|
) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record PyRagErrorEnvelope(ErrorBody error) {
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record ErrorBody(String code, String message, List<Map<String, Object>> details) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package com.superbiz.agent.client;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* py-rag 调用异常:携带契约错误码(E_*)与 HTTP 状态。
|
||||||
|
*
|
||||||
|
* <p>调用方按错误码分支:422 参数/数据问题不重试;
|
||||||
|
* 409 E_REBUILD_IN_PROGRESS 延迟重试;网络异常(E_NETWORK)可安全重试。</p>
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
public class PyRagClientException extends RuntimeException {
|
||||||
|
|
||||||
|
/** 契约错误码:E_INVALID_REQUEST / E_FRONTMATTER_INVALID / E_REBUILD_IN_PROGRESS / E_NETWORK 等 */
|
||||||
|
private final String code;
|
||||||
|
|
||||||
|
/** HTTP 状态码;网络层异常(未拿到响应)为 null */
|
||||||
|
private final Integer httpStatus;
|
||||||
|
|
||||||
|
public PyRagClientException(String code, String message, Integer httpStatus, Throwable cause) {
|
||||||
|
super("[" + code + "] " + message, cause);
|
||||||
|
this.code = code;
|
||||||
|
this.httpStatus = httpStatus;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package com.superbiz.agent.config;
|
|
||||||
|
|
||||||
import lombok.Getter;
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档分片配置
|
|
||||||
*/
|
|
||||||
@Getter
|
|
||||||
@Configuration
|
|
||||||
@ConfigurationProperties(prefix = "document.chunk")
|
|
||||||
public class DocumentChunkConfig {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 每个分片的最大字符数(保留向后兼容)
|
|
||||||
*/
|
|
||||||
private int maxSize = 800;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片之间的重叠字符数
|
|
||||||
*/
|
|
||||||
private int overlap = 100;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 每个分片的最大 token 数(中文~1:1,英文~0.25:1)
|
|
||||||
* 替代 maxSize 作为切割触发器
|
|
||||||
*/
|
|
||||||
private int maxTokens = 500;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 硬上限 token 数 = maxTokens × 1.2
|
|
||||||
* 仅在不可中断上下文(列表、代码块)内触发
|
|
||||||
*/
|
|
||||||
private int maxTokensHard = 600;
|
|
||||||
|
|
||||||
public void setMaxSize(int maxSize) {
|
|
||||||
this.maxSize = maxSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setOverlap(int overlap) {
|
|
||||||
this.overlap = overlap;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setMaxTokens(int maxTokens) {
|
|
||||||
this.maxTokens = maxTokens;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setMaxTokensHard(int maxTokensHard) {
|
|
||||||
this.maxTokensHard = maxTokensHard;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
package com.superbiz.agent.config;
|
|
||||||
|
|
||||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Milvus 知识路径配置说明(无额外 Bean 装配)。
|
|
||||||
*
|
|
||||||
* <p>知识库 RAG 唯一实现:{@link MilvusHybridKnowledgeStore}({@code MilvusClientV2})。</p>
|
|
||||||
* <ul>
|
|
||||||
* <li>支持 dense 与 dense+BM25 {@code hybridSearch}+RRF。</li>
|
|
||||||
* <li>不再为知识路径创建 legacy {@code MilvusServiceClient} Bean。</li>
|
|
||||||
* <li>Spring AI {@code VectorStore} starter 仍可存在于 classpath,但只作 sidecar,
|
|
||||||
* 不作 lookup_knowledge 主路径(starter 无 BM25 hybrid API)。</li>
|
|
||||||
* </ul>
|
|
||||||
*/
|
|
||||||
@Configuration
|
|
||||||
public class MilvusConfig {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(MilvusConfig.class);
|
|
||||||
|
|
||||||
public MilvusConfig() {
|
|
||||||
logger.info("Milvus knowledge path: MilvusClientV2 hybrid store only (legacy SDK search disabled)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,95 +0,0 @@
|
|||||||
package com.superbiz.agent.config;
|
|
||||||
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
|
|
||||||
@Configuration
|
|
||||||
@ConfigurationProperties(prefix = "milvus")
|
|
||||||
public class MilvusProperties {
|
|
||||||
|
|
||||||
private String host = "localhost";
|
|
||||||
private Integer port = 19530;
|
|
||||||
private String username = "";
|
|
||||||
private String password = "";
|
|
||||||
private String database = "default";
|
|
||||||
private Long timeout = 10000L;
|
|
||||||
private String token = "";
|
|
||||||
private boolean secure = false;
|
|
||||||
private int vectorDim = 1024;
|
|
||||||
|
|
||||||
public String getHost() {
|
|
||||||
return host;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setHost(String host) {
|
|
||||||
this.host = host;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Integer getPort() {
|
|
||||||
return port;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setPort(Integer port) {
|
|
||||||
this.port = port;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getUsername() {
|
|
||||||
return username;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setUsername(String username) {
|
|
||||||
this.username = username;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getPassword() {
|
|
||||||
return password;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setPassword(String password) {
|
|
||||||
this.password = password;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatabase() {
|
|
||||||
return database;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatabase(String database) {
|
|
||||||
this.database = database;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Long getTimeout() {
|
|
||||||
return timeout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setTimeout(Long timeout) {
|
|
||||||
this.timeout = timeout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getToken() {
|
|
||||||
return token;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setToken(String token) {
|
|
||||||
this.token = token;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isSecure() {
|
|
||||||
return secure;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSecure(boolean secure) {
|
|
||||||
this.secure = secure;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getVectorDim() {
|
|
||||||
return vectorDim;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setVectorDim(int vectorDim) {
|
|
||||||
this.vectorDim = vectorDim;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getAddress() {
|
|
||||||
return host + ":" + port;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +1,10 @@
|
|||||||
package com.superbiz.agent.config;
|
package com.superbiz.agent.config;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
import org.springframework.ai.chat.model.ChatModel;
|
||||||
import org.springframework.ai.embedding.EmbeddingModel;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
import org.springframework.context.annotation.Bean;
|
import org.springframework.context.annotation.Bean;
|
||||||
import org.springframework.context.annotation.Configuration;
|
import org.springframework.context.annotation.Configuration;
|
||||||
@@ -19,11 +17,11 @@ import org.springframework.context.annotation.Primary;
|
|||||||
* <pre>{@code
|
* <pre>{@code
|
||||||
* model-routing:
|
* model-routing:
|
||||||
* chat: deepseek
|
* chat: deepseek
|
||||||
* embedding: siliconflow
|
|
||||||
* }</pre>
|
* }</pre>
|
||||||
* <p>
|
* <p>
|
||||||
* 匹配优先级:Bean 名 > 类名(均不区分大小写)。
|
* 匹配优先级:Bean 名 > 类名(均不区分大小写)。
|
||||||
* 切换模型只改 yml + pom + 对应 api-key,Java 代码不动。
|
* 切换模型只改 yml + pom + 对应 api-key,Java 代码不动。
|
||||||
|
* (Embedding 路由已随 RAG 模块抽离至 py-rag 服务端,此处仅路由 Chat。)
|
||||||
*/
|
*/
|
||||||
@Configuration
|
@Configuration
|
||||||
public class ModelRoutingConfig {
|
public class ModelRoutingConfig {
|
||||||
@@ -33,9 +31,6 @@ public class ModelRoutingConfig {
|
|||||||
@Value("${model-routing.chat:deepseek}")
|
@Value("${model-routing.chat:deepseek}")
|
||||||
private String chatKeyword;
|
private String chatKeyword;
|
||||||
|
|
||||||
@Value("${model-routing.embedding:siliconflow}")
|
|
||||||
private String embeddingKeyword;
|
|
||||||
|
|
||||||
@Bean
|
@Bean
|
||||||
@Primary
|
@Primary
|
||||||
public ChatModel chatModel(List<ChatModel> chatModels) {
|
public ChatModel chatModel(List<ChatModel> chatModels) {
|
||||||
@@ -53,33 +48,6 @@ public class ModelRoutingConfig {
|
|||||||
return chatModels.get(0);
|
return chatModels.get(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Bean
|
|
||||||
@Primary
|
|
||||||
public EmbeddingModel embeddingModel(Map<String, EmbeddingModel> embeddingBeans) {
|
|
||||||
log.info("Embedding 路由: keyword='{}', 可用: {}", embeddingKeyword, embeddingBeans.keySet());
|
|
||||||
|
|
||||||
// 先按 Bean 名匹配
|
|
||||||
for (Map.Entry<String, EmbeddingModel> entry : embeddingBeans.entrySet()) {
|
|
||||||
if (containsIgnoreCase(entry.getKey(), embeddingKeyword)) {
|
|
||||||
log.info(" → Bean 名匹配: {} → {}", entry.getKey(),
|
|
||||||
entry.getValue().getClass().getSimpleName());
|
|
||||||
return entry.getValue();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 再按类名匹配
|
|
||||||
for (EmbeddingModel em : embeddingBeans.values()) {
|
|
||||||
if (matches(em.getClass(), embeddingKeyword)) {
|
|
||||||
log.info(" → 类名匹配: {}", em.getClass().getSimpleName());
|
|
||||||
return em;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var first = embeddingBeans.values().iterator().next();
|
|
||||||
log.warn(" → 未匹配, 回退到 {}", first.getClass().getSimpleName());
|
|
||||||
return first;
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean matches(Class<?> clazz, String keyword) {
|
private boolean matches(Class<?> clazz, String keyword) {
|
||||||
return containsIgnoreCase(clazz.getName(), keyword)
|
return containsIgnoreCase(clazz.getName(), keyword)
|
||||||
|| containsIgnoreCase(clazz.getSimpleName(), keyword);
|
|| containsIgnoreCase(clazz.getSimpleName(), keyword);
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package com.superbiz.agent.config;
|
||||||
|
|
||||||
|
import lombok.Getter;
|
||||||
|
import lombok.Setter;
|
||||||
|
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* py-rag 知识服务接入配置。
|
||||||
|
*
|
||||||
|
* <p>超时矩阵来自《py-rag 知识服务 · Java 接入文档》第 6 节:
|
||||||
|
* 服务含 embedding/rerank 外呼,search 正常 300–800ms、ingest 正常 1–5s。</p>
|
||||||
|
*/
|
||||||
|
@Getter
|
||||||
|
@Setter
|
||||||
|
@Configuration
|
||||||
|
@ConfigurationProperties(prefix = "pyrag")
|
||||||
|
public class PyRagProperties {
|
||||||
|
|
||||||
|
/** py-rag 服务根地址,如 http://py-rag:8000 */
|
||||||
|
private String baseUrl = "http://localhost:8000";
|
||||||
|
|
||||||
|
/** 连接超时(毫秒),全端点统一 */
|
||||||
|
private int connectTimeoutMs = 3000;
|
||||||
|
|
||||||
|
/** /api/v1/search 读取超时(毫秒) */
|
||||||
|
private int searchReadTimeoutMs = 5000;
|
||||||
|
|
||||||
|
/** /api/v1/documents:ingest 读取超时(毫秒) */
|
||||||
|
private int ingestReadTimeoutMs = 30000;
|
||||||
|
|
||||||
|
/** rebuild/tasks/stats/health 读取超时(毫秒) */
|
||||||
|
private int defaultReadTimeoutMs = 10000;
|
||||||
|
}
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
package com.superbiz.agent.config;
|
|
||||||
|
|
||||||
import lombok.Getter;
|
|
||||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
@Configuration
|
|
||||||
@ConfigurationProperties(prefix = "rag.sidecar.spring-ai")
|
|
||||||
public class RagSidecarProperties {
|
|
||||||
|
|
||||||
private boolean enabled = false;
|
|
||||||
|
|
||||||
private int contentPreviewLimit = 300;
|
|
||||||
|
|
||||||
public void setEnabled(boolean enabled) {
|
|
||||||
this.enabled = enabled;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setContentPreviewLimit(int contentPreviewLimit) {
|
|
||||||
this.contentPreviewLimit = contentPreviewLimit;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
package com.superbiz.agent.config;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.ai.document.MetadataMode;
|
|
||||||
import org.springframework.ai.embedding.EmbeddingModel;
|
|
||||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
|
||||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
|
||||||
import org.springframework.ai.openai.api.OpenAiApi;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
import org.springframework.web.client.RestClient;
|
|
||||||
import org.springframework.web.reactive.function.client.WebClient;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* SiliconFlow Embedding 配置(BGE-M3, OpenAI 兼容协议, 1024维)
|
|
||||||
* <p>
|
|
||||||
* Chat 走 DeepSeek、Embedding 走 SiliconFlow,两者都是 OpenAI 兼容但地址不同,
|
|
||||||
* 因此单独为 SiliconFlow 创建 OpenAiApi + EmbeddingModel Bean。
|
|
||||||
*/
|
|
||||||
@Configuration
|
|
||||||
public class SiliconFlowEmbeddingConfig {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(SiliconFlowEmbeddingConfig.class);
|
|
||||||
|
|
||||||
@Value("${siliconflow.api-key}")
|
|
||||||
private String apiKey;
|
|
||||||
|
|
||||||
@Value("${siliconflow.base-url}")
|
|
||||||
private String baseUrl;
|
|
||||||
|
|
||||||
@Value("${siliconflow.embedding.model}")
|
|
||||||
private String model;
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
public OpenAiApi siliconFlowApi(RestClient.Builder restClientBuilder, WebClient.Builder webClientBuilder) {
|
|
||||||
log.info("创建 SiliconFlow OpenAiApi: {}", baseUrl);
|
|
||||||
return OpenAiApi.builder()
|
|
||||||
.baseUrl(baseUrl)
|
|
||||||
.apiKey(apiKey)
|
|
||||||
.restClientBuilder(restClientBuilder)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Bean
|
|
||||||
public EmbeddingModel siliconFlowEmbeddingModel(OpenAiApi siliconFlowApi) {
|
|
||||||
log.info("创建 SiliconFlow EmbeddingModel, model: {}", model);
|
|
||||||
return new OpenAiEmbeddingModel(siliconFlowApi, MetadataMode.EMBED,
|
|
||||||
OpenAiEmbeddingOptions.builder()
|
|
||||||
.model(model)
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
package com.superbiz.agent.constant;
|
|
||||||
|
|
||||||
public class MilvusConstants {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Milvus 数据库名称
|
|
||||||
*/
|
|
||||||
public static final String MILVUS_DB_NAME = "default";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Default knowledge collection name (dense + BM25).
|
|
||||||
* Overridable via {@code milvus.collection}.
|
|
||||||
*/
|
|
||||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Alias kept for readability in hybrid-related code.
|
|
||||||
*/
|
|
||||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 向量维度(豆包 embedding 模型的维度)
|
|
||||||
*/
|
|
||||||
public static final int VECTOR_DIM = 1024; // 豆包模型返回1024维向量
|
|
||||||
|
|
||||||
/**
|
|
||||||
* ID字段最大长度
|
|
||||||
*/
|
|
||||||
public static final int ID_MAX_LENGTH = 256;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Content字段最大长度
|
|
||||||
*/
|
|
||||||
public static final int CONTENT_MAX_LENGTH = 8192;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 默认分片数
|
|
||||||
*/
|
|
||||||
public static final int DEFAULT_SHARD_NUMBER = 2;
|
|
||||||
|
|
||||||
private MilvusConstants() {
|
|
||||||
// 工具类,禁止实例化
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
package com.superbiz.agent.controller;
|
package com.superbiz.agent.controller;
|
||||||
|
|
||||||
|
import com.superbiz.agent.client.PyRagClient;
|
||||||
import com.superbiz.agent.config.FileUploadConfig;
|
import com.superbiz.agent.config.FileUploadConfig;
|
||||||
import com.superbiz.agent.dto.FileUploadRes;
|
import com.superbiz.agent.dto.FileUploadRes;
|
||||||
import com.superbiz.agent.service.VectorIndexService;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
@@ -29,10 +29,11 @@ public class FileUploadController {
|
|||||||
private FileUploadConfig fileUploadConfig;
|
private FileUploadConfig fileUploadConfig;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private VectorIndexService vectorIndexService;
|
private PyRagClient pyRagClient;
|
||||||
|
|
||||||
@PostMapping(value = "/api/upload", consumes = "multipart/form-data")
|
@PostMapping(value = "/api/upload", consumes = "multipart/form-data")
|
||||||
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file) {
|
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file,
|
||||||
|
@RequestParam(value = "category", required = false) String category) {
|
||||||
if (file.isEmpty()) {
|
if (file.isEmpty()) {
|
||||||
return ResponseEntity.badRequest().body("文件不能为空");
|
return ResponseEntity.badRequest().body("文件不能为空");
|
||||||
}
|
}
|
||||||
@@ -68,15 +69,17 @@ public class FileUploadController {
|
|||||||
|
|
||||||
logger.info("文件上传成功: {}", filePath);
|
logger.info("文件上传成功: {}", filePath);
|
||||||
|
|
||||||
// 文件上传成功后,自动调用向量索引服务
|
// 转发 py-rag 入库(同内容重传返回 unchanged)。入库失败不影响上传成功语义。
|
||||||
try {
|
try {
|
||||||
logger.info("开始为上传文件创建向量索引: {}", filePath);
|
String ingestCategory = (category == null || category.isBlank()) ? "default" : category;
|
||||||
vectorIndexService.indexSingleFile(filePath.toString());
|
logger.info("开始 py-rag 入库: {}, category={}", filePath, ingestCategory);
|
||||||
logger.info("向量索引创建成功: {}", filePath);
|
var ingest = pyRagClient.ingest(originalFilename, file.getBytes(), file.getContentType(),
|
||||||
|
ingestCategory, null, null, null);
|
||||||
|
logger.info("py-rag 入库完成: docId={}, status={}, chunks={}",
|
||||||
|
ingest.docId(), ingest.status(), ingest.chunkCount());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("向量索引创建失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
logger.error("py-rag 入库失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
||||||
// 注意:即使索引失败,文件上传仍然成功,只是记录错误日志
|
// 注意:即使入库失败,文件上传仍然成功,只是记录错误日志
|
||||||
// 可以根据业务需求决定是否要删除文件或返回错误
|
|
||||||
}
|
}
|
||||||
|
|
||||||
FileUploadRes response = new FileUploadRes(
|
FileUploadRes response = new FileUploadRes(
|
||||||
|
|||||||
@@ -1,147 +0,0 @@
|
|||||||
package com.superbiz.agent.controller;
|
|
||||||
|
|
||||||
import com.superbiz.agent.service.KnowledgeBaseInitService;
|
|
||||||
import lombok.Data;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.http.ResponseEntity;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库管理控制器
|
|
||||||
* 提供知识库初始化、查询等接口
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/knowledge")
|
|
||||||
public class KnowledgeBaseController {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeBaseInitService initService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 初始化知识库
|
|
||||||
* 扫描 knowledge_base 目录下的所有文档,去重后批量导入到数据库和 Milvus
|
|
||||||
*
|
|
||||||
* @param force 是否强制重新导入(跳过去重检查)
|
|
||||||
* @return 初始化结果
|
|
||||||
*/
|
|
||||||
@PostMapping("/init")
|
|
||||||
public ResponseEntity<?> initKnowledgeBase(@RequestParam(defaultValue = "false") boolean force) {
|
|
||||||
logger.info("收到知识库初始化请求, force={}", force);
|
|
||||||
|
|
||||||
try {
|
|
||||||
KnowledgeBaseInitService.InitResult result = initService.initializeKnowledgeBase(force);
|
|
||||||
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", true);
|
|
||||||
response.put("message", "知识库初始化完成");
|
|
||||||
response.put("scanned", result.getScanned());
|
|
||||||
response.put("skipped", result.getSkipped());
|
|
||||||
response.put("inserted", result.getInserted());
|
|
||||||
response.put("failed", result.getFailed());
|
|
||||||
response.put("details", result.getDetails());
|
|
||||||
|
|
||||||
logger.info("知识库初始化成功: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
|
||||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
|
||||||
|
|
||||||
return ResponseEntity.ok(response);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("知识库初始化失败", e);
|
|
||||||
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", false);
|
|
||||||
response.put("message", "初始化失败: " + e.getMessage());
|
|
||||||
|
|
||||||
return ResponseEntity.internalServerError().body(response);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
|
|
||||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
|
||||||
*
|
|
||||||
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
|
|
||||||
* 需要显式 confirm=REBUILD。</p>
|
|
||||||
*/
|
|
||||||
@PostMapping("/rebuild-hybrid")
|
|
||||||
public ResponseEntity<?> rebuildHybrid(
|
|
||||||
@RequestParam(defaultValue = "") String confirm) {
|
|
||||||
if (!"REBUILD".equals(confirm)) {
|
|
||||||
Map<String, Object> rejected = new HashMap<>();
|
|
||||||
rejected.put("success", false);
|
|
||||||
rejected.put("message", "拒绝执行:请传 confirm=REBUILD 以确认清空并重建");
|
|
||||||
rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
|
|
||||||
return ResponseEntity.badRequest().body(rejected);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
|
|
||||||
try {
|
|
||||||
KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", result.isSuccess());
|
|
||||||
response.put("message", result.isSuccess()
|
|
||||||
? "hybrid 知识库重建完成"
|
|
||||||
: "hybrid 知识库重建结束,但存在失败项");
|
|
||||||
response.put("collection", result.getCollection());
|
|
||||||
response.put("basePath", result.getBasePath());
|
|
||||||
response.put("milvus", result.getMilvus());
|
|
||||||
response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
|
|
||||||
response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
|
|
||||||
response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
|
|
||||||
response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
|
|
||||||
response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
|
|
||||||
if (result.getInit() != null) {
|
|
||||||
response.put("scanned", result.getInit().getScanned());
|
|
||||||
response.put("skipped", result.getInit().getSkipped());
|
|
||||||
response.put("inserted", result.getInit().getInserted());
|
|
||||||
response.put("failed", result.getInit().getFailed());
|
|
||||||
response.put("details", result.getInit().getDetails());
|
|
||||||
}
|
|
||||||
return result.isSuccess()
|
|
||||||
? ResponseEntity.ok(response)
|
|
||||||
: ResponseEntity.status(500).body(response);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("hybrid 知识库重建失败", e);
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", false);
|
|
||||||
response.put("message", "重建失败: " + e.getMessage());
|
|
||||||
return ResponseEntity.internalServerError().body(response);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 查询知识库统计信息
|
|
||||||
*
|
|
||||||
* @return 统计信息
|
|
||||||
*/
|
|
||||||
@GetMapping("/stats")
|
|
||||||
public ResponseEntity<?> getStats() {
|
|
||||||
try {
|
|
||||||
KnowledgeBaseInitService.Stats stats = initService.getStats();
|
|
||||||
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", true);
|
|
||||||
response.put("totalDocuments", stats.getTotalDocuments());
|
|
||||||
response.put("totalVectors", stats.getTotalVectors());
|
|
||||||
response.put("categories", stats.getCategoryCount());
|
|
||||||
|
|
||||||
return ResponseEntity.ok(response);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("查询统计信息失败", e);
|
|
||||||
|
|
||||||
Map<String, Object> response = new HashMap<>();
|
|
||||||
response.put("success", false);
|
|
||||||
response.put("message", "查询失败: " + e.getMessage());
|
|
||||||
|
|
||||||
return ResponseEntity.internalServerError().body(response);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
package com.superbiz.agent.controller;
|
|
||||||
|
|
||||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
|
||||||
import io.milvus.v2.service.collection.response.ListCollectionsResp;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.http.ResponseEntity;
|
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Milvus health check using the single V2 knowledge backend.
|
|
||||||
*/
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/milvus")
|
|
||||||
public class MilvusCheckController {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
|
||||||
|
|
||||||
@GetMapping("/health")
|
|
||||||
public ResponseEntity<Map<String, Object>> simpleHealth() {
|
|
||||||
Map<String, Object> result = new HashMap<>();
|
|
||||||
try {
|
|
||||||
ListCollectionsResp response = knowledgeStore.client().listCollections();
|
|
||||||
result.put("message", "ok");
|
|
||||||
result.put("backend", "milvus-client-v2");
|
|
||||||
result.put("knowledgeCollection", knowledgeStore.collectionName());
|
|
||||||
result.put("collections", response == null ? null : response.getCollectionNames());
|
|
||||||
return ResponseEntity.ok(result);
|
|
||||||
} catch (Exception e) {
|
|
||||||
result.put("error", e.getMessage());
|
|
||||||
return ResponseEntity.status(503).body(result);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
package com.superbiz.agent.controller;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.Result;
|
|
||||||
import com.superbiz.agent.service.VectorSearchService;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档检索控制器(测试用)
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@RestController
|
|
||||||
@RequestMapping("/api/search")
|
|
||||||
public class SearchController {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorSearchService vectorSearchService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 搜索相似文档
|
|
||||||
*/
|
|
||||||
@GetMapping("/similar")
|
|
||||||
public Result<List<VectorSearchService.SearchResult>> searchSimilar(
|
|
||||||
@RequestParam("query") String query,
|
|
||||||
@RequestParam(value = "topK", defaultValue = "5") int topK,
|
|
||||||
@RequestParam(value = "category", required = false) String category
|
|
||||||
) {
|
|
||||||
try {
|
|
||||||
log.info("收到检索请求,query: {}, topK: {}, category: {}", query, topK, category);
|
|
||||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(query, topK, category);
|
|
||||||
log.info("检索完成,返回 {} 条结果", results.size());
|
|
||||||
return Result.success(results);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("检索失败", e);
|
|
||||||
return Result.error(500, "检索失败: " + e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
package com.superbiz.agent.dto;
|
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
|
||||||
import lombok.Builder;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NoArgsConstructor;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档分片
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
@Builder
|
|
||||||
@NoArgsConstructor
|
|
||||||
@AllArgsConstructor
|
|
||||||
public class DocumentChunk {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片内容
|
|
||||||
*/
|
|
||||||
private String content;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片在原文档中的起始位置
|
|
||||||
*/
|
|
||||||
private int startOffset;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片在原文档中的结束位置
|
|
||||||
*/
|
|
||||||
private int endOffset;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片序号(从0开始)
|
|
||||||
*/
|
|
||||||
private int chunkIndex;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分片标题或上下文信息
|
|
||||||
*/
|
|
||||||
private String title;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 面包屑导航(完整标题层级路径)
|
|
||||||
* 例如: "故障诊断流程规范 > 应急响应流程 > 1. 初步评估"
|
|
||||||
*/
|
|
||||||
private String breadcrumb;
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package com.superbiz.agent.dto;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
|
||||||
import lombok.AllArgsConstructor;
|
|
||||||
import lombok.Builder;
|
|
||||||
import lombok.Data;
|
|
||||||
import lombok.NoArgsConstructor;
|
|
||||||
|
|
||||||
import java.time.LocalDate;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Frontmatter 数据模型
|
|
||||||
* 用于解析 Markdown 文件头的 YAML frontmatter
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
@Builder
|
|
||||||
@NoArgsConstructor
|
|
||||||
@AllArgsConstructor
|
|
||||||
public class Frontmatter {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档标题(必填)
|
|
||||||
*/
|
|
||||||
private String title;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 关键词列表(必填,用于 L0 精确匹配)
|
|
||||||
*/
|
|
||||||
private List<String> keywords;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档摘要(必填)
|
|
||||||
*/
|
|
||||||
private String summary;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档类别(可选)
|
|
||||||
*/
|
|
||||||
private String category;
|
|
||||||
|
|
||||||
private String source;
|
|
||||||
|
|
||||||
private String breadcrumb;
|
|
||||||
|
|
||||||
@JsonProperty("kb_scope")
|
|
||||||
private String kbScope;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 章节锚点(预留字段,MVP 不使用)
|
|
||||||
* Key: 章节标题,Value: 章节 Markdown 标题
|
|
||||||
*/
|
|
||||||
private Map<String, String> sections;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 版本号(预留字段)
|
|
||||||
*/
|
|
||||||
private String version;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 作者(预留字段)
|
|
||||||
*/
|
|
||||||
private String author;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 最后更新日期(预留字段)
|
|
||||||
*/
|
|
||||||
private LocalDate lastUpdated;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 业务场景标签,供 Planner 决策用(LLM 上传时自动生成)
|
|
||||||
*/
|
|
||||||
private List<String> covers;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档级检索时机(LLM 上传时自动生成)
|
|
||||||
*/
|
|
||||||
private String whenToRetrieve;
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
package com.superbiz.agent.dto;
|
|
||||||
|
|
||||||
import lombok.Builder;
|
|
||||||
import lombok.Data;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库索引条目
|
|
||||||
* L0 内存索引使用的数据结构
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
@Builder
|
|
||||||
public class KnowledgeEntry {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文件路径(如:knowledge_base/api/payment-errors.md)
|
|
||||||
*/
|
|
||||||
private String filePath;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档标题
|
|
||||||
*/
|
|
||||||
private String title;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 关键词列表(用于精确匹配)
|
|
||||||
*/
|
|
||||||
private List<String> keywords;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档摘要
|
|
||||||
*/
|
|
||||||
private String summary;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档类别(如:api、domain、troubleshooting)
|
|
||||||
*/
|
|
||||||
private String category;
|
|
||||||
|
|
||||||
private String kbScope;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 章节锚点(预留字段,MVP 不使用)
|
|
||||||
*/
|
|
||||||
private Map<String, String> sections;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 业务场景标签,供 Planner 决策用
|
|
||||||
*/
|
|
||||||
private List<String> covers;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档级检索时机
|
|
||||||
*/
|
|
||||||
private String whenToRetrieve;
|
|
||||||
}
|
|
||||||
@@ -6,9 +6,10 @@ import lombok.Data;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 检索前 query understanding 的输出(L0 -> pipeline 控制面)。
|
* 检索 pipeline 控制面参数。
|
||||||
*
|
*
|
||||||
* <p>由 {@code KnowledgeQueryTransformer} 生成,供 L1 过滤、规则 rerank 与 trace 使用。
|
* <p>L0 query 理解已下沉 py-rag 服务端;当前 {@code originalQuery} = {@code rewrittenQuery}、
|
||||||
|
* hint 字段恒为空、{@code categoryFilter} 恒为 null,结构保留供后处理与 trace 使用。
|
||||||
* 不是 Agent 可见契约。</p>
|
* 不是 Agent 可见契约。</p>
|
||||||
*/
|
*/
|
||||||
@Data
|
@Data
|
||||||
|
|||||||
@@ -1,446 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.config.DocumentChunkConfig;
|
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.regex.Matcher;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档切片服务(RAG 入库前处理)。
|
|
||||||
*
|
|
||||||
* <p>把长 Markdown/文本切成带 title/breadcrumb 的 {@link com.superbiz.agent.dto.DocumentChunk},
|
|
||||||
* 供 {@link VectorIndexService} 向量化。</p>
|
|
||||||
*
|
|
||||||
* <h3>策略摘要</h3>
|
|
||||||
* <ol>
|
|
||||||
* <li>先按 Markdown 标题分 section,并维护 breadcrumb 层级</li>
|
|
||||||
* <li>section 过长再按段落累积;用 token 估算做软边界 / 硬上限</li>
|
|
||||||
* <li>尽量不在有序/无序列表或未闭合代码块中间切断</li>
|
|
||||||
* <li>相邻 chunk 保留 overlap,减轻边界语义断裂</li>
|
|
||||||
* </ol>
|
|
||||||
*
|
|
||||||
* <p>检索命中单个 chunk 后,当前主链路不会自动回补同章节相邻 chunk
|
|
||||||
* (上下文重建仍是后续增强点)。</p>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class DocumentChunkService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(DocumentChunkService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentChunkConfig chunkConfig;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 智能分片文档
|
|
||||||
* 优先按照标题、段落边界进行分割,保持语义完整性
|
|
||||||
*
|
|
||||||
* @param content 文档内容
|
|
||||||
* @param filePath 文件路径(用于日志)
|
|
||||||
* @return 文档分片列表
|
|
||||||
*/
|
|
||||||
public List<DocumentChunk> chunkDocument(String content, String filePath) {
|
|
||||||
List<DocumentChunk> chunks = new ArrayList<>();
|
|
||||||
|
|
||||||
if (content == null || content.trim().isEmpty()) {
|
|
||||||
logger.warn("文档内容为空: {}", filePath);
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. 首先尝试按标题分割(Markdown格式)
|
|
||||||
List<Section> sections = splitByHeadings(content);
|
|
||||||
|
|
||||||
// 2. 对每个章节进行进一步分片
|
|
||||||
int globalChunkIndex = 0;
|
|
||||||
for (Section section : sections) {
|
|
||||||
List<DocumentChunk> sectionChunks = chunkSection(section, globalChunkIndex);
|
|
||||||
chunks.addAll(sectionChunks);
|
|
||||||
globalChunkIndex += sectionChunks.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 按照 Markdown 标题分割文档,同时构建面包屑层级路径
|
|
||||||
*/
|
|
||||||
private List<Section> splitByHeadings(String content) {
|
|
||||||
List<Section> sections = new ArrayList<>();
|
|
||||||
|
|
||||||
// 匹配 Markdown 标题:# 标题, ## 标题, ### 标题等
|
|
||||||
Pattern headingPattern = Pattern.compile("^(#{1,6})\\s+(.+)$", Pattern.MULTILINE);
|
|
||||||
Matcher matcher = headingPattern.matcher(content);
|
|
||||||
|
|
||||||
// 标题层级栈:维护当前标题的完整路径
|
|
||||||
List<String> headingStack = new ArrayList<>();
|
|
||||||
int lastEnd = 0;
|
|
||||||
String currentBreadcrumb = null;
|
|
||||||
|
|
||||||
while (matcher.find()) {
|
|
||||||
int level = matcher.group(1).length(); // #→1, ##→2, ###→3 ...
|
|
||||||
String title = matcher.group(2).trim();
|
|
||||||
|
|
||||||
// 保存上一个章节
|
|
||||||
if (lastEnd < matcher.start()) {
|
|
||||||
String sectionContent = content.substring(lastEnd, matcher.start()).trim();
|
|
||||||
if (!sectionContent.isEmpty()) {
|
|
||||||
sections.add(new Section(
|
|
||||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
|
||||||
level,
|
|
||||||
currentBreadcrumb,
|
|
||||||
sectionContent,
|
|
||||||
lastEnd));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 维护层级栈:同级别或更高级别 → 弹出,低级 → 追加
|
|
||||||
while (!headingStack.isEmpty() && headingStack.size() >= level) {
|
|
||||||
headingStack.remove(headingStack.size() - 1);
|
|
||||||
}
|
|
||||||
headingStack.add(title);
|
|
||||||
currentBreadcrumb = String.join(" > ", headingStack);
|
|
||||||
lastEnd = matcher.start();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 添加最后一个章节
|
|
||||||
if (lastEnd < content.length()) {
|
|
||||||
String sectionContent = content.substring(lastEnd).trim();
|
|
||||||
if (!sectionContent.isEmpty()) {
|
|
||||||
sections.add(new Section(
|
|
||||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
|
||||||
headingStack.size(),
|
|
||||||
currentBreadcrumb,
|
|
||||||
sectionContent,
|
|
||||||
lastEnd));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 如果没有找到任何标题,将整个文档作为一个章节
|
|
||||||
if (sections.isEmpty()) {
|
|
||||||
sections.add(new Section(null, 0, null, content, 0));
|
|
||||||
}
|
|
||||||
|
|
||||||
return sections;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 对单个章节进行分片
|
|
||||||
* <p>
|
|
||||||
* 核心改造(Phase 1):
|
|
||||||
* - Token 估算替代字符计数
|
|
||||||
* - 感知有序/无序列表结构,不在列表中间切断
|
|
||||||
* - 软边界(maxTokens)+ 硬上限(maxTokensHard)双重控制
|
|
||||||
* - 修复 currentStartIndex 漂移:用段落原始位置而非手工推算
|
|
||||||
*/
|
|
||||||
private List<DocumentChunk> chunkSection(Section section, int startChunkIndex) {
|
|
||||||
List<DocumentChunk> chunks = new ArrayList<>();
|
|
||||||
String content = section.content;
|
|
||||||
String title = section.title;
|
|
||||||
String breadcrumb = section.breadcrumb;
|
|
||||||
|
|
||||||
// 短章节直接作为一个分片(用 token 估算替代字符数做短路判断)
|
|
||||||
if (content.length() <= chunkConfig.getMaxSize()
|
|
||||||
&& estimateTokens(content) <= chunkConfig.getMaxTokens()) {
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.content(content)
|
|
||||||
.startOffset(section.startIndex)
|
|
||||||
.endOffset(section.startIndex + content.length())
|
|
||||||
.chunkIndex(startChunkIndex)
|
|
||||||
.title(title)
|
|
||||||
.breadcrumb(breadcrumb)
|
|
||||||
.build();
|
|
||||||
chunks.add(chunk);
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 章节内容较长,需要进一步分片
|
|
||||||
List<String> paragraphs = splitByParagraphs(content);
|
|
||||||
if (paragraphs.isEmpty()) {
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 定位每个段落在 section.content 中的位置(修复 index 漂移)
|
|
||||||
List<ParagraphPos> paraPositions = locateParagraphPositions(paragraphs, content);
|
|
||||||
|
|
||||||
// 当前分片的段落范围
|
|
||||||
int chunkParaStart = 0; // 当前分片第一个段落的索引(在 paragraphs 中)
|
|
||||||
StringBuilder buffer = new StringBuilder();
|
|
||||||
int tokenCount = 0;
|
|
||||||
int chunkIndex = startChunkIndex;
|
|
||||||
|
|
||||||
for (int i = 0; i < paragraphs.size(); i++) {
|
|
||||||
String paragraph = paragraphs.get(i);
|
|
||||||
int paraTokens = estimateTokens(paragraph);
|
|
||||||
|
|
||||||
// 判断是否需要切分
|
|
||||||
if (buffer.length() > 0 && tokenCount + paraTokens > chunkConfig.getMaxTokens()) {
|
|
||||||
|
|
||||||
// 检查是否处于不可中断的上下文中
|
|
||||||
if (isInUnbreakableContext(buffer.toString(), paragraph)) {
|
|
||||||
// 硬上限保护:即使不可中断也不能无限膨胀
|
|
||||||
if (tokenCount + paraTokens > chunkConfig.getMaxTokensHard()) {
|
|
||||||
logger.debug(" 触及硬上限 ({} tokens),强制切分", tokenCount + paraTokens);
|
|
||||||
chunkParaStart = saveChunkAndGetNextStart(
|
|
||||||
chunks, section, paraPositions,
|
|
||||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
|
||||||
chunkIndex++;
|
|
||||||
|
|
||||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
|
||||||
String overlap = getOverlapText(prevChunkContent);
|
|
||||||
buffer = new StringBuilder(overlap);
|
|
||||||
tokenCount = estimateTokens(overlap);
|
|
||||||
}
|
|
||||||
// 否则:容忍超出(软边界)
|
|
||||||
} else {
|
|
||||||
// 安全切点:段落边界
|
|
||||||
chunkParaStart = saveChunkAndGetNextStart(
|
|
||||||
chunks, section, paraPositions,
|
|
||||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
|
||||||
chunkIndex++;
|
|
||||||
|
|
||||||
// 新分片以重叠文本开头
|
|
||||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
|
||||||
String overlap = getOverlapText(prevChunkContent);
|
|
||||||
buffer = new StringBuilder(overlap);
|
|
||||||
tokenCount = estimateTokens(overlap);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
buffer.append(paragraph).append("\n\n");
|
|
||||||
tokenCount += paraTokens;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 保存最后一个分片
|
|
||||||
if (buffer.length() > 0 && chunkParaStart < paragraphs.size()) {
|
|
||||||
String chunkContent = buffer.toString().trim();
|
|
||||||
int actualStart = paraPositions.get(chunkParaStart).start;
|
|
||||||
int actualEnd = paraPositions.get(paragraphs.size() - 1).end;
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.content(chunkContent)
|
|
||||||
.startOffset(section.startIndex + actualStart)
|
|
||||||
.endOffset(section.startIndex + actualEnd)
|
|
||||||
.chunkIndex(chunkIndex)
|
|
||||||
.title(title)
|
|
||||||
.breadcrumb(breadcrumb)
|
|
||||||
.build();
|
|
||||||
chunks.add(chunk);
|
|
||||||
}
|
|
||||||
|
|
||||||
return chunks;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 保存当前分块,返回下一个分块的起始段落索引
|
|
||||||
* <p>
|
|
||||||
* 从 section.content 中提取原始文本(而非手工拼装),修复 index 漂移问题
|
|
||||||
*/
|
|
||||||
private int saveChunkAndGetNextStart(
|
|
||||||
List<DocumentChunk> chunks,
|
|
||||||
Section section,
|
|
||||||
List<ParagraphPos> paraPositions,
|
|
||||||
int fromPara,
|
|
||||||
int toPara,
|
|
||||||
String title,
|
|
||||||
String breadcrumb,
|
|
||||||
int chunkIndex) {
|
|
||||||
|
|
||||||
int actualStart = paraPositions.get(fromPara).start;
|
|
||||||
int actualEnd = paraPositions.get(toPara - 1).end;
|
|
||||||
String originalText = section.content.substring(actualStart, actualEnd);
|
|
||||||
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.content(originalText)
|
|
||||||
.startOffset(section.startIndex + actualStart)
|
|
||||||
.endOffset(section.startIndex + actualEnd)
|
|
||||||
.chunkIndex(chunkIndex)
|
|
||||||
.title(title)
|
|
||||||
.breadcrumb(breadcrumb)
|
|
||||||
.build();
|
|
||||||
chunks.add(chunk);
|
|
||||||
|
|
||||||
return toPara; // 下一个分块的起始段落索引
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 按段落分割文本
|
|
||||||
*/
|
|
||||||
private List<String> splitByParagraphs(String content) {
|
|
||||||
List<String> paragraphs = new ArrayList<>();
|
|
||||||
|
|
||||||
// 按双换行符分割段落
|
|
||||||
String[] parts = content.split("\n\n+");
|
|
||||||
for (String part : parts) {
|
|
||||||
String trimmed = part.trim();
|
|
||||||
if (!trimmed.isEmpty()) {
|
|
||||||
paragraphs.add(trimmed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return paragraphs;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 定位每个段落在原始文本中的字符偏移
|
|
||||||
*/
|
|
||||||
private List<ParagraphPos> locateParagraphPositions(List<String> paragraphs, String sectionContent) {
|
|
||||||
List<ParagraphPos> positions = new ArrayList<>();
|
|
||||||
int searchFrom = 0;
|
|
||||||
for (String p : paragraphs) {
|
|
||||||
int idx = sectionContent.indexOf(p, searchFrom);
|
|
||||||
if (idx >= 0) {
|
|
||||||
positions.add(new ParagraphPos(idx, idx + p.length()));
|
|
||||||
searchFrom = idx + p.length();
|
|
||||||
} else {
|
|
||||||
// fallback: 段落在原文中找不到(不应该发生)
|
|
||||||
positions.add(new ParagraphPos(searchFrom, searchFrom + p.length()));
|
|
||||||
searchFrom += p.length();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return positions;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 启发式 token 估算(无需外部依赖)
|
|
||||||
* <p>
|
|
||||||
* 中文(BMP): ~1 字符/token
|
|
||||||
* 英文/数字/标点: ~4 字符/token
|
|
||||||
* 空白字符忽略
|
|
||||||
*/
|
|
||||||
private int estimateTokens(String text) {
|
|
||||||
int nonCjkCount = 0;
|
|
||||||
int cjkCount = 0;
|
|
||||||
for (char c : text.toCharArray()) {
|
|
||||||
if (Character.isWhitespace(c)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
|
|
||||||
if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
|
|
||||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
|
|
||||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
|
|
||||||
|| block == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS) {
|
|
||||||
cjkCount++;
|
|
||||||
} else {
|
|
||||||
nonCjkCount++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return cjkCount + (nonCjkCount + 3) / 4; // 非中文每 4 字符算 1 token,向上取整
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 判断当前段落是否属于不可中断的结构
|
|
||||||
* <p>
|
|
||||||
* 不可中断结构包括:
|
|
||||||
* - 有序列表项("1. ", "2. " 格式)
|
|
||||||
* - 无序列表项("- " 或 "* " 格式)
|
|
||||||
* - 未闭合的代码块(``` 内)
|
|
||||||
*/
|
|
||||||
private boolean isInUnbreakableContext(String buffer, String nextParagraph) {
|
|
||||||
// 有序列表:判断 buffer 末尾和下一段是否都是列表项
|
|
||||||
if (nextParagraph.matches("^\\d{1,2}\\.\\s.*")) {
|
|
||||||
String lastLine = getLastNonEmptyLine(buffer);
|
|
||||||
if (lastLine != null && lastLine.matches("^\\d{1,2}\\.\\s.*")) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// 无序列表:"- " 或 "* " 格式
|
|
||||||
if (nextParagraph.matches("^[-*]\\s.*")) {
|
|
||||||
String lastLine = getLastNonEmptyLine(buffer);
|
|
||||||
if (lastLine != null && lastLine.matches("^[-*]\\s.*")) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// 代码块:``` 未闭合
|
|
||||||
if (buffer.contains("```")) {
|
|
||||||
int count = 0;
|
|
||||||
for (int i = 0; i <= buffer.length() - 3; i++) {
|
|
||||||
if (buffer.substring(i).startsWith("```")) {
|
|
||||||
count++;
|
|
||||||
i += 2;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (count % 2 == 1) {
|
|
||||||
return true; // 奇数个 ``` → 在代码块内部
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取 buffer 中最后一行非空白文本
|
|
||||||
*/
|
|
||||||
private String getLastNonEmptyLine(String buffer) {
|
|
||||||
String[] lines = buffer.split("\n");
|
|
||||||
for (int i = lines.length - 1; i >= 0; i--) {
|
|
||||||
String line = lines[i].trim();
|
|
||||||
if (!line.isEmpty()) {
|
|
||||||
return line;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取重叠文本
|
|
||||||
* 从文本末尾提取指定长度的内容作为下一个分片的开头
|
|
||||||
*/
|
|
||||||
private String getOverlapText(String text) {
|
|
||||||
int overlapSize = Math.min(chunkConfig.getOverlap(), text.length());
|
|
||||||
if (overlapSize <= 0) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
|
|
||||||
// 从末尾提取重叠内容
|
|
||||||
String overlap = text.substring(text.length() - overlapSize);
|
|
||||||
|
|
||||||
// 尝试在句子边界截断(查找最后一个句号、问号、感叹号)
|
|
||||||
int lastSentenceEnd = Math.max(
|
|
||||||
overlap.lastIndexOf('。'),
|
|
||||||
Math.max(overlap.lastIndexOf('?'), overlap.lastIndexOf('!'))
|
|
||||||
);
|
|
||||||
|
|
||||||
if (lastSentenceEnd > overlapSize / 2) {
|
|
||||||
return overlap.substring(lastSentenceEnd + 1).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
return overlap.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 段落在原文中的位置
|
|
||||||
*/
|
|
||||||
private static class ParagraphPos {
|
|
||||||
final int start;
|
|
||||||
final int end;
|
|
||||||
|
|
||||||
ParagraphPos(int start, int end) {
|
|
||||||
this.start = start;
|
|
||||||
this.end = end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 章节数据类
|
|
||||||
*/
|
|
||||||
private static class Section {
|
|
||||||
String title; // 最近一级标题名称
|
|
||||||
int level; // 标题级别(1-6),0=无标题
|
|
||||||
String breadcrumb; // 完整面包屑路径
|
|
||||||
String content; // 章节内容
|
|
||||||
int startIndex; // 在原文中的起始偏移
|
|
||||||
|
|
||||||
Section(String title, int level, String breadcrumb, String content, int startIndex) {
|
|
||||||
this.title = title;
|
|
||||||
this.level = level;
|
|
||||||
this.breadcrumb = breadcrumb;
|
|
||||||
this.content = content;
|
|
||||||
this.startIndex = startIndex;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,135 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
|
||||||
import org.springframework.ai.chat.prompt.Prompt;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.core.io.ClassPathResource;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import jakarta.annotation.PostConstruct;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档字段补全服务
|
|
||||||
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class DocumentFieldEnricher {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ChatModel chatModel;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeIndexService knowledgeIndexService;
|
|
||||||
|
|
||||||
private String promptTemplate;
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init() {
|
|
||||||
try {
|
|
||||||
promptTemplate = new String(
|
|
||||||
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
|
|
||||||
StandardCharsets.UTF_8);
|
|
||||||
log.info("DocumentFieldEnricher prompt 加载成功");
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("加载 doc-field-enricher-prompt.md 失败", e);
|
|
||||||
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void enrich(Frontmatter frontmatter, String bodyText) {
|
|
||||||
enrich(frontmatter, bodyText, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 为 Frontmatter 补全 covers 和 whenToRetrieve
|
|
||||||
* 若已有值则跳过;LLM 失败时降级,不阻断主流程
|
|
||||||
*
|
|
||||||
* @param frontmatter 待补全的 frontmatter
|
|
||||||
* @param bodyText 文档正文
|
|
||||||
* @param category 文档所属域(用于查找同域其他文档)
|
|
||||||
*/
|
|
||||||
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
|
|
||||||
if (frontmatter == null) return;
|
|
||||||
|
|
||||||
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
|
|
||||||
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
|
|
||||||
|
|
||||||
if (!needsCovers && !needsWhen) {
|
|
||||||
log.debug("covers 和 whenToRetrieve 已存在,跳过 LLM 生成");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
String snippet = bodyText != null && bodyText.length() > 1000
|
|
||||||
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
|
|
||||||
|
|
||||||
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
|
|
||||||
|
|
||||||
String promptText = String.format(promptTemplate,
|
|
||||||
frontmatter.getTitle(),
|
|
||||||
frontmatter.getSummary(),
|
|
||||||
sameDomainDocs,
|
|
||||||
snippet);
|
|
||||||
|
|
||||||
String response = chatModel.call(new Prompt(promptText))
|
|
||||||
.getResult().getOutput().getText();
|
|
||||||
|
|
||||||
// 提取 JSON 部分(防止模型输出多余文本)
|
|
||||||
String json = extractJson(response);
|
|
||||||
JsonNode node = objectMapper.readTree(json);
|
|
||||||
|
|
||||||
if (needsCovers && node.has("covers")) {
|
|
||||||
List<String> covers = new ArrayList<>();
|
|
||||||
node.get("covers").forEach(n -> covers.add(n.asText()));
|
|
||||||
frontmatter.setCovers(covers);
|
|
||||||
log.debug("LLM 生成 covers: {}", covers);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (needsWhen && node.has("whenToRetrieve")) {
|
|
||||||
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
|
|
||||||
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("LLM 生成文档字段失败,降级处理: title={}", frontmatter.getTitle(), e);
|
|
||||||
if (needsCovers) frontmatter.setCovers(List.of());
|
|
||||||
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractJson(String text) {
|
|
||||||
if (text == null) return "{}";
|
|
||||||
int start = text.indexOf('{');
|
|
||||||
int end = text.lastIndexOf('}');
|
|
||||||
if (start == -1 || end == -1 || end <= start) return "{}";
|
|
||||||
return text.substring(start, end + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建同域其他文档标题列表(供 LLM 做排除判断)
|
|
||||||
*/
|
|
||||||
private String buildSameDomainDocs(String currentTitle, String category) {
|
|
||||||
if (category == null || category.isBlank()) return "(无同域文档信息)";
|
|
||||||
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
|
|
||||||
.filter(e -> category.equals(e.getCategory()))
|
|
||||||
.map(KnowledgeEntry::getTitle)
|
|
||||||
.filter(t -> t != null && !t.equals(currentTitle))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
if (otherTitles.isEmpty()) return "(无同域其他文档)";
|
|
||||||
return String.join("、", otherTitles);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
package com.superbiz.agent.service;
|
package com.superbiz.agent.service;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.superbiz.agent.client.PyRagClient;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagIngestResponse;
|
||||||
|
import com.superbiz.agent.client.PyRagClientException;
|
||||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
import com.superbiz.agent.domain.enums.FaultCategory;
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import com.superbiz.agent.dto.DocumentQueryResponse;
|
import com.superbiz.agent.dto.DocumentQueryResponse;
|
||||||
import com.superbiz.agent.dto.DocumentUploadRequest;
|
import com.superbiz.agent.dto.DocumentUploadRequest;
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import com.superbiz.agent.exception.DocumentProcessException;
|
import com.superbiz.agent.exception.DocumentProcessException;
|
||||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -27,11 +27,13 @@ import java.security.MessageDigest;
|
|||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.UUID;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 文档管理服务
|
* 文档管理服务。
|
||||||
|
*
|
||||||
|
* <p>负责 MySQL 元数据({@link ApiDocument})、本地原件保存与业务查询/删除;
|
||||||
|
* 文档解析、frontmatter 校验、分块与向量索引全部由 py-rag 服务端 ingest 完成。</p>
|
||||||
*/
|
*/
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@@ -41,34 +43,19 @@ public class DocumentManagementService {
|
|||||||
private String knowledgeBasePath;
|
private String knowledgeBasePath;
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private TextExtractorService textExtractorService;
|
private PyRagClient pyRagClient;
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentChunkService documentChunkService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorIndexService vectorIndexService;
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private ApiDocumentRepository apiDocumentRepository;
|
private ApiDocumentRepository apiDocumentRepository;
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private FrontmatterParser frontmatterParser;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeIndexService knowledgeIndexService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentFieldEnricher documentFieldEnricher;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeDomainService knowledgeDomainService;
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private ObjectMapper objectMapper;
|
private ObjectMapper objectMapper;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 上传文档
|
* 上传文档。
|
||||||
|
*
|
||||||
|
* <p>流程:hash 去重 → 本地原件保存 → py-rag ingest(同步,服务端解析/分块/索引)→
|
||||||
|
* MySQL 元数据落库。docId 取 py-rag 返回的 doc_id,与检索 evidence_key 的 docId 段对齐。</p>
|
||||||
*
|
*
|
||||||
* @param request 上传请求
|
* @param request 上传请求
|
||||||
* @return 文档ID
|
* @return 文档ID
|
||||||
@@ -83,15 +70,7 @@ public class DocumentManagementService {
|
|||||||
log.info("开始上传文档,文件名: {}, 大小: {} bytes", fileName, file.getSize());
|
log.info("开始上传文档,文件名: {}, 大小: {} bytes", fileName, file.getSize());
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// 1. 验证文件格式
|
// 1. 计算文件 hash(去重)
|
||||||
if (!textExtractorService.isSupportedFormat(fileName)) {
|
|
||||||
throw new DocumentProcessException(
|
|
||||||
fileName, "upload",
|
|
||||||
"不支持的文件格式,仅支持 .md 和 .txt"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 计算文件 hash(去重)
|
|
||||||
long hashStart = System.currentTimeMillis();
|
long hashStart = System.currentTimeMillis();
|
||||||
String fileHash = calculateFileHash(file);
|
String fileHash = calculateFileHash(file);
|
||||||
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
|
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
|
||||||
@@ -105,16 +84,7 @@ public class DocumentManagementService {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. 提取文本
|
// 2. category 缺省处理 + 保存原始文件到本地
|
||||||
long extractStart = System.currentTimeMillis();
|
|
||||||
String text = textExtractorService.extractText(file, fileName);
|
|
||||||
log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart);
|
|
||||||
|
|
||||||
if (text == null || text.isBlank()) {
|
|
||||||
throw new DocumentProcessException(fileName, "upload", "文档内容为空");
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. 保存原始文件到本地
|
|
||||||
String category = request.getCategory();
|
String category = request.getCategory();
|
||||||
if (category == null || category.isBlank()) {
|
if (category == null || category.isBlank()) {
|
||||||
category = "default";
|
category = "default";
|
||||||
@@ -123,47 +93,33 @@ public class DocumentManagementService {
|
|||||||
localPath = saveToLocal(file, fileName, category);
|
localPath = saveToLocal(file, fileName, category);
|
||||||
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
|
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
|
||||||
|
|
||||||
// 5. 解析 frontmatter
|
// 3. py-rag 入库(格式校验/frontmatter/分块/向量索引都在服务端;同内容重传返回 unchanged)
|
||||||
long frontmatterStart = System.currentTimeMillis();
|
long ingestStart = System.currentTimeMillis();
|
||||||
Frontmatter frontmatter = null;
|
PyRagIngestResponse ingest;
|
||||||
String bodyText = text;
|
try {
|
||||||
if (frontmatterParser.hasFrontmatter(text)) {
|
ingest = pyRagClient.ingest(fileName, file.getBytes(), file.getContentType(),
|
||||||
frontmatter = frontmatterParser.parse(text);
|
category, null, null, null);
|
||||||
if (frontmatter != null) {
|
} catch (PyRagClientException | IOException e) {
|
||||||
// LLM 补全 covers / whenToRetrieve(已有值则跳过)
|
throw new DocumentProcessException(
|
||||||
bodyText = frontmatterParser.stripFrontmatter(text);
|
fileName, "ingest", "py-rag 入库失败: " + e.getMessage(), e
|
||||||
documentFieldEnricher.enrich(frontmatter, bodyText, category);
|
);
|
||||||
log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
|
|
||||||
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
|
|
||||||
} else {
|
|
||||||
log.warn("frontmatter解析失败,文件名: {}", fileName);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
log.debug("文件不包含frontmatter: {}", fileName);
|
|
||||||
}
|
}
|
||||||
|
log.info("py-rag 入库完成: docId={}, status={}, chunks={}, time={}ms",
|
||||||
|
ingest.docId(), ingest.status(), ingest.chunkCount(),
|
||||||
|
System.currentTimeMillis() - ingestStart);
|
||||||
|
|
||||||
// 6. 分块
|
// 4. 保存文档元数据
|
||||||
long chunkStart = System.currentTimeMillis();
|
|
||||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(bodyText, fileName);
|
|
||||||
if (chunks.isEmpty()) {
|
|
||||||
throw new DocumentProcessException(fileName, "upload", "文档分块失败");
|
|
||||||
}
|
|
||||||
log.info("文档分块完成: fileName={}, chunks={}, time={}ms",
|
|
||||||
fileName, chunks.size(), System.currentTimeMillis() - chunkStart);
|
|
||||||
|
|
||||||
// 7. 创建文档元数据
|
|
||||||
String docId = resolveDocumentId(frontmatter);
|
|
||||||
String metadataJson = null;
|
String metadataJson = null;
|
||||||
if (frontmatter != null) {
|
if (ingest.frontmatter() != null) {
|
||||||
try {
|
try {
|
||||||
metadataJson = objectMapper.writeValueAsString(frontmatter);
|
metadataJson = objectMapper.writeValueAsString(ingest.frontmatter());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.warn("Frontmatter序列化失败", e);
|
log.warn("frontmatter 序列化失败", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ApiDocument document = ApiDocument.builder()
|
ApiDocument document = ApiDocument.builder()
|
||||||
.docId(docId)
|
.docId(ingest.docId())
|
||||||
.fileName(fileName)
|
.fileName(fileName)
|
||||||
.filePath(localPath)
|
.filePath(localPath)
|
||||||
.metadata(metadataJson)
|
.metadata(metadataJson)
|
||||||
@@ -173,56 +129,17 @@ public class DocumentManagementService {
|
|||||||
.version(request.getVersion())
|
.version(request.getVersion())
|
||||||
.fileSize(file.getSize())
|
.fileSize(file.getSize())
|
||||||
.fileHash(fileHash)
|
.fileHash(fileHash)
|
||||||
.status("PROCESSING")
|
.status("INDEXED")
|
||||||
.chunkCount(chunks.size())
|
.chunkCount(ingest.chunkCount())
|
||||||
.build();
|
.build();
|
||||||
|
document.setIndexedAt(LocalDateTime.now());
|
||||||
apiDocumentRepository.save(document);
|
apiDocumentRepository.save(document);
|
||||||
log.info("文档元数据已保存: docId={}", docId);
|
log.info("文档元数据已保存: docId={}", document.getDocId());
|
||||||
|
|
||||||
// 8. 向量化并索引
|
log.info("文档上传完成: docId={}, fileName={}, ingestStatus={}, totalTime={}ms",
|
||||||
try {
|
document.getDocId(), fileName, ingest.status(), System.currentTimeMillis() - startTime);
|
||||||
long vectorStart = System.currentTimeMillis();
|
|
||||||
vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter);
|
|
||||||
document.setStatus("INDEXED");
|
|
||||||
document.setIndexedAt(LocalDateTime.now());
|
|
||||||
apiDocumentRepository.save(document);
|
|
||||||
log.info("文档向量索引完成: docId={}, category={}, time={}ms",
|
|
||||||
docId, category, System.currentTimeMillis() - vectorStart);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
return document.getDocId();
|
||||||
log.error("文档索引失败: docId={}", docId, e);
|
|
||||||
document.setStatus("FAILED");
|
|
||||||
apiDocumentRepository.save(document);
|
|
||||||
throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 9. 更新 L0 索引
|
|
||||||
if (frontmatter != null) {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath(localPath)
|
|
||||||
.title(frontmatter.getTitle())
|
|
||||||
.keywords(frontmatter.getKeywords())
|
|
||||||
.summary(frontmatter.getSummary())
|
|
||||||
.category(category)
|
|
||||||
.kbScope(frontmatter.getKbScope())
|
|
||||||
.sections(frontmatter.getSections())
|
|
||||||
.covers(frontmatter.getCovers())
|
|
||||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
knowledgeIndexService.addToIndex(entry);
|
|
||||||
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 触发域级聚合重算
|
|
||||||
knowledgeDomainService.onDocumentChange(category);
|
|
||||||
|
|
||||||
long totalTime = System.currentTimeMillis() - startTime;
|
|
||||||
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
|
|
||||||
docId, fileName, frontmatter != null, totalTime);
|
|
||||||
|
|
||||||
return docId;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
// 失败时清理本地文件
|
// 失败时清理本地文件
|
||||||
@@ -314,16 +231,6 @@ public class DocumentManagementService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String resolveDocumentId(Frontmatter frontmatter) {
|
|
||||||
if (frontmatter != null && frontmatter.getSource() != null) {
|
|
||||||
String source = frontmatter.getSource().trim();
|
|
||||||
if (!source.isEmpty() && source.length() <= 64) {
|
|
||||||
return source;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return UUID.randomUUID().toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 根据 docId 查询文档
|
* 根据 docId 查询文档
|
||||||
*/
|
*/
|
||||||
@@ -358,7 +265,10 @@ public class DocumentManagementService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除文档
|
* 删除文档(仅 MySQL 元数据与本地原件)。
|
||||||
|
*
|
||||||
|
* <p>py-rag v1 契约没有单文档删除端点:已入库内容需在其服务端
|
||||||
|
* 全量重建({@code /api/v1/collections:rebuild})后才会从知识库消失。</p>
|
||||||
*/
|
*/
|
||||||
@Transactional
|
@Transactional
|
||||||
public void deleteDocument(String docId) {
|
public void deleteDocument(String docId) {
|
||||||
@@ -379,47 +289,9 @@ public class DocumentManagementService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 删除 L0 索引
|
// 删除元数据(py-rag 侧索引留存,重建后失效)
|
||||||
if (doc.getFilePath() != null) {
|
|
||||||
knowledgeIndexService.removeFromIndex(doc.getFilePath());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 删除向量索引
|
|
||||||
try {
|
|
||||||
vectorIndexService.deleteDocumentChunks(docId);
|
|
||||||
log.info("文档向量索引已删除,docId: {}", docId);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("删除向量索引失败,docId: {}", docId, e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 删除元数据
|
|
||||||
apiDocumentRepository.delete(doc);
|
apiDocumentRepository.delete(doc);
|
||||||
log.info("文档已删除,docId: {}", docId);
|
log.info("文档已删除,docId={}(py-rag 侧需全量重建后生效)", docId);
|
||||||
|
|
||||||
// 触发域级聚合重算
|
|
||||||
String category = doc.getFilePath() != null
|
|
||||||
? resolveCategory(doc.getFilePath()) : null;
|
|
||||||
if (category != null) {
|
|
||||||
knowledgeDomainService.onDocumentChange(category);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 转换为响应 DTO
|
|
||||||
*/
|
|
||||||
/**
|
|
||||||
* 从 filePath 解析 category(取 knowledge_base/{category}/... 中的 category 段)
|
|
||||||
*/
|
|
||||||
private String resolveCategory(String filePath) {
|
|
||||||
try {
|
|
||||||
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
|
|
||||||
// filePath 形如 knowledge_base/payment/xxx.md,取倒数第二段
|
|
||||||
int nameCount = p.getNameCount();
|
|
||||||
if (nameCount >= 2) {
|
|
||||||
return p.getName(nameCount - 2).toString();
|
|
||||||
}
|
|
||||||
} catch (Exception ignored) {}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Path resolveLocalPath(String filePath) {
|
private Path resolveLocalPath(String filePath) {
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.yaml.snakeyaml.Yaml;
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Frontmatter 解析器
|
|
||||||
* 解析 Markdown 文件头的 YAML frontmatter
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class FrontmatterParser {
|
|
||||||
|
|
||||||
private final Yaml yaml = new Yaml();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 检查文件是否包含 frontmatter
|
|
||||||
*
|
|
||||||
* @param content 文件内容
|
|
||||||
* @return true 如果包含 frontmatter
|
|
||||||
*/
|
|
||||||
public boolean hasFrontmatter(String content) {
|
|
||||||
if (content == null || content.isEmpty()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return content.trim().startsWith("---");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 解析 Markdown frontmatter
|
|
||||||
*
|
|
||||||
* @param content 完整文件内容
|
|
||||||
* @return Frontmatter 对象,如果不存在或解析失败返回 null
|
|
||||||
*/
|
|
||||||
public Frontmatter parse(String content) {
|
|
||||||
if (!hasFrontmatter(content)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 1. 提取 frontmatter 部分(两个 --- 之间)
|
|
||||||
String frontmatterText = extractFrontmatter(content);
|
|
||||||
if (frontmatterText == null) {
|
|
||||||
log.warn("未找到有效的 frontmatter 结束标记");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 使用 SnakeYAML 解析
|
|
||||||
Map<String, Object> map = yaml.load(frontmatterText);
|
|
||||||
if (map == null || map.isEmpty()) {
|
|
||||||
log.warn("Frontmatter 解析结果为空");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 映射到 Frontmatter 对象
|
|
||||||
Frontmatter frontmatter = Frontmatter.builder()
|
|
||||||
.title((String) map.get("title"))
|
|
||||||
.keywords((java.util.List<String>) map.get("keywords"))
|
|
||||||
.summary((String) map.get("summary"))
|
|
||||||
.category((String) map.get("category"))
|
|
||||||
.source((String) map.get("source"))
|
|
||||||
.breadcrumb((String) map.get("breadcrumb"))
|
|
||||||
.kbScope(firstString(map, "kb_scope", "kbScope"))
|
|
||||||
.sections((Map<String, String>) map.get("sections"))
|
|
||||||
.version((String) map.get("version"))
|
|
||||||
.author((String) map.get("author"))
|
|
||||||
.covers((java.util.List<String>) map.get("covers"))
|
|
||||||
.whenToRetrieve((String) map.get("when_to_retrieve"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
// 4. 验证必填字段
|
|
||||||
if (frontmatter.getTitle() == null || frontmatter.getKeywords() == null ||
|
|
||||||
frontmatter.getSummary() == null) {
|
|
||||||
log.warn("Frontmatter 缺少必填字段: title={}, keywords={}, summary={}",
|
|
||||||
frontmatter.getTitle(), frontmatter.getKeywords(), frontmatter.getSummary());
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug("Frontmatter 解析成功: title={}, keywords=",
|
|
||||||
frontmatter.getTitle(), frontmatter.getKeywords());
|
|
||||||
return frontmatter;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Frontmatter 解析失败", e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public String stripFrontmatter(String content) {
|
|
||||||
if (!hasFrontmatter(content)) {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
String trimmed = content.trim();
|
|
||||||
int secondDelimiter = trimmed.indexOf("\n---", 3);
|
|
||||||
int delimiterLength = 4;
|
|
||||||
if (secondDelimiter == -1) {
|
|
||||||
secondDelimiter = trimmed.indexOf("\r\n---", 3);
|
|
||||||
delimiterLength = 5;
|
|
||||||
}
|
|
||||||
if (secondDelimiter == -1) {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
int bodyStart = secondDelimiter + delimiterLength;
|
|
||||||
if (bodyStart < trimmed.length()) {
|
|
||||||
char next = trimmed.charAt(bodyStart);
|
|
||||||
if (next == '\r') {
|
|
||||||
bodyStart++;
|
|
||||||
}
|
|
||||||
if (bodyStart < trimmed.length() && trimmed.charAt(bodyStart) == '\n') {
|
|
||||||
bodyStart++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return trimmed.substring(Math.min(bodyStart, trimmed.length())).stripLeading();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 提取 frontmatter 文本(两个 --- 之间的内容)
|
|
||||||
*
|
|
||||||
* @param content 完整文件内容
|
|
||||||
* @return frontmatter 文本,如果格式错误返回 null
|
|
||||||
*/
|
|
||||||
private String extractFrontmatter(String content) {
|
|
||||||
// 去除开头的空白
|
|
||||||
content = content.trim();
|
|
||||||
|
|
||||||
// 检查是否以 --- 开头
|
|
||||||
if (!content.startsWith("---")) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 查找第二个 ---(结束标记)
|
|
||||||
int secondDelimiter = content.indexOf("\n---", 3);
|
|
||||||
if (secondDelimiter == -1) {
|
|
||||||
// 尝试查找 Windows 风格换行
|
|
||||||
secondDelimiter = content.indexOf("\r\n---", 3);
|
|
||||||
if (secondDelimiter == -1) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 提取 frontmatter(不包含 --- 标记)
|
|
||||||
return content.substring(3, secondDelimiter).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String firstString(Map<String, Object> map, String... keys) {
|
|
||||||
for (String key : keys) {
|
|
||||||
Object value = map.get(key);
|
|
||||||
if (value instanceof String text && !text.isBlank()) {
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,407 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
|
||||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
|
||||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import lombok.Data;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.*;
|
|
||||||
import java.nio.file.attribute.BasicFileAttributes;
|
|
||||||
import java.time.LocalDateTime;
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库初始化服务
|
|
||||||
* 负责批量导入 knowledge_base 目录下的文档到数据库和 Milvus
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class KnowledgeBaseInitService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseInitService.class);
|
|
||||||
|
|
||||||
@Value("${knowledge.base-path:knowledge_base}")
|
|
||||||
private String knowledgeBasePath;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ApiDocumentRepository apiDocumentRepository;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private FrontmatterParser frontmatterParser;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentChunkService documentChunkService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorIndexService vectorIndexService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorEmbeddingService vectorEmbeddingService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeIndexService knowledgeIndexService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
|
||||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
|
||||||
*/
|
|
||||||
@Transactional(rollbackFor = Exception.class)
|
|
||||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
|
||||||
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
|
|
||||||
RebuildResult rebuild = new RebuildResult();
|
|
||||||
rebuild.setBasePath(knowledgeBasePath);
|
|
||||||
rebuild.setCollection(hybridKnowledgeStore.collectionName());
|
|
||||||
|
|
||||||
long mysqlBefore = apiDocumentRepository.count();
|
|
||||||
rebuild.setMysqlDocumentsBefore(mysqlBefore);
|
|
||||||
|
|
||||||
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
|
|
||||||
rebuild.setMilvus(milvus);
|
|
||||||
|
|
||||||
apiDocumentRepository.deleteAll();
|
|
||||||
apiDocumentRepository.flush();
|
|
||||||
knowledgeIndexService.clearIndex();
|
|
||||||
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
|
|
||||||
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
|
|
||||||
|
|
||||||
InitResult init = initializeKnowledgeBase(true);
|
|
||||||
rebuild.setInit(init);
|
|
||||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
|
||||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
|
||||||
// Success when at least one doc indexed and no hard failures.
|
|
||||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
|
||||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
|
||||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
|
||||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
|
||||||
return rebuild;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 初始化知识库
|
|
||||||
*
|
|
||||||
* @param force 是否强制重新导入(跳过去重检查)
|
|
||||||
* @return 初始化结果
|
|
||||||
*/
|
|
||||||
@Transactional(rollbackFor = Exception.class)
|
|
||||||
public InitResult initializeKnowledgeBase(boolean force) {
|
|
||||||
logger.info("开始初始化知识库: basePath={}, force={}", knowledgeBasePath, force);
|
|
||||||
|
|
||||||
InitResult result = new InitResult();
|
|
||||||
Path baseDir = Paths.get(knowledgeBasePath);
|
|
||||||
|
|
||||||
if (!Files.exists(baseDir)) {
|
|
||||||
logger.error("知识库目录不存在: {}", knowledgeBasePath);
|
|
||||||
throw new RuntimeException("知识库目录不存在: " + knowledgeBasePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. 扫描所有 Markdown 文件
|
|
||||||
List<Path> markdownFiles = scanMarkdownFiles(baseDir);
|
|
||||||
result.setScanned(markdownFiles.size());
|
|
||||||
logger.info("扫描到 {} 个 Markdown 文件", markdownFiles.size());
|
|
||||||
|
|
||||||
// 2. 如果非强制模式,获取已存在的文档(用于去重)
|
|
||||||
Set<String> existingFilePaths = new HashSet<>();
|
|
||||||
if (!force) {
|
|
||||||
existingFilePaths = apiDocumentRepository.findAll().stream()
|
|
||||||
.map(ApiDocument::getFilePath)
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
logger.info("已存在 个文档记录", existingFilePaths.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. 逐个处理文档
|
|
||||||
for (Path file : markdownFiles) {
|
|
||||||
String relativePath = baseDir.relativize(file).toString().replace("\\", "/");
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 去重检查
|
|
||||||
if (!force && existingFilePaths.contains(relativePath)) {
|
|
||||||
logger.debug("跳过已存在的文档: {}", relativePath);
|
|
||||||
result.incrementSkipped();
|
|
||||||
result.addDetail(relativePath, "已存在,跳过");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 解析文档
|
|
||||||
String content = Files.readString(file);
|
|
||||||
Frontmatter frontmatter = frontmatterParser.parse(content);
|
|
||||||
|
|
||||||
if (frontmatter == null) {
|
|
||||||
logger.warn("文档格式无效: {}, frontmatter 解析失败", relativePath);
|
|
||||||
result.incrementFailed();
|
|
||||||
result.addDetail(relativePath, "格式无效: frontmatter 解析失败");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 提取字段
|
|
||||||
String title = frontmatter.getTitle();
|
|
||||||
String summary = frontmatter.getSummary();
|
|
||||||
String category = frontmatter.getCategory() != null ? frontmatter.getCategory() : "general";
|
|
||||||
List<String> keywords = frontmatter.getKeywords();
|
|
||||||
|
|
||||||
if (title == null || title.isBlank()) {
|
|
||||||
logger.warn("文档缺少标题: {}", relativePath);
|
|
||||||
result.incrementFailed();
|
|
||||||
result.addDetail(relativePath, "缺少标题");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 保存到数据库
|
|
||||||
ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
|
|
||||||
|
|
||||||
// 提取文档正文(去除 frontmatter)
|
|
||||||
String body = extractBody(content);
|
|
||||||
|
|
||||||
// 文档分块
|
|
||||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
|
||||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
|
||||||
|
|
||||||
// 上传到 Milvus hybrid collection(dense + BM25 search_text)
|
|
||||||
try {
|
|
||||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
|
|
||||||
|
|
||||||
document.setStatus("INDEXED");
|
|
||||||
document.setChunkCount(chunks.size());
|
|
||||||
document.setIndexedAt(LocalDateTime.now());
|
|
||||||
apiDocumentRepository.save(document);
|
|
||||||
|
|
||||||
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
|
|
||||||
title, document.getDocId(), chunks.size());
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
|
||||||
|
|
||||||
document.setStatus("FAILED");
|
|
||||||
document.setErrorMessage(e.getMessage());
|
|
||||||
apiDocumentRepository.save(document);
|
|
||||||
|
|
||||||
result.incrementFailed();
|
|
||||||
result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
|
|
||||||
continue; // 跳过该文档,继续处理下一个
|
|
||||||
}
|
|
||||||
|
|
||||||
// 添加到 L0 内存索引
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath(relativePath)
|
|
||||||
.title(title)
|
|
||||||
.keywords(keywords)
|
|
||||||
.summary(summary)
|
|
||||||
.category(category)
|
|
||||||
.kbScope(frontmatter.getKbScope())
|
|
||||||
.build();
|
|
||||||
knowledgeIndexService.addToIndex(entry);
|
|
||||||
|
|
||||||
result.incrementInserted();
|
|
||||||
result.addDetail(relativePath, "导入成功(L0+L1)");
|
|
||||||
logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("处理文档失败: {}", relativePath, e);
|
|
||||||
result.incrementFailed();
|
|
||||||
result.addDetail(relativePath, "处理失败: " + e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("知识库初始化完成: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
|
||||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取知识库统计信息
|
|
||||||
*/
|
|
||||||
public Stats getStats() {
|
|
||||||
Stats stats = new Stats();
|
|
||||||
|
|
||||||
// 数据库中的文档数量
|
|
||||||
long totalDocuments = apiDocumentRepository.count();
|
|
||||||
stats.setTotalDocuments(totalDocuments);
|
|
||||||
|
|
||||||
// L0 索引中的文档数量
|
|
||||||
int indexSize = knowledgeIndexService.getIndexSize();
|
|
||||||
logger.debug("L0 索引大小: {}", indexSize);
|
|
||||||
|
|
||||||
// 按分类统计(从 fault_category 字段读取)
|
|
||||||
Map<String, Long> categoryCount = apiDocumentRepository.findAll().stream()
|
|
||||||
.collect(Collectors.groupingBy(
|
|
||||||
doc -> doc.getFaultCategory() != null ? doc.getFaultCategory().name() : "GENERAL",
|
|
||||||
Collectors.counting()
|
|
||||||
));
|
|
||||||
stats.setCategoryCount(categoryCount);
|
|
||||||
|
|
||||||
// Milvus 中的向量数量(需要实现)
|
|
||||||
// TODO: 查询 Milvus collection 的实体数量
|
|
||||||
stats.setTotalVectors(0L);
|
|
||||||
|
|
||||||
return stats;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 扫描目录下所有 Markdown 文件
|
|
||||||
*/
|
|
||||||
private List<Path> scanMarkdownFiles(Path baseDir) {
|
|
||||||
List<Path> files = new ArrayList<>();
|
|
||||||
|
|
||||||
try {
|
|
||||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
|
||||||
@Override
|
|
||||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
|
||||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
|
||||||
// Import content docs only; skip README/index markdown without frontmatter.
|
|
||||||
if (name.endsWith(".md")
|
|
||||||
&& !name.equalsIgnoreCase("README.md")
|
|
||||||
&& !name.equalsIgnoreCase("readme.md")) {
|
|
||||||
files.add(file);
|
|
||||||
}
|
|
||||||
return FileVisitResult.CONTINUE;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public FileVisitResult visitFileFailed(Path file, IOException exc) {
|
|
||||||
logger.warn("访问文件失败: {}", file, exc);
|
|
||||||
return FileVisitResult.CONTINUE;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (IOException e) {
|
|
||||||
logger.error("扫描目录失败: {}", baseDir, e);
|
|
||||||
throw new RuntimeException("扫描目录失败", e);
|
|
||||||
}
|
|
||||||
|
|
||||||
return files;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 保存文档到数据库
|
|
||||||
*/
|
|
||||||
private ApiDocument saveToDatabase(String filePath, String title, String summary,
|
|
||||||
String category, String content, List<String> keywords) {
|
|
||||||
ApiDocument document = new ApiDocument();
|
|
||||||
document.setDocId(UUID.randomUUID().toString());
|
|
||||||
document.setFileName(Paths.get(filePath).getFileName().toString());
|
|
||||||
document.setFilePath(filePath);
|
|
||||||
document.setApiName(title); // 使用 title 作为 apiName
|
|
||||||
document.setStatus("PENDING"); // 初始状态为 PENDING,索引成功后更新为 INDEXED
|
|
||||||
|
|
||||||
// 映射 category 到 FaultCategory 枚举
|
|
||||||
FaultCategory faultCategory = FaultCategory.fromString(category);
|
|
||||||
document.setFaultCategory(faultCategory);
|
|
||||||
|
|
||||||
// 将 frontmatter 信息保存到 metadata(JSON 格式)
|
|
||||||
String metadataJson = String.format(
|
|
||||||
"{\"title\":\"%s\",\"summary\":\"%s\",\"category\":\"%s\",\"keywords\":%s}",
|
|
||||||
escapeJson(title),
|
|
||||||
escapeJson(summary),
|
|
||||||
escapeJson(category),
|
|
||||||
"[\"" + String.join("\",\"", keywords.stream().map(this::escapeJson).toArray(String[]::new)) + "\"]"
|
|
||||||
);
|
|
||||||
document.setMetadata(metadataJson);
|
|
||||||
|
|
||||||
document.setFileSize((long) content.length());
|
|
||||||
|
|
||||||
return apiDocumentRepository.save(document);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* JSON 转义
|
|
||||||
*/
|
|
||||||
private String escapeJson(String str) {
|
|
||||||
if (str == null) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
return str.replace("\\", "\\\\")
|
|
||||||
.replace("\"", "\\\"")
|
|
||||||
.replace("\n", "\\n")
|
|
||||||
.replace("\r", "\\r");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 提取文档正文(去除 frontmatter)
|
|
||||||
*/
|
|
||||||
private String extractBody(String content) {
|
|
||||||
if (!content.trim().startsWith("---")) {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
int firstEnd = content.indexOf("---", 3);
|
|
||||||
if (firstEnd == -1) {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
int secondEnd = content.indexOf("---", firstEnd + 3);
|
|
||||||
if (secondEnd == -1) {
|
|
||||||
return content.substring(firstEnd + 3).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
return content.substring(secondEnd + 3).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 数据模型 ====================
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 初始化结果
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
public static class InitResult {
|
|
||||||
private int scanned; // 扫描到的文件数量
|
|
||||||
private int skipped; // 跳过的文件数量(已存在)
|
|
||||||
private int inserted; // 成功导入的文件数量
|
|
||||||
private int failed; // 失败的文件数量
|
|
||||||
private Map<String, String> details = new LinkedHashMap<>(); // 详细信息
|
|
||||||
|
|
||||||
public void incrementSkipped() {
|
|
||||||
this.skipped++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void incrementInserted() {
|
|
||||||
this.inserted++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void incrementFailed() {
|
|
||||||
this.failed++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addDetail(String filePath, String message) {
|
|
||||||
this.details.put(filePath, message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 统计信息
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
public static class Stats {
|
|
||||||
private long totalDocuments; // 数据库中的文档总数
|
|
||||||
private long totalVectors; // Milvus 中的向量总数
|
|
||||||
private Map<String, Long> categoryCount; // 按分类统计
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Full hybrid rebuild result.
|
|
||||||
*/
|
|
||||||
@Data
|
|
||||||
public static class RebuildResult {
|
|
||||||
private boolean success;
|
|
||||||
private String basePath;
|
|
||||||
private String collection;
|
|
||||||
private long mysqlDocumentsBefore;
|
|
||||||
private long mysqlDocumentsAfterClear;
|
|
||||||
private long mysqlDocumentsAfterInit;
|
|
||||||
private int l0IndexSizeAfterClear;
|
|
||||||
private int l0IndexSizeAfterInit;
|
|
||||||
private Map<String, Object> milvus;
|
|
||||||
private InitResult init;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,188 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.domain.entity.KnowledgeDomain;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
|
||||||
import org.springframework.ai.chat.prompt.Prompt;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.core.io.ClassPathResource;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import jakarta.annotation.PostConstruct;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识域服务
|
|
||||||
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class KnowledgeDomainService {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeIndexService knowledgeIndexService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ChatModel chatModel;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
private String domainPromptTemplate;
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void init() {
|
|
||||||
try {
|
|
||||||
domainPromptTemplate = new String(
|
|
||||||
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
|
|
||||||
StandardCharsets.UTF_8);
|
|
||||||
log.info("KnowledgeDomainService prompt 加载成功");
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("加载 domain-summary-prompt.md 失败", e);
|
|
||||||
throw new RuntimeException("Failed to load domain-summary prompt", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文档变更后重算指定域的 when_to_retrieve
|
|
||||||
*/
|
|
||||||
public void onDocumentChange(String category) {
|
|
||||||
if (category == null || category.isBlank()) return;
|
|
||||||
|
|
||||||
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
|
|
||||||
.filter(e -> category.equals(e.getCategory()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
|
|
||||||
buildDomainSummary(category, entries);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 聚合同域文档,调用 LLM 生成域级摘要,写入 DB
|
|
||||||
*/
|
|
||||||
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
|
|
||||||
if (entries.isEmpty()) {
|
|
||||||
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
|
|
||||||
d.setDocumentCount(0);
|
|
||||||
knowledgeDomainRepository.save(d);
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构建文档列表描述
|
|
||||||
StringBuilder docList = new StringBuilder();
|
|
||||||
for (KnowledgeEntry entry : entries) {
|
|
||||||
docList.append("- 文档:").append(entry.getTitle()).append("\n");
|
|
||||||
if (entry.getWhenToRetrieve() != null) {
|
|
||||||
docList.append(" 适用场景:").append(entry.getWhenToRetrieve()).append("\n");
|
|
||||||
}
|
|
||||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
|
||||||
docList.append(" 覆盖:").append(String.join("、", entry.getCovers())).append("\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
String description = entries.stream()
|
|
||||||
.map(KnowledgeEntry::getSummary)
|
|
||||||
.filter(s -> s != null && !s.isBlank())
|
|
||||||
.findFirst().orElse(category);
|
|
||||||
|
|
||||||
String whenToRetrieve = null;
|
|
||||||
try {
|
|
||||||
String otherDomainsInfo = buildOtherDomainsInfo(category);
|
|
||||||
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
|
|
||||||
whenToRetrieve = chatModel.call(new Prompt(promptText))
|
|
||||||
.getResult().getOutput().getText();
|
|
||||||
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("LLM 生成域级 when_to_retrieve 失败,保留旧值: domain={}", category, e);
|
|
||||||
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
|
|
||||||
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
|
|
||||||
}
|
|
||||||
|
|
||||||
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
|
|
||||||
.orElse(KnowledgeDomain.builder().domainId(category).build());
|
|
||||||
|
|
||||||
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
|
|
||||||
domain.setWhenToRetrieve(whenToRetrieve);
|
|
||||||
domain.setDocumentCount(entries.size());
|
|
||||||
knowledgeDomainRepository.save(domain);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建注入 Planner 的 knowledge map YAML 文本
|
|
||||||
*/
|
|
||||||
public String buildKnowledgeMap() {
|
|
||||||
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
|
|
||||||
if (domains.isEmpty()) return "";
|
|
||||||
|
|
||||||
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
|
|
||||||
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
|
|
||||||
.filter(e -> e.getCategory() != null)
|
|
||||||
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
|
|
||||||
|
|
||||||
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
|
|
||||||
|
|
||||||
for (KnowledgeDomain domain : domains) {
|
|
||||||
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
|
|
||||||
if (domain.getDescription() != null) {
|
|
||||||
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
|
|
||||||
}
|
|
||||||
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
|
|
||||||
yaml.append(" when_to_retrieve: \"")
|
|
||||||
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
|
|
||||||
}
|
|
||||||
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
|
|
||||||
|
|
||||||
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
|
|
||||||
if (!domainEntries.isEmpty()) {
|
|
||||||
yaml.append(" documents:\n");
|
|
||||||
for (KnowledgeEntry entry : domainEntries) {
|
|
||||||
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
|
|
||||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
|
||||||
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return yaml.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建其他域的摘要信息(用于 LLM 域级 prompt 的边界判断)
|
|
||||||
* 优先使用其他域的 when_to_retrieve(边界信号),而非 description
|
|
||||||
*/
|
|
||||||
private String buildOtherDomainsInfo(String currentCategory) {
|
|
||||||
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
for (KnowledgeDomain d : allDomains) {
|
|
||||||
if (d.getDomainId().equals(currentCategory)) continue;
|
|
||||||
sb.append("- ").append(d.getDomainId());
|
|
||||||
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
|
|
||||||
sb.append(":").append(d.getWhenToRetrieve());
|
|
||||||
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
|
|
||||||
sb.append("(").append(d.getDescription()).append(")");
|
|
||||||
}
|
|
||||||
sb.append("\n");
|
|
||||||
}
|
|
||||||
// 如果 DB 里还没有其他域的记录(首次启动),从 L0 索引补充
|
|
||||||
if (sb.isEmpty()) {
|
|
||||||
knowledgeIndexService.getAllEntries().stream()
|
|
||||||
.map(KnowledgeEntry::getCategory)
|
|
||||||
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
|
|
||||||
.distinct()
|
|
||||||
.forEach(c -> sb.append("- ").append(c).append("\n"));
|
|
||||||
}
|
|
||||||
return sb.isEmpty() ? "(无其他域信息)" : sb.toString();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,341 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
|
||||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
|
||||||
import org.springframework.context.annotation.Lazy;
|
|
||||||
import org.springframework.context.event.EventListener;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import jakarta.annotation.PostConstruct;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.Paths;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* L0 知识索引服务(关键词 / domain hint,不是向量库)。
|
|
||||||
*
|
|
||||||
* <h3>定位</h3>
|
|
||||||
* 从 MySQL {@code api_document.metadata}(frontmatter)加载文档级关键词与 category,
|
|
||||||
* 供检索前 query understanding 使用。L0 输出只作为:
|
|
||||||
* <ul>
|
|
||||||
* <li>可选 category filter(唯一 domain 时)</li>
|
|
||||||
* <li>rerank 的 domain/keyword/entity boost 信号</li>
|
|
||||||
* <li>trace 可解释信息</li>
|
|
||||||
* </ul>
|
|
||||||
* <b>L0 命中文档不会直接当作事实 evidence</b>;证据正文只来自 L1 向量召回。
|
|
||||||
*
|
|
||||||
* <h3>匹配方式(当前较粗)</h3>
|
|
||||||
* {@code query.contains(keyword) || keyword.contains(query)},大小写不敏感。
|
|
||||||
* 没有分词、别名归一或停用词;短词/泛词可能误命中。
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class KnowledgeIndexService {
|
|
||||||
|
|
||||||
@Value("${knowledge.base-path:knowledge_base}")
|
|
||||||
private String knowledgeBasePath;
|
|
||||||
|
|
||||||
@Value("${retrieval.kb-scope:}")
|
|
||||||
private String kbScope = "";
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ApiDocumentRepository apiDocumentRepository;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
|
||||||
|
|
||||||
@Lazy
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeDomainService knowledgeDomainService;
|
|
||||||
|
|
||||||
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
|
|
||||||
|
|
||||||
@PostConstruct
|
|
||||||
public void loadIndex() {
|
|
||||||
log.info("开始从数据库加载知识库索引");
|
|
||||||
|
|
||||||
try {
|
|
||||||
List<ApiDocument> documents = apiDocumentRepository.findAll();
|
|
||||||
|
|
||||||
int loaded = 0;
|
|
||||||
for (ApiDocument doc : documents) {
|
|
||||||
try {
|
|
||||||
KnowledgeEntry entry = parseDocumentToEntry(doc);
|
|
||||||
if (entry != null) {
|
|
||||||
knowledgeIndex.add(entry);
|
|
||||||
loaded++;
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("解析文档失败: docId={}, error={}", doc.getDocId(), e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("知识库索引加载完成,共 {} 个文档", loaded);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("知识库索引加载失败", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 应用就绪后,检查各域是否有 knowledge_domain 记录,无则触发生成
|
|
||||||
* 使用 ApplicationReadyEvent 而非 PostConstruct,避免循环依赖
|
|
||||||
*/
|
|
||||||
@EventListener(ApplicationReadyEvent.class)
|
|
||||||
public void onApplicationReady() {
|
|
||||||
try {
|
|
||||||
knowledgeIndex.stream()
|
|
||||||
.map(KnowledgeEntry::getCategory)
|
|
||||||
.filter(c -> c != null && !c.isBlank())
|
|
||||||
.distinct()
|
|
||||||
.forEach(category -> {
|
|
||||||
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
|
|
||||||
log.info("域 {} 无 knowledge_domain 记录,触发生成", category);
|
|
||||||
knowledgeDomainService.onDocumentChange(category);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("域级记录生成失败", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
|
|
||||||
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
|
|
||||||
|
|
||||||
return KnowledgeEntry.builder()
|
|
||||||
.filePath(doc.getFilePath())
|
|
||||||
.title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
|
|
||||||
.keywords(frontmatter.getKeywords())
|
|
||||||
.summary(frontmatter.getSummary())
|
|
||||||
.category(frontmatter.getCategory())
|
|
||||||
.kbScope(frontmatter.getKbScope())
|
|
||||||
.covers(frontmatter.getCovers())
|
|
||||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
|
||||||
.build();
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("解析 metadata 失败: {}", doc.getDocId(), e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 兼容旧调用:只返回命中的文档条目。 */
|
|
||||||
public List<KnowledgeEntry> exactMatch(String query) {
|
|
||||||
return analyzeQuery(query).matches();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 分析 query,产出 L0 hint。
|
|
||||||
* 遍历内存索引,收集匹配 keyword、domain、title;不做向量检索。
|
|
||||||
*/
|
|
||||||
public L0Hint analyzeQuery(String query) {
|
|
||||||
long startTime = System.currentTimeMillis();
|
|
||||||
|
|
||||||
if (query == null || query.trim().isEmpty()) {
|
|
||||||
log.debug("查询关键词为空,返回空结果");
|
|
||||||
return L0Hint.empty();
|
|
||||||
}
|
|
||||||
|
|
||||||
String queryLower = query.toLowerCase();
|
|
||||||
List<KnowledgeEntry> results = new ArrayList<>();
|
|
||||||
Set<String> matchedKeywords = new LinkedHashSet<>();
|
|
||||||
Set<String> domains = new LinkedHashSet<>();
|
|
||||||
Set<String> entities = new LinkedHashSet<>();
|
|
||||||
Set<String> titles = new LinkedHashSet<>();
|
|
||||||
|
|
||||||
for (KnowledgeEntry entry : knowledgeIndex) {
|
|
||||||
if (!matchesConfiguredScope(entry)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
List<String> entryMatchedKeywords = matchedKeywords(entry, queryLower);
|
|
||||||
if (entryMatchedKeywords.isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
results.add(entry);
|
|
||||||
matchedKeywords.addAll(entryMatchedKeywords);
|
|
||||||
entities.addAll(entryMatchedKeywords);
|
|
||||||
|
|
||||||
if (entry.getCategory() != null && !entry.getCategory().isBlank()) {
|
|
||||||
domains.add(entry.getCategory());
|
|
||||||
}
|
|
||||||
if (entry.getTitle() != null && !entry.getTitle().isBlank()) {
|
|
||||||
titles.add(entry.getTitle());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
long elapsedTime = System.currentTimeMillis() - startTime;
|
|
||||||
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
|
|
||||||
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
|
|
||||||
|
|
||||||
return new L0Hint(
|
|
||||||
List.copyOf(results),
|
|
||||||
List.copyOf(matchedKeywords),
|
|
||||||
List.copyOf(domains),
|
|
||||||
List.copyOf(entities),
|
|
||||||
List.copyOf(titles)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean matchesKeywords(KnowledgeEntry entry, String query) {
|
|
||||||
return !matchedKeywords(entry, query).isEmpty();
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean matchesConfiguredScope(KnowledgeEntry entry) {
|
|
||||||
String scope = trimToNull(kbScope);
|
|
||||||
if (scope == null) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return scope.equals(trimToNull(entry.getKbScope()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String trimToNull(String value) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 关键词双向包含匹配。
|
|
||||||
* query 已在调用方 lower-case;keyword 在此 lower-case。
|
|
||||||
* 例:query="mysql timeout" 可命中 keyword="mysql";
|
|
||||||
* 反过来 keyword="mysql connection pool timeout" 也可能被短 query 命中。
|
|
||||||
*/
|
|
||||||
private List<String> matchedKeywords(KnowledgeEntry entry, String query) {
|
|
||||||
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
|
|
||||||
List<String> matches = new ArrayList<>();
|
|
||||||
for (String keyword : entry.getKeywords()) {
|
|
||||||
String keywordLower = keyword.toLowerCase();
|
|
||||||
if (query.contains(keywordLower) || keywordLower.contains(query)) {
|
|
||||||
matches.add(keyword);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return matches;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String readDocument(String filePath, int maxChars) {
|
|
||||||
try {
|
|
||||||
Path fullPath = resolveDocumentPath(filePath);
|
|
||||||
if (!Files.exists(fullPath)) {
|
|
||||||
log.warn("读取文档失败,文件不存在: basePath={}, filePath={}, resolvedPath={}",
|
|
||||||
knowledgeBasePath, filePath, fullPath);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String content = Files.readString(fullPath);
|
|
||||||
|
|
||||||
if (content.length() > maxChars) {
|
|
||||||
return content.substring(0, maxChars) + "...";
|
|
||||||
}
|
|
||||||
|
|
||||||
return content;
|
|
||||||
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("读取文档失败: basePath={}, filePath={}", knowledgeBasePath, filePath, e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Path resolveDocumentPath(String filePath) {
|
|
||||||
if (filePath == null || filePath.isBlank()) {
|
|
||||||
throw new IllegalArgumentException("filePath cannot be blank");
|
|
||||||
}
|
|
||||||
|
|
||||||
Path path = Paths.get(filePath).normalize();
|
|
||||||
if (path.isAbsolute()) {
|
|
||||||
return path;
|
|
||||||
}
|
|
||||||
|
|
||||||
Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize();
|
|
||||||
Path baseName = basePath.getFileName();
|
|
||||||
if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) {
|
|
||||||
return basePath.getParent().resolve(path).normalize();
|
|
||||||
}
|
|
||||||
|
|
||||||
Path pathFromWorkingDir = path.toAbsolutePath().normalize();
|
|
||||||
if (pathFromWorkingDir.startsWith(basePath)) {
|
|
||||||
return pathFromWorkingDir;
|
|
||||||
}
|
|
||||||
|
|
||||||
return basePath.resolve(path).normalize();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addToIndex(KnowledgeEntry entry) {
|
|
||||||
knowledgeIndex.add(entry);
|
|
||||||
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
public void removeFromIndex(String filePath) {
|
|
||||||
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
|
|
||||||
log.debug("文档已从 L0 索引移除: {}", filePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Clear in-memory L0 entries (used by knowledge rebuild). */
|
|
||||||
public void clearIndex() {
|
|
||||||
knowledgeIndex.clear();
|
|
||||||
log.info("L0 knowledge index cleared");
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getIndexSize() {
|
|
||||||
return knowledgeIndex.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取所有索引条目(供域聚合使用)
|
|
||||||
*/
|
|
||||||
public List<KnowledgeEntry> getAllEntries() {
|
|
||||||
return List.copyOf(knowledgeIndex);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* L0 分析结果。
|
|
||||||
*
|
|
||||||
* @param matches 命中的文档条目(仅 hint,不是 evidence)
|
|
||||||
* @param matchedKeywords 命中的关键词
|
|
||||||
* @param domains 命中文档的 category 集合
|
|
||||||
* @param entities 当前实现等同 matchedKeywords,预留实体字段
|
|
||||||
* @param titles 命中文档标题
|
|
||||||
*/
|
|
||||||
public record L0Hint(
|
|
||||||
List<KnowledgeEntry> matches,
|
|
||||||
List<String> matchedKeywords,
|
|
||||||
List<String> domains,
|
|
||||||
List<String> entities,
|
|
||||||
List<String> titles
|
|
||||||
) {
|
|
||||||
public static L0Hint empty() {
|
|
||||||
return new L0Hint(List.of(), List.of(), List.of(), List.of(), List.of());
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 仅当恰好一个 domain 时返回,用于安全地加 category filter。 */
|
|
||||||
public String singleDomainOrNull() {
|
|
||||||
return domains.size() == 1 ? domains.get(0) : null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 检索前的 query 理解层(L0 出口)。
|
|
||||||
*
|
|
||||||
* <p>输入是 Agent 的原始检索句,输出 {@link KnowledgeQuery},供后续 L1 过滤与 rerank 使用。</p>
|
|
||||||
*
|
|
||||||
* <h3>当前能力边界</h3>
|
|
||||||
* <ul>
|
|
||||||
* <li>会做:关键词匹配、domain/entity/title hint、唯一 domain 时生成 categoryFilter</li>
|
|
||||||
* <li>不会做:真正的 query rewrite / 同义词扩展 / 多 query 改写
|
|
||||||
* ({@code rewrittenQuery} 目前等于 {@code originalQuery})</li>
|
|
||||||
* <li>L0 命中文档正文不会直接当作 evidence;证据只来自 L1 向量召回</li>
|
|
||||||
* </ul>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class KnowledgeQueryTransformer {
|
|
||||||
|
|
||||||
private final KnowledgeIndexService knowledgeIndexService;
|
|
||||||
|
|
||||||
public KnowledgeQueryTransformer(KnowledgeIndexService knowledgeIndexService) {
|
|
||||||
this.knowledgeIndexService = knowledgeIndexService;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将原始 query 转为检索控制结构。
|
|
||||||
*
|
|
||||||
* <p>{@code categoryFilter} 仅在 L0 恰好命中一个 domain 时非空;
|
|
||||||
* 多 domain 或零 domain 时为 null,避免错误收窄召回。</p>
|
|
||||||
*/
|
|
||||||
public KnowledgeQuery transform(String rawQuery) {
|
|
||||||
String normalized = rawQuery == null ? "" : rawQuery.trim();
|
|
||||||
KnowledgeIndexService.L0Hint hint = knowledgeIndexService.analyzeQuery(normalized);
|
|
||||||
return KnowledgeQuery.builder()
|
|
||||||
.originalQuery(normalized)
|
|
||||||
// 预留改写字段;当前未实现 rewrite,保持与 original 一致
|
|
||||||
.rewrittenQuery(normalized)
|
|
||||||
.domainHints(safeList(hint.domains()))
|
|
||||||
.matchedKeywords(safeList(hint.matchedKeywords()))
|
|
||||||
.entities(safeList(hint.entities()))
|
|
||||||
// 只有唯一 domain 才作为向量 metadata 的 category 过滤条件
|
|
||||||
.categoryFilter(hint.singleDomainOrNull())
|
|
||||||
.l0Titles(safeList(hint.titles()))
|
|
||||||
.l0MatchCount(hint.matches() == null ? 0 : hint.matches().size())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> safeList(List<String> values) {
|
|
||||||
return values == null ? List.of() : values;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,179 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.config.RagSidecarProperties;
|
|
||||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
|
||||||
import com.superbiz.agent.dto.RetrievalComparisonCase;
|
|
||||||
import com.superbiz.agent.dto.RetrievalComparisonReport;
|
|
||||||
import com.superbiz.agent.dto.RetrievalComparisonResult;
|
|
||||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.time.OffsetDateTime;
|
|
||||||
import java.time.ZoneOffset;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
@Service
|
|
||||||
public class RagRetrievalSidecarComparisonService {
|
|
||||||
|
|
||||||
private final VectorSearchService vectorSearchService;
|
|
||||||
private final SpringAiVectorStoreSidecarService sidecarService;
|
|
||||||
private final RetrievalResultNormalizer normalizer;
|
|
||||||
private final RagSidecarProperties properties;
|
|
||||||
private final ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
public RagRetrievalSidecarComparisonService(VectorSearchService vectorSearchService,
|
|
||||||
SpringAiVectorStoreSidecarService sidecarService,
|
|
||||||
RetrievalResultNormalizer normalizer,
|
|
||||||
RagSidecarProperties properties,
|
|
||||||
ObjectMapper objectMapper) {
|
|
||||||
this.vectorSearchService = vectorSearchService;
|
|
||||||
this.sidecarService = sidecarService;
|
|
||||||
this.normalizer = normalizer;
|
|
||||||
this.properties = properties;
|
|
||||||
this.objectMapper = objectMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
public RetrievalComparisonReport compare(List<RetrievalComparisonCase> cases, int topK) {
|
|
||||||
List<RetrievalComparisonResult> results = new ArrayList<>();
|
|
||||||
String sidecarStatus = "not_run";
|
|
||||||
for (RetrievalComparisonCase comparisonCase : cases) {
|
|
||||||
List<ComparableRetrievalResult> currentResults = normalizeCurrentResults(
|
|
||||||
vectorSearchService.searchSimilarDocuments(
|
|
||||||
comparisonCase.getQuery(),
|
|
||||||
topK,
|
|
||||||
comparisonCase.getCategory()
|
|
||||||
)
|
|
||||||
);
|
|
||||||
SidecarRetrievalResponse sidecar = sidecarService.search(
|
|
||||||
comparisonCase.getQuery(),
|
|
||||||
topK,
|
|
||||||
comparisonCase.getCategory()
|
|
||||||
);
|
|
||||||
sidecarStatus = sidecar.getStatus();
|
|
||||||
results.add(RetrievalComparisonResult.builder()
|
|
||||||
.caseId(comparisonCase.getCaseId())
|
|
||||||
.scenario(comparisonCase.getScenario())
|
|
||||||
.query(comparisonCase.getQuery())
|
|
||||||
.category(comparisonCase.getCategory())
|
|
||||||
.currentResults(currentResults)
|
|
||||||
.sidecar(sidecar)
|
|
||||||
.differences(compareDifferences(currentResults, sidecar.getResults()))
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
return RetrievalComparisonReport.builder()
|
|
||||||
.generatedAt(OffsetDateTime.now(ZoneOffset.UTC).toString())
|
|
||||||
.caseCount(cases.size())
|
|
||||||
.topK(topK)
|
|
||||||
.sidecarStatus(sidecarStatus)
|
|
||||||
.results(results)
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public RetrievalComparisonReport compareGoldenCases(Path caseFile) throws IOException {
|
|
||||||
var root = objectMapper.readTree(caseFile.toFile());
|
|
||||||
int topK = root.path("topK").asInt(5);
|
|
||||||
List<RetrievalComparisonCase> cases = new ArrayList<>();
|
|
||||||
for (var node : root.path("cases")) {
|
|
||||||
cases.add(RetrievalComparisonCase.builder()
|
|
||||||
.caseId(node.path("caseId").asText())
|
|
||||||
.scenario(node.path("scenario").asText())
|
|
||||||
.query(node.path("query").asText())
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
return compare(cases, topK);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void writeReports(RetrievalComparisonReport report, Path jsonPath, Path markdownPath) throws IOException {
|
|
||||||
createParentDirectories(jsonPath);
|
|
||||||
createParentDirectories(markdownPath);
|
|
||||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(jsonPath.toFile(), report);
|
|
||||||
Files.writeString(markdownPath, renderMarkdown(report));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void createParentDirectories(Path path) throws IOException {
|
|
||||||
Path parent = path.getParent();
|
|
||||||
if (parent != null) {
|
|
||||||
Files.createDirectories(parent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<ComparableRetrievalResult> normalizeCurrentResults(List<VectorSearchService.SearchResult> rawResults) {
|
|
||||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
|
||||||
for (int i = 0; i < rawResults.size(); i++) {
|
|
||||||
results.add(normalizer.fromCurrent(rawResults.get(i), i + 1, properties.getContentPreviewLimit()));
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<String> compareDifferences(List<ComparableRetrievalResult> currentResults,
|
|
||||||
List<ComparableRetrievalResult> sidecarResults) {
|
|
||||||
if (sidecarResults == null || sidecarResults.isEmpty()) {
|
|
||||||
return List.of("sidecar_unavailable_or_empty");
|
|
||||||
}
|
|
||||||
List<String> differences = new ArrayList<>();
|
|
||||||
String currentTopSource = currentResults.isEmpty() ? null : currentResults.get(0).getSource();
|
|
||||||
String sidecarTopSource = sidecarResults.get(0).getSource();
|
|
||||||
if (!Objects.equals(currentTopSource, sidecarTopSource)) {
|
|
||||||
differences.add("top_source_differs");
|
|
||||||
}
|
|
||||||
String currentTopBreadcrumb = currentResults.isEmpty() ? null : currentResults.get(0).getBreadcrumb();
|
|
||||||
String sidecarTopBreadcrumb = sidecarResults.get(0).getBreadcrumb();
|
|
||||||
if (!Objects.equals(currentTopBreadcrumb, sidecarTopBreadcrumb)) {
|
|
||||||
differences.add("top_breadcrumb_differs");
|
|
||||||
}
|
|
||||||
String currentScoreLabel = currentResults.isEmpty() ? null : currentResults.get(0).getScoreLabel();
|
|
||||||
String sidecarScoreLabel = sidecarResults.get(0).getScoreLabel();
|
|
||||||
if (!Objects.equals(currentScoreLabel, sidecarScoreLabel)) {
|
|
||||||
differences.add("score_label_differs");
|
|
||||||
}
|
|
||||||
return differences;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String renderMarkdown(RetrievalComparisonReport report) {
|
|
||||||
StringBuilder builder = new StringBuilder();
|
|
||||||
builder.append("# RAG Sidecar Retrieval Comparison\n\n");
|
|
||||||
builder.append("Generated at: `").append(report.getGeneratedAt()).append("`\n\n");
|
|
||||||
builder.append("- Cases: ").append(report.getCaseCount()).append("\n");
|
|
||||||
builder.append("- Top K: ").append(report.getTopK()).append("\n");
|
|
||||||
builder.append("- Sidecar status: `").append(report.getSidecarStatus()).append("`\n\n");
|
|
||||||
builder.append("| Case | Query | Current Top | Sidecar Top | Differences |\n");
|
|
||||||
builder.append("|---|---|---|---|---|\n");
|
|
||||||
for (RetrievalComparisonResult result : report.getResults()) {
|
|
||||||
builder.append("| ")
|
|
||||||
.append(nullToBlank(result.getCaseId()))
|
|
||||||
.append(" | ")
|
|
||||||
.append(escapePipe(result.getQuery()))
|
|
||||||
.append(" | ")
|
|
||||||
.append(formatTop(result.getCurrentResults()))
|
|
||||||
.append(" | ")
|
|
||||||
.append(formatTop(result.getSidecar() != null ? result.getSidecar().getResults() : List.of()))
|
|
||||||
.append(" | ")
|
|
||||||
.append(String.join("<br>", result.getDifferences()))
|
|
||||||
.append(" |\n");
|
|
||||||
}
|
|
||||||
return builder.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String formatTop(List<ComparableRetrievalResult> results) {
|
|
||||||
if (results == null || results.isEmpty()) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
ComparableRetrievalResult top = results.get(0);
|
|
||||||
return escapePipe(nullToBlank(top.getSource())) + " (" + nullToBlank(top.getScoreLabel()) + ")";
|
|
||||||
}
|
|
||||||
|
|
||||||
private String escapePipe(String value) {
|
|
||||||
return nullToBlank(value).replace("|", "\\|");
|
|
||||||
}
|
|
||||||
|
|
||||||
private String nullToBlank(String value) {
|
|
||||||
return value == null ? "" : value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,190 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
|
||||||
import org.springframework.ai.chat.messages.Message;
|
|
||||||
import org.springframework.ai.chat.messages.UserMessage;
|
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
|
||||||
import org.springframework.ai.chat.model.ChatResponse;
|
|
||||||
import org.springframework.ai.chat.prompt.Prompt;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import reactor.core.publisher.Flux;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* RAG (Retrieval-Augmented Generation) 服务
|
|
||||||
* 结合向量检索和大语言模型生成答案
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class RagService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(RagService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorSearchService vectorSearchService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ChatModel chatModel;
|
|
||||||
|
|
||||||
@Value("${rag.top-k:3}")
|
|
||||||
private int topK;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 流式处理用户问题(不带历史消息)
|
|
||||||
*
|
|
||||||
* @param question 用户问题
|
|
||||||
* @param callback 流式回调接口
|
|
||||||
*/
|
|
||||||
public void queryStream(String question, StreamCallback callback) {
|
|
||||||
queryStream(question, new ArrayList<>(), callback);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 流式处理用户问题(带历史消息)
|
|
||||||
*
|
|
||||||
* @param question 用户问题
|
|
||||||
* @param history 历史消息列表,格式:[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
|
|
||||||
* @param callback 流式回调接口
|
|
||||||
*/
|
|
||||||
public void queryStream(String question, List<Map<String, String>> history, StreamCallback callback) {
|
|
||||||
try {
|
|
||||||
logger.info("收到 RAG 流式查询: {}", question);
|
|
||||||
|
|
||||||
// 1. 从向量数据库检索相关文档
|
|
||||||
List<VectorSearchService.SearchResult> searchResults =
|
|
||||||
vectorSearchService.searchSimilarDocuments(question, topK);
|
|
||||||
|
|
||||||
// 发送检索结果
|
|
||||||
callback.onSearchResults(searchResults);
|
|
||||||
|
|
||||||
if (searchResults.isEmpty()) {
|
|
||||||
logger.warn("未找到相关文档");
|
|
||||||
callback.onComplete("抱歉,我在知识库中没有找到相关信息来回答您的问题。", "");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. 构建上下文和提示词
|
|
||||||
String context = buildContext(searchResults);
|
|
||||||
String prompt = buildPrompt(question, context);
|
|
||||||
|
|
||||||
// 3. 流式调用大语言模型(传入历史消息)
|
|
||||||
generateAnswerStream(prompt, history, callback);
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("RAG 流式查询失败", e);
|
|
||||||
callback.onError(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建上下文
|
|
||||||
*/
|
|
||||||
private String buildContext(List<VectorSearchService.SearchResult> searchResults) {
|
|
||||||
StringBuilder context = new StringBuilder();
|
|
||||||
|
|
||||||
for (int i = 0; i < searchResults.size(); i++) {
|
|
||||||
VectorSearchService.SearchResult result = searchResults.get(i);
|
|
||||||
context.append("【参考资料 ").append(i + 1).append("】\n");
|
|
||||||
context.append(result.getContent()).append("\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
return context.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构建提示词
|
|
||||||
*/
|
|
||||||
private String buildPrompt(String question, String context) {
|
|
||||||
return String.format(
|
|
||||||
"你是一个专业的AI助手。请根据以下参考资料回答用户的问题。\n\n" +
|
|
||||||
"参考资料:\n%s\n" +
|
|
||||||
"用户问题:%s\n\n" +
|
|
||||||
"请基于上述参考资料给出准确、详细的回答。如果参考资料中没有相关信息,请明确说明。",
|
|
||||||
context, question
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 生成答案(流式)
|
|
||||||
*
|
|
||||||
* @param prompt 当前问题的提示词
|
|
||||||
* @param history 历史消息列表
|
|
||||||
* @param callback 流式回调接口
|
|
||||||
*/
|
|
||||||
private void generateAnswerStream(String prompt, List<Map<String, String>> history, StreamCallback callback) {
|
|
||||||
// 构建消息列表:历史消息 + 当前问题
|
|
||||||
List<Message> messages = new ArrayList<>();
|
|
||||||
|
|
||||||
// 添加历史消息
|
|
||||||
for (Map<String, String> historyMsg : history) {
|
|
||||||
String role = historyMsg.get("role");
|
|
||||||
String content = historyMsg.get("content");
|
|
||||||
|
|
||||||
if ("user".equals(role)) {
|
|
||||||
messages.add(new UserMessage(content));
|
|
||||||
} else if ("assistant".equals(role)) {
|
|
||||||
messages.add(new AssistantMessage(content));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 添加当前用户问题
|
|
||||||
messages.add(new UserMessage(prompt));
|
|
||||||
|
|
||||||
logger.debug("发送给AI模型的消息数量: {}(包含 {} 条历史消息)",
|
|
||||||
messages.size(), history.size());
|
|
||||||
|
|
||||||
logger.info("开始调用AI模型流式接口...");
|
|
||||||
|
|
||||||
StringBuilder reasoningContent = new StringBuilder();
|
|
||||||
StringBuilder finalContent = new StringBuilder();
|
|
||||||
|
|
||||||
Flux<ChatResponse> flux = chatModel.stream(new Prompt(messages));
|
|
||||||
|
|
||||||
logger.info("开始接收AI模型流式响应...");
|
|
||||||
|
|
||||||
flux.subscribe(
|
|
||||||
response -> {
|
|
||||||
if (response.getResults() != null && !response.getResults().isEmpty()) {
|
|
||||||
String content = response.getResults().get(0).getOutput().getText();
|
|
||||||
|
|
||||||
if (content != null && !content.isEmpty()) {
|
|
||||||
logger.debug("收到AI模型内容块: {}", content);
|
|
||||||
|
|
||||||
finalContent.append(content);
|
|
||||||
callback.onContentChunk(content);
|
|
||||||
|
|
||||||
logger.debug("已调用 onContentChunk 回调");
|
|
||||||
} else {
|
|
||||||
logger.debug("收到空内容块,跳过");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
error -> {
|
|
||||||
logger.error("AI模型流式响应失败", error);
|
|
||||||
callback.onError(new Exception("AI模型流式响应失败: " + error.getMessage(), error));
|
|
||||||
},
|
|
||||||
() -> {
|
|
||||||
logger.info("AI模型流式响应完成,总内容长度: {}", finalContent.length());
|
|
||||||
callback.onComplete(finalContent.toString(), reasoningContent.toString());
|
|
||||||
logger.info("已调用 onComplete 回调");
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 流式回调接口
|
|
||||||
*/
|
|
||||||
public interface StreamCallback {
|
|
||||||
void onSearchResults(List<VectorSearchService.SearchResult> results);
|
|
||||||
void onReasoningChunk(String chunk);
|
|
||||||
void onContentChunk(String chunk);
|
|
||||||
void onComplete(String fullContent, String fullReasoning);
|
|
||||||
void onError(Exception e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,96 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
|
||||||
import org.springframework.ai.document.Document;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
public class RetrievalResultNormalizer {
|
|
||||||
|
|
||||||
private final ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
public RetrievalResultNormalizer(ObjectMapper objectMapper) {
|
|
||||||
this.objectMapper = objectMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ComparableRetrievalResult fromCurrent(VectorSearchService.SearchResult result, int rank, int previewLimit) {
|
|
||||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
|
||||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), result.getMetadata(), result.getId());
|
|
||||||
return ComparableRetrievalResult.builder()
|
|
||||||
.path("current")
|
|
||||||
.rank(rank)
|
|
||||||
.id(result.getId())
|
|
||||||
.source(source)
|
|
||||||
.docId(metadata.get("docId"))
|
|
||||||
.title(metadata.get("title"))
|
|
||||||
.breadcrumb(metadata.get("breadcrumb"))
|
|
||||||
.category(metadata.get("category"))
|
|
||||||
.contentPreview(truncate(result.getContent(), previewLimit))
|
|
||||||
.scoreLabel("l2_distance")
|
|
||||||
.scoreValue((double) result.getScore())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
public ComparableRetrievalResult fromSidecar(Document document, int rank, int previewLimit) {
|
|
||||||
Map<String, String> metadata = stringifyMetadata(document.getMetadata());
|
|
||||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), metadata.get("docId"), document.getId());
|
|
||||||
return ComparableRetrievalResult.builder()
|
|
||||||
.path("sidecar")
|
|
||||||
.rank(rank)
|
|
||||||
.id(document.getId())
|
|
||||||
.source(source)
|
|
||||||
.docId(metadata.get("docId"))
|
|
||||||
.title(metadata.get("title"))
|
|
||||||
.breadcrumb(metadata.get("breadcrumb"))
|
|
||||||
.category(metadata.get("category"))
|
|
||||||
.contentPreview(truncate(document.getText(), previewLimit))
|
|
||||||
.scoreLabel("similarity")
|
|
||||||
.scoreValue(document.getScore())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, String> parseMetadata(String metadata) {
|
|
||||||
if (metadata == null || metadata.isBlank()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
|
||||||
return stringifyMetadata(raw);
|
|
||||||
} catch (Exception e) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, String> stringifyMetadata(Map<?, ?> raw) {
|
|
||||||
if (raw == null || raw.isEmpty()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
Map<String, String> result = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
|
||||||
if (entry.getKey() != null && entry.getValue() != null) {
|
|
||||||
result.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String firstNonBlank(String... values) {
|
|
||||||
for (String value : values) {
|
|
||||||
if (value != null && !value.isBlank()) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String truncate(String text, int maxLength) {
|
|
||||||
if (text == null || text.length() <= maxLength) {
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
return text.substring(0, maxLength) + "...";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,106 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.config.RagSidecarProperties;
|
|
||||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
|
||||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.ai.document.Document;
|
|
||||||
import org.springframework.ai.vectorstore.SearchRequest;
|
|
||||||
import org.springframework.ai.vectorstore.VectorStore;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class SpringAiVectorStoreSidecarService {
|
|
||||||
|
|
||||||
private final RagSidecarProperties properties;
|
|
||||||
private final ObjectProvider<VectorStore> vectorStoreProvider;
|
|
||||||
private final RetrievalResultNormalizer normalizer;
|
|
||||||
|
|
||||||
@Value("${retrieval.kb-scope:}")
|
|
||||||
private String kbScope = "";
|
|
||||||
|
|
||||||
public SpringAiVectorStoreSidecarService(RagSidecarProperties properties,
|
|
||||||
ObjectProvider<VectorStore> vectorStoreProvider,
|
|
||||||
RetrievalResultNormalizer normalizer) {
|
|
||||||
this.properties = properties;
|
|
||||||
this.vectorStoreProvider = vectorStoreProvider;
|
|
||||||
this.normalizer = normalizer;
|
|
||||||
}
|
|
||||||
|
|
||||||
public SidecarRetrievalResponse search(String query, int topK, String category) {
|
|
||||||
if (!properties.isEnabled()) {
|
|
||||||
return unavailable("disabled", null);
|
|
||||||
}
|
|
||||||
|
|
||||||
VectorStore vectorStore = vectorStoreProvider.getIfAvailable();
|
|
||||||
if (vectorStore == null) {
|
|
||||||
return unavailable("missing_vector_store", "No Spring AI VectorStore bean is available");
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
SearchRequest.Builder builder = SearchRequest.builder()
|
|
||||||
.query(query)
|
|
||||||
.topK(topK)
|
|
||||||
.similarityThresholdAll();
|
|
||||||
String filterExpression = buildFilterExpression(category);
|
|
||||||
if (filterExpression != null) {
|
|
||||||
builder.filterExpression(filterExpression);
|
|
||||||
}
|
|
||||||
|
|
||||||
List<Document> documents = vectorStore.similaritySearch(builder.build());
|
|
||||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
|
||||||
for (int i = 0; i < documents.size(); i++) {
|
|
||||||
results.add(normalizer.fromSidecar(documents.get(i), i + 1, properties.getContentPreviewLimit()));
|
|
||||||
}
|
|
||||||
return SidecarRetrievalResponse.builder()
|
|
||||||
.enabled(true)
|
|
||||||
.available(true)
|
|
||||||
.status("available")
|
|
||||||
.results(results)
|
|
||||||
.build();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Spring AI sidecar retrieval failed: {}", e.getMessage());
|
|
||||||
return unavailable("query_failed", e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private SidecarRetrievalResponse unavailable(String status, String errorMessage) {
|
|
||||||
return SidecarRetrievalResponse.builder()
|
|
||||||
.enabled(properties.isEnabled())
|
|
||||||
.available(false)
|
|
||||||
.status(status)
|
|
||||||
.errorMessage(errorMessage)
|
|
||||||
.results(List.of())
|
|
||||||
.build();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String escapeFilterValue(String value) {
|
|
||||||
return value.replace("'", "\\'");
|
|
||||||
}
|
|
||||||
|
|
||||||
String buildFilterExpression(String category) {
|
|
||||||
List<String> parts = new ArrayList<>();
|
|
||||||
String categoryFilter = trimToNull(category);
|
|
||||||
if (categoryFilter != null) {
|
|
||||||
parts.add("category == '" + escapeFilterValue(categoryFilter) + "'");
|
|
||||||
}
|
|
||||||
String scopeFilter = trimToNull(kbScope);
|
|
||||||
if (scopeFilter != null) {
|
|
||||||
parts.add("kb_scope == '" + escapeFilterValue(scopeFilter) + "'");
|
|
||||||
}
|
|
||||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
|
||||||
}
|
|
||||||
|
|
||||||
private String trimToNull(String value) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.exception.DocumentProcessException;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.web.multipart.MultipartFile;
|
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.InputStreamReader;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 文本提取服务
|
|
||||||
* 仅支持 Markdown (.md) 和纯文本 (.txt) 格式
|
|
||||||
* 其他格式(.docx、.pdf 等)需要通过外部转换服务先转为 Markdown
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@Service
|
|
||||||
public class TextExtractorService {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 从文件中提取文本
|
|
||||||
*
|
|
||||||
* @param file 上传的文件
|
|
||||||
* @param fileName 文件名
|
|
||||||
* @return 提取的文本内容
|
|
||||||
*/
|
|
||||||
public String extractText(MultipartFile file, String fileName) {
|
|
||||||
if (file == null || file.isEmpty()) {
|
|
||||||
throw new DocumentProcessException(fileName, "extract", "文件为空");
|
|
||||||
}
|
|
||||||
|
|
||||||
String extension = getFileExtension(fileName);
|
|
||||||
log.info("开始提取文本,文件名: {}, 格式: {}, 大小: {} bytes", fileName, extension, file.getSize());
|
|
||||||
|
|
||||||
if (!isSupportedFormat(fileName)) {
|
|
||||||
throw new DocumentProcessException(
|
|
||||||
fileName, "extract",
|
|
||||||
"不支持的文件格式: " + extension + ",仅支持 .md 和 .txt。其他格式请先通过转换服务转为 Markdown。"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
String text = extractPlainText(file);
|
|
||||||
log.info("文本提取成功,文件名: {}, 提取字符数: {}", fileName, text.length());
|
|
||||||
return text;
|
|
||||||
|
|
||||||
} catch (IOException e) {
|
|
||||||
log.error("文本提取失败,文件名: {}", fileName, e);
|
|
||||||
throw new DocumentProcessException(fileName, "extract", "文件读取失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 提取纯文本(.txt、.md)
|
|
||||||
*/
|
|
||||||
private String extractPlainText(MultipartFile file) throws IOException {
|
|
||||||
StringBuilder content = new StringBuilder();
|
|
||||||
try (InputStream is = file.getInputStream();
|
|
||||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
|
|
||||||
|
|
||||||
String line;
|
|
||||||
while ((line = reader.readLine()) != null) {
|
|
||||||
content.append(line).append("\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return content.toString().trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 获取文件扩展名
|
|
||||||
*/
|
|
||||||
private String getFileExtension(String fileName) {
|
|
||||||
if (fileName == null || !fileName.contains(".")) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
return fileName.substring(fileName.lastIndexOf(".") + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 验证文件格式是否支持
|
|
||||||
*/
|
|
||||||
public boolean isSupportedFormat(String fileName) {
|
|
||||||
String extension = getFileExtension(fileName).toLowerCase();
|
|
||||||
return extension.equals("md") || extension.equals("txt");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,125 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.ai.embedding.EmbeddingModel;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 向量嵌入服务
|
|
||||||
* 使用阿里云 DashScope Text Embedding API
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class VectorEmbeddingService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(VectorEmbeddingService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private EmbeddingModel embeddingModel;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 生成向量嵌入
|
|
||||||
* 调用阿里云 DashScope Text Embedding API
|
|
||||||
*
|
|
||||||
* @param content 文本内容
|
|
||||||
* @return 向量嵌入(浮点数列表)
|
|
||||||
*/
|
|
||||||
public List<Float> generateEmbedding(String content) {
|
|
||||||
try {
|
|
||||||
if (content == null || content.trim().isEmpty()) {
|
|
||||||
logger.warn("内容为空,无法生成向量");
|
|
||||||
throw new IllegalArgumentException("内容不能为空");
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug("开始生成向量嵌入, 内容长度: {} 字符", content.length());
|
|
||||||
|
|
||||||
float[] embedding = embeddingModel.embed(content);
|
|
||||||
|
|
||||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
|
||||||
for (float v : embedding) {
|
|
||||||
floatEmbedding.add(v);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("成功生成向量嵌入, 内容长度: {} 字符, 向量维度: {}",
|
|
||||||
content.length(), floatEmbedding.size());
|
|
||||||
|
|
||||||
return floatEmbedding;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("生成向量嵌入失败, 内容长度: {}", content != null ? content.length() : 0, e);
|
|
||||||
throw new RuntimeException("生成向量嵌入失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<List<Float>> generateEmbeddings(List<String> contents) {
|
|
||||||
try {
|
|
||||||
if (contents == null || contents.isEmpty()) {
|
|
||||||
logger.warn("内容列表为空,无法生成向量");
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("开始批量生成向量嵌入, 数量: {}", contents.size());
|
|
||||||
|
|
||||||
List<float[]> embeddings = embeddingModel.embed(contents);
|
|
||||||
|
|
||||||
List<List<Float>> result = new ArrayList<>();
|
|
||||||
for (float[] embedding : embeddings) {
|
|
||||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
|
||||||
for (float v : embedding) {
|
|
||||||
floatEmbedding.add(v);
|
|
||||||
}
|
|
||||||
result.add(floatEmbedding);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("成功批量生成向量嵌入, 数量: {}, 维度: {}",
|
|
||||||
result.size(),
|
|
||||||
result.isEmpty() ? 0 : result.get(0).size());
|
|
||||||
|
|
||||||
return result;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("批量生成向量嵌入失败", e);
|
|
||||||
throw new RuntimeException("批量生成向量嵌入失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 生成查询向量
|
|
||||||
*
|
|
||||||
* @param query 查询文本
|
|
||||||
* @return 向量嵌入
|
|
||||||
*/
|
|
||||||
public List<Float> generateQueryVector(String query) {
|
|
||||||
return generateEmbedding(query);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 计算两个向量的余弦相似度
|
|
||||||
*
|
|
||||||
* @param vector1 向量1
|
|
||||||
* @param vector2 向量2
|
|
||||||
* @return 余弦相似度 [-1, 1]
|
|
||||||
*/
|
|
||||||
public float calculateCosineSimilarity(List<Float> vector1, List<Float> vector2) {
|
|
||||||
if (vector1.size() != vector2.size()) {
|
|
||||||
throw new IllegalArgumentException("向量维度不匹配");
|
|
||||||
}
|
|
||||||
|
|
||||||
float dotProduct = 0.0f;
|
|
||||||
float norm1 = 0.0f;
|
|
||||||
float norm2 = 0.0f;
|
|
||||||
|
|
||||||
for (int i = 0; i < vector1.size(); i++) {
|
|
||||||
dotProduct += vector1.get(i) * vector2.get(i);
|
|
||||||
norm1 += vector1.get(i) * vector1.get(i);
|
|
||||||
norm2 += vector2.get(i) * vector2.get(i);
|
|
||||||
}
|
|
||||||
|
|
||||||
return dotProduct / (float) (Math.sqrt(norm1) * Math.sqrt(norm2));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,380 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.Setter;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.nio.file.Paths;
|
|
||||||
import java.time.LocalDateTime;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 向量索引写入服务(RAG 入库侧)。
|
|
||||||
*
|
|
||||||
* <p>唯一后端 {@link MilvusHybridKnowledgeStore}(Milvus SDK v2):</p>
|
|
||||||
* <ul>
|
|
||||||
* <li>dense:应用侧 embedding → 字段 {@code vector}</li>
|
|
||||||
* <li>BM25:{@link #buildSearchText} → 字段 {@code search_text};
|
|
||||||
* sparse 由 collection 上 BM25 Function 自动生成,本类不写 sparse</li>
|
|
||||||
* </ul>
|
|
||||||
* <p>不再使用 legacy {@code MilvusServiceClient} insert/delete,
|
|
||||||
* 也不走 Spring AI {@code VectorStore#add}(starter 无 hybrid schema/BM25 Function)。</p>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class VectorIndexService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(VectorIndexService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorEmbeddingService embeddingService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentChunkService chunkService;
|
|
||||||
|
|
||||||
@Value("${file.upload.path}")
|
|
||||||
private String uploadPath;
|
|
||||||
|
|
||||||
public IndexingResult indexDirectory(String directoryPath) {
|
|
||||||
IndexingResult result = new IndexingResult();
|
|
||||||
result.setStartTime(LocalDateTime.now());
|
|
||||||
|
|
||||||
try {
|
|
||||||
String targetPath = (directoryPath != null && !directoryPath.trim().isEmpty())
|
|
||||||
? directoryPath : uploadPath;
|
|
||||||
|
|
||||||
Path dirPath = Paths.get(targetPath).normalize();
|
|
||||||
File directory = dirPath.toFile();
|
|
||||||
|
|
||||||
if (!directory.exists() || !directory.isDirectory()) {
|
|
||||||
throw new IllegalArgumentException("目录不存在或不是有效目录: " + targetPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
result.setDirectoryPath(directory.getAbsolutePath());
|
|
||||||
|
|
||||||
File[] files = directory.listFiles((dir, name) ->
|
|
||||||
name.endsWith(".txt") || name.endsWith(".md")
|
|
||||||
);
|
|
||||||
|
|
||||||
if (files == null || files.length == 0) {
|
|
||||||
logger.warn("目录中没有找到支持的文件: {}", targetPath);
|
|
||||||
result.setTotalFiles(0);
|
|
||||||
result.setSuccess(true);
|
|
||||||
result.setEndTime(LocalDateTime.now());
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
result.setTotalFiles(files.length);
|
|
||||||
logger.info("开始索引目录: {}, 找到 {} 个文件", targetPath, files.length);
|
|
||||||
|
|
||||||
for (File file : files) {
|
|
||||||
try {
|
|
||||||
indexSingleFile(file.getAbsolutePath());
|
|
||||||
result.incrementSuccessCount();
|
|
||||||
logger.info("文件索引成功: {}", file.getName());
|
|
||||||
} catch (Exception e) {
|
|
||||||
result.incrementFailCount();
|
|
||||||
result.addFailedFile(file.getAbsolutePath(), e.getMessage());
|
|
||||||
logger.error("文件索引失败: {}", file.getName(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
result.setSuccess(result.getFailCount() == 0);
|
|
||||||
result.setEndTime(LocalDateTime.now());
|
|
||||||
return result;
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("索引目录失败", e);
|
|
||||||
result.setSuccess(false);
|
|
||||||
result.setErrorMessage(e.getMessage());
|
|
||||||
result.setEndTime(LocalDateTime.now());
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void indexSingleFile(String filePath) throws Exception {
|
|
||||||
Path path = Paths.get(filePath).normalize();
|
|
||||||
File file = path.toFile();
|
|
||||||
|
|
||||||
if (!file.exists() || !file.isFile()) {
|
|
||||||
throw new IllegalArgumentException("文件不存在: " + filePath);
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("开始索引文件: {}", path);
|
|
||||||
String content = Files.readString(path);
|
|
||||||
deleteExistingData(path.toString());
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = chunkService.chunkDocument(content, path.toString());
|
|
||||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
|
||||||
|
|
||||||
for (int i = 0; i < chunks.size(); i++) {
|
|
||||||
DocumentChunk chunk = chunks.get(i);
|
|
||||||
try {
|
|
||||||
// dense embedding 与 BM25 search_text 同源(title/path 增强)
|
|
||||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
|
||||||
Map<String, Object> metadata = buildMetadata(path.toString(), chunk, chunks.size());
|
|
||||||
knowledgeStore.upsertChunk(
|
|
||||||
chunk.getContent(), // 返回原文
|
|
||||||
buildSearchText(chunk), // BM25 语料;sparse 由 Milvus Function 生成
|
|
||||||
vector, // dense 向量
|
|
||||||
metadata,
|
|
||||||
chunk.getChunkIndex());
|
|
||||||
logger.info("分片 {}/{} 索引成功", i + 1, chunks.size());
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("分片 {}/{} 索引失败", i + 1, chunks.size(), e);
|
|
||||||
throw new RuntimeException("分片索引失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
public void indexDocumentChunks(String docId, List<DocumentChunk> chunks, String category) throws Exception {
|
|
||||||
indexDocumentChunks(docId, chunks, category, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void indexDocumentChunks(String docId,
|
|
||||||
List<DocumentChunk> chunks,
|
|
||||||
String category,
|
|
||||||
Frontmatter frontmatter) throws Exception {
|
|
||||||
if (chunks == null || chunks.isEmpty()) {
|
|
||||||
throw new IllegalArgumentException("文档分块列表为空");
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("开始索引文档分块,docId: {}, 分块数: {}, 类别: {}", docId, chunks.size(), category);
|
|
||||||
deleteDocumentChunks(docId);
|
|
||||||
|
|
||||||
for (int i = 0; i < chunks.size(); i++) {
|
|
||||||
DocumentChunk chunk = chunks.get(i);
|
|
||||||
try {
|
|
||||||
// dense embedding 与 BM25 search_text 同源(title/path 增强)
|
|
||||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
|
||||||
Map<String, Object> metadata = buildDocumentMetadata(docId, chunk, chunks.size(), category, frontmatter);
|
|
||||||
knowledgeStore.upsertChunk(
|
|
||||||
chunk.getContent(), // 返回原文
|
|
||||||
buildSearchText(chunk), // BM25 语料;sparse 由 Milvus Function 生成
|
|
||||||
vector, // dense 向量
|
|
||||||
metadata,
|
|
||||||
chunk.getChunkIndex());
|
|
||||||
logger.info("文档分块 {}/{} 索引成功,docId: {}", i + 1, chunks.size(), docId);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("文档分块 {}/{} 索引失败,docId: {}", i + 1, chunks.size(), docId, e);
|
|
||||||
throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logger.info("文档索引完成,docId: {}, 共 {} 个分块,类别: {}", docId, chunks.size(), category);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteDocumentChunks(String docId) {
|
|
||||||
try {
|
|
||||||
knowledgeStore.deleteByDocId(docId);
|
|
||||||
logger.info("删除文档旧数据成功,docId: {}", docId);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn("删除文档旧数据异常,docId: {}", docId, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static Map<String, Object> buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks, String category) {
|
|
||||||
return buildDocumentMetadata(docId, chunk, totalChunks, category, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
static Map<String, Object> buildDocumentMetadata(String docId,
|
|
||||||
DocumentChunk chunk,
|
|
||||||
int totalChunks,
|
|
||||||
String category,
|
|
||||||
Frontmatter frontmatter) {
|
|
||||||
Map<String, Object> metadata = new HashMap<>();
|
|
||||||
String source = firstNonBlank(frontmatter != null ? frontmatter.getSource() : null, "upload:" + docId);
|
|
||||||
metadata.put("docId", docId);
|
|
||||||
metadata.put("_source", source);
|
|
||||||
metadata.put("source", source);
|
|
||||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
|
||||||
metadata.put("totalChunks", totalChunks);
|
|
||||||
|
|
||||||
String title = firstNonBlank(chunk.getTitle(), frontmatter != null ? frontmatter.getTitle() : null);
|
|
||||||
if (title != null) {
|
|
||||||
metadata.put("title", title);
|
|
||||||
}
|
|
||||||
String breadcrumb = firstNonBlank(frontmatter != null ? frontmatter.getBreadcrumb() : null, chunk.getBreadcrumb());
|
|
||||||
if (breadcrumb != null) {
|
|
||||||
metadata.put("breadcrumb", breadcrumb);
|
|
||||||
}
|
|
||||||
metadata.put("category", category != null && !category.isBlank() ? category : "upload");
|
|
||||||
String kbScope = trimToNull(frontmatter != null ? frontmatter.getKbScope() : null);
|
|
||||||
if (kbScope != null) {
|
|
||||||
metadata.put("kb_scope", kbScope);
|
|
||||||
}
|
|
||||||
return metadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dense embedding 输入。与 {@link #buildSearchText} 同源,保证 dense/BM25 看到同一增强文本。
|
|
||||||
*/
|
|
||||||
static String buildEmbeddingText(DocumentChunk chunk) {
|
|
||||||
return buildSearchText(chunk);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 构造写入 Milvus 的检索文本(BM25 {@code search_text},并复用为 dense embedding 输入)。
|
|
||||||
*
|
|
||||||
* <p>在正文前拼接 title / breadcrumb,提高「按标题或路径关键词」的 BM25 命中率,
|
|
||||||
* 同时让 dense 向量也编码结构信息。无标题路径时退回纯 content。</p>
|
|
||||||
*/
|
|
||||||
static String buildSearchText(DocumentChunk chunk) {
|
|
||||||
String content = trimToEmpty(chunk.getContent());
|
|
||||||
String title = trimToEmpty(chunk.getTitle());
|
|
||||||
String breadcrumb = trimToEmpty(chunk.getBreadcrumb());
|
|
||||||
|
|
||||||
if (title.isEmpty() && breadcrumb.isEmpty()) {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
|
|
||||||
StringBuilder text = new StringBuilder();
|
|
||||||
if (!title.isEmpty()) {
|
|
||||||
text.append("Title: ").append(title).append("\n");
|
|
||||||
}
|
|
||||||
if (!breadcrumb.isEmpty()) {
|
|
||||||
text.append("Path: ").append(breadcrumb).append("\n");
|
|
||||||
}
|
|
||||||
text.append("Content:\n").append(content);
|
|
||||||
return text.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String trimToEmpty(String value) {
|
|
||||||
return value == null ? "" : value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String trimToNull(String value) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String firstNonBlank(String... values) {
|
|
||||||
for (String value : values) {
|
|
||||||
String trimmed = trimToNull(value);
|
|
||||||
if (trimmed != null) {
|
|
||||||
return trimmed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void deleteExistingData(String filePath) {
|
|
||||||
try {
|
|
||||||
Path path = Paths.get(filePath).normalize();
|
|
||||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
|
||||||
knowledgeStore.deleteBySource(normalizedPath);
|
|
||||||
logger.info("已删除文件的旧数据: {}", normalizedPath);
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn("删除旧数据失败(可能是首次索引): {}", e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, Object> buildMetadata(String filePath, DocumentChunk chunk, int totalChunks) {
|
|
||||||
Map<String, Object> metadata = new HashMap<>();
|
|
||||||
Path path = Paths.get(filePath).normalize();
|
|
||||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
|
||||||
|
|
||||||
Path fileName = path.getFileName();
|
|
||||||
String fileNameStr = fileName != null ? fileName.toString() : "";
|
|
||||||
String extension = "";
|
|
||||||
int dotIndex = fileNameStr.lastIndexOf('.');
|
|
||||||
if (dotIndex > 0) {
|
|
||||||
extension = fileNameStr.substring(dotIndex);
|
|
||||||
}
|
|
||||||
|
|
||||||
metadata.put("_source", normalizedPath);
|
|
||||||
metadata.put("source", normalizedPath);
|
|
||||||
metadata.put("_extension", extension);
|
|
||||||
metadata.put("_file_name", fileNameStr);
|
|
||||||
|
|
||||||
String category = extractCategory(normalizedPath);
|
|
||||||
if (category != null && !category.isEmpty()) {
|
|
||||||
metadata.put("category", category);
|
|
||||||
}
|
|
||||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
|
||||||
metadata.put("totalChunks", totalChunks);
|
|
||||||
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
|
|
||||||
metadata.put("title", chunk.getTitle());
|
|
||||||
}
|
|
||||||
if (chunk.getBreadcrumb() != null && !chunk.getBreadcrumb().isEmpty()) {
|
|
||||||
metadata.put("breadcrumb", chunk.getBreadcrumb());
|
|
||||||
}
|
|
||||||
return metadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String extractCategory(String filePath) {
|
|
||||||
try {
|
|
||||||
String normalized = filePath.replace("\\", "/");
|
|
||||||
int docsIndex = normalized.indexOf("aiops-docs/");
|
|
||||||
if (docsIndex >= 0) {
|
|
||||||
String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
|
|
||||||
int slashIndex = afterDocs.indexOf("/");
|
|
||||||
if (slashIndex > 0) {
|
|
||||||
return afterDocs.substring(0, slashIndex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
int firstSlash = normalized.indexOf("/");
|
|
||||||
if (firstSlash > 0) {
|
|
||||||
return normalized.substring(0, firstSlash);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.warn("提取类别失败,路径: {}", filePath, e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Getter
|
|
||||||
public static class IndexingResult {
|
|
||||||
@Setter
|
|
||||||
private boolean success;
|
|
||||||
@Setter
|
|
||||||
private String directoryPath;
|
|
||||||
@Setter
|
|
||||||
private int totalFiles;
|
|
||||||
private int successCount;
|
|
||||||
private int failCount;
|
|
||||||
@Setter
|
|
||||||
private LocalDateTime startTime;
|
|
||||||
@Setter
|
|
||||||
private LocalDateTime endTime;
|
|
||||||
@Setter
|
|
||||||
private String errorMessage;
|
|
||||||
private Map<String, String> failedFiles = new HashMap<>();
|
|
||||||
|
|
||||||
public void incrementSuccessCount() {
|
|
||||||
this.successCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void incrementFailCount() {
|
|
||||||
this.failCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getDurationMs() {
|
|
||||||
if (startTime != null && endTime != null) {
|
|
||||||
return java.time.Duration.between(startTime, endTime).toMillis();
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void addFailedFile(String filePath, String error) {
|
|
||||||
this.failedFiles.put(filePath, error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
|
||||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
|
||||||
import lombok.Getter;
|
|
||||||
import lombok.Setter;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Locale;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库向量检索门面(lookup_knowledge / RAG 召回入口)。
|
|
||||||
*
|
|
||||||
* <p><b>唯一后端:</b>{@link MilvusHybridKnowledgeStore}(Milvus Java SDK v2)。</p>
|
|
||||||
*
|
|
||||||
* <h3>模式切换</h3>
|
|
||||||
* <p>{@code retrieval.search.mode}(同库查询算法,非两套写入):</p>
|
|
||||||
* <ul>
|
|
||||||
* <li>{@code hybrid} —— 线上主路径:dense + 服务端 BM25 + RRF</li>
|
|
||||||
* <li>{@code dense} —— 对照/评测:仅 dense ANN</li>
|
|
||||||
* </ul>
|
|
||||||
* <p>命中 {@link SearchResult#scoreLabel} 仅为 {@link RetrievalScoreLabels#DENSE} /
|
|
||||||
* {@link RetrievalScoreLabels#HYBRID}。质量分由后处理 {@code RetrievalScoreNormalizer} 统一计算。</p>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class VectorSearchService {
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(VectorSearchService.class);
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorEmbeddingService embeddingService;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 检索模式:{@code hybrid}(主路径)| {@code dense}(召回对照)。
|
|
||||||
*/
|
|
||||||
@Value("${retrieval.search.mode:dense}")
|
|
||||||
private String searchMode = "dense";
|
|
||||||
|
|
||||||
public List<SearchResult> searchSimilarDocuments(String query, int topK) {
|
|
||||||
return searchSimilarDocuments(query, topK, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public List<SearchResult> searchSimilarDocuments(String query, int topK, String category) {
|
|
||||||
String mode = searchMode == null ? "dense" : searchMode.trim().toLowerCase(Locale.ROOT);
|
|
||||||
List<Float> queryVector = embeddingService.generateQueryVector(query);
|
|
||||||
if ("hybrid".equals(mode)) {
|
|
||||||
logger.info("Hybrid dense+BM25 search topK={} category={} collection={}",
|
|
||||||
topK, category, knowledgeStore.collectionName());
|
|
||||||
return knowledgeStore.searchHybrid(query, queryVector, topK, category);
|
|
||||||
}
|
|
||||||
logger.info("Dense search topK={} category={} collection={}",
|
|
||||||
topK, category, knowledgeStore.collectionName());
|
|
||||||
return knowledgeStore.searchDense(query, queryVector, topK, category);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 单条召回结果。列表顺序即检索权威序(adapter 赋 originalRank=1..n)。
|
|
||||||
*
|
|
||||||
* <ul>
|
|
||||||
* <li>{@code scoreLabel=dense}:{@link #score} = L2 距离(越小越好)</li>
|
|
||||||
* <li>{@code scoreLabel=hybrid}:{@link #score}/{@link #rawScore} = 引擎融合分;
|
|
||||||
* 后处理 quality 主要按 rank 映射,不把 score 当 L2</li>
|
|
||||||
* </ul>
|
|
||||||
*/
|
|
||||||
@Setter
|
|
||||||
@Getter
|
|
||||||
public static class SearchResult {
|
|
||||||
private String id;
|
|
||||||
private String content;
|
|
||||||
/**
|
|
||||||
* 引擎主分:dense=L2;hybrid=融合分(量纲由 scoreLabel 解释)。
|
|
||||||
*/
|
|
||||||
private float score;
|
|
||||||
/** 引擎原始分(与 score 同源或更细,便于调试)。 */
|
|
||||||
private Double rawScore;
|
|
||||||
/** {@link RetrievalScoreLabels#DENSE} 或 {@link RetrievalScoreLabels#HYBRID}。 */
|
|
||||||
private String scoreLabel;
|
|
||||||
/**
|
|
||||||
* Optional dense L2 for the same id (hybrid path only).
|
|
||||||
* Used for absolute quality / low-quality gates; does <b>not</b> replace sort order.
|
|
||||||
*/
|
|
||||||
private Double denseDistance;
|
|
||||||
/** metadata JSON 字符串(docId、source、title…)。 */
|
|
||||||
private String metadata;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,547 +0,0 @@
|
|||||||
package com.superbiz.agent.service.milvus;
|
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
|
||||||
import com.google.gson.JsonObject;
|
|
||||||
import com.superbiz.agent.config.MilvusProperties;
|
|
||||||
import com.superbiz.agent.constant.MilvusConstants;
|
|
||||||
import com.superbiz.agent.service.VectorSearchService;
|
|
||||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
|
||||||
import io.milvus.common.clientenum.FunctionType;
|
|
||||||
import io.milvus.v2.client.ConnectConfig;
|
|
||||||
import io.milvus.v2.client.MilvusClientV2;
|
|
||||||
import io.milvus.v2.common.DataType;
|
|
||||||
import io.milvus.v2.common.IndexParam;
|
|
||||||
import io.milvus.v2.service.collection.request.AddFieldReq;
|
|
||||||
import io.milvus.v2.service.collection.request.CreateCollectionReq;
|
|
||||||
import io.milvus.v2.service.collection.request.DropCollectionReq;
|
|
||||||
import io.milvus.v2.service.collection.request.HasCollectionReq;
|
|
||||||
import io.milvus.v2.service.collection.request.LoadCollectionReq;
|
|
||||||
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
|
|
||||||
import io.milvus.v2.service.index.request.CreateIndexReq;
|
|
||||||
import io.milvus.v2.service.vector.request.AnnSearchReq;
|
|
||||||
import io.milvus.v2.service.vector.request.DeleteReq;
|
|
||||||
import io.milvus.v2.service.vector.request.HybridSearchReq;
|
|
||||||
import io.milvus.v2.service.vector.request.InsertReq;
|
|
||||||
import io.milvus.v2.service.vector.request.SearchReq;
|
|
||||||
import io.milvus.v2.service.vector.request.data.BaseVector;
|
|
||||||
import io.milvus.v2.service.vector.request.data.EmbeddedText;
|
|
||||||
import io.milvus.v2.service.vector.request.data.FloatVec;
|
|
||||||
import io.milvus.v2.service.vector.request.ranker.RRFRanker;
|
|
||||||
import io.milvus.v2.service.vector.response.SearchResp;
|
|
||||||
import jakarta.annotation.PreDestroy;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 知识库向量后端(Milvus Java SDK v2)—— dense + BM25 混合检索的唯一实现。
|
|
||||||
*
|
|
||||||
* <h3>为什么不用 Spring AI {@code spring-ai-starter-vector-store-milvus}</h3>
|
|
||||||
* <ul>
|
|
||||||
* <li>Spring AI Milvus starter(截至 2.0.0 / 1.1.8)只封装 dense {@code similaritySearch}。</li>
|
|
||||||
* <li>底层仍是 V1 {@code MilvusServiceClient} + 单路 {@code SearchParam},无 {@code hybridSearch} /
|
|
||||||
* BM25 Function / {@link RRFRanker}。</li>
|
|
||||||
* <li>真混合检索(dense ANN + 服务端 BM25 sparse,再 RRF 融合)必须走 Milvus SDK v2,
|
|
||||||
* 见 {@link #searchHybrid}。</li>
|
|
||||||
* </ul>
|
|
||||||
*
|
|
||||||
* <h3>Collection schema(默认名 {@code biz})</h3>
|
|
||||||
* <pre>
|
|
||||||
* id VarChar PK
|
|
||||||
* content VarChar —— 原文,返回给上层
|
|
||||||
* search_text VarChar+analyzer —— BM25 输入文本(可含 title/path 增强)
|
|
||||||
* sparse_vector SparseFloatVector —— 由 BM25 Function 从 search_text 自动生成,写入时不必填
|
|
||||||
* vector FloatVector —— dense 向量(应用侧 embedding)
|
|
||||||
* metadata JSON —— docId / source / category / kb_scope 等
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* <h3>检索模式</h3>
|
|
||||||
* <ul>
|
|
||||||
* <li>{@link #searchDense}:单路 L2 ANN;{@code scoreLabel=dense}。</li>
|
|
||||||
* <li>{@link #searchHybrid}:dense + BM25 + 服务端 {@link RRFRanker};{@code scoreLabel=hybrid};
|
|
||||||
* 返回序即 RRF 序,不再用 dense L2 覆盖主分。</li>
|
|
||||||
* </ul>
|
|
||||||
*
|
|
||||||
* <p>配置入口:{@code milvus.collection}、{@code retrieval.search.mode}、{@code retrieval.hybrid.rrf-k}。</p>
|
|
||||||
*/
|
|
||||||
@Service
|
|
||||||
public class MilvusHybridKnowledgeStore {
|
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MilvusHybridKnowledgeStore.class);
|
|
||||||
private static final Gson GSON = new Gson();
|
|
||||||
|
|
||||||
/** 主键(稳定 UUID,由 source + chunkIndex 派生,便于幂等重写)。 */
|
|
||||||
public static final String FIELD_ID = "id";
|
|
||||||
/** 返回给 LLM / 上层的原文 chunk。 */
|
|
||||||
public static final String FIELD_CONTENT = "content";
|
|
||||||
/**
|
|
||||||
* BM25 输入字段。写入明文;Milvus 侧 analyzer + BM25 Function 生成 {@link #FIELD_SPARSE}。
|
|
||||||
* 通常比 content 多带 title/path 等检索增强词。
|
|
||||||
*/
|
|
||||||
public static final String FIELD_SEARCH_TEXT = "search_text";
|
|
||||||
/** 稀疏向量字段;由 BM25 Function 自动产出,insert 时不要手动填。 */
|
|
||||||
public static final String FIELD_SPARSE = "sparse_vector";
|
|
||||||
/** Dense 向量字段(应用侧 EmbeddingModel 生成)。 */
|
|
||||||
public static final String FIELD_DENSE = "vector";
|
|
||||||
/** 业务元数据 JSON(过滤、证据身份、展示用)。 */
|
|
||||||
public static final String FIELD_METADATA = "metadata";
|
|
||||||
|
|
||||||
private final MilvusProperties milvusProperties;
|
|
||||||
|
|
||||||
@Value("${milvus.collection:biz}")
|
|
||||||
private String collectionName = "biz";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* RRF 平滑参数 k:score(d) = Σ 1/(k + rank_i(d))。
|
|
||||||
* k 越大,各路排名差异被压得越平;默认 60 与常见 RRF 设定一致。
|
|
||||||
*/
|
|
||||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
|
||||||
private int rrfK = 60;
|
|
||||||
|
|
||||||
/** 非空时追加 {@code metadata.kb_scope} 过滤,实现多知识域隔离。 */
|
|
||||||
@Value("${retrieval.kb-scope:}")
|
|
||||||
private String kbScope = "";
|
|
||||||
|
|
||||||
private volatile MilvusClientV2 client;
|
|
||||||
|
|
||||||
public MilvusHybridKnowledgeStore(MilvusProperties milvusProperties) {
|
|
||||||
this.milvusProperties = milvusProperties;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 懒连接:首次调用时建连、确保 collection schema 存在并 load。
|
|
||||||
* 线程安全;后续检索/写入复用同一 {@link MilvusClientV2}。
|
|
||||||
*/
|
|
||||||
public synchronized MilvusClientV2 client() {
|
|
||||||
if (client == null) {
|
|
||||||
client = connect();
|
|
||||||
ensureCollection(client);
|
|
||||||
loadCollection(client);
|
|
||||||
}
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String collectionName() {
|
|
||||||
return collectionName;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 写入单个 chunk(dense + BM25 所需明文)。
|
|
||||||
*
|
|
||||||
* <p>只插入 {@code content / search_text / vector / metadata};
|
|
||||||
* {@code sparse_vector} 由 collection 上的 BM25 Function 在服务端从 {@code search_text} 生成。</p>
|
|
||||||
*
|
|
||||||
* <p>id 由 {@code source|docId + chunkIndex} 的 nameUUID 派生,同一 chunk 重复写入会得到相同 id
|
|
||||||
*(配合先 delete 再 insert 的上层逻辑实现覆盖)。</p>
|
|
||||||
*
|
|
||||||
* @param content 原文(返回字段)
|
|
||||||
* @param searchText BM25 / 可与 dense embedding 同源的检索文本
|
|
||||||
* @param denseVector 应用侧 embedding
|
|
||||||
* @param metadata 须尽量带 {@code _source} 或 {@code docId},供 id 与过滤使用
|
|
||||||
* @param chunkIndex 分片序号
|
|
||||||
*/
|
|
||||||
public void upsertChunk(String content,
|
|
||||||
String searchText,
|
|
||||||
List<Float> denseVector,
|
|
||||||
Map<String, Object> metadata,
|
|
||||||
int chunkIndex) {
|
|
||||||
String source = metadata == null ? null : stringVal(metadata.get("_source"));
|
|
||||||
if (source == null) {
|
|
||||||
source = metadata == null ? null : stringVal(metadata.get("source"));
|
|
||||||
}
|
|
||||||
if (source == null) {
|
|
||||||
source = metadata == null ? null : stringVal(metadata.get("docId"));
|
|
||||||
}
|
|
||||||
String idSeed = (source == null ? "chunk" : source) + "_" + chunkIndex;
|
|
||||||
String id = UUID.nameUUIDFromBytes(idSeed.getBytes()).toString();
|
|
||||||
|
|
||||||
JsonObject row = new JsonObject();
|
|
||||||
row.addProperty(FIELD_ID, id);
|
|
||||||
row.addProperty(FIELD_CONTENT, content == null ? "" : content);
|
|
||||||
// 仅写明文;sparse 由 BM25 Function(search_text -> sparse_vector) 自动生成
|
|
||||||
row.addProperty(FIELD_SEARCH_TEXT, searchText == null ? "" : searchText);
|
|
||||||
row.add(FIELD_DENSE, GSON.toJsonTree(denseVector));
|
|
||||||
row.add(FIELD_METADATA, GSON.toJsonTree(metadata == null ? Map.of() : metadata));
|
|
||||||
|
|
||||||
client().insert(InsertReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.data(List.of(row))
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 按 metadata.docId 删除该文档全部 chunk(重建/覆盖前调用)。 */
|
|
||||||
public void deleteByDocId(String docId) {
|
|
||||||
if (docId == null || docId.isBlank()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
String filter = "metadata[\"docId\"] == \"" + escapeFilter(docId) + "\"";
|
|
||||||
client().delete(DeleteReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.filter(filter)
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 按 metadata._source(规范化路径)删除,用于按文件路径重索引。 */
|
|
||||||
public void deleteBySource(String sourcePath) {
|
|
||||||
if (sourcePath == null || sourcePath.isBlank()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
String normalized = sourcePath.replace('\\', '/');
|
|
||||||
String filter = "metadata[\"_source\"] == \"" + escapeFilter(normalized) + "\"";
|
|
||||||
client().delete(DeleteReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.filter(filter)
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 删除并重建当前知识 collection(空的 dense+BM25 schema)。
|
|
||||||
* 供 {@code /api/knowledge/rebuild-hybrid} 与重建脚本使用;会销毁该 collection 全部向量。
|
|
||||||
*/
|
|
||||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
|
||||||
Map<String, Object> result = new LinkedHashMap<>();
|
|
||||||
result.put("collection", collectionName);
|
|
||||||
MilvusClientV2 milvusClient = client();
|
|
||||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
result.put("existedBefore", Boolean.TRUE.equals(exists));
|
|
||||||
if (Boolean.TRUE.equals(exists)) {
|
|
||||||
try {
|
|
||||||
milvusClient.releaseCollection(ReleaseCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
|
|
||||||
}
|
|
||||||
milvusClient.dropCollection(DropCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
log.info("Dropped hybrid collection '{}'", collectionName);
|
|
||||||
result.put("dropped", true);
|
|
||||||
} else {
|
|
||||||
result.put("dropped", false);
|
|
||||||
}
|
|
||||||
ensureCollection(milvusClient);
|
|
||||||
loadCollection(milvusClient);
|
|
||||||
result.put("recreated", true);
|
|
||||||
result.put("loaded", true);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 单路 dense ANN(L2)。
|
|
||||||
* {@code score} = L2 距离(越小越好);{@code scoreLabel} = {@link RetrievalScoreLabels#DENSE}。
|
|
||||||
*/
|
|
||||||
public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
|
|
||||||
List<Float> queryVector,
|
|
||||||
int topK,
|
|
||||||
String category) {
|
|
||||||
String filter = buildFilter(category);
|
|
||||||
SearchReq.SearchReqBuilder builder = SearchReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.annsField(FIELD_DENSE)
|
|
||||||
.data(List.of(new FloatVec(queryVector)))
|
|
||||||
.topK(topK)
|
|
||||||
.outputFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
|
||||||
.metricType(IndexParam.MetricType.L2);
|
|
||||||
if (filter != null) {
|
|
||||||
builder.filter(filter);
|
|
||||||
}
|
|
||||||
SearchResp resp = client().search(builder.build());
|
|
||||||
return toSearchResults(resp, RetrievalScoreLabels.DENSE);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Dense + BM25 真混合检索(Milvus 服务端融合)。
|
|
||||||
*
|
|
||||||
* <ol>
|
|
||||||
* <li>dense 子路:{@code vector},L2</li>
|
|
||||||
* <li>BM25 子路:{@code sparse_vector} + {@link EmbeddedText}</li>
|
|
||||||
* <li>{@link HybridSearchReq} + {@link RRFRanker} → 返回序即权威序</li>
|
|
||||||
* </ol>
|
|
||||||
*
|
|
||||||
* <p>{@code scoreLabel=hybrid};{@code score}/{@code rawScore} 保留引擎融合分,
|
|
||||||
* <b>不</b>用 dense L2 覆盖主分或改 label。可选并行 dense 探测仅填充
|
|
||||||
* {@link VectorSearchService.SearchResult#setDenseDistance},供后处理绝对质量闸门
|
|
||||||
* (如 L0 filter low-quality → unfiltered retry),排序仍以 RRF 返回序为准。</p>
|
|
||||||
*/
|
|
||||||
public List<VectorSearchService.SearchResult> searchHybrid(String queryText,
|
|
||||||
List<Float> queryVector,
|
|
||||||
int topK,
|
|
||||||
String category) {
|
|
||||||
String filter = buildFilter(category);
|
|
||||||
int pathTopK = Math.max(topK, 10);
|
|
||||||
|
|
||||||
AnnSearchReq.AnnSearchReqBuilder denseAnn = AnnSearchReq.builder()
|
|
||||||
.vectorFieldName(FIELD_DENSE)
|
|
||||||
.vectors(List.of((BaseVector) new FloatVec(queryVector)))
|
|
||||||
.topK(pathTopK)
|
|
||||||
.metricType(IndexParam.MetricType.L2)
|
|
||||||
.params("{\"nprobe\":10}");
|
|
||||||
if (filter != null) {
|
|
||||||
denseAnn.filter(filter);
|
|
||||||
}
|
|
||||||
|
|
||||||
AnnSearchReq.AnnSearchReqBuilder sparseAnn = AnnSearchReq.builder()
|
|
||||||
.vectorFieldName(FIELD_SPARSE)
|
|
||||||
.vectors(List.of((BaseVector) new EmbeddedText(queryText == null ? "" : queryText)))
|
|
||||||
.topK(pathTopK)
|
|
||||||
.metricType(IndexParam.MetricType.BM25);
|
|
||||||
if (filter != null) {
|
|
||||||
sparseAnn.filter(filter);
|
|
||||||
}
|
|
||||||
|
|
||||||
HybridSearchReq hybridReq = HybridSearchReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.searchRequests(List.of(denseAnn.build(), sparseAnn.build()))
|
|
||||||
.ranker(new RRFRanker(Math.max(1, rrfK)))
|
|
||||||
.topK(topK)
|
|
||||||
.outFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
SearchResp hybridResp = client().hybridSearch(hybridReq);
|
|
||||||
List<VectorSearchService.SearchResult> fused = toSearchResults(hybridResp, RetrievalScoreLabels.HYBRID);
|
|
||||||
attachDenseDistances(fused, queryText, queryVector, pathTopK, category);
|
|
||||||
return fused;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Attach dense L2 by id for quality gates only — never overwrites hybrid score/label/order.
|
|
||||||
*/
|
|
||||||
private void attachDenseDistances(List<VectorSearchService.SearchResult> fused,
|
|
||||||
String queryText,
|
|
||||||
List<Float> queryVector,
|
|
||||||
int pathTopK,
|
|
||||||
String category) {
|
|
||||||
if (fused == null || fused.isEmpty()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Map<String, Float> denseById = new HashMap<>();
|
|
||||||
for (VectorSearchService.SearchResult denseHit :
|
|
||||||
searchDense(queryText, queryVector, pathTopK, category)) {
|
|
||||||
if (denseHit.getId() != null) {
|
|
||||||
denseById.put(denseHit.getId(), denseHit.getScore());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (VectorSearchService.SearchResult hit : fused) {
|
|
||||||
Float l2 = denseById.get(hit.getId());
|
|
||||||
if (l2 != null) {
|
|
||||||
hit.setDenseDistance(l2.doubleValue());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Dense distance attach for hybrid quality gate failed: {}", e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 将 Milvus {@link SearchResp} 映射为上层结果;列表顺序即检索权威序(adapter 赋 originalRank)。
|
|
||||||
*/
|
|
||||||
private List<VectorSearchService.SearchResult> toSearchResults(SearchResp resp, String scoreLabel) {
|
|
||||||
List<VectorSearchService.SearchResult> out = new ArrayList<>();
|
|
||||||
if (resp == null || resp.getSearchResults() == null || resp.getSearchResults().isEmpty()) {
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
List<SearchResp.SearchResult> first = resp.getSearchResults().get(0);
|
|
||||||
if (first == null) {
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
for (SearchResp.SearchResult row : first) {
|
|
||||||
VectorSearchService.SearchResult mapped = new VectorSearchService.SearchResult();
|
|
||||||
Object id = row.getId();
|
|
||||||
mapped.setId(id == null ? null : String.valueOf(id));
|
|
||||||
Map<String, Object> entity = row.getEntity() == null ? Map.of() : row.getEntity();
|
|
||||||
Object content = entity.get(FIELD_CONTENT);
|
|
||||||
mapped.setContent(content == null ? null : String.valueOf(content));
|
|
||||||
Object metadata = entity.get(FIELD_METADATA);
|
|
||||||
if (metadata instanceof JsonObject jsonObject) {
|
|
||||||
mapped.setMetadata(jsonObject.toString());
|
|
||||||
} else if (metadata instanceof Map<?, ?> map) {
|
|
||||||
mapped.setMetadata(GSON.toJson(map));
|
|
||||||
} else if (metadata != null) {
|
|
||||||
mapped.setMetadata(String.valueOf(metadata));
|
|
||||||
}
|
|
||||||
Float score = row.getScore();
|
|
||||||
mapped.setRawScore(score == null ? null : score.doubleValue());
|
|
||||||
mapped.setScoreLabel(scoreLabel);
|
|
||||||
// dense: L2;hybrid: 引擎融合分(后处理 quality 主要看 rank,不依赖此量纲)
|
|
||||||
mapped.setScore(score == null ? 0f : score);
|
|
||||||
out.add(mapped);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 组装标量过滤表达式:category、kb_scope(配置级)可叠加,用 {@code &&} 连接。
|
|
||||||
*/
|
|
||||||
private String buildFilter(String category) {
|
|
||||||
List<String> parts = new ArrayList<>();
|
|
||||||
String categoryFilter = trimToNull(category);
|
|
||||||
if (categoryFilter != null) {
|
|
||||||
parts.add("metadata[\"category\"] == \"" + escapeFilter(categoryFilter) + "\"");
|
|
||||||
}
|
|
||||||
String scope = trimToNull(kbScope);
|
|
||||||
if (scope != null) {
|
|
||||||
parts.add("metadata[\"kb_scope\"] == \"" + escapeFilter(scope) + "\"");
|
|
||||||
}
|
|
||||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
|
||||||
}
|
|
||||||
|
|
||||||
private MilvusClientV2 connect() {
|
|
||||||
String uri;
|
|
||||||
if (milvusProperties.isSecure() || milvusProperties.getPort() == 443) {
|
|
||||||
uri = "https://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
|
||||||
} else {
|
|
||||||
uri = "http://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
|
||||||
}
|
|
||||||
ConnectConfig.ConnectConfigBuilder builder = ConnectConfig.builder()
|
|
||||||
.uri(uri)
|
|
||||||
.connectTimeoutMs(milvusProperties.getTimeout() == null ? 10000L : milvusProperties.getTimeout());
|
|
||||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isBlank()) {
|
|
||||||
builder.token(milvusProperties.getToken());
|
|
||||||
builder.secure(true);
|
|
||||||
} else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isBlank()) {
|
|
||||||
builder.username(milvusProperties.getUsername());
|
|
||||||
builder.password(milvusProperties.getPassword());
|
|
||||||
}
|
|
||||||
if (milvusProperties.getDatabase() != null && !milvusProperties.getDatabase().isBlank()) {
|
|
||||||
builder.dbName(milvusProperties.getDatabase());
|
|
||||||
}
|
|
||||||
log.info("Connecting MilvusClientV2 uri={} db={} collection={}",
|
|
||||||
uri, milvusProperties.getDatabase(), collectionName);
|
|
||||||
return new MilvusClientV2(builder.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 若不存在则创建 dense+BM25 hybrid collection。
|
|
||||||
*
|
|
||||||
* <p>关键点:</p>
|
|
||||||
* <ul>
|
|
||||||
* <li>{@code search_text} 开启 analyzer,作为 BM25 语料。</li>
|
|
||||||
* <li>{@link FunctionType#BM25}:input={@code search_text} → output={@code sparse_vector}。</li>
|
|
||||||
* <li>dense:IVF_FLAT + L2;sparse:SPARSE_INVERTED_INDEX + BM25。</li>
|
|
||||||
* </ul>
|
|
||||||
* <p>已存在的 collection 不会改 schema;schema 变更需走 {@link #dropAndRecreateCollection()}。</p>
|
|
||||||
*/
|
|
||||||
private void ensureCollection(MilvusClientV2 milvusClient) {
|
|
||||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
if (Boolean.TRUE.equals(exists)) {
|
|
||||||
log.info("Hybrid collection '{}' already exists", collectionName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
log.info("Creating hybrid collection '{}'", collectionName);
|
|
||||||
|
|
||||||
CreateCollectionReq.CollectionSchema schema = milvusClient.createSchema();
|
|
||||||
schema.setEnableDynamicField(false);
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_ID)
|
|
||||||
.dataType(DataType.VarChar)
|
|
||||||
.maxLength(MilvusConstants.ID_MAX_LENGTH)
|
|
||||||
.isPrimaryKey(true)
|
|
||||||
.autoID(false)
|
|
||||||
.build());
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_CONTENT)
|
|
||||||
.dataType(DataType.VarChar)
|
|
||||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
|
||||||
.build());
|
|
||||||
// BM25 语料字段:必须 enableAnalyzer,Function 才能从文本生成 sparse
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_SEARCH_TEXT)
|
|
||||||
.dataType(DataType.VarChar)
|
|
||||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
|
||||||
.enableAnalyzer(true)
|
|
||||||
.build());
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_SPARSE)
|
|
||||||
.dataType(DataType.SparseFloatVector)
|
|
||||||
.build());
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_DENSE)
|
|
||||||
.dataType(DataType.FloatVector)
|
|
||||||
.dimension(milvusProperties.getVectorDim())
|
|
||||||
.build());
|
|
||||||
schema.addField(AddFieldReq.builder()
|
|
||||||
.fieldName(FIELD_METADATA)
|
|
||||||
.dataType(DataType.JSON)
|
|
||||||
.build());
|
|
||||||
// 写入 search_text 时,Milvus 自动维护 sparse_vector(应用层 insert 不填 sparse)
|
|
||||||
schema.addFunction(CreateCollectionReq.Function.builder()
|
|
||||||
.functionType(FunctionType.BM25)
|
|
||||||
.name("bm25_fn")
|
|
||||||
.inputFieldNames(List.of(FIELD_SEARCH_TEXT))
|
|
||||||
.outputFieldNames(List.of(FIELD_SPARSE))
|
|
||||||
.build());
|
|
||||||
|
|
||||||
milvusClient.createCollection(CreateCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.collectionSchema(schema)
|
|
||||||
.description("Knowledge hybrid dense+BM25 collection")
|
|
||||||
.numShards(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
List<IndexParam> indexes = List.of(
|
|
||||||
IndexParam.builder()
|
|
||||||
.fieldName(FIELD_DENSE)
|
|
||||||
.indexType(IndexParam.IndexType.IVF_FLAT)
|
|
||||||
.metricType(IndexParam.MetricType.L2)
|
|
||||||
.extraParams(Map.of("nlist", 128))
|
|
||||||
.build(),
|
|
||||||
IndexParam.builder()
|
|
||||||
.fieldName(FIELD_SPARSE)
|
|
||||||
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
|
|
||||||
.metricType(IndexParam.MetricType.BM25)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
milvusClient.createIndex(CreateIndexReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.indexParams(indexes)
|
|
||||||
.build());
|
|
||||||
log.info("Hybrid collection '{}' created with dense+BM25 indexes", collectionName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void loadCollection(MilvusClientV2 milvusClient) {
|
|
||||||
milvusClient.loadCollection(LoadCollectionReq.builder()
|
|
||||||
.collectionName(collectionName)
|
|
||||||
.build());
|
|
||||||
}
|
|
||||||
|
|
||||||
@PreDestroy
|
|
||||||
public void close() {
|
|
||||||
if (client != null) {
|
|
||||||
try {
|
|
||||||
client.close();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.warn("Error closing MilvusClientV2: {}", e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 过滤表达式字符串转义,防止引号打断 expr。 */
|
|
||||||
private static String escapeFilter(String value) {
|
|
||||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String trimToNull(String value) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String stringVal(Object value) {
|
|
||||||
return value == null ? null : String.valueOf(value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,9 +5,9 @@ import java.util.List;
|
|||||||
/**
|
/**
|
||||||
* 知识语义检索的应用边界端口。
|
* 知识语义检索的应用边界端口。
|
||||||
*
|
*
|
||||||
* <p>实现可对接 dense / hybrid 等引擎,但不得向上层泄漏 SDK 类型。
|
* <p>实现可对接 dense / hybrid 等引擎,但不得向上层泄漏远端 API 类型。
|
||||||
* 当前实现:{@link VectorKnowledgeSearchAdapter} → {@code VectorSearchService}
|
* 当前实现:{@link PyRagKnowledgeSearchAdapter}(py-rag 知识服务 /api/v1/search,
|
||||||
* → {@code MilvusHybridKnowledgeStore}(Milvus SDK v2 dense 或 dense+BM25 RRF)。</p>
|
* 服务端负责 hybrid 融合、BM25、rerank 与判级)。RAG 模块抽离后为唯一实现。</p>
|
||||||
*/
|
*/
|
||||||
public interface KnowledgeSearchPort {
|
public interface KnowledgeSearchPort {
|
||||||
|
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
package com.superbiz.agent.service.retrieval;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.LinkedHashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sparse-lite lexical ranking over already recalled candidates.
|
|
||||||
* Not a substitute for inverted-index BM25; expands ordering signal only.
|
|
||||||
*/
|
|
||||||
public final class LexicalRanker {
|
|
||||||
|
|
||||||
private LexicalRanker() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<KnowledgeSearchHit> rank(String query, List<KnowledgeSearchHit> candidates) {
|
|
||||||
if (candidates == null || candidates.isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
Set<String> terms = tokenize(query);
|
|
||||||
if (terms.isEmpty()) {
|
|
||||||
return List.copyOf(candidates);
|
|
||||||
}
|
|
||||||
List<ScoredHit> scored = new ArrayList<>(candidates.size());
|
|
||||||
for (KnowledgeSearchHit hit : candidates) {
|
|
||||||
String haystack = (nullToEmpty(hit.title()) + " "
|
|
||||||
+ nullToEmpty(hit.breadcrumb()) + " "
|
|
||||||
+ nullToEmpty(hit.content())).toLowerCase(Locale.ROOT);
|
|
||||||
int hits = 0;
|
|
||||||
for (String term : terms) {
|
|
||||||
if (haystack.contains(term)) {
|
|
||||||
hits++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
double coverage = hits / (double) terms.size();
|
|
||||||
scored.add(new ScoredHit(hit, coverage, hits));
|
|
||||||
}
|
|
||||||
scored.sort(Comparator
|
|
||||||
.comparingDouble((ScoredHit s) -> s.coverage).reversed()
|
|
||||||
.thenComparingInt((ScoredHit s) -> s.hits).reversed()
|
|
||||||
.thenComparingInt(s -> s.hit.originalRank()));
|
|
||||||
return scored.stream().map(s -> s.hit).toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
static Set<String> tokenize(String query) {
|
|
||||||
if (query == null || query.isBlank()) {
|
|
||||||
return Set.of();
|
|
||||||
}
|
|
||||||
String normalized = query.toLowerCase(Locale.ROOT);
|
|
||||||
String[] parts = normalized.split("[^\\p{IsAlphabetic}\\p{IsDigit}]+");
|
|
||||||
Set<String> terms = new LinkedHashSet<>();
|
|
||||||
for (String part : parts) {
|
|
||||||
if (part == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String term = part.trim();
|
|
||||||
if (term.length() >= 2) {
|
|
||||||
terms.add(term);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return terms;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String nullToEmpty(String value) {
|
|
||||||
return value == null ? "" : value;
|
|
||||||
}
|
|
||||||
|
|
||||||
private record ScoredHit(KnowledgeSearchHit hit, double coverage, int hits) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package com.superbiz.agent.service.retrieval;
|
||||||
|
|
||||||
|
import com.superbiz.agent.client.PyRagClient;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link KnowledgeSearchPort} 的 py-rag 远端实现(RAG 模块抽离后的唯一检索后端)。
|
||||||
|
*
|
||||||
|
* <p>原进程内链路(VectorKnowledgeSearchAdapter → VectorSearchService → MilvusHybridKnowledgeStore)
|
||||||
|
* 已由 py-rag 服务端接管:hybrid 融合、BM25、rerank、chunk 去重、判级阈值全部下沉。
|
||||||
|
* Java 侧只做请求映射与命中结构归一化,不碰检索算法。</p>
|
||||||
|
*
|
||||||
|
* <h3>映射约定</h3>
|
||||||
|
* <ul>
|
||||||
|
* <li>mode:{@link KnowledgeSearchMode#DENSE} → {@code semantic},{@link KnowledgeSearchMode#HYBRID} → {@code hybrid}</li>
|
||||||
|
* <li>retrieve_k = return_n = topK:返回 topK 条精排后命中,chunk 去重/截断仍由
|
||||||
|
* {@code KnowledgeEvidencePostProcessor} 统一负责,故 max_chunks_per_document 同步放大避免服务端预截断</li>
|
||||||
|
* <li>category:{@code categoryFilter} 透传;null = 不过滤;kb_scope 不传,由服务端部署配置决定</li>
|
||||||
|
* <li>score:py-rag rerank 绝对相关分([0,1],越大越好),scoreLabel =
|
||||||
|
* {@link RetrievalScoreLabels#RERANK}(quality 原样采用,不做 L2/rank 归一化)</li>
|
||||||
|
* <li>evidence_key:{@code docId#chunk-N},与 EvidenceGuard 验真约定一致;
|
||||||
|
* {@code evidence_status=no_evidence} 时服务端保证 hits=[],按"无知识"正常返回</li>
|
||||||
|
* </ul>
|
||||||
|
*/
|
||||||
|
@Component
|
||||||
|
public class PyRagKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
||||||
|
|
||||||
|
private static final String CHUNK_MARK = "#chunk-";
|
||||||
|
|
||||||
|
private final PyRagClient pyRagClient;
|
||||||
|
|
||||||
|
public PyRagKnowledgeSearchAdapter(PyRagClient pyRagClient) {
|
||||||
|
this.pyRagClient = pyRagClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
||||||
|
PyRagSearchResponse response = pyRagClient.search(toPyRagRequest(request));
|
||||||
|
if (response == null || response.hits() == null || response.hits().isEmpty()) {
|
||||||
|
return List.of();
|
||||||
|
}
|
||||||
|
List<KnowledgeSearchHit> hits = new ArrayList<>(response.hits().size());
|
||||||
|
for (int i = 0; i < response.hits().size(); i++) {
|
||||||
|
hits.add(toHit(response.hits().get(i), i + 1));
|
||||||
|
}
|
||||||
|
return hits;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 请求映射:topK 同时作为召回宽度与返回条数,服务端不预截断 chunk。 */
|
||||||
|
private PyRagSearchRequest toPyRagRequest(KnowledgeSearchRequest request) {
|
||||||
|
return new PyRagSearchRequest(
|
||||||
|
request.query(),
|
||||||
|
request.mode() == KnowledgeSearchMode.HYBRID ? "hybrid" : "semantic",
|
||||||
|
request.topK(),
|
||||||
|
request.topK(),
|
||||||
|
request.topK(),
|
||||||
|
blankToNull(request.categoryFilter()),
|
||||||
|
null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private KnowledgeSearchHit toHit(PyRagSearchHit hit, int originalRank) {
|
||||||
|
String docId = blankToNull(hit.documentId());
|
||||||
|
Integer chunkIndex = parseChunkIndex(hit.evidenceKey());
|
||||||
|
String evidenceKey = EvidenceIdentity.firstNonBlank(
|
||||||
|
hit.evidenceKey(),
|
||||||
|
EvidenceIdentity.evidenceKey(docId, chunkIndex, null, originalRank));
|
||||||
|
Double score = hit.qualityScore() == null ? 0.0 : hit.qualityScore();
|
||||||
|
return new KnowledgeSearchHit(
|
||||||
|
firstNonBlank(evidenceKey, docId, "rank:" + originalRank),
|
||||||
|
hit.excerpt(),
|
||||||
|
score,
|
||||||
|
score,
|
||||||
|
RetrievalScoreLabels.RERANK,
|
||||||
|
null,
|
||||||
|
Map.of(),
|
||||||
|
docId,
|
||||||
|
chunkIndex,
|
||||||
|
evidenceKey,
|
||||||
|
hit.source(),
|
||||||
|
hit.title(),
|
||||||
|
hit.breadcrumb(),
|
||||||
|
originalRank,
|
||||||
|
null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** evidence_key 形如 {@code docId#chunk-N},解析末尾 chunk 序号;不符返回 null。 */
|
||||||
|
private Integer parseChunkIndex(String evidenceKey) {
|
||||||
|
String key = blankToNull(evidenceKey);
|
||||||
|
if (key == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
int mark = key.lastIndexOf(CHUNK_MARK);
|
||||||
|
if (mark < 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return Integer.valueOf(key.substring(mark + CHUNK_MARK.length()).trim());
|
||||||
|
} catch (NumberFormatException ignored) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String blankToNull(String value) {
|
||||||
|
return value == null || value.isBlank() ? null : value.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String firstNonBlank(String... values) {
|
||||||
|
return EvidenceIdentity.firstNonBlank(values);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,8 +3,13 @@ package com.superbiz.agent.service.retrieval;
|
|||||||
/**
|
/**
|
||||||
* 检索结果一级 {@code scoreLabel} 约定。
|
* 检索结果一级 {@code scoreLabel} 约定。
|
||||||
*
|
*
|
||||||
* <p>只区分两种检索形态(与 {@code retrieval.search.mode} 对齐),
|
* <p>三种检索形态:</p>
|
||||||
* 不再使用 {@code bm25_only_*} 等作为正式一级 label。</p>
|
* <ul>
|
||||||
|
* <li>{@link #DENSE} —— 单路向量 ANN(L2 距离,越小越好)</li>
|
||||||
|
* <li>{@link #HYBRID} —— dense + BM25 + RRF 融合(质量主要看 rank)</li>
|
||||||
|
* <li>{@link #RERANK} —— py-rag 服务端 rerank 绝对相关分([0,1],越大越好);
|
||||||
|
* RAG 模块抽离后的线上主路径</li>
|
||||||
|
* </ul>
|
||||||
*/
|
*/
|
||||||
public final class RetrievalScoreLabels {
|
public final class RetrievalScoreLabels {
|
||||||
|
|
||||||
@@ -14,11 +19,14 @@ public final class RetrievalScoreLabels {
|
|||||||
/** hybrid(dense+BM25+RRF):{@code score}/raw 为融合侧信号;质量分主要看 rank。 */
|
/** hybrid(dense+BM25+RRF):{@code score}/raw 为融合侧信号;质量分主要看 rank。 */
|
||||||
public static final String HYBRID = "hybrid";
|
public static final String HYBRID = "hybrid";
|
||||||
|
|
||||||
|
/** py-rag 服务端 rerank 绝对分:{@code score} 即归一化质量([0,1],越大越好)。 */
|
||||||
|
public static final String RERANK = "rerank";
|
||||||
|
|
||||||
private RetrievalScoreLabels() {
|
private RetrievalScoreLabels() {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将历史/别名 label 归一到 {@link #DENSE} 或 {@link #HYBRID}。
|
* 将历史/别名 label 归一到 {@link #DENSE}、{@link #HYBRID} 或 {@link #RERANK}。
|
||||||
* 未知或空 → dense(保守,按 L2 解释失败时 quality 偏低)。
|
* 未知或空 → dense(保守,按 L2 解释失败时 quality 偏低)。
|
||||||
*/
|
*/
|
||||||
public static String canonicalize(String scoreLabel) {
|
public static String canonicalize(String scoreLabel) {
|
||||||
@@ -29,12 +37,19 @@ public final class RetrievalScoreLabels {
|
|||||||
return switch (label) {
|
return switch (label) {
|
||||||
case DENSE, "l2_distance", "l2" -> DENSE;
|
case DENSE, "l2_distance", "l2" -> DENSE;
|
||||||
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
|
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
|
||||||
default -> label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
case RERANK, "rerank_score", "quality_score" -> RERANK;
|
||||||
? HYBRID
|
default -> label.contains("rerank") || label.contains("quality")
|
||||||
: DENSE;
|
? RERANK
|
||||||
|
: label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
||||||
|
? HYBRID
|
||||||
|
: DENSE;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static boolean isRerank(String scoreLabel) {
|
||||||
|
return RERANK.equals(canonicalize(scoreLabel));
|
||||||
|
}
|
||||||
|
|
||||||
public static boolean isHybrid(String scoreLabel) {
|
public static boolean isHybrid(String scoreLabel) {
|
||||||
return HYBRID.equals(canonicalize(scoreLabel));
|
return HYBRID.equals(canonicalize(scoreLabel));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ package com.superbiz.agent.service.retrieval;
|
|||||||
* <p>后处理排序仍按 {@code originalRank};本类只负责质量闸门 / relevance 用分。</p>
|
* <p>后处理排序仍按 {@code originalRank};本类只负责质量闸门 / relevance 用分。</p>
|
||||||
*
|
*
|
||||||
* <ul>
|
* <ul>
|
||||||
|
* <li>{@link RetrievalScoreLabels#RERANK}:py-rag 服务端 rerank 绝对分,
|
||||||
|
* {@code score} 已归一化,原样 clamp 到 [0,1](RAG 抽离后的主路径)</li>
|
||||||
* <li>{@link RetrievalScoreLabels#DENSE}:{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li>
|
* <li>{@link RetrievalScoreLabels#DENSE}:{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li>
|
||||||
* <li>{@link RetrievalScoreLabels#HYBRID}:优先用可选 {@code denseDistance} 做绝对质量
|
* <li>{@link RetrievalScoreLabels#HYBRID}:优先用可选 {@code denseDistance} 做绝对质量
|
||||||
* (恢复 L0 filter low-quality 等闸门);无 dense 时回退 rank 映射</li>
|
* (恢复 L0 filter low-quality 等闸门);无 dense 时回退 rank 映射</li>
|
||||||
@@ -17,8 +19,8 @@ public final class RetrievalScoreNormalizer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param scoreLabel {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
* @param scoreLabel {@link RetrievalScoreLabels#RERANK} / {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
||||||
* @param score 引擎主分:dense=L2;hybrid=融合分(hybrid 质量不依赖其量纲)
|
* @param score 引擎主分:rerank=绝对相关分[0,1];dense=L2;hybrid=融合分(hybrid 质量不依赖其量纲)
|
||||||
* @param originalRank 检索名次(1-based)
|
* @param originalRank 检索名次(1-based)
|
||||||
* @param batchSize 本轮候选数(rank 回退映射用)
|
* @param batchSize 本轮候选数(rank 回退映射用)
|
||||||
* @param maxL2Distance L2 上界
|
* @param maxL2Distance L2 上界
|
||||||
@@ -31,6 +33,12 @@ public final class RetrievalScoreNormalizer {
|
|||||||
double maxL2Distance,
|
double maxL2Distance,
|
||||||
Double denseDistance) {
|
Double denseDistance) {
|
||||||
String label = RetrievalScoreLabels.canonicalize(scoreLabel);
|
String label = RetrievalScoreLabels.canonicalize(scoreLabel);
|
||||||
|
if (RetrievalScoreLabels.RERANK.equals(label)) {
|
||||||
|
if (score == null) {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
return Math.max(0.0, Math.min(1.0, score));
|
||||||
|
}
|
||||||
if (RetrievalScoreLabels.HYBRID.equals(label)) {
|
if (RetrievalScoreLabels.HYBRID.equals(label)) {
|
||||||
if (denseDistance != null) {
|
if (denseDistance != null) {
|
||||||
return l2ToQuality(denseDistance, maxL2Distance);
|
return l2ToQuality(denseDistance, maxL2Distance);
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
package com.superbiz.agent.service.retrieval;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.function.Function;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reciprocal Rank Fusion 工具:把多路检索的排名列表融合成一个分数排序。
|
|
||||||
*
|
|
||||||
* <pre>
|
|
||||||
* RRF_w(d) = Σ w_i / (k + rank_i(d))
|
|
||||||
* </pre>
|
|
||||||
*
|
|
||||||
* <p>只依赖排名不依赖原始分数——屏蔽跨路分数尺度不可比的问题;
|
|
||||||
* 每路可加权(w <= 0 时按 1.0 等权),k 是平滑参数(默认 60,可配)。
|
|
||||||
*/
|
|
||||||
public final class RrfFusion {
|
|
||||||
|
|
||||||
private RrfFusion() {
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 融合多路排名:对每路的每个 item 累加 w/(k+rank),按总分降序输出。
|
|
||||||
*
|
|
||||||
* @param paths 多路排名(每路带 name / items / weight)
|
|
||||||
* @param rrfK 平滑参数 k(至少 1)
|
|
||||||
* @param identityFn 跨路识别同一 item 的身份函数(如 evidenceKey)
|
|
||||||
* @return 融合后排序(含每路排名明细)
|
|
||||||
*/
|
|
||||||
public static <T> List<Scored<T>> fuse(List<RankedPath<T>> paths,
|
|
||||||
int rrfK,
|
|
||||||
Function<T, String> identityFn) {
|
|
||||||
if (paths == null || paths.isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
int k = Math.max(1, rrfK);
|
|
||||||
Map<String, Acc<T>> acc = new LinkedHashMap<>();
|
|
||||||
for (RankedPath<T> path : paths) {
|
|
||||||
if (path == null || path.items() == null || path.items().isEmpty()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
double weight = path.weight() <= 0 ? 1.0 : path.weight();
|
|
||||||
List<T> items = path.items();
|
|
||||||
for (int i = 0; i < items.size(); i++) {
|
|
||||||
T item = items.get(i);
|
|
||||||
if (item == null) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String id = identityFn.apply(item);
|
|
||||||
if (id == null || id.isBlank()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
int rank = i + 1;
|
|
||||||
double contrib = weight / (k + rank); // 排名越前贡献越大
|
|
||||||
Acc<T> bucket = acc.computeIfAbsent(id, ignored -> new Acc<>(item));
|
|
||||||
bucket.score += contrib;
|
|
||||||
bucket.ranks.put(path.name(), rank);
|
|
||||||
// Prefer first-seen item payload; callers should put preferred path first if needed.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
List<Scored<T>> scored = new ArrayList<>(acc.size());
|
|
||||||
for (Map.Entry<String, Acc<T>> entry : acc.entrySet()) {
|
|
||||||
Acc<T> value = entry.getValue();
|
|
||||||
scored.add(new Scored<>(entry.getKey(), value.item, value.score, Map.copyOf(value.ranks)));
|
|
||||||
}
|
|
||||||
// 总分降序(两路共识的靠前),同分按身份稳定排序
|
|
||||||
scored.sort(Comparator
|
|
||||||
.comparingDouble((Scored<T> s) -> s.rrfScore()).reversed()
|
|
||||||
.thenComparing(Scored::identity));
|
|
||||||
return scored;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 一路检索结果:name(路名)+ items(按排名顺序)+ weight(可选加权,≤0 视为等权)。 */
|
|
||||||
public record RankedPath<T>(String name, List<T> items, double weight) {
|
|
||||||
public RankedPath {
|
|
||||||
Objects.requireNonNull(name, "name");
|
|
||||||
items = items == null ? List.of() : List.copyOf(items);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 融合后的单个 item:identity + 原始 item + rrfScore + 每路排名明细。 */
|
|
||||||
public record Scored<T>(String identity, T item, double rrfScore, Map<String, Integer> ranks) {
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 跨路累加器:同一 identity 的 item 累加 RRF 分并记录各路排名。 */
|
|
||||||
private static final class Acc<T> {
|
|
||||||
private final T item;
|
|
||||||
private double score;
|
|
||||||
private final Map<String, Integer> ranks = new HashMap<>();
|
|
||||||
|
|
||||||
private Acc(T item) {
|
|
||||||
this.item = item;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
package com.superbiz.agent.service.retrieval;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.service.VectorSearchService;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* {@link KnowledgeSearchPort} 适配器:把向量检索结果映射为带 evidenceKey 的命中结构。
|
|
||||||
*
|
|
||||||
* <p>委托 {@link VectorSearchService}(背后仅 {@code MilvusHybridKnowledgeStore}):
|
|
||||||
* dense 或 dense+BM25 hybrid 由配置 {@code retrieval.search.mode} 选择。
|
|
||||||
* 本类负责 metadata 解析、docId/chunk 身份与 evidenceKey,不碰 SDK。</p>
|
|
||||||
*/
|
|
||||||
@Component
|
|
||||||
public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
|
||||||
|
|
||||||
private final VectorSearchService vectorSearchService;
|
|
||||||
private final ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
|
|
||||||
this.vectorSearchService = vectorSearchService;
|
|
||||||
this.objectMapper = objectMapper;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
|
||||||
// Mode is global on VectorSearchService; request.mode is advisory for future per-call overrides.
|
|
||||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(
|
|
||||||
request.query(),
|
|
||||||
request.topK(),
|
|
||||||
request.categoryFilter());
|
|
||||||
return toHits(results);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<KnowledgeSearchHit> toHits(List<VectorSearchService.SearchResult> results) {
|
|
||||||
if (results == null || results.isEmpty()) {
|
|
||||||
return List.of();
|
|
||||||
}
|
|
||||||
List<KnowledgeSearchHit> hits = new ArrayList<>(results.size());
|
|
||||||
for (int i = 0; i < results.size(); i++) {
|
|
||||||
hits.add(toHit(results.get(i), i + 1));
|
|
||||||
}
|
|
||||||
return hits;
|
|
||||||
}
|
|
||||||
|
|
||||||
private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
|
|
||||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
|
||||||
String docId = EvidenceIdentity.extractDocId(
|
|
||||||
metadata,
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "source"));
|
|
||||||
Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
|
|
||||||
String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
|
|
||||||
String source = EvidenceIdentity.firstNonBlank(
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "source"),
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "filePath"),
|
|
||||||
docId,
|
|
||||||
result.getId());
|
|
||||||
return new KnowledgeSearchHit(
|
|
||||||
result.getId(),
|
|
||||||
result.getContent(),
|
|
||||||
(double) result.getScore(),
|
|
||||||
result.getRawScore(),
|
|
||||||
result.getScoreLabel(),
|
|
||||||
result.getMetadata(),
|
|
||||||
metadata,
|
|
||||||
docId,
|
|
||||||
chunkIndex,
|
|
||||||
evidenceKey,
|
|
||||||
source,
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "title"),
|
|
||||||
EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
|
|
||||||
originalRank,
|
|
||||||
result.getDenseDistance()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Map<String, String> parseMetadata(String metadata) {
|
|
||||||
if (metadata == null || metadata.isBlank()) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
|
||||||
Map<String, String> parsed = new LinkedHashMap<>();
|
|
||||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
|
||||||
if (entry.getKey() != null && entry.getValue() != null) {
|
|
||||||
parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parsed;
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
return Map.of();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
package com.superbiz.agent.tool;
|
|
||||||
|
|
||||||
import io.milvus.client.MilvusServiceClient;
|
|
||||||
import io.milvus.param.ConnectParam;
|
|
||||||
import io.milvus.param.R;
|
|
||||||
import io.milvus.param.RpcStatus;
|
|
||||||
import io.milvus.param.collection.DropCollectionParam;
|
|
||||||
import io.milvus.param.collection.HasCollectionParam;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 删除 Milvus Collection 的工具类
|
|
||||||
* 用于重建 Collection 时清理旧数据
|
|
||||||
*/
|
|
||||||
public class DropCollection {
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
MilvusServiceClient client = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
// 连接到 Milvus
|
|
||||||
System.out.println("正在连接到 Milvus localhost:19530...");
|
|
||||||
client = new MilvusServiceClient(
|
|
||||||
ConnectParam.newBuilder()
|
|
||||||
.withHost("localhost")
|
|
||||||
.withPort(19530)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
System.out.println("✓ 连接成功");
|
|
||||||
|
|
||||||
String collectionName = "biz";
|
|
||||||
|
|
||||||
// 检查 Collection 是否存在
|
|
||||||
R<Boolean> hasResponse = client.hasCollection(
|
|
||||||
HasCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
|
|
||||||
if (hasResponse.getData()) {
|
|
||||||
System.out.println("发现 Collection: " + collectionName);
|
|
||||||
System.out.println("正在删除...");
|
|
||||||
|
|
||||||
// 删除 Collection
|
|
||||||
R<RpcStatus> dropResponse = client.dropCollection(
|
|
||||||
DropCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(collectionName)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
|
|
||||||
if (dropResponse.getStatus() == 0) {
|
|
||||||
System.out.println("✓ Collection 已成功删除");
|
|
||||||
System.out.println("\n请重启 Spring Boot 应用,它会自动创建新的 FloatVector Collection");
|
|
||||||
} else {
|
|
||||||
System.err.println("✗ 删除失败: " + dropResponse.getMessage());
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
System.out.println("Collection '" + collectionName + "' 不存在");
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.err.println("错误: " + e.getMessage());
|
|
||||||
e.printStackTrace();
|
|
||||||
} finally {
|
|
||||||
if (client != null) {
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import com.superbiz.agent.dto.RetrievalTrace;
|
|||||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
|
||||||
import com.superbiz.agent.service.LookupResultAssembler;
|
import com.superbiz.agent.service.LookupResultAssembler;
|
||||||
import jakarta.annotation.PostConstruct;
|
import jakarta.annotation.PostConstruct;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -29,10 +28,9 @@ import java.util.Map;
|
|||||||
*
|
*
|
||||||
* <h3>主链路</h3>
|
* <h3>主链路</h3>
|
||||||
* <pre>
|
* <pre>
|
||||||
* query
|
* query(原始句直传;L0 query 理解已下沉 py-rag 服务端)
|
||||||
* -> KnowledgeQueryTransformer
|
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort → py-rag, retrieve-k)
|
||||||
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k)
|
* -> KnowledgeEvidencePostProcessor (qualityScore / chunk dedup / caps / return-n)
|
||||||
* -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
|
|
||||||
* -> [optional] unfiltered retry
|
* -> [optional] unfiltered retry
|
||||||
* -> KnowledgeContextPacker
|
* -> KnowledgeContextPacker
|
||||||
* -> LookupResultAssembler
|
* -> LookupResultAssembler
|
||||||
@@ -62,9 +60,6 @@ public class LookupKnowledgeTool {
|
|||||||
|
|
||||||
private int retrieveK = 20;
|
private int retrieveK = 20;
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private KnowledgeQueryTransformer queryTransformer;
|
|
||||||
|
|
||||||
@Autowired
|
@Autowired
|
||||||
private KnowledgeDocumentRetriever documentRetriever;
|
private KnowledgeDocumentRetriever documentRetriever;
|
||||||
|
|
||||||
@@ -98,10 +93,10 @@ public class LookupKnowledgeTool {
|
|||||||
*
|
*
|
||||||
* <p>流程(模块化三段):
|
* <p>流程(模块化三段):
|
||||||
* <ol>
|
* <ol>
|
||||||
* <li>检索前:QueryTransformer.transform → KnowledgeQuery(分类过滤/域/关键词);</li>
|
* <li>检索前:原始 query 直传(L0 domain 分析已下沉 py-rag,categoryFilter 恒为 null);</li>
|
||||||
* <li>检索:DocumentRetriever.retrieve(FILTERED 或 UNFILTERED,retrieveK 候选);</li>
|
* <li>检索:DocumentRetriever.retrieve(UNFILTERED,retrieveK 候选,py-rag 服务端融合+精排);</li>
|
||||||
* <li>检索后:PostProcessor.process(qualityScore/去重/判级);</li>
|
* <li>检索后:PostProcessor.process(qualityScore/去重/判级);</li>
|
||||||
* <li>低质量降级:带分类过滤结果低质 → 去掉过滤、用原始 query 重查;</li>
|
* <li>低质量降级:带分类过滤结果低质 → 去掉过滤、用原始 query 重查(L0 移除后保留兜底语义);</li>
|
||||||
* <li>打包 + 组装:ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li>
|
* <li>打包 + 组装:ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li>
|
||||||
* </ol>
|
* </ol>
|
||||||
*
|
*
|
||||||
@@ -113,12 +108,17 @@ public class LookupKnowledgeTool {
|
|||||||
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
|
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
|
||||||
log.info("----------------------------------------");
|
log.info("----------------------------------------");
|
||||||
|
|
||||||
// ── 检索前:查询理解(L0)──
|
// ── 检索前:原始 query 直传(L0 已下沉 py-rag,不做 Java 侧 category 收窄)──
|
||||||
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
|
String normalized = query == null ? "" : query.trim();
|
||||||
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
|
KnowledgeQuery knowledgeQuery = KnowledgeQuery.builder()
|
||||||
knowledgeQuery.getCategoryFilter(),
|
.originalQuery(normalized)
|
||||||
knowledgeQuery.getDomainHints().size(),
|
.rewrittenQuery(normalized)
|
||||||
knowledgeQuery.getMatchedKeywords().size());
|
.domainHints(List.of())
|
||||||
|
.matchedKeywords(List.of())
|
||||||
|
.entities(List.of())
|
||||||
|
.l0Titles(List.of())
|
||||||
|
.l0MatchCount(0)
|
||||||
|
.build();
|
||||||
|
|
||||||
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
|
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
|
||||||
String fallbackReason = null;
|
String fallbackReason = null;
|
||||||
|
|||||||
@@ -15,30 +15,28 @@ file:
|
|||||||
knowledge:
|
knowledge:
|
||||||
base-path: knowledge_base/
|
base-path: knowledge_base/
|
||||||
|
|
||||||
milvus:
|
# =====================================================
|
||||||
host: in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com
|
# py-rag 知识服务接入
|
||||||
port: 443
|
# =====================================================
|
||||||
username: ""
|
# RAG 检索与文档入库均由 py-rag 服务承担(契约见 py-rag 仓库 docs/Java接入文档.md):
|
||||||
password: ""
|
# 检索 /api/v1/search,入库 /api/v1/documents:ingest,全量重建 /api/v1/collections:rebuild。
|
||||||
database: db_4a578da0f27ce9d
|
pyrag:
|
||||||
timeout: 10000
|
base-url: ${PYRAG_BASE_URL:http://localhost:8000}
|
||||||
token: ${MILVUS_TOKEN}
|
connect-timeout-ms: 3000
|
||||||
secure: true
|
search-read-timeout-ms: 5000 # 正常 300–800ms(含 rerank 外呼)
|
||||||
vector-dim: 1024 # BGE-M3 = 1024,换模型时同步改
|
ingest-read-timeout-ms: 30000 # 正常 1–5s
|
||||||
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
|
default-read-timeout-ms: 10000
|
||||||
collection: biz
|
|
||||||
|
|
||||||
# =====================================================
|
# =====================================================
|
||||||
# 模型路由配置
|
# 模型路由配置
|
||||||
# =====================================================
|
# =====================================================
|
||||||
# 通过关键字匹配 Bean,切换模型只改这里 + 对应 api-key
|
# 通过关键字匹配 Bean,切换模型只改这里 + 对应 api-key
|
||||||
# Chat: deepseek | openai | ollama | ...
|
# Chat: deepseek | openai | ollama | ...
|
||||||
# Embedding: siliconflow | openai | ollama | dashscope | ...
|
# (Embedding 已随 RAG 抽离至 py-rag 服务端)
|
||||||
# =====================================================
|
# =====================================================
|
||||||
|
|
||||||
model-routing:
|
model-routing:
|
||||||
chat: deepseek
|
chat: deepseek
|
||||||
embedding: siliconflow
|
|
||||||
|
|
||||||
spring:
|
spring:
|
||||||
config:
|
config:
|
||||||
@@ -102,30 +100,6 @@ spring:
|
|||||||
retry:
|
retry:
|
||||||
max-attempts: 1
|
max-attempts: 1
|
||||||
|
|
||||||
vectorstore:
|
|
||||||
type: milvus
|
|
||||||
milvus:
|
|
||||||
initialize-schema: false
|
|
||||||
database-name: ${milvus.database}
|
|
||||||
collection-name: biz
|
|
||||||
embedding-dimension: ${milvus.vector-dim}
|
|
||||||
index-type: IVF_FLAT
|
|
||||||
metric-type: L2
|
|
||||||
index-parameters: '{"nlist":128}'
|
|
||||||
id-field-name: id
|
|
||||||
auto-id: false
|
|
||||||
content-field-name: content
|
|
||||||
metadata-field-name: metadata
|
|
||||||
embedding-field-name: vector
|
|
||||||
client:
|
|
||||||
host: ${milvus.host}
|
|
||||||
port: ${milvus.port}
|
|
||||||
token: ${milvus.token}
|
|
||||||
username: ${milvus.username}
|
|
||||||
password: ${milvus.password}
|
|
||||||
secure: ${milvus.secure}
|
|
||||||
connect-timeout-ms: ${milvus.timeout}
|
|
||||||
|
|
||||||
# --- Chat: DeepSeek (原生) ---
|
# --- Chat: DeepSeek (原生) ---
|
||||||
deepseek:
|
deepseek:
|
||||||
api-key: ${DEEPSEEK_API_KEY}
|
api-key: ${DEEPSEEK_API_KEY}
|
||||||
@@ -134,53 +108,27 @@ spring:
|
|||||||
options:
|
options:
|
||||||
model: deepseek-v4-flash
|
model: deepseek-v4-flash
|
||||||
|
|
||||||
# --- OpenAI 模块供 SiliconFlow Embedding 复用 ---
|
|
||||||
openai:
|
|
||||||
api-key: unused
|
|
||||||
|
|
||||||
# Spring AI MCP 客户端配置
|
# Spring AI MCP 客户端配置
|
||||||
mcp:
|
mcp:
|
||||||
client:
|
client:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
# --- Embedding: SiliconFlow BGE-M3 ---
|
|
||||||
siliconflow:
|
|
||||||
api-key: ${SILICONFLOW_API_KEY}
|
|
||||||
base-url: https://api.siliconflow.cn
|
|
||||||
embedding:
|
|
||||||
model: BAAI/bge-m3
|
|
||||||
|
|
||||||
# 文档分片配置
|
|
||||||
document:
|
|
||||||
chunk:
|
|
||||||
max-size: 800
|
|
||||||
overlap: 100
|
|
||||||
|
|
||||||
# RAG 配置
|
# RAG 配置
|
||||||
rag:
|
rag:
|
||||||
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
||||||
retrieve-k: 20
|
retrieve-k: 20
|
||||||
return-n: 5
|
return-n: 5
|
||||||
max-chunks-per-document: 2
|
max-chunks-per-document: 2
|
||||||
sidecar:
|
|
||||||
spring-ai:
|
|
||||||
enabled: false
|
|
||||||
content-preview-limit: 300
|
|
||||||
|
|
||||||
# 检索配置
|
# 检索配置
|
||||||
# 知识主路径:Milvus Java SDK v2(MilvusHybridKnowledgeStore),非 Spring AI VectorStore starter。
|
# 知识主路径:py-rag 知识服务(PyRagKnowledgeSearchAdapter → /api/v1/search)。
|
||||||
# 原因:starter(含 2.0.0)仅 dense similarity,无 hybridSearch / BM25 Function / RRFRanker。
|
# 服务端负责 dense+BM25 融合、rerank(BGE-Reranker)与判级;Java 侧只做请求映射与后处理。
|
||||||
# 已移除 legacy sdk/spring/auto 多后端路由。
|
|
||||||
retrieval:
|
retrieval:
|
||||||
kb-scope: "" # 非空则过滤 metadata.kb_scope;空=不过滤
|
|
||||||
search:
|
search:
|
||||||
# hybrid=线上主路径;dense=同库对照/评测/排障(非第二套线上策略)。见 mvp/architecture/rag-knowledge-retrieval-architecture.md §6.0
|
# hybrid=线上主路径;dense 为对照/排障(映射 py-rag mode:hybrid→hybrid,dense→semantic)
|
||||||
mode: hybrid # dense=单路L2对照 | hybrid=dense+服务端BM25+RRF
|
mode: hybrid
|
||||||
hybrid:
|
|
||||||
rrf-k: 60 # RRF 平滑参数 k,score=Σ 1/(k+rank)
|
|
||||||
normalization:
|
normalization:
|
||||||
max-l2-distance: 2.0 # dense quality:L2 上界(单位向量 ≈ 2.0)
|
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE(与 py-rag 判级阈值一致)
|
||||||
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE(hybrid 为序数分,见架构 §6)
|
|
||||||
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE;低于则低质/可 unfiltered retry
|
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE;低于则低质/可 unfiltered retry
|
||||||
|
|
||||||
# Prometheus 配置
|
# Prometheus 配置
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import com.superbiz.agent.tool.LookupKnowledgeTool;
|
|||||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
|
||||||
import com.superbiz.agent.service.LookupResultAssembler;
|
import com.superbiz.agent.service.LookupResultAssembler;
|
||||||
import com.superbiz.agent.repository.AgentStepRepository;
|
import com.superbiz.agent.repository.AgentStepRepository;
|
||||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||||
@@ -84,7 +83,6 @@ class HarnessChatConfigurationTest {
|
|||||||
.withBean(ChatModel.class, () -> mock(ChatModel.class))
|
.withBean(ChatModel.class, () -> mock(ChatModel.class))
|
||||||
.withBean(RedisTemplate.class, () -> mock(RedisTemplate.class))
|
.withBean(RedisTemplate.class, () -> mock(RedisTemplate.class))
|
||||||
.withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class))
|
.withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class))
|
||||||
.withBean(KnowledgeQueryTransformer.class, () -> mock(KnowledgeQueryTransformer.class))
|
|
||||||
.withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class))
|
.withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class))
|
||||||
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
|
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
|
||||||
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
|
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
|
||||||
|
|||||||
@@ -4,18 +4,13 @@ import org.junit.jupiter.api.Test;
|
|||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
import org.springframework.data.redis.core.RedisTemplate;
|
import org.springframework.data.redis.core.RedisTemplate;
|
||||||
import org.springframework.test.context.TestPropertySource;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 单独测试 Redis 连接
|
* 单独测试 Redis 连接(需要可达的 Redis 实例)。
|
||||||
* 禁用 Milvus 以避免启动失败
|
|
||||||
*/
|
*/
|
||||||
@SpringBootTest
|
@SpringBootTest
|
||||||
@TestPropertySource(properties = {
|
|
||||||
"spring.autoconfigure.exclude=org.example.config.MilvusConfig"
|
|
||||||
})
|
|
||||||
class RedisConnectionTest {
|
class RedisConnectionTest {
|
||||||
|
|
||||||
@Autowired(required = false)
|
@Autowired(required = false)
|
||||||
|
|||||||
@@ -1,93 +0,0 @@
|
|||||||
package com.superbiz.agent.eval;
|
|
||||||
|
|
||||||
import com.superbiz.agent.Main;
|
|
||||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
|
||||||
import com.superbiz.agent.dto.DocumentUploadRequest;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
|
||||||
import com.superbiz.agent.service.DocumentManagementService;
|
|
||||||
import com.superbiz.agent.service.FrontmatterParser;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
|
||||||
import org.springframework.mock.web.MockMultipartFile;
|
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Imports canonical RAG eval documents through the real document pipeline.
|
|
||||||
*
|
|
||||||
* <p>Disabled by default because it writes DB rows, local knowledge files, and
|
|
||||||
* vector index records in the configured runtime environment.</p>
|
|
||||||
*/
|
|
||||||
@SpringBootTest(
|
|
||||||
classes = Main.class,
|
|
||||||
webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
|
||||||
properties = "spring.main.web-application-type=none"
|
|
||||||
)
|
|
||||||
@EnabledIfSystemProperty(named = "rag.seed.enabled", matches = "true")
|
|
||||||
class RagEvalSeedImporterTest {
|
|
||||||
|
|
||||||
private static final Path DEFAULT_SEED_DOCS = Path.of("eval/rag-retrieval/seed-docs");
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private DocumentManagementService documentManagementService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private FrontmatterParser frontmatterParser;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ApiDocumentRepository apiDocumentRepository;
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void importSeedDocuments() throws Exception {
|
|
||||||
Path seedDir = Path.of(System.getProperty("rag.seed.docs", DEFAULT_SEED_DOCS.toString()));
|
|
||||||
List<Path> docs;
|
|
||||||
try (var stream = Files.list(seedDir)) {
|
|
||||||
docs = stream
|
|
||||||
.filter(path -> path.getFileName().toString().endsWith(".md"))
|
|
||||||
.sorted()
|
|
||||||
.toList();
|
|
||||||
}
|
|
||||||
assertFalse(docs.isEmpty(), "seed docs directory must contain markdown files");
|
|
||||||
|
|
||||||
for (Path docPath : docs) {
|
|
||||||
String content = Files.readString(docPath, StandardCharsets.UTF_8);
|
|
||||||
Frontmatter frontmatter = frontmatterParser.parse(content);
|
|
||||||
if (frontmatter == null || frontmatter.getSource() == null || frontmatter.getSource().isBlank()) {
|
|
||||||
throw new IllegalArgumentException("seed doc must include frontmatter source: " + docPath);
|
|
||||||
}
|
|
||||||
|
|
||||||
apiDocumentRepository.findByDocId(frontmatter.getSource().trim())
|
|
||||||
.map(ApiDocument::getDocId)
|
|
||||||
.ifPresent(documentManagementService::deleteDocument);
|
|
||||||
|
|
||||||
String fileName = docPath.getFileName().toString();
|
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
|
||||||
"file",
|
|
||||||
fileName,
|
|
||||||
"text/markdown",
|
|
||||||
content.getBytes(StandardCharsets.UTF_8)
|
|
||||||
);
|
|
||||||
DocumentUploadRequest request = DocumentUploadRequest.builder()
|
|
||||||
.file(file)
|
|
||||||
.category(resolveCategory(frontmatter))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
documentManagementService.uploadDocument(request);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private String resolveCategory(Frontmatter frontmatter) {
|
|
||||||
if (frontmatter.getCategory() != null && !frontmatter.getCategory().isBlank()) {
|
|
||||||
return frontmatter.getCategory().trim();
|
|
||||||
}
|
|
||||||
return "rag-eval";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
package com.superbiz.agent.eval;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
|
||||||
import com.superbiz.agent.Main;
|
|
||||||
import com.superbiz.agent.dto.LookupResult;
|
|
||||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
|
||||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
|
||||||
import org.springframework.test.context.DynamicPropertySource;
|
|
||||||
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.time.Instant;
|
|
||||||
import java.util.Locale;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
|
|
||||||
*
|
|
||||||
* <p>Disabled by default: writes repository files and needs the live retrieval stack
|
|
||||||
* (embedding + Milvus hybrid collection + optional MySQL/L0).</p>
|
|
||||||
*
|
|
||||||
* <p>System properties (via Maven {@code -D}):</p>
|
|
||||||
* <ul>
|
|
||||||
* <li>{@code rag.snapshot.enabled=true} — required to run</li>
|
|
||||||
* <li>{@code retrieval.search.mode=hybrid|dense} — default hybrid</li>
|
|
||||||
* <li>{@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})</li>
|
|
||||||
* <li>{@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}</li>
|
|
||||||
* </ul>
|
|
||||||
*/
|
|
||||||
@SpringBootTest(
|
|
||||||
classes = Main.class,
|
|
||||||
webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
|
||||||
properties = "spring.main.web-application-type=none"
|
|
||||||
)
|
|
||||||
@EnabledIfSystemProperty(named = "rag.snapshot.enabled", matches = "true")
|
|
||||||
class RagLookupSnapshotGeneratorTest {
|
|
||||||
|
|
||||||
private static final Path DEFAULT_CASES = Path.of("eval/rag-retrieval/cases/golden-cases.json");
|
|
||||||
private static final Path DEFAULT_FIXTURES = Path.of("eval/rag-retrieval/fixtures");
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private LookupKnowledgeTool lookupKnowledgeTool;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ObjectMapper objectMapper;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
|
|
||||||
*/
|
|
||||||
@DynamicPropertySource
|
|
||||||
static void retrievalProperties(DynamicPropertyRegistry registry) {
|
|
||||||
String mode = System.getProperty("retrieval.search.mode", "hybrid");
|
|
||||||
if (mode == null || mode.isBlank()) {
|
|
||||||
mode = "hybrid";
|
|
||||||
}
|
|
||||||
String normalized = mode.trim().toLowerCase(Locale.ROOT);
|
|
||||||
registry.add("retrieval.search.mode", () -> normalized);
|
|
||||||
|
|
||||||
String kbScope = System.getProperty("retrieval.kb-scope", "");
|
|
||||||
if (kbScope != null && !kbScope.isBlank()) {
|
|
||||||
registry.add("retrieval.kb-scope", kbScope::trim);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void generateLookupResultFixtures() throws Exception {
|
|
||||||
Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
|
|
||||||
Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
|
|
||||||
String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
|
|
||||||
String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
|
|
||||||
String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
|
|
||||||
|
|
||||||
JsonNode root = objectMapper.readTree(casesPath.toFile());
|
|
||||||
JsonNode cases = root.path("cases");
|
|
||||||
assertTrue(cases.isArray(), "golden cases file must contain a cases array");
|
|
||||||
|
|
||||||
Files.createDirectories(fixturesDir);
|
|
||||||
for (JsonNode testCase : cases) {
|
|
||||||
String caseId = requiredText(testCase, "caseId");
|
|
||||||
String query = requiredText(testCase, "query");
|
|
||||||
|
|
||||||
LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
|
|
||||||
|
|
||||||
ObjectNode fixture = objectMapper.createObjectNode();
|
|
||||||
fixture.put("caseId", caseId);
|
|
||||||
fixture.put("query", query);
|
|
||||||
fixture.put("retrievedAt", retrievedAt);
|
|
||||||
fixture.put("searchMode", searchMode);
|
|
||||||
if (kbScope != null) {
|
|
||||||
fixture.put("kbScope", kbScope);
|
|
||||||
}
|
|
||||||
fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
|
|
||||||
|
|
||||||
Path output = fixturesDir.resolve(caseId + ".json");
|
|
||||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), fixture);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String normalizeMode(String mode) {
|
|
||||||
if (mode == null || mode.isBlank()) {
|
|
||||||
return "hybrid";
|
|
||||||
}
|
|
||||||
return mode.trim().toLowerCase(Locale.ROOT);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String blankToNull(String value) {
|
|
||||||
if (value == null || value.isBlank()) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return value.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
private String requiredText(JsonNode node, String fieldName) {
|
|
||||||
JsonNode value = node.get(fieldName);
|
|
||||||
if (value == null || value.asText().isBlank()) {
|
|
||||||
throw new IllegalArgumentException("golden case is missing required field: " + fieldName);
|
|
||||||
}
|
|
||||||
return value.asText();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,539 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.config.DocumentChunkConfig;
|
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.DisplayName;
|
|
||||||
import org.junit.jupiter.api.Nested;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 当前分片策略的单元测试 — 覆盖旧能力回归 + Phase 1 新增能力
|
|
||||||
*/
|
|
||||||
@DisplayName("DocumentChunkService 分片策略")
|
|
||||||
class DocumentChunkServiceTest {
|
|
||||||
|
|
||||||
private DocumentChunkService service;
|
|
||||||
private DocumentChunkConfig config;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
config = new DocumentChunkConfig();
|
|
||||||
config.setMaxSize(800);
|
|
||||||
config.setMaxTokens(500);
|
|
||||||
config.setMaxTokensHard(600);
|
|
||||||
config.setOverlap(100);
|
|
||||||
service = new DocumentChunkService();
|
|
||||||
try {
|
|
||||||
var field = DocumentChunkService.class.getDeclaredField("chunkConfig");
|
|
||||||
field.setAccessible(true);
|
|
||||||
field.set(service, config);
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 回归:边界条件 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("边界条件")
|
|
||||||
class BoundaryTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("null 内容 → 空列表")
|
|
||||||
void nullContent_returnsEmpty() {
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(null, "/test/null.md");
|
|
||||||
assertTrue(chunks.isEmpty());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("空字符串 → 空列表")
|
|
||||||
void emptyContent_returnsEmpty() {
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(" \n ", "/test/empty.md");
|
|
||||||
assertTrue(chunks.isEmpty());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("短文档(≤maxSize)→ 1个分块")
|
|
||||||
void shortDocument_singleChunk() {
|
|
||||||
String content = "这是一篇短文档,内容不超过800个字符。";
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/short.md");
|
|
||||||
|
|
||||||
assertEquals(1, chunks.size());
|
|
||||||
assertEquals(content, chunks.get(0).getContent());
|
|
||||||
assertEquals(0, chunks.get(0).getChunkIndex());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("恰好 maxSize 边界 → 1个分块")
|
|
||||||
void exactlyMaxSize_singleChunk() {
|
|
||||||
String content = "A".repeat(800);
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/boundary.md");
|
|
||||||
assertEquals(1, chunks.size());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 回归:标题分割 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("Markdown 标题分割")
|
|
||||||
class HeadingSplitTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("单个 H1 标题 → section 继承标题")
|
|
||||||
void singleHeading_titlePropagates() {
|
|
||||||
String content = "# CPU高负载问题\n\n这是CPU高负载的描述内容。";
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/cpu.md");
|
|
||||||
|
|
||||||
assertEquals(1, chunks.size());
|
|
||||||
assertEquals("CPU高负载问题", chunks.get(0).getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("多个标题 → 按标题边界分割")
|
|
||||||
void multipleHeadings_splitAtHeadings() {
|
|
||||||
String content =
|
|
||||||
"# CPU高负载\n\nCPU问题的详细描述。\n\n" +
|
|
||||||
"# 内存高负载\n\n内存问题的详细描述。";
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/multi.md");
|
|
||||||
|
|
||||||
assertEquals(2, chunks.size());
|
|
||||||
assertEquals("CPU高负载", chunks.get(0).getTitle());
|
|
||||||
assertEquals("内存高负载", chunks.get(1).getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("多级标题(H1/H2/H3)→ 标题独立不冲突")
|
|
||||||
void multiLevelHeadings() {
|
|
||||||
String content =
|
|
||||||
"# 一级标题\n\n一级内容。\n\n" +
|
|
||||||
"## 二级标题\n\n二级内容。\n\n" +
|
|
||||||
"### 三级标题\n\n三级内容。";
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/levels.md");
|
|
||||||
assertEquals(3, chunks.size());
|
|
||||||
assertEquals("一级标题", chunks.get(0).getTitle());
|
|
||||||
assertEquals("二级标题", chunks.get(1).getTitle());
|
|
||||||
assertEquals("三级标题", chunks.get(2).getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("H1-H6 全部支持")
|
|
||||||
void allHeadingLevels() {
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
for (int i = 1; i <= 6; i++) {
|
|
||||||
sb.append("#".repeat(i)).append(" 标题").append(i).append("\n\n内容").append(i).append("。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/h1h6.md");
|
|
||||||
assertEquals(6, chunks.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("无标题文档 → 整个文档作为1个 section")
|
|
||||||
void noHeadings_entireAsOneSection() {
|
|
||||||
String content = "纯文本没有标题。\n\n第二段内容。\n\n第三段内容。";
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/nohead.md");
|
|
||||||
assertFalse(chunks.isEmpty());
|
|
||||||
assertNull(chunks.get(0).getTitle());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 回归:段落边界切分 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("超长章节 — 段落边界切分")
|
|
||||||
class ParagraphSplitTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("短章节(≤maxSize)→ 不进入段落切割")
|
|
||||||
void shortSection_noParagraphSplit() {
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 测试\n\n");
|
|
||||||
for (int i = 0; i < 5; i++) {
|
|
||||||
sb.append("段落").append(i).append(":这是一段短内容。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/short_sec.md");
|
|
||||||
assertEquals(1, chunks.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("超长章节 → 在段落边界切分")
|
|
||||||
void longSection_splitsAtParagraphBoundaries() {
|
|
||||||
config.setMaxSize(50);
|
|
||||||
config.setMaxTokens(30);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 长章节\n\n");
|
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
sb.append("段落").append(i).append(":ABCDEFGHIJKLMNOPQRSTUVWXYZ。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_sec.md");
|
|
||||||
assertTrue(chunks.size() >= 2, "超长章节应切分为多个分块,实际: " + chunks.size());
|
|
||||||
|
|
||||||
// 所有分块携带相同的 title
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
assertEquals("长章节", c.getTitle());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 回归:chunkIndex 元数据 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("分块元数据")
|
|
||||||
class ChunkMetadataTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("chunkIndex 自增且唯一")
|
|
||||||
void chunkIndexSequential() {
|
|
||||||
config.setMaxSize(50);
|
|
||||||
config.setMaxTokens(30);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder("# Meta\n\n");
|
|
||||||
for (int i = 0; i < 10; i++) {
|
|
||||||
sb.append("段落").append(i).append(":填充内容以触发切分机制。ABCDE。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/meta.md");
|
|
||||||
assertTrue(chunks.size() >= 2);
|
|
||||||
|
|
||||||
for (int i = 0; i < chunks.size(); i++) {
|
|
||||||
assertEquals(i, chunks.get(i).getChunkIndex(),
|
|
||||||
"chunkIndex 应从0开始连续递增");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("startIndex/endIndex 范围合法 — 无漂移")
|
|
||||||
void indexRangeValid_noDrift() {
|
|
||||||
String content = "# 标题\n\n测试内容。";
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/index.md");
|
|
||||||
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
assertTrue(c.getStartOffset() >= 0);
|
|
||||||
assertTrue(c.getEndOffset() > c.getStartOffset(),
|
|
||||||
"endIndex(" + c.getEndOffset() + ") 应 > startIndex(" + c.getStartOffset() + ")");
|
|
||||||
assertTrue(c.getEndOffset() <= content.length());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 新增:Token 估算 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("Token 估算")
|
|
||||||
class TokenEstimationTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("纯中文 800 字符 ≈ 800 tokens → 短章节不切")
|
|
||||||
void pureChinese_fewerTokensThanMax() {
|
|
||||||
config.setMaxTokens(400);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 中文测试\n\n");
|
|
||||||
// 纯中文 ~300 字符 ≈ 300 tokens
|
|
||||||
for (int i = 0; i < 3; i++) {
|
|
||||||
sb.append("这是纯中文测试内容的第十").append(i).append("段落。");
|
|
||||||
sb.append("每个中文字符大约占用一个令牌的位置。");
|
|
||||||
sb.append("因此这段文本的令牌数大致等于字符数。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/cn_tokens.md");
|
|
||||||
// 300 字符 ≈ 300 tokens < 400 maxTokens → 1 个分块
|
|
||||||
assertEquals(1, chunks.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("纯英文 2000 字符 ≈ 500 tokens → 刚好不超过上限")
|
|
||||||
void pureEnglish_moreCharactersSameTokens() {
|
|
||||||
config.setMaxTokens(200);
|
|
||||||
config.setMaxTokensHard(250);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# English Test\n\n");
|
|
||||||
for (int i = 0; i < 8; i++) {
|
|
||||||
sb.append("This is paragraph number ").append(i)
|
|
||||||
.append(" containing English text. ")
|
|
||||||
.append("English characters are much cheaper in tokens. ")
|
|
||||||
.append("More filler text here to reach the limit properly. ")
|
|
||||||
.append("Yet another sentence for good measure. ")
|
|
||||||
.append("Still more words needed to reach token limit here.\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/en_tokens.md");
|
|
||||||
// 大量英文才占少量 token → 分块数应少于用字符计数的版本
|
|
||||||
assertTrue(chunks.size() >= 2, "1200+ 字符英文应切分");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 新增:列表结构感知 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("列表结构感知")
|
|
||||||
class ListStructureTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("有序列表项之间不切分 — 即使超过 maxTokens")
|
|
||||||
void orderedList_notSplitBetweenItems() {
|
|
||||||
config.setMaxTokens(80);
|
|
||||||
config.setMaxTokensHard(200);
|
|
||||||
config.setOverlap(30);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 排查步骤\n\n");
|
|
||||||
// 5个有序列表项,每项 ~40 字符 ≈ 40 tokens,总共 ~200 tokens
|
|
||||||
for (int i = 1; i <= 5; i++) {
|
|
||||||
sb.append(i).append(". 这是排查步骤第").append(i)
|
|
||||||
.append("项,包含具体的操作指引和注意事项说明。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/ordered_list.md");
|
|
||||||
|
|
||||||
// 5项应保持在一起(未触及 hard 上限)
|
|
||||||
assertEquals(1, chunks.size(),
|
|
||||||
"有序列表项不应被拆散,实际分块数: " + chunks.size());
|
|
||||||
|
|
||||||
String content = chunks.get(0).getContent();
|
|
||||||
assertTrue(content.contains("1. "), "应包含第1项");
|
|
||||||
assertTrue(content.contains("5. "), "应包含第5项");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("有序列表触及硬上限 → 在列表项边界强制切分")
|
|
||||||
void orderedList_hardLimitSplits() {
|
|
||||||
config.setMaxTokens(50);
|
|
||||||
config.setMaxTokensHard(100);
|
|
||||||
config.setOverlap(20);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 长列表\n\n");
|
|
||||||
// 每项 ~60 tokens,硬上限 100 → 最多装 1 项多
|
|
||||||
for (int i = 1; i <= 6; i++) {
|
|
||||||
sb.append(i).append(". 这是很长的排查步骤内容,包含详细的说明信息。")
|
|
||||||
.append("每个步骤都要执行多个检查操作。继续填充文本以增加令牌计数。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/long_list.md");
|
|
||||||
|
|
||||||
System.out.println(" 长列表硬上限测试 — 实际分块数: " + chunks.size());
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
System.out.println(" Chunk #" + c.getChunkIndex() + ": " + c.getContent().length() + "字符 "
|
|
||||||
+ "| start=" + c.getStartOffset() + " end=" + c.getEndOffset()
|
|
||||||
+ " | preview=" + c.getContent().substring(0, Math.min(60, c.getContent().length())).replace("\n", "\\n"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 硬上限会强制切分,但每个分块内的列表项应保持连续
|
|
||||||
assertTrue(chunks.size() >= 2, "长列表应至少触发1次切分,实际: " + chunks.size());
|
|
||||||
|
|
||||||
// 验证:除了第一个分块(可能是标题),其余应包含列表项
|
|
||||||
for (int i = 1; i < chunks.size(); i++) {
|
|
||||||
DocumentChunk c = chunks.get(i);
|
|
||||||
assertFalse(c.getContent().isEmpty());
|
|
||||||
assertTrue(c.getContent().matches("(?s).*\\d+\\.\\s.*"),
|
|
||||||
"非标题分块应包含列表项,Chunk #" + c.getChunkIndex()
|
|
||||||
+ " preview: " + c.getContent().substring(0, Math.min(60, c.getContent().length())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("无序列表项之间不切分")
|
|
||||||
void unorderedList_notSplitBetweenItems() {
|
|
||||||
config.setMaxTokens(80);
|
|
||||||
config.setMaxTokensHard(200);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 检查清单\n\n");
|
|
||||||
for (int i = 1; i <= 5; i++) {
|
|
||||||
sb.append("- 检查项").append(i).append(":确认服务运行状态正常并记录相关指标。\n\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/unordered_list.md");
|
|
||||||
assertEquals(1, chunks.size(), "无序列表项不应被拆散");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("列表结束后普通段落应从下一段落开始新分块")
|
|
||||||
void listEnds_normalParagraphStartsNewChunk() {
|
|
||||||
config.setMaxTokens(150);
|
|
||||||
config.setMaxTokensHard(250);
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("# 文档\n\n");
|
|
||||||
// 先一个普通段落
|
|
||||||
sb.append("这是介绍段落,描述系统的整体架构和设计思路。\n\n");
|
|
||||||
// 有序列表
|
|
||||||
for (int i = 1; i <= 3; i++) {
|
|
||||||
sb.append(i).append(". 列表项第").append(i).append("条,包含操作说明。\n\n");
|
|
||||||
}
|
|
||||||
// 普通段落
|
|
||||||
sb.append("这是总结段落,包含上述操作完成后需要关注的监控指标。\n\n");
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(sb.toString(), "/test/list_mixed.md");
|
|
||||||
assertTrue(chunks.size() >= 1);
|
|
||||||
// 列表项应保持在一起
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
String content = c.getContent();
|
|
||||||
// 分块中不应有孤立的单个列表项(除非只有一个)
|
|
||||||
if (content.contains("1. ") && content.contains("3. ")) {
|
|
||||||
// 这个分块包含了全部3个列表项 → 正确
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 新增:代码块结构感知 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("代码块结构感知")
|
|
||||||
class CodeBlockTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("代码块内部不切分")
|
|
||||||
void codeBlock_notSplitInside() {
|
|
||||||
config.setMaxTokens(60);
|
|
||||||
config.setMaxTokensHard(200);
|
|
||||||
config.setOverlap(20);
|
|
||||||
|
|
||||||
String content =
|
|
||||||
"# 代码示例\n\n" +
|
|
||||||
"以下是配置代码:\n\n" +
|
|
||||||
"```yaml\n" +
|
|
||||||
"server:\n" +
|
|
||||||
" port: 8080\n" +
|
|
||||||
" host: localhost\n" +
|
|
||||||
" timeout: 30s\n" +
|
|
||||||
"```\n\n" +
|
|
||||||
"配置说明结束。";
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(content, "/test/code.md");
|
|
||||||
|
|
||||||
// 代码块应保持完整(未触及硬上限)
|
|
||||||
// 验证:至少有一个分块包含完整的 ```...```
|
|
||||||
boolean foundCompleteBlock = false;
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
String text = c.getContent();
|
|
||||||
if (text.contains("```yaml") && text.contains("```") &&
|
|
||||||
text.indexOf("```yaml") < text.lastIndexOf("```")) {
|
|
||||||
foundCompleteBlock = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// 可能整体在一个分块中
|
|
||||||
assertTrue(chunks.size() >= 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== 可视化 ====================
|
|
||||||
|
|
||||||
@Nested
|
|
||||||
@DisplayName("可视化 — 打印切分结果")
|
|
||||||
class VisualInspectionTests {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("模拟运维文档 — 展示新策略效果")
|
|
||||||
void realWorldAIOpsDoc() {
|
|
||||||
config.setMaxTokens(150);
|
|
||||||
config.setMaxTokensHard(200);
|
|
||||||
config.setOverlap(40);
|
|
||||||
|
|
||||||
String doc = """
|
|
||||||
# CPU高负载问题排查指南
|
|
||||||
|
|
||||||
## 问题现象
|
|
||||||
|
|
||||||
服务器CPU使用率持续超过90%,系统响应变慢,用户反馈页面加载超时。
|
|
||||||
监控告警系统连续发出多条CPU使用率告警。
|
|
||||||
|
|
||||||
## 排查步骤
|
|
||||||
|
|
||||||
1. 登录服务器,执行 top 命令查看当前CPU使用率最高的进程。记录进程ID和CPU占用百分比。
|
|
||||||
|
|
||||||
2. 使用 ps aux | grep {进程名} 确认相关服务的运行状态。检查是否有异常进程占用资源。
|
|
||||||
|
|
||||||
3. 查看应用日志,重点关注最近15分钟的ERROR级别日志。使用 tail -n 500 命令。
|
|
||||||
|
|
||||||
4. 检查数据库连接池状态,确认是否有慢查询或连接泄漏。查看慢查询日志。
|
|
||||||
|
|
||||||
5. 检查JVM内存使用情况和GC日志。使用 jstat -gcutil {pid} 1000 命令观察GC频率。
|
|
||||||
|
|
||||||
## 常见原因
|
|
||||||
|
|
||||||
1. 死循环或递归调用导致CPU满载。检查是否有未设置退出条件的循环逻辑。
|
|
||||||
2. 大量正则表达式匹配操作。检查是否有未编译的正则在循环中使用。
|
|
||||||
|
|
||||||
## 解决方案
|
|
||||||
|
|
||||||
根据排查结果采取对应措施:代码问题则回滚或热修复;资源不足则扩容。
|
|
||||||
处理完成后持续观察监控指标30分钟,确认CPU使用率恢复正常。
|
|
||||||
""";
|
|
||||||
|
|
||||||
List<DocumentChunk> chunks = service.chunkDocument(doc, "/kb/cpu_high_usage.md");
|
|
||||||
|
|
||||||
System.out.println("========================================");
|
|
||||||
System.out.println(" Phase 1 新策略效果 — 模拟运维文档");
|
|
||||||
System.out.println(" 配置: maxTokens=150, hard=200, overlap=40");
|
|
||||||
System.out.println(" 总字符数: " + doc.length());
|
|
||||||
System.out.println(" 总分块数: " + chunks.size());
|
|
||||||
System.out.println("========================================\n");
|
|
||||||
|
|
||||||
for (DocumentChunk c : chunks) {
|
|
||||||
System.out.println("┌─ Chunk #" + c.getChunkIndex());
|
|
||||||
System.out.println("│ Title: " + (c.getTitle() != null ? c.getTitle() : "(无)"));
|
|
||||||
System.out.println("│ Range: [" + c.getStartOffset() + "→" + c.getEndOffset() + "] (" + c.getContent().length() + "字符)");
|
|
||||||
// 显示前150字符
|
|
||||||
String preview = c.getContent().length() > 120
|
|
||||||
? c.getContent().substring(0, 120).replace("\n", "\\n") + "..."
|
|
||||||
: c.getContent().replace("\n", "\\n");
|
|
||||||
System.out.println("│ Preview: " + preview);
|
|
||||||
System.out.println("└──────────────────────\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTrue(chunks.size() >= 3, "应产生多个分块");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@DisplayName("中英混排对比 — token vs 字符计数差异")
|
|
||||||
void mixedContentComparison() {
|
|
||||||
config.setMaxTokens(100);
|
|
||||||
config.setMaxTokensHard(150);
|
|
||||||
config.setOverlap(30);
|
|
||||||
|
|
||||||
String chinese = "这是中文内容示范。中文每个字符在LLM中约占用1个token。" +
|
|
||||||
"因此这段文本在上下文窗口中占用的token数较多。" +
|
|
||||||
"继续填充文字以触发切分逻辑,验证中文token估算是否合理。" +
|
|
||||||
"更多中文文本来增加令牌计数。";
|
|
||||||
|
|
||||||
String english = "This is English content. Each word may take one or two tokens. " +
|
|
||||||
"A sentence like this one actually consumes relatively few tokens compared to " +
|
|
||||||
"Chinese characters. More English text to reach the same token count as above. " +
|
|
||||||
"Still need more words because English is very efficient in tokenization. " +
|
|
||||||
"Adding even more content to make this paragraph long enough to test properly.";
|
|
||||||
|
|
||||||
List<DocumentChunk> cnChunks = service.chunkDocument("# CN\n\n" + chinese + "\n\n" + chinese, "/test/cn.md");
|
|
||||||
List<DocumentChunk> enChunks = service.chunkDocument("# EN\n\n" + english + "\n\n" + english, "/test/en.md");
|
|
||||||
|
|
||||||
System.out.println("========================================");
|
|
||||||
System.out.println(" Token 计数对比");
|
|
||||||
System.out.println(" 配置: maxTokens=100, overlap=30");
|
|
||||||
System.out.println("========================================");
|
|
||||||
System.out.println(" 中文文档: " + (chinese.length() * 2) + "字符 → " + cnChunks.size() + "个分块");
|
|
||||||
System.out.println(" 英文文档: " + (english.length() * 2) + "字符 → " + enChunks.size() + "个分块");
|
|
||||||
|
|
||||||
for (DocumentChunk c : cnChunks) {
|
|
||||||
System.out.println(" 中文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
|
|
||||||
}
|
|
||||||
for (DocumentChunk c : enChunks) {
|
|
||||||
System.out.println(" 英文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
|
|
||||||
}
|
|
||||||
System.out.println(" ★ 现在中文和英文的分块数更接近(基于 token 而非字符)");
|
|
||||||
System.out.println("========================================");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
package com.superbiz.agent.service;
|
package com.superbiz.agent.service;
|
||||||
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
import org.springframework.mock.web.MockMultipartFile;
|
import org.springframework.mock.web.MockMultipartFile;
|
||||||
@@ -39,16 +38,4 @@ class DocumentManagementServiceTest {
|
|||||||
assertEquals("payment/runbook.md", storedPath);
|
assertEquals("payment/runbook.md", storedPath);
|
||||||
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
|
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
void resolveDocumentIdUsesFrontmatterSourceWhenItFitsDatabaseColumn() {
|
|
||||||
DocumentManagementService service = new DocumentManagementService();
|
|
||||||
Frontmatter frontmatter = Frontmatter.builder()
|
|
||||||
.source("mysql-connection-pool")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
String docId = ReflectionTestUtils.invokeMethod(service, "resolveDocumentId", frontmatter);
|
|
||||||
|
|
||||||
assertEquals("mysql-connection-pool", docId);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,193 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* FrontmatterParser 单元测试
|
|
||||||
*/
|
|
||||||
class FrontmatterParserTest {
|
|
||||||
|
|
||||||
private FrontmatterParser parser;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
parser = new FrontmatterParser();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testHasFrontmatter_withValidFrontmatter() {
|
|
||||||
String content = "---\ntitle: Test\n---\nContent";
|
|
||||||
assertTrue(parser.hasFrontmatter(content));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testHasFrontmatter_withoutFrontmatter() {
|
|
||||||
String content = "# Just a title\nContent";
|
|
||||||
assertFalse(parser.hasFrontmatter(content));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testHasFrontmatter_nullContent() {
|
|
||||||
assertFalse(parser.hasFrontmatter(null));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testHasFrontmatter_emptyContent() {
|
|
||||||
assertFalse(parser.hasFrontmatter(""));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_validFrontmatter() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: 支付网关错误码
|
|
||||||
keywords: [ERR_TIMEOUT, 超时, 支付网关]
|
|
||||||
summary: 记录了支付网关所有核心错误码
|
|
||||||
category: api
|
|
||||||
---
|
|
||||||
|
|
||||||
# 正文内容
|
|
||||||
""";
|
|
||||||
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals("支付网关错误码", result.getTitle());
|
|
||||||
assertEquals(3, result.getKeywords().size());
|
|
||||||
assertTrue(result.getKeywords().contains("ERR_TIMEOUT"));
|
|
||||||
assertEquals("记录了支付网关所有核心错误码", result.getSummary());
|
|
||||||
assertEquals("api", result.getCategory());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_withoutFrontmatter() {
|
|
||||||
String content = "# Just content\nNo frontmatter here";
|
|
||||||
assertNull(parser.parse(content));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_missingRequiredFields() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: Only Title
|
|
||||||
---
|
|
||||||
Content
|
|
||||||
""";
|
|
||||||
|
|
||||||
// 缺少 keywords 和 summary,应返回 null
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
assertNull(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_malformedYaml() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: Test
|
|
||||||
keywords: [unclosed array
|
|
||||||
---
|
|
||||||
Content
|
|
||||||
""";
|
|
||||||
|
|
||||||
// YAML 格式错误,应返回 null
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
assertNull(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_noClosingDelimiter() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: Test
|
|
||||||
keywords: [test]
|
|
||||||
summary: Test summary
|
|
||||||
|
|
||||||
Content without closing ---
|
|
||||||
""";
|
|
||||||
|
|
||||||
// 缺少结束标记,应返回 null
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
assertNull(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_windowsLineEndings() {
|
|
||||||
String content = "---\r\ntitle: Test\r\nkeywords: [test]\r\nsummary: Summary\r\n---\r\nContent";
|
|
||||||
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals("Test", result.getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_withOptionalFields() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: Test Document
|
|
||||||
keywords: [test, doc]
|
|
||||||
summary: A test document
|
|
||||||
version: 1.0.0
|
|
||||||
author: Test Author
|
|
||||||
---
|
|
||||||
Content
|
|
||||||
""";
|
|
||||||
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals("Test Document", result.getTitle());
|
|
||||||
assertEquals("1.0.0", result.getVersion());
|
|
||||||
assertEquals("Test Author", result.getAuthor());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testParse_withRetrievalMetadata() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: MySQL Connection Pool
|
|
||||||
keywords: [connection pool, HikariCP]
|
|
||||||
summary: Diagnose exhausted MySQL connection pools
|
|
||||||
category: database
|
|
||||||
source: mysql-connection-pool
|
|
||||||
breadcrumb: Database > MySQL > Connection Pool
|
|
||||||
kb_scope: rag-eval
|
|
||||||
---
|
|
||||||
Content
|
|
||||||
""";
|
|
||||||
|
|
||||||
Frontmatter result = parser.parse(content);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals("mysql-connection-pool", result.getSource());
|
|
||||||
assertEquals("Database > MySQL > Connection Pool", result.getBreadcrumb());
|
|
||||||
assertEquals("rag-eval", result.getKbScope());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testStripFrontmatter_returnsMarkdownBodyOnly() {
|
|
||||||
String content = """
|
|
||||||
---
|
|
||||||
title: Test
|
|
||||||
keywords: [frontmatter-only]
|
|
||||||
summary: Summary
|
|
||||||
---
|
|
||||||
|
|
||||||
# Body
|
|
||||||
|
|
||||||
Body content
|
|
||||||
""";
|
|
||||||
|
|
||||||
String body = parser.stripFrontmatter(content);
|
|
||||||
|
|
||||||
assertFalse(body.contains("frontmatter-only"));
|
|
||||||
assertTrue(body.startsWith("# Body"));
|
|
||||||
assertTrue(body.contains("Body content"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.DisplayName;
|
|
||||||
import org.junit.jupiter.api.MethodOrderer;
|
|
||||||
import org.junit.jupiter.api.Order;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.TestMethodOrder;
|
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
|
||||||
import org.springframework.ai.chat.prompt.Prompt;
|
|
||||||
import org.springframework.ai.embedding.EmbeddingModel;
|
|
||||||
import org.springframework.ai.tool.ToolCallback;
|
|
||||||
import org.springframework.ai.tool.ToolCallbackProvider;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.boot.test.context.SpringBootTest;
|
|
||||||
import org.springframework.boot.test.context.TestConfiguration;
|
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 全链路验证:DeepSeek → BGE-M3 → Milvus
|
|
||||||
*/
|
|
||||||
@SpringBootTest
|
|
||||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
|
||||||
@DisplayName("DeepSeek → BGE-M3 → Milvus 全链路")
|
|
||||||
class FullPipelineSmokeTest {
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private ChatModel chatModel;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private EmbeddingModel embeddingModel;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorEmbeddingService vectorEmbeddingService;
|
|
||||||
|
|
||||||
@Autowired
|
|
||||||
private VectorSearchService vectorSearchService;
|
|
||||||
|
|
||||||
@TestConfiguration
|
|
||||||
static class MockToolConfig {
|
|
||||||
@Bean
|
|
||||||
public ToolCallbackProvider toolCallbackProvider() {
|
|
||||||
return () -> new ToolCallback[0];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ===== ① Chat: DeepSeek =====
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(1)
|
|
||||||
@DisplayName("Chat: DeepSeek 聊天验证")
|
|
||||||
void chatDeepSeekWorks() {
|
|
||||||
System.out.println("\n===== ① Chat: DeepSeek =====");
|
|
||||||
System.out.println("ChatModel: " + chatModel.getClass().getSimpleName());
|
|
||||||
System.out.println("ChatOptions: " + chatModel.toString());
|
|
||||||
|
|
||||||
// 直接调用 chat
|
|
||||||
var response = chatModel.call(new Prompt("请用一句话介绍你自己"));
|
|
||||||
String text = response.getResult().getOutput().getText();
|
|
||||||
assertNotNull(text);
|
|
||||||
assertFalse(text.isEmpty());
|
|
||||||
System.out.println("Response: " + text.substring(0, Math.min(200, text.length())) + "...");
|
|
||||||
System.out.println("Chat ✓");
|
|
||||||
}
|
|
||||||
|
|
||||||
// ===== ② Embedding: BGE-M3 via SiliconFlow =====
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(2)
|
|
||||||
@DisplayName("Embedding: BGE-M3 向量生成验证")
|
|
||||||
void embeddingBgeM3Works() {
|
|
||||||
System.out.println("\n===== ② Embedding: BGE-M3 (SiliconFlow) =====");
|
|
||||||
System.out.println("EmbeddingModel: " + embeddingModel.getClass().getSimpleName());
|
|
||||||
|
|
||||||
String text = "你好,这是一条测试文本";
|
|
||||||
List<Float> vector = vectorEmbeddingService.generateEmbedding(text);
|
|
||||||
|
|
||||||
assertNotNull(vector);
|
|
||||||
assertFalse(vector.isEmpty());
|
|
||||||
assertEquals(1024, vector.size(), "BGE-M3 应返回 1024 维向量");
|
|
||||||
|
|
||||||
// 非零校验
|
|
||||||
boolean hasNonZero = vector.stream().anyMatch(v -> Math.abs(v) > 1e-6);
|
|
||||||
assertTrue(hasNonZero, "向量不能全为零");
|
|
||||||
|
|
||||||
// L2 范数校验:BGE-M3 输出应为 L2 归一化的单位向量
|
|
||||||
double norm = Math.sqrt(vector.stream().mapToDouble(v -> (double) v * v).sum());
|
|
||||||
|
|
||||||
System.out.println("维度: " + vector.size());
|
|
||||||
System.out.println("前5维: " + vector.subList(0, Math.min(5, vector.size())));
|
|
||||||
System.out.println("L2 范数: " + String.format("%.10f", norm));
|
|
||||||
System.out.println("是否归一化 (|norm - 1.0| < 0.01): " + (Math.abs(norm - 1.0) < 0.01));
|
|
||||||
|
|
||||||
assertEquals(1.0, norm, 0.01, "BGE-M3 向量应为 L2 归一化单位向量,实际范数=" + norm);
|
|
||||||
System.out.println("Embedding ✓");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(3)
|
|
||||||
@DisplayName("Embedding: BGE-M3 批量向量生成验证")
|
|
||||||
void embeddingBatchWorks() {
|
|
||||||
System.out.println("\n===== ③ Embedding 批量 =====");
|
|
||||||
List<String> texts = List.of("文本一", "文本二", "文本三");
|
|
||||||
List<List<Float>> results = vectorEmbeddingService.generateEmbeddings(texts);
|
|
||||||
|
|
||||||
assertEquals(3, results.size());
|
|
||||||
for (List<Float> r : results) {
|
|
||||||
assertEquals(1024, r.size());
|
|
||||||
}
|
|
||||||
System.out.println("批量生成: " + results.size() + " 个 向量,各 " + results.get(0).size() + " 维 ✓");
|
|
||||||
}
|
|
||||||
|
|
||||||
// ===== ③ Milvus: 向量搜索 =====
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(4)
|
|
||||||
@DisplayName("Milvus: 连接 + 搜索验证")
|
|
||||||
void milvusSearchWorks() {
|
|
||||||
System.out.println("\n===== ④ Milvus: 向量搜索 =====");
|
|
||||||
|
|
||||||
// 用 BGE-M3 生成查询向量
|
|
||||||
String query = "内部文档";
|
|
||||||
List<Float> queryVector = vectorEmbeddingService.generateQueryVector(query);
|
|
||||||
assertNotNull(queryVector);
|
|
||||||
assertEquals(1024, queryVector.size());
|
|
||||||
|
|
||||||
// 搜索
|
|
||||||
List<VectorSearchService.SearchResult> results =
|
|
||||||
vectorSearchService.searchSimilarDocuments(query, 3);
|
|
||||||
|
|
||||||
assertNotNull(results);
|
|
||||||
System.out.println("查询: " + query);
|
|
||||||
System.out.println("返回: " + results.size() + " 条");
|
|
||||||
|
|
||||||
if (!results.isEmpty()) {
|
|
||||||
// 至少有结果,验证结构
|
|
||||||
for (int i = 0; i < results.size(); i++) {
|
|
||||||
var r = results.get(i);
|
|
||||||
assertNotNull(r.getId());
|
|
||||||
assertNotNull(r.getContent());
|
|
||||||
System.out.println(" [" + (i + 1) + "] id=" + r.getId()
|
|
||||||
+ ", score=" + String.format("%.4f", r.getScore())
|
|
||||||
+ ", content=" + r.getContent().substring(0, Math.min(50, r.getContent().length())) + "...");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
System.out.println("(Milvus 中暂无数据,但连接正常)");
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("Milvus ✓");
|
|
||||||
}
|
|
||||||
|
|
||||||
// ===== 汇总 =====
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(5)
|
|
||||||
@DisplayName("总结")
|
|
||||||
void summary() {
|
|
||||||
System.out.println("\n==========================================");
|
|
||||||
System.out.println("全链路验证完成:");
|
|
||||||
System.out.println(" ① Chat → DeepSeek ✓");
|
|
||||||
System.out.println(" ② Embedding → BGE-M3 ✓ (SiliconFlow, 1024维)");
|
|
||||||
System.out.println(" ③ 向量存储 → Milvus ✓ (Zilliz Cloud)");
|
|
||||||
System.out.println("==========================================");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,316 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
|
||||||
import org.springframework.test.util.ReflectionTestUtils;
|
|
||||||
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* KnowledgeIndexService 单元测试
|
|
||||||
*/
|
|
||||||
class KnowledgeIndexServiceTest {
|
|
||||||
|
|
||||||
private KnowledgeIndexService service;
|
|
||||||
|
|
||||||
@TempDir
|
|
||||||
Path tempDir;
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
void setUp() {
|
|
||||||
service = new KnowledgeIndexService();
|
|
||||||
ReflectionTestUtils.setField(service, "knowledgeBasePath", tempDir.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_singleMatch() {
|
|
||||||
// 准备测试数据
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("test.md")
|
|
||||||
.title("Test")
|
|
||||||
.keywords(List.of("ERR_TIMEOUT", "超时"))
|
|
||||||
.summary("Test summary")
|
|
||||||
.category("api")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
// 测试匹配
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("ERR_TIMEOUT");
|
|
||||||
|
|
||||||
assertEquals(1, results.size());
|
|
||||||
assertEquals("Test", results.get(0).getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_caseInsensitive() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("test.md")
|
|
||||||
.keywords(List.of("ERR_TIMEOUT"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
// 小写查询应该匹配
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("err_timeout");
|
|
||||||
assertEquals(1, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_partialMatch() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("test.md")
|
|
||||||
.keywords(List.of("支付网关"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
// 包含关键词的查询应该匹配
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("支付网关超时问题");
|
|
||||||
assertEquals(1, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_multipleMatches() {
|
|
||||||
KnowledgeEntry entry1 = KnowledgeEntry.builder()
|
|
||||||
.filePath("doc1.md")
|
|
||||||
.title("Doc 1")
|
|
||||||
.keywords(List.of("超时"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
KnowledgeEntry entry2 = KnowledgeEntry.builder()
|
|
||||||
.filePath("doc2.md")
|
|
||||||
.title("Doc 2")
|
|
||||||
.keywords(List.of("超时", "错误"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry1);
|
|
||||||
service.addToIndex(entry2);
|
|
||||||
|
|
||||||
// 应该匹配两个文档
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("超时");
|
|
||||||
assertEquals(2, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testAnalyzeQuery_returnsStructuredHint() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("mysql.md")
|
|
||||||
.title("MySQL Doc")
|
|
||||||
.keywords(List.of("mysql", "connection pool"))
|
|
||||||
.category("database")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("mysql connection pool timeout");
|
|
||||||
|
|
||||||
assertEquals(1, hint.matches().size());
|
|
||||||
assertEquals(List.of("mysql", "connection pool"), hint.matchedKeywords());
|
|
||||||
assertEquals(List.of("database"), hint.domains());
|
|
||||||
assertEquals(List.of("mysql", "connection pool"), hint.entities());
|
|
||||||
assertEquals(List.of("MySQL Doc"), hint.titles());
|
|
||||||
assertEquals("database", hint.singleDomainOrNull());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testAnalyzeQuery_multipleDomainsHasNoSingleDomain() {
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("mysql.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("database")
|
|
||||||
.build());
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("api.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("api")
|
|
||||||
.build());
|
|
||||||
|
|
||||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
|
||||||
|
|
||||||
assertEquals(2, hint.matches().size());
|
|
||||||
assertNull(hint.singleDomainOrNull());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testAnalyzeQuery_filtersByConfiguredKbScope() {
|
|
||||||
ReflectionTestUtils.setField(service, "kbScope", "rag-eval");
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("legacy.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("legacy")
|
|
||||||
.build());
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("eval.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("eval")
|
|
||||||
.kbScope("rag-eval")
|
|
||||||
.build());
|
|
||||||
|
|
||||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
|
||||||
|
|
||||||
assertEquals(1, hint.matches().size());
|
|
||||||
assertEquals(List.of("eval"), hint.domains());
|
|
||||||
assertEquals("eval", hint.singleDomainOrNull());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testAnalyzeQuery_keepsLegacyEntriesWhenNoScopeConfigured() {
|
|
||||||
ReflectionTestUtils.setField(service, "kbScope", "");
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("legacy.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("legacy")
|
|
||||||
.build());
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("eval.md")
|
|
||||||
.keywords(List.of("timeout"))
|
|
||||||
.category("eval")
|
|
||||||
.kbScope("rag-eval")
|
|
||||||
.build());
|
|
||||||
|
|
||||||
KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
|
|
||||||
|
|
||||||
assertEquals(2, hint.matches().size());
|
|
||||||
assertNull(hint.singleDomainOrNull());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_noMatch() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("test.md")
|
|
||||||
.keywords(List.of("错误码"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
// 不匹配的查询
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("限流");
|
|
||||||
assertEquals(0, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_emptyQuery() {
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("");
|
|
||||||
assertEquals(0, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testExactMatch_nullQuery() {
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch(null);
|
|
||||||
assertEquals(0, results.size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testReadDocument_success() throws Exception {
|
|
||||||
// 创建测试文件
|
|
||||||
Path testFile = tempDir.resolve("test.md");
|
|
||||||
String content = "Test content line 1\nTest content line 2\n";
|
|
||||||
Files.writeString(testFile, content);
|
|
||||||
|
|
||||||
// 读取文件
|
|
||||||
String result = service.readDocument(testFile.toString(), 100);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertTrue(result.contains("Test content"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testReadDocument_relativePathUnderBasePath() throws Exception {
|
|
||||||
Path categoryDir = tempDir.resolve("payment");
|
|
||||||
Files.createDirectories(categoryDir);
|
|
||||||
Path testFile = categoryDir.resolve("relative.md");
|
|
||||||
Files.writeString(testFile, "Relative content");
|
|
||||||
|
|
||||||
String result = service.readDocument("payment/relative.md", 100);
|
|
||||||
|
|
||||||
assertEquals("Relative content", result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testReadDocument_legacyPathAlreadyContainsBasePath() throws Exception {
|
|
||||||
Path categoryDir = tempDir.resolve("payment");
|
|
||||||
Files.createDirectories(categoryDir);
|
|
||||||
Path testFile = categoryDir.resolve("legacy.md");
|
|
||||||
Files.writeString(testFile, "Legacy content");
|
|
||||||
|
|
||||||
String result = service.readDocument(tempDir.getFileName() + "/payment/legacy.md", 100);
|
|
||||||
|
|
||||||
assertEquals("Legacy content", result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testReadDocument_exceedsMaxChars() throws Exception {
|
|
||||||
// 创建超长内容
|
|
||||||
String longContent = "x".repeat(3000);
|
|
||||||
Path testFile = tempDir.resolve("long.md");
|
|
||||||
Files.writeString(testFile, longContent);
|
|
||||||
|
|
||||||
// 读取限制字符数
|
|
||||||
String result = service.readDocument(testFile.toString(), 2000);
|
|
||||||
|
|
||||||
assertNotNull(result);
|
|
||||||
assertEquals(2003, result.length()); // 2000 + "..."
|
|
||||||
assertTrue(result.endsWith("..."));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testReadDocument_fileNotFound() {
|
|
||||||
String result = service.readDocument("nonexistent.md", 100);
|
|
||||||
assertNull(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testAddToIndex() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("new.md")
|
|
||||||
.title("New Document")
|
|
||||||
.keywords(List.of("test"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
|
|
||||||
List<KnowledgeEntry> results = service.exactMatch("test");
|
|
||||||
assertEquals(1, results.size());
|
|
||||||
assertEquals("New Document", results.get(0).getTitle());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testRemoveFromIndex() {
|
|
||||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
|
||||||
.filePath("remove.md")
|
|
||||||
.keywords(List.of("test"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
service.addToIndex(entry);
|
|
||||||
assertEquals(1, service.exactMatch("test").size());
|
|
||||||
|
|
||||||
service.removeFromIndex("remove.md");
|
|
||||||
assertEquals(0, service.exactMatch("test").size());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testGetIndexSize() {
|
|
||||||
assertEquals(0, service.getIndexSize());
|
|
||||||
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("doc1.md")
|
|
||||||
.keywords(List.of("test"))
|
|
||||||
.build());
|
|
||||||
|
|
||||||
assertEquals(1, service.getIndexSize());
|
|
||||||
|
|
||||||
service.addToIndex(KnowledgeEntry.builder()
|
|
||||||
.filePath("doc2.md")
|
|
||||||
.keywords(List.of("test"))
|
|
||||||
.build());
|
|
||||||
|
|
||||||
assertEquals(2, service.getIndexSize());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,237 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import io.milvus.client.MilvusServiceClient;
|
|
||||||
import io.milvus.grpc.DataType;
|
|
||||||
import io.milvus.grpc.FlushResponse;
|
|
||||||
import io.milvus.grpc.MutationResult;
|
|
||||||
import io.milvus.grpc.SearchResults;
|
|
||||||
import io.milvus.grpc.ShowCollectionsResponse;
|
|
||||||
import io.milvus.common.clientenum.ConsistencyLevelEnum;
|
|
||||||
import io.milvus.param.ConnectParam;
|
|
||||||
import io.milvus.param.IndexType;
|
|
||||||
import io.milvus.param.MetricType;
|
|
||||||
import io.milvus.param.R;
|
|
||||||
import io.milvus.param.RpcStatus;
|
|
||||||
import io.milvus.param.collection.*;
|
|
||||||
import io.milvus.param.dml.InsertParam;
|
|
||||||
import io.milvus.param.dml.SearchParam;
|
|
||||||
import io.milvus.param.index.CreateIndexParam;
|
|
||||||
import io.milvus.response.SearchResultsWrapper;
|
|
||||||
import org.junit.jupiter.api.*;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
@DisplayName("Milvus 连接验证")
|
|
||||||
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
|
|
||||||
class MilvusConnectionTest {
|
|
||||||
|
|
||||||
private static final String COLLECTION = "conn_test";
|
|
||||||
private static final int DIM = 128;
|
|
||||||
|
|
||||||
private static MilvusServiceClient client;
|
|
||||||
|
|
||||||
@BeforeAll
|
|
||||||
static void connect() {
|
|
||||||
String host = envOrDefault("MILVUS_HOST",
|
|
||||||
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
|
|
||||||
int port = Integer.parseInt(envOrDefault("MILVUS_PORT", "443"));
|
|
||||||
String token = System.getenv("MILVUS_TOKEN");
|
|
||||||
|
|
||||||
assertNotNull(token, "环境变量 MILVUS_TOKEN 未设置");
|
|
||||||
|
|
||||||
ConnectParam connectParam = ConnectParam.newBuilder()
|
|
||||||
.withHost(host)
|
|
||||||
.withPort(port)
|
|
||||||
.withToken(token)
|
|
||||||
.withSecure(true)
|
|
||||||
.withDatabaseName("db_4a578da0f27ce9d")
|
|
||||||
.withConnectTimeout(30, TimeUnit.SECONDS)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
client = new MilvusServiceClient(connectParam);
|
|
||||||
System.out.println("连接目标: " + host + ":" + port);
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterAll
|
|
||||||
static void disconnect() {
|
|
||||||
if (client != null) {
|
|
||||||
try {
|
|
||||||
client.dropCollection(DropCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION).build());
|
|
||||||
} catch (Exception ignored) {}
|
|
||||||
client.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String safeMsg(R<?> resp) {
|
|
||||||
try {
|
|
||||||
return resp.getMessage();
|
|
||||||
} catch (Exception e) {
|
|
||||||
return "(no message)";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(1)
|
|
||||||
@DisplayName("1. 连接成功 - 能列出 collection")
|
|
||||||
void listCollections() {
|
|
||||||
R<ShowCollectionsResponse> resp = client.showCollections(
|
|
||||||
ShowCollectionsParam.newBuilder().build());
|
|
||||||
|
|
||||||
System.out.println("listCollections status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
|
||||||
assertEquals(0, resp.getStatus(), "连接失败,status=" + resp.getStatus());
|
|
||||||
|
|
||||||
List<String> names = resp.getData().getCollectionNamesList();
|
|
||||||
System.out.println("现有 collections: " + names);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(2)
|
|
||||||
@DisplayName("2. 创建测试 collection")
|
|
||||||
void createCollection() {
|
|
||||||
client.dropCollection(DropCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION).build());
|
|
||||||
|
|
||||||
FieldType idField = FieldType.newBuilder()
|
|
||||||
.withName("id")
|
|
||||||
.withDataType(DataType.Int64)
|
|
||||||
.withPrimaryKey(true)
|
|
||||||
.withAutoID(true)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
FieldType vectorField = FieldType.newBuilder()
|
|
||||||
.withName("vector")
|
|
||||||
.withDataType(DataType.FloatVector)
|
|
||||||
.withDimension(DIM)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
|
|
||||||
.addFieldType(idField)
|
|
||||||
.addFieldType(vectorField)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
R<RpcStatus> resp = client.createCollection(
|
|
||||||
CreateCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION)
|
|
||||||
.withSchema(schema)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
System.out.println("createCollection status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
|
||||||
assertEquals(0, resp.getStatus(), "创建 collection 失败");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(3)
|
|
||||||
@DisplayName("3. 插入数据 + flush")
|
|
||||||
void insertAndFlush() {
|
|
||||||
List<Float> vec1 = makeVector(1.0f);
|
|
||||||
List<Float> vec2 = makeVector(2.0f);
|
|
||||||
List<Float> vec3 = makeVector(3.0f);
|
|
||||||
|
|
||||||
List<InsertParam.Field> fields = Collections.singletonList(
|
|
||||||
new InsertParam.Field("vector", Arrays.asList(vec1, vec2, vec3))
|
|
||||||
);
|
|
||||||
|
|
||||||
R<MutationResult> insertResp = client.insert(
|
|
||||||
InsertParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION)
|
|
||||||
.withFields(fields)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
System.out.println("insert status: " + insertResp.getStatus() + ", msg: " + safeMsg(insertResp));
|
|
||||||
assertEquals(0, insertResp.getStatus(), "插入失败");
|
|
||||||
|
|
||||||
// 官方示例要求:insert 后必须 flush,数据才对搜索可见
|
|
||||||
R<FlushResponse> flushResp = client.flush(FlushParam.newBuilder()
|
|
||||||
.withCollectionNames(Collections.singletonList(COLLECTION))
|
|
||||||
.withSyncFlush(true)
|
|
||||||
.withSyncFlushWaitingTimeout(30L)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
System.out.println("flush status: " + flushResp.getStatus() + ", msg: " + safeMsg(flushResp));
|
|
||||||
assertEquals(0, flushResp.getStatus(), "flush 失败");
|
|
||||||
System.out.println("插入 3 条数据并 flush 完成");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(4)
|
|
||||||
@DisplayName("4. 创建索引 + 加载")
|
|
||||||
void createIndexAndLoad() {
|
|
||||||
R<RpcStatus> indexResp = client.createIndex(
|
|
||||||
CreateIndexParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION)
|
|
||||||
.withFieldName("vector")
|
|
||||||
.withIndexType(IndexType.AUTOINDEX)
|
|
||||||
.withMetricType(MetricType.L2)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
System.out.println("createIndex status: " + indexResp.getStatus() + ", msg: " + safeMsg(indexResp));
|
|
||||||
assertEquals(0, indexResp.getStatus(), "创建索引失败");
|
|
||||||
|
|
||||||
R<RpcStatus> loadResp = client.loadCollection(
|
|
||||||
LoadCollectionParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION)
|
|
||||||
.withSyncLoad(true)
|
|
||||||
.withSyncLoadWaitingTimeout(30L)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
System.out.println("load status: " + loadResp.getStatus() + ", msg: " + safeMsg(loadResp));
|
|
||||||
assertEquals(0, loadResp.getStatus(), "加载失败");
|
|
||||||
System.out.println("索引创建 + 加载完成");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
@Order(5)
|
|
||||||
@DisplayName("5. 向量搜索")
|
|
||||||
void search() throws InterruptedException {
|
|
||||||
Thread.sleep(3000);
|
|
||||||
|
|
||||||
List<Float> queryVec = makeVector(1.1f);
|
|
||||||
|
|
||||||
R<SearchResults> resp = null;
|
|
||||||
for (int retry = 0; retry < 10; retry++) {
|
|
||||||
resp = client.search(
|
|
||||||
SearchParam.newBuilder()
|
|
||||||
.withCollectionName(COLLECTION)
|
|
||||||
.withMetricType(MetricType.L2)
|
|
||||||
.withTopK(2)
|
|
||||||
.withVectors(Collections.singletonList(queryVec))
|
|
||||||
.withVectorFieldName("vector")
|
|
||||||
.withParams("{}")
|
|
||||||
.withConsistencyLevel(ConsistencyLevelEnum.STRONG)
|
|
||||||
.build());
|
|
||||||
|
|
||||||
if (resp.getStatus() == 0) break;
|
|
||||||
System.out.println("search retry " + (retry + 1) + ": status=" + resp.getStatus() + ", msg=" + safeMsg(resp));
|
|
||||||
Thread.sleep(5000);
|
|
||||||
}
|
|
||||||
|
|
||||||
System.out.println("search status: " + resp.getStatus() + ", msg: " + safeMsg(resp));
|
|
||||||
assertEquals(0, resp.getStatus(), "搜索失败");
|
|
||||||
|
|
||||||
SearchResultsWrapper wrapper = new SearchResultsWrapper(resp.getData().getResults());
|
|
||||||
List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(0);
|
|
||||||
|
|
||||||
assertFalse(scores.isEmpty(), "搜索结果不应为空");
|
|
||||||
System.out.println("搜索结果 (top " + scores.size() + "):");
|
|
||||||
for (SearchResultsWrapper.IDScore idScore : scores) {
|
|
||||||
System.out.println(" score=" + idScore.getScore() + ", id=" + idScore.getLongID());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<Float> makeVector(float val) {
|
|
||||||
Float[] arr = new Float[DIM];
|
|
||||||
Arrays.fill(arr, val);
|
|
||||||
return Arrays.asList(arr);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String envOrDefault(String key, String defaultVal) {
|
|
||||||
String val = System.getenv(key);
|
|
||||||
return (val != null && !val.isEmpty()) ? val : defaultVal;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-117
@@ -1,117 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.config.RagSidecarProperties;
|
|
||||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
|
||||||
import com.superbiz.agent.dto.RetrievalComparisonCase;
|
|
||||||
import com.superbiz.agent.dto.RetrievalComparisonReport;
|
|
||||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.junit.jupiter.api.io.TempDir;
|
|
||||||
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
class RagRetrievalSidecarComparisonServiceTest {
|
|
||||||
|
|
||||||
@TempDir
|
|
||||||
Path tempDir;
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void compareWritesSeparateSidecarReports() throws Exception {
|
|
||||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
|
||||||
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
|
|
||||||
RagSidecarProperties properties = new RagSidecarProperties();
|
|
||||||
RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
|
|
||||||
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
|
|
||||||
vectorSearchService,
|
|
||||||
sidecarService,
|
|
||||||
normalizer,
|
|
||||||
properties,
|
|
||||||
new ObjectMapper()
|
|
||||||
);
|
|
||||||
|
|
||||||
VectorSearchService.SearchResult current = new VectorSearchService.SearchResult();
|
|
||||||
current.setId("current-1");
|
|
||||||
current.setMetadata("{\"_source\":\"current.md\",\"breadcrumb\":\"A\",\"category\":\"api\"}");
|
|
||||||
current.setContent("current content");
|
|
||||||
current.setScore(0.1f);
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("timeout", 3, "api"))
|
|
||||||
.thenReturn(List.of(current));
|
|
||||||
when(sidecarService.search("timeout", 3, "api"))
|
|
||||||
.thenReturn(SidecarRetrievalResponse.builder()
|
|
||||||
.enabled(true)
|
|
||||||
.available(true)
|
|
||||||
.status("available")
|
|
||||||
.results(List.of(ComparableRetrievalResult.builder()
|
|
||||||
.path("sidecar")
|
|
||||||
.rank(1)
|
|
||||||
.source("sidecar.md")
|
|
||||||
.breadcrumb("B")
|
|
||||||
.scoreLabel("similarity")
|
|
||||||
.scoreValue(0.9)
|
|
||||||
.build()))
|
|
||||||
.build());
|
|
||||||
|
|
||||||
RetrievalComparisonReport report = comparisonService.compare(List.of(
|
|
||||||
RetrievalComparisonCase.builder()
|
|
||||||
.caseId("case-1")
|
|
||||||
.scenario("aiops")
|
|
||||||
.query("timeout")
|
|
||||||
.category("api")
|
|
||||||
.build()
|
|
||||||
), 3);
|
|
||||||
|
|
||||||
assertEquals(1, report.getCaseCount());
|
|
||||||
assertEquals("available", report.getSidecarStatus());
|
|
||||||
assertTrue(report.getResults().get(0).getDifferences().contains("top_source_differs"));
|
|
||||||
Path json = tempDir.resolve("sidecar.json");
|
|
||||||
Path markdown = tempDir.resolve("sidecar.md");
|
|
||||||
comparisonService.writeReports(report, json, markdown);
|
|
||||||
|
|
||||||
assertTrue(Files.readString(json).contains("\"sidecarStatus\""));
|
|
||||||
assertTrue(Files.readString(markdown).contains("RAG Sidecar Retrieval Comparison"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void compareGoldenCasesLoadsExistingCaseShape() throws Exception {
|
|
||||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
|
||||||
SpringAiVectorStoreSidecarService sidecarService = mock(SpringAiVectorStoreSidecarService.class);
|
|
||||||
RagRetrievalSidecarComparisonService comparisonService = new RagRetrievalSidecarComparisonService(
|
|
||||||
vectorSearchService,
|
|
||||||
sidecarService,
|
|
||||||
new RetrievalResultNormalizer(new ObjectMapper()),
|
|
||||||
new RagSidecarProperties(),
|
|
||||||
new ObjectMapper()
|
|
||||||
);
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("query", 2, null)).thenReturn(List.of());
|
|
||||||
when(sidecarService.search("query", 2, null))
|
|
||||||
.thenReturn(SidecarRetrievalResponse.builder()
|
|
||||||
.enabled(false)
|
|
||||||
.available(false)
|
|
||||||
.status("disabled")
|
|
||||||
.results(List.of())
|
|
||||||
.build());
|
|
||||||
Path cases = tempDir.resolve("cases.json");
|
|
||||||
Files.writeString(cases, """
|
|
||||||
{
|
|
||||||
"topK": 2,
|
|
||||||
"cases": [
|
|
||||||
{"caseId": "case-1", "scenario": "chat", "query": "query"}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
""");
|
|
||||||
|
|
||||||
RetrievalComparisonReport report = comparisonService.compareGoldenCases(cases);
|
|
||||||
|
|
||||||
assertEquals(1, report.getCaseCount());
|
|
||||||
assertEquals(2, report.getTopK());
|
|
||||||
assertEquals("disabled", report.getSidecarStatus());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.springframework.ai.document.Document;
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
|
|
||||||
class RetrievalResultNormalizerTest {
|
|
||||||
|
|
||||||
private final RetrievalResultNormalizer normalizer = new RetrievalResultNormalizer(new ObjectMapper());
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void fromCurrentParsesMetadataAndLabelsDistanceScore() {
|
|
||||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
|
||||||
result.setId("vec-1");
|
|
||||||
result.setMetadata("{\"docId\":\"doc-1\",\"_source\":\"docs/api.md\",\"title\":\"API\",\"breadcrumb\":\"A > B\",\"category\":\"api\"}");
|
|
||||||
result.setContent("abcdef");
|
|
||||||
result.setScore(0.25f);
|
|
||||||
|
|
||||||
ComparableRetrievalResult comparable = normalizer.fromCurrent(result, 1, 3);
|
|
||||||
|
|
||||||
assertEquals("current", comparable.getPath());
|
|
||||||
assertEquals("docs/api.md", comparable.getSource());
|
|
||||||
assertEquals("doc-1", comparable.getDocId());
|
|
||||||
assertEquals("API", comparable.getTitle());
|
|
||||||
assertEquals("A > B", comparable.getBreadcrumb());
|
|
||||||
assertEquals("api", comparable.getCategory());
|
|
||||||
assertEquals("abc...", comparable.getContentPreview());
|
|
||||||
assertEquals("l2_distance", comparable.getScoreLabel());
|
|
||||||
assertEquals(0.25, comparable.getScoreValue(), 0.0001);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void fromSidecarNormalizesDocumentMetadataAndLabelsSimilarityScore() {
|
|
||||||
Document document = Document.builder()
|
|
||||||
.id("doc-vector")
|
|
||||||
.text("sidecar content")
|
|
||||||
.metadata(Map.of(
|
|
||||||
"docId", "doc-2",
|
|
||||||
"_source", "docs/sidecar.md",
|
|
||||||
"title", "Sidecar",
|
|
||||||
"breadcrumb", "Root > Sidecar",
|
|
||||||
"category", "rag"
|
|
||||||
))
|
|
||||||
.score(0.91)
|
|
||||||
.build();
|
|
||||||
|
|
||||||
ComparableRetrievalResult comparable = normalizer.fromSidecar(document, 2, 100);
|
|
||||||
|
|
||||||
assertEquals("sidecar", comparable.getPath());
|
|
||||||
assertEquals(2, comparable.getRank());
|
|
||||||
assertEquals("docs/sidecar.md", comparable.getSource());
|
|
||||||
assertEquals("similarity", comparable.getScoreLabel());
|
|
||||||
assertEquals(0.91, comparable.getScoreValue(), 0.0001);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,66 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import io.milvus.client.MilvusServiceClient;
|
|
||||||
import io.milvus.param.ConnectParam;
|
|
||||||
import io.milvus.param.R;
|
|
||||||
import io.milvus.param.collection.HasCollectionParam;
|
|
||||||
import org.junit.jupiter.api.Assumptions;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 简单的 Milvus 连接测试
|
|
||||||
*/
|
|
||||||
public class SimpleMilvusTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testConnection() {
|
|
||||||
String host = System.getenv().getOrDefault(
|
|
||||||
"MILVUS_HOST",
|
|
||||||
"in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com");
|
|
||||||
int port = Integer.parseInt(System.getenv().getOrDefault("MILVUS_PORT", "443"));
|
|
||||||
String token = System.getenv("MILVUS_TOKEN");
|
|
||||||
Assumptions.assumeTrue(token != null && !token.isBlank(), "MILVUS_TOKEN is required");
|
|
||||||
|
|
||||||
System.out.println("尝试连接 Milvus...");
|
|
||||||
System.out.println("Host: " + host);
|
|
||||||
System.out.println("Port: " + port);
|
|
||||||
|
|
||||||
try {
|
|
||||||
MilvusServiceClient client = new MilvusServiceClient(
|
|
||||||
ConnectParam.newBuilder()
|
|
||||||
.withHost(host)
|
|
||||||
.withPort(port)
|
|
||||||
.withToken(token)
|
|
||||||
.withSecure(true)
|
|
||||||
.withConnectTimeout(10L, java.util.concurrent.TimeUnit.SECONDS)
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
|
|
||||||
System.out.println("✓ 客户端创建成功");
|
|
||||||
|
|
||||||
// 测试连接:查询是否存在某个 collection
|
|
||||||
R<Boolean> response = client.hasCollection(
|
|
||||||
HasCollectionParam.newBuilder()
|
|
||||||
.withCollectionName("test_collection")
|
|
||||||
.build()
|
|
||||||
);
|
|
||||||
|
|
||||||
System.out.println("✓ 连接成功!");
|
|
||||||
System.out.println("Status Code: " + response.getStatus());
|
|
||||||
|
|
||||||
if (response.getStatus() == 0 || response.getStatus() == io.milvus.param.R.Status.Success.getCode()) {
|
|
||||||
System.out.println("✓ Milvus 集群状态:正常运行");
|
|
||||||
} else {
|
|
||||||
System.out.println("✗ 响应状态异常: " + response.getStatus());
|
|
||||||
}
|
|
||||||
|
|
||||||
client.close();
|
|
||||||
System.out.println("✓ 连接已关闭");
|
|
||||||
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.err.println("✗ 连接失败:" + e.getMessage());
|
|
||||||
e.printStackTrace();
|
|
||||||
throw new RuntimeException("Milvus 连接失败", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.config.RagSidecarProperties;
|
|
||||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.springframework.ai.vectorstore.VectorStore;
|
|
||||||
import org.springframework.beans.factory.ObjectProvider;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.never;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
class SpringAiVectorStoreSidecarServiceTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void disabledSidecarDoesNotRequestVectorStore() {
|
|
||||||
RagSidecarProperties properties = new RagSidecarProperties();
|
|
||||||
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
|
|
||||||
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
|
|
||||||
properties,
|
|
||||||
provider,
|
|
||||||
new RetrievalResultNormalizer(new ObjectMapper())
|
|
||||||
);
|
|
||||||
|
|
||||||
SidecarRetrievalResponse response = service.search("query", 3, null);
|
|
||||||
|
|
||||||
assertFalse(response.isEnabled());
|
|
||||||
assertFalse(response.isAvailable());
|
|
||||||
assertEquals("disabled", response.getStatus());
|
|
||||||
verify(provider, never()).getIfAvailable();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void enabledSidecarReportsMissingVectorStore() {
|
|
||||||
RagSidecarProperties properties = new RagSidecarProperties();
|
|
||||||
properties.setEnabled(true);
|
|
||||||
ObjectProvider<VectorStore> provider = mock(ObjectProvider.class);
|
|
||||||
when(provider.getIfAvailable()).thenReturn(null);
|
|
||||||
SpringAiVectorStoreSidecarService service = new SpringAiVectorStoreSidecarService(
|
|
||||||
properties,
|
|
||||||
provider,
|
|
||||||
new RetrievalResultNormalizer(new ObjectMapper())
|
|
||||||
);
|
|
||||||
|
|
||||||
SidecarRetrievalResponse response = service.search("query", 3, "api");
|
|
||||||
|
|
||||||
assertEquals("missing_vector_store", response.getStatus());
|
|
||||||
assertFalse(response.isAvailable());
|
|
||||||
assertEquals(0, response.getResults().size());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.dto.DocumentChunk;
|
|
||||||
import com.superbiz.agent.dto.Frontmatter;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
|
|
||||||
class VectorIndexServiceTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void buildEmbeddingTextIncludesTitleAndBreadcrumb() {
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.title("Connection Pool")
|
|
||||||
.breadcrumb("Database > MySQL > Connection Pool")
|
|
||||||
.content("Check active connections and leak detection.")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
String embeddingText = VectorIndexService.buildEmbeddingText(chunk);
|
|
||||||
|
|
||||||
assertEquals("""
|
|
||||||
Title: Connection Pool
|
|
||||||
Path: Database > MySQL > Connection Pool
|
|
||||||
Content:
|
|
||||||
Check active connections and leak detection.""", embeddingText);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void buildEmbeddingTextKeepsPlainContentWhenNoStructureExists() {
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.title(" ")
|
|
||||||
.breadcrumb(null)
|
|
||||||
.content("Plain chunk content.")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
assertEquals("Plain chunk content.", VectorIndexService.buildEmbeddingText(chunk));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void buildDocumentMetadataUsesFrontmatterRetrievalFields() {
|
|
||||||
DocumentChunk chunk = DocumentChunk.builder()
|
|
||||||
.chunkIndex(0)
|
|
||||||
.title("Chunk Title")
|
|
||||||
.breadcrumb("Chunk > Path")
|
|
||||||
.content("content")
|
|
||||||
.build();
|
|
||||||
Frontmatter frontmatter = Frontmatter.builder()
|
|
||||||
.title("Document Title")
|
|
||||||
.source("mysql-connection-pool")
|
|
||||||
.breadcrumb("Database > MySQL > Connection Pool")
|
|
||||||
.kbScope("rag-eval")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
Map<String, Object> metadata = VectorIndexService.buildDocumentMetadata(
|
|
||||||
"mysql-connection-pool",
|
|
||||||
chunk,
|
|
||||||
2,
|
|
||||||
"database",
|
|
||||||
frontmatter
|
|
||||||
);
|
|
||||||
|
|
||||||
assertEquals("mysql-connection-pool", metadata.get("docId"));
|
|
||||||
assertEquals("mysql-connection-pool", metadata.get("_source"));
|
|
||||||
assertEquals("mysql-connection-pool", metadata.get("source"));
|
|
||||||
assertEquals("database", metadata.get("category"));
|
|
||||||
assertEquals("rag-eval", metadata.get("kb_scope"));
|
|
||||||
assertEquals("Database > MySQL > Connection Pool", metadata.get("breadcrumb"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
package com.superbiz.agent.service;
|
|
||||||
|
|
||||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
import org.springframework.test.util.ReflectionTestUtils;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
|
||||||
import static org.mockito.ArgumentMatchers.isNull;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
class VectorSearchServiceTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void denseModeCallsHybridStoreDenseSearch() {
|
|
||||||
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
|
|
||||||
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
|
|
||||||
when(embeddingService.generateQueryVector("query")).thenReturn(List.of(0.1f, 0.2f));
|
|
||||||
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
|
|
||||||
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
|
|
||||||
.thenReturn(List.of(expected));
|
|
||||||
when(store.collectionName()).thenReturn("biz");
|
|
||||||
|
|
||||||
VectorSearchService service = new VectorSearchService();
|
|
||||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
|
||||||
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
|
|
||||||
ReflectionTestUtils.setField(service, "searchMode", "dense");
|
|
||||||
|
|
||||||
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("query", 3, null);
|
|
||||||
|
|
||||||
assertEquals(List.of(expected), results);
|
|
||||||
verify(store).searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void hybridModeCallsHybridStoreHybridSearch() {
|
|
||||||
MilvusHybridKnowledgeStore store = mock(MilvusHybridKnowledgeStore.class);
|
|
||||||
VectorEmbeddingService embeddingService = mock(VectorEmbeddingService.class);
|
|
||||||
when(embeddingService.generateQueryVector("pool")).thenReturn(List.of(0.3f));
|
|
||||||
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
|
|
||||||
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
|
|
||||||
.thenReturn(List.of(expected));
|
|
||||||
when(store.collectionName()).thenReturn("biz");
|
|
||||||
|
|
||||||
VectorSearchService service = new VectorSearchService();
|
|
||||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
|
||||||
ReflectionTestUtils.setField(service, "embeddingService", embeddingService);
|
|
||||||
ReflectionTestUtils.setField(service, "searchMode", "hybrid");
|
|
||||||
|
|
||||||
List<VectorSearchService.SearchResult> results = service.searchSimilarDocuments("pool", 5, "mysql");
|
|
||||||
|
|
||||||
assertEquals(1, results.size());
|
|
||||||
assertEquals("doc-h", results.get(0).getId());
|
|
||||||
verify(store).searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql"));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static VectorSearchService.SearchResult result(String id, float score) {
|
|
||||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
|
||||||
result.setId(id);
|
|
||||||
result.setScore(score);
|
|
||||||
result.setRawScore((double) score);
|
|
||||||
result.setScoreLabel("dense");
|
|
||||||
result.setContent("content");
|
|
||||||
result.setMetadata("{}");
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+136
@@ -0,0 +1,136 @@
|
|||||||
|
package com.superbiz.agent.service.retrieval;
|
||||||
|
|
||||||
|
import com.superbiz.agent.client.PyRagClient;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
|
||||||
|
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link PyRagKnowledgeSearchAdapter} 请求/响应映射契约测试。
|
||||||
|
*/
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class PyRagKnowledgeSearchAdapterTest {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private PyRagClient pyRagClient;
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void mapsRequestModeTopKAndCategory() {
|
||||||
|
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||||
|
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||||
|
"q", "hybrid", List.of(), "PRECISE", "supported", null));
|
||||||
|
|
||||||
|
adapter.search(new KnowledgeSearchRequest("网关超时怎么排查", 20, "gateway", KnowledgeSearchMode.HYBRID));
|
||||||
|
|
||||||
|
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
|
||||||
|
verify(pyRagClient).search(captor.capture());
|
||||||
|
PyRagSearchRequest request = captor.getValue();
|
||||||
|
assertEquals("网关超时怎么排查", request.query());
|
||||||
|
assertEquals("hybrid", request.mode());
|
||||||
|
assertEquals(20, request.retrieveK());
|
||||||
|
assertEquals(20, request.returnN());
|
||||||
|
assertEquals(20, request.maxChunksPerDocument());
|
||||||
|
assertEquals("gateway", request.category());
|
||||||
|
assertNull(request.kbScope());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void denseModeMapsToSemanticWithoutCategory() {
|
||||||
|
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||||
|
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||||
|
"q", "semantic", List.of(), null, "supported", null));
|
||||||
|
|
||||||
|
adapter.search(KnowledgeSearchRequest.dense("性能优化", 5, null));
|
||||||
|
|
||||||
|
ArgumentCaptor<PyRagSearchRequest> captor = ArgumentCaptor.forClass(PyRagSearchRequest.class);
|
||||||
|
verify(pyRagClient).search(captor.capture());
|
||||||
|
assertEquals("semantic", captor.getValue().mode());
|
||||||
|
assertNull(captor.getValue().category());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void mapsHitsWithRerankScoreAndEvidenceIdentity() {
|
||||||
|
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||||
|
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||||
|
"网关超时怎么排查",
|
||||||
|
"hybrid",
|
||||||
|
List.of(new PyRagSearchHit(
|
||||||
|
"e2e-gateway-b9c1fa12-md-34223174#chunk-1",
|
||||||
|
"e2e-gateway-b9c1fa12-md-34223174",
|
||||||
|
"e2e-gateway-b9c1fa12-md",
|
||||||
|
"网关超时排查",
|
||||||
|
"网关超时排查 > 处理步骤",
|
||||||
|
"网关超时先检查 upstream 配置…",
|
||||||
|
0.9147,
|
||||||
|
"PRECISE")),
|
||||||
|
"PRECISE",
|
||||||
|
"supported",
|
||||||
|
null));
|
||||||
|
|
||||||
|
List<KnowledgeSearchHit> hits = adapter.search(
|
||||||
|
new KnowledgeSearchRequest("网关超时怎么排查", 5, "gateway", KnowledgeSearchMode.HYBRID));
|
||||||
|
|
||||||
|
assertEquals(1, hits.size());
|
||||||
|
KnowledgeSearchHit hit = hits.get(0);
|
||||||
|
assertEquals("e2e-gateway-b9c1fa12-md-34223174#chunk-1", hit.evidenceKey());
|
||||||
|
assertEquals("e2e-gateway-b9c1fa12-md-34223174", hit.docId());
|
||||||
|
assertEquals(1, hit.chunkIndex());
|
||||||
|
assertEquals("网关超时先检查 upstream 配置…", hit.content());
|
||||||
|
assertEquals(0.9147, hit.score(), 1e-9);
|
||||||
|
assertEquals(0.9147, hit.rawScore(), 1e-9);
|
||||||
|
assertEquals(RetrievalScoreLabels.RERANK, hit.scoreLabel());
|
||||||
|
assertEquals("e2e-gateway-b9c1fa12-md", hit.source());
|
||||||
|
assertEquals("网关超时排查", hit.title());
|
||||||
|
assertEquals("网关超时排查 > 处理步骤", hit.breadcrumb());
|
||||||
|
assertEquals(1, hit.originalRank());
|
||||||
|
assertNull(hit.denseDistance());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void noEvidenceReturnsEmptyListAsNormalBusinessResult() {
|
||||||
|
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||||
|
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||||
|
"乱码查询", "hybrid", List.of(), null, "no_evidence", null));
|
||||||
|
|
||||||
|
List<KnowledgeSearchHit> hits = adapter.search(
|
||||||
|
KnowledgeSearchRequest.dense("乱码查询", 5, null));
|
||||||
|
|
||||||
|
assertTrue(hits.isEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void hitWithoutChunkMarkerStillGetsStableIdentity() {
|
||||||
|
PyRagKnowledgeSearchAdapter adapter = new PyRagKnowledgeSearchAdapter(pyRagClient);
|
||||||
|
when(pyRagClient.search(any())).thenReturn(new PyRagSearchResponse(
|
||||||
|
"q",
|
||||||
|
"hybrid",
|
||||||
|
List.of(new PyRagSearchHit(
|
||||||
|
"legacy-doc", "legacy-doc", "legacy-doc", "Legacy",
|
||||||
|
null, "content", 0.42, "REFERENCE")),
|
||||||
|
"REFERENCE",
|
||||||
|
"supported",
|
||||||
|
null));
|
||||||
|
|
||||||
|
List<KnowledgeSearchHit> hits = adapter.search(
|
||||||
|
KnowledgeSearchRequest.dense("q", 3, null));
|
||||||
|
|
||||||
|
assertEquals(1, hits.size());
|
||||||
|
// 无 #chunk-N 标记:chunkIndex 为 null,evidenceKey 保留服务端原值
|
||||||
|
assertNull(hits.get(0).chunkIndex());
|
||||||
|
assertEquals("legacy-doc", hits.get(0).evidenceKey());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,10 +47,25 @@ class RetrievalScoreNormalizerTest {
|
|||||||
assertTrue(hybridRank2 < hybridRank);
|
assertTrue(hybridRank2 < hybridRank);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void rerankScorePassesThroughClamped() {
|
||||||
|
// py-rag rerank 绝对分:quality = score 原样(越大越好),不受 L2/rank 分支影响
|
||||||
|
assertEquals(0.9147, RetrievalScoreNormalizer.toQualityScore(
|
||||||
|
RetrievalScoreLabels.RERANK, 0.9147, 1, 20, 2.0, null), 1e-9);
|
||||||
|
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
|
||||||
|
RetrievalScoreLabels.RERANK, null, 1, 20, 2.0, null), 1e-9);
|
||||||
|
assertEquals(1.0, RetrievalScoreNormalizer.toQualityScore(
|
||||||
|
RetrievalScoreLabels.RERANK, 1.7, 1, 20, 2.0, null), 1e-9);
|
||||||
|
assertEquals(0.0, RetrievalScoreNormalizer.toQualityScore(
|
||||||
|
RetrievalScoreLabels.RERANK, -0.3, 1, 20, 2.0, null), 1e-9);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void canonicalizeAliases() {
|
void canonicalizeAliases() {
|
||||||
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
|
assertEquals(RetrievalScoreLabels.DENSE, RetrievalScoreLabels.canonicalize("l2_distance"));
|
||||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
|
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("rrf_fused"));
|
||||||
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
|
assertEquals(RetrievalScoreLabels.HYBRID, RetrievalScoreLabels.canonicalize("bm25_only_no_dense"));
|
||||||
|
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("rerank"));
|
||||||
|
assertEquals(RetrievalScoreLabels.RERANK, RetrievalScoreLabels.canonicalize("quality_score"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
package com.superbiz.agent.service.retrieval;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
class RrfFusionTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void multiPathAgreementOutranksSinglePathHead() {
|
|
||||||
List<String> dense = List.of("a", "b", "c");
|
|
||||||
List<String> lexical = List.of("c", "b", "d");
|
|
||||||
|
|
||||||
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
|
|
||||||
List.of(
|
|
||||||
new RrfFusion.RankedPath<>("dense", dense, 1.0),
|
|
||||||
new RrfFusion.RankedPath<>("lexical", lexical, 1.0)
|
|
||||||
),
|
|
||||||
60,
|
|
||||||
s -> s
|
|
||||||
);
|
|
||||||
|
|
||||||
// c: dense#3 + lexical#1 ; b: dense#2 + lexical#2 ; a: dense#1 only
|
|
||||||
// With k=60, c edges b slightly, and both beat single-path a.
|
|
||||||
assertEquals("c", fused.get(0).identity());
|
|
||||||
assertEquals("b", fused.get(1).identity());
|
|
||||||
assertEquals("a", fused.get(2).identity());
|
|
||||||
assertTrue(fused.get(0).rrfScore() > fused.get(2).rrfScore());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void pathWeightCanElevateSecondaryPath() {
|
|
||||||
List<String> dense = List.of("a", "b");
|
|
||||||
List<String> lexical = List.of("b", "a");
|
|
||||||
|
|
||||||
List<RrfFusion.Scored<String>> fused = RrfFusion.fuse(
|
|
||||||
List.of(
|
|
||||||
new RrfFusion.RankedPath<>("dense", dense, 1.0),
|
|
||||||
new RrfFusion.RankedPath<>("lexical", lexical, 2.0)
|
|
||||||
),
|
|
||||||
60,
|
|
||||||
s -> s
|
|
||||||
);
|
|
||||||
|
|
||||||
assertEquals("b", fused.get(0).identity());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-48
@@ -1,48 +0,0 @@
|
|||||||
package com.superbiz.agent.service.retrieval;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.service.VectorSearchService;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
||||||
import static org.mockito.Mockito.mock;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
class VectorKnowledgeSearchAdapterHybridTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void adapterMapsStoreHitsWithChunkIdentity() {
|
|
||||||
VectorSearchService vectorSearchService = mock(VectorSearchService.class);
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("pool timeout", 3, "mysql")).thenReturn(List.of(
|
|
||||||
result("id-1",
|
|
||||||
"{\"_source\":\"c.md\",\"docId\":\"c\",\"chunkIndex\":0,\"title\":\"mysql pool timeout\"}",
|
|
||||||
"mysql pool timeout runbook",
|
|
||||||
0.35f)
|
|
||||||
));
|
|
||||||
|
|
||||||
VectorKnowledgeSearchAdapter adapter =
|
|
||||||
new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper());
|
|
||||||
|
|
||||||
List<KnowledgeSearchHit> hits = adapter.search(
|
|
||||||
new KnowledgeSearchRequest("pool timeout", 3, "mysql", KnowledgeSearchMode.HYBRID));
|
|
||||||
|
|
||||||
assertEquals(1, hits.size());
|
|
||||||
assertEquals("c#chunk-0", hits.get(0).evidenceKey());
|
|
||||||
assertEquals("c.md", hits.get(0).source());
|
|
||||||
verify(vectorSearchService).searchSimilarDocuments("pool timeout", 3, "mysql");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static VectorSearchService.SearchResult result(String id, String metadata, String content, float score) {
|
|
||||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
|
||||||
result.setId(id);
|
|
||||||
result.setMetadata(metadata);
|
|
||||||
result.setContent(content);
|
|
||||||
result.setScore(score);
|
|
||||||
result.setRawScore((double) score);
|
|
||||||
result.setScoreLabel("dense");
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +1,14 @@
|
|||||||
package com.superbiz.agent.tool;
|
package com.superbiz.agent.tool;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
|
||||||
import com.superbiz.agent.dto.LookupResult;
|
import com.superbiz.agent.dto.LookupResult;
|
||||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||||
import com.superbiz.agent.service.KnowledgeIndexService;
|
|
||||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
|
||||||
import com.superbiz.agent.service.LookupResultAssembler;
|
import com.superbiz.agent.service.LookupResultAssembler;
|
||||||
import com.superbiz.agent.service.VectorSearchService;
|
import com.superbiz.agent.service.retrieval.KnowledgeSearchHit;
|
||||||
import com.superbiz.agent.service.retrieval.VectorKnowledgeSearchAdapter;
|
import com.superbiz.agent.service.retrieval.KnowledgeSearchPort;
|
||||||
|
import com.superbiz.agent.service.retrieval.KnowledgeSearchRequest;
|
||||||
|
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import org.mockito.Mock;
|
import org.mockito.Mock;
|
||||||
@@ -19,25 +17,26 @@ import org.springframework.test.util.ReflectionTestUtils;
|
|||||||
|
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* LookupKnowledgeTool evidence-first contract tests.
|
* LookupKnowledgeTool evidence-first contract tests.
|
||||||
|
*
|
||||||
|
* <p>检索后端为 py-rag(经 {@link KnowledgeSearchPort} mock);L0 query 理解已下沉服务端,
|
||||||
|
* categoryFilter 恒为 null,走 UNFILTERED_VECTOR 单 attempt 主路径。</p>
|
||||||
*/
|
*/
|
||||||
class LookupKnowledgeToolTest {
|
class LookupKnowledgeToolTest {
|
||||||
|
|
||||||
@Mock
|
@Mock
|
||||||
private KnowledgeIndexService knowledgeIndexService;
|
private KnowledgeSearchPort knowledgeSearchPort;
|
||||||
|
|
||||||
@Mock
|
|
||||||
private VectorSearchService vectorSearchService;
|
|
||||||
|
|
||||||
private LookupKnowledgeTool tool;
|
private LookupKnowledgeTool tool;
|
||||||
|
|
||||||
@@ -50,9 +49,8 @@ class LookupKnowledgeToolTest {
|
|||||||
ReflectionTestUtils.setField(postProcessor, "maxChunksPerDocument", 2);
|
ReflectionTestUtils.setField(postProcessor, "maxChunksPerDocument", 2);
|
||||||
KnowledgeContextPacker contextPacker = new KnowledgeContextPacker();
|
KnowledgeContextPacker contextPacker = new KnowledgeContextPacker();
|
||||||
tool = new LookupKnowledgeTool();
|
tool = new LookupKnowledgeTool();
|
||||||
ReflectionTestUtils.setField(tool, "queryTransformer", new KnowledgeQueryTransformer(knowledgeIndexService));
|
|
||||||
ReflectionTestUtils.setField(tool, "documentRetriever",
|
ReflectionTestUtils.setField(tool, "documentRetriever",
|
||||||
new KnowledgeDocumentRetriever(new VectorKnowledgeSearchAdapter(vectorSearchService, new ObjectMapper())));
|
new KnowledgeDocumentRetriever(knowledgeSearchPort));
|
||||||
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
|
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
|
||||||
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
|
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
|
||||||
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
|
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
|
||||||
@@ -60,17 +58,11 @@ class LookupKnowledgeToolTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void filteredL1SuccessDoesNotRetry() {
|
void lookupSuccessSingleAttemptUnfiltered() {
|
||||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
KnowledgeSearchHit hit = hit("db#chunk-0", "db", 0, "db.md", "Database Doc",
|
||||||
VectorSearchService.SearchResult result = searchResult(
|
"mysql timeout runbook", 0.9);
|
||||||
"vec-1",
|
|
||||||
"{\"_source\":\"db.md\",\"docId\":\"db\",\"chunkIndex\":0,\"title\":\"Database Doc\",\"category\":\"database\"}",
|
|
||||||
"mysql timeout runbook",
|
|
||||||
0.2f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
|
||||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
||||||
.thenReturn(List.of(result));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
||||||
|
|
||||||
@@ -80,76 +72,30 @@ class LookupKnowledgeToolTest {
|
|||||||
assertEquals("db#chunk-0", lookup.getEvidenceBlocks().get(0).getEvidenceKey());
|
assertEquals("db#chunk-0", lookup.getEvidenceBlocks().get(0).getEvidenceKey());
|
||||||
assertNotNull(lookup.getContextPack());
|
assertNotNull(lookup.getContextPack());
|
||||||
assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook"));
|
assertTrue(lookup.getContextPack().getPackedText().contains("mysql timeout runbook"));
|
||||||
assertEquals("FILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
||||||
assertEquals(1, lookup.getRetrievalTrace().getAttempts().size());
|
assertEquals(1, lookup.getRetrievalTrace().getAttempts().size());
|
||||||
assertEquals("PRECISE", lookup.getRelevanceLevel());
|
assertEquals("PRECISE", lookup.getRelevanceLevel());
|
||||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
// 原始 query 直传、无 category 收窄(L0 已下沉 py-rag)
|
||||||
verify(vectorSearchService, never()).searchSimilarDocuments("mysql timeout", 3, null);
|
verify(knowledgeSearchPort).search(new KnowledgeSearchRequest("mysql timeout", 3, null, null));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void filteredLowQualityTriggersRawUnfilteredRetry() {
|
void rerankScoreDrivesRelevanceLevel() {
|
||||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
// rerank 绝对分 0.6:quality 原样采用 → REFERENCE(>=0.5 且 <0.75)
|
||||||
VectorSearchService.SearchResult weak = searchResult(
|
KnowledgeSearchHit hit = hit("ref#chunk-0", "ref", 0, "ref.md", "Reference Doc",
|
||||||
"weak",
|
"reference level content", 0.6);
|
||||||
"{\"_source\":\"weak.md\",\"docId\":\"weak\",\"chunkIndex\":0,\"title\":\"Weak\"}",
|
|
||||||
"weak candidate",
|
|
||||||
1.4f);
|
|
||||||
VectorSearchService.SearchResult strong = searchResult(
|
|
||||||
"strong",
|
|
||||||
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
|
|
||||||
"mysql timeout strong runbook",
|
|
||||||
0.2f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
when(knowledgeSearchPort.search(any())).thenReturn(List.of(hit));
|
||||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
||||||
.thenReturn(List.of(weak));
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
|
||||||
.thenReturn(List.of(strong));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
LookupResult lookup = tool.lookupKnowledge("reference query");
|
||||||
|
|
||||||
assertTrue(lookup.isFound());
|
assertTrue(lookup.isFound());
|
||||||
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
assertEquals("REFERENCE", lookup.getRelevanceLevel());
|
||||||
assertEquals("filtered_vector_low_quality", lookup.getRetrievalTrace().getFallbackReason());
|
|
||||||
assertEquals(2, lookup.getRetrievalTrace().getAttempts().size());
|
|
||||||
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
||||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, "database");
|
|
||||||
verify(vectorSearchService).searchSimilarDocuments("mysql timeout", 3, null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void filteredNoEvidenceTriggersRawUnfilteredRetry() {
|
void emptyHitsYieldNoEvidence() {
|
||||||
KnowledgeEntry entry = entry("db.md", "Database Doc", "mysql", "database");
|
when(knowledgeSearchPort.search(any())).thenReturn(Collections.emptyList());
|
||||||
VectorSearchService.SearchResult strong = searchResult(
|
|
||||||
"strong",
|
|
||||||
"{\"_source\":\"strong.md\",\"docId\":\"strong\",\"chunkIndex\":0,\"title\":\"Strong\"}",
|
|
||||||
"mysql timeout strong runbook",
|
|
||||||
0.2f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("mysql timeout")).thenReturn(hint(entry));
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, "database"))
|
|
||||||
.thenReturn(Collections.emptyList());
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("mysql timeout", 3, null))
|
|
||||||
.thenReturn(List.of(strong));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("mysql timeout");
|
|
||||||
|
|
||||||
assertTrue(lookup.isFound());
|
|
||||||
assertEquals("UNFILTERED_VECTOR_RETRY", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
||||||
assertEquals("filtered_vector_no_evidence", lookup.getRetrievalTrace().getFallbackReason());
|
|
||||||
assertEquals("strong.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void l0HintsDoNotBecomeStandaloneEvidenceWhenL1Fails() {
|
|
||||||
KnowledgeEntry entry = entry("fallback.md", "Fallback Doc", "fallback", "database");
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("fallback")).thenReturn(hint(entry));
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("fallback", 3, "database"))
|
|
||||||
.thenReturn(Collections.emptyList());
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("fallback", 3, null))
|
|
||||||
.thenReturn(Collections.emptyList());
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("fallback");
|
LookupResult lookup = tool.lookupKnowledge("fallback");
|
||||||
|
|
||||||
@@ -157,52 +103,26 @@ class LookupKnowledgeToolTest {
|
|||||||
assertEquals(0, lookup.getEvidenceBlockCount());
|
assertEquals(0, lookup.getEvidenceBlockCount());
|
||||||
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
||||||
assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus());
|
assertEquals("no_evidence", lookup.getRetrievalTrace().getEvidenceStatus());
|
||||||
assertTrue(String.valueOf(lookup.getRetrievalTrace().getQueryHints()).contains("Fallback Doc"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void noL0HintUsesUnfilteredVectorSearch() {
|
|
||||||
VectorSearchService.SearchResult result = searchResult(
|
|
||||||
"vec-1",
|
|
||||||
"{\"_source\":\"perf.md\",\"docId\":\"perf\",\"chunkIndex\":0,\"title\":\"Perf\"}",
|
|
||||||
"performance tuning guide",
|
|
||||||
0.3f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("性能优化"))
|
|
||||||
.thenReturn(KnowledgeIndexService.L0Hint.empty());
|
|
||||||
when(vectorSearchService.searchSimilarDocuments("性能优化", 3, null))
|
|
||||||
.thenReturn(List.of(result));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("性能优化");
|
|
||||||
|
|
||||||
assertTrue(lookup.isFound());
|
|
||||||
assertEquals("UNFILTERED_VECTOR", lookup.getRetrievalTrace().getSelectedAttempt());
|
|
||||||
assertEquals("perf.md", lookup.getEvidenceBlocks().get(0).getSource());
|
|
||||||
verify(vectorSearchService).searchSimilarDocuments("性能优化", 3, null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
|
void preservesRetrievalOrderAndContextPackMetadataWithoutBoostRerank() {
|
||||||
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
KnowledgeSearchHit first = hit("a#chunk-0", "a", 0, "a.md", "Generic",
|
||||||
VectorSearchService.SearchResult first = searchResult(
|
"generic troubleshooting", 0.95);
|
||||||
"a",
|
KnowledgeSearchHit second = hit("b#chunk-0", "b", 0, "b.md", "Payment ERR_TIMEOUT",
|
||||||
"{\"_source\":\"a.md\",\"docId\":\"a\",\"chunkIndex\":0,\"title\":\"Generic\",\"category\":\"other\"}",
|
"payment ERR_TIMEOUT timeout diagnosis", 0.55);
|
||||||
"generic troubleshooting",
|
second = new KnowledgeSearchHit(
|
||||||
0.4f);
|
second.id(), second.content(), second.score(), second.rawScore(), second.scoreLabel(),
|
||||||
VectorSearchService.SearchResult second = searchResult(
|
second.metadataJson(), second.metadata(), second.docId(), second.chunkIndex(),
|
||||||
"b",
|
second.evidenceKey(), second.source(), second.title(), "Payment > Timeout",
|
||||||
"{\"_source\":\"b.md\",\"docId\":\"b\",\"chunkIndex\":0,\"title\":\"Payment ERR_TIMEOUT\",\"breadcrumb\":\"Payment > Timeout\",\"category\":\"payment\"}",
|
2, second.denseDistance());
|
||||||
"payment ERR_TIMEOUT timeout diagnosis",
|
|
||||||
0.45f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
|
when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
|
||||||
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
|
|
||||||
.thenReturn(List.of(first, second));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
||||||
|
|
||||||
assertTrue(lookup.isFound());
|
assertTrue(lookup.isFound());
|
||||||
// originalRank order wins; keyword/domain boost must not promote second over first
|
// originalRank 顺序权威;服务端 rerank 分不得改变主序
|
||||||
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
|
assertEquals("a.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||||
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
|
assertEquals("b.md", lookup.getEvidenceBlocks().get(1).getSource());
|
||||||
assertTrue(lookup.getRerankTrace().getItems().stream()
|
assertTrue(lookup.getRerankTrace().getItems().stream()
|
||||||
@@ -216,21 +136,12 @@ class LookupKnowledgeToolTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void keepsDistinctChunksFromSameSource() {
|
void keepsDistinctChunksFromSameSource() {
|
||||||
KnowledgeEntry entry = entry("shared.md", "Shared", "shared", "payment");
|
KnowledgeSearchHit first = hit("shared#chunk-0", "shared", 0, "shared.md", "Shared",
|
||||||
VectorSearchService.SearchResult first = searchResult(
|
"shared content 1", 0.9);
|
||||||
"a",
|
KnowledgeSearchHit second = hit("shared#chunk-1", "shared", 1, "shared.md", "Shared",
|
||||||
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":0,\"title\":\"Shared\"}",
|
"shared content 2", 0.85);
|
||||||
"shared content 1",
|
|
||||||
0.2f);
|
|
||||||
VectorSearchService.SearchResult second = searchResult(
|
|
||||||
"b",
|
|
||||||
"{\"_source\":\"shared.md\",\"docId\":\"shared\",\"chunkIndex\":1,\"title\":\"Shared\"}",
|
|
||||||
"shared content 2",
|
|
||||||
0.25f);
|
|
||||||
|
|
||||||
when(knowledgeIndexService.analyzeQuery("shared")).thenReturn(hint(entry));
|
when(knowledgeSearchPort.search(any())).thenReturn(List.of(first, second));
|
||||||
when(vectorSearchService.searchSimilarDocuments("shared", 3, "payment"))
|
|
||||||
.thenReturn(List.of(first, second));
|
|
||||||
|
|
||||||
LookupResult lookup = tool.lookupKnowledge("shared");
|
LookupResult lookup = tool.lookupKnowledge("shared");
|
||||||
|
|
||||||
@@ -243,48 +154,29 @@ class LookupKnowledgeToolTest {
|
|||||||
assertEquals("shared content 2", lookup.getEvidenceBlocks().get(1).getContent());
|
assertEquals("shared content 2", lookup.getEvidenceBlocks().get(1).getContent());
|
||||||
}
|
}
|
||||||
|
|
||||||
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) {
|
/** 构造 py-rag 形态的命中:rerank 绝对分 + evidenceKey(docId#chunk-N)。 */
|
||||||
return KnowledgeEntry.builder()
|
private KnowledgeSearchHit hit(String evidenceKey,
|
||||||
.filePath(filePath)
|
String docId,
|
||||||
.title(title)
|
int chunkIndex,
|
||||||
.keywords(List.of(keyword))
|
String source,
|
||||||
.summary(title + " summary")
|
String title,
|
||||||
.category(category)
|
String content,
|
||||||
.build();
|
double score) {
|
||||||
}
|
return new KnowledgeSearchHit(
|
||||||
|
evidenceKey,
|
||||||
private VectorSearchService.SearchResult searchResult(String id,
|
content,
|
||||||
String metadata,
|
score,
|
||||||
String content,
|
score,
|
||||||
float score) {
|
RetrievalScoreLabels.RERANK,
|
||||||
VectorSearchService.SearchResult result = new VectorSearchService.SearchResult();
|
null,
|
||||||
result.setId(id);
|
Map.of(),
|
||||||
result.setMetadata(metadata);
|
docId,
|
||||||
result.setContent(content);
|
chunkIndex,
|
||||||
result.setScore(score);
|
evidenceKey,
|
||||||
result.setRawScore((double) score);
|
source,
|
||||||
result.setScoreLabel("dense");
|
title,
|
||||||
return result;
|
null,
|
||||||
}
|
1,
|
||||||
|
null);
|
||||||
private KnowledgeIndexService.L0Hint hint(KnowledgeEntry... entries) {
|
|
||||||
List<KnowledgeEntry> matches = List.of(entries);
|
|
||||||
List<String> keywords = matches.stream()
|
|
||||||
.flatMap(entry -> entry.getKeywords() == null
|
|
||||||
? java.util.stream.Stream.empty()
|
|
||||||
: entry.getKeywords().stream())
|
|
||||||
.distinct()
|
|
||||||
.toList();
|
|
||||||
List<String> domains = matches.stream()
|
|
||||||
.map(KnowledgeEntry::getCategory)
|
|
||||||
.filter(category -> category != null && !category.isBlank())
|
|
||||||
.distinct()
|
|
||||||
.toList();
|
|
||||||
List<String> titles = matches.stream()
|
|
||||||
.map(KnowledgeEntry::getTitle)
|
|
||||||
.filter(title -> title != null && !title.isBlank())
|
|
||||||
.distinct()
|
|
||||||
.toList();
|
|
||||||
return new KnowledgeIndexService.L0Hint(matches, keywords, domains, keywords, titles);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,73 +0,0 @@
|
|||||||
services:
|
|
||||||
etcd:
|
|
||||||
container_name: milvus-etcd
|
|
||||||
image: quay.io/coreos/etcd:v3.5.18
|
|
||||||
environment:
|
|
||||||
- ETCD_AUTO_COMPACTION_MODE=revision
|
|
||||||
- ETCD_AUTO_COMPACTION_RETENTION=1000
|
|
||||||
- ETCD_QUOTA_BACKEND_BYTES=4294967296
|
|
||||||
- ETCD_SNAPSHOT_COUNT=50000
|
|
||||||
volumes:
|
|
||||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
|
|
||||||
command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
minio:
|
|
||||||
container_name: milvus-minio
|
|
||||||
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
|
|
||||||
environment:
|
|
||||||
MINIO_ACCESS_KEY: minioadmin
|
|
||||||
MINIO_SECRET_KEY: minioadmin
|
|
||||||
ports:
|
|
||||||
- "9001:9001"
|
|
||||||
- "9000:9000"
|
|
||||||
volumes:
|
|
||||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data
|
|
||||||
command: minio server /minio_data --console-address ":9001"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
standalone:
|
|
||||||
container_name: milvus-standalone
|
|
||||||
image: milvusdb/milvus:v2.5.10
|
|
||||||
command: ["milvus", "run", "standalone"]
|
|
||||||
security_opt:
|
|
||||||
- seccomp:unconfined
|
|
||||||
environment:
|
|
||||||
ETCD_ENDPOINTS: etcd:2379
|
|
||||||
MINIO_ADDRESS: minio:9000
|
|
||||||
volumes:
|
|
||||||
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
|
||||||
interval: 30s
|
|
||||||
start_period: 90s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
ports:
|
|
||||||
- "19530:19530"
|
|
||||||
- "9091:9091"
|
|
||||||
depends_on:
|
|
||||||
- "etcd"
|
|
||||||
- "minio"
|
|
||||||
# 这是新增的 Attu 服务哦!
|
|
||||||
attu:
|
|
||||||
container_name: milvus-attu
|
|
||||||
image: zilliz/attu:v2.5
|
|
||||||
ports:
|
|
||||||
- "8000:3000" # 把本地的 8000 端口映射到容器的 3000 端口 (Attu 默认端口)
|
|
||||||
environment:
|
|
||||||
# MILVUS_URL 指向 Docker 网络里的 Milvus standalone 服务
|
|
||||||
MILVUS_URL: standalone:19530
|
|
||||||
depends_on:
|
|
||||||
- standalone # 确保 Milvus 启动后再启动 Attu
|
|
||||||
networks:
|
|
||||||
default:
|
|
||||||
name: milvus
|
|
||||||
Reference in New Issue
Block a user