> details) {
+ }
+ }
+}
diff --git a/src/main/java/com/superbiz/agent/client/PyRagClientException.java b/src/main/java/com/superbiz/agent/client/PyRagClientException.java
new file mode 100644
index 0000000..807c912
--- /dev/null
+++ b/src/main/java/com/superbiz/agent/client/PyRagClientException.java
@@ -0,0 +1,25 @@
+package com.superbiz.agent.client;
+
+import lombok.Getter;
+
+/**
+ * py-rag 调用异常：携带契约错误码（E_*）与 HTTP 状态。
+ *
+ * 调用方按错误码分支：422 参数/数据问题不重试；
+ * 409 E_REBUILD_IN_PROGRESS 延迟重试；网络异常（E_NETWORK）可安全重试。
+ */
+@Getter
+public class PyRagClientException extends RuntimeException {
+
+ /** 契约错误码：E_INVALID_REQUEST / E_FRONTMATTER_INVALID / E_REBUILD_IN_PROGRESS / E_NETWORK 等 */
+ private final String code;
+
+ /** HTTP 状态码；网络层异常（未拿到响应）为 null */
+ private final Integer httpStatus;
+
+ public PyRagClientException(String code, String message, Integer httpStatus, Throwable cause) {
+ super("[" + code + "] " + message, cause);
+ this.code = code;
+ this.httpStatus = httpStatus;
+ }
+}
diff --git a/src/main/java/com/superbiz/agent/config/DocumentChunkConfig.java b/src/main/java/com/superbiz/agent/config/DocumentChunkConfig.java
deleted file mode 100644
index 240352e..0000000
--- a/src/main/java/com/superbiz/agent/config/DocumentChunkConfig.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package com.superbiz.agent.config;
-
-import lombok.Getter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-/**
- * 文档分片配置
- */
-@Getter
-@Configuration
-@ConfigurationProperties(prefix = "document.chunk")
-public class DocumentChunkConfig {
-
- /**
- * 每个分片的最大字符数（保留向后兼容）
- */
- private int maxSize = 800;
-
- /**
- * 分片之间的重叠字符数
- */
- private int overlap = 100;
-
- /**
- * 每个分片的最大 token 数（中文~1:1，英文~0.25:1）
- * 替代 maxSize 作为切割触发器
- */
- private int maxTokens = 500;
-
- /**
- * 硬上限 token 数 = maxTokens × 1.2
- * 仅在不可中断上下文（列表、代码块）内触发
- */
- private int maxTokensHard = 600;
-
- public void setMaxSize(int maxSize) {
- this.maxSize = maxSize;
- }
-
- public void setOverlap(int overlap) {
- this.overlap = overlap;
- }
-
- public void setMaxTokens(int maxTokens) {
- this.maxTokens = maxTokens;
- }
-
- public void setMaxTokensHard(int maxTokensHard) {
- this.maxTokensHard = maxTokensHard;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/config/MilvusConfig.java b/src/main/java/com/superbiz/agent/config/MilvusConfig.java
deleted file mode 100644
index 3a07993..0000000
--- a/src/main/java/com/superbiz/agent/config/MilvusConfig.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.superbiz.agent.config;
-
-import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Configuration;
-
-/**
- * Milvus 知识路径配置说明（无额外 Bean 装配）。
- *
- * 知识库 RAG 唯一实现：{@link MilvusHybridKnowledgeStore}（{@code MilvusClientV2}）。
- *
- * - 支持 dense 与 dense+BM25 {@code hybridSearch}+RRF。
- * - 不再为知识路径创建 legacy {@code MilvusServiceClient} Bean。
- * - Spring AI {@code VectorStore} starter 仍可存在于 classpath，但只作 sidecar，
- * 不作 lookup_knowledge 主路径（starter 无 BM25 hybrid API）。
- *
- */
-@Configuration
-public class MilvusConfig {
-
- private static final Logger logger = LoggerFactory.getLogger(MilvusConfig.class);
-
- public MilvusConfig() {
- logger.info("Milvus knowledge path: MilvusClientV2 hybrid store only (legacy SDK search disabled)");
- }
-}
diff --git a/src/main/java/com/superbiz/agent/config/MilvusProperties.java b/src/main/java/com/superbiz/agent/config/MilvusProperties.java
deleted file mode 100644
index 628a828..0000000
--- a/src/main/java/com/superbiz/agent/config/MilvusProperties.java
+++ /dev/null
@@ -1,95 +0,0 @@
-package com.superbiz.agent.config;
-
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@ConfigurationProperties(prefix = "milvus")
-public class MilvusProperties {
-
- private String host = "localhost";
- private Integer port = 19530;
- private String username = "";
- private String password = "";
- private String database = "default";
- private Long timeout = 10000L;
- private String token = "";
- private boolean secure = false;
- private int vectorDim = 1024;
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public Integer getPort() {
- return port;
- }
-
- public void setPort(Integer port) {
- this.port = port;
- }
-
- public String getUsername() {
- return username;
- }
-
- public void setUsername(String username) {
- this.username = username;
- }
-
- public String getPassword() {
- return password;
- }
-
- public void setPassword(String password) {
- this.password = password;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public void setDatabase(String database) {
- this.database = database;
- }
-
- public Long getTimeout() {
- return timeout;
- }
-
- public void setTimeout(Long timeout) {
- this.timeout = timeout;
- }
-
- public String getToken() {
- return token;
- }
-
- public void setToken(String token) {
- this.token = token;
- }
-
- public boolean isSecure() {
- return secure;
- }
-
- public void setSecure(boolean secure) {
- this.secure = secure;
- }
-
- public int getVectorDim() {
- return vectorDim;
- }
-
- public void setVectorDim(int vectorDim) {
- this.vectorDim = vectorDim;
- }
-
- public String getAddress() {
- return host + ":" + port;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/config/ModelRoutingConfig.java b/src/main/java/com/superbiz/agent/config/ModelRoutingConfig.java
index 1b54a59..055d282 100644
--- a/src/main/java/com/superbiz/agent/config/ModelRoutingConfig.java
+++ b/src/main/java/com/superbiz/agent/config/ModelRoutingConfig.java
@@ -1,12 +1,10 @@
package com.superbiz.agent.config;
import java.util.List;
-import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.model.ChatModel;
-import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -19,11 +17,11 @@ import org.springframework.context.annotation.Primary;
* {@code
* model-routing:
* chat: deepseek
- * embedding: siliconflow
* }
*
* 匹配优先级：Bean 名 > 类名（均不区分大小写）。
* 切换模型只改 yml + pom + 对应 api-key，Java 代码不动。
+ * （Embedding 路由已随 RAG 模块抽离至 py-rag 服务端，此处仅路由 Chat。）
*/
@Configuration
public class ModelRoutingConfig {
@@ -33,9 +31,6 @@ public class ModelRoutingConfig {
@Value("${model-routing.chat:deepseek}")
private String chatKeyword;
- @Value("${model-routing.embedding:siliconflow}")
- private String embeddingKeyword;
-
@Bean
@Primary
public ChatModel chatModel(List chatModels) {
@@ -53,33 +48,6 @@ public class ModelRoutingConfig {
return chatModels.get(0);
}
- @Bean
- @Primary
- public EmbeddingModel embeddingModel(Map embeddingBeans) {
- log.info("Embedding 路由: keyword='{}', 可用: {}", embeddingKeyword, embeddingBeans.keySet());
-
- // 先按 Bean 名匹配
- for (Map.Entry entry : embeddingBeans.entrySet()) {
- if (containsIgnoreCase(entry.getKey(), embeddingKeyword)) {
- log.info(" → Bean 名匹配: {} → {}", entry.getKey(),
- entry.getValue().getClass().getSimpleName());
- return entry.getValue();
- }
- }
-
- // 再按类名匹配
- for (EmbeddingModel em : embeddingBeans.values()) {
- if (matches(em.getClass(), embeddingKeyword)) {
- log.info(" → 类名匹配: {}", em.getClass().getSimpleName());
- return em;
- }
- }
-
- var first = embeddingBeans.values().iterator().next();
- log.warn(" → 未匹配, 回退到 {}", first.getClass().getSimpleName());
- return first;
- }
-
private boolean matches(Class clazz, String keyword) {
return containsIgnoreCase(clazz.getName(), keyword)
|| containsIgnoreCase(clazz.getSimpleName(), keyword);
diff --git a/src/main/java/com/superbiz/agent/config/PyRagProperties.java b/src/main/java/com/superbiz/agent/config/PyRagProperties.java
new file mode 100644
index 0000000..037bc62
--- /dev/null
+++ b/src/main/java/com/superbiz/agent/config/PyRagProperties.java
@@ -0,0 +1,34 @@
+package com.superbiz.agent.config;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * py-rag 知识服务接入配置。
+ *
+ * 超时矩阵来自《py-rag 知识服务 · Java 接入文档》第 6 节：
+ * 服务含 embedding/rerank 外呼，search 正常 300–800ms、ingest 正常 1–5s。
+ */
+@Getter
+@Setter
+@Configuration
+@ConfigurationProperties(prefix = "pyrag")
+public class PyRagProperties {
+
+ /** py-rag 服务根地址，如 http://py-rag:8000 */
+ private String baseUrl = "http://localhost:8000";
+
+ /** 连接超时（毫秒），全端点统一 */
+ private int connectTimeoutMs = 3000;
+
+ /** /api/v1/search 读取超时（毫秒） */
+ private int searchReadTimeoutMs = 5000;
+
+ /** /api/v1/documents:ingest 读取超时（毫秒） */
+ private int ingestReadTimeoutMs = 30000;
+
+ /** rebuild/tasks/stats/health 读取超时（毫秒） */
+ private int defaultReadTimeoutMs = 10000;
+}
diff --git a/src/main/java/com/superbiz/agent/config/RagSidecarProperties.java b/src/main/java/com/superbiz/agent/config/RagSidecarProperties.java
deleted file mode 100644
index eb8dd34..0000000
--- a/src/main/java/com/superbiz/agent/config/RagSidecarProperties.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.superbiz.agent.config;
-
-import lombok.Getter;
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.context.annotation.Configuration;
-
-@Getter
-@Configuration
-@ConfigurationProperties(prefix = "rag.sidecar.spring-ai")
-public class RagSidecarProperties {
-
- private boolean enabled = false;
-
- private int contentPreviewLimit = 300;
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-
- public void setContentPreviewLimit(int contentPreviewLimit) {
- this.contentPreviewLimit = contentPreviewLimit;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/config/SiliconFlowEmbeddingConfig.java b/src/main/java/com/superbiz/agent/config/SiliconFlowEmbeddingConfig.java
deleted file mode 100644
index 7af1d8a..0000000
--- a/src/main/java/com/superbiz/agent/config/SiliconFlowEmbeddingConfig.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package com.superbiz.agent.config;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.ai.document.MetadataMode;
-import org.springframework.ai.embedding.EmbeddingModel;
-import org.springframework.ai.openai.OpenAiEmbeddingModel;
-import org.springframework.ai.openai.OpenAiEmbeddingOptions;
-import org.springframework.ai.openai.api.OpenAiApi;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.web.client.RestClient;
-import org.springframework.web.reactive.function.client.WebClient;
-
-/**
- * SiliconFlow Embedding 配置（BGE-M3, OpenAI 兼容协议, 1024维）
- *
- * Chat 走 DeepSeek、Embedding 走 SiliconFlow，两者都是 OpenAI 兼容但地址不同，
- * 因此单独为 SiliconFlow 创建 OpenAiApi + EmbeddingModel Bean。
- */
-@Configuration
-public class SiliconFlowEmbeddingConfig {
-
- private static final Logger log = LoggerFactory.getLogger(SiliconFlowEmbeddingConfig.class);
-
- @Value("${siliconflow.api-key}")
- private String apiKey;
-
- @Value("${siliconflow.base-url}")
- private String baseUrl;
-
- @Value("${siliconflow.embedding.model}")
- private String model;
-
- @Bean
- public OpenAiApi siliconFlowApi(RestClient.Builder restClientBuilder, WebClient.Builder webClientBuilder) {
- log.info("创建 SiliconFlow OpenAiApi: {}", baseUrl);
- return OpenAiApi.builder()
- .baseUrl(baseUrl)
- .apiKey(apiKey)
- .restClientBuilder(restClientBuilder)
- .build();
- }
-
- @Bean
- public EmbeddingModel siliconFlowEmbeddingModel(OpenAiApi siliconFlowApi) {
- log.info("创建 SiliconFlow EmbeddingModel, model: {}", model);
- return new OpenAiEmbeddingModel(siliconFlowApi, MetadataMode.EMBED,
- OpenAiEmbeddingOptions.builder()
- .model(model)
- .build());
- }
-}
diff --git a/src/main/java/com/superbiz/agent/constant/MilvusConstants.java b/src/main/java/com/superbiz/agent/constant/MilvusConstants.java
deleted file mode 100644
index 99e0a21..0000000
--- a/src/main/java/com/superbiz/agent/constant/MilvusConstants.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package com.superbiz.agent.constant;
-
-public class MilvusConstants {
-
- /**
- * Milvus 数据库名称
- */
- public static final String MILVUS_DB_NAME = "default";
-
- /**
- * Default knowledge collection name (dense + BM25).
- * Overridable via {@code milvus.collection}.
- */
- public static final String MILVUS_COLLECTION_NAME = "biz";
-
- /**
- * Alias kept for readability in hybrid-related code.
- */
- public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
-
- /**
- * 向量维度（豆包 embedding 模型的维度）
- */
- public static final int VECTOR_DIM = 1024; // 豆包模型返回1024维向量
-
- /**
- * ID字段最大长度
- */
- public static final int ID_MAX_LENGTH = 256;
-
- /**
- * Content字段最大长度
- */
- public static final int CONTENT_MAX_LENGTH = 8192;
-
- /**
- * 默认分片数
- */
- public static final int DEFAULT_SHARD_NUMBER = 2;
-
- private MilvusConstants() {
- // 工具类，禁止实例化
- }
-}
diff --git a/src/main/java/com/superbiz/agent/controller/FileUploadController.java b/src/main/java/com/superbiz/agent/controller/FileUploadController.java
index 3b5b91e..b87569c 100644
--- a/src/main/java/com/superbiz/agent/controller/FileUploadController.java
+++ b/src/main/java/com/superbiz/agent/controller/FileUploadController.java
@@ -1,8 +1,8 @@
package com.superbiz.agent.controller;
+import com.superbiz.agent.client.PyRagClient;
import com.superbiz.agent.config.FileUploadConfig;
import com.superbiz.agent.dto.FileUploadRes;
-import com.superbiz.agent.service.VectorIndexService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -29,10 +29,11 @@ public class FileUploadController {
private FileUploadConfig fileUploadConfig;
@Autowired
- private VectorIndexService vectorIndexService;
+ private PyRagClient pyRagClient;
@PostMapping(value = "/api/upload", consumes = "multipart/form-data")
- public ResponseEntity upload(@RequestParam("file") MultipartFile file) {
+ public ResponseEntity upload(@RequestParam("file") MultipartFile file,
+ @RequestParam(value = "category", required = false) String category) {
if (file.isEmpty()) {
return ResponseEntity.badRequest().body("文件不能为空");
}
@@ -68,15 +69,17 @@ public class FileUploadController {
logger.info("文件上传成功: {}", filePath);
- // 文件上传成功后，自动调用向量索引服务
+ // 转发 py-rag 入库（同内容重传返回 unchanged）。入库失败不影响上传成功语义。
try {
- logger.info("开始为上传文件创建向量索引: {}", filePath);
- vectorIndexService.indexSingleFile(filePath.toString());
- logger.info("向量索引创建成功: {}", filePath);
+ String ingestCategory = (category == null || category.isBlank()) ? "default" : category;
+ logger.info("开始 py-rag 入库: {}, category={}", filePath, ingestCategory);
+ var ingest = pyRagClient.ingest(originalFilename, file.getBytes(), file.getContentType(),
+ ingestCategory, null, null, null);
+ logger.info("py-rag 入库完成: docId={}, status={}, chunks={}",
+ ingest.docId(), ingest.status(), ingest.chunkCount());
} catch (Exception e) {
- logger.error("向量索引创建失败: {}, 错误: {}", filePath, e.getMessage(), e);
- // 注意：即使索引失败，文件上传仍然成功，只是记录错误日志
- // 可以根据业务需求决定是否要删除文件或返回错误
+ logger.error("py-rag 入库失败: {}, 错误: {}", filePath, e.getMessage(), e);
+ // 注意：即使入库失败，文件上传仍然成功，只是记录错误日志
}
FileUploadRes response = new FileUploadRes(
diff --git a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java b/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java
deleted file mode 100644
index c19fe8b..0000000
--- a/src/main/java/com/superbiz/agent/controller/KnowledgeBaseController.java
+++ /dev/null
@@ -1,147 +0,0 @@
-package com.superbiz.agent.controller;
-
-import com.superbiz.agent.service.KnowledgeBaseInitService;
-import lombok.Data;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * 知识库管理控制器
- * 提供知识库初始化、查询等接口
- */
-@RestController
-@RequestMapping("/api/knowledge")
-public class KnowledgeBaseController {
-
- private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
-
- @Autowired
- private KnowledgeBaseInitService initService;
-
- /**
- * 初始化知识库
- * 扫描 knowledge_base 目录下的所有文档，去重后批量导入到数据库和 Milvus
- *
- * @param force 是否强制重新导入（跳过去重检查）
- * @return 初始化结果
- */
- @PostMapping("/init")
- public ResponseEntity initKnowledgeBase(@RequestParam(defaultValue = "false") boolean force) {
- logger.info("收到知识库初始化请求, force={}", force);
-
- try {
- KnowledgeBaseInitService.InitResult result = initService.initializeKnowledgeBase(force);
-
- Map response = new HashMap<>();
- response.put("success", true);
- response.put("message", "知识库初始化完成");
- response.put("scanned", result.getScanned());
- response.put("skipped", result.getSkipped());
- response.put("inserted", result.getInserted());
- response.put("failed", result.getFailed());
- response.put("details", result.getDetails());
-
- logger.info("知识库初始化成功: 扫描={}, 跳过={}, 新增={}, 失败={}",
- result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
-
- return ResponseEntity.ok(response);
-
- } catch (Exception e) {
- logger.error("知识库初始化失败", e);
-
- Map response = new HashMap<>();
- response.put("success", false);
- response.put("message", "初始化失败: " + e.getMessage());
-
- return ResponseEntity.internalServerError().body(response);
- }
- }
-
- /**
- * 清空 hybrid collection + MySQL api_document + L0 内存索引，
- * 再从 knowledge_base 全量重建 dense+BM25 索引。
- *
- * 危险操作：会删除 {@code milvus.collection}（默认 {@code biz}）与文档元数据表数据。
- * 需要显式 confirm=REBUILD。
- */
- @PostMapping("/rebuild-hybrid")
- public ResponseEntity rebuildHybrid(
- @RequestParam(defaultValue = "") String confirm) {
- if (!"REBUILD".equals(confirm)) {
- Map rejected = new HashMap<>();
- rejected.put("success", false);
- rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建");
- rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
- return ResponseEntity.badRequest().body(rejected);
- }
-
- logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
- try {
- KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
- Map response = new HashMap<>();
- response.put("success", result.isSuccess());
- response.put("message", result.isSuccess()
- ? "hybrid 知识库重建完成"
- : "hybrid 知识库重建结束，但存在失败项");
- response.put("collection", result.getCollection());
- response.put("basePath", result.getBasePath());
- response.put("milvus", result.getMilvus());
- response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
- response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
- response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
- response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
- response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
- if (result.getInit() != null) {
- response.put("scanned", result.getInit().getScanned());
- response.put("skipped", result.getInit().getSkipped());
- response.put("inserted", result.getInit().getInserted());
- response.put("failed", result.getInit().getFailed());
- response.put("details", result.getInit().getDetails());
- }
- return result.isSuccess()
- ? ResponseEntity.ok(response)
- : ResponseEntity.status(500).body(response);
- } catch (Exception e) {
- logger.error("hybrid 知识库重建失败", e);
- Map response = new HashMap<>();
- response.put("success", false);
- response.put("message", "重建失败: " + e.getMessage());
- return ResponseEntity.internalServerError().body(response);
- }
- }
-
- /**
- * 查询知识库统计信息
- *
- * @return 统计信息
- */
- @GetMapping("/stats")
- public ResponseEntity getStats() {
- try {
- KnowledgeBaseInitService.Stats stats = initService.getStats();
-
- Map response = new HashMap<>();
- response.put("success", true);
- response.put("totalDocuments", stats.getTotalDocuments());
- response.put("totalVectors", stats.getTotalVectors());
- response.put("categories", stats.getCategoryCount());
-
- return ResponseEntity.ok(response);
-
- } catch (Exception e) {
- logger.error("查询统计信息失败", e);
-
- Map response = new HashMap<>();
- response.put("success", false);
- response.put("message", "查询失败: " + e.getMessage());
-
- return ResponseEntity.internalServerError().body(response);
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/controller/MilvusCheckController.java b/src/main/java/com/superbiz/agent/controller/MilvusCheckController.java
deleted file mode 100644
index 458b604..0000000
--- a/src/main/java/com/superbiz/agent/controller/MilvusCheckController.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.superbiz.agent.controller;
-
-import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
-import io.milvus.v2.service.collection.response.ListCollectionsResp;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Milvus health check using the single V2 knowledge backend.
- */
-@RestController
-@RequestMapping("/milvus")
-public class MilvusCheckController {
-
- @Autowired
- private MilvusHybridKnowledgeStore knowledgeStore;
-
- @GetMapping("/health")
- public ResponseEntity> simpleHealth() {
- Map result = new HashMap<>();
- try {
- ListCollectionsResp response = knowledgeStore.client().listCollections();
- result.put("message", "ok");
- result.put("backend", "milvus-client-v2");
- result.put("knowledgeCollection", knowledgeStore.collectionName());
- result.put("collections", response == null ? null : response.getCollectionNames());
- return ResponseEntity.ok(result);
- } catch (Exception e) {
- result.put("error", e.getMessage());
- return ResponseEntity.status(503).body(result);
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/controller/SearchController.java b/src/main/java/com/superbiz/agent/controller/SearchController.java
deleted file mode 100644
index 52b23a9..0000000
--- a/src/main/java/com/superbiz/agent/controller/SearchController.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package com.superbiz.agent.controller;
-
-import com.superbiz.agent.dto.Result;
-import com.superbiz.agent.service.VectorSearchService;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
-
-import java.util.List;
-
-/**
- * 文档检索控制器（测试用）
- */
-@Slf4j
-@RestController
-@RequestMapping("/api/search")
-public class SearchController {
-
- @Autowired
- private VectorSearchService vectorSearchService;
-
- /**
- * 搜索相似文档
- */
- @GetMapping("/similar")
- public Result> searchSimilar(
- @RequestParam("query") String query,
- @RequestParam(value = "topK", defaultValue = "5") int topK,
- @RequestParam(value = "category", required = false) String category
- ) {
- try {
- log.info("收到检索请求，query: {}, topK: {}, category: {}", query, topK, category);
- List results = vectorSearchService.searchSimilarDocuments(query, topK, category);
- log.info("检索完成，返回 {} 条结果", results.size());
- return Result.success(results);
-
- } catch (Exception e) {
- log.error("检索失败", e);
- return Result.error(500, "检索失败: " + e.getMessage());
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/dto/DocumentChunk.java b/src/main/java/com/superbiz/agent/dto/DocumentChunk.java
deleted file mode 100644
index 7ea2662..0000000
--- a/src/main/java/com/superbiz/agent/dto/DocumentChunk.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package com.superbiz.agent.dto;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-/**
- * 文档分片
- */
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class DocumentChunk {
-
- /**
- * 分片内容
- */
- private String content;
-
- /**
- * 分片在原文档中的起始位置
- */
- private int startOffset;
-
- /**
- * 分片在原文档中的结束位置
- */
- private int endOffset;
-
- /**
- * 分片序号（从0开始）
- */
- private int chunkIndex;
-
- /**
- * 分片标题或上下文信息
- */
- private String title;
-
- /**
- * 面包屑导航（完整标题层级路径）
- * 例如: "故障诊断流程规范 > 应急响应流程 > 1. 初步评估"
- */
- private String breadcrumb;
-}
diff --git a/src/main/java/com/superbiz/agent/dto/Frontmatter.java b/src/main/java/com/superbiz/agent/dto/Frontmatter.java
deleted file mode 100644
index 5e97045..0000000
--- a/src/main/java/com/superbiz/agent/dto/Frontmatter.java
+++ /dev/null
@@ -1,80 +0,0 @@
-package com.superbiz.agent.dto;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-import java.time.LocalDate;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Frontmatter 数据模型
- * 用于解析 Markdown 文件头的 YAML frontmatter
- */
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class Frontmatter {
-
- /**
- * 文档标题（必填）
- */
- private String title;
-
- /**
- * 关键词列表（必填，用于 L0 精确匹配）
- */
- private List keywords;
-
- /**
- * 文档摘要（必填）
- */
- private String summary;
-
- /**
- * 文档类别（可选）
- */
- private String category;
-
- private String source;
-
- private String breadcrumb;
-
- @JsonProperty("kb_scope")
- private String kbScope;
-
- /**
- * 章节锚点（预留字段，MVP 不使用）
- * Key: 章节标题，Value: 章节 Markdown 标题
- */
- private Map sections;
-
- /**
- * 版本号（预留字段）
- */
- private String version;
-
- /**
- * 作者（预留字段）
- */
- private String author;
-
- /**
- * 最后更新日期（预留字段）
- */
- private LocalDate lastUpdated;
-
- /**
- * 业务场景标签，供 Planner 决策用（LLM 上传时自动生成）
- */
- private List covers;
-
- /**
- * 文档级检索时机（LLM 上传时自动生成）
- */
- private String whenToRetrieve;
-}
diff --git a/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java b/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java
deleted file mode 100644
index 5335a73..0000000
--- a/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package com.superbiz.agent.dto;
-
-import lombok.Builder;
-import lombok.Data;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * 知识库索引条目
- * L0 内存索引使用的数据结构
- */
-@Data
-@Builder
-public class KnowledgeEntry {
-
- /**
- * 文件路径（如：knowledge_base/api/payment-errors.md）
- */
- private String filePath;
-
- /**
- * 文档标题
- */
- private String title;
-
- /**
- * 关键词列表（用于精确匹配）
- */
- private List keywords;
-
- /**
- * 文档摘要
- */
- private String summary;
-
- /**
- * 文档类别（如：api、domain、troubleshooting）
- */
- private String category;
-
- private String kbScope;
-
- /**
- * 章节锚点（预留字段，MVP 不使用）
- */
- private Map sections;
-
- /**
- * 业务场景标签，供 Planner 决策用
- */
- private List covers;
-
- /**
- * 文档级检索时机
- */
- private String whenToRetrieve;
-}
diff --git a/src/main/java/com/superbiz/agent/dto/KnowledgeQuery.java b/src/main/java/com/superbiz/agent/dto/KnowledgeQuery.java
index 86a2123..71c729e 100644
--- a/src/main/java/com/superbiz/agent/dto/KnowledgeQuery.java
+++ b/src/main/java/com/superbiz/agent/dto/KnowledgeQuery.java
@@ -6,9 +6,10 @@ import lombok.Data;
import java.util.List;
/**
- * 检索前 query understanding 的输出（L0 -> pipeline 控制面）。
+ * 检索 pipeline 控制面参数。
*
- * 由 {@code KnowledgeQueryTransformer} 生成，供 L1 过滤、规则 rerank 与 trace 使用。
+ *
L0 query 理解已下沉 py-rag 服务端；当前 {@code originalQuery} = {@code rewrittenQuery}、
+ * hint 字段恒为空、{@code categoryFilter} 恒为 null，结构保留供后处理与 trace 使用。
* 不是 Agent 可见契约。
*/
@Data
diff --git a/src/main/java/com/superbiz/agent/service/DocumentChunkService.java b/src/main/java/com/superbiz/agent/service/DocumentChunkService.java
deleted file mode 100644
index fe98b64..0000000
--- a/src/main/java/com/superbiz/agent/service/DocumentChunkService.java
+++ /dev/null
@@ -1,446 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.config.DocumentChunkConfig;
-import com.superbiz.agent.dto.DocumentChunk;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * 文档切片服务（RAG 入库前处理）。
- *
- * 把长 Markdown/文本切成带 title/breadcrumb 的 {@link com.superbiz.agent.dto.DocumentChunk}，
- * 供 {@link VectorIndexService} 向量化。
- *
- * 策略摘要
- *
- * - 先按 Markdown 标题分 section，并维护 breadcrumb 层级
- * - section 过长再按段落累积；用 token 估算做软边界 / 硬上限
- * - 尽量不在有序/无序列表或未闭合代码块中间切断
- * - 相邻 chunk 保留 overlap，减轻边界语义断裂
- *
- *
- * 检索命中单个 chunk 后，当前主链路不会自动回补同章节相邻 chunk
- * （上下文重建仍是后续增强点）。
- */
-@Service
-public class DocumentChunkService {
-
- private static final Logger logger = LoggerFactory.getLogger(DocumentChunkService.class);
-
- @Autowired
- private DocumentChunkConfig chunkConfig;
-
- /**
- * 智能分片文档
- * 优先按照标题、段落边界进行分割，保持语义完整性
- *
- * @param content 文档内容
- * @param filePath 文件路径（用于日志）
- * @return 文档分片列表
- */
- public List chunkDocument(String content, String filePath) {
- List chunks = new ArrayList<>();
-
- if (content == null || content.trim().isEmpty()) {
- logger.warn("文档内容为空: {}", filePath);
- return chunks;
- }
-
- // 1. 首先尝试按标题分割（Markdown格式）
- List sections = splitByHeadings(content);
-
- // 2. 对每个章节进行进一步分片
- int globalChunkIndex = 0;
- for (Section section : sections) {
- List sectionChunks = chunkSection(section, globalChunkIndex);
- chunks.addAll(sectionChunks);
- globalChunkIndex += sectionChunks.size();
- }
-
- logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
- return chunks;
- }
-
- /**
- * 按照 Markdown 标题分割文档，同时构建面包屑层级路径
- */
- private List splitByHeadings(String content) {
- List sections = new ArrayList<>();
-
- // 匹配 Markdown 标题：# 标题, ## 标题, ### 标题等
- Pattern headingPattern = Pattern.compile("^(#{1,6})\\s+(.+)$", Pattern.MULTILINE);
- Matcher matcher = headingPattern.matcher(content);
-
- // 标题层级栈：维护当前标题的完整路径
- List headingStack = new ArrayList<>();
- int lastEnd = 0;
- String currentBreadcrumb = null;
-
- while (matcher.find()) {
- int level = matcher.group(1).length(); // #→1, ##→2, ###→3 ...
- String title = matcher.group(2).trim();
-
- // 保存上一个章节
- if (lastEnd < matcher.start()) {
- String sectionContent = content.substring(lastEnd, matcher.start()).trim();
- if (!sectionContent.isEmpty()) {
- sections.add(new Section(
- headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
- level,
- currentBreadcrumb,
- sectionContent,
- lastEnd));
- }
- }
-
- // 维护层级栈：同级别或更高级别 → 弹出，低级 → 追加
- while (!headingStack.isEmpty() && headingStack.size() >= level) {
- headingStack.remove(headingStack.size() - 1);
- }
- headingStack.add(title);
- currentBreadcrumb = String.join(" > ", headingStack);
- lastEnd = matcher.start();
- }
-
- // 添加最后一个章节
- if (lastEnd < content.length()) {
- String sectionContent = content.substring(lastEnd).trim();
- if (!sectionContent.isEmpty()) {
- sections.add(new Section(
- headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
- headingStack.size(),
- currentBreadcrumb,
- sectionContent,
- lastEnd));
- }
- }
-
- // 如果没有找到任何标题，将整个文档作为一个章节
- if (sections.isEmpty()) {
- sections.add(new Section(null, 0, null, content, 0));
- }
-
- return sections;
- }
-
- /**
- * 对单个章节进行分片
- *
- * 核心改造（Phase 1）：
- * - Token 估算替代字符计数
- * - 感知有序/无序列表结构，不在列表中间切断
- * - 软边界（maxTokens）+ 硬上限（maxTokensHard）双重控制
- * - 修复 currentStartIndex 漂移：用段落原始位置而非手工推算
- */
- private List chunkSection(Section section, int startChunkIndex) {
- List chunks = new ArrayList<>();
- String content = section.content;
- String title = section.title;
- String breadcrumb = section.breadcrumb;
-
- // 短章节直接作为一个分片（用 token 估算替代字符数做短路判断）
- if (content.length() <= chunkConfig.getMaxSize()
- && estimateTokens(content) <= chunkConfig.getMaxTokens()) {
- DocumentChunk chunk = DocumentChunk.builder()
- .content(content)
- .startOffset(section.startIndex)
- .endOffset(section.startIndex + content.length())
- .chunkIndex(startChunkIndex)
- .title(title)
- .breadcrumb(breadcrumb)
- .build();
- chunks.add(chunk);
- return chunks;
- }
-
- // 章节内容较长，需要进一步分片
- List paragraphs = splitByParagraphs(content);
- if (paragraphs.isEmpty()) {
- return chunks;
- }
-
- // 定位每个段落在 section.content 中的位置（修复 index 漂移）
- List paraPositions = locateParagraphPositions(paragraphs, content);
-
- // 当前分片的段落范围
- int chunkParaStart = 0; // 当前分片第一个段落的索引（在 paragraphs 中）
- StringBuilder buffer = new StringBuilder();
- int tokenCount = 0;
- int chunkIndex = startChunkIndex;
-
- for (int i = 0; i < paragraphs.size(); i++) {
- String paragraph = paragraphs.get(i);
- int paraTokens = estimateTokens(paragraph);
-
- // 判断是否需要切分
- if (buffer.length() > 0 && tokenCount + paraTokens > chunkConfig.getMaxTokens()) {
-
- // 检查是否处于不可中断的上下文中
- if (isInUnbreakableContext(buffer.toString(), paragraph)) {
- // 硬上限保护：即使不可中断也不能无限膨胀
- if (tokenCount + paraTokens > chunkConfig.getMaxTokensHard()) {
- logger.debug(" 触及硬上限 ({} tokens)，强制切分", tokenCount + paraTokens);
- chunkParaStart = saveChunkAndGetNextStart(
- chunks, section, paraPositions,
- chunkParaStart, i, title, breadcrumb, chunkIndex);
- chunkIndex++;
-
- String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
- String overlap = getOverlapText(prevChunkContent);
- buffer = new StringBuilder(overlap);
- tokenCount = estimateTokens(overlap);
- }
- // 否则：容忍超出（软边界）
- } else {
- // 安全切点：段落边界
- chunkParaStart = saveChunkAndGetNextStart(
- chunks, section, paraPositions,
- chunkParaStart, i, title, breadcrumb, chunkIndex);
- chunkIndex++;
-
- // 新分片以重叠文本开头
- String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
- String overlap = getOverlapText(prevChunkContent);
- buffer = new StringBuilder(overlap);
- tokenCount = estimateTokens(overlap);
- }
- }
-
- buffer.append(paragraph).append("\n\n");
- tokenCount += paraTokens;
- }
-
- // 保存最后一个分片
- if (buffer.length() > 0 && chunkParaStart < paragraphs.size()) {
- String chunkContent = buffer.toString().trim();
- int actualStart = paraPositions.get(chunkParaStart).start;
- int actualEnd = paraPositions.get(paragraphs.size() - 1).end;
- DocumentChunk chunk = DocumentChunk.builder()
- .content(chunkContent)
- .startOffset(section.startIndex + actualStart)
- .endOffset(section.startIndex + actualEnd)
- .chunkIndex(chunkIndex)
- .title(title)
- .breadcrumb(breadcrumb)
- .build();
- chunks.add(chunk);
- }
-
- return chunks;
- }
-
- /**
- * 保存当前分块，返回下一个分块的起始段落索引
- *
- * 从 section.content 中提取原始文本（而非手工拼装），修复 index 漂移问题
- */
- private int saveChunkAndGetNextStart(
- List chunks,
- Section section,
- List paraPositions,
- int fromPara,
- int toPara,
- String title,
- String breadcrumb,
- int chunkIndex) {
-
- int actualStart = paraPositions.get(fromPara).start;
- int actualEnd = paraPositions.get(toPara - 1).end;
- String originalText = section.content.substring(actualStart, actualEnd);
-
- DocumentChunk chunk = DocumentChunk.builder()
- .content(originalText)
- .startOffset(section.startIndex + actualStart)
- .endOffset(section.startIndex + actualEnd)
- .chunkIndex(chunkIndex)
- .title(title)
- .breadcrumb(breadcrumb)
- .build();
- chunks.add(chunk);
-
- return toPara; // 下一个分块的起始段落索引
- }
-
- /**
- * 按段落分割文本
- */
- private List splitByParagraphs(String content) {
- List paragraphs = new ArrayList<>();
-
- // 按双换行符分割段落
- String[] parts = content.split("\n\n+");
- for (String part : parts) {
- String trimmed = part.trim();
- if (!trimmed.isEmpty()) {
- paragraphs.add(trimmed);
- }
- }
-
- return paragraphs;
- }
-
- /**
- * 定位每个段落在原始文本中的字符偏移
- */
- private List locateParagraphPositions(List paragraphs, String sectionContent) {
- List positions = new ArrayList<>();
- int searchFrom = 0;
- for (String p : paragraphs) {
- int idx = sectionContent.indexOf(p, searchFrom);
- if (idx >= 0) {
- positions.add(new ParagraphPos(idx, idx + p.length()));
- searchFrom = idx + p.length();
- } else {
- // fallback: 段落在原文中找不到（不应该发生）
- positions.add(new ParagraphPos(searchFrom, searchFrom + p.length()));
- searchFrom += p.length();
- }
- }
- return positions;
- }
-
- /**
- * 启发式 token 估算（无需外部依赖）
- *
- * 中文(BMP): ~1 字符/token
- * 英文/数字/标点: ~4 字符/token
- * 空白字符忽略
- */
- private int estimateTokens(String text) {
- int nonCjkCount = 0;
- int cjkCount = 0;
- for (char c : text.toCharArray()) {
- if (Character.isWhitespace(c)) {
- continue;
- }
- Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
- if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
- || block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
- || block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
- || block == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS) {
- cjkCount++;
- } else {
- nonCjkCount++;
- }
- }
- return cjkCount + (nonCjkCount + 3) / 4; // 非中文每 4 字符算 1 token，向上取整
- }
-
- /**
- * 判断当前段落是否属于不可中断的结构
- *
- * 不可中断结构包括：
- * - 有序列表项（"1. ", "2. " 格式）
- * - 无序列表项（"- " 或 "* " 格式）
- * - 未闭合的代码块（``` 内）
- */
- private boolean isInUnbreakableContext(String buffer, String nextParagraph) {
- // 有序列表：判断 buffer 末尾和下一段是否都是列表项
- if (nextParagraph.matches("^\\d{1,2}\\.\\s.*")) {
- String lastLine = getLastNonEmptyLine(buffer);
- if (lastLine != null && lastLine.matches("^\\d{1,2}\\.\\s.*")) {
- return true;
- }
- }
- // 无序列表："- " 或 "* " 格式
- if (nextParagraph.matches("^[-*]\\s.*")) {
- String lastLine = getLastNonEmptyLine(buffer);
- if (lastLine != null && lastLine.matches("^[-*]\\s.*")) {
- return true;
- }
- }
- // 代码块：``` 未闭合
- if (buffer.contains("```")) {
- int count = 0;
- for (int i = 0; i <= buffer.length() - 3; i++) {
- if (buffer.substring(i).startsWith("```")) {
- count++;
- i += 2;
- }
- }
- if (count % 2 == 1) {
- return true; // 奇数个 ``` → 在代码块内部
- }
- }
- return false;
- }
-
- /**
- * 获取 buffer 中最后一行非空白文本
- */
- private String getLastNonEmptyLine(String buffer) {
- String[] lines = buffer.split("\n");
- for (int i = lines.length - 1; i >= 0; i--) {
- String line = lines[i].trim();
- if (!line.isEmpty()) {
- return line;
- }
- }
- return null;
- }
-
- /**
- * 获取重叠文本
- * 从文本末尾提取指定长度的内容作为下一个分片的开头
- */
- private String getOverlapText(String text) {
- int overlapSize = Math.min(chunkConfig.getOverlap(), text.length());
- if (overlapSize <= 0) {
- return "";
- }
-
- // 从末尾提取重叠内容
- String overlap = text.substring(text.length() - overlapSize);
-
- // 尝试在句子边界截断（查找最后一个句号、问号、感叹号）
- int lastSentenceEnd = Math.max(
- overlap.lastIndexOf('。'),
- Math.max(overlap.lastIndexOf('？'), overlap.lastIndexOf('！'))
- );
-
- if (lastSentenceEnd > overlapSize / 2) {
- return overlap.substring(lastSentenceEnd + 1).trim();
- }
-
- return overlap.trim();
- }
-
- /**
- * 段落在原文中的位置
- */
- private static class ParagraphPos {
- final int start;
- final int end;
-
- ParagraphPos(int start, int end) {
- this.start = start;
- this.end = end;
- }
- }
-
- /**
- * 章节数据类
- */
- private static class Section {
- String title; // 最近一级标题名称
- int level; // 标题级别（1-6），0=无标题
- String breadcrumb; // 完整面包屑路径
- String content; // 章节内容
- int startIndex; // 在原文中的起始偏移
-
- Section(String title, int level, String breadcrumb, String content, int startIndex) {
- this.title = title;
- this.level = level;
- this.breadcrumb = breadcrumb;
- this.content = content;
- this.startIndex = startIndex;
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java b/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java
deleted file mode 100644
index 927e73b..0000000
--- a/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java
+++ /dev/null
@@ -1,135 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.dto.KnowledgeEntry;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.ai.chat.model.ChatModel;
-import org.springframework.ai.chat.prompt.Prompt;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.stereotype.Service;
-
-import jakarta.annotation.PostConstruct;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * 文档字段补全服务
- * 上传时调用 LLM 生成 covers 和 whenToRetrieve
- */
-@Slf4j
-@Service
-public class DocumentFieldEnricher {
-
- @Autowired
- private ChatModel chatModel;
-
- @Autowired
- private ObjectMapper objectMapper;
-
- @Autowired
- private KnowledgeIndexService knowledgeIndexService;
-
- private String promptTemplate;
-
- @PostConstruct
- public void init() {
- try {
- promptTemplate = new String(
- new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
- StandardCharsets.UTF_8);
- log.info("DocumentFieldEnricher prompt 加载成功");
- } catch (IOException e) {
- log.error("加载 doc-field-enricher-prompt.md 失败", e);
- throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
- }
- }
-
- public void enrich(Frontmatter frontmatter, String bodyText) {
- enrich(frontmatter, bodyText, null);
- }
-
- /**
- * 为 Frontmatter 补全 covers 和 whenToRetrieve
- * 若已有值则跳过；LLM 失败时降级，不阻断主流程
- *
- * @param frontmatter 待补全的 frontmatter
- * @param bodyText 文档正文
- * @param category 文档所属域（用于查找同域其他文档）
- */
- public void enrich(Frontmatter frontmatter, String bodyText, String category) {
- if (frontmatter == null) return;
-
- boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
- boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
-
- if (!needsCovers && !needsWhen) {
- log.debug("covers 和 whenToRetrieve 已存在，跳过 LLM 生成");
- return;
- }
-
- try {
- String snippet = bodyText != null && bodyText.length() > 1000
- ? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
-
- String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
-
- String promptText = String.format(promptTemplate,
- frontmatter.getTitle(),
- frontmatter.getSummary(),
- sameDomainDocs,
- snippet);
-
- String response = chatModel.call(new Prompt(promptText))
- .getResult().getOutput().getText();
-
- // 提取 JSON 部分（防止模型输出多余文本）
- String json = extractJson(response);
- JsonNode node = objectMapper.readTree(json);
-
- if (needsCovers && node.has("covers")) {
- List covers = new ArrayList<>();
- node.get("covers").forEach(n -> covers.add(n.asText()));
- frontmatter.setCovers(covers);
- log.debug("LLM 生成 covers: {}", covers);
- }
-
- if (needsWhen && node.has("whenToRetrieve")) {
- frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
- log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
- }
-
- } catch (Exception e) {
- log.warn("LLM 生成文档字段失败，降级处理: title={}", frontmatter.getTitle(), e);
- if (needsCovers) frontmatter.setCovers(List.of());
- if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
- }
- }
-
- private String extractJson(String text) {
- if (text == null) return "{}";
- int start = text.indexOf('{');
- int end = text.lastIndexOf('}');
- if (start == -1 || end == -1 || end <= start) return "{}";
- return text.substring(start, end + 1);
- }
-
- /**
- * 构建同域其他文档标题列表（供 LLM 做排除判断）
- */
- private String buildSameDomainDocs(String currentTitle, String category) {
- if (category == null || category.isBlank()) return "（无同域文档信息）";
- List otherTitles = knowledgeIndexService.getAllEntries().stream()
- .filter(e -> category.equals(e.getCategory()))
- .map(KnowledgeEntry::getTitle)
- .filter(t -> t != null && !t.equals(currentTitle))
- .collect(Collectors.toList());
- if (otherTitles.isEmpty()) return "（无同域其他文档）";
- return String.join("、", otherTitles);
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java
index 7ad1ecb..81515e8 100644
--- a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java
+++ b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java
@@ -1,13 +1,13 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.superbiz.agent.client.PyRagClient;
+import com.superbiz.agent.client.PyRagClient.PyRagIngestResponse;
+import com.superbiz.agent.client.PyRagClientException;
import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.domain.enums.FaultCategory;
-import com.superbiz.agent.dto.DocumentChunk;
import com.superbiz.agent.dto.DocumentQueryResponse;
import com.superbiz.agent.dto.DocumentUploadRequest;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.exception.DocumentProcessException;
import com.superbiz.agent.repository.ApiDocumentRepository;
import lombok.extern.slf4j.Slf4j;
@@ -27,11 +27,13 @@ import java.security.MessageDigest;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
-import java.util.UUID;
import java.util.stream.Collectors;
/**
- * 文档管理服务
+ * 文档管理服务。
+ *
+ * 负责 MySQL 元数据（{@link ApiDocument}）、本地原件保存与业务查询/删除；
+ * 文档解析、frontmatter 校验、分块与向量索引全部由 py-rag 服务端 ingest 完成。
*/
@Slf4j
@Service
@@ -41,34 +43,19 @@ public class DocumentManagementService {
private String knowledgeBasePath;
@Autowired
- private TextExtractorService textExtractorService;
-
- @Autowired
- private DocumentChunkService documentChunkService;
-
- @Autowired
- private VectorIndexService vectorIndexService;
+ private PyRagClient pyRagClient;
@Autowired
private ApiDocumentRepository apiDocumentRepository;
- @Autowired
- private FrontmatterParser frontmatterParser;
-
- @Autowired
- private KnowledgeIndexService knowledgeIndexService;
-
- @Autowired
- private DocumentFieldEnricher documentFieldEnricher;
-
- @Autowired
- private KnowledgeDomainService knowledgeDomainService;
-
@Autowired
private ObjectMapper objectMapper;
/**
- * 上传文档
+ * 上传文档。
+ *
+ * 流程：hash 去重 → 本地原件保存 → py-rag ingest（同步，服务端解析/分块/索引）→
+ * MySQL 元数据落库。docId 取 py-rag 返回的 doc_id，与检索 evidence_key 的 docId 段对齐。
*
* @param request 上传请求
* @return 文档ID
@@ -83,15 +70,7 @@ public class DocumentManagementService {
log.info("开始上传文档，文件名: {}, 大小: {} bytes", fileName, file.getSize());
try {
- // 1. 验证文件格式
- if (!textExtractorService.isSupportedFormat(fileName)) {
- throw new DocumentProcessException(
- fileName, "upload",
- "不支持的文件格式，仅支持 .md 和 .txt"
- );
- }
-
- // 2. 计算文件 hash（去重）
+ // 1. 计算文件 hash（去重）
long hashStart = System.currentTimeMillis();
String fileHash = calculateFileHash(file);
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
@@ -105,16 +84,7 @@ public class DocumentManagementService {
);
}
- // 3. 提取文本
- long extractStart = System.currentTimeMillis();
- String text = textExtractorService.extractText(file, fileName);
- log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart);
-
- if (text == null || text.isBlank()) {
- throw new DocumentProcessException(fileName, "upload", "文档内容为空");
- }
-
- // 4. 保存原始文件到本地
+ // 2. category 缺省处理 + 保存原始文件到本地
String category = request.getCategory();
if (category == null || category.isBlank()) {
category = "default";
@@ -123,47 +93,33 @@ public class DocumentManagementService {
localPath = saveToLocal(file, fileName, category);
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
- // 5. 解析 frontmatter
- long frontmatterStart = System.currentTimeMillis();
- Frontmatter frontmatter = null;
- String bodyText = text;
- if (frontmatterParser.hasFrontmatter(text)) {
- frontmatter = frontmatterParser.parse(text);
- if (frontmatter != null) {
- // LLM 补全 covers / whenToRetrieve（已有值则跳过）
- bodyText = frontmatterParser.stripFrontmatter(text);
- documentFieldEnricher.enrich(frontmatter, bodyText, category);
- log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
- frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
- } else {
- log.warn("frontmatter解析失败，文件名: {}", fileName);
- }
- } else {
- log.debug("文件不包含frontmatter: {}", fileName);
+ // 3. py-rag 入库（格式校验/frontmatter/分块/向量索引都在服务端；同内容重传返回 unchanged）
+ long ingestStart = System.currentTimeMillis();
+ PyRagIngestResponse ingest;
+ try {
+ ingest = pyRagClient.ingest(fileName, file.getBytes(), file.getContentType(),
+ category, null, null, null);
+ } catch (PyRagClientException | IOException e) {
+ throw new DocumentProcessException(
+ fileName, "ingest", "py-rag 入库失败: " + e.getMessage(), e
+ );
}
+ log.info("py-rag 入库完成: docId={}, status={}, chunks={}, time={}ms",
+ ingest.docId(), ingest.status(), ingest.chunkCount(),
+ System.currentTimeMillis() - ingestStart);
- // 6. 分块
- long chunkStart = System.currentTimeMillis();
- List chunks = documentChunkService.chunkDocument(bodyText, fileName);
- if (chunks.isEmpty()) {
- throw new DocumentProcessException(fileName, "upload", "文档分块失败");
- }
- log.info("文档分块完成: fileName={}, chunks={}, time={}ms",
- fileName, chunks.size(), System.currentTimeMillis() - chunkStart);
-
- // 7. 创建文档元数据
- String docId = resolveDocumentId(frontmatter);
+ // 4. 保存文档元数据
String metadataJson = null;
- if (frontmatter != null) {
+ if (ingest.frontmatter() != null) {
try {
- metadataJson = objectMapper.writeValueAsString(frontmatter);
+ metadataJson = objectMapper.writeValueAsString(ingest.frontmatter());
} catch (Exception e) {
- log.warn("Frontmatter序列化失败", e);
+ log.warn("frontmatter 序列化失败", e);
}
}
ApiDocument document = ApiDocument.builder()
- .docId(docId)
+ .docId(ingest.docId())
.fileName(fileName)
.filePath(localPath)
.metadata(metadataJson)
@@ -173,56 +129,17 @@ public class DocumentManagementService {
.version(request.getVersion())
.fileSize(file.getSize())
.fileHash(fileHash)
- .status("PROCESSING")
- .chunkCount(chunks.size())
+ .status("INDEXED")
+ .chunkCount(ingest.chunkCount())
.build();
-
+ document.setIndexedAt(LocalDateTime.now());
apiDocumentRepository.save(document);
- log.info("文档元数据已保存: docId={}", docId);
+ log.info("文档元数据已保存: docId={}", document.getDocId());
- // 8. 向量化并索引
- try {
- long vectorStart = System.currentTimeMillis();
- vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter);
- document.setStatus("INDEXED");
- document.setIndexedAt(LocalDateTime.now());
- apiDocumentRepository.save(document);
- log.info("文档向量索引完成: docId={}, category={}, time={}ms",
- docId, category, System.currentTimeMillis() - vectorStart);
+ log.info("文档上传完成: docId={}, fileName={}, ingestStatus={}, totalTime={}ms",
+ document.getDocId(), fileName, ingest.status(), System.currentTimeMillis() - startTime);
- } catch (Exception e) {
- log.error("文档索引失败: docId={}", docId, e);
- document.setStatus("FAILED");
- apiDocumentRepository.save(document);
- throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
- }
-
- // 9. 更新 L0 索引
- if (frontmatter != null) {
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath(localPath)
- .title(frontmatter.getTitle())
- .keywords(frontmatter.getKeywords())
- .summary(frontmatter.getSummary())
- .category(category)
- .kbScope(frontmatter.getKbScope())
- .sections(frontmatter.getSections())
- .covers(frontmatter.getCovers())
- .whenToRetrieve(frontmatter.getWhenToRetrieve())
- .build();
-
- knowledgeIndexService.addToIndex(entry);
- log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
- }
-
- // 触发域级聚合重算
- knowledgeDomainService.onDocumentChange(category);
-
- long totalTime = System.currentTimeMillis() - startTime;
- log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
- docId, fileName, frontmatter != null, totalTime);
-
- return docId;
+ return document.getDocId();
} catch (Exception e) {
// 失败时清理本地文件
@@ -314,16 +231,6 @@ public class DocumentManagementService {
}
}
- private String resolveDocumentId(Frontmatter frontmatter) {
- if (frontmatter != null && frontmatter.getSource() != null) {
- String source = frontmatter.getSource().trim();
- if (!source.isEmpty() && source.length() <= 64) {
- return source;
- }
- }
- return UUID.randomUUID().toString();
- }
-
/**
* 根据 docId 查询文档
*/
@@ -358,7 +265,10 @@ public class DocumentManagementService {
}
/**
- * 删除文档
+ * 删除文档（仅 MySQL 元数据与本地原件）。
+ *
+ * py-rag v1 契约没有单文档删除端点：已入库内容需在其服务端
+ * 全量重建（{@code /api/v1/collections:rebuild}）后才会从知识库消失。
*/
@Transactional
public void deleteDocument(String docId) {
@@ -379,47 +289,9 @@ public class DocumentManagementService {
}
}
- // 删除 L0 索引
- if (doc.getFilePath() != null) {
- knowledgeIndexService.removeFromIndex(doc.getFilePath());
- }
-
- // 删除向量索引
- try {
- vectorIndexService.deleteDocumentChunks(docId);
- log.info("文档向量索引已删除，docId: {}", docId);
- } catch (Exception e) {
- log.warn("删除向量索引失败，docId: {}", docId, e);
- }
-
- // 删除元数据
+ // 删除元数据（py-rag 侧索引留存，重建后失效）
apiDocumentRepository.delete(doc);
- log.info("文档已删除，docId: {}", docId);
-
- // 触发域级聚合重算
- String category = doc.getFilePath() != null
- ? resolveCategory(doc.getFilePath()) : null;
- if (category != null) {
- knowledgeDomainService.onDocumentChange(category);
- }
- }
-
- /**
- * 转换为响应 DTO
- */
- /**
- * 从 filePath 解析 category（取 knowledge_base/{category}/... 中的 category 段）
- */
- private String resolveCategory(String filePath) {
- try {
- java.nio.file.Path p = java.nio.file.Paths.get(filePath);
- // filePath 形如 knowledge_base/payment/xxx.md，取倒数第二段
- int nameCount = p.getNameCount();
- if (nameCount >= 2) {
- return p.getName(nameCount - 2).toString();
- }
- } catch (Exception ignored) {}
- return null;
+ log.info("文档已删除，docId={}（py-rag 侧需全量重建后生效）", docId);
}
private Path resolveLocalPath(String filePath) {
diff --git a/src/main/java/com/superbiz/agent/service/FrontmatterParser.java b/src/main/java/com/superbiz/agent/service/FrontmatterParser.java
deleted file mode 100644
index 350e921..0000000
--- a/src/main/java/com/superbiz/agent/service/FrontmatterParser.java
+++ /dev/null
@@ -1,160 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.dto.Frontmatter;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import org.yaml.snakeyaml.Yaml;
-
-import java.util.Map;
-
-/**
- * Frontmatter 解析器
- * 解析 Markdown 文件头的 YAML frontmatter
- */
-@Slf4j
-@Service
-public class FrontmatterParser {
-
- private final Yaml yaml = new Yaml();
-
- /**
- * 检查文件是否包含 frontmatter
- *
- * @param content 文件内容
- * @return true 如果包含 frontmatter
- */
- public boolean hasFrontmatter(String content) {
- if (content == null || content.isEmpty()) {
- return false;
- }
- return content.trim().startsWith("---");
- }
-
- /**
- * 解析 Markdown frontmatter
- *
- * @param content 完整文件内容
- * @return Frontmatter 对象，如果不存在或解析失败返回 null
- */
- public Frontmatter parse(String content) {
- if (!hasFrontmatter(content)) {
- return null;
- }
-
- try {
- // 1. 提取 frontmatter 部分（两个 --- 之间）
- String frontmatterText = extractFrontmatter(content);
- if (frontmatterText == null) {
- log.warn("未找到有效的 frontmatter 结束标记");
- return null;
- }
-
- // 2. 使用 SnakeYAML 解析
- Map map = yaml.load(frontmatterText);
- if (map == null || map.isEmpty()) {
- log.warn("Frontmatter 解析结果为空");
- return null;
- }
-
- // 3. 映射到 Frontmatter 对象
- Frontmatter frontmatter = Frontmatter.builder()
- .title((String) map.get("title"))
- .keywords((java.util.List) map.get("keywords"))
- .summary((String) map.get("summary"))
- .category((String) map.get("category"))
- .source((String) map.get("source"))
- .breadcrumb((String) map.get("breadcrumb"))
- .kbScope(firstString(map, "kb_scope", "kbScope"))
- .sections((Map) map.get("sections"))
- .version((String) map.get("version"))
- .author((String) map.get("author"))
- .covers((java.util.List) map.get("covers"))
- .whenToRetrieve((String) map.get("when_to_retrieve"))
- .build();
-
- // 4. 验证必填字段
- if (frontmatter.getTitle() == null || frontmatter.getKeywords() == null ||
- frontmatter.getSummary() == null) {
- log.warn("Frontmatter 缺少必填字段: title={}, keywords={}, summary={}",
- frontmatter.getTitle(), frontmatter.getKeywords(), frontmatter.getSummary());
- return null;
- }
-
- log.debug("Frontmatter 解析成功: title={}, keywords=",
- frontmatter.getTitle(), frontmatter.getKeywords());
- return frontmatter;
-
- } catch (Exception e) {
- log.warn("Frontmatter 解析失败", e);
- return null;
- }
- }
-
- public String stripFrontmatter(String content) {
- if (!hasFrontmatter(content)) {
- return content;
- }
-
- String trimmed = content.trim();
- int secondDelimiter = trimmed.indexOf("\n---", 3);
- int delimiterLength = 4;
- if (secondDelimiter == -1) {
- secondDelimiter = trimmed.indexOf("\r\n---", 3);
- delimiterLength = 5;
- }
- if (secondDelimiter == -1) {
- return content;
- }
-
- int bodyStart = secondDelimiter + delimiterLength;
- if (bodyStart < trimmed.length()) {
- char next = trimmed.charAt(bodyStart);
- if (next == '\r') {
- bodyStart++;
- }
- if (bodyStart < trimmed.length() && trimmed.charAt(bodyStart) == '\n') {
- bodyStart++;
- }
- }
- return trimmed.substring(Math.min(bodyStart, trimmed.length())).stripLeading();
- }
-
- /**
- * 提取 frontmatter 文本（两个 --- 之间的内容）
- *
- * @param content 完整文件内容
- * @return frontmatter 文本，如果格式错误返回 null
- */
- private String extractFrontmatter(String content) {
- // 去除开头的空白
- content = content.trim();
-
- // 检查是否以 --- 开头
- if (!content.startsWith("---")) {
- return null;
- }
-
- // 查找第二个 ---（结束标记）
- int secondDelimiter = content.indexOf("\n---", 3);
- if (secondDelimiter == -1) {
- // 尝试查找 Windows 风格换行
- secondDelimiter = content.indexOf("\r\n---", 3);
- if (secondDelimiter == -1) {
- return null;
- }
- }
-
- // 提取 frontmatter（不包含 --- 标记）
- return content.substring(3, secondDelimiter).trim();
- }
-
- private String firstString(Map map, String... keys) {
- for (String key : keys) {
- Object value = map.get(key);
- if (value instanceof String text && !text.isBlank()) {
- return text;
- }
- }
- return null;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java b/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java
deleted file mode 100644
index c0a9efc..0000000
--- a/src/main/java/com/superbiz/agent/service/KnowledgeBaseInitService.java
+++ /dev/null
@@ -1,407 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.domain.entity.ApiDocument;
-import com.superbiz.agent.domain.enums.FaultCategory;
-import com.superbiz.agent.repository.ApiDocumentRepository;
-import com.superbiz.agent.dto.KnowledgeEntry;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.dto.DocumentChunk;
-import lombok.Data;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
-
-import java.io.IOException;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.time.LocalDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Collectors;
-
-/**
- * 知识库初始化服务
- * 负责批量导入 knowledge_base 目录下的文档到数据库和 Milvus
- */
-@Service
-public class KnowledgeBaseInitService {
-
- private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseInitService.class);
-
- @Value("${knowledge.base-path:knowledge_base}")
- private String knowledgeBasePath;
-
- @Autowired
- private ApiDocumentRepository apiDocumentRepository;
-
- @Autowired
- private FrontmatterParser frontmatterParser;
-
- @Autowired
- private DocumentChunkService documentChunkService;
-
- @Autowired
- private VectorIndexService vectorIndexService;
-
- @Autowired
- private VectorEmbeddingService vectorEmbeddingService;
-
- @Autowired
- private KnowledgeIndexService knowledgeIndexService;
-
- @Autowired
- private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
-
- /**
- * Drop hybrid collection, clear MySQL api_document + L0 memory index,
- * then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
- */
- @Transactional(rollbackFor = Exception.class)
- public RebuildResult rebuildHybridFromKnowledgeBase() {
- logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
- RebuildResult rebuild = new RebuildResult();
- rebuild.setBasePath(knowledgeBasePath);
- rebuild.setCollection(hybridKnowledgeStore.collectionName());
-
- long mysqlBefore = apiDocumentRepository.count();
- rebuild.setMysqlDocumentsBefore(mysqlBefore);
-
- Map milvus = hybridKnowledgeStore.dropAndRecreateCollection();
- rebuild.setMilvus(milvus);
-
- apiDocumentRepository.deleteAll();
- apiDocumentRepository.flush();
- knowledgeIndexService.clearIndex();
- rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
- rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
-
- InitResult init = initializeKnowledgeBase(true);
- rebuild.setInit(init);
- rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
- rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
- // Success when at least one doc indexed and no hard failures.
- // README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
- rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
- logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
- rebuild.isSuccess(), init.getInserted(), init.getFailed());
- return rebuild;
- }
-
- /**
- * 初始化知识库
- *
- * @param force 是否强制重新导入（跳过去重检查）
- * @return 初始化结果
- */
- @Transactional(rollbackFor = Exception.class)
- public InitResult initializeKnowledgeBase(boolean force) {
- logger.info("开始初始化知识库: basePath={}, force={}", knowledgeBasePath, force);
-
- InitResult result = new InitResult();
- Path baseDir = Paths.get(knowledgeBasePath);
-
- if (!Files.exists(baseDir)) {
- logger.error("知识库目录不存在: {}", knowledgeBasePath);
- throw new RuntimeException("知识库目录不存在: " + knowledgeBasePath);
- }
-
- // 1. 扫描所有 Markdown 文件
- List markdownFiles = scanMarkdownFiles(baseDir);
- result.setScanned(markdownFiles.size());
- logger.info("扫描到 {} 个 Markdown 文件", markdownFiles.size());
-
- // 2. 如果非强制模式，获取已存在的文档（用于去重）
- Set existingFilePaths = new HashSet<>();
- if (!force) {
- existingFilePaths = apiDocumentRepository.findAll().stream()
- .map(ApiDocument::getFilePath)
- .collect(Collectors.toSet());
- logger.info("已存在 个文档记录", existingFilePaths.size());
- }
-
- // 3. 逐个处理文档
- for (Path file : markdownFiles) {
- String relativePath = baseDir.relativize(file).toString().replace("\\", "/");
-
- try {
- // 去重检查
- if (!force && existingFilePaths.contains(relativePath)) {
- logger.debug("跳过已存在的文档: {}", relativePath);
- result.incrementSkipped();
- result.addDetail(relativePath, "已存在，跳过");
- continue;
- }
-
- // 解析文档
- String content = Files.readString(file);
- Frontmatter frontmatter = frontmatterParser.parse(content);
-
- if (frontmatter == null) {
- logger.warn("文档格式无效: {}, frontmatter 解析失败", relativePath);
- result.incrementFailed();
- result.addDetail(relativePath, "格式无效: frontmatter 解析失败");
- continue;
- }
-
- // 提取字段
- String title = frontmatter.getTitle();
- String summary = frontmatter.getSummary();
- String category = frontmatter.getCategory() != null ? frontmatter.getCategory() : "general";
- List keywords = frontmatter.getKeywords();
-
- if (title == null || title.isBlank()) {
- logger.warn("文档缺少标题: {}", relativePath);
- result.incrementFailed();
- result.addDetail(relativePath, "缺少标题");
- continue;
- }
-
- // 保存到数据库
- ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
-
- // 提取文档正文（去除 frontmatter）
- String body = extractBody(content);
-
- // 文档分块
- List chunks = documentChunkService.chunkDocument(body, relativePath);
- logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
-
- // 上传到 Milvus hybrid collection（dense + BM25 search_text）
- try {
- vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
-
- document.setStatus("INDEXED");
- document.setChunkCount(chunks.size());
- document.setIndexedAt(LocalDateTime.now());
- apiDocumentRepository.save(document);
-
- logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
- title, document.getDocId(), chunks.size());
- } catch (Exception e) {
- logger.error("上传到 Milvus 失败: {}", relativePath, e);
-
- document.setStatus("FAILED");
- document.setErrorMessage(e.getMessage());
- apiDocumentRepository.save(document);
-
- result.incrementFailed();
- result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
- continue; // 跳过该文档，继续处理下一个
- }
-
- // 添加到 L0 内存索引
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath(relativePath)
- .title(title)
- .keywords(keywords)
- .summary(summary)
- .category(category)
- .kbScope(frontmatter.getKbScope())
- .build();
- knowledgeIndexService.addToIndex(entry);
-
- result.incrementInserted();
- result.addDetail(relativePath, "导入成功（L0+L1）");
- logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
-
- } catch (Exception e) {
- logger.error("处理文档失败: {}", relativePath, e);
- result.incrementFailed();
- result.addDetail(relativePath, "处理失败: " + e.getMessage());
- }
- }
-
- logger.info("知识库初始化完成: 扫描={}, 跳过={}, 新增={}, 失败={}",
- result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
-
- return result;
- }
-
- /**
- * 获取知识库统计信息
- */
- public Stats getStats() {
- Stats stats = new Stats();
-
- // 数据库中的文档数量
- long totalDocuments = apiDocumentRepository.count();
- stats.setTotalDocuments(totalDocuments);
-
- // L0 索引中的文档数量
- int indexSize = knowledgeIndexService.getIndexSize();
- logger.debug("L0 索引大小: {}", indexSize);
-
- // 按分类统计（从 fault_category 字段读取）
- Map categoryCount = apiDocumentRepository.findAll().stream()
- .collect(Collectors.groupingBy(
- doc -> doc.getFaultCategory() != null ? doc.getFaultCategory().name() : "GENERAL",
- Collectors.counting()
- ));
- stats.setCategoryCount(categoryCount);
-
- // Milvus 中的向量数量（需要实现）
- // TODO: 查询 Milvus collection 的实体数量
- stats.setTotalVectors(0L);
-
- return stats;
- }
-
- /**
- * 扫描目录下所有 Markdown 文件
- */
- private List scanMarkdownFiles(Path baseDir) {
- List files = new ArrayList<>();
-
- try {
- Files.walkFileTree(baseDir, new SimpleFileVisitor() {
- @Override
- public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
- String name = file.getFileName() == null ? "" : file.getFileName().toString();
- // Import content docs only; skip README/index markdown without frontmatter.
- if (name.endsWith(".md")
- && !name.equalsIgnoreCase("README.md")
- && !name.equalsIgnoreCase("readme.md")) {
- files.add(file);
- }
- return FileVisitResult.CONTINUE;
- }
-
- @Override
- public FileVisitResult visitFileFailed(Path file, IOException exc) {
- logger.warn("访问文件失败: {}", file, exc);
- return FileVisitResult.CONTINUE;
- }
- });
- } catch (IOException e) {
- logger.error("扫描目录失败: {}", baseDir, e);
- throw new RuntimeException("扫描目录失败", e);
- }
-
- return files;
- }
-
- /**
- * 保存文档到数据库
- */
- private ApiDocument saveToDatabase(String filePath, String title, String summary,
- String category, String content, List keywords) {
- ApiDocument document = new ApiDocument();
- document.setDocId(UUID.randomUUID().toString());
- document.setFileName(Paths.get(filePath).getFileName().toString());
- document.setFilePath(filePath);
- document.setApiName(title); // 使用 title 作为 apiName
- document.setStatus("PENDING"); // 初始状态为 PENDING，索引成功后更新为 INDEXED
-
- // 映射 category 到 FaultCategory 枚举
- FaultCategory faultCategory = FaultCategory.fromString(category);
- document.setFaultCategory(faultCategory);
-
- // 将 frontmatter 信息保存到 metadata（JSON 格式）
- String metadataJson = String.format(
- "{\"title\":\"%s\",\"summary\":\"%s\",\"category\":\"%s\",\"keywords\":%s}",
- escapeJson(title),
- escapeJson(summary),
- escapeJson(category),
- "[\"" + String.join("\",\"", keywords.stream().map(this::escapeJson).toArray(String[]::new)) + "\"]"
- );
- document.setMetadata(metadataJson);
-
- document.setFileSize((long) content.length());
-
- return apiDocumentRepository.save(document);
- }
-
- /**
- * JSON 转义
- */
- private String escapeJson(String str) {
- if (str == null) {
- return "";
- }
- return str.replace("\\", "\\\\")
- .replace("\"", "\\\"")
- .replace("\n", "\\n")
- .replace("\r", "\\r");
- }
-
- /**
- * 提取文档正文（去除 frontmatter）
- */
- private String extractBody(String content) {
- if (!content.trim().startsWith("---")) {
- return content;
- }
-
- int firstEnd = content.indexOf("---", 3);
- if (firstEnd == -1) {
- return content;
- }
-
- int secondEnd = content.indexOf("---", firstEnd + 3);
- if (secondEnd == -1) {
- return content.substring(firstEnd + 3).trim();
- }
-
- return content.substring(secondEnd + 3).trim();
- }
-
- // ==================== 数据模型 ====================
-
- /**
- * 初始化结果
- */
- @Data
- public static class InitResult {
- private int scanned; // 扫描到的文件数量
- private int skipped; // 跳过的文件数量（已存在）
- private int inserted; // 成功导入的文件数量
- private int failed; // 失败的文件数量
- private Map details = new LinkedHashMap<>(); // 详细信息
-
- public void incrementSkipped() {
- this.skipped++;
- }
-
- public void incrementInserted() {
- this.inserted++;
- }
-
- public void incrementFailed() {
- this.failed++;
- }
-
- public void addDetail(String filePath, String message) {
- this.details.put(filePath, message);
- }
- }
-
- /**
- * 统计信息
- */
- @Data
- public static class Stats {
- private long totalDocuments; // 数据库中的文档总数
- private long totalVectors; // Milvus 中的向量总数
- private Map categoryCount; // 按分类统计
- }
-
- /**
- * Full hybrid rebuild result.
- */
- @Data
- public static class RebuildResult {
- private boolean success;
- private String basePath;
- private String collection;
- private long mysqlDocumentsBefore;
- private long mysqlDocumentsAfterClear;
- private long mysqlDocumentsAfterInit;
- private int l0IndexSizeAfterClear;
- private int l0IndexSizeAfterInit;
- private Map milvus;
- private InitResult init;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java b/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java
deleted file mode 100644
index d5f6971..0000000
--- a/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java
+++ /dev/null
@@ -1,188 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.domain.entity.KnowledgeDomain;
-import com.superbiz.agent.dto.KnowledgeEntry;
-import com.superbiz.agent.repository.KnowledgeDomainRepository;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.ai.chat.model.ChatModel;
-import org.springframework.ai.chat.prompt.Prompt;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.stereotype.Service;
-
-import jakarta.annotation.PostConstruct;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-/**
- * 知识域服务
- * 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
- */
-@Slf4j
-@Service
-public class KnowledgeDomainService {
-
- @Autowired
- private KnowledgeDomainRepository knowledgeDomainRepository;
-
- @Autowired
- private KnowledgeIndexService knowledgeIndexService;
-
- @Autowired
- private ChatModel chatModel;
-
- @Autowired
- private ObjectMapper objectMapper;
-
- private String domainPromptTemplate;
-
- @PostConstruct
- public void init() {
- try {
- domainPromptTemplate = new String(
- new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
- StandardCharsets.UTF_8);
- log.info("KnowledgeDomainService prompt 加载成功");
- } catch (IOException e) {
- log.error("加载 domain-summary-prompt.md 失败", e);
- throw new RuntimeException("Failed to load domain-summary prompt", e);
- }
- }
-
- /**
- * 文档变更后重算指定域的 when_to_retrieve
- */
- public void onDocumentChange(String category) {
- if (category == null || category.isBlank()) return;
-
- List entries = knowledgeIndexService.getAllEntries().stream()
- .filter(e -> category.equals(e.getCategory()))
- .collect(Collectors.toList());
-
- buildDomainSummary(category, entries);
- }
-
- /**
- * 聚合同域文档，调用 LLM 生成域级摘要，写入 DB
- */
- public void buildDomainSummary(String category, List entries) {
- if (entries.isEmpty()) {
- knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
- d.setDocumentCount(0);
- knowledgeDomainRepository.save(d);
- });
- return;
- }
-
- // 构建文档列表描述
- StringBuilder docList = new StringBuilder();
- for (KnowledgeEntry entry : entries) {
- docList.append("- 文档：").append(entry.getTitle()).append("\n");
- if (entry.getWhenToRetrieve() != null) {
- docList.append(" 适用场景：").append(entry.getWhenToRetrieve()).append("\n");
- }
- if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
- docList.append(" 覆盖：").append(String.join("、", entry.getCovers())).append("\n");
- }
- }
-
- String description = entries.stream()
- .map(KnowledgeEntry::getSummary)
- .filter(s -> s != null && !s.isBlank())
- .findFirst().orElse(category);
-
- String whenToRetrieve = null;
- try {
- String otherDomainsInfo = buildOtherDomainsInfo(category);
- String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
- whenToRetrieve = chatModel.call(new Prompt(promptText))
- .getResult().getOutput().getText();
- log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
- } catch (Exception e) {
- log.warn("LLM 生成域级 when_to_retrieve 失败，保留旧值: domain={}", category, e);
- Optional existing = knowledgeDomainRepository.findByDomainId(category);
- whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
- }
-
- KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
- .orElse(KnowledgeDomain.builder().domainId(category).build());
-
- domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
- domain.setWhenToRetrieve(whenToRetrieve);
- domain.setDocumentCount(entries.size());
- knowledgeDomainRepository.save(domain);
- }
-
- /**
- * 构建注入 Planner 的 knowledge map YAML 文本
- */
- public String buildKnowledgeMap() {
- List domains = knowledgeDomainRepository.findAll();
- if (domains.isEmpty()) return "";
-
- List allEntries = knowledgeIndexService.getAllEntries();
- Map> byCategory = allEntries.stream()
- .filter(e -> e.getCategory() != null)
- .collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
-
- StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
-
- for (KnowledgeDomain domain : domains) {
- yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
- if (domain.getDescription() != null) {
- yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
- }
- if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
- yaml.append(" when_to_retrieve: \"")
- .append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
- }
- yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
-
- List domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
- if (!domainEntries.isEmpty()) {
- yaml.append(" documents:\n");
- for (KnowledgeEntry entry : domainEntries) {
- yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
- if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
- yaml.append(" covers: ").append(entry.getCovers()).append("\n");
- }
- }
- }
- }
-
- return yaml.toString();
- }
-
- /**
- * 构建其他域的摘要信息（用于 LLM 域级 prompt 的边界判断）
- * 优先使用其他域的 when_to_retrieve（边界信号），而非 description
- */
- private String buildOtherDomainsInfo(String currentCategory) {
- List allDomains = knowledgeDomainRepository.findAll();
- StringBuilder sb = new StringBuilder();
- for (KnowledgeDomain d : allDomains) {
- if (d.getDomainId().equals(currentCategory)) continue;
- sb.append("- ").append(d.getDomainId());
- if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
- sb.append("：").append(d.getWhenToRetrieve());
- } else if (d.getDescription() != null && !d.getDescription().isBlank()) {
- sb.append("（").append(d.getDescription()).append("）");
- }
- sb.append("\n");
- }
- // 如果 DB 里还没有其他域的记录（首次启动），从 L0 索引补充
- if (sb.isEmpty()) {
- knowledgeIndexService.getAllEntries().stream()
- .map(KnowledgeEntry::getCategory)
- .filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
- .distinct()
- .forEach(c -> sb.append("- ").append(c).append("\n"));
- }
- return sb.isEmpty() ? "（无其他域信息）" : sb.toString();
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java b/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java
deleted file mode 100644
index 509a086..0000000
--- a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java
+++ /dev/null
@@ -1,341 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.domain.entity.ApiDocument;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.dto.KnowledgeEntry;
-import com.superbiz.agent.repository.ApiDocumentRepository;
-import com.superbiz.agent.repository.KnowledgeDomainRepository;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.annotation.Lazy;
-import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
-
-import jakarta.annotation.PostConstruct;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-/**
- * L0 知识索引服务（关键词 / domain hint，不是向量库）。
- *
- * 定位
- * 从 MySQL {@code api_document.metadata}（frontmatter）加载文档级关键词与 category，
- * 供检索前 query understanding 使用。L0 输出只作为：
- *
- * - 可选 category filter（唯一 domain 时）
- * - rerank 的 domain/keyword/entity boost 信号
- * - trace 可解释信息
- *
- * L0 命中文档不会直接当作事实 evidence；证据正文只来自 L1 向量召回。
- *
- * 匹配方式（当前较粗）
- * {@code query.contains(keyword) || keyword.contains(query)}，大小写不敏感。
- * 没有分词、别名归一或停用词；短词/泛词可能误命中。
- */
-@Slf4j
-@Service
-public class KnowledgeIndexService {
-
- @Value("${knowledge.base-path:knowledge_base}")
- private String knowledgeBasePath;
-
- @Value("${retrieval.kb-scope:}")
- private String kbScope = "";
-
- @Autowired
- private ApiDocumentRepository apiDocumentRepository;
-
- @Autowired
- private ObjectMapper objectMapper;
-
- @Autowired
- private KnowledgeDomainRepository knowledgeDomainRepository;
-
- @Lazy
- @Autowired
- private KnowledgeDomainService knowledgeDomainService;
-
- private final List knowledgeIndex = new CopyOnWriteArrayList<>();
-
- @PostConstruct
- public void loadIndex() {
- log.info("开始从数据库加载知识库索引");
-
- try {
- List documents = apiDocumentRepository.findAll();
-
- int loaded = 0;
- for (ApiDocument doc : documents) {
- try {
- KnowledgeEntry entry = parseDocumentToEntry(doc);
- if (entry != null) {
- knowledgeIndex.add(entry);
- loaded++;
- }
- } catch (Exception e) {
- log.warn("解析文档失败: docId={}, error={}", doc.getDocId(), e.getMessage());
- }
- }
-
- log.info("知识库索引加载完成，共 {} 个文档", loaded);
-
- } catch (Exception e) {
- log.error("知识库索引加载失败", e);
- }
- }
-
- /**
- * 应用就绪后，检查各域是否有 knowledge_domain 记录，无则触发生成
- * 使用 ApplicationReadyEvent 而非 PostConstruct，避免循环依赖
- */
- @EventListener(ApplicationReadyEvent.class)
- public void onApplicationReady() {
- try {
- knowledgeIndex.stream()
- .map(KnowledgeEntry::getCategory)
- .filter(c -> c != null && !c.isBlank())
- .distinct()
- .forEach(category -> {
- if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
- log.info("域 {} 无 knowledge_domain 记录，触发生成", category);
- knowledgeDomainService.onDocumentChange(category);
- }
- });
- } catch (Exception e) {
- log.error("域级记录生成失败", e);
- }
- }
-
- private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
- if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
- return null;
- }
-
- try {
- Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
-
- return KnowledgeEntry.builder()
- .filePath(doc.getFilePath())
- .title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
- .keywords(frontmatter.getKeywords())
- .summary(frontmatter.getSummary())
- .category(frontmatter.getCategory())
- .kbScope(frontmatter.getKbScope())
- .covers(frontmatter.getCovers())
- .whenToRetrieve(frontmatter.getWhenToRetrieve())
- .build();
-
- } catch (Exception e) {
- log.warn("解析 metadata 失败: {}", doc.getDocId(), e);
- return null;
- }
- }
-
- /** 兼容旧调用：只返回命中的文档条目。 */
- public List exactMatch(String query) {
- return analyzeQuery(query).matches();
- }
-
- /**
- * 分析 query，产出 L0 hint。
- * 遍历内存索引，收集匹配 keyword、domain、title；不做向量检索。
- */
- public L0Hint analyzeQuery(String query) {
- long startTime = System.currentTimeMillis();
-
- if (query == null || query.trim().isEmpty()) {
- log.debug("查询关键词为空，返回空结果");
- return L0Hint.empty();
- }
-
- String queryLower = query.toLowerCase();
- List results = new ArrayList<>();
- Set matchedKeywords = new LinkedHashSet<>();
- Set domains = new LinkedHashSet<>();
- Set entities = new LinkedHashSet<>();
- Set titles = new LinkedHashSet<>();
-
- for (KnowledgeEntry entry : knowledgeIndex) {
- if (!matchesConfiguredScope(entry)) {
- continue;
- }
- List entryMatchedKeywords = matchedKeywords(entry, queryLower);
- if (entryMatchedKeywords.isEmpty()) {
- continue;
- }
-
- results.add(entry);
- matchedKeywords.addAll(entryMatchedKeywords);
- entities.addAll(entryMatchedKeywords);
-
- if (entry.getCategory() != null && !entry.getCategory().isBlank()) {
- domains.add(entry.getCategory());
- }
- if (entry.getTitle() != null && !entry.getTitle().isBlank()) {
- titles.add(entry.getTitle());
- }
- }
-
- long elapsedTime = System.currentTimeMillis() - startTime;
- log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
- results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
-
- return new L0Hint(
- List.copyOf(results),
- List.copyOf(matchedKeywords),
- List.copyOf(domains),
- List.copyOf(entities),
- List.copyOf(titles)
- );
- }
-
- private boolean matchesKeywords(KnowledgeEntry entry, String query) {
- return !matchedKeywords(entry, query).isEmpty();
- }
-
- private boolean matchesConfiguredScope(KnowledgeEntry entry) {
- String scope = trimToNull(kbScope);
- if (scope == null) {
- return true;
- }
- return scope.equals(trimToNull(entry.getKbScope()));
- }
-
- private String trimToNull(String value) {
- if (value == null || value.isBlank()) {
- return null;
- }
- return value.trim();
- }
-
- /**
- * 关键词双向包含匹配。
- * query 已在调用方 lower-case；keyword 在此 lower-case。
- * 例：query="mysql timeout" 可命中 keyword="mysql"；
- * 反过来 keyword="mysql connection pool timeout" 也可能被短 query 命中。
- */
- private List matchedKeywords(KnowledgeEntry entry, String query) {
- if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
- return List.of();
- }
-
- List matches = new ArrayList<>();
- for (String keyword : entry.getKeywords()) {
- String keywordLower = keyword.toLowerCase();
- if (query.contains(keywordLower) || keywordLower.contains(query)) {
- matches.add(keyword);
- }
- }
-
- return matches;
- }
-
- public String readDocument(String filePath, int maxChars) {
- try {
- Path fullPath = resolveDocumentPath(filePath);
- if (!Files.exists(fullPath)) {
- log.warn("读取文档失败，文件不存在: basePath={}, filePath={}, resolvedPath={}",
- knowledgeBasePath, filePath, fullPath);
- return null;
- }
- String content = Files.readString(fullPath);
-
- if (content.length() > maxChars) {
- return content.substring(0, maxChars) + "...";
- }
-
- return content;
-
- } catch (IOException e) {
- log.error("读取文档失败: basePath={}, filePath={}", knowledgeBasePath, filePath, e);
- return null;
- }
- }
-
- Path resolveDocumentPath(String filePath) {
- if (filePath == null || filePath.isBlank()) {
- throw new IllegalArgumentException("filePath cannot be blank");
- }
-
- Path path = Paths.get(filePath).normalize();
- if (path.isAbsolute()) {
- return path;
- }
-
- Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize();
- Path baseName = basePath.getFileName();
- if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) {
- return basePath.getParent().resolve(path).normalize();
- }
-
- Path pathFromWorkingDir = path.toAbsolutePath().normalize();
- if (pathFromWorkingDir.startsWith(basePath)) {
- return pathFromWorkingDir;
- }
-
- return basePath.resolve(path).normalize();
- }
-
- public void addToIndex(KnowledgeEntry entry) {
- knowledgeIndex.add(entry);
- log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
- }
-
- public void removeFromIndex(String filePath) {
- knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
- log.debug("文档已从 L0 索引移除: {}", filePath);
- }
-
- /** Clear in-memory L0 entries (used by knowledge rebuild). */
- public void clearIndex() {
- knowledgeIndex.clear();
- log.info("L0 knowledge index cleared");
- }
-
- public int getIndexSize() {
- return knowledgeIndex.size();
- }
-
- /**
- * 获取所有索引条目（供域聚合使用）
- */
- public List getAllEntries() {
- return List.copyOf(knowledgeIndex);
- }
-
- /**
- * L0 分析结果。
- *
- * @param matches 命中的文档条目（仅 hint，不是 evidence）
- * @param matchedKeywords 命中的关键词
- * @param domains 命中文档的 category 集合
- * @param entities 当前实现等同 matchedKeywords，预留实体字段
- * @param titles 命中文档标题
- */
- public record L0Hint(
- List matches,
- List matchedKeywords,
- List domains,
- List entities,
- List titles
- ) {
- public static L0Hint empty() {
- return new L0Hint(List.of(), List.of(), List.of(), List.of(), List.of());
- }
-
- /** 仅当恰好一个 domain 时返回，用于安全地加 category filter。 */
- public String singleDomainOrNull() {
- return domains.size() == 1 ? domains.get(0) : null;
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeQueryTransformer.java b/src/main/java/com/superbiz/agent/service/KnowledgeQueryTransformer.java
deleted file mode 100644
index dcc40d2..0000000
--- a/src/main/java/com/superbiz/agent/service/KnowledgeQueryTransformer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.dto.KnowledgeQuery;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-
-/**
- * 检索前的 query 理解层（L0 出口）。
- *
- * 输入是 Agent 的原始检索句，输出 {@link KnowledgeQuery}，供后续 L1 过滤与 rerank 使用。
- *
- * 当前能力边界
- *
- * - 会做：关键词匹配、domain/entity/title hint、唯一 domain 时生成 categoryFilter
- * - 不会做：真正的 query rewrite / 同义词扩展 / 多 query 改写
- * （{@code rewrittenQuery} 目前等于 {@code originalQuery}）
- * - L0 命中文档正文不会直接当作 evidence；证据只来自 L1 向量召回
- *
- */
-@Service
-public class KnowledgeQueryTransformer {
-
- private final KnowledgeIndexService knowledgeIndexService;
-
- public KnowledgeQueryTransformer(KnowledgeIndexService knowledgeIndexService) {
- this.knowledgeIndexService = knowledgeIndexService;
- }
-
- /**
- * 将原始 query 转为检索控制结构。
- *
- * {@code categoryFilter} 仅在 L0 恰好命中一个 domain 时非空；
- * 多 domain 或零 domain 时为 null，避免错误收窄召回。
- */
- public KnowledgeQuery transform(String rawQuery) {
- String normalized = rawQuery == null ? "" : rawQuery.trim();
- KnowledgeIndexService.L0Hint hint = knowledgeIndexService.analyzeQuery(normalized);
- return KnowledgeQuery.builder()
- .originalQuery(normalized)
- // 预留改写字段；当前未实现 rewrite，保持与 original 一致
- .rewrittenQuery(normalized)
- .domainHints(safeList(hint.domains()))
- .matchedKeywords(safeList(hint.matchedKeywords()))
- .entities(safeList(hint.entities()))
- // 只有唯一 domain 才作为向量 metadata 的 category 过滤条件
- .categoryFilter(hint.singleDomainOrNull())
- .l0Titles(safeList(hint.titles()))
- .l0MatchCount(hint.matches() == null ? 0 : hint.matches().size())
- .build();
- }
-
- private List safeList(List values) {
- return values == null ? List.of() : values;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/RagRetrievalSidecarComparisonService.java b/src/main/java/com/superbiz/agent/service/RagRetrievalSidecarComparisonService.java
deleted file mode 100644
index 2a8ef0d..0000000
--- a/src/main/java/com/superbiz/agent/service/RagRetrievalSidecarComparisonService.java
+++ /dev/null
@@ -1,179 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.config.RagSidecarProperties;
-import com.superbiz.agent.dto.ComparableRetrievalResult;
-import com.superbiz.agent.dto.RetrievalComparisonCase;
-import com.superbiz.agent.dto.RetrievalComparisonReport;
-import com.superbiz.agent.dto.RetrievalComparisonResult;
-import com.superbiz.agent.dto.SidecarRetrievalResponse;
-import org.springframework.stereotype.Service;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-@Service
-public class RagRetrievalSidecarComparisonService {
-
- private final VectorSearchService vectorSearchService;
- private final SpringAiVectorStoreSidecarService sidecarService;
- private final RetrievalResultNormalizer normalizer;
- private final RagSidecarProperties properties;
- private final ObjectMapper objectMapper;
-
- public RagRetrievalSidecarComparisonService(VectorSearchService vectorSearchService,
- SpringAiVectorStoreSidecarService sidecarService,
- RetrievalResultNormalizer normalizer,
- RagSidecarProperties properties,
- ObjectMapper objectMapper) {
- this.vectorSearchService = vectorSearchService;
- this.sidecarService = sidecarService;
- this.normalizer = normalizer;
- this.properties = properties;
- this.objectMapper = objectMapper;
- }
-
- public RetrievalComparisonReport compare(List cases, int topK) {
- List results = new ArrayList<>();
- String sidecarStatus = "not_run";
- for (RetrievalComparisonCase comparisonCase : cases) {
- List currentResults = normalizeCurrentResults(
- vectorSearchService.searchSimilarDocuments(
- comparisonCase.getQuery(),
- topK,
- comparisonCase.getCategory()
- )
- );
- SidecarRetrievalResponse sidecar = sidecarService.search(
- comparisonCase.getQuery(),
- topK,
- comparisonCase.getCategory()
- );
- sidecarStatus = sidecar.getStatus();
- results.add(RetrievalComparisonResult.builder()
- .caseId(comparisonCase.getCaseId())
- .scenario(comparisonCase.getScenario())
- .query(comparisonCase.getQuery())
- .category(comparisonCase.getCategory())
- .currentResults(currentResults)
- .sidecar(sidecar)
- .differences(compareDifferences(currentResults, sidecar.getResults()))
- .build());
- }
-
- return RetrievalComparisonReport.builder()
- .generatedAt(OffsetDateTime.now(ZoneOffset.UTC).toString())
- .caseCount(cases.size())
- .topK(topK)
- .sidecarStatus(sidecarStatus)
- .results(results)
- .build();
- }
-
- public RetrievalComparisonReport compareGoldenCases(Path caseFile) throws IOException {
- var root = objectMapper.readTree(caseFile.toFile());
- int topK = root.path("topK").asInt(5);
- List cases = new ArrayList<>();
- for (var node : root.path("cases")) {
- cases.add(RetrievalComparisonCase.builder()
- .caseId(node.path("caseId").asText())
- .scenario(node.path("scenario").asText())
- .query(node.path("query").asText())
- .build());
- }
- return compare(cases, topK);
- }
-
- public void writeReports(RetrievalComparisonReport report, Path jsonPath, Path markdownPath) throws IOException {
- createParentDirectories(jsonPath);
- createParentDirectories(markdownPath);
- objectMapper.writerWithDefaultPrettyPrinter().writeValue(jsonPath.toFile(), report);
- Files.writeString(markdownPath, renderMarkdown(report));
- }
-
- private void createParentDirectories(Path path) throws IOException {
- Path parent = path.getParent();
- if (parent != null) {
- Files.createDirectories(parent);
- }
- }
-
- private List normalizeCurrentResults(List rawResults) {
- List results = new ArrayList<>();
- for (int i = 0; i < rawResults.size(); i++) {
- results.add(normalizer.fromCurrent(rawResults.get(i), i + 1, properties.getContentPreviewLimit()));
- }
- return results;
- }
-
- private List compareDifferences(List currentResults,
- List sidecarResults) {
- if (sidecarResults == null || sidecarResults.isEmpty()) {
- return List.of("sidecar_unavailable_or_empty");
- }
- List differences = new ArrayList<>();
- String currentTopSource = currentResults.isEmpty() ? null : currentResults.get(0).getSource();
- String sidecarTopSource = sidecarResults.get(0).getSource();
- if (!Objects.equals(currentTopSource, sidecarTopSource)) {
- differences.add("top_source_differs");
- }
- String currentTopBreadcrumb = currentResults.isEmpty() ? null : currentResults.get(0).getBreadcrumb();
- String sidecarTopBreadcrumb = sidecarResults.get(0).getBreadcrumb();
- if (!Objects.equals(currentTopBreadcrumb, sidecarTopBreadcrumb)) {
- differences.add("top_breadcrumb_differs");
- }
- String currentScoreLabel = currentResults.isEmpty() ? null : currentResults.get(0).getScoreLabel();
- String sidecarScoreLabel = sidecarResults.get(0).getScoreLabel();
- if (!Objects.equals(currentScoreLabel, sidecarScoreLabel)) {
- differences.add("score_label_differs");
- }
- return differences;
- }
-
- private String renderMarkdown(RetrievalComparisonReport report) {
- StringBuilder builder = new StringBuilder();
- builder.append("# RAG Sidecar Retrieval Comparison\n\n");
- builder.append("Generated at: `").append(report.getGeneratedAt()).append("`\n\n");
- builder.append("- Cases: ").append(report.getCaseCount()).append("\n");
- builder.append("- Top K: ").append(report.getTopK()).append("\n");
- builder.append("- Sidecar status: `").append(report.getSidecarStatus()).append("`\n\n");
- builder.append("| Case | Query | Current Top | Sidecar Top | Differences |\n");
- builder.append("|---|---|---|---|---|\n");
- for (RetrievalComparisonResult result : report.getResults()) {
- builder.append("| ")
- .append(nullToBlank(result.getCaseId()))
- .append(" | ")
- .append(escapePipe(result.getQuery()))
- .append(" | ")
- .append(formatTop(result.getCurrentResults()))
- .append(" | ")
- .append(formatTop(result.getSidecar() != null ? result.getSidecar().getResults() : List.of()))
- .append(" | ")
- .append(String.join("
", result.getDifferences()))
- .append(" |\n");
- }
- return builder.toString();
- }
-
- private String formatTop(List results) {
- if (results == null || results.isEmpty()) {
- return "";
- }
- ComparableRetrievalResult top = results.get(0);
- return escapePipe(nullToBlank(top.getSource())) + " (" + nullToBlank(top.getScoreLabel()) + ")";
- }
-
- private String escapePipe(String value) {
- return nullToBlank(value).replace("|", "\\|");
- }
-
- private String nullToBlank(String value) {
- return value == null ? "" : value;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/RagService.java b/src/main/java/com/superbiz/agent/service/RagService.java
deleted file mode 100644
index ae38712..0000000
--- a/src/main/java/com/superbiz/agent/service/RagService.java
+++ /dev/null
@@ -1,190 +0,0 @@
-package com.superbiz.agent.service;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.ai.chat.messages.AssistantMessage;
-import org.springframework.ai.chat.messages.Message;
-import org.springframework.ai.chat.messages.UserMessage;
-import org.springframework.ai.chat.model.ChatModel;
-import org.springframework.ai.chat.model.ChatResponse;
-import org.springframework.ai.chat.prompt.Prompt;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-import reactor.core.publisher.Flux;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * RAG (Retrieval-Augmented Generation) 服务
- * 结合向量检索和大语言模型生成答案
- */
-@Service
-public class RagService {
-
- private static final Logger logger = LoggerFactory.getLogger(RagService.class);
-
- @Autowired
- private VectorSearchService vectorSearchService;
-
- @Autowired
- private ChatModel chatModel;
-
- @Value("${rag.top-k:3}")
- private int topK;
-
- /**
- * 流式处理用户问题（不带历史消息）
- *
- * @param question 用户问题
- * @param callback 流式回调接口
- */
- public void queryStream(String question, StreamCallback callback) {
- queryStream(question, new ArrayList<>(), callback);
- }
-
- /**
- * 流式处理用户问题（带历史消息）
- *
- * @param question 用户问题
- * @param history 历史消息列表，格式：[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
- * @param callback 流式回调接口
- */
- public void queryStream(String question, List> history, StreamCallback callback) {
- try {
- logger.info("收到 RAG 流式查询: {}", question);
-
- // 1. 从向量数据库检索相关文档
- List searchResults =
- vectorSearchService.searchSimilarDocuments(question, topK);
-
- // 发送检索结果
- callback.onSearchResults(searchResults);
-
- if (searchResults.isEmpty()) {
- logger.warn("未找到相关文档");
- callback.onComplete("抱歉，我在知识库中没有找到相关信息来回答您的问题。", "");
- return;
- }
-
- // 2. 构建上下文和提示词
- String context = buildContext(searchResults);
- String prompt = buildPrompt(question, context);
-
- // 3. 流式调用大语言模型（传入历史消息）
- generateAnswerStream(prompt, history, callback);
-
- } catch (Exception e) {
- logger.error("RAG 流式查询失败", e);
- callback.onError(e);
- }
- }
-
- /**
- * 构建上下文
- */
- private String buildContext(List searchResults) {
- StringBuilder context = new StringBuilder();
-
- for (int i = 0; i < searchResults.size(); i++) {
- VectorSearchService.SearchResult result = searchResults.get(i);
- context.append("【参考资料 ").append(i + 1).append("】\n");
- context.append(result.getContent()).append("\n\n");
- }
-
- return context.toString();
- }
-
- /**
- * 构建提示词
- */
- private String buildPrompt(String question, String context) {
- return String.format(
- "你是一个专业的AI助手。请根据以下参考资料回答用户的问题。\n\n" +
- "参考资料：\n%s\n" +
- "用户问题：%s\n\n" +
- "请基于上述参考资料给出准确、详细的回答。如果参考资料中没有相关信息，请明确说明。",
- context, question
- );
- }
-
- /**
- * 生成答案（流式）
- *
- * @param prompt 当前问题的提示词
- * @param history 历史消息列表
- * @param callback 流式回调接口
- */
- private void generateAnswerStream(String prompt, List> history, StreamCallback callback) {
- // 构建消息列表：历史消息 + 当前问题
- List messages = new ArrayList<>();
-
- // 添加历史消息
- for (Map historyMsg : history) {
- String role = historyMsg.get("role");
- String content = historyMsg.get("content");
-
- if ("user".equals(role)) {
- messages.add(new UserMessage(content));
- } else if ("assistant".equals(role)) {
- messages.add(new AssistantMessage(content));
- }
- }
-
- // 添加当前用户问题
- messages.add(new UserMessage(prompt));
-
- logger.debug("发送给AI模型的消息数量: {}（包含 {} 条历史消息）",
- messages.size(), history.size());
-
- logger.info("开始调用AI模型流式接口...");
-
- StringBuilder reasoningContent = new StringBuilder();
- StringBuilder finalContent = new StringBuilder();
-
- Flux flux = chatModel.stream(new Prompt(messages));
-
- logger.info("开始接收AI模型流式响应...");
-
- flux.subscribe(
- response -> {
- if (response.getResults() != null && !response.getResults().isEmpty()) {
- String content = response.getResults().get(0).getOutput().getText();
-
- if (content != null && !content.isEmpty()) {
- logger.debug("收到AI模型内容块: {}", content);
-
- finalContent.append(content);
- callback.onContentChunk(content);
-
- logger.debug("已调用 onContentChunk 回调");
- } else {
- logger.debug("收到空内容块，跳过");
- }
- }
- },
- error -> {
- logger.error("AI模型流式响应失败", error);
- callback.onError(new Exception("AI模型流式响应失败: " + error.getMessage(), error));
- },
- () -> {
- logger.info("AI模型流式响应完成，总内容长度: {}", finalContent.length());
- callback.onComplete(finalContent.toString(), reasoningContent.toString());
- logger.info("已调用 onComplete 回调");
- }
- );
- }
-
- /**
- * 流式回调接口
- */
- public interface StreamCallback {
- void onSearchResults(List results);
- void onReasoningChunk(String chunk);
- void onContentChunk(String chunk);
- void onComplete(String fullContent, String fullReasoning);
- void onError(Exception e);
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/RetrievalResultNormalizer.java b/src/main/java/com/superbiz/agent/service/RetrievalResultNormalizer.java
deleted file mode 100644
index 3424b32..0000000
--- a/src/main/java/com/superbiz/agent/service/RetrievalResultNormalizer.java
+++ /dev/null
@@ -1,96 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.dto.ComparableRetrievalResult;
-import org.springframework.ai.document.Document;
-import org.springframework.stereotype.Component;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-@Component
-public class RetrievalResultNormalizer {
-
- private final ObjectMapper objectMapper;
-
- public RetrievalResultNormalizer(ObjectMapper objectMapper) {
- this.objectMapper = objectMapper;
- }
-
- public ComparableRetrievalResult fromCurrent(VectorSearchService.SearchResult result, int rank, int previewLimit) {
- Map metadata = parseMetadata(result.getMetadata());
- String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), result.getMetadata(), result.getId());
- return ComparableRetrievalResult.builder()
- .path("current")
- .rank(rank)
- .id(result.getId())
- .source(source)
- .docId(metadata.get("docId"))
- .title(metadata.get("title"))
- .breadcrumb(metadata.get("breadcrumb"))
- .category(metadata.get("category"))
- .contentPreview(truncate(result.getContent(), previewLimit))
- .scoreLabel("l2_distance")
- .scoreValue((double) result.getScore())
- .build();
- }
-
- public ComparableRetrievalResult fromSidecar(Document document, int rank, int previewLimit) {
- Map metadata = stringifyMetadata(document.getMetadata());
- String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), metadata.get("docId"), document.getId());
- return ComparableRetrievalResult.builder()
- .path("sidecar")
- .rank(rank)
- .id(document.getId())
- .source(source)
- .docId(metadata.get("docId"))
- .title(metadata.get("title"))
- .breadcrumb(metadata.get("breadcrumb"))
- .category(metadata.get("category"))
- .contentPreview(truncate(document.getText(), previewLimit))
- .scoreLabel("similarity")
- .scoreValue(document.getScore())
- .build();
- }
-
- private Map parseMetadata(String metadata) {
- if (metadata == null || metadata.isBlank()) {
- return Map.of();
- }
- try {
- Map raw = objectMapper.readValue(metadata, Map.class);
- return stringifyMetadata(raw);
- } catch (Exception e) {
- return Map.of();
- }
- }
-
- private Map stringifyMetadata(Map raw) {
- if (raw == null || raw.isEmpty()) {
- return Map.of();
- }
- Map result = new LinkedHashMap<>();
- for (Map.Entry entry : raw.entrySet()) {
- if (entry.getKey() != null && entry.getValue() != null) {
- result.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
- }
- }
- return result;
- }
-
- private String firstNonBlank(String... values) {
- for (String value : values) {
- if (value != null && !value.isBlank()) {
- return value;
- }
- }
- return null;
- }
-
- private String truncate(String text, int maxLength) {
- if (text == null || text.length() <= maxLength) {
- return text;
- }
- return text.substring(0, maxLength) + "...";
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/SpringAiVectorStoreSidecarService.java b/src/main/java/com/superbiz/agent/service/SpringAiVectorStoreSidecarService.java
deleted file mode 100644
index d8911ed..0000000
--- a/src/main/java/com/superbiz/agent/service/SpringAiVectorStoreSidecarService.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.config.RagSidecarProperties;
-import com.superbiz.agent.dto.ComparableRetrievalResult;
-import com.superbiz.agent.dto.SidecarRetrievalResponse;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.ai.document.Document;
-import org.springframework.ai.vectorstore.SearchRequest;
-import org.springframework.ai.vectorstore.VectorStore;
-import org.springframework.beans.factory.ObjectProvider;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@Slf4j
-@Service
-public class SpringAiVectorStoreSidecarService {
-
- private final RagSidecarProperties properties;
- private final ObjectProvider vectorStoreProvider;
- private final RetrievalResultNormalizer normalizer;
-
- @Value("${retrieval.kb-scope:}")
- private String kbScope = "";
-
- public SpringAiVectorStoreSidecarService(RagSidecarProperties properties,
- ObjectProvider vectorStoreProvider,
- RetrievalResultNormalizer normalizer) {
- this.properties = properties;
- this.vectorStoreProvider = vectorStoreProvider;
- this.normalizer = normalizer;
- }
-
- public SidecarRetrievalResponse search(String query, int topK, String category) {
- if (!properties.isEnabled()) {
- return unavailable("disabled", null);
- }
-
- VectorStore vectorStore = vectorStoreProvider.getIfAvailable();
- if (vectorStore == null) {
- return unavailable("missing_vector_store", "No Spring AI VectorStore bean is available");
- }
-
- try {
- SearchRequest.Builder builder = SearchRequest.builder()
- .query(query)
- .topK(topK)
- .similarityThresholdAll();
- String filterExpression = buildFilterExpression(category);
- if (filterExpression != null) {
- builder.filterExpression(filterExpression);
- }
-
- List documents = vectorStore.similaritySearch(builder.build());
- List results = new ArrayList<>();
- for (int i = 0; i < documents.size(); i++) {
- results.add(normalizer.fromSidecar(documents.get(i), i + 1, properties.getContentPreviewLimit()));
- }
- return SidecarRetrievalResponse.builder()
- .enabled(true)
- .available(true)
- .status("available")
- .results(results)
- .build();
- } catch (Exception e) {
- log.warn("Spring AI sidecar retrieval failed: {}", e.getMessage());
- return unavailable("query_failed", e.getMessage());
- }
- }
-
- private SidecarRetrievalResponse unavailable(String status, String errorMessage) {
- return SidecarRetrievalResponse.builder()
- .enabled(properties.isEnabled())
- .available(false)
- .status(status)
- .errorMessage(errorMessage)
- .results(List.of())
- .build();
- }
-
- private String escapeFilterValue(String value) {
- return value.replace("'", "\\'");
- }
-
- String buildFilterExpression(String category) {
- List parts = new ArrayList<>();
- String categoryFilter = trimToNull(category);
- if (categoryFilter != null) {
- parts.add("category == '" + escapeFilterValue(categoryFilter) + "'");
- }
- String scopeFilter = trimToNull(kbScope);
- if (scopeFilter != null) {
- parts.add("kb_scope == '" + escapeFilterValue(scopeFilter) + "'");
- }
- return parts.isEmpty() ? null : String.join(" && ", parts);
- }
-
- private String trimToNull(String value) {
- if (value == null || value.isBlank()) {
- return null;
- }
- return value.trim();
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/TextExtractorService.java b/src/main/java/com/superbiz/agent/service/TextExtractorService.java
deleted file mode 100644
index cf89146..0000000
--- a/src/main/java/com/superbiz/agent/service/TextExtractorService.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.exception.DocumentProcessException;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-import org.springframework.web.multipart.MultipartFile;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-
-/**
- * 文本提取服务
- * 仅支持 Markdown (.md) 和纯文本 (.txt) 格式
- * 其他格式（.docx、.pdf 等）需要通过外部转换服务先转为 Markdown
- */
-@Slf4j
-@Service
-public class TextExtractorService {
-
- /**
- * 从文件中提取文本
- *
- * @param file 上传的文件
- * @param fileName 文件名
- * @return 提取的文本内容
- */
- public String extractText(MultipartFile file, String fileName) {
- if (file == null || file.isEmpty()) {
- throw new DocumentProcessException(fileName, "extract", "文件为空");
- }
-
- String extension = getFileExtension(fileName);
- log.info("开始提取文本，文件名: {}, 格式: {}, 大小: {} bytes", fileName, extension, file.getSize());
-
- if (!isSupportedFormat(fileName)) {
- throw new DocumentProcessException(
- fileName, "extract",
- "不支持的文件格式: " + extension + "，仅支持 .md 和 .txt。其他格式请先通过转换服务转为 Markdown。"
- );
- }
-
- try {
- String text = extractPlainText(file);
- log.info("文本提取成功，文件名: {}, 提取字符数: {}", fileName, text.length());
- return text;
-
- } catch (IOException e) {
- log.error("文本提取失败，文件名: {}", fileName, e);
- throw new DocumentProcessException(fileName, "extract", "文件读取失败: " + e.getMessage(), e);
- }
- }
-
- /**
- * 提取纯文本（.txt、.md）
- */
- private String extractPlainText(MultipartFile file) throws IOException {
- StringBuilder content = new StringBuilder();
- try (InputStream is = file.getInputStream();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
-
- String line;
- while ((line = reader.readLine()) != null) {
- content.append(line).append("\n");
- }
- }
- return content.toString().trim();
- }
-
- /**
- * 获取文件扩展名
- */
- private String getFileExtension(String fileName) {
- if (fileName == null || !fileName.contains(".")) {
- return "";
- }
- return fileName.substring(fileName.lastIndexOf(".") + 1);
- }
-
- /**
- * 验证文件格式是否支持
- */
- public boolean isSupportedFormat(String fileName) {
- String extension = getFileExtension(fileName).toLowerCase();
- return extension.equals("md") || extension.equals("txt");
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/VectorEmbeddingService.java b/src/main/java/com/superbiz/agent/service/VectorEmbeddingService.java
deleted file mode 100644
index 2fc46b2..0000000
--- a/src/main/java/com/superbiz/agent/service/VectorEmbeddingService.java
+++ /dev/null
@@ -1,125 +0,0 @@
-package com.superbiz.agent.service;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.ai.embedding.EmbeddingModel;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * 向量嵌入服务
- * 使用阿里云 DashScope Text Embedding API
- */
-@Service
-public class VectorEmbeddingService {
-
- private static final Logger logger = LoggerFactory.getLogger(VectorEmbeddingService.class);
-
- @Autowired
- private EmbeddingModel embeddingModel;
-
- /**
- * 生成向量嵌入
- * 调用阿里云 DashScope Text Embedding API
- *
- * @param content 文本内容
- * @return 向量嵌入（浮点数列表）
- */
- public List generateEmbedding(String content) {
- try {
- if (content == null || content.trim().isEmpty()) {
- logger.warn("内容为空，无法生成向量");
- throw new IllegalArgumentException("内容不能为空");
- }
-
- logger.debug("开始生成向量嵌入, 内容长度: {} 字符", content.length());
-
- float[] embedding = embeddingModel.embed(content);
-
- List floatEmbedding = new ArrayList<>(embedding.length);
- for (float v : embedding) {
- floatEmbedding.add(v);
- }
-
- logger.info("成功生成向量嵌入, 内容长度: {} 字符, 向量维度: {}",
- content.length(), floatEmbedding.size());
-
- return floatEmbedding;
-
- } catch (Exception e) {
- logger.error("生成向量嵌入失败, 内容长度: {}", content != null ? content.length() : 0, e);
- throw new RuntimeException("生成向量嵌入失败: " + e.getMessage(), e);
- }
- }
-
- public List> generateEmbeddings(List contents) {
- try {
- if (contents == null || contents.isEmpty()) {
- logger.warn("内容列表为空，无法生成向量");
- return Collections.emptyList();
- }
-
- logger.info("开始批量生成向量嵌入, 数量: {}", contents.size());
-
- List embeddings = embeddingModel.embed(contents);
-
- List> result = new ArrayList<>();
- for (float[] embedding : embeddings) {
- List floatEmbedding = new ArrayList<>(embedding.length);
- for (float v : embedding) {
- floatEmbedding.add(v);
- }
- result.add(floatEmbedding);
- }
-
- logger.info("成功批量生成向量嵌入, 数量: {}, 维度: {}",
- result.size(),
- result.isEmpty() ? 0 : result.get(0).size());
-
- return result;
-
- } catch (Exception e) {
- logger.error("批量生成向量嵌入失败", e);
- throw new RuntimeException("批量生成向量嵌入失败: " + e.getMessage(), e);
- }
- }
-
- /**
- * 生成查询向量
- *
- * @param query 查询文本
- * @return 向量嵌入
- */
- public List generateQueryVector(String query) {
- return generateEmbedding(query);
- }
-
- /**
- * 计算两个向量的余弦相似度
- *
- * @param vector1 向量1
- * @param vector2 向量2
- * @return 余弦相似度 [-1, 1]
- */
- public float calculateCosineSimilarity(List vector1, List vector2) {
- if (vector1.size() != vector2.size()) {
- throw new IllegalArgumentException("向量维度不匹配");
- }
-
- float dotProduct = 0.0f;
- float norm1 = 0.0f;
- float norm2 = 0.0f;
-
- for (int i = 0; i < vector1.size(); i++) {
- dotProduct += vector1.get(i) * vector2.get(i);
- norm1 += vector1.get(i) * vector1.get(i);
- norm2 += vector2.get(i) * vector2.get(i);
- }
-
- return dotProduct / (float) (Math.sqrt(norm1) * Math.sqrt(norm2));
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/VectorIndexService.java b/src/main/java/com/superbiz/agent/service/VectorIndexService.java
deleted file mode 100644
index 63d606e..0000000
--- a/src/main/java/com/superbiz/agent/service/VectorIndexService.java
+++ /dev/null
@@ -1,380 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.dto.DocumentChunk;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
-import lombok.Getter;
-import lombok.Setter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.io.File;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.time.LocalDateTime;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 向量索引写入服务（RAG 入库侧）。
- *
- * 唯一后端 {@link MilvusHybridKnowledgeStore}（Milvus SDK v2）：
- *
- * - dense：应用侧 embedding → 字段 {@code vector}
- * - BM25：{@link #buildSearchText} → 字段 {@code search_text}；
- * sparse 由 collection 上 BM25 Function 自动生成，本类不写 sparse
- *
- * 不再使用 legacy {@code MilvusServiceClient} insert/delete，
- * 也不走 Spring AI {@code VectorStore#add}（starter 无 hybrid schema/BM25 Function）。
- */
-@Service
-public class VectorIndexService {
-
- private static final Logger logger = LoggerFactory.getLogger(VectorIndexService.class);
-
- @Autowired
- private MilvusHybridKnowledgeStore knowledgeStore;
-
- @Autowired
- private VectorEmbeddingService embeddingService;
-
- @Autowired
- private DocumentChunkService chunkService;
-
- @Value("${file.upload.path}")
- private String uploadPath;
-
- public IndexingResult indexDirectory(String directoryPath) {
- IndexingResult result = new IndexingResult();
- result.setStartTime(LocalDateTime.now());
-
- try {
- String targetPath = (directoryPath != null && !directoryPath.trim().isEmpty())
- ? directoryPath : uploadPath;
-
- Path dirPath = Paths.get(targetPath).normalize();
- File directory = dirPath.toFile();
-
- if (!directory.exists() || !directory.isDirectory()) {
- throw new IllegalArgumentException("目录不存在或不是有效目录: " + targetPath);
- }
-
- result.setDirectoryPath(directory.getAbsolutePath());
-
- File[] files = directory.listFiles((dir, name) ->
- name.endsWith(".txt") || name.endsWith(".md")
- );
-
- if (files == null || files.length == 0) {
- logger.warn("目录中没有找到支持的文件: {}", targetPath);
- result.setTotalFiles(0);
- result.setSuccess(true);
- result.setEndTime(LocalDateTime.now());
- return result;
- }
-
- result.setTotalFiles(files.length);
- logger.info("开始索引目录: {}, 找到 {} 个文件", targetPath, files.length);
-
- for (File file : files) {
- try {
- indexSingleFile(file.getAbsolutePath());
- result.incrementSuccessCount();
- logger.info("文件索引成功: {}", file.getName());
- } catch (Exception e) {
- result.incrementFailCount();
- result.addFailedFile(file.getAbsolutePath(), e.getMessage());
- logger.error("文件索引失败: {}", file.getName(), e);
- }
- }
-
- result.setSuccess(result.getFailCount() == 0);
- result.setEndTime(LocalDateTime.now());
- return result;
-
- } catch (Exception e) {
- logger.error("索引目录失败", e);
- result.setSuccess(false);
- result.setErrorMessage(e.getMessage());
- result.setEndTime(LocalDateTime.now());
- return result;
- }
- }
-
- public void indexSingleFile(String filePath) throws Exception {
- Path path = Paths.get(filePath).normalize();
- File file = path.toFile();
-
- if (!file.exists() || !file.isFile()) {
- throw new IllegalArgumentException("文件不存在: " + filePath);
- }
-
- logger.info("开始索引文件: {}", path);
- String content = Files.readString(path);
- deleteExistingData(path.toString());
-
- List chunks = chunkService.chunkDocument(content, path.toString());
- logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
-
- for (int i = 0; i < chunks.size(); i++) {
- DocumentChunk chunk = chunks.get(i);
- try {
- // dense embedding 与 BM25 search_text 同源（title/path 增强）
- List vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
- Map metadata = buildMetadata(path.toString(), chunk, chunks.size());
- knowledgeStore.upsertChunk(
- chunk.getContent(), // 返回原文
- buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
- vector, // dense 向量
- metadata,
- chunk.getChunkIndex());
- logger.info("分片 {}/{} 索引成功", i + 1, chunks.size());
- } catch (Exception e) {
- logger.error("分片 {}/{} 索引失败", i + 1, chunks.size(), e);
- throw new RuntimeException("分片索引失败: " + e.getMessage(), e);
- }
- }
- logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
- }
-
- public void indexDocumentChunks(String docId, List chunks, String category) throws Exception {
- indexDocumentChunks(docId, chunks, category, null);
- }
-
- public void indexDocumentChunks(String docId,
- List chunks,
- String category,
- Frontmatter frontmatter) throws Exception {
- if (chunks == null || chunks.isEmpty()) {
- throw new IllegalArgumentException("文档分块列表为空");
- }
-
- logger.info("开始索引文档分块，docId: {}, 分块数: {}, 类别: {}", docId, chunks.size(), category);
- deleteDocumentChunks(docId);
-
- for (int i = 0; i < chunks.size(); i++) {
- DocumentChunk chunk = chunks.get(i);
- try {
- // dense embedding 与 BM25 search_text 同源（title/path 增强）
- List vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
- Map metadata = buildDocumentMetadata(docId, chunk, chunks.size(), category, frontmatter);
- knowledgeStore.upsertChunk(
- chunk.getContent(), // 返回原文
- buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
- vector, // dense 向量
- metadata,
- chunk.getChunkIndex());
- logger.info("文档分块 {}/{} 索引成功，docId: {}", i + 1, chunks.size(), docId);
- } catch (Exception e) {
- logger.error("文档分块 {}/{} 索引失败，docId: {}", i + 1, chunks.size(), docId, e);
- throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
- }
- }
- logger.info("文档索引完成，docId: {}, 共 {} 个分块，类别: {}", docId, chunks.size(), category);
- }
-
- public void deleteDocumentChunks(String docId) {
- try {
- knowledgeStore.deleteByDocId(docId);
- logger.info("删除文档旧数据成功，docId: {}", docId);
- } catch (Exception e) {
- logger.warn("删除文档旧数据异常，docId: {}", docId, e);
- }
- }
-
- static Map buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks, String category) {
- return buildDocumentMetadata(docId, chunk, totalChunks, category, null);
- }
-
- static Map buildDocumentMetadata(String docId,
- DocumentChunk chunk,
- int totalChunks,
- String category,
- Frontmatter frontmatter) {
- Map metadata = new HashMap<>();
- String source = firstNonBlank(frontmatter != null ? frontmatter.getSource() : null, "upload:" + docId);
- metadata.put("docId", docId);
- metadata.put("_source", source);
- metadata.put("source", source);
- metadata.put("chunkIndex", chunk.getChunkIndex());
- metadata.put("totalChunks", totalChunks);
-
- String title = firstNonBlank(chunk.getTitle(), frontmatter != null ? frontmatter.getTitle() : null);
- if (title != null) {
- metadata.put("title", title);
- }
- String breadcrumb = firstNonBlank(frontmatter != null ? frontmatter.getBreadcrumb() : null, chunk.getBreadcrumb());
- if (breadcrumb != null) {
- metadata.put("breadcrumb", breadcrumb);
- }
- metadata.put("category", category != null && !category.isBlank() ? category : "upload");
- String kbScope = trimToNull(frontmatter != null ? frontmatter.getKbScope() : null);
- if (kbScope != null) {
- metadata.put("kb_scope", kbScope);
- }
- return metadata;
- }
-
- /**
- * Dense embedding 输入。与 {@link #buildSearchText} 同源，保证 dense/BM25 看到同一增强文本。
- */
- static String buildEmbeddingText(DocumentChunk chunk) {
- return buildSearchText(chunk);
- }
-
- /**
- * 构造写入 Milvus 的检索文本（BM25 {@code search_text}，并复用为 dense embedding 输入）。
- *
- * 在正文前拼接 title / breadcrumb，提高「按标题或路径关键词」的 BM25 命中率，
- * 同时让 dense 向量也编码结构信息。无标题路径时退回纯 content。
- */
- static String buildSearchText(DocumentChunk chunk) {
- String content = trimToEmpty(chunk.getContent());
- String title = trimToEmpty(chunk.getTitle());
- String breadcrumb = trimToEmpty(chunk.getBreadcrumb());
-
- if (title.isEmpty() && breadcrumb.isEmpty()) {
- return content;
- }
-
- StringBuilder text = new StringBuilder();
- if (!title.isEmpty()) {
- text.append("Title: ").append(title).append("\n");
- }
- if (!breadcrumb.isEmpty()) {
- text.append("Path: ").append(breadcrumb).append("\n");
- }
- text.append("Content:\n").append(content);
- return text.toString();
- }
-
- private static String trimToEmpty(String value) {
- return value == null ? "" : value.trim();
- }
-
- private static String trimToNull(String value) {
- if (value == null || value.isBlank()) {
- return null;
- }
- return value.trim();
- }
-
- private static String firstNonBlank(String... values) {
- for (String value : values) {
- String trimmed = trimToNull(value);
- if (trimmed != null) {
- return trimmed;
- }
- }
- return null;
- }
-
- private void deleteExistingData(String filePath) {
- try {
- Path path = Paths.get(filePath).normalize();
- String normalizedPath = path.toString().replace(File.separator, "/");
- knowledgeStore.deleteBySource(normalizedPath);
- logger.info("已删除文件的旧数据: {}", normalizedPath);
- } catch (Exception e) {
- logger.warn("删除旧数据失败（可能是首次索引）: {}", e.getMessage());
- }
- }
-
- private Map buildMetadata(String filePath, DocumentChunk chunk, int totalChunks) {
- Map metadata = new HashMap<>();
- Path path = Paths.get(filePath).normalize();
- String normalizedPath = path.toString().replace(File.separator, "/");
-
- Path fileName = path.getFileName();
- String fileNameStr = fileName != null ? fileName.toString() : "";
- String extension = "";
- int dotIndex = fileNameStr.lastIndexOf('.');
- if (dotIndex > 0) {
- extension = fileNameStr.substring(dotIndex);
- }
-
- metadata.put("_source", normalizedPath);
- metadata.put("source", normalizedPath);
- metadata.put("_extension", extension);
- metadata.put("_file_name", fileNameStr);
-
- String category = extractCategory(normalizedPath);
- if (category != null && !category.isEmpty()) {
- metadata.put("category", category);
- }
- metadata.put("chunkIndex", chunk.getChunkIndex());
- metadata.put("totalChunks", totalChunks);
- if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
- metadata.put("title", chunk.getTitle());
- }
- if (chunk.getBreadcrumb() != null && !chunk.getBreadcrumb().isEmpty()) {
- metadata.put("breadcrumb", chunk.getBreadcrumb());
- }
- return metadata;
- }
-
- private String extractCategory(String filePath) {
- try {
- String normalized = filePath.replace("\\", "/");
- int docsIndex = normalized.indexOf("aiops-docs/");
- if (docsIndex >= 0) {
- String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
- int slashIndex = afterDocs.indexOf("/");
- if (slashIndex > 0) {
- return afterDocs.substring(0, slashIndex);
- }
- }
- int firstSlash = normalized.indexOf("/");
- if (firstSlash > 0) {
- return normalized.substring(0, firstSlash);
- }
- return null;
- } catch (Exception e) {
- logger.warn("提取类别失败，路径: {}", filePath, e);
- return null;
- }
- }
-
- @Getter
- public static class IndexingResult {
- @Setter
- private boolean success;
- @Setter
- private String directoryPath;
- @Setter
- private int totalFiles;
- private int successCount;
- private int failCount;
- @Setter
- private LocalDateTime startTime;
- @Setter
- private LocalDateTime endTime;
- @Setter
- private String errorMessage;
- private Map failedFiles = new HashMap<>();
-
- public void incrementSuccessCount() {
- this.successCount++;
- }
-
- public void incrementFailCount() {
- this.failCount++;
- }
-
- public long getDurationMs() {
- if (startTime != null && endTime != null) {
- return java.time.Duration.between(startTime, endTime).toMillis();
- }
- return 0;
- }
-
- public void addFailedFile(String filePath, String error) {
- this.failedFiles.put(filePath, error);
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/VectorSearchService.java b/src/main/java/com/superbiz/agent/service/VectorSearchService.java
deleted file mode 100644
index 4b6deee..0000000
--- a/src/main/java/com/superbiz/agent/service/VectorSearchService.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
-import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
-import lombok.Getter;
-import lombok.Setter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.Locale;
-
-/**
- * 知识库向量检索门面（lookup_knowledge / RAG 召回入口）。
- *
- * 唯一后端：{@link MilvusHybridKnowledgeStore}（Milvus Java SDK v2）。
- *
- * 模式切换
- * {@code retrieval.search.mode}（同库查询算法，非两套写入）：
- *
- * - {@code hybrid} —— 线上主路径：dense + 服务端 BM25 + RRF
- * - {@code dense} —— 对照/评测：仅 dense ANN
- *
- * 命中 {@link SearchResult#scoreLabel} 仅为 {@link RetrievalScoreLabels#DENSE} /
- * {@link RetrievalScoreLabels#HYBRID}。质量分由后处理 {@code RetrievalScoreNormalizer} 统一计算。
- */
-@Service
-public class VectorSearchService {
-
- private static final Logger logger = LoggerFactory.getLogger(VectorSearchService.class);
-
- @Autowired
- private MilvusHybridKnowledgeStore knowledgeStore;
-
- @Autowired
- private VectorEmbeddingService embeddingService;
-
- /**
- * 检索模式：{@code hybrid}（主路径）| {@code dense}（召回对照）。
- */
- @Value("${retrieval.search.mode:dense}")
- private String searchMode = "dense";
-
- public List searchSimilarDocuments(String query, int topK) {
- return searchSimilarDocuments(query, topK, null);
- }
-
- public List searchSimilarDocuments(String query, int topK, String category) {
- String mode = searchMode == null ? "dense" : searchMode.trim().toLowerCase(Locale.ROOT);
- List queryVector = embeddingService.generateQueryVector(query);
- if ("hybrid".equals(mode)) {
- logger.info("Hybrid dense+BM25 search topK={} category={} collection={}",
- topK, category, knowledgeStore.collectionName());
- return knowledgeStore.searchHybrid(query, queryVector, topK, category);
- }
- logger.info("Dense search topK={} category={} collection={}",
- topK, category, knowledgeStore.collectionName());
- return knowledgeStore.searchDense(query, queryVector, topK, category);
- }
-
- /**
- * 单条召回结果。列表顺序即检索权威序（adapter 赋 originalRank=1..n）。
- *
- *
- * - {@code scoreLabel=dense}：{@link #score} = L2 距离（越小越好）
- * - {@code scoreLabel=hybrid}：{@link #score}/{@link #rawScore} = 引擎融合分；
- * 后处理 quality 主要按 rank 映射，不把 score 当 L2
- *
- */
- @Setter
- @Getter
- public static class SearchResult {
- private String id;
- private String content;
- /**
- * 引擎主分：dense=L2；hybrid=融合分（量纲由 scoreLabel 解释）。
- */
- private float score;
- /** 引擎原始分（与 score 同源或更细，便于调试）。 */
- private Double rawScore;
- /** {@link RetrievalScoreLabels#DENSE} 或 {@link RetrievalScoreLabels#HYBRID}。 */
- private String scoreLabel;
- /**
- * Optional dense L2 for the same id (hybrid path only).
- * Used for absolute quality / low-quality gates; does not replace sort order.
- */
- private Double denseDistance;
- /** metadata JSON 字符串（docId、source、title…）。 */
- private String metadata;
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java b/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java
deleted file mode 100644
index 3f3c6e3..0000000
--- a/src/main/java/com/superbiz/agent/service/milvus/MilvusHybridKnowledgeStore.java
+++ /dev/null
@@ -1,547 +0,0 @@
-package com.superbiz.agent.service.milvus;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.superbiz.agent.config.MilvusProperties;
-import com.superbiz.agent.constant.MilvusConstants;
-import com.superbiz.agent.service.VectorSearchService;
-import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
-import io.milvus.common.clientenum.FunctionType;
-import io.milvus.v2.client.ConnectConfig;
-import io.milvus.v2.client.MilvusClientV2;
-import io.milvus.v2.common.DataType;
-import io.milvus.v2.common.IndexParam;
-import io.milvus.v2.service.collection.request.AddFieldReq;
-import io.milvus.v2.service.collection.request.CreateCollectionReq;
-import io.milvus.v2.service.collection.request.DropCollectionReq;
-import io.milvus.v2.service.collection.request.HasCollectionReq;
-import io.milvus.v2.service.collection.request.LoadCollectionReq;
-import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
-import io.milvus.v2.service.index.request.CreateIndexReq;
-import io.milvus.v2.service.vector.request.AnnSearchReq;
-import io.milvus.v2.service.vector.request.DeleteReq;
-import io.milvus.v2.service.vector.request.HybridSearchReq;
-import io.milvus.v2.service.vector.request.InsertReq;
-import io.milvus.v2.service.vector.request.SearchReq;
-import io.milvus.v2.service.vector.request.data.BaseVector;
-import io.milvus.v2.service.vector.request.data.EmbeddedText;
-import io.milvus.v2.service.vector.request.data.FloatVec;
-import io.milvus.v2.service.vector.request.ranker.RRFRanker;
-import io.milvus.v2.service.vector.response.SearchResp;
-import jakarta.annotation.PreDestroy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * 知识库向量后端（Milvus Java SDK v2）—— dense + BM25 混合检索的唯一实现。
- *
- * 为什么不用 Spring AI {@code spring-ai-starter-vector-store-milvus}
- *
- * - Spring AI Milvus starter（截至 2.0.0 / 1.1.8）只封装 dense {@code similaritySearch}。
- * - 底层仍是 V1 {@code MilvusServiceClient} + 单路 {@code SearchParam}，无 {@code hybridSearch} /
- * BM25 Function / {@link RRFRanker}。
- * - 真混合检索（dense ANN + 服务端 BM25 sparse，再 RRF 融合）必须走 Milvus SDK v2，
- * 见 {@link #searchHybrid}。
- *
- *
- * Collection schema（默认名 {@code biz}）
- *
- * id VarChar PK
- * content VarChar —— 原文，返回给上层
- * search_text VarChar+analyzer —— BM25 输入文本（可含 title/path 增强）
- * sparse_vector SparseFloatVector —— 由 BM25 Function 从 search_text 自动生成，写入时不必填
- * vector FloatVector —— dense 向量（应用侧 embedding）
- * metadata JSON —— docId / source / category / kb_scope 等
- *
- *
- * 检索模式
- *
- * - {@link #searchDense}：单路 L2 ANN；{@code scoreLabel=dense}。
- * - {@link #searchHybrid}：dense + BM25 + 服务端 {@link RRFRanker}；{@code scoreLabel=hybrid}；
- * 返回序即 RRF 序，不再用 dense L2 覆盖主分。
- *
- *
- * 配置入口：{@code milvus.collection}、{@code retrieval.search.mode}、{@code retrieval.hybrid.rrf-k}。
- */
-@Service
-public class MilvusHybridKnowledgeStore {
-
- private static final Logger log = LoggerFactory.getLogger(MilvusHybridKnowledgeStore.class);
- private static final Gson GSON = new Gson();
-
- /** 主键（稳定 UUID，由 source + chunkIndex 派生，便于幂等重写）。 */
- public static final String FIELD_ID = "id";
- /** 返回给 LLM / 上层的原文 chunk。 */
- public static final String FIELD_CONTENT = "content";
- /**
- * BM25 输入字段。写入明文；Milvus 侧 analyzer + BM25 Function 生成 {@link #FIELD_SPARSE}。
- * 通常比 content 多带 title/path 等检索增强词。
- */
- public static final String FIELD_SEARCH_TEXT = "search_text";
- /** 稀疏向量字段；由 BM25 Function 自动产出，insert 时不要手动填。 */
- public static final String FIELD_SPARSE = "sparse_vector";
- /** Dense 向量字段（应用侧 EmbeddingModel 生成）。 */
- public static final String FIELD_DENSE = "vector";
- /** 业务元数据 JSON（过滤、证据身份、展示用）。 */
- public static final String FIELD_METADATA = "metadata";
-
- private final MilvusProperties milvusProperties;
-
- @Value("${milvus.collection:biz}")
- private String collectionName = "biz";
-
- /**
- * RRF 平滑参数 k：score(d) = Σ 1/(k + rank_i(d))。
- * k 越大，各路排名差异被压得越平；默认 60 与常见 RRF 设定一致。
- */
- @Value("${retrieval.hybrid.rrf-k:60}")
- private int rrfK = 60;
-
- /** 非空时追加 {@code metadata.kb_scope} 过滤，实现多知识域隔离。 */
- @Value("${retrieval.kb-scope:}")
- private String kbScope = "";
-
- private volatile MilvusClientV2 client;
-
- public MilvusHybridKnowledgeStore(MilvusProperties milvusProperties) {
- this.milvusProperties = milvusProperties;
- }
-
- /**
- * 懒连接：首次调用时建连、确保 collection schema 存在并 load。
- * 线程安全；后续检索/写入复用同一 {@link MilvusClientV2}。
- */
- public synchronized MilvusClientV2 client() {
- if (client == null) {
- client = connect();
- ensureCollection(client);
- loadCollection(client);
- }
- return client;
- }
-
- public String collectionName() {
- return collectionName;
- }
-
- /**
- * 写入单个 chunk（dense + BM25 所需明文）。
- *
- * 只插入 {@code content / search_text / vector / metadata}；
- * {@code sparse_vector} 由 collection 上的 BM25 Function 在服务端从 {@code search_text} 生成。
- *
- * id 由 {@code source|docId + chunkIndex} 的 nameUUID 派生，同一 chunk 重复写入会得到相同 id
- *（配合先 delete 再 insert 的上层逻辑实现覆盖）。
- *
- * @param content 原文（返回字段）
- * @param searchText BM25 / 可与 dense embedding 同源的检索文本
- * @param denseVector 应用侧 embedding
- * @param metadata 须尽量带 {@code _source} 或 {@code docId}，供 id 与过滤使用
- * @param chunkIndex 分片序号
- */
- public void upsertChunk(String content,
- String searchText,
- List denseVector,
- Map metadata,
- int chunkIndex) {
- String source = metadata == null ? null : stringVal(metadata.get("_source"));
- if (source == null) {
- source = metadata == null ? null : stringVal(metadata.get("source"));
- }
- if (source == null) {
- source = metadata == null ? null : stringVal(metadata.get("docId"));
- }
- String idSeed = (source == null ? "chunk" : source) + "_" + chunkIndex;
- String id = UUID.nameUUIDFromBytes(idSeed.getBytes()).toString();
-
- JsonObject row = new JsonObject();
- row.addProperty(FIELD_ID, id);
- row.addProperty(FIELD_CONTENT, content == null ? "" : content);
- // 仅写明文；sparse 由 BM25 Function(search_text -> sparse_vector) 自动生成
- row.addProperty(FIELD_SEARCH_TEXT, searchText == null ? "" : searchText);
- row.add(FIELD_DENSE, GSON.toJsonTree(denseVector));
- row.add(FIELD_METADATA, GSON.toJsonTree(metadata == null ? Map.of() : metadata));
-
- client().insert(InsertReq.builder()
- .collectionName(collectionName)
- .data(List.of(row))
- .build());
- }
-
- /** 按 metadata.docId 删除该文档全部 chunk（重建/覆盖前调用）。 */
- public void deleteByDocId(String docId) {
- if (docId == null || docId.isBlank()) {
- return;
- }
- String filter = "metadata[\"docId\"] == \"" + escapeFilter(docId) + "\"";
- client().delete(DeleteReq.builder()
- .collectionName(collectionName)
- .filter(filter)
- .build());
- }
-
- /** 按 metadata._source（规范化路径）删除，用于按文件路径重索引。 */
- public void deleteBySource(String sourcePath) {
- if (sourcePath == null || sourcePath.isBlank()) {
- return;
- }
- String normalized = sourcePath.replace('\\', '/');
- String filter = "metadata[\"_source\"] == \"" + escapeFilter(normalized) + "\"";
- client().delete(DeleteReq.builder()
- .collectionName(collectionName)
- .filter(filter)
- .build());
- }
-
- /**
- * 删除并重建当前知识 collection（空的 dense+BM25 schema）。
- * 供 {@code /api/knowledge/rebuild-hybrid} 与重建脚本使用；会销毁该 collection 全部向量。
- */
- public synchronized Map dropAndRecreateCollection() {
- Map result = new LinkedHashMap<>();
- result.put("collection", collectionName);
- MilvusClientV2 milvusClient = client();
- Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
- .collectionName(collectionName)
- .build());
- result.put("existedBefore", Boolean.TRUE.equals(exists));
- if (Boolean.TRUE.equals(exists)) {
- try {
- milvusClient.releaseCollection(ReleaseCollectionReq.builder()
- .collectionName(collectionName)
- .build());
- } catch (Exception e) {
- log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
- }
- milvusClient.dropCollection(DropCollectionReq.builder()
- .collectionName(collectionName)
- .build());
- log.info("Dropped hybrid collection '{}'", collectionName);
- result.put("dropped", true);
- } else {
- result.put("dropped", false);
- }
- ensureCollection(milvusClient);
- loadCollection(milvusClient);
- result.put("recreated", true);
- result.put("loaded", true);
- return result;
- }
-
- /**
- * 单路 dense ANN（L2）。
- * {@code score} = L2 距离（越小越好）；{@code scoreLabel} = {@link RetrievalScoreLabels#DENSE}。
- */
- public List searchDense(String queryEmbeddingText,
- List queryVector,
- int topK,
- String category) {
- String filter = buildFilter(category);
- SearchReq.SearchReqBuilder builder = SearchReq.builder()
- .collectionName(collectionName)
- .annsField(FIELD_DENSE)
- .data(List.of(new FloatVec(queryVector)))
- .topK(topK)
- .outputFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
- .metricType(IndexParam.MetricType.L2);
- if (filter != null) {
- builder.filter(filter);
- }
- SearchResp resp = client().search(builder.build());
- return toSearchResults(resp, RetrievalScoreLabels.DENSE);
- }
-
- /**
- * Dense + BM25 真混合检索（Milvus 服务端融合）。
- *
- *
- * - dense 子路：{@code vector}，L2
- * - BM25 子路：{@code sparse_vector} + {@link EmbeddedText}
- * - {@link HybridSearchReq} + {@link RRFRanker} → 返回序即权威序
- *
- *
- * {@code scoreLabel=hybrid}；{@code score}/{@code rawScore} 保留引擎融合分，
- * 不用 dense L2 覆盖主分或改 label。可选并行 dense 探测仅填充
- * {@link VectorSearchService.SearchResult#setDenseDistance}，供后处理绝对质量闸门
- * （如 L0 filter low-quality → unfiltered retry），排序仍以 RRF 返回序为准。
- */
- public List searchHybrid(String queryText,
- List queryVector,
- int topK,
- String category) {
- String filter = buildFilter(category);
- int pathTopK = Math.max(topK, 10);
-
- AnnSearchReq.AnnSearchReqBuilder denseAnn = AnnSearchReq.builder()
- .vectorFieldName(FIELD_DENSE)
- .vectors(List.of((BaseVector) new FloatVec(queryVector)))
- .topK(pathTopK)
- .metricType(IndexParam.MetricType.L2)
- .params("{\"nprobe\":10}");
- if (filter != null) {
- denseAnn.filter(filter);
- }
-
- AnnSearchReq.AnnSearchReqBuilder sparseAnn = AnnSearchReq.builder()
- .vectorFieldName(FIELD_SPARSE)
- .vectors(List.of((BaseVector) new EmbeddedText(queryText == null ? "" : queryText)))
- .topK(pathTopK)
- .metricType(IndexParam.MetricType.BM25);
- if (filter != null) {
- sparseAnn.filter(filter);
- }
-
- HybridSearchReq hybridReq = HybridSearchReq.builder()
- .collectionName(collectionName)
- .searchRequests(List.of(denseAnn.build(), sparseAnn.build()))
- .ranker(new RRFRanker(Math.max(1, rrfK)))
- .topK(topK)
- .outFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
- .build();
-
- SearchResp hybridResp = client().hybridSearch(hybridReq);
- List fused = toSearchResults(hybridResp, RetrievalScoreLabels.HYBRID);
- attachDenseDistances(fused, queryText, queryVector, pathTopK, category);
- return fused;
- }
-
- /**
- * Attach dense L2 by id for quality gates only — never overwrites hybrid score/label/order.
- */
- private void attachDenseDistances(List fused,
- String queryText,
- List queryVector,
- int pathTopK,
- String category) {
- if (fused == null || fused.isEmpty()) {
- return;
- }
- try {
- Map denseById = new HashMap<>();
- for (VectorSearchService.SearchResult denseHit :
- searchDense(queryText, queryVector, pathTopK, category)) {
- if (denseHit.getId() != null) {
- denseById.put(denseHit.getId(), denseHit.getScore());
- }
- }
- for (VectorSearchService.SearchResult hit : fused) {
- Float l2 = denseById.get(hit.getId());
- if (l2 != null) {
- hit.setDenseDistance(l2.doubleValue());
- }
- }
- } catch (Exception e) {
- log.warn("Dense distance attach for hybrid quality gate failed: {}", e.getMessage());
- }
- }
-
- /**
- * 将 Milvus {@link SearchResp} 映射为上层结果；列表顺序即检索权威序（adapter 赋 originalRank）。
- */
- private List toSearchResults(SearchResp resp, String scoreLabel) {
- List out = new ArrayList<>();
- if (resp == null || resp.getSearchResults() == null || resp.getSearchResults().isEmpty()) {
- return out;
- }
- List first = resp.getSearchResults().get(0);
- if (first == null) {
- return out;
- }
- for (SearchResp.SearchResult row : first) {
- VectorSearchService.SearchResult mapped = new VectorSearchService.SearchResult();
- Object id = row.getId();
- mapped.setId(id == null ? null : String.valueOf(id));
- Map entity = row.getEntity() == null ? Map.of() : row.getEntity();
- Object content = entity.get(FIELD_CONTENT);
- mapped.setContent(content == null ? null : String.valueOf(content));
- Object metadata = entity.get(FIELD_METADATA);
- if (metadata instanceof JsonObject jsonObject) {
- mapped.setMetadata(jsonObject.toString());
- } else if (metadata instanceof Map map) {
- mapped.setMetadata(GSON.toJson(map));
- } else if (metadata != null) {
- mapped.setMetadata(String.valueOf(metadata));
- }
- Float score = row.getScore();
- mapped.setRawScore(score == null ? null : score.doubleValue());
- mapped.setScoreLabel(scoreLabel);
- // dense: L2；hybrid: 引擎融合分（后处理 quality 主要看 rank，不依赖此量纲）
- mapped.setScore(score == null ? 0f : score);
- out.add(mapped);
- }
- return out;
- }
-
- /**
- * 组装标量过滤表达式：category、kb_scope（配置级）可叠加，用 {@code &&} 连接。
- */
- private String buildFilter(String category) {
- List parts = new ArrayList<>();
- String categoryFilter = trimToNull(category);
- if (categoryFilter != null) {
- parts.add("metadata[\"category\"] == \"" + escapeFilter(categoryFilter) + "\"");
- }
- String scope = trimToNull(kbScope);
- if (scope != null) {
- parts.add("metadata[\"kb_scope\"] == \"" + escapeFilter(scope) + "\"");
- }
- return parts.isEmpty() ? null : String.join(" && ", parts);
- }
-
- private MilvusClientV2 connect() {
- String uri;
- if (milvusProperties.isSecure() || milvusProperties.getPort() == 443) {
- uri = "https://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
- } else {
- uri = "http://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
- }
- ConnectConfig.ConnectConfigBuilder builder = ConnectConfig.builder()
- .uri(uri)
- .connectTimeoutMs(milvusProperties.getTimeout() == null ? 10000L : milvusProperties.getTimeout());
- if (milvusProperties.getToken() != null && !milvusProperties.getToken().isBlank()) {
- builder.token(milvusProperties.getToken());
- builder.secure(true);
- } else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isBlank()) {
- builder.username(milvusProperties.getUsername());
- builder.password(milvusProperties.getPassword());
- }
- if (milvusProperties.getDatabase() != null && !milvusProperties.getDatabase().isBlank()) {
- builder.dbName(milvusProperties.getDatabase());
- }
- log.info("Connecting MilvusClientV2 uri={} db={} collection={}",
- uri, milvusProperties.getDatabase(), collectionName);
- return new MilvusClientV2(builder.build());
- }
-
- /**
- * 若不存在则创建 dense+BM25 hybrid collection。
- *
- * 关键点：
- *
- * - {@code search_text} 开启 analyzer，作为 BM25 语料。
- * - {@link FunctionType#BM25}：input={@code search_text} → output={@code sparse_vector}。
- * - dense：IVF_FLAT + L2；sparse：SPARSE_INVERTED_INDEX + BM25。
- *
- * 已存在的 collection 不会改 schema；schema 变更需走 {@link #dropAndRecreateCollection()}。
- */
- private void ensureCollection(MilvusClientV2 milvusClient) {
- Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
- .collectionName(collectionName)
- .build());
- if (Boolean.TRUE.equals(exists)) {
- log.info("Hybrid collection '{}' already exists", collectionName);
- return;
- }
- log.info("Creating hybrid collection '{}'", collectionName);
-
- CreateCollectionReq.CollectionSchema schema = milvusClient.createSchema();
- schema.setEnableDynamicField(false);
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_ID)
- .dataType(DataType.VarChar)
- .maxLength(MilvusConstants.ID_MAX_LENGTH)
- .isPrimaryKey(true)
- .autoID(false)
- .build());
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_CONTENT)
- .dataType(DataType.VarChar)
- .maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
- .build());
- // BM25 语料字段：必须 enableAnalyzer，Function 才能从文本生成 sparse
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_SEARCH_TEXT)
- .dataType(DataType.VarChar)
- .maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
- .enableAnalyzer(true)
- .build());
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_SPARSE)
- .dataType(DataType.SparseFloatVector)
- .build());
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_DENSE)
- .dataType(DataType.FloatVector)
- .dimension(milvusProperties.getVectorDim())
- .build());
- schema.addField(AddFieldReq.builder()
- .fieldName(FIELD_METADATA)
- .dataType(DataType.JSON)
- .build());
- // 写入 search_text 时，Milvus 自动维护 sparse_vector（应用层 insert 不填 sparse）
- schema.addFunction(CreateCollectionReq.Function.builder()
- .functionType(FunctionType.BM25)
- .name("bm25_fn")
- .inputFieldNames(List.of(FIELD_SEARCH_TEXT))
- .outputFieldNames(List.of(FIELD_SPARSE))
- .build());
-
- milvusClient.createCollection(CreateCollectionReq.builder()
- .collectionName(collectionName)
- .collectionSchema(schema)
- .description("Knowledge hybrid dense+BM25 collection")
- .numShards(MilvusConstants.DEFAULT_SHARD_NUMBER)
- .build());
-
- List indexes = List.of(
- IndexParam.builder()
- .fieldName(FIELD_DENSE)
- .indexType(IndexParam.IndexType.IVF_FLAT)
- .metricType(IndexParam.MetricType.L2)
- .extraParams(Map.of("nlist", 128))
- .build(),
- IndexParam.builder()
- .fieldName(FIELD_SPARSE)
- .indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
- .metricType(IndexParam.MetricType.BM25)
- .build()
- );
- milvusClient.createIndex(CreateIndexReq.builder()
- .collectionName(collectionName)
- .indexParams(indexes)
- .build());
- log.info("Hybrid collection '{}' created with dense+BM25 indexes", collectionName);
- }
-
- private void loadCollection(MilvusClientV2 milvusClient) {
- milvusClient.loadCollection(LoadCollectionReq.builder()
- .collectionName(collectionName)
- .build());
- }
-
- @PreDestroy
- public void close() {
- if (client != null) {
- try {
- client.close();
- } catch (Exception e) {
- log.warn("Error closing MilvusClientV2: {}", e.getMessage());
- }
- }
- }
-
- /** 过滤表达式字符串转义，防止引号打断 expr。 */
- private static String escapeFilter(String value) {
- return value.replace("\\", "\\\\").replace("\"", "\\\"");
- }
-
- private static String trimToNull(String value) {
- if (value == null || value.isBlank()) {
- return null;
- }
- return value.trim();
- }
-
- private static String stringVal(Object value) {
- return value == null ? null : String.valueOf(value);
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/KnowledgeSearchPort.java b/src/main/java/com/superbiz/agent/service/retrieval/KnowledgeSearchPort.java
index b878891..efb6edf 100644
--- a/src/main/java/com/superbiz/agent/service/retrieval/KnowledgeSearchPort.java
+++ b/src/main/java/com/superbiz/agent/service/retrieval/KnowledgeSearchPort.java
@@ -5,9 +5,9 @@ import java.util.List;
/**
* 知识语义检索的应用边界端口。
*
- * 实现可对接 dense / hybrid 等引擎，但不得向上层泄漏 SDK 类型。
- * 当前实现：{@link VectorKnowledgeSearchAdapter} → {@code VectorSearchService}
- * → {@code MilvusHybridKnowledgeStore}（Milvus SDK v2 dense 或 dense+BM25 RRF）。
+ * 实现可对接 dense / hybrid 等引擎，但不得向上层泄漏远端 API 类型。
+ * 当前实现：{@link PyRagKnowledgeSearchAdapter}（py-rag 知识服务 /api/v1/search，
+ * 服务端负责 hybrid 融合、BM25、rerank 与判级）。RAG 模块抽离后为唯一实现。
*/
public interface KnowledgeSearchPort {
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/LexicalRanker.java b/src/main/java/com/superbiz/agent/service/retrieval/LexicalRanker.java
deleted file mode 100644
index 154c9e8..0000000
--- a/src/main/java/com/superbiz/agent/service/retrieval/LexicalRanker.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package com.superbiz.agent.service.retrieval;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-
-/**
- * Sparse-lite lexical ranking over already recalled candidates.
- * Not a substitute for inverted-index BM25; expands ordering signal only.
- */
-public final class LexicalRanker {
-
- private LexicalRanker() {
- }
-
- public static List rank(String query, List candidates) {
- if (candidates == null || candidates.isEmpty()) {
- return List.of();
- }
- Set terms = tokenize(query);
- if (terms.isEmpty()) {
- return List.copyOf(candidates);
- }
- List scored = new ArrayList<>(candidates.size());
- for (KnowledgeSearchHit hit : candidates) {
- String haystack = (nullToEmpty(hit.title()) + " "
- + nullToEmpty(hit.breadcrumb()) + " "
- + nullToEmpty(hit.content())).toLowerCase(Locale.ROOT);
- int hits = 0;
- for (String term : terms) {
- if (haystack.contains(term)) {
- hits++;
- }
- }
- double coverage = hits / (double) terms.size();
- scored.add(new ScoredHit(hit, coverage, hits));
- }
- scored.sort(Comparator
- .comparingDouble((ScoredHit s) -> s.coverage).reversed()
- .thenComparingInt((ScoredHit s) -> s.hits).reversed()
- .thenComparingInt(s -> s.hit.originalRank()));
- return scored.stream().map(s -> s.hit).toList();
- }
-
- static Set tokenize(String query) {
- if (query == null || query.isBlank()) {
- return Set.of();
- }
- String normalized = query.toLowerCase(Locale.ROOT);
- String[] parts = normalized.split("[^\\p{IsAlphabetic}\\p{IsDigit}]+");
- Set terms = new LinkedHashSet<>();
- for (String part : parts) {
- if (part == null) {
- continue;
- }
- String term = part.trim();
- if (term.length() >= 2) {
- terms.add(term);
- }
- }
- return terms;
- }
-
- private static String nullToEmpty(String value) {
- return value == null ? "" : value;
- }
-
- private record ScoredHit(KnowledgeSearchHit hit, double coverage, int hits) {
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/PyRagKnowledgeSearchAdapter.java b/src/main/java/com/superbiz/agent/service/retrieval/PyRagKnowledgeSearchAdapter.java
new file mode 100644
index 0000000..4012db9
--- /dev/null
+++ b/src/main/java/com/superbiz/agent/service/retrieval/PyRagKnowledgeSearchAdapter.java
@@ -0,0 +1,118 @@
+package com.superbiz.agent.service.retrieval;
+
+import com.superbiz.agent.client.PyRagClient;
+import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
+import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
+import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * {@link KnowledgeSearchPort} 的 py-rag 远端实现（RAG 模块抽离后的唯一检索后端）。
+ *
+ * 原进程内链路（VectorKnowledgeSearchAdapter → VectorSearchService → MilvusHybridKnowledgeStore）
+ * 已由 py-rag 服务端接管：hybrid 融合、BM25、rerank、chunk 去重、判级阈值全部下沉。
+ * Java 侧只做请求映射与命中结构归一化，不碰检索算法。
+ *
+ * 映射约定
+ *
+ * - mode：{@link KnowledgeSearchMode#DENSE} → {@code semantic}，{@link KnowledgeSearchMode#HYBRID} → {@code hybrid}
+ * - retrieve_k = return_n = topK：返回 topK 条精排后命中，chunk 去重/截断仍由
+ * {@code KnowledgeEvidencePostProcessor} 统一负责，故 max_chunks_per_document 同步放大避免服务端预截断
+ * - category：{@code categoryFilter} 透传；null = 不过滤；kb_scope 不传，由服务端部署配置决定
+ * - score：py-rag rerank 绝对相关分（[0,1]，越大越好），scoreLabel =
+ * {@link RetrievalScoreLabels#RERANK}（quality 原样采用，不做 L2/rank 归一化）
+ * - evidence_key：{@code docId#chunk-N}，与 EvidenceGuard 验真约定一致；
+ * {@code evidence_status=no_evidence} 时服务端保证 hits=[]，按"无知识"正常返回
+ *
+ */
+@Component
+public class PyRagKnowledgeSearchAdapter implements KnowledgeSearchPort {
+
+ private static final String CHUNK_MARK = "#chunk-";
+
+ private final PyRagClient pyRagClient;
+
+ public PyRagKnowledgeSearchAdapter(PyRagClient pyRagClient) {
+ this.pyRagClient = pyRagClient;
+ }
+
+ @Override
+ public List search(KnowledgeSearchRequest request) {
+ PyRagSearchResponse response = pyRagClient.search(toPyRagRequest(request));
+ if (response == null || response.hits() == null || response.hits().isEmpty()) {
+ return List.of();
+ }
+ List hits = new ArrayList<>(response.hits().size());
+ for (int i = 0; i < response.hits().size(); i++) {
+ hits.add(toHit(response.hits().get(i), i + 1));
+ }
+ return hits;
+ }
+
+ /** 请求映射：topK 同时作为召回宽度与返回条数，服务端不预截断 chunk。 */
+ private PyRagSearchRequest toPyRagRequest(KnowledgeSearchRequest request) {
+ return new PyRagSearchRequest(
+ request.query(),
+ request.mode() == KnowledgeSearchMode.HYBRID ? "hybrid" : "semantic",
+ request.topK(),
+ request.topK(),
+ request.topK(),
+ blankToNull(request.categoryFilter()),
+ null);
+ }
+
+ private KnowledgeSearchHit toHit(PyRagSearchHit hit, int originalRank) {
+ String docId = blankToNull(hit.documentId());
+ Integer chunkIndex = parseChunkIndex(hit.evidenceKey());
+ String evidenceKey = EvidenceIdentity.firstNonBlank(
+ hit.evidenceKey(),
+ EvidenceIdentity.evidenceKey(docId, chunkIndex, null, originalRank));
+ Double score = hit.qualityScore() == null ? 0.0 : hit.qualityScore();
+ return new KnowledgeSearchHit(
+ firstNonBlank(evidenceKey, docId, "rank:" + originalRank),
+ hit.excerpt(),
+ score,
+ score,
+ RetrievalScoreLabels.RERANK,
+ null,
+ Map.of(),
+ docId,
+ chunkIndex,
+ evidenceKey,
+ hit.source(),
+ hit.title(),
+ hit.breadcrumb(),
+ originalRank,
+ null);
+ }
+
+ /** evidence_key 形如 {@code docId#chunk-N}，解析末尾 chunk 序号；不符返回 null。 */
+ private Integer parseChunkIndex(String evidenceKey) {
+ String key = blankToNull(evidenceKey);
+ if (key == null) {
+ return null;
+ }
+ int mark = key.lastIndexOf(CHUNK_MARK);
+ if (mark < 0) {
+ return null;
+ }
+ try {
+ return Integer.valueOf(key.substring(mark + CHUNK_MARK.length()).trim());
+ } catch (NumberFormatException ignored) {
+ return null;
+ }
+ }
+
+ private String blankToNull(String value) {
+ return value == null || value.isBlank() ? null : value.trim();
+ }
+
+ private String firstNonBlank(String... values) {
+ return EvidenceIdentity.firstNonBlank(values);
+ }
+}
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreLabels.java b/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreLabels.java
index ee9bf7e..dab9ffc 100644
--- a/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreLabels.java
+++ b/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreLabels.java
@@ -3,8 +3,13 @@ package com.superbiz.agent.service.retrieval;
/**
* 检索结果一级 {@code scoreLabel} 约定。
*
- * 只区分两种检索形态（与 {@code retrieval.search.mode} 对齐），
- * 不再使用 {@code bm25_only_*} 等作为正式一级 label。
+ * 三种检索形态：
+ *
+ * - {@link #DENSE} —— 单路向量 ANN（L2 距离，越小越好）
+ * - {@link #HYBRID} —— dense + BM25 + RRF 融合（质量主要看 rank）
+ * - {@link #RERANK} —— py-rag 服务端 rerank 绝对相关分（[0,1]，越大越好）；
+ * RAG 模块抽离后的线上主路径
+ *
*/
public final class RetrievalScoreLabels {
@@ -14,11 +19,14 @@ public final class RetrievalScoreLabels {
/** hybrid（dense+BM25+RRF）：{@code score}/raw 为融合侧信号；质量分主要看 rank。 */
public static final String HYBRID = "hybrid";
+ /** py-rag 服务端 rerank 绝对分：{@code score} 即归一化质量（[0,1]，越大越好）。 */
+ public static final String RERANK = "rerank";
+
private RetrievalScoreLabels() {
}
/**
- * 将历史/别名 label 归一到 {@link #DENSE} 或 {@link #HYBRID}。
+ * 将历史/别名 label 归一到 {@link #DENSE}、{@link #HYBRID} 或 {@link #RERANK}。
* 未知或空 → dense（保守，按 L2 解释失败时 quality 偏低）。
*/
public static String canonicalize(String scoreLabel) {
@@ -29,12 +37,19 @@ public final class RetrievalScoreLabels {
return switch (label) {
case DENSE, "l2_distance", "l2" -> DENSE;
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
- default -> label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
- ? HYBRID
- : DENSE;
+ case RERANK, "rerank_score", "quality_score" -> RERANK;
+ default -> label.contains("rerank") || label.contains("quality")
+ ? RERANK
+ : label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
+ ? HYBRID
+ : DENSE;
};
}
+ public static boolean isRerank(String scoreLabel) {
+ return RERANK.equals(canonicalize(scoreLabel));
+ }
+
public static boolean isHybrid(String scoreLabel) {
return HYBRID.equals(canonicalize(scoreLabel));
}
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreNormalizer.java b/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreNormalizer.java
index 0e68d4a..77b686a 100644
--- a/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreNormalizer.java
+++ b/src/main/java/com/superbiz/agent/service/retrieval/RetrievalScoreNormalizer.java
@@ -6,6 +6,8 @@ package com.superbiz.agent.service.retrieval;
* 后处理排序仍按 {@code originalRank}；本类只负责质量闸门 / relevance 用分。
*
*
+ * - {@link RetrievalScoreLabels#RERANK}：py-rag 服务端 rerank 绝对分，
+ * {@code score} 已归一化，原样 clamp 到 [0,1]（RAG 抽离后的主路径）
* - {@link RetrievalScoreLabels#DENSE}：{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}
* - {@link RetrievalScoreLabels#HYBRID}：优先用可选 {@code denseDistance} 做绝对质量
* （恢复 L0 filter low-quality 等闸门）；无 dense 时回退 rank 映射
@@ -17,8 +19,8 @@ public final class RetrievalScoreNormalizer {
}
/**
- * @param scoreLabel {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
- * @param score 引擎主分：dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
+ * @param scoreLabel {@link RetrievalScoreLabels#RERANK} / {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
+ * @param score 引擎主分：rerank=绝对相关分[0,1]；dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
* @param originalRank 检索名次（1-based）
* @param batchSize 本轮候选数（rank 回退映射用）
* @param maxL2Distance L2 上界
@@ -31,6 +33,12 @@ public final class RetrievalScoreNormalizer {
double maxL2Distance,
Double denseDistance) {
String label = RetrievalScoreLabels.canonicalize(scoreLabel);
+ if (RetrievalScoreLabels.RERANK.equals(label)) {
+ if (score == null) {
+ return 0.0;
+ }
+ return Math.max(0.0, Math.min(1.0, score));
+ }
if (RetrievalScoreLabels.HYBRID.equals(label)) {
if (denseDistance != null) {
return l2ToQuality(denseDistance, maxL2Distance);
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/RrfFusion.java b/src/main/java/com/superbiz/agent/service/retrieval/RrfFusion.java
deleted file mode 100644
index f14e283..0000000
--- a/src/main/java/com/superbiz/agent/service/retrieval/RrfFusion.java
+++ /dev/null
@@ -1,100 +0,0 @@
-package com.superbiz.agent.service.retrieval;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.Function;
-
-/**
- * Reciprocal Rank Fusion 工具：把多路检索的排名列表融合成一个分数排序。
- *
- *
- * RRF_w(d) = Σ w_i / (k + rank_i(d))
- *
- *
- * 只依赖排名不依赖原始分数——屏蔽跨路分数尺度不可比的问题；
- * 每路可加权（w <= 0 时按 1.0 等权），k 是平滑参数（默认 60，可配）。
- */
-public final class RrfFusion {
-
- private RrfFusion() {
- }
-
- /**
- * 融合多路排名：对每路的每个 item 累加 w/(k+rank)，按总分降序输出。
- *
- * @param paths 多路排名（每路带 name / items / weight）
- * @param rrfK 平滑参数 k（至少 1）
- * @param identityFn 跨路识别同一 item 的身份函数（如 evidenceKey）
- * @return 融合后排序（含每路排名明细）
- */
- public static List> fuse(List> paths,
- int rrfK,
- Function identityFn) {
- if (paths == null || paths.isEmpty()) {
- return List.of();
- }
- int k = Math.max(1, rrfK);
- Map> acc = new LinkedHashMap<>();
- for (RankedPath path : paths) {
- if (path == null || path.items() == null || path.items().isEmpty()) {
- continue;
- }
- double weight = path.weight() <= 0 ? 1.0 : path.weight();
- List items = path.items();
- for (int i = 0; i < items.size(); i++) {
- T item = items.get(i);
- if (item == null) {
- continue;
- }
- String id = identityFn.apply(item);
- if (id == null || id.isBlank()) {
- continue;
- }
- int rank = i + 1;
- double contrib = weight / (k + rank); // 排名越前贡献越大
- Acc bucket = acc.computeIfAbsent(id, ignored -> new Acc<>(item));
- bucket.score += contrib;
- bucket.ranks.put(path.name(), rank);
- // Prefer first-seen item payload; callers should put preferred path first if needed.
- }
- }
- List> scored = new ArrayList<>(acc.size());
- for (Map.Entry> entry : acc.entrySet()) {
- Acc value = entry.getValue();
- scored.add(new Scored<>(entry.getKey(), value.item, value.score, Map.copyOf(value.ranks)));
- }
- // 总分降序（两路共识的靠前），同分按身份稳定排序
- scored.sort(Comparator
- .comparingDouble((Scored s) -> s.rrfScore()).reversed()
- .thenComparing(Scored::identity));
- return scored;
- }
-
- /** 一路检索结果：name（路名）+ items（按排名顺序）+ weight（可选加权，≤0 视为等权）。 */
- public record RankedPath(String name, List items, double weight) {
- public RankedPath {
- Objects.requireNonNull(name, "name");
- items = items == null ? List.of() : List.copyOf(items);
- }
- }
-
- /** 融合后的单个 item：identity + 原始 item + rrfScore + 每路排名明细。 */
- public record Scored(String identity, T item, double rrfScore, Map ranks) {
- }
-
- /** 跨路累加器：同一 identity 的 item 累加 RRF 分并记录各路排名。 */
- private static final class Acc {
- private final T item;
- private double score;
- private final Map ranks = new HashMap<>();
-
- private Acc(T item) {
- this.item = item;
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/service/retrieval/VectorKnowledgeSearchAdapter.java b/src/main/java/com/superbiz/agent/service/retrieval/VectorKnowledgeSearchAdapter.java
deleted file mode 100644
index 1dc60dc..0000000
--- a/src/main/java/com/superbiz/agent/service/retrieval/VectorKnowledgeSearchAdapter.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package com.superbiz.agent.service.retrieval;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.superbiz.agent.service.VectorSearchService;
-import org.springframework.stereotype.Component;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * {@link KnowledgeSearchPort} 适配器：把向量检索结果映射为带 evidenceKey 的命中结构。
- *
- * 委托 {@link VectorSearchService}（背后仅 {@code MilvusHybridKnowledgeStore}）：
- * dense 或 dense+BM25 hybrid 由配置 {@code retrieval.search.mode} 选择。
- * 本类负责 metadata 解析、docId/chunk 身份与 evidenceKey，不碰 SDK。
- */
-@Component
-public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
-
- private final VectorSearchService vectorSearchService;
- private final ObjectMapper objectMapper;
-
- public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
- this.vectorSearchService = vectorSearchService;
- this.objectMapper = objectMapper;
- }
-
- @Override
- public List search(KnowledgeSearchRequest request) {
- // Mode is global on VectorSearchService; request.mode is advisory for future per-call overrides.
- List results = vectorSearchService.searchSimilarDocuments(
- request.query(),
- request.topK(),
- request.categoryFilter());
- return toHits(results);
- }
-
- private List toHits(List results) {
- if (results == null || results.isEmpty()) {
- return List.of();
- }
- List hits = new ArrayList<>(results.size());
- for (int i = 0; i < results.size(); i++) {
- hits.add(toHit(results.get(i), i + 1));
- }
- return hits;
- }
-
- private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
- Map metadata = parseMetadata(result.getMetadata());
- String docId = EvidenceIdentity.extractDocId(
- metadata,
- EvidenceIdentity.metadataValue(metadata, "_source"),
- EvidenceIdentity.metadataValue(metadata, "source"));
- Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
- String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
- String source = EvidenceIdentity.firstNonBlank(
- EvidenceIdentity.metadataValue(metadata, "_source"),
- EvidenceIdentity.metadataValue(metadata, "source"),
- EvidenceIdentity.metadataValue(metadata, "filePath"),
- docId,
- result.getId());
- return new KnowledgeSearchHit(
- result.getId(),
- result.getContent(),
- (double) result.getScore(),
- result.getRawScore(),
- result.getScoreLabel(),
- result.getMetadata(),
- metadata,
- docId,
- chunkIndex,
- evidenceKey,
- source,
- EvidenceIdentity.metadataValue(metadata, "title"),
- EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
- originalRank,
- result.getDenseDistance()
- );
- }
-
- private Map parseMetadata(String metadata) {
- if (metadata == null || metadata.isBlank()) {
- return Map.of();
- }
- try {
- Map raw = objectMapper.readValue(metadata, Map.class);
- Map parsed = new LinkedHashMap<>();
- for (Map.Entry entry : raw.entrySet()) {
- if (entry.getKey() != null && entry.getValue() != null) {
- parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
- }
- }
- return parsed;
- } catch (Exception ignored) {
- return Map.of();
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/tool/DropCollection.java b/src/main/java/com/superbiz/agent/tool/DropCollection.java
deleted file mode 100644
index 523a4c8..0000000
--- a/src/main/java/com/superbiz/agent/tool/DropCollection.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package com.superbiz.agent.tool;
-
-import io.milvus.client.MilvusServiceClient;
-import io.milvus.param.ConnectParam;
-import io.milvus.param.R;
-import io.milvus.param.RpcStatus;
-import io.milvus.param.collection.DropCollectionParam;
-import io.milvus.param.collection.HasCollectionParam;
-
-/**
- * 删除 Milvus Collection 的工具类
- * 用于重建 Collection 时清理旧数据
- */
-public class DropCollection {
-
- public static void main(String[] args) {
- MilvusServiceClient client = null;
-
- try {
- // 连接到 Milvus
- System.out.println("正在连接到 Milvus localhost:19530...");
- client = new MilvusServiceClient(
- ConnectParam.newBuilder()
- .withHost("localhost")
- .withPort(19530)
- .build()
- );
- System.out.println("✓ 连接成功");
-
- String collectionName = "biz";
-
- // 检查 Collection 是否存在
- R hasResponse = client.hasCollection(
- HasCollectionParam.newBuilder()
- .withCollectionName(collectionName)
- .build()
- );
-
- if (hasResponse.getData()) {
- System.out.println("发现 Collection: " + collectionName);
- System.out.println("正在删除...");
-
- // 删除 Collection
- R dropResponse = client.dropCollection(
- DropCollectionParam.newBuilder()
- .withCollectionName(collectionName)
- .build()
- );
-
- if (dropResponse.getStatus() == 0) {
- System.out.println("✓ Collection 已成功删除");
- System.out.println("\n请重启 Spring Boot 应用，它会自动创建新的 FloatVector Collection");
- } else {
- System.err.println("✗ 删除失败: " + dropResponse.getMessage());
- }
- } else {
- System.out.println("Collection '" + collectionName + "' 不存在");
- }
-
- } catch (Exception e) {
- System.err.println("错误: " + e.getMessage());
- e.printStackTrace();
- } finally {
- if (client != null) {
- client.close();
- }
- }
- }
-}
diff --git a/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java b/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java
index 414142b..bc066fb 100644
--- a/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java
+++ b/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java
@@ -8,7 +8,6 @@ import com.superbiz.agent.dto.RetrievalTrace;
import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
-import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
@@ -29,10 +28,9 @@ import java.util.Map;
*
* 主链路
*
- * query
- * -> KnowledgeQueryTransformer
- * -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k)
- * -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
+ * query（原始句直传；L0 query 理解已下沉 py-rag 服务端）
+ * -> KnowledgeDocumentRetriever (via KnowledgeSearchPort → py-rag, retrieve-k)
+ * -> KnowledgeEvidencePostProcessor (qualityScore / chunk dedup / caps / return-n)
* -> [optional] unfiltered retry
* -> KnowledgeContextPacker
* -> LookupResultAssembler
@@ -62,9 +60,6 @@ public class LookupKnowledgeTool {
private int retrieveK = 20;
- @Autowired
- private KnowledgeQueryTransformer queryTransformer;
-
@Autowired
private KnowledgeDocumentRetriever documentRetriever;
@@ -98,10 +93,10 @@ public class LookupKnowledgeTool {
*
* 流程（模块化三段）：
*
- * - 检索前：QueryTransformer.transform → KnowledgeQuery（分类过滤/域/关键词）；
- * - 检索：DocumentRetriever.retrieve（FILTERED 或 UNFILTERED，retrieveK 候选）；
+ * - 检索前：原始 query 直传（L0 domain 分析已下沉 py-rag，categoryFilter 恒为 null）；
+ * - 检索：DocumentRetriever.retrieve（UNFILTERED，retrieveK 候选，py-rag 服务端融合+精排）；
* - 检索后：PostProcessor.process（qualityScore/去重/判级）；
- * - 低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查；
+ * - 低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查（L0 移除后保留兜底语义）；
* - 打包 + 组装：ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。
*
*
@@ -113,12 +108,17 @@ public class LookupKnowledgeTool {
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
log.info("----------------------------------------");
- // ── 检索前：查询理解（L0）──
- KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
- log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
- knowledgeQuery.getCategoryFilter(),
- knowledgeQuery.getDomainHints().size(),
- knowledgeQuery.getMatchedKeywords().size());
+ // ── 检索前：原始 query 直传（L0 已下沉 py-rag，不做 Java 侧 category 收窄）──
+ String normalized = query == null ? "" : query.trim();
+ KnowledgeQuery knowledgeQuery = KnowledgeQuery.builder()
+ .originalQuery(normalized)
+ .rewrittenQuery(normalized)
+ .domainHints(List.of())
+ .matchedKeywords(List.of())
+ .entities(List.of())
+ .l0Titles(List.of())
+ .l0MatchCount(0)
+ .build();
List attempts = new ArrayList<>();
String fallbackReason = null;
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 7a4d177..9d0e365 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -15,30 +15,28 @@ file:
knowledge:
base-path: knowledge_base/
-milvus:
- host: in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com
- port: 443
- username: ""
- password: ""
- database: db_4a578da0f27ce9d
- timeout: 10000
- token: ${MILVUS_TOKEN}
- secure: true
- vector-dim: 1024 # BGE-M3 = 1024，换模型时同步改
- # knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
- collection: biz
+# =====================================================
+# py-rag 知识服务接入
+# =====================================================
+# RAG 检索与文档入库均由 py-rag 服务承担（契约见 py-rag 仓库 docs/Java接入文档.md）：
+# 检索 /api/v1/search，入库 /api/v1/documents:ingest，全量重建 /api/v1/collections:rebuild。
+pyrag:
+ base-url: ${PYRAG_BASE_URL:http://localhost:8000}
+ connect-timeout-ms: 3000
+ search-read-timeout-ms: 5000 # 正常 300–800ms（含 rerank 外呼）
+ ingest-read-timeout-ms: 30000 # 正常 1–5s
+ default-read-timeout-ms: 10000
# =====================================================
# 模型路由配置
# =====================================================
# 通过关键字匹配 Bean，切换模型只改这里 + 对应 api-key
-# Chat: deepseek | openai | ollama | ...
-# Embedding: siliconflow | openai | ollama | dashscope | ...
+# Chat: deepseek | openai | ollama | ...
+# （Embedding 已随 RAG 抽离至 py-rag 服务端）
# =====================================================
model-routing:
chat: deepseek
- embedding: siliconflow
spring:
config:
@@ -102,30 +100,6 @@ spring:
retry:
max-attempts: 1
- vectorstore:
- type: milvus
- milvus:
- initialize-schema: false
- database-name: ${milvus.database}
- collection-name: biz
- embedding-dimension: ${milvus.vector-dim}
- index-type: IVF_FLAT
- metric-type: L2
- index-parameters: '{"nlist":128}'
- id-field-name: id
- auto-id: false
- content-field-name: content
- metadata-field-name: metadata
- embedding-field-name: vector
- client:
- host: ${milvus.host}
- port: ${milvus.port}
- token: ${milvus.token}
- username: ${milvus.username}
- password: ${milvus.password}
- secure: ${milvus.secure}
- connect-timeout-ms: ${milvus.timeout}
-
# --- Chat: DeepSeek (原生) ---
deepseek:
api-key: ${DEEPSEEK_API_KEY}
@@ -134,53 +108,27 @@ spring:
options:
model: deepseek-v4-flash
- # --- OpenAI 模块供 SiliconFlow Embedding 复用 ---
- openai:
- api-key: unused
-
# Spring AI MCP 客户端配置
mcp:
client:
enabled: false
-# --- Embedding: SiliconFlow BGE-M3 ---
-siliconflow:
- api-key: ${SILICONFLOW_API_KEY}
- base-url: https://api.siliconflow.cn
- embedding:
- model: BAAI/bge-m3
-
-# 文档分片配置
-document:
- chunk:
- max-size: 800
- overlap: 100
-
# RAG 配置
rag:
top-k: 3 # legacy fallback when retrieve-k/return-n absent
retrieve-k: 20
return-n: 5
max-chunks-per-document: 2
- sidecar:
- spring-ai:
- enabled: false
- content-preview-limit: 300
# 检索配置
-# 知识主路径：Milvus Java SDK v2（MilvusHybridKnowledgeStore），非 Spring AI VectorStore starter。
-# 原因：starter（含 2.0.0）仅 dense similarity，无 hybridSearch / BM25 Function / RRFRanker。
-# 已移除 legacy sdk/spring/auto 多后端路由。
+# 知识主路径：py-rag 知识服务（PyRagKnowledgeSearchAdapter → /api/v1/search）。
+# 服务端负责 dense+BM25 融合、rerank（BGE-Reranker）与判级；Java 侧只做请求映射与后处理。
retrieval:
- kb-scope: "" # 非空则过滤 metadata.kb_scope；空=不过滤
search:
- # hybrid=线上主路径；dense=同库对照/评测/排障（非第二套线上策略）。见 mvp/architecture/rag-knowledge-retrieval-architecture.md §6.0
- mode: hybrid # dense=单路L2对照 | hybrid=dense+服务端BM25+RRF
- hybrid:
- rrf-k: 60 # RRF 平滑参数 k，score=Σ 1/(k+rank)
+ # hybrid=线上主路径；dense 为对照/排障（映射 py-rag mode：hybrid→hybrid，dense→semantic）
+ mode: hybrid
normalization:
- max-l2-distance: 2.0 # dense quality：L2 上界（单位向量 ≈ 2.0）
- highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（hybrid 为序数分，见架构 §6）
+ highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（与 py-rag 判级阈值一致）
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE；低于则低质/可 unfiltered retry
# Prometheus 配置
diff --git a/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java b/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java
index 34b9e89..ddd3825 100644
--- a/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java
+++ b/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java
@@ -11,7 +11,6 @@ import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.service.KnowledgeContextPacker;
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
-import com.superbiz.agent.service.KnowledgeQueryTransformer;
import com.superbiz.agent.service.LookupResultAssembler;
import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
@@ -84,7 +83,6 @@ class HarnessChatConfigurationTest {
.withBean(ChatModel.class, () -> mock(ChatModel.class))
.withBean(RedisTemplate.class, () -> mock(RedisTemplate.class))
.withBean(LookupKnowledgeTool.class, () -> mock(LookupKnowledgeTool.class))
- .withBean(KnowledgeQueryTransformer.class, () -> mock(KnowledgeQueryTransformer.class))
.withBean(KnowledgeDocumentRetriever.class, () -> mock(KnowledgeDocumentRetriever.class))
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
diff --git a/src/test/java/com/superbiz/agent/config/RedisConnectionTest.java b/src/test/java/com/superbiz/agent/config/RedisConnectionTest.java
index a2d97b9..5af4b09 100644
--- a/src/test/java/com/superbiz/agent/config/RedisConnectionTest.java
+++ b/src/test/java/com/superbiz/agent/config/RedisConnectionTest.java
@@ -4,18 +4,13 @@ import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.redis.core.RedisTemplate;
-import org.springframework.test.context.TestPropertySource;
import static org.junit.jupiter.api.Assertions.*;
/**
- * 单独测试 Redis 连接
- * 禁用 Milvus 以避免启动失败
+ * 单独测试 Redis 连接（需要可达的 Redis 实例）。
*/
@SpringBootTest
-@TestPropertySource(properties = {
- "spring.autoconfigure.exclude=org.example.config.MilvusConfig"
-})
class RedisConnectionTest {
@Autowired(required = false)
diff --git a/src/test/java/com/superbiz/agent/eval/RagEvalSeedImporterTest.java b/src/test/java/com/superbiz/agent/eval/RagEvalSeedImporterTest.java
deleted file mode 100644
index 6f9e0e4..0000000
--- a/src/test/java/com/superbiz/agent/eval/RagEvalSeedImporterTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-package com.superbiz.agent.eval;
-
-import com.superbiz.agent.Main;
-import com.superbiz.agent.domain.entity.ApiDocument;
-import com.superbiz.agent.dto.DocumentUploadRequest;
-import com.superbiz.agent.dto.Frontmatter;
-import com.superbiz.agent.repository.ApiDocumentRepository;
-import com.superbiz.agent.service.DocumentManagementService;
-import com.superbiz.agent.service.FrontmatterParser;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.mock.web.MockMultipartFile;
-
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.assertFalse;
-
-/**
- * Imports canonical RAG eval documents through the real document pipeline.
- *
- * Disabled by default because it writes DB rows, local knowledge files, and
- * vector index records in the configured runtime environment.
- */
-@SpringBootTest(
- classes = Main.class,
- webEnvironment = SpringBootTest.WebEnvironment.NONE,
- properties = "spring.main.web-application-type=none"
-)
-@EnabledIfSystemProperty(named = "rag.seed.enabled", matches = "true")
-class RagEvalSeedImporterTest {
-
- private static final Path DEFAULT_SEED_DOCS = Path.of("eval/rag-retrieval/seed-docs");
-
- @Autowired
- private DocumentManagementService documentManagementService;
-
- @Autowired
- private FrontmatterParser frontmatterParser;
-
- @Autowired
- private ApiDocumentRepository apiDocumentRepository;
-
- @Test
- void importSeedDocuments() throws Exception {
- Path seedDir = Path.of(System.getProperty("rag.seed.docs", DEFAULT_SEED_DOCS.toString()));
- List docs;
- try (var stream = Files.list(seedDir)) {
- docs = stream
- .filter(path -> path.getFileName().toString().endsWith(".md"))
- .sorted()
- .toList();
- }
- assertFalse(docs.isEmpty(), "seed docs directory must contain markdown files");
-
- for (Path docPath : docs) {
- String content = Files.readString(docPath, StandardCharsets.UTF_8);
- Frontmatter frontmatter = frontmatterParser.parse(content);
- if (frontmatter == null || frontmatter.getSource() == null || frontmatter.getSource().isBlank()) {
- throw new IllegalArgumentException("seed doc must include frontmatter source: " + docPath);
- }
-
- apiDocumentRepository.findByDocId(frontmatter.getSource().trim())
- .map(ApiDocument::getDocId)
- .ifPresent(documentManagementService::deleteDocument);
-
- String fileName = docPath.getFileName().toString();
- MockMultipartFile file = new MockMultipartFile(
- "file",
- fileName,
- "text/markdown",
- content.getBytes(StandardCharsets.UTF_8)
- );
- DocumentUploadRequest request = DocumentUploadRequest.builder()
- .file(file)
- .category(resolveCategory(frontmatter))
- .build();
-
- documentManagementService.uploadDocument(request);
- }
- }
-
- private String resolveCategory(Frontmatter frontmatter) {
- if (frontmatter.getCategory() != null && !frontmatter.getCategory().isBlank()) {
- return frontmatter.getCategory().trim();
- }
- return "rag-eval";
- }
-}
diff --git a/src/test/java/com/superbiz/agent/eval/RagLookupSnapshotGeneratorTest.java b/src/test/java/com/superbiz/agent/eval/RagLookupSnapshotGeneratorTest.java
deleted file mode 100644
index 011a47a..0000000
--- a/src/test/java/com/superbiz/agent/eval/RagLookupSnapshotGeneratorTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-package com.superbiz.agent.eval;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.superbiz.agent.Main;
-import com.superbiz.agent.dto.LookupResult;
-import com.superbiz.agent.tool.LookupKnowledgeTool;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.test.context.DynamicPropertyRegistry;
-import org.springframework.test.context.DynamicPropertySource;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Instant;
-import java.util.Locale;
-
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-/**
- * Generates RAG retrieval fixtures from the real {@link LookupKnowledgeTool} bean.
- *
- * Disabled by default: writes repository files and needs the live retrieval stack
- * (embedding + Milvus hybrid collection + optional MySQL/L0).
- *
- * System properties (via Maven {@code -D}):
- *
- * - {@code rag.snapshot.enabled=true} — required to run
- * - {@code retrieval.search.mode=hybrid|dense} — default hybrid
- * - {@code retrieval.kb-scope} — default empty unless set (scripts use {@code rag-eval})
- * - {@code rag.snapshot.cases} / {@code rag.snapshot.fixtures} / {@code rag.snapshot.retrievedAt}
- *
- */
-@SpringBootTest(
- classes = Main.class,
- webEnvironment = SpringBootTest.WebEnvironment.NONE,
- properties = "spring.main.web-application-type=none"
-)
-@EnabledIfSystemProperty(named = "rag.snapshot.enabled", matches = "true")
-class RagLookupSnapshotGeneratorTest {
-
- private static final Path DEFAULT_CASES = Path.of("eval/rag-retrieval/cases/golden-cases.json");
- private static final Path DEFAULT_FIXTURES = Path.of("eval/rag-retrieval/fixtures");
-
- @Autowired
- private LookupKnowledgeTool lookupKnowledgeTool;
-
- @Autowired
- private ObjectMapper objectMapper;
-
- /**
- * Bind retrieval mode/scope early so {@code VectorSearchService} / store filters see them.
- */
- @DynamicPropertySource
- static void retrievalProperties(DynamicPropertyRegistry registry) {
- String mode = System.getProperty("retrieval.search.mode", "hybrid");
- if (mode == null || mode.isBlank()) {
- mode = "hybrid";
- }
- String normalized = mode.trim().toLowerCase(Locale.ROOT);
- registry.add("retrieval.search.mode", () -> normalized);
-
- String kbScope = System.getProperty("retrieval.kb-scope", "");
- if (kbScope != null && !kbScope.isBlank()) {
- registry.add("retrieval.kb-scope", kbScope::trim);
- }
- }
-
- @Test
- void generateLookupResultFixtures() throws Exception {
- Path casesPath = Path.of(System.getProperty("rag.snapshot.cases", DEFAULT_CASES.toString()));
- Path fixturesDir = Path.of(System.getProperty("rag.snapshot.fixtures", DEFAULT_FIXTURES.toString()));
- String retrievedAt = System.getProperty("rag.snapshot.retrievedAt", Instant.now().toString());
- String searchMode = normalizeMode(System.getProperty("retrieval.search.mode", "hybrid"));
- String kbScope = blankToNull(System.getProperty("retrieval.kb-scope", ""));
-
- JsonNode root = objectMapper.readTree(casesPath.toFile());
- JsonNode cases = root.path("cases");
- assertTrue(cases.isArray(), "golden cases file must contain a cases array");
-
- Files.createDirectories(fixturesDir);
- for (JsonNode testCase : cases) {
- String caseId = requiredText(testCase, "caseId");
- String query = requiredText(testCase, "query");
-
- LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
-
- ObjectNode fixture = objectMapper.createObjectNode();
- fixture.put("caseId", caseId);
- fixture.put("query", query);
- fixture.put("retrievedAt", retrievedAt);
- fixture.put("searchMode", searchMode);
- if (kbScope != null) {
- fixture.put("kbScope", kbScope);
- }
- fixture.set("lookupResult", objectMapper.valueToTree(lookupResult));
-
- Path output = fixturesDir.resolve(caseId + ".json");
- objectMapper.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), fixture);
- }
- }
-
- private static String normalizeMode(String mode) {
- if (mode == null || mode.isBlank()) {
- return "hybrid";
- }
- return mode.trim().toLowerCase(Locale.ROOT);
- }
-
- private static String blankToNull(String value) {
- if (value == null || value.isBlank()) {
- return null;
- }
- return value.trim();
- }
-
- private String requiredText(JsonNode node, String fieldName) {
- JsonNode value = node.get(fieldName);
- if (value == null || value.asText().isBlank()) {
- throw new IllegalArgumentException("golden case is missing required field: " + fieldName);
- }
- return value.asText();
- }
-}
diff --git a/src/test/java/com/superbiz/agent/service/DocumentChunkServiceTest.java b/src/test/java/com/superbiz/agent/service/DocumentChunkServiceTest.java
deleted file mode 100644
index 479b683..0000000
--- a/src/test/java/com/superbiz/agent/service/DocumentChunkServiceTest.java
+++ /dev/null
@@ -1,539 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.config.DocumentChunkConfig;
-import com.superbiz.agent.dto.DocumentChunk;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
-
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-/**
- * 当前分片策略的单元测试 — 覆盖旧能力回归 + Phase 1 新增能力
- */
-@DisplayName("DocumentChunkService 分片策略")
-class DocumentChunkServiceTest {
-
- private DocumentChunkService service;
- private DocumentChunkConfig config;
-
- @BeforeEach
- void setUp() {
- config = new DocumentChunkConfig();
- config.setMaxSize(800);
- config.setMaxTokens(500);
- config.setMaxTokensHard(600);
- config.setOverlap(100);
- service = new DocumentChunkService();
- try {
- var field = DocumentChunkService.class.getDeclaredField("chunkConfig");
- field.setAccessible(true);
- field.set(service, config);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- // ==================== 回归：边界条件 ====================
-
- @Nested
- @DisplayName("边界条件")
- class BoundaryTests {
-
- @Test
- @DisplayName("null 内容 → 空列表")
- void nullContent_returnsEmpty() {
- List chunks = service.chunkDocument(null, "/test/null.md");
- assertTrue(chunks.isEmpty());
- }
-
- @Test
- @DisplayName("空字符串 → 空列表")
- void emptyContent_returnsEmpty() {
- List chunks = service.chunkDocument(" \n ", "/test/empty.md");
- assertTrue(chunks.isEmpty());
- }
-
- @Test
- @DisplayName("短文档（≤maxSize）→ 1个分块")
- void shortDocument_singleChunk() {
- String content = "这是一篇短文档，内容不超过800个字符。";
- List chunks = service.chunkDocument(content, "/test/short.md");
-
- assertEquals(1, chunks.size());
- assertEquals(content, chunks.get(0).getContent());
- assertEquals(0, chunks.get(0).getChunkIndex());
- }
-
- @Test
- @DisplayName("恰好 maxSize 边界 → 1个分块")
- void exactlyMaxSize_singleChunk() {
- String content = "A".repeat(800);
- List chunks = service.chunkDocument(content, "/test/boundary.md");
- assertEquals(1, chunks.size());
- }
- }
-
- // ==================== 回归：标题分割 ====================
-
- @Nested
- @DisplayName("Markdown 标题分割")
- class HeadingSplitTests {
-
- @Test
- @DisplayName("单个 H1 标题 → section 继承标题")
- void singleHeading_titlePropagates() {
- String content = "# CPU高负载问题\n\n这是CPU高负载的描述内容。";
- List chunks = service.chunkDocument(content, "/test/cpu.md");
-
- assertEquals(1, chunks.size());
- assertEquals("CPU高负载问题", chunks.get(0).getTitle());
- }
-
- @Test
- @DisplayName("多个标题 → 按标题边界分割")
- void multipleHeadings_splitAtHeadings() {
- String content =
- "# CPU高负载\n\nCPU问题的详细描述。\n\n" +
- "# 内存高负载\n\n内存问题的详细描述。";
-
- List chunks = service.chunkDocument(content, "/test/multi.md");
-
- assertEquals(2, chunks.size());
- assertEquals("CPU高负载", chunks.get(0).getTitle());
- assertEquals("内存高负载", chunks.get(1).getTitle());
- }
-
- @Test
- @DisplayName("多级标题（H1/H2/H3）→ 标题独立不冲突")
- void multiLevelHeadings() {
- String content =
- "# 一级标题\n\n一级内容。\n\n" +
- "## 二级标题\n\n二级内容。\n\n" +
- "### 三级标题\n\n三级内容。";
-
- List chunks = service.chunkDocument(content, "/test/levels.md");
- assertEquals(3, chunks.size());
- assertEquals("一级标题", chunks.get(0).getTitle());
- assertEquals("二级标题", chunks.get(1).getTitle());
- assertEquals("三级标题", chunks.get(2).getTitle());
- }
-
- @Test
- @DisplayName("H1-H6 全部支持")
- void allHeadingLevels() {
- StringBuilder sb = new StringBuilder();
- for (int i = 1; i <= 6; i++) {
- sb.append("#".repeat(i)).append(" 标题").append(i).append("\n\n内容").append(i).append("。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/h1h6.md");
- assertEquals(6, chunks.size());
- }
-
- @Test
- @DisplayName("无标题文档 → 整个文档作为1个 section")
- void noHeadings_entireAsOneSection() {
- String content = "纯文本没有标题。\n\n第二段内容。\n\n第三段内容。";
- List chunks = service.chunkDocument(content, "/test/nohead.md");
- assertFalse(chunks.isEmpty());
- assertNull(chunks.get(0).getTitle());
- }
- }
-
- // ==================== 回归：段落边界切分 ====================
-
- @Nested
- @DisplayName("超长章节 — 段落边界切分")
- class ParagraphSplitTests {
-
- @Test
- @DisplayName("短章节（≤maxSize）→ 不进入段落切割")
- void shortSection_noParagraphSplit() {
- StringBuilder sb = new StringBuilder();
- sb.append("# 测试\n\n");
- for (int i = 0; i < 5; i++) {
- sb.append("段落").append(i).append("：这是一段短内容。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/short_sec.md");
- assertEquals(1, chunks.size());
- }
-
- @Test
- @DisplayName("超长章节 → 在段落边界切分")
- void longSection_splitsAtParagraphBoundaries() {
- config.setMaxSize(50);
- config.setMaxTokens(30);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 长章节\n\n");
- for (int i = 0; i < 10; i++) {
- sb.append("段落").append(i).append("：ABCDEFGHIJKLMNOPQRSTUVWXYZ。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/long_sec.md");
- assertTrue(chunks.size() >= 2, "超长章节应切分为多个分块，实际: " + chunks.size());
-
- // 所有分块携带相同的 title
- for (DocumentChunk c : chunks) {
- assertEquals("长章节", c.getTitle());
- }
- }
- }
-
- // ==================== 回归：chunkIndex 元数据 ====================
-
- @Nested
- @DisplayName("分块元数据")
- class ChunkMetadataTests {
-
- @Test
- @DisplayName("chunkIndex 自增且唯一")
- void chunkIndexSequential() {
- config.setMaxSize(50);
- config.setMaxTokens(30);
-
- StringBuilder sb = new StringBuilder("# Meta\n\n");
- for (int i = 0; i < 10; i++) {
- sb.append("段落").append(i).append("：填充内容以触发切分机制。ABCDE。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/meta.md");
- assertTrue(chunks.size() >= 2);
-
- for (int i = 0; i < chunks.size(); i++) {
- assertEquals(i, chunks.get(i).getChunkIndex(),
- "chunkIndex 应从0开始连续递增");
- }
- }
-
- @Test
- @DisplayName("startIndex/endIndex 范围合法 — 无漂移")
- void indexRangeValid_noDrift() {
- String content = "# 标题\n\n测试内容。";
- List chunks = service.chunkDocument(content, "/test/index.md");
-
- for (DocumentChunk c : chunks) {
- assertTrue(c.getStartOffset() >= 0);
- assertTrue(c.getEndOffset() > c.getStartOffset(),
- "endIndex(" + c.getEndOffset() + ") 应 > startIndex(" + c.getStartOffset() + ")");
- assertTrue(c.getEndOffset() <= content.length());
- }
- }
- }
-
- // ==================== 新增：Token 估算 ====================
-
- @Nested
- @DisplayName("Token 估算")
- class TokenEstimationTests {
-
- @Test
- @DisplayName("纯中文 800 字符 ≈ 800 tokens → 短章节不切")
- void pureChinese_fewerTokensThanMax() {
- config.setMaxTokens(400);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 中文测试\n\n");
- // 纯中文 ~300 字符 ≈ 300 tokens
- for (int i = 0; i < 3; i++) {
- sb.append("这是纯中文测试内容的第十").append(i).append("段落。");
- sb.append("每个中文字符大约占用一个令牌的位置。");
- sb.append("因此这段文本的令牌数大致等于字符数。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/cn_tokens.md");
- // 300 字符 ≈ 300 tokens < 400 maxTokens → 1 个分块
- assertEquals(1, chunks.size());
- }
-
- @Test
- @DisplayName("纯英文 2000 字符 ≈ 500 tokens → 刚好不超过上限")
- void pureEnglish_moreCharactersSameTokens() {
- config.setMaxTokens(200);
- config.setMaxTokensHard(250);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# English Test\n\n");
- for (int i = 0; i < 8; i++) {
- sb.append("This is paragraph number ").append(i)
- .append(" containing English text. ")
- .append("English characters are much cheaper in tokens. ")
- .append("More filler text here to reach the limit properly. ")
- .append("Yet another sentence for good measure. ")
- .append("Still more words needed to reach token limit here.\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/en_tokens.md");
- // 大量英文才占少量 token → 分块数应少于用字符计数的版本
- assertTrue(chunks.size() >= 2, "1200+ 字符英文应切分");
- }
- }
-
- // ==================== 新增：列表结构感知 ====================
-
- @Nested
- @DisplayName("列表结构感知")
- class ListStructureTests {
-
- @Test
- @DisplayName("有序列表项之间不切分 — 即使超过 maxTokens")
- void orderedList_notSplitBetweenItems() {
- config.setMaxTokens(80);
- config.setMaxTokensHard(200);
- config.setOverlap(30);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 排查步骤\n\n");
- // 5个有序列表项，每项 ~40 字符 ≈ 40 tokens，总共 ~200 tokens
- for (int i = 1; i <= 5; i++) {
- sb.append(i).append(". 这是排查步骤第").append(i)
- .append("项，包含具体的操作指引和注意事项说明。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/ordered_list.md");
-
- // 5项应保持在一起（未触及 hard 上限）
- assertEquals(1, chunks.size(),
- "有序列表项不应被拆散，实际分块数: " + chunks.size());
-
- String content = chunks.get(0).getContent();
- assertTrue(content.contains("1. "), "应包含第1项");
- assertTrue(content.contains("5. "), "应包含第5项");
- }
-
- @Test
- @DisplayName("有序列表触及硬上限 → 在列表项边界强制切分")
- void orderedList_hardLimitSplits() {
- config.setMaxTokens(50);
- config.setMaxTokensHard(100);
- config.setOverlap(20);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 长列表\n\n");
- // 每项 ~60 tokens，硬上限 100 → 最多装 1 项多
- for (int i = 1; i <= 6; i++) {
- sb.append(i).append(". 这是很长的排查步骤内容，包含详细的说明信息。")
- .append("每个步骤都要执行多个检查操作。继续填充文本以增加令牌计数。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/long_list.md");
-
- System.out.println(" 长列表硬上限测试 — 实际分块数: " + chunks.size());
- for (DocumentChunk c : chunks) {
- System.out.println(" Chunk #" + c.getChunkIndex() + ": " + c.getContent().length() + "字符 "
- + "| start=" + c.getStartOffset() + " end=" + c.getEndOffset()
- + " | preview=" + c.getContent().substring(0, Math.min(60, c.getContent().length())).replace("\n", "\\n"));
- }
-
- // 硬上限会强制切分，但每个分块内的列表项应保持连续
- assertTrue(chunks.size() >= 2, "长列表应至少触发1次切分，实际: " + chunks.size());
-
- // 验证：除了第一个分块（可能是标题），其余应包含列表项
- for (int i = 1; i < chunks.size(); i++) {
- DocumentChunk c = chunks.get(i);
- assertFalse(c.getContent().isEmpty());
- assertTrue(c.getContent().matches("(?s).*\\d+\\.\\s.*"),
- "非标题分块应包含列表项，Chunk #" + c.getChunkIndex()
- + " preview: " + c.getContent().substring(0, Math.min(60, c.getContent().length())));
- }
- }
-
- @Test
- @DisplayName("无序列表项之间不切分")
- void unorderedList_notSplitBetweenItems() {
- config.setMaxTokens(80);
- config.setMaxTokensHard(200);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 检查清单\n\n");
- for (int i = 1; i <= 5; i++) {
- sb.append("- 检查项").append(i).append("：确认服务运行状态正常并记录相关指标。\n\n");
- }
-
- List chunks = service.chunkDocument(sb.toString(), "/test/unordered_list.md");
- assertEquals(1, chunks.size(), "无序列表项不应被拆散");
- }
-
- @Test
- @DisplayName("列表结束后普通段落应从下一段落开始新分块")
- void listEnds_normalParagraphStartsNewChunk() {
- config.setMaxTokens(150);
- config.setMaxTokensHard(250);
-
- StringBuilder sb = new StringBuilder();
- sb.append("# 文档\n\n");
- // 先一个普通段落
- sb.append("这是介绍段落，描述系统的整体架构和设计思路。\n\n");
- // 有序列表
- for (int i = 1; i <= 3; i++) {
- sb.append(i).append(". 列表项第").append(i).append("条，包含操作说明。\n\n");
- }
- // 普通段落
- sb.append("这是总结段落，包含上述操作完成后需要关注的监控指标。\n\n");
-
- List chunks = service.chunkDocument(sb.toString(), "/test/list_mixed.md");
- assertTrue(chunks.size() >= 1);
- // 列表项应保持在一起
- for (DocumentChunk c : chunks) {
- String content = c.getContent();
- // 分块中不应有孤立的单个列表项（除非只有一个）
- if (content.contains("1. ") && content.contains("3. ")) {
- // 这个分块包含了全部3个列表项 → 正确
- }
- }
- }
- }
-
- // ==================== 新增：代码块结构感知 ====================
-
- @Nested
- @DisplayName("代码块结构感知")
- class CodeBlockTests {
-
- @Test
- @DisplayName("代码块内部不切分")
- void codeBlock_notSplitInside() {
- config.setMaxTokens(60);
- config.setMaxTokensHard(200);
- config.setOverlap(20);
-
- String content =
- "# 代码示例\n\n" +
- "以下是配置代码：\n\n" +
- "```yaml\n" +
- "server:\n" +
- " port: 8080\n" +
- " host: localhost\n" +
- " timeout: 30s\n" +
- "```\n\n" +
- "配置说明结束。";
-
- List chunks = service.chunkDocument(content, "/test/code.md");
-
- // 代码块应保持完整（未触及硬上限）
- // 验证：至少有一个分块包含完整的 ```...```
- boolean foundCompleteBlock = false;
- for (DocumentChunk c : chunks) {
- String text = c.getContent();
- if (text.contains("```yaml") && text.contains("```") &&
- text.indexOf("```yaml") < text.lastIndexOf("```")) {
- foundCompleteBlock = true;
- }
- }
- // 可能整体在一个分块中
- assertTrue(chunks.size() >= 1);
- }
- }
-
- // ==================== 可视化 ====================
-
- @Nested
- @DisplayName("可视化 — 打印切分结果")
- class VisualInspectionTests {
-
- @Test
- @DisplayName("模拟运维文档 — 展示新策略效果")
- void realWorldAIOpsDoc() {
- config.setMaxTokens(150);
- config.setMaxTokensHard(200);
- config.setOverlap(40);
-
- String doc = """
- # CPU高负载问题排查指南
-
- ## 问题现象
-
- 服务器CPU使用率持续超过90%，系统响应变慢，用户反馈页面加载超时。
- 监控告警系统连续发出多条CPU使用率告警。
-
- ## 排查步骤
-
- 1. 登录服务器，执行 top 命令查看当前CPU使用率最高的进程。记录进程ID和CPU占用百分比。
-
- 2. 使用 ps aux | grep {进程名} 确认相关服务的运行状态。检查是否有异常进程占用资源。
-
- 3. 查看应用日志，重点关注最近15分钟的ERROR级别日志。使用 tail -n 500 命令。
-
- 4. 检查数据库连接池状态，确认是否有慢查询或连接泄漏。查看慢查询日志。
-
- 5. 检查JVM内存使用情况和GC日志。使用 jstat -gcutil {pid} 1000 命令观察GC频率。
-
- ## 常见原因
-
- 1. 死循环或递归调用导致CPU满载。检查是否有未设置退出条件的循环逻辑。
- 2. 大量正则表达式匹配操作。检查是否有未编译的正则在循环中使用。
-
- ## 解决方案
-
- 根据排查结果采取对应措施：代码问题则回滚或热修复；资源不足则扩容。
- 处理完成后持续观察监控指标30分钟，确认CPU使用率恢复正常。
- """;
-
- List chunks = service.chunkDocument(doc, "/kb/cpu_high_usage.md");
-
- System.out.println("========================================");
- System.out.println(" Phase 1 新策略效果 — 模拟运维文档");
- System.out.println(" 配置: maxTokens=150, hard=200, overlap=40");
- System.out.println(" 总字符数: " + doc.length());
- System.out.println(" 总分块数: " + chunks.size());
- System.out.println("========================================\n");
-
- for (DocumentChunk c : chunks) {
- System.out.println("┌─ Chunk #" + c.getChunkIndex());
- System.out.println("│ Title: " + (c.getTitle() != null ? c.getTitle() : "(无)"));
- System.out.println("│ Range: [" + c.getStartOffset() + "→" + c.getEndOffset() + "] (" + c.getContent().length() + "字符)");
- // 显示前150字符
- String preview = c.getContent().length() > 120
- ? c.getContent().substring(0, 120).replace("\n", "\\n") + "..."
- : c.getContent().replace("\n", "\\n");
- System.out.println("│ Preview: " + preview);
- System.out.println("└──────────────────────\n");
- }
-
- assertTrue(chunks.size() >= 3, "应产生多个分块");
- }
-
- @Test
- @DisplayName("中英混排对比 — token vs 字符计数差异")
- void mixedContentComparison() {
- config.setMaxTokens(100);
- config.setMaxTokensHard(150);
- config.setOverlap(30);
-
- String chinese = "这是中文内容示范。中文每个字符在LLM中约占用1个token。" +
- "因此这段文本在上下文窗口中占用的token数较多。" +
- "继续填充文字以触发切分逻辑，验证中文token估算是否合理。" +
- "更多中文文本来增加令牌计数。";
-
- String english = "This is English content. Each word may take one or two tokens. " +
- "A sentence like this one actually consumes relatively few tokens compared to " +
- "Chinese characters. More English text to reach the same token count as above. " +
- "Still need more words because English is very efficient in tokenization. " +
- "Adding even more content to make this paragraph long enough to test properly.";
-
- List cnChunks = service.chunkDocument("# CN\n\n" + chinese + "\n\n" + chinese, "/test/cn.md");
- List enChunks = service.chunkDocument("# EN\n\n" + english + "\n\n" + english, "/test/en.md");
-
- System.out.println("========================================");
- System.out.println(" Token 计数对比");
- System.out.println(" 配置: maxTokens=100, overlap=30");
- System.out.println("========================================");
- System.out.println(" 中文文档: " + (chinese.length() * 2) + "字符 → " + cnChunks.size() + "个分块");
- System.out.println(" 英文文档: " + (english.length() * 2) + "字符 → " + enChunks.size() + "个分块");
-
- for (DocumentChunk c : cnChunks) {
- System.out.println(" 中文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
- }
- for (DocumentChunk c : enChunks) {
- System.out.println(" 英文Chunk#" + c.getChunkIndex() + ": " + c.getContent().length() + "字符");
- }
- System.out.println(" ★ 现在中文和英文的分块数更接近（基于 token 而非字符）");
- System.out.println("========================================");
- }
- }
-}
diff --git a/src/test/java/com/superbiz/agent/service/DocumentManagementServiceTest.java b/src/test/java/com/superbiz/agent/service/DocumentManagementServiceTest.java
index f3c44b6..584332d 100644
--- a/src/test/java/com/superbiz/agent/service/DocumentManagementServiceTest.java
+++ b/src/test/java/com/superbiz/agent/service/DocumentManagementServiceTest.java
@@ -1,6 +1,5 @@
package com.superbiz.agent.service;
-import com.superbiz.agent.dto.Frontmatter;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.mock.web.MockMultipartFile;
@@ -39,16 +38,4 @@ class DocumentManagementServiceTest {
assertEquals("payment/runbook.md", storedPath);
assertTrue(Files.exists(tempDir.resolve("payment").resolve("runbook.md")));
}
-
- @Test
- void resolveDocumentIdUsesFrontmatterSourceWhenItFitsDatabaseColumn() {
- DocumentManagementService service = new DocumentManagementService();
- Frontmatter frontmatter = Frontmatter.builder()
- .source("mysql-connection-pool")
- .build();
-
- String docId = ReflectionTestUtils.invokeMethod(service, "resolveDocumentId", frontmatter);
-
- assertEquals("mysql-connection-pool", docId);
- }
}
diff --git a/src/test/java/com/superbiz/agent/service/FrontmatterParserTest.java b/src/test/java/com/superbiz/agent/service/FrontmatterParserTest.java
deleted file mode 100644
index fac2fe7..0000000
--- a/src/test/java/com/superbiz/agent/service/FrontmatterParserTest.java
+++ /dev/null
@@ -1,193 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.dto.Frontmatter;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-/**
- * FrontmatterParser 单元测试
- */
-class FrontmatterParserTest {
-
- private FrontmatterParser parser;
-
- @BeforeEach
- void setUp() {
- parser = new FrontmatterParser();
- }
-
- @Test
- void testHasFrontmatter_withValidFrontmatter() {
- String content = "---\ntitle: Test\n---\nContent";
- assertTrue(parser.hasFrontmatter(content));
- }
-
- @Test
- void testHasFrontmatter_withoutFrontmatter() {
- String content = "# Just a title\nContent";
- assertFalse(parser.hasFrontmatter(content));
- }
-
- @Test
- void testHasFrontmatter_nullContent() {
- assertFalse(parser.hasFrontmatter(null));
- }
-
- @Test
- void testHasFrontmatter_emptyContent() {
- assertFalse(parser.hasFrontmatter(""));
- }
-
- @Test
- void testParse_validFrontmatter() {
- String content = """
- ---
- title: 支付网关错误码
- keywords: [ERR_TIMEOUT, 超时, 支付网关]
- summary: 记录了支付网关所有核心错误码
- category: api
- ---
-
- # 正文内容
- """;
-
- Frontmatter result = parser.parse(content);
-
- assertNotNull(result);
- assertEquals("支付网关错误码", result.getTitle());
- assertEquals(3, result.getKeywords().size());
- assertTrue(result.getKeywords().contains("ERR_TIMEOUT"));
- assertEquals("记录了支付网关所有核心错误码", result.getSummary());
- assertEquals("api", result.getCategory());
- }
-
- @Test
- void testParse_withoutFrontmatter() {
- String content = "# Just content\nNo frontmatter here";
- assertNull(parser.parse(content));
- }
-
- @Test
- void testParse_missingRequiredFields() {
- String content = """
- ---
- title: Only Title
- ---
- Content
- """;
-
- // 缺少 keywords 和 summary，应返回 null
- Frontmatter result = parser.parse(content);
- assertNull(result);
- }
-
- @Test
- void testParse_malformedYaml() {
- String content = """
- ---
- title: Test
- keywords: [unclosed array
- ---
- Content
- """;
-
- // YAML 格式错误，应返回 null
- Frontmatter result = parser.parse(content);
- assertNull(result);
- }
-
- @Test
- void testParse_noClosingDelimiter() {
- String content = """
- ---
- title: Test
- keywords: [test]
- summary: Test summary
-
- Content without closing ---
- """;
-
- // 缺少结束标记，应返回 null
- Frontmatter result = parser.parse(content);
- assertNull(result);
- }
-
- @Test
- void testParse_windowsLineEndings() {
- String content = "---\r\ntitle: Test\r\nkeywords: [test]\r\nsummary: Summary\r\n---\r\nContent";
-
- Frontmatter result = parser.parse(content);
-
- assertNotNull(result);
- assertEquals("Test", result.getTitle());
- }
-
- @Test
- void testParse_withOptionalFields() {
- String content = """
- ---
- title: Test Document
- keywords: [test, doc]
- summary: A test document
- version: 1.0.0
- author: Test Author
- ---
- Content
- """;
-
- Frontmatter result = parser.parse(content);
-
- assertNotNull(result);
- assertEquals("Test Document", result.getTitle());
- assertEquals("1.0.0", result.getVersion());
- assertEquals("Test Author", result.getAuthor());
- }
-
- @Test
- void testParse_withRetrievalMetadata() {
- String content = """
- ---
- title: MySQL Connection Pool
- keywords: [connection pool, HikariCP]
- summary: Diagnose exhausted MySQL connection pools
- category: database
- source: mysql-connection-pool
- breadcrumb: Database > MySQL > Connection Pool
- kb_scope: rag-eval
- ---
- Content
- """;
-
- Frontmatter result = parser.parse(content);
-
- assertNotNull(result);
- assertEquals("mysql-connection-pool", result.getSource());
- assertEquals("Database > MySQL > Connection Pool", result.getBreadcrumb());
- assertEquals("rag-eval", result.getKbScope());
- }
-
- @Test
- void testStripFrontmatter_returnsMarkdownBodyOnly() {
- String content = """
- ---
- title: Test
- keywords: [frontmatter-only]
- summary: Summary
- ---
-
- # Body
-
- Body content
- """;
-
- String body = parser.stripFrontmatter(content);
-
- assertFalse(body.contains("frontmatter-only"));
- assertTrue(body.startsWith("# Body"));
- assertTrue(body.contains("Body content"));
- }
-}
diff --git a/src/test/java/com/superbiz/agent/service/FullPipelineSmokeTest.java b/src/test/java/com/superbiz/agent/service/FullPipelineSmokeTest.java
deleted file mode 100644
index 957a391..0000000
--- a/src/test/java/com/superbiz/agent/service/FullPipelineSmokeTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-package com.superbiz.agent.service;
-
-import org.junit.jupiter.api.DisplayName;
-import org.junit.jupiter.api.MethodOrderer;
-import org.junit.jupiter.api.Order;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestMethodOrder;
-import org.springframework.ai.chat.model.ChatModel;
-import org.springframework.ai.chat.prompt.Prompt;
-import org.springframework.ai.embedding.EmbeddingModel;
-import org.springframework.ai.tool.ToolCallback;
-import org.springframework.ai.tool.ToolCallbackProvider;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.boot.test.context.TestConfiguration;
-import org.springframework.context.annotation.Bean;
-
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-/**
- * 全链路验证：DeepSeek → BGE-M3 → Milvus
- */
-@SpringBootTest
-@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
-@DisplayName("DeepSeek → BGE-M3 → Milvus 全链路")
-class FullPipelineSmokeTest {
-
- @Autowired
- private ChatModel chatModel;
-
- @Autowired
- private EmbeddingModel embeddingModel;
-
- @Autowired
- private VectorEmbeddingService vectorEmbeddingService;
-
- @Autowired
- private VectorSearchService vectorSearchService;
-
- @TestConfiguration
- static class MockToolConfig {
- @Bean
- public ToolCallbackProvider toolCallbackProvider() {
- return () -> new ToolCallback[0];
- }
- }
-
- // ===== ① Chat: DeepSeek =====
-
- @Test
- @Order(1)
- @DisplayName("Chat: DeepSeek 聊天验证")
- void chatDeepSeekWorks() {
- System.out.println("\n===== ① Chat: DeepSeek =====");
- System.out.println("ChatModel: " + chatModel.getClass().getSimpleName());
- System.out.println("ChatOptions: " + chatModel.toString());
-
- // 直接调用 chat
- var response = chatModel.call(new Prompt("请用一句话介绍你自己"));
- String text = response.getResult().getOutput().getText();
- assertNotNull(text);
- assertFalse(text.isEmpty());
- System.out.println("Response: " + text.substring(0, Math.min(200, text.length())) + "...");
- System.out.println("Chat ✓");
- }
-
- // ===== ② Embedding: BGE-M3 via SiliconFlow =====
-
- @Test
- @Order(2)
- @DisplayName("Embedding: BGE-M3 向量生成验证")
- void embeddingBgeM3Works() {
- System.out.println("\n===== ② Embedding: BGE-M3 (SiliconFlow) =====");
- System.out.println("EmbeddingModel: " + embeddingModel.getClass().getSimpleName());
-
- String text = "你好，这是一条测试文本";
- List vector = vectorEmbeddingService.generateEmbedding(text);
-
- assertNotNull(vector);
- assertFalse(vector.isEmpty());
- assertEquals(1024, vector.size(), "BGE-M3 应返回 1024 维向量");
-
- // 非零校验
- boolean hasNonZero = vector.stream().anyMatch(v -> Math.abs(v) > 1e-6);
- assertTrue(hasNonZero, "向量不能全为零");
-
- // L2 范数校验：BGE-M3 输出应为 L2 归一化的单位向量
- double norm = Math.sqrt(vector.stream().mapToDouble(v -> (double) v * v).sum());
-
- System.out.println("维度: " + vector.size());
- System.out.println("前5维: " + vector.subList(0, Math.min(5, vector.size())));
- System.out.println("L2 范数: " + String.format("%.10f", norm));
- System.out.println("是否归一化 (|norm - 1.0| < 0.01): " + (Math.abs(norm - 1.0) < 0.01));
-
- assertEquals(1.0, norm, 0.01, "BGE-M3 向量应为 L2 归一化单位向量，实际范数=" + norm);
- System.out.println("Embedding ✓");
- }
-
- @Test
- @Order(3)
- @DisplayName("Embedding: BGE-M3 批量向量生成验证")
- void embeddingBatchWorks() {
- System.out.println("\n===== ③ Embedding 批量 =====");
- List texts = List.of("文本一", "文本二", "文本三");
- List> results = vectorEmbeddingService.generateEmbeddings(texts);
-
- assertEquals(3, results.size());
- for (List r : results) {
- assertEquals(1024, r.size());
- }
- System.out.println("批量生成: " + results.size() + " 个 向量，各 " + results.get(0).size() + " 维 ✓");
- }
-
- // ===== ③ Milvus: 向量搜索 =====
-
- @Test
- @Order(4)
- @DisplayName("Milvus: 连接 + 搜索验证")
- void milvusSearchWorks() {
- System.out.println("\n===== ④ Milvus: 向量搜索 =====");
-
- // 用 BGE-M3 生成查询向量
- String query = "内部文档";
- List queryVector = vectorEmbeddingService.generateQueryVector(query);
- assertNotNull(queryVector);
- assertEquals(1024, queryVector.size());
-
- // 搜索
- List results =
- vectorSearchService.searchSimilarDocuments(query, 3);
-
- assertNotNull(results);
- System.out.println("查询: " + query);
- System.out.println("返回: " + results.size() + " 条");
-
- if (!results.isEmpty()) {
- // 至少有结果，验证结构
- for (int i = 0; i < results.size(); i++) {
- var r = results.get(i);
- assertNotNull(r.getId());
- assertNotNull(r.getContent());
- System.out.println(" [" + (i + 1) + "] id=" + r.getId()
- + ", score=" + String.format("%.4f", r.getScore())
- + ", content=" + r.getContent().substring(0, Math.min(50, r.getContent().length())) + "...");
- }
- } else {
- System.out.println("(Milvus 中暂无数据，但连接正常)");
- }
-
- System.out.println("Milvus ✓");
- }
-
- // ===== 汇总 =====
-
- @Test
- @Order(5)
- @DisplayName("总结")
- void summary() {
- System.out.println("\n==========================================");
- System.out.println("全链路验证完成：");
- System.out.println(" ① Chat → DeepSeek ✓");
- System.out.println(" ② Embedding → BGE-M3 ✓ (SiliconFlow, 1024维)");
- System.out.println(" ③ 向量存储 → Milvus ✓ (Zilliz Cloud)");
- System.out.println("==========================================");
- }
-}
diff --git a/src/test/java/com/superbiz/agent/service/KnowledgeIndexServiceTest.java b/src/test/java/com/superbiz/agent/service/KnowledgeIndexServiceTest.java
deleted file mode 100644
index 1943926..0000000
--- a/src/test/java/com/superbiz/agent/service/KnowledgeIndexServiceTest.java
+++ /dev/null
@@ -1,316 +0,0 @@
-package com.superbiz.agent.service;
-
-import com.superbiz.agent.dto.KnowledgeEntry;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.springframework.test.util.ReflectionTestUtils;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-/**
- * KnowledgeIndexService 单元测试
- */
-class KnowledgeIndexServiceTest {
-
- private KnowledgeIndexService service;
-
- @TempDir
- Path tempDir;
-
- @BeforeEach
- void setUp() {
- service = new KnowledgeIndexService();
- ReflectionTestUtils.setField(service, "knowledgeBasePath", tempDir.toString());
- }
-
- @Test
- void testExactMatch_singleMatch() {
- // 准备测试数据
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath("test.md")
- .title("Test")
- .keywords(List.of("ERR_TIMEOUT", "超时"))
- .summary("Test summary")
- .category("api")
- .build();
-
- service.addToIndex(entry);
-
- // 测试匹配
- List results = service.exactMatch("ERR_TIMEOUT");
-
- assertEquals(1, results.size());
- assertEquals("Test", results.get(0).getTitle());
- }
-
- @Test
- void testExactMatch_caseInsensitive() {
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath("test.md")
- .keywords(List.of("ERR_TIMEOUT"))
- .build();
-
- service.addToIndex(entry);
-
- // 小写查询应该匹配
- List results = service.exactMatch("err_timeout");
- assertEquals(1, results.size());
- }
-
- @Test
- void testExactMatch_partialMatch() {
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath("test.md")
- .keywords(List.of("支付网关"))
- .build();
-
- service.addToIndex(entry);
-
- // 包含关键词的查询应该匹配
- List results = service.exactMatch("支付网关超时问题");
- assertEquals(1, results.size());
- }
-
- @Test
- void testExactMatch_multipleMatches() {
- KnowledgeEntry entry1 = KnowledgeEntry.builder()
- .filePath("doc1.md")
- .title("Doc 1")
- .keywords(List.of("超时"))
- .build();
-
- KnowledgeEntry entry2 = KnowledgeEntry.builder()
- .filePath("doc2.md")
- .title("Doc 2")
- .keywords(List.of("超时", "错误"))
- .build();
-
- service.addToIndex(entry1);
- service.addToIndex(entry2);
-
- // 应该匹配两个文档
- List results = service.exactMatch("超时");
- assertEquals(2, results.size());
- }
-
- @Test
- void testAnalyzeQuery_returnsStructuredHint() {
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath("mysql.md")
- .title("MySQL Doc")
- .keywords(List.of("mysql", "connection pool"))
- .category("database")
- .build();
-
- service.addToIndex(entry);
-
- KnowledgeIndexService.L0Hint hint = service.analyzeQuery("mysql connection pool timeout");
-
- assertEquals(1, hint.matches().size());
- assertEquals(List.of("mysql", "connection pool"), hint.matchedKeywords());
- assertEquals(List.of("database"), hint.domains());
- assertEquals(List.of("mysql", "connection pool"), hint.entities());
- assertEquals(List.of("MySQL Doc"), hint.titles());
- assertEquals("database", hint.singleDomainOrNull());
- }
-
- @Test
- void testAnalyzeQuery_multipleDomainsHasNoSingleDomain() {
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("mysql.md")
- .keywords(List.of("timeout"))
- .category("database")
- .build());
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("api.md")
- .keywords(List.of("timeout"))
- .category("api")
- .build());
-
- KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
-
- assertEquals(2, hint.matches().size());
- assertNull(hint.singleDomainOrNull());
- }
-
- @Test
- void testAnalyzeQuery_filtersByConfiguredKbScope() {
- ReflectionTestUtils.setField(service, "kbScope", "rag-eval");
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("legacy.md")
- .keywords(List.of("timeout"))
- .category("legacy")
- .build());
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("eval.md")
- .keywords(List.of("timeout"))
- .category("eval")
- .kbScope("rag-eval")
- .build());
-
- KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
-
- assertEquals(1, hint.matches().size());
- assertEquals(List.of("eval"), hint.domains());
- assertEquals("eval", hint.singleDomainOrNull());
- }
-
- @Test
- void testAnalyzeQuery_keepsLegacyEntriesWhenNoScopeConfigured() {
- ReflectionTestUtils.setField(service, "kbScope", "");
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("legacy.md")
- .keywords(List.of("timeout"))
- .category("legacy")
- .build());
- service.addToIndex(KnowledgeEntry.builder()
- .filePath("eval.md")
- .keywords(List.of("timeout"))
- .category("eval")
- .kbScope("rag-eval")
- .build());
-
- KnowledgeIndexService.L0Hint hint = service.analyzeQuery("timeout");
-
- assertEquals(2, hint.matches().size());
- assertNull(hint.singleDomainOrNull());
- }
-
- @Test
- void testExactMatch_noMatch() {
- KnowledgeEntry entry = KnowledgeEntry.builder()
- .filePath("test.md")
- .keywords(List.of("错误码"))
- .build();
-
- service.addToIndex(entry);
-
- // 不匹配的查询
- List results = service.exactMatch("限流");
- assertEquals(0, results.size());
- }
-
- @Test
- void testExactMatch_emptyQuery() {
- List