refactor(rag): extract retrieval and ingest to py-rag service
- replace in-process Milvus stack with PyRagClient + PyRagKnowledgeSearchAdapter behind KnowledgeSearchPort (RERANK score passthrough) - move document ingest to py-rag /documents:ingest; DocumentManagementService keeps MySQL ledger + local files - sink L0 query understanding to py-rag; drop KnowledgeQueryTransformer, single UNFILTERED_VECTOR attempt - remove Milvus deps, config classes, dead demo services and obsolete rebuild scripts - compose/Makefile reduced to MySQL/Redis; add pyrag.* config
This commit is contained in:
zhuyongxin
@@ -1,196 +0,0 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.grpc.DataType;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.IndexType;
|
||||
import io.milvus.param.MetricType;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.RpcStatus;
|
||||
import io.milvus.param.collection.*;
|
||||
import io.milvus.param.index.CreateIndexParam;
|
||||
import com.superbiz.agent.config.MilvusProperties;
|
||||
import com.superbiz.agent.constant.MilvusConstants;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Milvus 客户端工厂类
|
||||
* 负责创建和初始化 Milvus 客户端连接
|
||||
*/
|
||||
@Component
|
||||
public class MilvusClientFactory {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MilvusClientFactory.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusProperties milvusProperties;
|
||||
|
||||
/**
|
||||
* 创建并初始化 Milvus 客户端
|
||||
*
|
||||
* 简化版本：直接连接并创建 collection
|
||||
*
|
||||
* @return MilvusServiceClient 实例
|
||||
* @throws RuntimeException 如果连接或初始化失败
|
||||
*/
|
||||
public MilvusServiceClient createClient() {
|
||||
MilvusServiceClient client = null;
|
||||
|
||||
try {
|
||||
// 1. 连接到 Milvus
|
||||
logger.info("正在连接到 Milvus: {}:{}", milvusProperties.getHost(), milvusProperties.getPort());
|
||||
client = connectToMilvus();
|
||||
logger.info("成功连接到 Milvus");
|
||||
|
||||
// 2. 检查并创建 biz collection（如果不存在）
|
||||
if (!collectionExists(client, MilvusConstants.MILVUS_COLLECTION_NAME)) {
|
||||
logger.info("collection '{}' 不存在，正在创建...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
createBizCollection(client);
|
||||
logger.info("成功创建 collection '{}'", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
|
||||
// 创建索引
|
||||
createIndexes(client);
|
||||
logger.info("成功创建索引");
|
||||
} else {
|
||||
logger.info("collection '{}' 已存在", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
}
|
||||
|
||||
// 3. 加载 collection 到内存（搜索必须）
|
||||
logger.info("正在加载 collection '{}' 到内存...", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
R<RpcStatus> loadResp = client.loadCollection(LoadCollectionParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.build());
|
||||
if (loadResp.getStatus() == 0) {
|
||||
logger.info("collection '{}' 已加载", MilvusConstants.MILVUS_COLLECTION_NAME);
|
||||
} else {
|
||||
logger.warn("collection '{}' 加载失败: {}", MilvusConstants.MILVUS_COLLECTION_NAME, loadResp.getMessage());
|
||||
}
|
||||
|
||||
return client;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("创建 Milvus 客户端失败", e);
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
throw new RuntimeException("创建 Milvus 客户端失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 连接到 Milvus
|
||||
*/
|
||||
private MilvusServiceClient connectToMilvus() {
|
||||
ConnectParam.Builder builder = ConnectParam.newBuilder()
|
||||
.withHost(milvusProperties.getHost())
|
||||
.withPort(milvusProperties.getPort())
|
||||
.withDatabaseName(milvusProperties.getDatabase())
|
||||
.withConnectTimeout(milvusProperties.getTimeout(), TimeUnit.MILLISECONDS);
|
||||
|
||||
// Zilliz Cloud: token + SSL
|
||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isEmpty()) {
|
||||
builder.withToken(milvusProperties.getToken());
|
||||
builder.withSecure(true);
|
||||
}
|
||||
// 本地 Milvus: username + password
|
||||
else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isEmpty()) {
|
||||
builder.withAuthorization(milvusProperties.getUsername(), milvusProperties.getPassword());
|
||||
}
|
||||
|
||||
return new MilvusServiceClient(builder.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查 collection 是否存在
|
||||
*/
|
||||
private boolean collectionExists(MilvusServiceClient client, String collectionName) {
|
||||
R<Boolean> response = client.hasCollection(HasCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build());
|
||||
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("检查 collection 失败: " + response.getMessage());
|
||||
}
|
||||
|
||||
return response.getData();
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建 biz collection
|
||||
*/
|
||||
private void createBizCollection(MilvusServiceClient client) {
|
||||
// 定义字段
|
||||
FieldType idField = FieldType.newBuilder()
|
||||
.withName("id")
|
||||
.withDataType(DataType.VarChar)
|
||||
.withMaxLength(MilvusConstants.ID_MAX_LENGTH)
|
||||
.withPrimaryKey(true)
|
||||
.build();
|
||||
|
||||
FieldType vectorField = FieldType.newBuilder()
|
||||
.withName("vector")
|
||||
.withDataType(DataType.FloatVector) // 改为 FloatVector
|
||||
.withDimension(milvusProperties.getVectorDim())
|
||||
.build();
|
||||
|
||||
FieldType contentField = FieldType.newBuilder()
|
||||
.withName("content")
|
||||
.withDataType(DataType.VarChar)
|
||||
.withMaxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.build();
|
||||
|
||||
FieldType metadataField = FieldType.newBuilder()
|
||||
.withName("metadata")
|
||||
.withDataType(DataType.JSON)
|
||||
.build();
|
||||
|
||||
// 创建 collection schema
|
||||
CollectionSchemaParam schema = CollectionSchemaParam.newBuilder()
|
||||
.withEnableDynamicField(false)
|
||||
.addFieldType(idField)
|
||||
.addFieldType(vectorField)
|
||||
.addFieldType(contentField)
|
||||
.addFieldType(metadataField)
|
||||
.build();
|
||||
|
||||
// 创建 collection
|
||||
CreateCollectionParam createParam = CreateCollectionParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.withDescription("Business knowledge collection")
|
||||
.withSchema(schema)
|
||||
.withShardsNum(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
||||
.build();
|
||||
|
||||
R<RpcStatus> response = client.createCollection(createParam);
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("创建 collection 失败: " + response.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 collection 创建索引
|
||||
*/
|
||||
private void createIndexes(MilvusServiceClient client) {
|
||||
// 为 vector 字段创建索引（FloatVector 使用 IVF_FLAT 和 L2 距离）
|
||||
CreateIndexParam vectorIndexParam = CreateIndexParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.withFieldName("vector")
|
||||
.withIndexType(IndexType.IVF_FLAT)
|
||||
.withMetricType(MetricType.L2) // L2 距离（欧氏距离）
|
||||
.withExtraParam("{\"nlist\":128}")
|
||||
.withSyncMode(Boolean.FALSE)
|
||||
.build();
|
||||
|
||||
R<RpcStatus> response = client.createIndex(vectorIndexParam);
|
||||
if (response.getStatus() != 0) {
|
||||
throw new RuntimeException("创建 vector 索引失败: " + response.getMessage());
|
||||
}
|
||||
|
||||
logger.info("成功为 vector 字段创建索引");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import com.superbiz.agent.config.PyRagProperties;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.core.io.ByteArrayResource;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.client.SimpleClientHttpRequestFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* py-rag 知识服务 HTTP 客户端（API v1，契约见 py-rag 仓库 docs/Java接入文档.md）。
|
||||
*
|
||||
* <p>错误信封：4xx/5xx 一律 {@code {"error":{"code":"E_XXX","message":…,"details":[…]}}}，
|
||||
* 统一抛出 {@link PyRagClientException}；网络异常包装为 {@code E_NETWORK}。
|
||||
* {@code evidence_status=no_evidence} 是 200 正常业务响应，不作为错误。</p>
|
||||
*
|
||||
* <p>每个请求携带 {@code X-Request-ID}（UUID）用于跨服务日志关联；
|
||||
* 超时按接入文档矩阵分端点配置（见 {@link PyRagProperties}）。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PyRagClient {
|
||||
|
||||
private static final String REQUEST_ID_HEADER = "X-Request-ID";
|
||||
|
||||
private final PyRagProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final RestClient searchClient;
|
||||
private final RestClient ingestClient;
|
||||
private final RestClient defaultClient;
|
||||
|
||||
public PyRagClient(PyRagProperties properties, ObjectMapper objectMapper) {
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
this.searchClient = buildRestClient(properties.getSearchReadTimeoutMs());
|
||||
this.ingestClient = buildRestClient(properties.getIngestReadTimeoutMs());
|
||||
this.defaultClient = buildRestClient(properties.getDefaultReadTimeoutMs());
|
||||
}
|
||||
|
||||
private RestClient buildRestClient(int readTimeoutMs) {
|
||||
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
|
||||
factory.setConnectTimeout(properties.getConnectTimeoutMs());
|
||||
factory.setReadTimeout(readTimeoutMs);
|
||||
return RestClient.builder()
|
||||
.baseUrl(properties.getBaseUrl())
|
||||
.requestFactory(factory)
|
||||
.build();
|
||||
}
|
||||
|
||||
// ── 检索 ────────────────────────────────────────────────
|
||||
|
||||
public PyRagSearchResponse search(PyRagSearchRequest request) {
|
||||
return exchange(() -> searchClient.post()
|
||||
.uri("/api/v1/search")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(request), PyRagSearchResponse.class);
|
||||
}
|
||||
|
||||
// ── 文档入库 ────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* multipart 文档入库。category 为契约必填；title/breadcrumb/kbScope 可选（null 不传）。
|
||||
* 同内容重复上传返回 unchanged（幂等），网络超时可安全重试。
|
||||
*/
|
||||
public PyRagIngestResponse ingest(String filename,
|
||||
byte[] content,
|
||||
String contentType,
|
||||
String category,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
String kbScope) {
|
||||
MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
|
||||
body.add("file", new ByteArrayResource(content) {
|
||||
@Override
|
||||
public String getFilename() {
|
||||
return filename;
|
||||
}
|
||||
});
|
||||
body.add("category", category);
|
||||
if (title != null && !title.isBlank()) {
|
||||
body.add("title", title);
|
||||
}
|
||||
if (breadcrumb != null && !breadcrumb.isBlank()) {
|
||||
body.add("breadcrumb", breadcrumb);
|
||||
}
|
||||
if (kbScope != null && !kbScope.isBlank()) {
|
||||
body.add("kb_scope", kbScope);
|
||||
}
|
||||
return exchange(() -> ingestClient.post()
|
||||
.uri("/api/v1/documents:ingest")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.MULTIPART_FORM_DATA)
|
||||
.body(body), PyRagIngestResponse.class);
|
||||
}
|
||||
|
||||
// ── 全量重建（异步任务） ────────────────────────────────
|
||||
|
||||
/** 202 返回任务号；已有 rebuild 执行中抛 E_REBUILD_IN_PROGRESS。 */
|
||||
public PyRagTaskAccepted rebuild() {
|
||||
return exchange(() -> defaultClient.post()
|
||||
.uri("/api/v1/collections:rebuild?confirm=REBUILD")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.body(Map.of()), PyRagTaskAccepted.class);
|
||||
}
|
||||
|
||||
public PyRagTaskStatus task(String taskId) {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/tasks/{id}", taskId)
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagTaskStatus.class);
|
||||
}
|
||||
|
||||
// ── 统计与健康 ──────────────────────────────────────────
|
||||
|
||||
public PyRagStats stats() {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/stats")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagStats.class);
|
||||
}
|
||||
|
||||
public PyRagHealth health() {
|
||||
return exchange(() -> defaultClient.get()
|
||||
.uri("/api/v1/health")
|
||||
.header(REQUEST_ID_HEADER, UUID.randomUUID().toString()), PyRagHealth.class);
|
||||
}
|
||||
|
||||
// ── 内部 ────────────────────────────────────────────────
|
||||
|
||||
private <T> T exchange(Supplier<RestClient.RequestHeadersSpec<?>> spec, Class<T> type) {
|
||||
try {
|
||||
return spec.get().exchange((request, response) -> {
|
||||
if (response.getStatusCode().isError()) {
|
||||
throw toClientException(response.getStatusCode().value(), response.getBody());
|
||||
}
|
||||
return objectMapper.readValue(response.getBody(), type);
|
||||
});
|
||||
} catch (PyRagClientException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
log.error("py-rag 调用失败: {}", e.getMessage(), e);
|
||||
throw new PyRagClientException("E_NETWORK",
|
||||
"py-rag 调用失败: " + e.getMessage(), null, e);
|
||||
}
|
||||
}
|
||||
|
||||
private PyRagClientException toClientException(int httpStatus, java.io.InputStream body) {
|
||||
String code = "E_HTTP_" + httpStatus;
|
||||
String message = "HTTP " + httpStatus;
|
||||
try {
|
||||
PyRagErrorEnvelope envelope = objectMapper.readValue(body, PyRagErrorEnvelope.class);
|
||||
if (envelope != null && envelope.error() != null) {
|
||||
code = envelope.error().code() == null ? code : envelope.error().code();
|
||||
message = envelope.error().message() == null ? message : envelope.error().message();
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// 错误响应体不是契约信封（如网关 502 页面），保留 HTTP 默认语义
|
||||
}
|
||||
return new PyRagClientException(code, message, httpStatus, null);
|
||||
}
|
||||
|
||||
// ── 契约 DTO（snake_case 对齐 py-rag API） ──────────────
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
public record PyRagSearchRequest(
|
||||
String query,
|
||||
/** hybrid=dense+BM25 融合；semantic=纯向量 */
|
||||
String mode,
|
||||
Integer retrieveK,
|
||||
Integer returnN,
|
||||
Integer maxChunksPerDocument,
|
||||
String category,
|
||||
String kbScope
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagSearchHit(
|
||||
String evidenceKey,
|
||||
String documentId,
|
||||
String source,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
String excerpt,
|
||||
Double qualityScore,
|
||||
String relevanceLevel
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagRetrievalTrace(
|
||||
String mode,
|
||||
Map<String, Object> filters,
|
||||
Integer recallCount,
|
||||
String rerankModel,
|
||||
String noEvidenceBasis
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagSearchResponse(
|
||||
String query,
|
||||
String mode,
|
||||
List<PyRagSearchHit> hits,
|
||||
String relevanceLevel,
|
||||
/** supported | no_evidence（no_evidence 时 hits=[]，属正常业务响应） */
|
||||
String evidenceStatus,
|
||||
PyRagRetrievalTrace retrievalTrace
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagIngestResponse(
|
||||
String docId,
|
||||
String source,
|
||||
/** created | updated | unchanged */
|
||||
String status,
|
||||
Integer chunkCount,
|
||||
List<String> warnings,
|
||||
Map<String, Object> frontmatter
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagTaskAccepted(
|
||||
String taskId,
|
||||
String status
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagTaskStatus(
|
||||
String taskId,
|
||||
String status,
|
||||
Integer documents,
|
||||
String detail,
|
||||
String createdAt,
|
||||
String finishedAt
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagStats(
|
||||
String collection,
|
||||
Integer rowCount
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagHealth(
|
||||
String status,
|
||||
String version,
|
||||
Map<String, String> checks
|
||||
) {
|
||||
}
|
||||
|
||||
@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record PyRagErrorEnvelope(ErrorBody error) {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public record ErrorBody(String code, String message, List<Map<String, Object>> details) {
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.superbiz.agent.client;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
/**
|
||||
* py-rag 调用异常：携带契约错误码（E_*）与 HTTP 状态。
|
||||
*
|
||||
* <p>调用方按错误码分支：422 参数/数据问题不重试；
|
||||
* 409 E_REBUILD_IN_PROGRESS 延迟重试；网络异常（E_NETWORK）可安全重试。</p>
|
||||
*/
|
||||
@Getter
|
||||
public class PyRagClientException extends RuntimeException {
|
||||
|
||||
/** 契约错误码：E_INVALID_REQUEST / E_FRONTMATTER_INVALID / E_REBUILD_IN_PROGRESS / E_NETWORK 等 */
|
||||
private final String code;
|
||||
|
||||
/** HTTP 状态码；网络层异常（未拿到响应）为 null */
|
||||
private final Integer httpStatus;
|
||||
|
||||
public PyRagClientException(String code, String message, Integer httpStatus, Throwable cause) {
|
||||
super("[" + code + "] " + message, cause);
|
||||
this.code = code;
|
||||
this.httpStatus = httpStatus;
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* 文档分片配置
|
||||
*/
|
||||
@Getter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "document.chunk")
|
||||
public class DocumentChunkConfig {
|
||||
|
||||
/**
|
||||
* 每个分片的最大字符数（保留向后兼容）
|
||||
*/
|
||||
private int maxSize = 800;
|
||||
|
||||
/**
|
||||
* 分片之间的重叠字符数
|
||||
*/
|
||||
private int overlap = 100;
|
||||
|
||||
/**
|
||||
* 每个分片的最大 token 数（中文~1:1，英文~0.25:1）
|
||||
* 替代 maxSize 作为切割触发器
|
||||
*/
|
||||
private int maxTokens = 500;
|
||||
|
||||
/**
|
||||
* 硬上限 token 数 = maxTokens × 1.2
|
||||
* 仅在不可中断上下文（列表、代码块）内触发
|
||||
*/
|
||||
private int maxTokensHard = 600;
|
||||
|
||||
public void setMaxSize(int maxSize) {
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
public void setOverlap(int overlap) {
|
||||
this.overlap = overlap;
|
||||
}
|
||||
|
||||
public void setMaxTokens(int maxTokens) {
|
||||
this.maxTokens = maxTokens;
|
||||
}
|
||||
|
||||
public void setMaxTokensHard(int maxTokensHard) {
|
||||
this.maxTokensHard = maxTokensHard;
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* Milvus 知识路径配置说明（无额外 Bean 装配）。
|
||||
*
|
||||
* <p>知识库 RAG 唯一实现：{@link MilvusHybridKnowledgeStore}（{@code MilvusClientV2}）。</p>
|
||||
* <ul>
|
||||
* <li>支持 dense 与 dense+BM25 {@code hybridSearch}+RRF。</li>
|
||||
* <li>不再为知识路径创建 legacy {@code MilvusServiceClient} Bean。</li>
|
||||
* <li>Spring AI {@code VectorStore} starter 仍可存在于 classpath，但只作 sidecar，
|
||||
* 不作 lookup_knowledge 主路径（starter 无 BM25 hybrid API）。</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Configuration
|
||||
public class MilvusConfig {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(MilvusConfig.class);
|
||||
|
||||
public MilvusConfig() {
|
||||
logger.info("Milvus knowledge path: MilvusClientV2 hybrid store only (legacy SDK search disabled)");
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "milvus")
|
||||
public class MilvusProperties {
|
||||
|
||||
private String host = "localhost";
|
||||
private Integer port = 19530;
|
||||
private String username = "";
|
||||
private String password = "";
|
||||
private String database = "default";
|
||||
private Long timeout = 10000L;
|
||||
private String token = "";
|
||||
private boolean secure = false;
|
||||
private int vectorDim = 1024;
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
public Integer getPort() {
|
||||
return port;
|
||||
}
|
||||
|
||||
public void setPort(Integer port) {
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
|
||||
public String getPassword() {
|
||||
return password;
|
||||
}
|
||||
|
||||
public void setPassword(String password) {
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public Long getTimeout() {
|
||||
return timeout;
|
||||
}
|
||||
|
||||
public void setTimeout(Long timeout) {
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
public String getToken() {
|
||||
return token;
|
||||
}
|
||||
|
||||
public void setToken(String token) {
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
public boolean isSecure() {
|
||||
return secure;
|
||||
}
|
||||
|
||||
public void setSecure(boolean secure) {
|
||||
this.secure = secure;
|
||||
}
|
||||
|
||||
public int getVectorDim() {
|
||||
return vectorDim;
|
||||
}
|
||||
|
||||
public void setVectorDim(int vectorDim) {
|
||||
this.vectorDim = vectorDim;
|
||||
}
|
||||
|
||||
public String getAddress() {
|
||||
return host + ":" + port;
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
@@ -19,11 +17,11 @@ import org.springframework.context.annotation.Primary;
|
||||
* <pre>{@code
|
||||
* model-routing:
|
||||
* chat: deepseek
|
||||
* embedding: siliconflow
|
||||
* }</pre>
|
||||
* <p>
|
||||
* 匹配优先级：Bean 名 > 类名（均不区分大小写）。
|
||||
* 切换模型只改 yml + pom + 对应 api-key，Java 代码不动。
|
||||
* （Embedding 路由已随 RAG 模块抽离至 py-rag 服务端，此处仅路由 Chat。）
|
||||
*/
|
||||
@Configuration
|
||||
public class ModelRoutingConfig {
|
||||
@@ -33,9 +31,6 @@ public class ModelRoutingConfig {
|
||||
@Value("${model-routing.chat:deepseek}")
|
||||
private String chatKeyword;
|
||||
|
||||
@Value("${model-routing.embedding:siliconflow}")
|
||||
private String embeddingKeyword;
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public ChatModel chatModel(List<ChatModel> chatModels) {
|
||||
@@ -53,33 +48,6 @@ public class ModelRoutingConfig {
|
||||
return chatModels.get(0);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Primary
|
||||
public EmbeddingModel embeddingModel(Map<String, EmbeddingModel> embeddingBeans) {
|
||||
log.info("Embedding 路由: keyword='{}', 可用: {}", embeddingKeyword, embeddingBeans.keySet());
|
||||
|
||||
// 先按 Bean 名匹配
|
||||
for (Map.Entry<String, EmbeddingModel> entry : embeddingBeans.entrySet()) {
|
||||
if (containsIgnoreCase(entry.getKey(), embeddingKeyword)) {
|
||||
log.info(" → Bean 名匹配: {} → {}", entry.getKey(),
|
||||
entry.getValue().getClass().getSimpleName());
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
// 再按类名匹配
|
||||
for (EmbeddingModel em : embeddingBeans.values()) {
|
||||
if (matches(em.getClass(), embeddingKeyword)) {
|
||||
log.info(" → 类名匹配: {}", em.getClass().getSimpleName());
|
||||
return em;
|
||||
}
|
||||
}
|
||||
|
||||
var first = embeddingBeans.values().iterator().next();
|
||||
log.warn(" → 未匹配, 回退到 {}", first.getClass().getSimpleName());
|
||||
return first;
|
||||
}
|
||||
|
||||
private boolean matches(Class<?> clazz, String keyword) {
|
||||
return containsIgnoreCase(clazz.getName(), keyword)
|
||||
|| containsIgnoreCase(clazz.getSimpleName(), keyword);
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* py-rag 知识服务接入配置。
|
||||
*
|
||||
* <p>超时矩阵来自《py-rag 知识服务 · Java 接入文档》第 6 节：
|
||||
* 服务含 embedding/rerank 外呼，search 正常 300–800ms、ingest 正常 1–5s。</p>
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "pyrag")
|
||||
public class PyRagProperties {
|
||||
|
||||
/** py-rag 服务根地址，如 http://py-rag:8000 */
|
||||
private String baseUrl = "http://localhost:8000";
|
||||
|
||||
/** 连接超时（毫秒），全端点统一 */
|
||||
private int connectTimeoutMs = 3000;
|
||||
|
||||
/** /api/v1/search 读取超时（毫秒） */
|
||||
private int searchReadTimeoutMs = 5000;
|
||||
|
||||
/** /api/v1/documents:ingest 读取超时（毫秒） */
|
||||
private int ingestReadTimeoutMs = 30000;
|
||||
|
||||
/** rebuild/tasks/stats/health 读取超时（毫秒） */
|
||||
private int defaultReadTimeoutMs = 10000;
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import lombok.Getter;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@Getter
|
||||
@Configuration
|
||||
@ConfigurationProperties(prefix = "rag.sidecar.spring-ai")
|
||||
public class RagSidecarProperties {
|
||||
|
||||
private boolean enabled = false;
|
||||
|
||||
private int contentPreviewLimit = 300;
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public void setContentPreviewLimit(int contentPreviewLimit) {
|
||||
this.contentPreviewLimit = contentPreviewLimit;
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.MetadataMode;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingModel;
|
||||
import org.springframework.ai.openai.OpenAiEmbeddingOptions;
|
||||
import org.springframework.ai.openai.api.OpenAiApi;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestClient;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
|
||||
/**
|
||||
* SiliconFlow Embedding 配置（BGE-M3, OpenAI 兼容协议, 1024维）
|
||||
* <p>
|
||||
* Chat 走 DeepSeek、Embedding 走 SiliconFlow，两者都是 OpenAI 兼容但地址不同，
|
||||
* 因此单独为 SiliconFlow 创建 OpenAiApi + EmbeddingModel Bean。
|
||||
*/
|
||||
@Configuration
|
||||
public class SiliconFlowEmbeddingConfig {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SiliconFlowEmbeddingConfig.class);
|
||||
|
||||
@Value("${siliconflow.api-key}")
|
||||
private String apiKey;
|
||||
|
||||
@Value("${siliconflow.base-url}")
|
||||
private String baseUrl;
|
||||
|
||||
@Value("${siliconflow.embedding.model}")
|
||||
private String model;
|
||||
|
||||
@Bean
|
||||
public OpenAiApi siliconFlowApi(RestClient.Builder restClientBuilder, WebClient.Builder webClientBuilder) {
|
||||
log.info("创建 SiliconFlow OpenAiApi: {}", baseUrl);
|
||||
return OpenAiApi.builder()
|
||||
.baseUrl(baseUrl)
|
||||
.apiKey(apiKey)
|
||||
.restClientBuilder(restClientBuilder)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public EmbeddingModel siliconFlowEmbeddingModel(OpenAiApi siliconFlowApi) {
|
||||
log.info("创建 SiliconFlow EmbeddingModel, model: {}", model);
|
||||
return new OpenAiEmbeddingModel(siliconFlowApi, MetadataMode.EMBED,
|
||||
OpenAiEmbeddingOptions.builder()
|
||||
.model(model)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package com.superbiz.agent.constant;
|
||||
|
||||
public class MilvusConstants {
|
||||
|
||||
/**
|
||||
* Milvus 数据库名称
|
||||
*/
|
||||
public static final String MILVUS_DB_NAME = "default";
|
||||
|
||||
/**
|
||||
* Default knowledge collection name (dense + BM25).
|
||||
* Overridable via {@code milvus.collection}.
|
||||
*/
|
||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
||||
|
||||
/**
|
||||
* Alias kept for readability in hybrid-related code.
|
||||
*/
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
||||
|
||||
/**
|
||||
* 向量维度（豆包 embedding 模型的维度）
|
||||
*/
|
||||
public static final int VECTOR_DIM = 1024; // 豆包模型返回1024维向量
|
||||
|
||||
/**
|
||||
* ID字段最大长度
|
||||
*/
|
||||
public static final int ID_MAX_LENGTH = 256;
|
||||
|
||||
/**
|
||||
* Content字段最大长度
|
||||
*/
|
||||
public static final int CONTENT_MAX_LENGTH = 8192;
|
||||
|
||||
/**
|
||||
* 默认分片数
|
||||
*/
|
||||
public static final int DEFAULT_SHARD_NUMBER = 2;
|
||||
|
||||
private MilvusConstants() {
|
||||
// 工具类，禁止实例化
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.config.FileUploadConfig;
|
||||
import com.superbiz.agent.dto.FileUploadRes;
|
||||
import com.superbiz.agent.service.VectorIndexService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -29,10 +29,11 @@ public class FileUploadController {
|
||||
private FileUploadConfig fileUploadConfig;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
private PyRagClient pyRagClient;
|
||||
|
||||
@PostMapping(value = "/api/upload", consumes = "multipart/form-data")
|
||||
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file) {
|
||||
public ResponseEntity<?> upload(@RequestParam("file") MultipartFile file,
|
||||
@RequestParam(value = "category", required = false) String category) {
|
||||
if (file.isEmpty()) {
|
||||
return ResponseEntity.badRequest().body("文件不能为空");
|
||||
}
|
||||
@@ -68,15 +69,17 @@ public class FileUploadController {
|
||||
|
||||
logger.info("文件上传成功: {}", filePath);
|
||||
|
||||
// 文件上传成功后，自动调用向量索引服务
|
||||
// 转发 py-rag 入库（同内容重传返回 unchanged）。入库失败不影响上传成功语义。
|
||||
try {
|
||||
logger.info("开始为上传文件创建向量索引: {}", filePath);
|
||||
vectorIndexService.indexSingleFile(filePath.toString());
|
||||
logger.info("向量索引创建成功: {}", filePath);
|
||||
String ingestCategory = (category == null || category.isBlank()) ? "default" : category;
|
||||
logger.info("开始 py-rag 入库: {}, category={}", filePath, ingestCategory);
|
||||
var ingest = pyRagClient.ingest(originalFilename, file.getBytes(), file.getContentType(),
|
||||
ingestCategory, null, null, null);
|
||||
logger.info("py-rag 入库完成: docId={}, status={}, chunks={}",
|
||||
ingest.docId(), ingest.status(), ingest.chunkCount());
|
||||
} catch (Exception e) {
|
||||
logger.error("向量索引创建失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
||||
// 注意：即使索引失败，文件上传仍然成功，只是记录错误日志
|
||||
// 可以根据业务需求决定是否要删除文件或返回错误
|
||||
logger.error("py-rag 入库失败: {}, 错误: {}", filePath, e.getMessage(), e);
|
||||
// 注意：即使入库失败，文件上传仍然成功，只是记录错误日志
|
||||
}
|
||||
|
||||
FileUploadRes response = new FileUploadRes(
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.service.KnowledgeBaseInitService;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 知识库管理控制器
|
||||
* 提供知识库初始化、查询等接口
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/api/knowledge")
|
||||
public class KnowledgeBaseController {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseController.class);
|
||||
|
||||
@Autowired
|
||||
private KnowledgeBaseInitService initService;
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
* 扫描 knowledge_base 目录下的所有文档，去重后批量导入到数据库和 Milvus
|
||||
*
|
||||
* @param force 是否强制重新导入（跳过去重检查）
|
||||
* @return 初始化结果
|
||||
*/
|
||||
@PostMapping("/init")
|
||||
public ResponseEntity<?> initKnowledgeBase(@RequestParam(defaultValue = "false") boolean force) {
|
||||
logger.info("收到知识库初始化请求, force={}", force);
|
||||
|
||||
try {
|
||||
KnowledgeBaseInitService.InitResult result = initService.initializeKnowledgeBase(force);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", true);
|
||||
response.put("message", "知识库初始化完成");
|
||||
response.put("scanned", result.getScanned());
|
||||
response.put("skipped", result.getSkipped());
|
||||
response.put("inserted", result.getInserted());
|
||||
response.put("failed", result.getFailed());
|
||||
response.put("details", result.getDetails());
|
||||
|
||||
logger.info("知识库初始化成功: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("知识库初始化失败", e);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "初始化失败: " + e.getMessage());
|
||||
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引，
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作：会删除 {@code milvus.collection}（默认 {@code biz}）与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
public ResponseEntity<?> rebuildHybrid(
|
||||
@RequestParam(defaultValue = "") String confirm) {
|
||||
if (!"REBUILD".equals(confirm)) {
|
||||
Map<String, Object> rejected = new HashMap<>();
|
||||
rejected.put("success", false);
|
||||
rejected.put("message", "拒绝执行：请传 confirm=REBUILD 以确认清空并重建");
|
||||
rejected.put("hint", "POST /api/knowledge/rebuild-hybrid?confirm=REBUILD");
|
||||
return ResponseEntity.badRequest().body(rejected);
|
||||
}
|
||||
|
||||
logger.warn("收到 hybrid 知识库全量重建请求 confirm={}", confirm);
|
||||
try {
|
||||
KnowledgeBaseInitService.RebuildResult result = initService.rebuildHybridFromKnowledgeBase();
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", result.isSuccess());
|
||||
response.put("message", result.isSuccess()
|
||||
? "hybrid 知识库重建完成"
|
||||
: "hybrid 知识库重建结束，但存在失败项");
|
||||
response.put("collection", result.getCollection());
|
||||
response.put("basePath", result.getBasePath());
|
||||
response.put("milvus", result.getMilvus());
|
||||
response.put("mysqlDocumentsBefore", result.getMysqlDocumentsBefore());
|
||||
response.put("mysqlDocumentsAfterClear", result.getMysqlDocumentsAfterClear());
|
||||
response.put("mysqlDocumentsAfterInit", result.getMysqlDocumentsAfterInit());
|
||||
response.put("l0IndexSizeAfterClear", result.getL0IndexSizeAfterClear());
|
||||
response.put("l0IndexSizeAfterInit", result.getL0IndexSizeAfterInit());
|
||||
if (result.getInit() != null) {
|
||||
response.put("scanned", result.getInit().getScanned());
|
||||
response.put("skipped", result.getInit().getSkipped());
|
||||
response.put("inserted", result.getInit().getInserted());
|
||||
response.put("failed", result.getInit().getFailed());
|
||||
response.put("details", result.getInit().getDetails());
|
||||
}
|
||||
return result.isSuccess()
|
||||
? ResponseEntity.ok(response)
|
||||
: ResponseEntity.status(500).body(response);
|
||||
} catch (Exception e) {
|
||||
logger.error("hybrid 知识库重建失败", e);
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "重建失败: " + e.getMessage());
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询知识库统计信息
|
||||
*
|
||||
* @return 统计信息
|
||||
*/
|
||||
@GetMapping("/stats")
|
||||
public ResponseEntity<?> getStats() {
|
||||
try {
|
||||
KnowledgeBaseInitService.Stats stats = initService.getStats();
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", true);
|
||||
response.put("totalDocuments", stats.getTotalDocuments());
|
||||
response.put("totalVectors", stats.getTotalVectors());
|
||||
response.put("categories", stats.getCategoryCount());
|
||||
|
||||
return ResponseEntity.ok(response);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("查询统计信息失败", e);
|
||||
|
||||
Map<String, Object> response = new HashMap<>();
|
||||
response.put("success", false);
|
||||
response.put("message", "查询失败: " + e.getMessage());
|
||||
|
||||
return ResponseEntity.internalServerError().body(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import io.milvus.v2.service.collection.response.ListCollectionsResp;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Milvus health check using the single V2 knowledge backend.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/milvus")
|
||||
public class MilvusCheckController {
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@GetMapping("/health")
|
||||
public ResponseEntity<Map<String, Object>> simpleHealth() {
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
try {
|
||||
ListCollectionsResp response = knowledgeStore.client().listCollections();
|
||||
result.put("message", "ok");
|
||||
result.put("backend", "milvus-client-v2");
|
||||
result.put("knowledgeCollection", knowledgeStore.collectionName());
|
||||
result.put("collections", response == null ? null : response.getCollectionNames());
|
||||
return ResponseEntity.ok(result);
|
||||
} catch (Exception e) {
|
||||
result.put("error", e.getMessage());
|
||||
return ResponseEntity.status(503).body(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import com.superbiz.agent.dto.Result;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 文档检索控制器（测试用）
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/search")
|
||||
public class SearchController {
|
||||
|
||||
@Autowired
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
/**
|
||||
* 搜索相似文档
|
||||
*/
|
||||
@GetMapping("/similar")
|
||||
public Result<List<VectorSearchService.SearchResult>> searchSimilar(
|
||||
@RequestParam("query") String query,
|
||||
@RequestParam(value = "topK", defaultValue = "5") int topK,
|
||||
@RequestParam(value = "category", required = false) String category
|
||||
) {
|
||||
try {
|
||||
log.info("收到检索请求，query: {}, topK: {}, category: {}", query, topK, category);
|
||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(query, topK, category);
|
||||
log.info("检索完成，返回 {} 条结果", results.size());
|
||||
return Result.success(results);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("检索失败", e);
|
||||
return Result.error(500, "检索失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* 文档分片
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class DocumentChunk {
|
||||
|
||||
/**
|
||||
* 分片内容
|
||||
*/
|
||||
private String content;
|
||||
|
||||
/**
|
||||
* 分片在原文档中的起始位置
|
||||
*/
|
||||
private int startOffset;
|
||||
|
||||
/**
|
||||
* 分片在原文档中的结束位置
|
||||
*/
|
||||
private int endOffset;
|
||||
|
||||
/**
|
||||
* 分片序号（从0开始）
|
||||
*/
|
||||
private int chunkIndex;
|
||||
|
||||
/**
|
||||
* 分片标题或上下文信息
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 面包屑导航（完整标题层级路径）
|
||||
* 例如: "故障诊断流程规范 > 应急响应流程 > 1. 初步评估"
|
||||
*/
|
||||
private String breadcrumb;
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Frontmatter 数据模型
|
||||
* 用于解析 Markdown 文件头的 YAML frontmatter
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class Frontmatter {
|
||||
|
||||
/**
|
||||
* 文档标题（必填）
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 关键词列表（必填，用于 L0 精确匹配）
|
||||
*/
|
||||
private List<String> keywords;
|
||||
|
||||
/**
|
||||
* 文档摘要（必填）
|
||||
*/
|
||||
private String summary;
|
||||
|
||||
/**
|
||||
* 文档类别（可选）
|
||||
*/
|
||||
private String category;
|
||||
|
||||
private String source;
|
||||
|
||||
private String breadcrumb;
|
||||
|
||||
@JsonProperty("kb_scope")
|
||||
private String kbScope;
|
||||
|
||||
/**
|
||||
* 章节锚点（预留字段，MVP 不使用）
|
||||
* Key: 章节标题，Value: 章节 Markdown 标题
|
||||
*/
|
||||
private Map<String, String> sections;
|
||||
|
||||
/**
|
||||
* 版本号（预留字段）
|
||||
*/
|
||||
private String version;
|
||||
|
||||
/**
|
||||
* 作者（预留字段）
|
||||
*/
|
||||
private String author;
|
||||
|
||||
/**
|
||||
* 最后更新日期（预留字段）
|
||||
*/
|
||||
private LocalDate lastUpdated;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用（LLM 上传时自动生成）
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机（LLM 上传时自动生成）
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package com.superbiz.agent.dto;
|
||||
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 知识库索引条目
|
||||
* L0 内存索引使用的数据结构
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
public class KnowledgeEntry {
|
||||
|
||||
/**
|
||||
* 文件路径（如：knowledge_base/api/payment-errors.md）
|
||||
*/
|
||||
private String filePath;
|
||||
|
||||
/**
|
||||
* 文档标题
|
||||
*/
|
||||
private String title;
|
||||
|
||||
/**
|
||||
* 关键词列表（用于精确匹配）
|
||||
*/
|
||||
private List<String> keywords;
|
||||
|
||||
/**
|
||||
* 文档摘要
|
||||
*/
|
||||
private String summary;
|
||||
|
||||
/**
|
||||
* 文档类别（如：api、domain、troubleshooting）
|
||||
*/
|
||||
private String category;
|
||||
|
||||
private String kbScope;
|
||||
|
||||
/**
|
||||
* 章节锚点（预留字段，MVP 不使用）
|
||||
*/
|
||||
private Map<String, String> sections;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
@@ -6,9 +6,10 @@ import lombok.Data;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 检索前 query understanding 的输出（L0 -> pipeline 控制面）。
|
||||
* 检索 pipeline 控制面参数。
|
||||
*
|
||||
* <p>由 {@code KnowledgeQueryTransformer} 生成，供 L1 过滤、规则 rerank 与 trace 使用。
|
||||
* <p>L0 query 理解已下沉 py-rag 服务端；当前 {@code originalQuery} = {@code rewrittenQuery}、
|
||||
* hint 字段恒为空、{@code categoryFilter} 恒为 null，结构保留供后处理与 trace 使用。
|
||||
* 不是 Agent 可见契约。</p>
|
||||
*/
|
||||
@Data
|
||||
|
||||
@@ -1,446 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.config.DocumentChunkConfig;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* 文档切片服务（RAG 入库前处理）。
|
||||
*
|
||||
* <p>把长 Markdown/文本切成带 title/breadcrumb 的 {@link com.superbiz.agent.dto.DocumentChunk}，
|
||||
* 供 {@link VectorIndexService} 向量化。</p>
|
||||
*
|
||||
* <h3>策略摘要</h3>
|
||||
* <ol>
|
||||
* <li>先按 Markdown 标题分 section，并维护 breadcrumb 层级</li>
|
||||
* <li>section 过长再按段落累积；用 token 估算做软边界 / 硬上限</li>
|
||||
* <li>尽量不在有序/无序列表或未闭合代码块中间切断</li>
|
||||
* <li>相邻 chunk 保留 overlap，减轻边界语义断裂</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>检索命中单个 chunk 后，当前主链路不会自动回补同章节相邻 chunk
|
||||
* （上下文重建仍是后续增强点）。</p>
|
||||
*/
|
||||
@Service
|
||||
public class DocumentChunkService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DocumentChunkService.class);
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkConfig chunkConfig;
|
||||
|
||||
/**
|
||||
* 智能分片文档
|
||||
* 优先按照标题、段落边界进行分割，保持语义完整性
|
||||
*
|
||||
* @param content 文档内容
|
||||
* @param filePath 文件路径（用于日志）
|
||||
* @return 文档分片列表
|
||||
*/
|
||||
public List<DocumentChunk> chunkDocument(String content, String filePath) {
|
||||
List<DocumentChunk> chunks = new ArrayList<>();
|
||||
|
||||
if (content == null || content.trim().isEmpty()) {
|
||||
logger.warn("文档内容为空: {}", filePath);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 1. 首先尝试按标题分割（Markdown格式）
|
||||
List<Section> sections = splitByHeadings(content);
|
||||
|
||||
// 2. 对每个章节进行进一步分片
|
||||
int globalChunkIndex = 0;
|
||||
for (Section section : sections) {
|
||||
List<DocumentChunk> sectionChunks = chunkSection(section, globalChunkIndex);
|
||||
chunks.addAll(sectionChunks);
|
||||
globalChunkIndex += sectionChunks.size();
|
||||
}
|
||||
|
||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按照 Markdown 标题分割文档，同时构建面包屑层级路径
|
||||
*/
|
||||
private List<Section> splitByHeadings(String content) {
|
||||
List<Section> sections = new ArrayList<>();
|
||||
|
||||
// 匹配 Markdown 标题：# 标题, ## 标题, ### 标题等
|
||||
Pattern headingPattern = Pattern.compile("^(#{1,6})\\s+(.+)$", Pattern.MULTILINE);
|
||||
Matcher matcher = headingPattern.matcher(content);
|
||||
|
||||
// 标题层级栈：维护当前标题的完整路径
|
||||
List<String> headingStack = new ArrayList<>();
|
||||
int lastEnd = 0;
|
||||
String currentBreadcrumb = null;
|
||||
|
||||
while (matcher.find()) {
|
||||
int level = matcher.group(1).length(); // #→1, ##→2, ###→3 ...
|
||||
String title = matcher.group(2).trim();
|
||||
|
||||
// 保存上一个章节
|
||||
if (lastEnd < matcher.start()) {
|
||||
String sectionContent = content.substring(lastEnd, matcher.start()).trim();
|
||||
if (!sectionContent.isEmpty()) {
|
||||
sections.add(new Section(
|
||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
||||
level,
|
||||
currentBreadcrumb,
|
||||
sectionContent,
|
||||
lastEnd));
|
||||
}
|
||||
}
|
||||
|
||||
// 维护层级栈：同级别或更高级别 → 弹出，低级 → 追加
|
||||
while (!headingStack.isEmpty() && headingStack.size() >= level) {
|
||||
headingStack.remove(headingStack.size() - 1);
|
||||
}
|
||||
headingStack.add(title);
|
||||
currentBreadcrumb = String.join(" > ", headingStack);
|
||||
lastEnd = matcher.start();
|
||||
}
|
||||
|
||||
// 添加最后一个章节
|
||||
if (lastEnd < content.length()) {
|
||||
String sectionContent = content.substring(lastEnd).trim();
|
||||
if (!sectionContent.isEmpty()) {
|
||||
sections.add(new Section(
|
||||
headingStack.isEmpty() ? null : headingStack.get(headingStack.size() - 1),
|
||||
headingStack.size(),
|
||||
currentBreadcrumb,
|
||||
sectionContent,
|
||||
lastEnd));
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有找到任何标题，将整个文档作为一个章节
|
||||
if (sections.isEmpty()) {
|
||||
sections.add(new Section(null, 0, null, content, 0));
|
||||
}
|
||||
|
||||
return sections;
|
||||
}
|
||||
|
||||
/**
|
||||
* 对单个章节进行分片
|
||||
* <p>
|
||||
* 核心改造（Phase 1）：
|
||||
* - Token 估算替代字符计数
|
||||
* - 感知有序/无序列表结构，不在列表中间切断
|
||||
* - 软边界（maxTokens）+ 硬上限（maxTokensHard）双重控制
|
||||
* - 修复 currentStartIndex 漂移：用段落原始位置而非手工推算
|
||||
*/
|
||||
private List<DocumentChunk> chunkSection(Section section, int startChunkIndex) {
|
||||
List<DocumentChunk> chunks = new ArrayList<>();
|
||||
String content = section.content;
|
||||
String title = section.title;
|
||||
String breadcrumb = section.breadcrumb;
|
||||
|
||||
// 短章节直接作为一个分片（用 token 估算替代字符数做短路判断）
|
||||
if (content.length() <= chunkConfig.getMaxSize()
|
||||
&& estimateTokens(content) <= chunkConfig.getMaxTokens()) {
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(content)
|
||||
.startOffset(section.startIndex)
|
||||
.endOffset(section.startIndex + content.length())
|
||||
.chunkIndex(startChunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 章节内容较长，需要进一步分片
|
||||
List<String> paragraphs = splitByParagraphs(content);
|
||||
if (paragraphs.isEmpty()) {
|
||||
return chunks;
|
||||
}
|
||||
|
||||
// 定位每个段落在 section.content 中的位置（修复 index 漂移）
|
||||
List<ParagraphPos> paraPositions = locateParagraphPositions(paragraphs, content);
|
||||
|
||||
// 当前分片的段落范围
|
||||
int chunkParaStart = 0; // 当前分片第一个段落的索引（在 paragraphs 中）
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
int tokenCount = 0;
|
||||
int chunkIndex = startChunkIndex;
|
||||
|
||||
for (int i = 0; i < paragraphs.size(); i++) {
|
||||
String paragraph = paragraphs.get(i);
|
||||
int paraTokens = estimateTokens(paragraph);
|
||||
|
||||
// 判断是否需要切分
|
||||
if (buffer.length() > 0 && tokenCount + paraTokens > chunkConfig.getMaxTokens()) {
|
||||
|
||||
// 检查是否处于不可中断的上下文中
|
||||
if (isInUnbreakableContext(buffer.toString(), paragraph)) {
|
||||
// 硬上限保护：即使不可中断也不能无限膨胀
|
||||
if (tokenCount + paraTokens > chunkConfig.getMaxTokensHard()) {
|
||||
logger.debug(" 触及硬上限 ({} tokens)，强制切分", tokenCount + paraTokens);
|
||||
chunkParaStart = saveChunkAndGetNextStart(
|
||||
chunks, section, paraPositions,
|
||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
||||
chunkIndex++;
|
||||
|
||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
||||
String overlap = getOverlapText(prevChunkContent);
|
||||
buffer = new StringBuilder(overlap);
|
||||
tokenCount = estimateTokens(overlap);
|
||||
}
|
||||
// 否则：容忍超出（软边界）
|
||||
} else {
|
||||
// 安全切点：段落边界
|
||||
chunkParaStart = saveChunkAndGetNextStart(
|
||||
chunks, section, paraPositions,
|
||||
chunkParaStart, i, title, breadcrumb, chunkIndex);
|
||||
chunkIndex++;
|
||||
|
||||
// 新分片以重叠文本开头
|
||||
String prevChunkContent = chunks.get(chunks.size() - 1).getContent();
|
||||
String overlap = getOverlapText(prevChunkContent);
|
||||
buffer = new StringBuilder(overlap);
|
||||
tokenCount = estimateTokens(overlap);
|
||||
}
|
||||
}
|
||||
|
||||
buffer.append(paragraph).append("\n\n");
|
||||
tokenCount += paraTokens;
|
||||
}
|
||||
|
||||
// 保存最后一个分片
|
||||
if (buffer.length() > 0 && chunkParaStart < paragraphs.size()) {
|
||||
String chunkContent = buffer.toString().trim();
|
||||
int actualStart = paraPositions.get(chunkParaStart).start;
|
||||
int actualEnd = paraPositions.get(paragraphs.size() - 1).end;
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(chunkContent)
|
||||
.startOffset(section.startIndex + actualStart)
|
||||
.endOffset(section.startIndex + actualEnd)
|
||||
.chunkIndex(chunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存当前分块，返回下一个分块的起始段落索引
|
||||
* <p>
|
||||
* 从 section.content 中提取原始文本（而非手工拼装），修复 index 漂移问题
|
||||
*/
|
||||
private int saveChunkAndGetNextStart(
|
||||
List<DocumentChunk> chunks,
|
||||
Section section,
|
||||
List<ParagraphPos> paraPositions,
|
||||
int fromPara,
|
||||
int toPara,
|
||||
String title,
|
||||
String breadcrumb,
|
||||
int chunkIndex) {
|
||||
|
||||
int actualStart = paraPositions.get(fromPara).start;
|
||||
int actualEnd = paraPositions.get(toPara - 1).end;
|
||||
String originalText = section.content.substring(actualStart, actualEnd);
|
||||
|
||||
DocumentChunk chunk = DocumentChunk.builder()
|
||||
.content(originalText)
|
||||
.startOffset(section.startIndex + actualStart)
|
||||
.endOffset(section.startIndex + actualEnd)
|
||||
.chunkIndex(chunkIndex)
|
||||
.title(title)
|
||||
.breadcrumb(breadcrumb)
|
||||
.build();
|
||||
chunks.add(chunk);
|
||||
|
||||
return toPara; // 下一个分块的起始段落索引
|
||||
}
|
||||
|
||||
/**
|
||||
* 按段落分割文本
|
||||
*/
|
||||
private List<String> splitByParagraphs(String content) {
|
||||
List<String> paragraphs = new ArrayList<>();
|
||||
|
||||
// 按双换行符分割段落
|
||||
String[] parts = content.split("\n\n+");
|
||||
for (String part : parts) {
|
||||
String trimmed = part.trim();
|
||||
if (!trimmed.isEmpty()) {
|
||||
paragraphs.add(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
return paragraphs;
|
||||
}
|
||||
|
||||
/**
|
||||
* 定位每个段落在原始文本中的字符偏移
|
||||
*/
|
||||
private List<ParagraphPos> locateParagraphPositions(List<String> paragraphs, String sectionContent) {
|
||||
List<ParagraphPos> positions = new ArrayList<>();
|
||||
int searchFrom = 0;
|
||||
for (String p : paragraphs) {
|
||||
int idx = sectionContent.indexOf(p, searchFrom);
|
||||
if (idx >= 0) {
|
||||
positions.add(new ParagraphPos(idx, idx + p.length()));
|
||||
searchFrom = idx + p.length();
|
||||
} else {
|
||||
// fallback: 段落在原文中找不到（不应该发生）
|
||||
positions.add(new ParagraphPos(searchFrom, searchFrom + p.length()));
|
||||
searchFrom += p.length();
|
||||
}
|
||||
}
|
||||
return positions;
|
||||
}
|
||||
|
||||
/**
|
||||
* 启发式 token 估算（无需外部依赖）
|
||||
* <p>
|
||||
* 中文(BMP): ~1 字符/token
|
||||
* 英文/数字/标点: ~4 字符/token
|
||||
* 空白字符忽略
|
||||
*/
|
||||
private int estimateTokens(String text) {
|
||||
int nonCjkCount = 0;
|
||||
int cjkCount = 0;
|
||||
for (char c : text.toCharArray()) {
|
||||
if (Character.isWhitespace(c)) {
|
||||
continue;
|
||||
}
|
||||
Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
|
||||
if (block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
|
||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
|
||||
|| block == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
|
||||
|| block == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS) {
|
||||
cjkCount++;
|
||||
} else {
|
||||
nonCjkCount++;
|
||||
}
|
||||
}
|
||||
return cjkCount + (nonCjkCount + 3) / 4; // 非中文每 4 字符算 1 token，向上取整
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断当前段落是否属于不可中断的结构
|
||||
* <p>
|
||||
* 不可中断结构包括：
|
||||
* - 有序列表项（"1. ", "2. " 格式）
|
||||
* - 无序列表项（"- " 或 "* " 格式）
|
||||
* - 未闭合的代码块（``` 内）
|
||||
*/
|
||||
private boolean isInUnbreakableContext(String buffer, String nextParagraph) {
|
||||
// 有序列表：判断 buffer 末尾和下一段是否都是列表项
|
||||
if (nextParagraph.matches("^\\d{1,2}\\.\\s.*")) {
|
||||
String lastLine = getLastNonEmptyLine(buffer);
|
||||
if (lastLine != null && lastLine.matches("^\\d{1,2}\\.\\s.*")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// 无序列表："- " 或 "* " 格式
|
||||
if (nextParagraph.matches("^[-*]\\s.*")) {
|
||||
String lastLine = getLastNonEmptyLine(buffer);
|
||||
if (lastLine != null && lastLine.matches("^[-*]\\s.*")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
// 代码块：``` 未闭合
|
||||
if (buffer.contains("```")) {
|
||||
int count = 0;
|
||||
for (int i = 0; i <= buffer.length() - 3; i++) {
|
||||
if (buffer.substring(i).startsWith("```")) {
|
||||
count++;
|
||||
i += 2;
|
||||
}
|
||||
}
|
||||
if (count % 2 == 1) {
|
||||
return true; // 奇数个 ``` → 在代码块内部
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 buffer 中最后一行非空白文本
|
||||
*/
|
||||
private String getLastNonEmptyLine(String buffer) {
|
||||
String[] lines = buffer.split("\n");
|
||||
for (int i = lines.length - 1; i >= 0; i--) {
|
||||
String line = lines[i].trim();
|
||||
if (!line.isEmpty()) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取重叠文本
|
||||
* 从文本末尾提取指定长度的内容作为下一个分片的开头
|
||||
*/
|
||||
private String getOverlapText(String text) {
|
||||
int overlapSize = Math.min(chunkConfig.getOverlap(), text.length());
|
||||
if (overlapSize <= 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// 从末尾提取重叠内容
|
||||
String overlap = text.substring(text.length() - overlapSize);
|
||||
|
||||
// 尝试在句子边界截断（查找最后一个句号、问号、感叹号）
|
||||
int lastSentenceEnd = Math.max(
|
||||
overlap.lastIndexOf('。'),
|
||||
Math.max(overlap.lastIndexOf('？'), overlap.lastIndexOf('！'))
|
||||
);
|
||||
|
||||
if (lastSentenceEnd > overlapSize / 2) {
|
||||
return overlap.substring(lastSentenceEnd + 1).trim();
|
||||
}
|
||||
|
||||
return overlap.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 段落在原文中的位置
|
||||
*/
|
||||
private static class ParagraphPos {
|
||||
final int start;
|
||||
final int end;
|
||||
|
||||
ParagraphPos(int start, int end) {
|
||||
this.start = start;
|
||||
this.end = end;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 章节数据类
|
||||
*/
|
||||
private static class Section {
|
||||
String title; // 最近一级标题名称
|
||||
int level; // 标题级别（1-6），0=无标题
|
||||
String breadcrumb; // 完整面包屑路径
|
||||
String content; // 章节内容
|
||||
int startIndex; // 在原文中的起始偏移
|
||||
|
||||
Section(String title, int level, String breadcrumb, String content, int startIndex) {
|
||||
this.title = title;
|
||||
this.level = level;
|
||||
this.breadcrumb = breadcrumb;
|
||||
this.content = content;
|
||||
this.startIndex = startIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 文档字段补全服务
|
||||
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DocumentFieldEnricher {
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
private String promptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
promptTemplate = new String(
|
||||
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("DocumentFieldEnricher prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 doc-field-enricher-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void enrich(Frontmatter frontmatter, String bodyText) {
|
||||
enrich(frontmatter, bodyText, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 Frontmatter 补全 covers 和 whenToRetrieve
|
||||
* 若已有值则跳过；LLM 失败时降级，不阻断主流程
|
||||
*
|
||||
* @param frontmatter 待补全的 frontmatter
|
||||
* @param bodyText 文档正文
|
||||
* @param category 文档所属域（用于查找同域其他文档）
|
||||
*/
|
||||
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
|
||||
if (frontmatter == null) return;
|
||||
|
||||
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
|
||||
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
|
||||
|
||||
if (!needsCovers && !needsWhen) {
|
||||
log.debug("covers 和 whenToRetrieve 已存在，跳过 LLM 生成");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String snippet = bodyText != null && bodyText.length() > 1000
|
||||
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
|
||||
|
||||
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
|
||||
|
||||
String promptText = String.format(promptTemplate,
|
||||
frontmatter.getTitle(),
|
||||
frontmatter.getSummary(),
|
||||
sameDomainDocs,
|
||||
snippet);
|
||||
|
||||
String response = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
|
||||
// 提取 JSON 部分（防止模型输出多余文本）
|
||||
String json = extractJson(response);
|
||||
JsonNode node = objectMapper.readTree(json);
|
||||
|
||||
if (needsCovers && node.has("covers")) {
|
||||
List<String> covers = new ArrayList<>();
|
||||
node.get("covers").forEach(n -> covers.add(n.asText()));
|
||||
frontmatter.setCovers(covers);
|
||||
log.debug("LLM 生成 covers: {}", covers);
|
||||
}
|
||||
|
||||
if (needsWhen && node.has("whenToRetrieve")) {
|
||||
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
|
||||
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成文档字段失败，降级处理: title={}", frontmatter.getTitle(), e);
|
||||
if (needsCovers) frontmatter.setCovers(List.of());
|
||||
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
|
||||
}
|
||||
}
|
||||
|
||||
private String extractJson(String text) {
|
||||
if (text == null) return "{}";
|
||||
int start = text.indexOf('{');
|
||||
int end = text.lastIndexOf('}');
|
||||
if (start == -1 || end == -1 || end <= start) return "{}";
|
||||
return text.substring(start, end + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建同域其他文档标题列表（供 LLM 做排除判断）
|
||||
*/
|
||||
private String buildSameDomainDocs(String currentTitle, String category) {
|
||||
if (category == null || category.isBlank()) return "（无同域文档信息）";
|
||||
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.map(KnowledgeEntry::getTitle)
|
||||
.filter(t -> t != null && !t.equals(currentTitle))
|
||||
.collect(Collectors.toList());
|
||||
if (otherTitles.isEmpty()) return "（无同域其他文档）";
|
||||
return String.join("、", otherTitles);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagIngestResponse;
|
||||
import com.superbiz.agent.client.PyRagClientException;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import com.superbiz.agent.dto.DocumentQueryResponse;
|
||||
import com.superbiz.agent.dto.DocumentUploadRequest;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.exception.DocumentProcessException;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -27,11 +27,13 @@ import java.security.MessageDigest;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 文档管理服务
|
||||
* 文档管理服务。
|
||||
*
|
||||
* <p>负责 MySQL 元数据（{@link ApiDocument}）、本地原件保存与业务查询/删除；
|
||||
* 文档解析、frontmatter 校验、分块与向量索引全部由 py-rag 服务端 ingest 完成。</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@@ -41,34 +43,19 @@ public class DocumentManagementService {
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Autowired
|
||||
private TextExtractorService textExtractorService;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService documentChunkService;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
private PyRagClient pyRagClient;
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private DocumentFieldEnricher documentFieldEnricher;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
/**
|
||||
* 上传文档
|
||||
* 上传文档。
|
||||
*
|
||||
* <p>流程：hash 去重 → 本地原件保存 → py-rag ingest（同步，服务端解析/分块/索引）→
|
||||
* MySQL 元数据落库。docId 取 py-rag 返回的 doc_id，与检索 evidence_key 的 docId 段对齐。</p>
|
||||
*
|
||||
* @param request 上传请求
|
||||
* @return 文档ID
|
||||
@@ -83,15 +70,7 @@ public class DocumentManagementService {
|
||||
log.info("开始上传文档，文件名: {}, 大小: {} bytes", fileName, file.getSize());
|
||||
|
||||
try {
|
||||
// 1. 验证文件格式
|
||||
if (!textExtractorService.isSupportedFormat(fileName)) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "upload",
|
||||
"不支持的文件格式，仅支持 .md 和 .txt"
|
||||
);
|
||||
}
|
||||
|
||||
// 2. 计算文件 hash（去重）
|
||||
// 1. 计算文件 hash（去重）
|
||||
long hashStart = System.currentTimeMillis();
|
||||
String fileHash = calculateFileHash(file);
|
||||
log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart);
|
||||
@@ -105,16 +84,7 @@ public class DocumentManagementService {
|
||||
);
|
||||
}
|
||||
|
||||
// 3. 提取文本
|
||||
long extractStart = System.currentTimeMillis();
|
||||
String text = textExtractorService.extractText(file, fileName);
|
||||
log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart);
|
||||
|
||||
if (text == null || text.isBlank()) {
|
||||
throw new DocumentProcessException(fileName, "upload", "文档内容为空");
|
||||
}
|
||||
|
||||
// 4. 保存原始文件到本地
|
||||
// 2. category 缺省处理 + 保存原始文件到本地
|
||||
String category = request.getCategory();
|
||||
if (category == null || category.isBlank()) {
|
||||
category = "default";
|
||||
@@ -123,47 +93,33 @@ public class DocumentManagementService {
|
||||
localPath = saveToLocal(file, fileName, category);
|
||||
log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart);
|
||||
|
||||
// 5. 解析 frontmatter
|
||||
long frontmatterStart = System.currentTimeMillis();
|
||||
Frontmatter frontmatter = null;
|
||||
String bodyText = text;
|
||||
if (frontmatterParser.hasFrontmatter(text)) {
|
||||
frontmatter = frontmatterParser.parse(text);
|
||||
if (frontmatter != null) {
|
||||
// LLM 补全 covers / whenToRetrieve（已有值则跳过）
|
||||
bodyText = frontmatterParser.stripFrontmatter(text);
|
||||
documentFieldEnricher.enrich(frontmatter, bodyText, category);
|
||||
log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
|
||||
} else {
|
||||
log.warn("frontmatter解析失败，文件名: {}", fileName);
|
||||
}
|
||||
} else {
|
||||
log.debug("文件不包含frontmatter: {}", fileName);
|
||||
// 3. py-rag 入库（格式校验/frontmatter/分块/向量索引都在服务端；同内容重传返回 unchanged）
|
||||
long ingestStart = System.currentTimeMillis();
|
||||
PyRagIngestResponse ingest;
|
||||
try {
|
||||
ingest = pyRagClient.ingest(fileName, file.getBytes(), file.getContentType(),
|
||||
category, null, null, null);
|
||||
} catch (PyRagClientException | IOException e) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "ingest", "py-rag 入库失败: " + e.getMessage(), e
|
||||
);
|
||||
}
|
||||
log.info("py-rag 入库完成: docId={}, status={}, chunks={}, time={}ms",
|
||||
ingest.docId(), ingest.status(), ingest.chunkCount(),
|
||||
System.currentTimeMillis() - ingestStart);
|
||||
|
||||
// 6. 分块
|
||||
long chunkStart = System.currentTimeMillis();
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(bodyText, fileName);
|
||||
if (chunks.isEmpty()) {
|
||||
throw new DocumentProcessException(fileName, "upload", "文档分块失败");
|
||||
}
|
||||
log.info("文档分块完成: fileName={}, chunks={}, time={}ms",
|
||||
fileName, chunks.size(), System.currentTimeMillis() - chunkStart);
|
||||
|
||||
// 7. 创建文档元数据
|
||||
String docId = resolveDocumentId(frontmatter);
|
||||
// 4. 保存文档元数据
|
||||
String metadataJson = null;
|
||||
if (frontmatter != null) {
|
||||
if (ingest.frontmatter() != null) {
|
||||
try {
|
||||
metadataJson = objectMapper.writeValueAsString(frontmatter);
|
||||
metadataJson = objectMapper.writeValueAsString(ingest.frontmatter());
|
||||
} catch (Exception e) {
|
||||
log.warn("Frontmatter序列化失败", e);
|
||||
log.warn("frontmatter 序列化失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
ApiDocument document = ApiDocument.builder()
|
||||
.docId(docId)
|
||||
.docId(ingest.docId())
|
||||
.fileName(fileName)
|
||||
.filePath(localPath)
|
||||
.metadata(metadataJson)
|
||||
@@ -173,56 +129,17 @@ public class DocumentManagementService {
|
||||
.version(request.getVersion())
|
||||
.fileSize(file.getSize())
|
||||
.fileHash(fileHash)
|
||||
.status("PROCESSING")
|
||||
.chunkCount(chunks.size())
|
||||
.status("INDEXED")
|
||||
.chunkCount(ingest.chunkCount())
|
||||
.build();
|
||||
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
log.info("文档元数据已保存: docId={}", docId);
|
||||
log.info("文档元数据已保存: docId={}", document.getDocId());
|
||||
|
||||
// 8. 向量化并索引
|
||||
try {
|
||||
long vectorStart = System.currentTimeMillis();
|
||||
vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter);
|
||||
document.setStatus("INDEXED");
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
log.info("文档向量索引完成: docId={}, category={}, time={}ms",
|
||||
docId, category, System.currentTimeMillis() - vectorStart);
|
||||
log.info("文档上传完成: docId={}, fileName={}, ingestStatus={}, totalTime={}ms",
|
||||
document.getDocId(), fileName, ingest.status(), System.currentTimeMillis() - startTime);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("文档索引失败: docId={}", docId, e);
|
||||
document.setStatus("FAILED");
|
||||
apiDocumentRepository.save(document);
|
||||
throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
|
||||
// 9. 更新 L0 索引
|
||||
if (frontmatter != null) {
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath(localPath)
|
||||
.title(frontmatter.getTitle())
|
||||
.keywords(frontmatter.getKeywords())
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.sections(frontmatter.getSections())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
|
||||
}
|
||||
|
||||
// 触发域级聚合重算
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
|
||||
docId, fileName, frontmatter != null, totalTime);
|
||||
|
||||
return docId;
|
||||
return document.getDocId();
|
||||
|
||||
} catch (Exception e) {
|
||||
// 失败时清理本地文件
|
||||
@@ -314,16 +231,6 @@ public class DocumentManagementService {
|
||||
}
|
||||
}
|
||||
|
||||
private String resolveDocumentId(Frontmatter frontmatter) {
|
||||
if (frontmatter != null && frontmatter.getSource() != null) {
|
||||
String source = frontmatter.getSource().trim();
|
||||
if (!source.isEmpty() && source.length() <= 64) {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 docId 查询文档
|
||||
*/
|
||||
@@ -358,7 +265,10 @@ public class DocumentManagementService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文档
|
||||
* 删除文档（仅 MySQL 元数据与本地原件）。
|
||||
*
|
||||
* <p>py-rag v1 契约没有单文档删除端点：已入库内容需在其服务端
|
||||
* 全量重建（{@code /api/v1/collections:rebuild}）后才会从知识库消失。</p>
|
||||
*/
|
||||
@Transactional
|
||||
public void deleteDocument(String docId) {
|
||||
@@ -379,47 +289,9 @@ public class DocumentManagementService {
|
||||
}
|
||||
}
|
||||
|
||||
// 删除 L0 索引
|
||||
if (doc.getFilePath() != null) {
|
||||
knowledgeIndexService.removeFromIndex(doc.getFilePath());
|
||||
}
|
||||
|
||||
// 删除向量索引
|
||||
try {
|
||||
vectorIndexService.deleteDocumentChunks(docId);
|
||||
log.info("文档向量索引已删除，docId: {}", docId);
|
||||
} catch (Exception e) {
|
||||
log.warn("删除向量索引失败，docId: {}", docId, e);
|
||||
}
|
||||
|
||||
// 删除元数据
|
||||
// 删除元数据（py-rag 侧索引留存，重建后失效）
|
||||
apiDocumentRepository.delete(doc);
|
||||
log.info("文档已删除，docId: {}", docId);
|
||||
|
||||
// 触发域级聚合重算
|
||||
String category = doc.getFilePath() != null
|
||||
? resolveCategory(doc.getFilePath()) : null;
|
||||
if (category != null) {
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为响应 DTO
|
||||
*/
|
||||
/**
|
||||
* 从 filePath 解析 category（取 knowledge_base/{category}/... 中的 category 段）
|
||||
*/
|
||||
private String resolveCategory(String filePath) {
|
||||
try {
|
||||
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
|
||||
// filePath 形如 knowledge_base/payment/xxx.md，取倒数第二段
|
||||
int nameCount = p.getNameCount();
|
||||
if (nameCount >= 2) {
|
||||
return p.getName(nameCount - 2).toString();
|
||||
}
|
||||
} catch (Exception ignored) {}
|
||||
return null;
|
||||
log.info("文档已删除，docId={}（py-rag 侧需全量重建后生效）", docId);
|
||||
}
|
||||
|
||||
private Path resolveLocalPath(String filePath) {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Frontmatter 解析器
|
||||
* 解析 Markdown 文件头的 YAML frontmatter
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class FrontmatterParser {
|
||||
|
||||
private final Yaml yaml = new Yaml();
|
||||
|
||||
/**
|
||||
* 检查文件是否包含 frontmatter
|
||||
*
|
||||
* @param content 文件内容
|
||||
* @return true 如果包含 frontmatter
|
||||
*/
|
||||
public boolean hasFrontmatter(String content) {
|
||||
if (content == null || content.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return content.trim().startsWith("---");
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析 Markdown frontmatter
|
||||
*
|
||||
* @param content 完整文件内容
|
||||
* @return Frontmatter 对象，如果不存在或解析失败返回 null
|
||||
*/
|
||||
public Frontmatter parse(String content) {
|
||||
if (!hasFrontmatter(content)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. 提取 frontmatter 部分（两个 --- 之间）
|
||||
String frontmatterText = extractFrontmatter(content);
|
||||
if (frontmatterText == null) {
|
||||
log.warn("未找到有效的 frontmatter 结束标记");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 2. 使用 SnakeYAML 解析
|
||||
Map<String, Object> map = yaml.load(frontmatterText);
|
||||
if (map == null || map.isEmpty()) {
|
||||
log.warn("Frontmatter 解析结果为空");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 3. 映射到 Frontmatter 对象
|
||||
Frontmatter frontmatter = Frontmatter.builder()
|
||||
.title((String) map.get("title"))
|
||||
.keywords((java.util.List<String>) map.get("keywords"))
|
||||
.summary((String) map.get("summary"))
|
||||
.category((String) map.get("category"))
|
||||
.source((String) map.get("source"))
|
||||
.breadcrumb((String) map.get("breadcrumb"))
|
||||
.kbScope(firstString(map, "kb_scope", "kbScope"))
|
||||
.sections((Map<String, String>) map.get("sections"))
|
||||
.version((String) map.get("version"))
|
||||
.author((String) map.get("author"))
|
||||
.covers((java.util.List<String>) map.get("covers"))
|
||||
.whenToRetrieve((String) map.get("when_to_retrieve"))
|
||||
.build();
|
||||
|
||||
// 4. 验证必填字段
|
||||
if (frontmatter.getTitle() == null || frontmatter.getKeywords() == null ||
|
||||
frontmatter.getSummary() == null) {
|
||||
log.warn("Frontmatter 缺少必填字段: title={}, keywords={}, summary={}",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords(), frontmatter.getSummary());
|
||||
return null;
|
||||
}
|
||||
|
||||
log.debug("Frontmatter 解析成功: title={}, keywords=",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords());
|
||||
return frontmatter;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("Frontmatter 解析失败", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public String stripFrontmatter(String content) {
|
||||
if (!hasFrontmatter(content)) {
|
||||
return content;
|
||||
}
|
||||
|
||||
String trimmed = content.trim();
|
||||
int secondDelimiter = trimmed.indexOf("\n---", 3);
|
||||
int delimiterLength = 4;
|
||||
if (secondDelimiter == -1) {
|
||||
secondDelimiter = trimmed.indexOf("\r\n---", 3);
|
||||
delimiterLength = 5;
|
||||
}
|
||||
if (secondDelimiter == -1) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int bodyStart = secondDelimiter + delimiterLength;
|
||||
if (bodyStart < trimmed.length()) {
|
||||
char next = trimmed.charAt(bodyStart);
|
||||
if (next == '\r') {
|
||||
bodyStart++;
|
||||
}
|
||||
if (bodyStart < trimmed.length() && trimmed.charAt(bodyStart) == '\n') {
|
||||
bodyStart++;
|
||||
}
|
||||
}
|
||||
return trimmed.substring(Math.min(bodyStart, trimmed.length())).stripLeading();
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取 frontmatter 文本（两个 --- 之间的内容）
|
||||
*
|
||||
* @param content 完整文件内容
|
||||
* @return frontmatter 文本，如果格式错误返回 null
|
||||
*/
|
||||
private String extractFrontmatter(String content) {
|
||||
// 去除开头的空白
|
||||
content = content.trim();
|
||||
|
||||
// 检查是否以 --- 开头
|
||||
if (!content.startsWith("---")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// 查找第二个 ---（结束标记）
|
||||
int secondDelimiter = content.indexOf("\n---", 3);
|
||||
if (secondDelimiter == -1) {
|
||||
// 尝试查找 Windows 风格换行
|
||||
secondDelimiter = content.indexOf("\r\n---", 3);
|
||||
if (secondDelimiter == -1) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 提取 frontmatter（不包含 --- 标记）
|
||||
return content.substring(3, secondDelimiter).trim();
|
||||
}
|
||||
|
||||
private String firstString(Map<String, Object> map, String... keys) {
|
||||
for (String key : keys) {
|
||||
Object value = map.get(key);
|
||||
if (value instanceof String text && !text.isBlank()) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,407 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.domain.enums.FaultCategory;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识库初始化服务
|
||||
* 负责批量导入 knowledge_base 目录下的文档到数据库和 Milvus
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeBaseInitService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(KnowledgeBaseInitService.class);
|
||||
|
||||
@Value("${knowledge.base-path:knowledge_base}")
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService documentChunkService;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService vectorEmbeddingService;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore hybridKnowledgeStore;
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
logger.info("Starting hybrid knowledge rebuild from {}", knowledgeBasePath);
|
||||
RebuildResult rebuild = new RebuildResult();
|
||||
rebuild.setBasePath(knowledgeBasePath);
|
||||
rebuild.setCollection(hybridKnowledgeStore.collectionName());
|
||||
|
||||
long mysqlBefore = apiDocumentRepository.count();
|
||||
rebuild.setMysqlDocumentsBefore(mysqlBefore);
|
||||
|
||||
Map<String, Object> milvus = hybridKnowledgeStore.dropAndRecreateCollection();
|
||||
rebuild.setMilvus(milvus);
|
||||
|
||||
apiDocumentRepository.deleteAll();
|
||||
apiDocumentRepository.flush();
|
||||
knowledgeIndexService.clearIndex();
|
||||
rebuild.setMysqlDocumentsAfterClear(apiDocumentRepository.count());
|
||||
rebuild.setL0IndexSizeAfterClear(knowledgeIndexService.getIndexSize());
|
||||
|
||||
InitResult init = initializeKnowledgeBase(true);
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
// Success when at least one doc indexed and no hard failures.
|
||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
return rebuild;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化知识库
|
||||
*
|
||||
* @param force 是否强制重新导入（跳过去重检查）
|
||||
* @return 初始化结果
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public InitResult initializeKnowledgeBase(boolean force) {
|
||||
logger.info("开始初始化知识库: basePath={}, force={}", knowledgeBasePath, force);
|
||||
|
||||
InitResult result = new InitResult();
|
||||
Path baseDir = Paths.get(knowledgeBasePath);
|
||||
|
||||
if (!Files.exists(baseDir)) {
|
||||
logger.error("知识库目录不存在: {}", knowledgeBasePath);
|
||||
throw new RuntimeException("知识库目录不存在: " + knowledgeBasePath);
|
||||
}
|
||||
|
||||
// 1. 扫描所有 Markdown 文件
|
||||
List<Path> markdownFiles = scanMarkdownFiles(baseDir);
|
||||
result.setScanned(markdownFiles.size());
|
||||
logger.info("扫描到 {} 个 Markdown 文件", markdownFiles.size());
|
||||
|
||||
// 2. 如果非强制模式，获取已存在的文档（用于去重）
|
||||
Set<String> existingFilePaths = new HashSet<>();
|
||||
if (!force) {
|
||||
existingFilePaths = apiDocumentRepository.findAll().stream()
|
||||
.map(ApiDocument::getFilePath)
|
||||
.collect(Collectors.toSet());
|
||||
logger.info("已存在 个文档记录", existingFilePaths.size());
|
||||
}
|
||||
|
||||
// 3. 逐个处理文档
|
||||
for (Path file : markdownFiles) {
|
||||
String relativePath = baseDir.relativize(file).toString().replace("\\", "/");
|
||||
|
||||
try {
|
||||
// 去重检查
|
||||
if (!force && existingFilePaths.contains(relativePath)) {
|
||||
logger.debug("跳过已存在的文档: {}", relativePath);
|
||||
result.incrementSkipped();
|
||||
result.addDetail(relativePath, "已存在，跳过");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 解析文档
|
||||
String content = Files.readString(file);
|
||||
Frontmatter frontmatter = frontmatterParser.parse(content);
|
||||
|
||||
if (frontmatter == null) {
|
||||
logger.warn("文档格式无效: {}, frontmatter 解析失败", relativePath);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "格式无效: frontmatter 解析失败");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 提取字段
|
||||
String title = frontmatter.getTitle();
|
||||
String summary = frontmatter.getSummary();
|
||||
String category = frontmatter.getCategory() != null ? frontmatter.getCategory() : "general";
|
||||
List<String> keywords = frontmatter.getKeywords();
|
||||
|
||||
if (title == null || title.isBlank()) {
|
||||
logger.warn("文档缺少标题: {}", relativePath);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "缺少标题");
|
||||
continue;
|
||||
}
|
||||
|
||||
// 保存到数据库
|
||||
ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
|
||||
|
||||
// 提取文档正文（去除 frontmatter）
|
||||
String body = extractBody(content);
|
||||
|
||||
// 文档分块
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
||||
|
||||
// 上传到 Milvus hybrid collection（dense + BM25 search_text）
|
||||
try {
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category, frontmatter);
|
||||
|
||||
document.setStatus("INDEXED");
|
||||
document.setChunkCount(chunks.size());
|
||||
document.setIndexedAt(LocalDateTime.now());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
logger.info("文档已索引到 Milvus hybrid: {} (docId={}, chunks={})",
|
||||
title, document.getDocId(), chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
||||
|
||||
document.setStatus("FAILED");
|
||||
document.setErrorMessage(e.getMessage());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
|
||||
continue; // 跳过该文档，继续处理下一个
|
||||
}
|
||||
|
||||
// 添加到 L0 内存索引
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath(relativePath)
|
||||
.title(title)
|
||||
.keywords(keywords)
|
||||
.summary(summary)
|
||||
.category(category)
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.build();
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
|
||||
result.incrementInserted();
|
||||
result.addDetail(relativePath, "导入成功（L0+L1）");
|
||||
logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("处理文档失败: {}", relativePath, e);
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "处理失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
logger.info("知识库初始化完成: 扫描={}, 跳过={}, 新增={}, 失败={}",
|
||||
result.getScanned(), result.getSkipped(), result.getInserted(), result.getFailed());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取知识库统计信息
|
||||
*/
|
||||
public Stats getStats() {
|
||||
Stats stats = new Stats();
|
||||
|
||||
// 数据库中的文档数量
|
||||
long totalDocuments = apiDocumentRepository.count();
|
||||
stats.setTotalDocuments(totalDocuments);
|
||||
|
||||
// L0 索引中的文档数量
|
||||
int indexSize = knowledgeIndexService.getIndexSize();
|
||||
logger.debug("L0 索引大小: {}", indexSize);
|
||||
|
||||
// 按分类统计（从 fault_category 字段读取）
|
||||
Map<String, Long> categoryCount = apiDocumentRepository.findAll().stream()
|
||||
.collect(Collectors.groupingBy(
|
||||
doc -> doc.getFaultCategory() != null ? doc.getFaultCategory().name() : "GENERAL",
|
||||
Collectors.counting()
|
||||
));
|
||||
stats.setCategoryCount(categoryCount);
|
||||
|
||||
// Milvus 中的向量数量（需要实现）
|
||||
// TODO: 查询 Milvus collection 的实体数量
|
||||
stats.setTotalVectors(0L);
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* 扫描目录下所有 Markdown 文件
|
||||
*/
|
||||
private List<Path> scanMarkdownFiles(Path baseDir) {
|
||||
List<Path> files = new ArrayList<>();
|
||||
|
||||
try {
|
||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
||||
// Import content docs only; skip README/index markdown without frontmatter.
|
||||
if (name.endsWith(".md")
|
||||
&& !name.equalsIgnoreCase("README.md")
|
||||
&& !name.equalsIgnoreCase("readme.md")) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult visitFileFailed(Path file, IOException exc) {
|
||||
logger.warn("访问文件失败: {}", file, exc);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
} catch (IOException e) {
|
||||
logger.error("扫描目录失败: {}", baseDir, e);
|
||||
throw new RuntimeException("扫描目录失败", e);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存文档到数据库
|
||||
*/
|
||||
private ApiDocument saveToDatabase(String filePath, String title, String summary,
|
||||
String category, String content, List<String> keywords) {
|
||||
ApiDocument document = new ApiDocument();
|
||||
document.setDocId(UUID.randomUUID().toString());
|
||||
document.setFileName(Paths.get(filePath).getFileName().toString());
|
||||
document.setFilePath(filePath);
|
||||
document.setApiName(title); // 使用 title 作为 apiName
|
||||
document.setStatus("PENDING"); // 初始状态为 PENDING，索引成功后更新为 INDEXED
|
||||
|
||||
// 映射 category 到 FaultCategory 枚举
|
||||
FaultCategory faultCategory = FaultCategory.fromString(category);
|
||||
document.setFaultCategory(faultCategory);
|
||||
|
||||
// 将 frontmatter 信息保存到 metadata（JSON 格式）
|
||||
String metadataJson = String.format(
|
||||
"{\"title\":\"%s\",\"summary\":\"%s\",\"category\":\"%s\",\"keywords\":%s}",
|
||||
escapeJson(title),
|
||||
escapeJson(summary),
|
||||
escapeJson(category),
|
||||
"[\"" + String.join("\",\"", keywords.stream().map(this::escapeJson).toArray(String[]::new)) + "\"]"
|
||||
);
|
||||
document.setMetadata(metadataJson);
|
||||
|
||||
document.setFileSize((long) content.length());
|
||||
|
||||
return apiDocumentRepository.save(document);
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON 转义
|
||||
*/
|
||||
private String escapeJson(String str) {
|
||||
if (str == null) {
|
||||
return "";
|
||||
}
|
||||
return str.replace("\\", "\\\\")
|
||||
.replace("\"", "\\\"")
|
||||
.replace("\n", "\\n")
|
||||
.replace("\r", "\\r");
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取文档正文（去除 frontmatter）
|
||||
*/
|
||||
private String extractBody(String content) {
|
||||
if (!content.trim().startsWith("---")) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int firstEnd = content.indexOf("---", 3);
|
||||
if (firstEnd == -1) {
|
||||
return content;
|
||||
}
|
||||
|
||||
int secondEnd = content.indexOf("---", firstEnd + 3);
|
||||
if (secondEnd == -1) {
|
||||
return content.substring(firstEnd + 3).trim();
|
||||
}
|
||||
|
||||
return content.substring(secondEnd + 3).trim();
|
||||
}
|
||||
|
||||
// ==================== 数据模型 ====================
|
||||
|
||||
/**
|
||||
* 初始化结果
|
||||
*/
|
||||
@Data
|
||||
public static class InitResult {
|
||||
private int scanned; // 扫描到的文件数量
|
||||
private int skipped; // 跳过的文件数量（已存在）
|
||||
private int inserted; // 成功导入的文件数量
|
||||
private int failed; // 失败的文件数量
|
||||
private Map<String, String> details = new LinkedHashMap<>(); // 详细信息
|
||||
|
||||
public void incrementSkipped() {
|
||||
this.skipped++;
|
||||
}
|
||||
|
||||
public void incrementInserted() {
|
||||
this.inserted++;
|
||||
}
|
||||
|
||||
public void incrementFailed() {
|
||||
this.failed++;
|
||||
}
|
||||
|
||||
public void addDetail(String filePath, String message) {
|
||||
this.details.put(filePath, message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计信息
|
||||
*/
|
||||
@Data
|
||||
public static class Stats {
|
||||
private long totalDocuments; // 数据库中的文档总数
|
||||
private long totalVectors; // Milvus 中的向量总数
|
||||
private Map<String, Long> categoryCount; // 按分类统计
|
||||
}
|
||||
|
||||
/**
|
||||
* Full hybrid rebuild result.
|
||||
*/
|
||||
@Data
|
||||
public static class RebuildResult {
|
||||
private boolean success;
|
||||
private String basePath;
|
||||
private String collection;
|
||||
private long mysqlDocumentsBefore;
|
||||
private long mysqlDocumentsAfterClear;
|
||||
private long mysqlDocumentsAfterInit;
|
||||
private int l0IndexSizeAfterClear;
|
||||
private int l0IndexSizeAfterInit;
|
||||
private Map<String, Object> milvus;
|
||||
private InitResult init;
|
||||
}
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.KnowledgeDomain;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识域服务
|
||||
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KnowledgeDomainService {
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
private String domainPromptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
domainPromptTemplate = new String(
|
||||
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("KnowledgeDomainService prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 domain-summary-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load domain-summary prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文档变更后重算指定域的 when_to_retrieve
|
||||
*/
|
||||
public void onDocumentChange(String category) {
|
||||
if (category == null || category.isBlank()) return;
|
||||
|
||||
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
buildDomainSummary(category, entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合同域文档，调用 LLM 生成域级摘要，写入 DB
|
||||
*/
|
||||
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
|
||||
if (entries.isEmpty()) {
|
||||
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
|
||||
d.setDocumentCount(0);
|
||||
knowledgeDomainRepository.save(d);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 构建文档列表描述
|
||||
StringBuilder docList = new StringBuilder();
|
||||
for (KnowledgeEntry entry : entries) {
|
||||
docList.append("- 文档：").append(entry.getTitle()).append("\n");
|
||||
if (entry.getWhenToRetrieve() != null) {
|
||||
docList.append(" 适用场景：").append(entry.getWhenToRetrieve()).append("\n");
|
||||
}
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
docList.append(" 覆盖：").append(String.join("、", entry.getCovers())).append("\n");
|
||||
}
|
||||
}
|
||||
|
||||
String description = entries.stream()
|
||||
.map(KnowledgeEntry::getSummary)
|
||||
.filter(s -> s != null && !s.isBlank())
|
||||
.findFirst().orElse(category);
|
||||
|
||||
String whenToRetrieve = null;
|
||||
try {
|
||||
String otherDomainsInfo = buildOtherDomainsInfo(category);
|
||||
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
|
||||
whenToRetrieve = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成域级 when_to_retrieve 失败，保留旧值: domain={}", category, e);
|
||||
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
|
||||
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
|
||||
}
|
||||
|
||||
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
|
||||
.orElse(KnowledgeDomain.builder().domainId(category).build());
|
||||
|
||||
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
|
||||
domain.setWhenToRetrieve(whenToRetrieve);
|
||||
domain.setDocumentCount(entries.size());
|
||||
knowledgeDomainRepository.save(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建注入 Planner 的 knowledge map YAML 文本
|
||||
*/
|
||||
public String buildKnowledgeMap() {
|
||||
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
|
||||
if (domains.isEmpty()) return "";
|
||||
|
||||
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
|
||||
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
|
||||
.filter(e -> e.getCategory() != null)
|
||||
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
|
||||
|
||||
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
|
||||
|
||||
for (KnowledgeDomain domain : domains) {
|
||||
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
|
||||
if (domain.getDescription() != null) {
|
||||
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
|
||||
}
|
||||
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
|
||||
yaml.append(" when_to_retrieve: \"")
|
||||
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
|
||||
}
|
||||
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
|
||||
|
||||
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
|
||||
if (!domainEntries.isEmpty()) {
|
||||
yaml.append(" documents:\n");
|
||||
for (KnowledgeEntry entry : domainEntries) {
|
||||
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建其他域的摘要信息（用于 LLM 域级 prompt 的边界判断）
|
||||
* 优先使用其他域的 when_to_retrieve（边界信号），而非 description
|
||||
*/
|
||||
private String buildOtherDomainsInfo(String currentCategory) {
|
||||
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (KnowledgeDomain d : allDomains) {
|
||||
if (d.getDomainId().equals(currentCategory)) continue;
|
||||
sb.append("- ").append(d.getDomainId());
|
||||
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
|
||||
sb.append("：").append(d.getWhenToRetrieve());
|
||||
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
|
||||
sb.append("（").append(d.getDescription()).append("）");
|
||||
}
|
||||
sb.append("\n");
|
||||
}
|
||||
// 如果 DB 里还没有其他域的记录（首次启动），从 L0 索引补充
|
||||
if (sb.isEmpty()) {
|
||||
knowledgeIndexService.getAllEntries().stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
|
||||
.distinct()
|
||||
.forEach(c -> sb.append("- ").append(c).append("\n"));
|
||||
}
|
||||
return sb.isEmpty() ? "（无其他域信息）" : sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,341 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
/**
|
||||
* L0 知识索引服务（关键词 / domain hint，不是向量库）。
|
||||
*
|
||||
* <h3>定位</h3>
|
||||
* 从 MySQL {@code api_document.metadata}（frontmatter）加载文档级关键词与 category，
|
||||
* 供检索前 query understanding 使用。L0 输出只作为：
|
||||
* <ul>
|
||||
* <li>可选 category filter（唯一 domain 时）</li>
|
||||
* <li>rerank 的 domain/keyword/entity boost 信号</li>
|
||||
* <li>trace 可解释信息</li>
|
||||
* </ul>
|
||||
* <b>L0 命中文档不会直接当作事实 evidence</b>；证据正文只来自 L1 向量召回。
|
||||
*
|
||||
* <h3>匹配方式（当前较粗）</h3>
|
||||
* {@code query.contains(keyword) || keyword.contains(query)}，大小写不敏感。
|
||||
* 没有分词、别名归一或停用词；短词/泛词可能误命中。
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KnowledgeIndexService {
|
||||
|
||||
@Value("${knowledge.base-path:knowledge_base}")
|
||||
private String knowledgeBasePath;
|
||||
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Lazy
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
|
||||
|
||||
@PostConstruct
|
||||
public void loadIndex() {
|
||||
log.info("开始从数据库加载知识库索引");
|
||||
|
||||
try {
|
||||
List<ApiDocument> documents = apiDocumentRepository.findAll();
|
||||
|
||||
int loaded = 0;
|
||||
for (ApiDocument doc : documents) {
|
||||
try {
|
||||
KnowledgeEntry entry = parseDocumentToEntry(doc);
|
||||
if (entry != null) {
|
||||
knowledgeIndex.add(entry);
|
||||
loaded++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("解析文档失败: docId={}, error={}", doc.getDocId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.info("知识库索引加载完成，共 {} 个文档", loaded);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("知识库索引加载失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 应用就绪后，检查各域是否有 knowledge_domain 记录，无则触发生成
|
||||
* 使用 ApplicationReadyEvent 而非 PostConstruct，避免循环依赖
|
||||
*/
|
||||
@EventListener(ApplicationReadyEvent.class)
|
||||
public void onApplicationReady() {
|
||||
try {
|
||||
knowledgeIndex.stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank())
|
||||
.distinct()
|
||||
.forEach(category -> {
|
||||
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
|
||||
log.info("域 {} 无 knowledge_domain 记录，触发生成", category);
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("域级记录生成失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
|
||||
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
|
||||
|
||||
return KnowledgeEntry.builder()
|
||||
.filePath(doc.getFilePath())
|
||||
.title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
|
||||
.keywords(frontmatter.getKeywords())
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(frontmatter.getCategory())
|
||||
.kbScope(frontmatter.getKbScope())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("解析 metadata 失败: {}", doc.getDocId(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** 兼容旧调用：只返回命中的文档条目。 */
|
||||
public List<KnowledgeEntry> exactMatch(String query) {
|
||||
return analyzeQuery(query).matches();
|
||||
}
|
||||
|
||||
/**
|
||||
* 分析 query，产出 L0 hint。
|
||||
* 遍历内存索引，收集匹配 keyword、domain、title；不做向量检索。
|
||||
*/
|
||||
public L0Hint analyzeQuery(String query) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
if (query == null || query.trim().isEmpty()) {
|
||||
log.debug("查询关键词为空，返回空结果");
|
||||
return L0Hint.empty();
|
||||
}
|
||||
|
||||
String queryLower = query.toLowerCase();
|
||||
List<KnowledgeEntry> results = new ArrayList<>();
|
||||
Set<String> matchedKeywords = new LinkedHashSet<>();
|
||||
Set<String> domains = new LinkedHashSet<>();
|
||||
Set<String> entities = new LinkedHashSet<>();
|
||||
Set<String> titles = new LinkedHashSet<>();
|
||||
|
||||
for (KnowledgeEntry entry : knowledgeIndex) {
|
||||
if (!matchesConfiguredScope(entry)) {
|
||||
continue;
|
||||
}
|
||||
List<String> entryMatchedKeywords = matchedKeywords(entry, queryLower);
|
||||
if (entryMatchedKeywords.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
results.add(entry);
|
||||
matchedKeywords.addAll(entryMatchedKeywords);
|
||||
entities.addAll(entryMatchedKeywords);
|
||||
|
||||
if (entry.getCategory() != null && !entry.getCategory().isBlank()) {
|
||||
domains.add(entry.getCategory());
|
||||
}
|
||||
if (entry.getTitle() != null && !entry.getTitle().isBlank()) {
|
||||
titles.add(entry.getTitle());
|
||||
}
|
||||
}
|
||||
|
||||
long elapsedTime = System.currentTimeMillis() - startTime;
|
||||
log.debug("L0 Hint分析: matches={}, domainCount={}, keywordCount={}, indexSize={}, time={}ms",
|
||||
results.size(), domains.size(), matchedKeywords.size(), knowledgeIndex.size(), elapsedTime);
|
||||
|
||||
return new L0Hint(
|
||||
List.copyOf(results),
|
||||
List.copyOf(matchedKeywords),
|
||||
List.copyOf(domains),
|
||||
List.copyOf(entities),
|
||||
List.copyOf(titles)
|
||||
);
|
||||
}
|
||||
|
||||
private boolean matchesKeywords(KnowledgeEntry entry, String query) {
|
||||
return !matchedKeywords(entry, query).isEmpty();
|
||||
}
|
||||
|
||||
private boolean matchesConfiguredScope(KnowledgeEntry entry) {
|
||||
String scope = trimToNull(kbScope);
|
||||
if (scope == null) {
|
||||
return true;
|
||||
}
|
||||
return scope.equals(trimToNull(entry.getKbScope()));
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关键词双向包含匹配。
|
||||
* query 已在调用方 lower-case；keyword 在此 lower-case。
|
||||
* 例：query="mysql timeout" 可命中 keyword="mysql"；
|
||||
* 反过来 keyword="mysql connection pool timeout" 也可能被短 query 命中。
|
||||
*/
|
||||
private List<String> matchedKeywords(KnowledgeEntry entry, String query) {
|
||||
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
List<String> matches = new ArrayList<>();
|
||||
for (String keyword : entry.getKeywords()) {
|
||||
String keywordLower = keyword.toLowerCase();
|
||||
if (query.contains(keywordLower) || keywordLower.contains(query)) {
|
||||
matches.add(keyword);
|
||||
}
|
||||
}
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
public String readDocument(String filePath, int maxChars) {
|
||||
try {
|
||||
Path fullPath = resolveDocumentPath(filePath);
|
||||
if (!Files.exists(fullPath)) {
|
||||
log.warn("读取文档失败，文件不存在: basePath={}, filePath={}, resolvedPath={}",
|
||||
knowledgeBasePath, filePath, fullPath);
|
||||
return null;
|
||||
}
|
||||
String content = Files.readString(fullPath);
|
||||
|
||||
if (content.length() > maxChars) {
|
||||
return content.substring(0, maxChars) + "...";
|
||||
}
|
||||
|
||||
return content;
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("读取文档失败: basePath={}, filePath={}", knowledgeBasePath, filePath, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Path resolveDocumentPath(String filePath) {
|
||||
if (filePath == null || filePath.isBlank()) {
|
||||
throw new IllegalArgumentException("filePath cannot be blank");
|
||||
}
|
||||
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
if (path.isAbsolute()) {
|
||||
return path;
|
||||
}
|
||||
|
||||
Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize();
|
||||
Path baseName = basePath.getFileName();
|
||||
if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) {
|
||||
return basePath.getParent().resolve(path).normalize();
|
||||
}
|
||||
|
||||
Path pathFromWorkingDir = path.toAbsolutePath().normalize();
|
||||
if (pathFromWorkingDir.startsWith(basePath)) {
|
||||
return pathFromWorkingDir;
|
||||
}
|
||||
|
||||
return basePath.resolve(path).normalize();
|
||||
}
|
||||
|
||||
public void addToIndex(KnowledgeEntry entry) {
|
||||
knowledgeIndex.add(entry);
|
||||
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
|
||||
}
|
||||
|
||||
public void removeFromIndex(String filePath) {
|
||||
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
|
||||
log.debug("文档已从 L0 索引移除: {}", filePath);
|
||||
}
|
||||
|
||||
/** Clear in-memory L0 entries (used by knowledge rebuild). */
|
||||
public void clearIndex() {
|
||||
knowledgeIndex.clear();
|
||||
log.info("L0 knowledge index cleared");
|
||||
}
|
||||
|
||||
public int getIndexSize() {
|
||||
return knowledgeIndex.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有索引条目（供域聚合使用）
|
||||
*/
|
||||
public List<KnowledgeEntry> getAllEntries() {
|
||||
return List.copyOf(knowledgeIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* L0 分析结果。
|
||||
*
|
||||
* @param matches 命中的文档条目（仅 hint，不是 evidence）
|
||||
* @param matchedKeywords 命中的关键词
|
||||
* @param domains 命中文档的 category 集合
|
||||
* @param entities 当前实现等同 matchedKeywords，预留实体字段
|
||||
* @param titles 命中文档标题
|
||||
*/
|
||||
public record L0Hint(
|
||||
List<KnowledgeEntry> matches,
|
||||
List<String> matchedKeywords,
|
||||
List<String> domains,
|
||||
List<String> entities,
|
||||
List<String> titles
|
||||
) {
|
||||
public static L0Hint empty() {
|
||||
return new L0Hint(List.of(), List.of(), List.of(), List.of(), List.of());
|
||||
}
|
||||
|
||||
/** 仅当恰好一个 domain 时返回，用于安全地加 category filter。 */
|
||||
public String singleDomainOrNull() {
|
||||
return domains.size() == 1 ? domains.get(0) : null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 检索前的 query 理解层（L0 出口）。
|
||||
*
|
||||
* <p>输入是 Agent 的原始检索句，输出 {@link KnowledgeQuery}，供后续 L1 过滤与 rerank 使用。</p>
|
||||
*
|
||||
* <h3>当前能力边界</h3>
|
||||
* <ul>
|
||||
* <li>会做：关键词匹配、domain/entity/title hint、唯一 domain 时生成 categoryFilter</li>
|
||||
* <li>不会做：真正的 query rewrite / 同义词扩展 / 多 query 改写
|
||||
* （{@code rewrittenQuery} 目前等于 {@code originalQuery}）</li>
|
||||
* <li>L0 命中文档正文不会直接当作 evidence；证据只来自 L1 向量召回</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Service
|
||||
public class KnowledgeQueryTransformer {
|
||||
|
||||
private final KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
public KnowledgeQueryTransformer(KnowledgeIndexService knowledgeIndexService) {
|
||||
this.knowledgeIndexService = knowledgeIndexService;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将原始 query 转为检索控制结构。
|
||||
*
|
||||
* <p>{@code categoryFilter} 仅在 L0 恰好命中一个 domain 时非空；
|
||||
* 多 domain 或零 domain 时为 null，避免错误收窄召回。</p>
|
||||
*/
|
||||
public KnowledgeQuery transform(String rawQuery) {
|
||||
String normalized = rawQuery == null ? "" : rawQuery.trim();
|
||||
KnowledgeIndexService.L0Hint hint = knowledgeIndexService.analyzeQuery(normalized);
|
||||
return KnowledgeQuery.builder()
|
||||
.originalQuery(normalized)
|
||||
// 预留改写字段；当前未实现 rewrite，保持与 original 一致
|
||||
.rewrittenQuery(normalized)
|
||||
.domainHints(safeList(hint.domains()))
|
||||
.matchedKeywords(safeList(hint.matchedKeywords()))
|
||||
.entities(safeList(hint.entities()))
|
||||
// 只有唯一 domain 才作为向量 metadata 的 category 过滤条件
|
||||
.categoryFilter(hint.singleDomainOrNull())
|
||||
.l0Titles(safeList(hint.titles()))
|
||||
.l0MatchCount(hint.matches() == null ? 0 : hint.matches().size())
|
||||
.build();
|
||||
}
|
||||
|
||||
private List<String> safeList(List<String> values) {
|
||||
return values == null ? List.of() : values;
|
||||
}
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonCase;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonReport;
|
||||
import com.superbiz.agent.dto.RetrievalComparisonResult;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
@Service
|
||||
public class RagRetrievalSidecarComparisonService {
|
||||
|
||||
private final VectorSearchService vectorSearchService;
|
||||
private final SpringAiVectorStoreSidecarService sidecarService;
|
||||
private final RetrievalResultNormalizer normalizer;
|
||||
private final RagSidecarProperties properties;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public RagRetrievalSidecarComparisonService(VectorSearchService vectorSearchService,
|
||||
SpringAiVectorStoreSidecarService sidecarService,
|
||||
RetrievalResultNormalizer normalizer,
|
||||
RagSidecarProperties properties,
|
||||
ObjectMapper objectMapper) {
|
||||
this.vectorSearchService = vectorSearchService;
|
||||
this.sidecarService = sidecarService;
|
||||
this.normalizer = normalizer;
|
||||
this.properties = properties;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public RetrievalComparisonReport compare(List<RetrievalComparisonCase> cases, int topK) {
|
||||
List<RetrievalComparisonResult> results = new ArrayList<>();
|
||||
String sidecarStatus = "not_run";
|
||||
for (RetrievalComparisonCase comparisonCase : cases) {
|
||||
List<ComparableRetrievalResult> currentResults = normalizeCurrentResults(
|
||||
vectorSearchService.searchSimilarDocuments(
|
||||
comparisonCase.getQuery(),
|
||||
topK,
|
||||
comparisonCase.getCategory()
|
||||
)
|
||||
);
|
||||
SidecarRetrievalResponse sidecar = sidecarService.search(
|
||||
comparisonCase.getQuery(),
|
||||
topK,
|
||||
comparisonCase.getCategory()
|
||||
);
|
||||
sidecarStatus = sidecar.getStatus();
|
||||
results.add(RetrievalComparisonResult.builder()
|
||||
.caseId(comparisonCase.getCaseId())
|
||||
.scenario(comparisonCase.getScenario())
|
||||
.query(comparisonCase.getQuery())
|
||||
.category(comparisonCase.getCategory())
|
||||
.currentResults(currentResults)
|
||||
.sidecar(sidecar)
|
||||
.differences(compareDifferences(currentResults, sidecar.getResults()))
|
||||
.build());
|
||||
}
|
||||
|
||||
return RetrievalComparisonReport.builder()
|
||||
.generatedAt(OffsetDateTime.now(ZoneOffset.UTC).toString())
|
||||
.caseCount(cases.size())
|
||||
.topK(topK)
|
||||
.sidecarStatus(sidecarStatus)
|
||||
.results(results)
|
||||
.build();
|
||||
}
|
||||
|
||||
public RetrievalComparisonReport compareGoldenCases(Path caseFile) throws IOException {
|
||||
var root = objectMapper.readTree(caseFile.toFile());
|
||||
int topK = root.path("topK").asInt(5);
|
||||
List<RetrievalComparisonCase> cases = new ArrayList<>();
|
||||
for (var node : root.path("cases")) {
|
||||
cases.add(RetrievalComparisonCase.builder()
|
||||
.caseId(node.path("caseId").asText())
|
||||
.scenario(node.path("scenario").asText())
|
||||
.query(node.path("query").asText())
|
||||
.build());
|
||||
}
|
||||
return compare(cases, topK);
|
||||
}
|
||||
|
||||
public void writeReports(RetrievalComparisonReport report, Path jsonPath, Path markdownPath) throws IOException {
|
||||
createParentDirectories(jsonPath);
|
||||
createParentDirectories(markdownPath);
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(jsonPath.toFile(), report);
|
||||
Files.writeString(markdownPath, renderMarkdown(report));
|
||||
}
|
||||
|
||||
private void createParentDirectories(Path path) throws IOException {
|
||||
Path parent = path.getParent();
|
||||
if (parent != null) {
|
||||
Files.createDirectories(parent);
|
||||
}
|
||||
}
|
||||
|
||||
private List<ComparableRetrievalResult> normalizeCurrentResults(List<VectorSearchService.SearchResult> rawResults) {
|
||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
||||
for (int i = 0; i < rawResults.size(); i++) {
|
||||
results.add(normalizer.fromCurrent(rawResults.get(i), i + 1, properties.getContentPreviewLimit()));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private List<String> compareDifferences(List<ComparableRetrievalResult> currentResults,
|
||||
List<ComparableRetrievalResult> sidecarResults) {
|
||||
if (sidecarResults == null || sidecarResults.isEmpty()) {
|
||||
return List.of("sidecar_unavailable_or_empty");
|
||||
}
|
||||
List<String> differences = new ArrayList<>();
|
||||
String currentTopSource = currentResults.isEmpty() ? null : currentResults.get(0).getSource();
|
||||
String sidecarTopSource = sidecarResults.get(0).getSource();
|
||||
if (!Objects.equals(currentTopSource, sidecarTopSource)) {
|
||||
differences.add("top_source_differs");
|
||||
}
|
||||
String currentTopBreadcrumb = currentResults.isEmpty() ? null : currentResults.get(0).getBreadcrumb();
|
||||
String sidecarTopBreadcrumb = sidecarResults.get(0).getBreadcrumb();
|
||||
if (!Objects.equals(currentTopBreadcrumb, sidecarTopBreadcrumb)) {
|
||||
differences.add("top_breadcrumb_differs");
|
||||
}
|
||||
String currentScoreLabel = currentResults.isEmpty() ? null : currentResults.get(0).getScoreLabel();
|
||||
String sidecarScoreLabel = sidecarResults.get(0).getScoreLabel();
|
||||
if (!Objects.equals(currentScoreLabel, sidecarScoreLabel)) {
|
||||
differences.add("score_label_differs");
|
||||
}
|
||||
return differences;
|
||||
}
|
||||
|
||||
private String renderMarkdown(RetrievalComparisonReport report) {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("# RAG Sidecar Retrieval Comparison\n\n");
|
||||
builder.append("Generated at: `").append(report.getGeneratedAt()).append("`\n\n");
|
||||
builder.append("- Cases: ").append(report.getCaseCount()).append("\n");
|
||||
builder.append("- Top K: ").append(report.getTopK()).append("\n");
|
||||
builder.append("- Sidecar status: `").append(report.getSidecarStatus()).append("`\n\n");
|
||||
builder.append("| Case | Query | Current Top | Sidecar Top | Differences |\n");
|
||||
builder.append("|---|---|---|---|---|\n");
|
||||
for (RetrievalComparisonResult result : report.getResults()) {
|
||||
builder.append("| ")
|
||||
.append(nullToBlank(result.getCaseId()))
|
||||
.append(" | ")
|
||||
.append(escapePipe(result.getQuery()))
|
||||
.append(" | ")
|
||||
.append(formatTop(result.getCurrentResults()))
|
||||
.append(" | ")
|
||||
.append(formatTop(result.getSidecar() != null ? result.getSidecar().getResults() : List.of()))
|
||||
.append(" | ")
|
||||
.append(String.join("<br>", result.getDifferences()))
|
||||
.append(" |\n");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private String formatTop(List<ComparableRetrievalResult> results) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
ComparableRetrievalResult top = results.get(0);
|
||||
return escapePipe(nullToBlank(top.getSource())) + " (" + nullToBlank(top.getScoreLabel()) + ")";
|
||||
}
|
||||
|
||||
private String escapePipe(String value) {
|
||||
return nullToBlank(value).replace("|", "\\|");
|
||||
}
|
||||
|
||||
private String nullToBlank(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
}
|
||||
@@ -1,190 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import reactor.core.publisher.Flux;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* RAG (Retrieval-Augmented Generation) 服务
|
||||
* 结合向量检索和大语言模型生成答案
|
||||
*/
|
||||
@Service
|
||||
public class RagService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RagService.class);
|
||||
|
||||
@Autowired
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Value("${rag.top-k:3}")
|
||||
private int topK;
|
||||
|
||||
/**
|
||||
* 流式处理用户问题（不带历史消息）
|
||||
*
|
||||
* @param question 用户问题
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
public void queryStream(String question, StreamCallback callback) {
|
||||
queryStream(question, new ArrayList<>(), callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式处理用户问题（带历史消息）
|
||||
*
|
||||
* @param question 用户问题
|
||||
* @param history 历史消息列表，格式：[{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
public void queryStream(String question, List<Map<String, String>> history, StreamCallback callback) {
|
||||
try {
|
||||
logger.info("收到 RAG 流式查询: {}", question);
|
||||
|
||||
// 1. 从向量数据库检索相关文档
|
||||
List<VectorSearchService.SearchResult> searchResults =
|
||||
vectorSearchService.searchSimilarDocuments(question, topK);
|
||||
|
||||
// 发送检索结果
|
||||
callback.onSearchResults(searchResults);
|
||||
|
||||
if (searchResults.isEmpty()) {
|
||||
logger.warn("未找到相关文档");
|
||||
callback.onComplete("抱歉，我在知识库中没有找到相关信息来回答您的问题。", "");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 构建上下文和提示词
|
||||
String context = buildContext(searchResults);
|
||||
String prompt = buildPrompt(question, context);
|
||||
|
||||
// 3. 流式调用大语言模型（传入历史消息）
|
||||
generateAnswerStream(prompt, history, callback);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("RAG 流式查询失败", e);
|
||||
callback.onError(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建上下文
|
||||
*/
|
||||
private String buildContext(List<VectorSearchService.SearchResult> searchResults) {
|
||||
StringBuilder context = new StringBuilder();
|
||||
|
||||
for (int i = 0; i < searchResults.size(); i++) {
|
||||
VectorSearchService.SearchResult result = searchResults.get(i);
|
||||
context.append("【参考资料 ").append(i + 1).append("】\n");
|
||||
context.append(result.getContent()).append("\n\n");
|
||||
}
|
||||
|
||||
return context.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建提示词
|
||||
*/
|
||||
private String buildPrompt(String question, String context) {
|
||||
return String.format(
|
||||
"你是一个专业的AI助手。请根据以下参考资料回答用户的问题。\n\n" +
|
||||
"参考资料：\n%s\n" +
|
||||
"用户问题：%s\n\n" +
|
||||
"请基于上述参考资料给出准确、详细的回答。如果参考资料中没有相关信息，请明确说明。",
|
||||
context, question
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成答案（流式）
|
||||
*
|
||||
* @param prompt 当前问题的提示词
|
||||
* @param history 历史消息列表
|
||||
* @param callback 流式回调接口
|
||||
*/
|
||||
private void generateAnswerStream(String prompt, List<Map<String, String>> history, StreamCallback callback) {
|
||||
// 构建消息列表：历史消息 + 当前问题
|
||||
List<Message> messages = new ArrayList<>();
|
||||
|
||||
// 添加历史消息
|
||||
for (Map<String, String> historyMsg : history) {
|
||||
String role = historyMsg.get("role");
|
||||
String content = historyMsg.get("content");
|
||||
|
||||
if ("user".equals(role)) {
|
||||
messages.add(new UserMessage(content));
|
||||
} else if ("assistant".equals(role)) {
|
||||
messages.add(new AssistantMessage(content));
|
||||
}
|
||||
}
|
||||
|
||||
// 添加当前用户问题
|
||||
messages.add(new UserMessage(prompt));
|
||||
|
||||
logger.debug("发送给AI模型的消息数量: {}（包含 {} 条历史消息）",
|
||||
messages.size(), history.size());
|
||||
|
||||
logger.info("开始调用AI模型流式接口...");
|
||||
|
||||
StringBuilder reasoningContent = new StringBuilder();
|
||||
StringBuilder finalContent = new StringBuilder();
|
||||
|
||||
Flux<ChatResponse> flux = chatModel.stream(new Prompt(messages));
|
||||
|
||||
logger.info("开始接收AI模型流式响应...");
|
||||
|
||||
flux.subscribe(
|
||||
response -> {
|
||||
if (response.getResults() != null && !response.getResults().isEmpty()) {
|
||||
String content = response.getResults().get(0).getOutput().getText();
|
||||
|
||||
if (content != null && !content.isEmpty()) {
|
||||
logger.debug("收到AI模型内容块: {}", content);
|
||||
|
||||
finalContent.append(content);
|
||||
callback.onContentChunk(content);
|
||||
|
||||
logger.debug("已调用 onContentChunk 回调");
|
||||
} else {
|
||||
logger.debug("收到空内容块，跳过");
|
||||
}
|
||||
}
|
||||
},
|
||||
error -> {
|
||||
logger.error("AI模型流式响应失败", error);
|
||||
callback.onError(new Exception("AI模型流式响应失败: " + error.getMessage(), error));
|
||||
},
|
||||
() -> {
|
||||
logger.info("AI模型流式响应完成，总内容长度: {}", finalContent.length());
|
||||
callback.onComplete(finalContent.toString(), reasoningContent.toString());
|
||||
logger.info("已调用 onComplete 回调");
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 流式回调接口
|
||||
*/
|
||||
public interface StreamCallback {
|
||||
void onSearchResults(List<VectorSearchService.SearchResult> results);
|
||||
void onReasoningChunk(String chunk);
|
||||
void onContentChunk(String chunk);
|
||||
void onComplete(String fullContent, String fullReasoning);
|
||||
void onError(Exception e);
|
||||
}
|
||||
}
|
||||
@@ -1,96 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@Component
|
||||
public class RetrievalResultNormalizer {
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public RetrievalResultNormalizer(ObjectMapper objectMapper) {
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
public ComparableRetrievalResult fromCurrent(VectorSearchService.SearchResult result, int rank, int previewLimit) {
|
||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), result.getMetadata(), result.getId());
|
||||
return ComparableRetrievalResult.builder()
|
||||
.path("current")
|
||||
.rank(rank)
|
||||
.id(result.getId())
|
||||
.source(source)
|
||||
.docId(metadata.get("docId"))
|
||||
.title(metadata.get("title"))
|
||||
.breadcrumb(metadata.get("breadcrumb"))
|
||||
.category(metadata.get("category"))
|
||||
.contentPreview(truncate(result.getContent(), previewLimit))
|
||||
.scoreLabel("l2_distance")
|
||||
.scoreValue((double) result.getScore())
|
||||
.build();
|
||||
}
|
||||
|
||||
public ComparableRetrievalResult fromSidecar(Document document, int rank, int previewLimit) {
|
||||
Map<String, String> metadata = stringifyMetadata(document.getMetadata());
|
||||
String source = firstNonBlank(metadata.get("_source"), metadata.get("source"), metadata.get("docId"), document.getId());
|
||||
return ComparableRetrievalResult.builder()
|
||||
.path("sidecar")
|
||||
.rank(rank)
|
||||
.id(document.getId())
|
||||
.source(source)
|
||||
.docId(metadata.get("docId"))
|
||||
.title(metadata.get("title"))
|
||||
.breadcrumb(metadata.get("breadcrumb"))
|
||||
.category(metadata.get("category"))
|
||||
.contentPreview(truncate(document.getText(), previewLimit))
|
||||
.scoreLabel("similarity")
|
||||
.scoreValue(document.getScore())
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, String> parseMetadata(String metadata) {
|
||||
if (metadata == null || metadata.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
||||
return stringifyMetadata(raw);
|
||||
} catch (Exception e) {
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> stringifyMetadata(Map<?, ?> raw) {
|
||||
if (raw == null || raw.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, String> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
result.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String truncate(String text, int maxLength) {
|
||||
if (text == null || text.length() <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
return text.substring(0, maxLength) + "...";
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.config.RagSidecarProperties;
|
||||
import com.superbiz.agent.dto.ComparableRetrievalResult;
|
||||
import com.superbiz.agent.dto.SidecarRetrievalResponse;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.vectorstore.SearchRequest;
|
||||
import org.springframework.ai.vectorstore.VectorStore;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Service
|
||||
public class SpringAiVectorStoreSidecarService {
|
||||
|
||||
private final RagSidecarProperties properties;
|
||||
private final ObjectProvider<VectorStore> vectorStoreProvider;
|
||||
private final RetrievalResultNormalizer normalizer;
|
||||
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
public SpringAiVectorStoreSidecarService(RagSidecarProperties properties,
|
||||
ObjectProvider<VectorStore> vectorStoreProvider,
|
||||
RetrievalResultNormalizer normalizer) {
|
||||
this.properties = properties;
|
||||
this.vectorStoreProvider = vectorStoreProvider;
|
||||
this.normalizer = normalizer;
|
||||
}
|
||||
|
||||
public SidecarRetrievalResponse search(String query, int topK, String category) {
|
||||
if (!properties.isEnabled()) {
|
||||
return unavailable("disabled", null);
|
||||
}
|
||||
|
||||
VectorStore vectorStore = vectorStoreProvider.getIfAvailable();
|
||||
if (vectorStore == null) {
|
||||
return unavailable("missing_vector_store", "No Spring AI VectorStore bean is available");
|
||||
}
|
||||
|
||||
try {
|
||||
SearchRequest.Builder builder = SearchRequest.builder()
|
||||
.query(query)
|
||||
.topK(topK)
|
||||
.similarityThresholdAll();
|
||||
String filterExpression = buildFilterExpression(category);
|
||||
if (filterExpression != null) {
|
||||
builder.filterExpression(filterExpression);
|
||||
}
|
||||
|
||||
List<Document> documents = vectorStore.similaritySearch(builder.build());
|
||||
List<ComparableRetrievalResult> results = new ArrayList<>();
|
||||
for (int i = 0; i < documents.size(); i++) {
|
||||
results.add(normalizer.fromSidecar(documents.get(i), i + 1, properties.getContentPreviewLimit()));
|
||||
}
|
||||
return SidecarRetrievalResponse.builder()
|
||||
.enabled(true)
|
||||
.available(true)
|
||||
.status("available")
|
||||
.results(results)
|
||||
.build();
|
||||
} catch (Exception e) {
|
||||
log.warn("Spring AI sidecar retrieval failed: {}", e.getMessage());
|
||||
return unavailable("query_failed", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private SidecarRetrievalResponse unavailable(String status, String errorMessage) {
|
||||
return SidecarRetrievalResponse.builder()
|
||||
.enabled(properties.isEnabled())
|
||||
.available(false)
|
||||
.status(status)
|
||||
.errorMessage(errorMessage)
|
||||
.results(List.of())
|
||||
.build();
|
||||
}
|
||||
|
||||
private String escapeFilterValue(String value) {
|
||||
return value.replace("'", "\\'");
|
||||
}
|
||||
|
||||
String buildFilterExpression(String category) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
String categoryFilter = trimToNull(category);
|
||||
if (categoryFilter != null) {
|
||||
parts.add("category == '" + escapeFilterValue(categoryFilter) + "'");
|
||||
}
|
||||
String scopeFilter = trimToNull(kbScope);
|
||||
if (scopeFilter != null) {
|
||||
parts.add("kb_scope == '" + escapeFilterValue(scopeFilter) + "'");
|
||||
}
|
||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
||||
}
|
||||
|
||||
private String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.exception.DocumentProcessException;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* 文本提取服务
|
||||
* 仅支持 Markdown (.md) 和纯文本 (.txt) 格式
|
||||
* 其他格式（.docx、.pdf 等）需要通过外部转换服务先转为 Markdown
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class TextExtractorService {
|
||||
|
||||
/**
|
||||
* 从文件中提取文本
|
||||
*
|
||||
* @param file 上传的文件
|
||||
* @param fileName 文件名
|
||||
* @return 提取的文本内容
|
||||
*/
|
||||
public String extractText(MultipartFile file, String fileName) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new DocumentProcessException(fileName, "extract", "文件为空");
|
||||
}
|
||||
|
||||
String extension = getFileExtension(fileName);
|
||||
log.info("开始提取文本，文件名: {}, 格式: {}, 大小: {} bytes", fileName, extension, file.getSize());
|
||||
|
||||
if (!isSupportedFormat(fileName)) {
|
||||
throw new DocumentProcessException(
|
||||
fileName, "extract",
|
||||
"不支持的文件格式: " + extension + "，仅支持 .md 和 .txt。其他格式请先通过转换服务转为 Markdown。"
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
String text = extractPlainText(file);
|
||||
log.info("文本提取成功，文件名: {}, 提取字符数: {}", fileName, text.length());
|
||||
return text;
|
||||
|
||||
} catch (IOException e) {
|
||||
log.error("文本提取失败，文件名: {}", fileName, e);
|
||||
throw new DocumentProcessException(fileName, "extract", "文件读取失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取纯文本（.txt、.md）
|
||||
*/
|
||||
private String extractPlainText(MultipartFile file) throws IOException {
|
||||
StringBuilder content = new StringBuilder();
|
||||
try (InputStream is = file.getInputStream();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
|
||||
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
content.append(line).append("\n");
|
||||
}
|
||||
}
|
||||
return content.toString().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件扩展名
|
||||
*/
|
||||
private String getFileExtension(String fileName) {
|
||||
if (fileName == null || !fileName.contains(".")) {
|
||||
return "";
|
||||
}
|
||||
return fileName.substring(fileName.lastIndexOf(".") + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 验证文件格式是否支持
|
||||
*/
|
||||
public boolean isSupportedFormat(String fileName) {
|
||||
String extension = getFileExtension(fileName).toLowerCase();
|
||||
return extension.equals("md") || extension.equals("txt");
|
||||
}
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 向量嵌入服务
|
||||
* 使用阿里云 DashScope Text Embedding API
|
||||
*/
|
||||
@Service
|
||||
public class VectorEmbeddingService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorEmbeddingService.class);
|
||||
|
||||
@Autowired
|
||||
private EmbeddingModel embeddingModel;
|
||||
|
||||
/**
|
||||
* 生成向量嵌入
|
||||
* 调用阿里云 DashScope Text Embedding API
|
||||
*
|
||||
* @param content 文本内容
|
||||
* @return 向量嵌入（浮点数列表）
|
||||
*/
|
||||
public List<Float> generateEmbedding(String content) {
|
||||
try {
|
||||
if (content == null || content.trim().isEmpty()) {
|
||||
logger.warn("内容为空，无法生成向量");
|
||||
throw new IllegalArgumentException("内容不能为空");
|
||||
}
|
||||
|
||||
logger.debug("开始生成向量嵌入, 内容长度: {} 字符", content.length());
|
||||
|
||||
float[] embedding = embeddingModel.embed(content);
|
||||
|
||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
||||
for (float v : embedding) {
|
||||
floatEmbedding.add(v);
|
||||
}
|
||||
|
||||
logger.info("成功生成向量嵌入, 内容长度: {} 字符, 向量维度: {}",
|
||||
content.length(), floatEmbedding.size());
|
||||
|
||||
return floatEmbedding;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("生成向量嵌入失败, 内容长度: {}", content != null ? content.length() : 0, e);
|
||||
throw new RuntimeException("生成向量嵌入失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public List<List<Float>> generateEmbeddings(List<String> contents) {
|
||||
try {
|
||||
if (contents == null || contents.isEmpty()) {
|
||||
logger.warn("内容列表为空，无法生成向量");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
logger.info("开始批量生成向量嵌入, 数量: {}", contents.size());
|
||||
|
||||
List<float[]> embeddings = embeddingModel.embed(contents);
|
||||
|
||||
List<List<Float>> result = new ArrayList<>();
|
||||
for (float[] embedding : embeddings) {
|
||||
List<Float> floatEmbedding = new ArrayList<>(embedding.length);
|
||||
for (float v : embedding) {
|
||||
floatEmbedding.add(v);
|
||||
}
|
||||
result.add(floatEmbedding);
|
||||
}
|
||||
|
||||
logger.info("成功批量生成向量嵌入, 数量: {}, 维度: {}",
|
||||
result.size(),
|
||||
result.isEmpty() ? 0 : result.get(0).size());
|
||||
|
||||
return result;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("批量生成向量嵌入失败", e);
|
||||
throw new RuntimeException("批量生成向量嵌入失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成查询向量
|
||||
*
|
||||
* @param query 查询文本
|
||||
* @return 向量嵌入
|
||||
*/
|
||||
public List<Float> generateQueryVector(String query) {
|
||||
return generateEmbedding(query);
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算两个向量的余弦相似度
|
||||
*
|
||||
* @param vector1 向量1
|
||||
* @param vector2 向量2
|
||||
* @return 余弦相似度 [-1, 1]
|
||||
*/
|
||||
public float calculateCosineSimilarity(List<Float> vector1, List<Float> vector2) {
|
||||
if (vector1.size() != vector2.size()) {
|
||||
throw new IllegalArgumentException("向量维度不匹配");
|
||||
}
|
||||
|
||||
float dotProduct = 0.0f;
|
||||
float norm1 = 0.0f;
|
||||
float norm2 = 0.0f;
|
||||
|
||||
for (int i = 0; i < vector1.size(); i++) {
|
||||
dotProduct += vector1.get(i) * vector2.get(i);
|
||||
norm1 += vector1.get(i) * vector1.get(i);
|
||||
norm2 += vector2.get(i) * vector2.get(i);
|
||||
}
|
||||
|
||||
return dotProduct / (float) (Math.sqrt(norm1) * Math.sqrt(norm2));
|
||||
}
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 向量索引写入服务（RAG 入库侧）。
|
||||
*
|
||||
* <p>唯一后端 {@link MilvusHybridKnowledgeStore}（Milvus SDK v2）：</p>
|
||||
* <ul>
|
||||
* <li>dense：应用侧 embedding → 字段 {@code vector}</li>
|
||||
* <li>BM25：{@link #buildSearchText} → 字段 {@code search_text}；
|
||||
* sparse 由 collection 上 BM25 Function 自动生成，本类不写 sparse</li>
|
||||
* </ul>
|
||||
* <p>不再使用 legacy {@code MilvusServiceClient} insert/delete，
|
||||
* 也不走 Spring AI {@code VectorStore#add}（starter 无 hybrid schema/BM25 Function）。</p>
|
||||
*/
|
||||
@Service
|
||||
public class VectorIndexService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorIndexService.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService embeddingService;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService chunkService;
|
||||
|
||||
@Value("${file.upload.path}")
|
||||
private String uploadPath;
|
||||
|
||||
public IndexingResult indexDirectory(String directoryPath) {
|
||||
IndexingResult result = new IndexingResult();
|
||||
result.setStartTime(LocalDateTime.now());
|
||||
|
||||
try {
|
||||
String targetPath = (directoryPath != null && !directoryPath.trim().isEmpty())
|
||||
? directoryPath : uploadPath;
|
||||
|
||||
Path dirPath = Paths.get(targetPath).normalize();
|
||||
File directory = dirPath.toFile();
|
||||
|
||||
if (!directory.exists() || !directory.isDirectory()) {
|
||||
throw new IllegalArgumentException("目录不存在或不是有效目录: " + targetPath);
|
||||
}
|
||||
|
||||
result.setDirectoryPath(directory.getAbsolutePath());
|
||||
|
||||
File[] files = directory.listFiles((dir, name) ->
|
||||
name.endsWith(".txt") || name.endsWith(".md")
|
||||
);
|
||||
|
||||
if (files == null || files.length == 0) {
|
||||
logger.warn("目录中没有找到支持的文件: {}", targetPath);
|
||||
result.setTotalFiles(0);
|
||||
result.setSuccess(true);
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
}
|
||||
|
||||
result.setTotalFiles(files.length);
|
||||
logger.info("开始索引目录: {}, 找到 {} 个文件", targetPath, files.length);
|
||||
|
||||
for (File file : files) {
|
||||
try {
|
||||
indexSingleFile(file.getAbsolutePath());
|
||||
result.incrementSuccessCount();
|
||||
logger.info("文件索引成功: {}", file.getName());
|
||||
} catch (Exception e) {
|
||||
result.incrementFailCount();
|
||||
result.addFailedFile(file.getAbsolutePath(), e.getMessage());
|
||||
logger.error("文件索引失败: {}", file.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
result.setSuccess(result.getFailCount() == 0);
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("索引目录失败", e);
|
||||
result.setSuccess(false);
|
||||
result.setErrorMessage(e.getMessage());
|
||||
result.setEndTime(LocalDateTime.now());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public void indexSingleFile(String filePath) throws Exception {
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
File file = path.toFile();
|
||||
|
||||
if (!file.exists() || !file.isFile()) {
|
||||
throw new IllegalArgumentException("文件不存在: " + filePath);
|
||||
}
|
||||
|
||||
logger.info("开始索引文件: {}", path);
|
||||
String content = Files.readString(path);
|
||||
deleteExistingData(path.toString());
|
||||
|
||||
List<DocumentChunk> chunks = chunkService.chunkDocument(content, path.toString());
|
||||
logger.info("文档分片完成: {} -> {} 个分片", filePath, chunks.size());
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
DocumentChunk chunk = chunks.get(i);
|
||||
try {
|
||||
// dense embedding 与 BM25 search_text 同源（title/path 增强）
|
||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
||||
Map<String, Object> metadata = buildMetadata(path.toString(), chunk, chunks.size());
|
||||
knowledgeStore.upsertChunk(
|
||||
chunk.getContent(), // 返回原文
|
||||
buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
|
||||
vector, // dense 向量
|
||||
metadata,
|
||||
chunk.getChunkIndex());
|
||||
logger.info("分片 {}/{} 索引成功", i + 1, chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("分片 {}/{} 索引失败", i + 1, chunks.size(), e);
|
||||
throw new RuntimeException("分片索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
|
||||
}
|
||||
|
||||
public void indexDocumentChunks(String docId, List<DocumentChunk> chunks, String category) throws Exception {
|
||||
indexDocumentChunks(docId, chunks, category, null);
|
||||
}
|
||||
|
||||
public void indexDocumentChunks(String docId,
|
||||
List<DocumentChunk> chunks,
|
||||
String category,
|
||||
Frontmatter frontmatter) throws Exception {
|
||||
if (chunks == null || chunks.isEmpty()) {
|
||||
throw new IllegalArgumentException("文档分块列表为空");
|
||||
}
|
||||
|
||||
logger.info("开始索引文档分块，docId: {}, 分块数: {}, 类别: {}", docId, chunks.size(), category);
|
||||
deleteDocumentChunks(docId);
|
||||
|
||||
for (int i = 0; i < chunks.size(); i++) {
|
||||
DocumentChunk chunk = chunks.get(i);
|
||||
try {
|
||||
// dense embedding 与 BM25 search_text 同源（title/path 增强）
|
||||
List<Float> vector = embeddingService.generateEmbedding(buildEmbeddingText(chunk));
|
||||
Map<String, Object> metadata = buildDocumentMetadata(docId, chunk, chunks.size(), category, frontmatter);
|
||||
knowledgeStore.upsertChunk(
|
||||
chunk.getContent(), // 返回原文
|
||||
buildSearchText(chunk), // BM25 语料；sparse 由 Milvus Function 生成
|
||||
vector, // dense 向量
|
||||
metadata,
|
||||
chunk.getChunkIndex());
|
||||
logger.info("文档分块 {}/{} 索引成功，docId: {}", i + 1, chunks.size(), docId);
|
||||
} catch (Exception e) {
|
||||
logger.error("文档分块 {}/{} 索引失败，docId: {}", i + 1, chunks.size(), docId, e);
|
||||
throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
logger.info("文档索引完成，docId: {}, 共 {} 个分块，类别: {}", docId, chunks.size(), category);
|
||||
}
|
||||
|
||||
public void deleteDocumentChunks(String docId) {
|
||||
try {
|
||||
knowledgeStore.deleteByDocId(docId);
|
||||
logger.info("删除文档旧数据成功，docId: {}", docId);
|
||||
} catch (Exception e) {
|
||||
logger.warn("删除文档旧数据异常，docId: {}", docId, e);
|
||||
}
|
||||
}
|
||||
|
||||
static Map<String, Object> buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks, String category) {
|
||||
return buildDocumentMetadata(docId, chunk, totalChunks, category, null);
|
||||
}
|
||||
|
||||
static Map<String, Object> buildDocumentMetadata(String docId,
|
||||
DocumentChunk chunk,
|
||||
int totalChunks,
|
||||
String category,
|
||||
Frontmatter frontmatter) {
|
||||
Map<String, Object> metadata = new HashMap<>();
|
||||
String source = firstNonBlank(frontmatter != null ? frontmatter.getSource() : null, "upload:" + docId);
|
||||
metadata.put("docId", docId);
|
||||
metadata.put("_source", source);
|
||||
metadata.put("source", source);
|
||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||
metadata.put("totalChunks", totalChunks);
|
||||
|
||||
String title = firstNonBlank(chunk.getTitle(), frontmatter != null ? frontmatter.getTitle() : null);
|
||||
if (title != null) {
|
||||
metadata.put("title", title);
|
||||
}
|
||||
String breadcrumb = firstNonBlank(frontmatter != null ? frontmatter.getBreadcrumb() : null, chunk.getBreadcrumb());
|
||||
if (breadcrumb != null) {
|
||||
metadata.put("breadcrumb", breadcrumb);
|
||||
}
|
||||
metadata.put("category", category != null && !category.isBlank() ? category : "upload");
|
||||
String kbScope = trimToNull(frontmatter != null ? frontmatter.getKbScope() : null);
|
||||
if (kbScope != null) {
|
||||
metadata.put("kb_scope", kbScope);
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dense embedding 输入。与 {@link #buildSearchText} 同源，保证 dense/BM25 看到同一增强文本。
|
||||
*/
|
||||
static String buildEmbeddingText(DocumentChunk chunk) {
|
||||
return buildSearchText(chunk);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造写入 Milvus 的检索文本（BM25 {@code search_text}，并复用为 dense embedding 输入）。
|
||||
*
|
||||
* <p>在正文前拼接 title / breadcrumb，提高「按标题或路径关键词」的 BM25 命中率，
|
||||
* 同时让 dense 向量也编码结构信息。无标题路径时退回纯 content。</p>
|
||||
*/
|
||||
static String buildSearchText(DocumentChunk chunk) {
|
||||
String content = trimToEmpty(chunk.getContent());
|
||||
String title = trimToEmpty(chunk.getTitle());
|
||||
String breadcrumb = trimToEmpty(chunk.getBreadcrumb());
|
||||
|
||||
if (title.isEmpty() && breadcrumb.isEmpty()) {
|
||||
return content;
|
||||
}
|
||||
|
||||
StringBuilder text = new StringBuilder();
|
||||
if (!title.isEmpty()) {
|
||||
text.append("Title: ").append(title).append("\n");
|
||||
}
|
||||
if (!breadcrumb.isEmpty()) {
|
||||
text.append("Path: ").append(breadcrumb).append("\n");
|
||||
}
|
||||
text.append("Content:\n").append(content);
|
||||
return text.toString();
|
||||
}
|
||||
|
||||
private static String trimToEmpty(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
private static String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
String trimmed = trimToNull(value);
|
||||
if (trimmed != null) {
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void deleteExistingData(String filePath) {
|
||||
try {
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
||||
knowledgeStore.deleteBySource(normalizedPath);
|
||||
logger.info("已删除文件的旧数据: {}", normalizedPath);
|
||||
} catch (Exception e) {
|
||||
logger.warn("删除旧数据失败（可能是首次索引）: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> buildMetadata(String filePath, DocumentChunk chunk, int totalChunks) {
|
||||
Map<String, Object> metadata = new HashMap<>();
|
||||
Path path = Paths.get(filePath).normalize();
|
||||
String normalizedPath = path.toString().replace(File.separator, "/");
|
||||
|
||||
Path fileName = path.getFileName();
|
||||
String fileNameStr = fileName != null ? fileName.toString() : "";
|
||||
String extension = "";
|
||||
int dotIndex = fileNameStr.lastIndexOf('.');
|
||||
if (dotIndex > 0) {
|
||||
extension = fileNameStr.substring(dotIndex);
|
||||
}
|
||||
|
||||
metadata.put("_source", normalizedPath);
|
||||
metadata.put("source", normalizedPath);
|
||||
metadata.put("_extension", extension);
|
||||
metadata.put("_file_name", fileNameStr);
|
||||
|
||||
String category = extractCategory(normalizedPath);
|
||||
if (category != null && !category.isEmpty()) {
|
||||
metadata.put("category", category);
|
||||
}
|
||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||
metadata.put("totalChunks", totalChunks);
|
||||
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
|
||||
metadata.put("title", chunk.getTitle());
|
||||
}
|
||||
if (chunk.getBreadcrumb() != null && !chunk.getBreadcrumb().isEmpty()) {
|
||||
metadata.put("breadcrumb", chunk.getBreadcrumb());
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private String extractCategory(String filePath) {
|
||||
try {
|
||||
String normalized = filePath.replace("\\", "/");
|
||||
int docsIndex = normalized.indexOf("aiops-docs/");
|
||||
if (docsIndex >= 0) {
|
||||
String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
|
||||
int slashIndex = afterDocs.indexOf("/");
|
||||
if (slashIndex > 0) {
|
||||
return afterDocs.substring(0, slashIndex);
|
||||
}
|
||||
}
|
||||
int firstSlash = normalized.indexOf("/");
|
||||
if (firstSlash > 0) {
|
||||
return normalized.substring(0, firstSlash);
|
||||
}
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
logger.warn("提取类别失败，路径: {}", filePath, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
public static class IndexingResult {
|
||||
@Setter
|
||||
private boolean success;
|
||||
@Setter
|
||||
private String directoryPath;
|
||||
@Setter
|
||||
private int totalFiles;
|
||||
private int successCount;
|
||||
private int failCount;
|
||||
@Setter
|
||||
private LocalDateTime startTime;
|
||||
@Setter
|
||||
private LocalDateTime endTime;
|
||||
@Setter
|
||||
private String errorMessage;
|
||||
private Map<String, String> failedFiles = new HashMap<>();
|
||||
|
||||
public void incrementSuccessCount() {
|
||||
this.successCount++;
|
||||
}
|
||||
|
||||
public void incrementFailCount() {
|
||||
this.failCount++;
|
||||
}
|
||||
|
||||
public long getDurationMs() {
|
||||
if (startTime != null && endTime != null) {
|
||||
return java.time.Duration.between(startTime, endTime).toMillis();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public void addFailedFile(String filePath, String error) {
|
||||
this.failedFiles.put(filePath, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.service.milvus.MilvusHybridKnowledgeStore;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* 知识库向量检索门面（lookup_knowledge / RAG 召回入口）。
|
||||
*
|
||||
* <p><b>唯一后端：</b>{@link MilvusHybridKnowledgeStore}（Milvus Java SDK v2）。</p>
|
||||
*
|
||||
* <h3>模式切换</h3>
|
||||
* <p>{@code retrieval.search.mode}（同库查询算法，非两套写入）：</p>
|
||||
* <ul>
|
||||
* <li>{@code hybrid} —— 线上主路径：dense + 服务端 BM25 + RRF</li>
|
||||
* <li>{@code dense} —— 对照/评测：仅 dense ANN</li>
|
||||
* </ul>
|
||||
* <p>命中 {@link SearchResult#scoreLabel} 仅为 {@link RetrievalScoreLabels#DENSE} /
|
||||
* {@link RetrievalScoreLabels#HYBRID}。质量分由后处理 {@code RetrievalScoreNormalizer} 统一计算。</p>
|
||||
*/
|
||||
@Service
|
||||
public class VectorSearchService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(VectorSearchService.class);
|
||||
|
||||
@Autowired
|
||||
private MilvusHybridKnowledgeStore knowledgeStore;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService embeddingService;
|
||||
|
||||
/**
|
||||
* 检索模式：{@code hybrid}（主路径）| {@code dense}（召回对照）。
|
||||
*/
|
||||
@Value("${retrieval.search.mode:dense}")
|
||||
private String searchMode = "dense";
|
||||
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK) {
|
||||
return searchSimilarDocuments(query, topK, null);
|
||||
}
|
||||
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK, String category) {
|
||||
String mode = searchMode == null ? "dense" : searchMode.trim().toLowerCase(Locale.ROOT);
|
||||
List<Float> queryVector = embeddingService.generateQueryVector(query);
|
||||
if ("hybrid".equals(mode)) {
|
||||
logger.info("Hybrid dense+BM25 search topK={} category={} collection={}",
|
||||
topK, category, knowledgeStore.collectionName());
|
||||
return knowledgeStore.searchHybrid(query, queryVector, topK, category);
|
||||
}
|
||||
logger.info("Dense search topK={} category={} collection={}",
|
||||
topK, category, knowledgeStore.collectionName());
|
||||
return knowledgeStore.searchDense(query, queryVector, topK, category);
|
||||
}
|
||||
|
||||
/**
|
||||
* 单条召回结果。列表顺序即检索权威序（adapter 赋 originalRank=1..n）。
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code scoreLabel=dense}：{@link #score} = L2 距离（越小越好）</li>
|
||||
* <li>{@code scoreLabel=hybrid}：{@link #score}/{@link #rawScore} = 引擎融合分；
|
||||
* 后处理 quality 主要按 rank 映射，不把 score 当 L2</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Setter
|
||||
@Getter
|
||||
public static class SearchResult {
|
||||
private String id;
|
||||
private String content;
|
||||
/**
|
||||
* 引擎主分：dense=L2；hybrid=融合分（量纲由 scoreLabel 解释）。
|
||||
*/
|
||||
private float score;
|
||||
/** 引擎原始分（与 score 同源或更细，便于调试）。 */
|
||||
private Double rawScore;
|
||||
/** {@link RetrievalScoreLabels#DENSE} 或 {@link RetrievalScoreLabels#HYBRID}。 */
|
||||
private String scoreLabel;
|
||||
/**
|
||||
* Optional dense L2 for the same id (hybrid path only).
|
||||
* Used for absolute quality / low-quality gates; does <b>not</b> replace sort order.
|
||||
*/
|
||||
private Double denseDistance;
|
||||
/** metadata JSON 字符串（docId、source、title…）。 */
|
||||
private String metadata;
|
||||
}
|
||||
}
|
||||
@@ -1,547 +0,0 @@
|
||||
package com.superbiz.agent.service.milvus;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.superbiz.agent.config.MilvusProperties;
|
||||
import com.superbiz.agent.constant.MilvusConstants;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import com.superbiz.agent.service.retrieval.RetrievalScoreLabels;
|
||||
import io.milvus.common.clientenum.FunctionType;
|
||||
import io.milvus.v2.client.ConnectConfig;
|
||||
import io.milvus.v2.client.MilvusClientV2;
|
||||
import io.milvus.v2.common.DataType;
|
||||
import io.milvus.v2.common.IndexParam;
|
||||
import io.milvus.v2.service.collection.request.AddFieldReq;
|
||||
import io.milvus.v2.service.collection.request.CreateCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.DropCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.HasCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.LoadCollectionReq;
|
||||
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
|
||||
import io.milvus.v2.service.index.request.CreateIndexReq;
|
||||
import io.milvus.v2.service.vector.request.AnnSearchReq;
|
||||
import io.milvus.v2.service.vector.request.DeleteReq;
|
||||
import io.milvus.v2.service.vector.request.HybridSearchReq;
|
||||
import io.milvus.v2.service.vector.request.InsertReq;
|
||||
import io.milvus.v2.service.vector.request.SearchReq;
|
||||
import io.milvus.v2.service.vector.request.data.BaseVector;
|
||||
import io.milvus.v2.service.vector.request.data.EmbeddedText;
|
||||
import io.milvus.v2.service.vector.request.data.FloatVec;
|
||||
import io.milvus.v2.service.vector.request.ranker.RRFRanker;
|
||||
import io.milvus.v2.service.vector.response.SearchResp;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
* 知识库向量后端（Milvus Java SDK v2）—— dense + BM25 混合检索的唯一实现。
|
||||
*
|
||||
* <h3>为什么不用 Spring AI {@code spring-ai-starter-vector-store-milvus}</h3>
|
||||
* <ul>
|
||||
* <li>Spring AI Milvus starter（截至 2.0.0 / 1.1.8）只封装 dense {@code similaritySearch}。</li>
|
||||
* <li>底层仍是 V1 {@code MilvusServiceClient} + 单路 {@code SearchParam}，无 {@code hybridSearch} /
|
||||
* BM25 Function / {@link RRFRanker}。</li>
|
||||
* <li>真混合检索（dense ANN + 服务端 BM25 sparse，再 RRF 融合）必须走 Milvus SDK v2，
|
||||
* 见 {@link #searchHybrid}。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <h3>Collection schema（默认名 {@code biz}）</h3>
|
||||
* <pre>
|
||||
* id VarChar PK
|
||||
* content VarChar —— 原文，返回给上层
|
||||
* search_text VarChar+analyzer —— BM25 输入文本（可含 title/path 增强）
|
||||
* sparse_vector SparseFloatVector —— 由 BM25 Function 从 search_text 自动生成，写入时不必填
|
||||
* vector FloatVector —— dense 向量（应用侧 embedding）
|
||||
* metadata JSON —— docId / source / category / kb_scope 等
|
||||
* </pre>
|
||||
*
|
||||
* <h3>检索模式</h3>
|
||||
* <ul>
|
||||
* <li>{@link #searchDense}：单路 L2 ANN；{@code scoreLabel=dense}。</li>
|
||||
* <li>{@link #searchHybrid}：dense + BM25 + 服务端 {@link RRFRanker}；{@code scoreLabel=hybrid}；
|
||||
* 返回序即 RRF 序，不再用 dense L2 覆盖主分。</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>配置入口：{@code milvus.collection}、{@code retrieval.search.mode}、{@code retrieval.hybrid.rrf-k}。</p>
|
||||
*/
|
||||
@Service
|
||||
public class MilvusHybridKnowledgeStore {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MilvusHybridKnowledgeStore.class);
|
||||
private static final Gson GSON = new Gson();
|
||||
|
||||
/** 主键（稳定 UUID，由 source + chunkIndex 派生，便于幂等重写）。 */
|
||||
public static final String FIELD_ID = "id";
|
||||
/** 返回给 LLM / 上层的原文 chunk。 */
|
||||
public static final String FIELD_CONTENT = "content";
|
||||
/**
|
||||
* BM25 输入字段。写入明文；Milvus 侧 analyzer + BM25 Function 生成 {@link #FIELD_SPARSE}。
|
||||
* 通常比 content 多带 title/path 等检索增强词。
|
||||
*/
|
||||
public static final String FIELD_SEARCH_TEXT = "search_text";
|
||||
/** 稀疏向量字段；由 BM25 Function 自动产出，insert 时不要手动填。 */
|
||||
public static final String FIELD_SPARSE = "sparse_vector";
|
||||
/** Dense 向量字段（应用侧 EmbeddingModel 生成）。 */
|
||||
public static final String FIELD_DENSE = "vector";
|
||||
/** 业务元数据 JSON（过滤、证据身份、展示用）。 */
|
||||
public static final String FIELD_METADATA = "metadata";
|
||||
|
||||
private final MilvusProperties milvusProperties;
|
||||
|
||||
@Value("${milvus.collection:biz}")
|
||||
private String collectionName = "biz";
|
||||
|
||||
/**
|
||||
* RRF 平滑参数 k：score(d) = Σ 1/(k + rank_i(d))。
|
||||
* k 越大，各路排名差异被压得越平；默认 60 与常见 RRF 设定一致。
|
||||
*/
|
||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
||||
private int rrfK = 60;
|
||||
|
||||
/** 非空时追加 {@code metadata.kb_scope} 过滤，实现多知识域隔离。 */
|
||||
@Value("${retrieval.kb-scope:}")
|
||||
private String kbScope = "";
|
||||
|
||||
private volatile MilvusClientV2 client;
|
||||
|
||||
public MilvusHybridKnowledgeStore(MilvusProperties milvusProperties) {
|
||||
this.milvusProperties = milvusProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* 懒连接：首次调用时建连、确保 collection schema 存在并 load。
|
||||
* 线程安全；后续检索/写入复用同一 {@link MilvusClientV2}。
|
||||
*/
|
||||
public synchronized MilvusClientV2 client() {
|
||||
if (client == null) {
|
||||
client = connect();
|
||||
ensureCollection(client);
|
||||
loadCollection(client);
|
||||
}
|
||||
return client;
|
||||
}
|
||||
|
||||
public String collectionName() {
|
||||
return collectionName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写入单个 chunk（dense + BM25 所需明文）。
|
||||
*
|
||||
* <p>只插入 {@code content / search_text / vector / metadata}；
|
||||
* {@code sparse_vector} 由 collection 上的 BM25 Function 在服务端从 {@code search_text} 生成。</p>
|
||||
*
|
||||
* <p>id 由 {@code source|docId + chunkIndex} 的 nameUUID 派生，同一 chunk 重复写入会得到相同 id
|
||||
*（配合先 delete 再 insert 的上层逻辑实现覆盖）。</p>
|
||||
*
|
||||
* @param content 原文（返回字段）
|
||||
* @param searchText BM25 / 可与 dense embedding 同源的检索文本
|
||||
* @param denseVector 应用侧 embedding
|
||||
* @param metadata 须尽量带 {@code _source} 或 {@code docId}，供 id 与过滤使用
|
||||
* @param chunkIndex 分片序号
|
||||
*/
|
||||
public void upsertChunk(String content,
|
||||
String searchText,
|
||||
List<Float> denseVector,
|
||||
Map<String, Object> metadata,
|
||||
int chunkIndex) {
|
||||
String source = metadata == null ? null : stringVal(metadata.get("_source"));
|
||||
if (source == null) {
|
||||
source = metadata == null ? null : stringVal(metadata.get("source"));
|
||||
}
|
||||
if (source == null) {
|
||||
source = metadata == null ? null : stringVal(metadata.get("docId"));
|
||||
}
|
||||
String idSeed = (source == null ? "chunk" : source) + "_" + chunkIndex;
|
||||
String id = UUID.nameUUIDFromBytes(idSeed.getBytes()).toString();
|
||||
|
||||
JsonObject row = new JsonObject();
|
||||
row.addProperty(FIELD_ID, id);
|
||||
row.addProperty(FIELD_CONTENT, content == null ? "" : content);
|
||||
// 仅写明文；sparse 由 BM25 Function(search_text -> sparse_vector) 自动生成
|
||||
row.addProperty(FIELD_SEARCH_TEXT, searchText == null ? "" : searchText);
|
||||
row.add(FIELD_DENSE, GSON.toJsonTree(denseVector));
|
||||
row.add(FIELD_METADATA, GSON.toJsonTree(metadata == null ? Map.of() : metadata));
|
||||
|
||||
client().insert(InsertReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.data(List.of(row))
|
||||
.build());
|
||||
}
|
||||
|
||||
/** 按 metadata.docId 删除该文档全部 chunk（重建/覆盖前调用）。 */
|
||||
public void deleteByDocId(String docId) {
|
||||
if (docId == null || docId.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String filter = "metadata[\"docId\"] == \"" + escapeFilter(docId) + "\"";
|
||||
client().delete(DeleteReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.filter(filter)
|
||||
.build());
|
||||
}
|
||||
|
||||
/** 按 metadata._source（规范化路径）删除，用于按文件路径重索引。 */
|
||||
public void deleteBySource(String sourcePath) {
|
||||
if (sourcePath == null || sourcePath.isBlank()) {
|
||||
return;
|
||||
}
|
||||
String normalized = sourcePath.replace('\\', '/');
|
||||
String filter = "metadata[\"_source\"] == \"" + escapeFilter(normalized) + "\"";
|
||||
client().delete(DeleteReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.filter(filter)
|
||||
.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除并重建当前知识 collection（空的 dense+BM25 schema）。
|
||||
* 供 {@code /api/knowledge/rebuild-hybrid} 与重建脚本使用；会销毁该 collection 全部向量。
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("collection", collectionName);
|
||||
MilvusClientV2 milvusClient = client();
|
||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
result.put("existedBefore", Boolean.TRUE.equals(exists));
|
||||
if (Boolean.TRUE.equals(exists)) {
|
||||
try {
|
||||
milvusClient.releaseCollection(ReleaseCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
} catch (Exception e) {
|
||||
log.warn("Release collection before drop failed (continuing): {}", e.getMessage());
|
||||
}
|
||||
milvusClient.dropCollection(DropCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
log.info("Dropped hybrid collection '{}'", collectionName);
|
||||
result.put("dropped", true);
|
||||
} else {
|
||||
result.put("dropped", false);
|
||||
}
|
||||
ensureCollection(milvusClient);
|
||||
loadCollection(milvusClient);
|
||||
result.put("recreated", true);
|
||||
result.put("loaded", true);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单路 dense ANN（L2）。
|
||||
* {@code score} = L2 距离（越小越好）；{@code scoreLabel} = {@link RetrievalScoreLabels#DENSE}。
|
||||
*/
|
||||
public List<VectorSearchService.SearchResult> searchDense(String queryEmbeddingText,
|
||||
List<Float> queryVector,
|
||||
int topK,
|
||||
String category) {
|
||||
String filter = buildFilter(category);
|
||||
SearchReq.SearchReqBuilder builder = SearchReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.annsField(FIELD_DENSE)
|
||||
.data(List.of(new FloatVec(queryVector)))
|
||||
.topK(topK)
|
||||
.outputFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
||||
.metricType(IndexParam.MetricType.L2);
|
||||
if (filter != null) {
|
||||
builder.filter(filter);
|
||||
}
|
||||
SearchResp resp = client().search(builder.build());
|
||||
return toSearchResults(resp, RetrievalScoreLabels.DENSE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dense + BM25 真混合检索（Milvus 服务端融合）。
|
||||
*
|
||||
* <ol>
|
||||
* <li>dense 子路：{@code vector}，L2</li>
|
||||
* <li>BM25 子路：{@code sparse_vector} + {@link EmbeddedText}</li>
|
||||
* <li>{@link HybridSearchReq} + {@link RRFRanker} → 返回序即权威序</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>{@code scoreLabel=hybrid}；{@code score}/{@code rawScore} 保留引擎融合分，
|
||||
* <b>不</b>用 dense L2 覆盖主分或改 label。可选并行 dense 探测仅填充
|
||||
* {@link VectorSearchService.SearchResult#setDenseDistance}，供后处理绝对质量闸门
|
||||
* （如 L0 filter low-quality → unfiltered retry），排序仍以 RRF 返回序为准。</p>
|
||||
*/
|
||||
public List<VectorSearchService.SearchResult> searchHybrid(String queryText,
|
||||
List<Float> queryVector,
|
||||
int topK,
|
||||
String category) {
|
||||
String filter = buildFilter(category);
|
||||
int pathTopK = Math.max(topK, 10);
|
||||
|
||||
AnnSearchReq.AnnSearchReqBuilder denseAnn = AnnSearchReq.builder()
|
||||
.vectorFieldName(FIELD_DENSE)
|
||||
.vectors(List.of((BaseVector) new FloatVec(queryVector)))
|
||||
.topK(pathTopK)
|
||||
.metricType(IndexParam.MetricType.L2)
|
||||
.params("{\"nprobe\":10}");
|
||||
if (filter != null) {
|
||||
denseAnn.filter(filter);
|
||||
}
|
||||
|
||||
AnnSearchReq.AnnSearchReqBuilder sparseAnn = AnnSearchReq.builder()
|
||||
.vectorFieldName(FIELD_SPARSE)
|
||||
.vectors(List.of((BaseVector) new EmbeddedText(queryText == null ? "" : queryText)))
|
||||
.topK(pathTopK)
|
||||
.metricType(IndexParam.MetricType.BM25);
|
||||
if (filter != null) {
|
||||
sparseAnn.filter(filter);
|
||||
}
|
||||
|
||||
HybridSearchReq hybridReq = HybridSearchReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.searchRequests(List.of(denseAnn.build(), sparseAnn.build()))
|
||||
.ranker(new RRFRanker(Math.max(1, rrfK)))
|
||||
.topK(topK)
|
||||
.outFields(List.of(FIELD_ID, FIELD_CONTENT, FIELD_METADATA))
|
||||
.build();
|
||||
|
||||
SearchResp hybridResp = client().hybridSearch(hybridReq);
|
||||
List<VectorSearchService.SearchResult> fused = toSearchResults(hybridResp, RetrievalScoreLabels.HYBRID);
|
||||
attachDenseDistances(fused, queryText, queryVector, pathTopK, category);
|
||||
return fused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach dense L2 by id for quality gates only — never overwrites hybrid score/label/order.
|
||||
*/
|
||||
private void attachDenseDistances(List<VectorSearchService.SearchResult> fused,
|
||||
String queryText,
|
||||
List<Float> queryVector,
|
||||
int pathTopK,
|
||||
String category) {
|
||||
if (fused == null || fused.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Map<String, Float> denseById = new HashMap<>();
|
||||
for (VectorSearchService.SearchResult denseHit :
|
||||
searchDense(queryText, queryVector, pathTopK, category)) {
|
||||
if (denseHit.getId() != null) {
|
||||
denseById.put(denseHit.getId(), denseHit.getScore());
|
||||
}
|
||||
}
|
||||
for (VectorSearchService.SearchResult hit : fused) {
|
||||
Float l2 = denseById.get(hit.getId());
|
||||
if (l2 != null) {
|
||||
hit.setDenseDistance(l2.doubleValue());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Dense distance attach for hybrid quality gate failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Milvus {@link SearchResp} 映射为上层结果；列表顺序即检索权威序（adapter 赋 originalRank）。
|
||||
*/
|
||||
private List<VectorSearchService.SearchResult> toSearchResults(SearchResp resp, String scoreLabel) {
|
||||
List<VectorSearchService.SearchResult> out = new ArrayList<>();
|
||||
if (resp == null || resp.getSearchResults() == null || resp.getSearchResults().isEmpty()) {
|
||||
return out;
|
||||
}
|
||||
List<SearchResp.SearchResult> first = resp.getSearchResults().get(0);
|
||||
if (first == null) {
|
||||
return out;
|
||||
}
|
||||
for (SearchResp.SearchResult row : first) {
|
||||
VectorSearchService.SearchResult mapped = new VectorSearchService.SearchResult();
|
||||
Object id = row.getId();
|
||||
mapped.setId(id == null ? null : String.valueOf(id));
|
||||
Map<String, Object> entity = row.getEntity() == null ? Map.of() : row.getEntity();
|
||||
Object content = entity.get(FIELD_CONTENT);
|
||||
mapped.setContent(content == null ? null : String.valueOf(content));
|
||||
Object metadata = entity.get(FIELD_METADATA);
|
||||
if (metadata instanceof JsonObject jsonObject) {
|
||||
mapped.setMetadata(jsonObject.toString());
|
||||
} else if (metadata instanceof Map<?, ?> map) {
|
||||
mapped.setMetadata(GSON.toJson(map));
|
||||
} else if (metadata != null) {
|
||||
mapped.setMetadata(String.valueOf(metadata));
|
||||
}
|
||||
Float score = row.getScore();
|
||||
mapped.setRawScore(score == null ? null : score.doubleValue());
|
||||
mapped.setScoreLabel(scoreLabel);
|
||||
// dense: L2；hybrid: 引擎融合分（后处理 quality 主要看 rank，不依赖此量纲）
|
||||
mapped.setScore(score == null ? 0f : score);
|
||||
out.add(mapped);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* 组装标量过滤表达式：category、kb_scope（配置级）可叠加，用 {@code &&} 连接。
|
||||
*/
|
||||
private String buildFilter(String category) {
|
||||
List<String> parts = new ArrayList<>();
|
||||
String categoryFilter = trimToNull(category);
|
||||
if (categoryFilter != null) {
|
||||
parts.add("metadata[\"category\"] == \"" + escapeFilter(categoryFilter) + "\"");
|
||||
}
|
||||
String scope = trimToNull(kbScope);
|
||||
if (scope != null) {
|
||||
parts.add("metadata[\"kb_scope\"] == \"" + escapeFilter(scope) + "\"");
|
||||
}
|
||||
return parts.isEmpty() ? null : String.join(" && ", parts);
|
||||
}
|
||||
|
||||
private MilvusClientV2 connect() {
|
||||
String uri;
|
||||
if (milvusProperties.isSecure() || milvusProperties.getPort() == 443) {
|
||||
uri = "https://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
||||
} else {
|
||||
uri = "http://" + milvusProperties.getHost() + ":" + milvusProperties.getPort();
|
||||
}
|
||||
ConnectConfig.ConnectConfigBuilder builder = ConnectConfig.builder()
|
||||
.uri(uri)
|
||||
.connectTimeoutMs(milvusProperties.getTimeout() == null ? 10000L : milvusProperties.getTimeout());
|
||||
if (milvusProperties.getToken() != null && !milvusProperties.getToken().isBlank()) {
|
||||
builder.token(milvusProperties.getToken());
|
||||
builder.secure(true);
|
||||
} else if (milvusProperties.getUsername() != null && !milvusProperties.getUsername().isBlank()) {
|
||||
builder.username(milvusProperties.getUsername());
|
||||
builder.password(milvusProperties.getPassword());
|
||||
}
|
||||
if (milvusProperties.getDatabase() != null && !milvusProperties.getDatabase().isBlank()) {
|
||||
builder.dbName(milvusProperties.getDatabase());
|
||||
}
|
||||
log.info("Connecting MilvusClientV2 uri={} db={} collection={}",
|
||||
uri, milvusProperties.getDatabase(), collectionName);
|
||||
return new MilvusClientV2(builder.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* 若不存在则创建 dense+BM25 hybrid collection。
|
||||
*
|
||||
* <p>关键点：</p>
|
||||
* <ul>
|
||||
* <li>{@code search_text} 开启 analyzer，作为 BM25 语料。</li>
|
||||
* <li>{@link FunctionType#BM25}：input={@code search_text} → output={@code sparse_vector}。</li>
|
||||
* <li>dense：IVF_FLAT + L2；sparse：SPARSE_INVERTED_INDEX + BM25。</li>
|
||||
* </ul>
|
||||
* <p>已存在的 collection 不会改 schema；schema 变更需走 {@link #dropAndRecreateCollection()}。</p>
|
||||
*/
|
||||
private void ensureCollection(MilvusClientV2 milvusClient) {
|
||||
Boolean exists = milvusClient.hasCollection(HasCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
if (Boolean.TRUE.equals(exists)) {
|
||||
log.info("Hybrid collection '{}' already exists", collectionName);
|
||||
return;
|
||||
}
|
||||
log.info("Creating hybrid collection '{}'", collectionName);
|
||||
|
||||
CreateCollectionReq.CollectionSchema schema = milvusClient.createSchema();
|
||||
schema.setEnableDynamicField(false);
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_ID)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.ID_MAX_LENGTH)
|
||||
.isPrimaryKey(true)
|
||||
.autoID(false)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_CONTENT)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.build());
|
||||
// BM25 语料字段：必须 enableAnalyzer，Function 才能从文本生成 sparse
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_SEARCH_TEXT)
|
||||
.dataType(DataType.VarChar)
|
||||
.maxLength(MilvusConstants.CONTENT_MAX_LENGTH)
|
||||
.enableAnalyzer(true)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_SPARSE)
|
||||
.dataType(DataType.SparseFloatVector)
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_DENSE)
|
||||
.dataType(DataType.FloatVector)
|
||||
.dimension(milvusProperties.getVectorDim())
|
||||
.build());
|
||||
schema.addField(AddFieldReq.builder()
|
||||
.fieldName(FIELD_METADATA)
|
||||
.dataType(DataType.JSON)
|
||||
.build());
|
||||
// 写入 search_text 时，Milvus 自动维护 sparse_vector（应用层 insert 不填 sparse）
|
||||
schema.addFunction(CreateCollectionReq.Function.builder()
|
||||
.functionType(FunctionType.BM25)
|
||||
.name("bm25_fn")
|
||||
.inputFieldNames(List.of(FIELD_SEARCH_TEXT))
|
||||
.outputFieldNames(List.of(FIELD_SPARSE))
|
||||
.build());
|
||||
|
||||
milvusClient.createCollection(CreateCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.collectionSchema(schema)
|
||||
.description("Knowledge hybrid dense+BM25 collection")
|
||||
.numShards(MilvusConstants.DEFAULT_SHARD_NUMBER)
|
||||
.build());
|
||||
|
||||
List<IndexParam> indexes = List.of(
|
||||
IndexParam.builder()
|
||||
.fieldName(FIELD_DENSE)
|
||||
.indexType(IndexParam.IndexType.IVF_FLAT)
|
||||
.metricType(IndexParam.MetricType.L2)
|
||||
.extraParams(Map.of("nlist", 128))
|
||||
.build(),
|
||||
IndexParam.builder()
|
||||
.fieldName(FIELD_SPARSE)
|
||||
.indexType(IndexParam.IndexType.SPARSE_INVERTED_INDEX)
|
||||
.metricType(IndexParam.MetricType.BM25)
|
||||
.build()
|
||||
);
|
||||
milvusClient.createIndex(CreateIndexReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.indexParams(indexes)
|
||||
.build());
|
||||
log.info("Hybrid collection '{}' created with dense+BM25 indexes", collectionName);
|
||||
}
|
||||
|
||||
private void loadCollection(MilvusClientV2 milvusClient) {
|
||||
milvusClient.loadCollection(LoadCollectionReq.builder()
|
||||
.collectionName(collectionName)
|
||||
.build());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void close() {
|
||||
if (client != null) {
|
||||
try {
|
||||
client.close();
|
||||
} catch (Exception e) {
|
||||
log.warn("Error closing MilvusClientV2: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 过滤表达式字符串转义，防止引号打断 expr。 */
|
||||
private static String escapeFilter(String value) {
|
||||
return value.replace("\\", "\\\\").replace("\"", "\\\"");
|
||||
}
|
||||
|
||||
private static String trimToNull(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
private static String stringVal(Object value) {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,9 @@ import java.util.List;
|
||||
/**
|
||||
* 知识语义检索的应用边界端口。
|
||||
*
|
||||
* <p>实现可对接 dense / hybrid 等引擎，但不得向上层泄漏 SDK 类型。
|
||||
* 当前实现：{@link VectorKnowledgeSearchAdapter} → {@code VectorSearchService}
|
||||
* → {@code MilvusHybridKnowledgeStore}（Milvus SDK v2 dense 或 dense+BM25 RRF）。</p>
|
||||
* <p>实现可对接 dense / hybrid 等引擎，但不得向上层泄漏远端 API 类型。
|
||||
* 当前实现：{@link PyRagKnowledgeSearchAdapter}（py-rag 知识服务 /api/v1/search，
|
||||
* 服务端负责 hybrid 融合、BM25、rerank 与判级）。RAG 模块抽离后为唯一实现。</p>
|
||||
*/
|
||||
public interface KnowledgeSearchPort {
|
||||
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Sparse-lite lexical ranking over already recalled candidates.
|
||||
* Not a substitute for inverted-index BM25; expands ordering signal only.
|
||||
*/
|
||||
public final class LexicalRanker {
|
||||
|
||||
private LexicalRanker() {
|
||||
}
|
||||
|
||||
public static List<KnowledgeSearchHit> rank(String query, List<KnowledgeSearchHit> candidates) {
|
||||
if (candidates == null || candidates.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
Set<String> terms = tokenize(query);
|
||||
if (terms.isEmpty()) {
|
||||
return List.copyOf(candidates);
|
||||
}
|
||||
List<ScoredHit> scored = new ArrayList<>(candidates.size());
|
||||
for (KnowledgeSearchHit hit : candidates) {
|
||||
String haystack = (nullToEmpty(hit.title()) + " "
|
||||
+ nullToEmpty(hit.breadcrumb()) + " "
|
||||
+ nullToEmpty(hit.content())).toLowerCase(Locale.ROOT);
|
||||
int hits = 0;
|
||||
for (String term : terms) {
|
||||
if (haystack.contains(term)) {
|
||||
hits++;
|
||||
}
|
||||
}
|
||||
double coverage = hits / (double) terms.size();
|
||||
scored.add(new ScoredHit(hit, coverage, hits));
|
||||
}
|
||||
scored.sort(Comparator
|
||||
.comparingDouble((ScoredHit s) -> s.coverage).reversed()
|
||||
.thenComparingInt((ScoredHit s) -> s.hits).reversed()
|
||||
.thenComparingInt(s -> s.hit.originalRank()));
|
||||
return scored.stream().map(s -> s.hit).toList();
|
||||
}
|
||||
|
||||
static Set<String> tokenize(String query) {
|
||||
if (query == null || query.isBlank()) {
|
||||
return Set.of();
|
||||
}
|
||||
String normalized = query.toLowerCase(Locale.ROOT);
|
||||
String[] parts = normalized.split("[^\\p{IsAlphabetic}\\p{IsDigit}]+");
|
||||
Set<String> terms = new LinkedHashSet<>();
|
||||
for (String part : parts) {
|
||||
if (part == null) {
|
||||
continue;
|
||||
}
|
||||
String term = part.trim();
|
||||
if (term.length() >= 2) {
|
||||
terms.add(term);
|
||||
}
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
|
||||
private static String nullToEmpty(String value) {
|
||||
return value == null ? "" : value;
|
||||
}
|
||||
|
||||
private record ScoredHit(KnowledgeSearchHit hit, double coverage, int hits) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.superbiz.agent.client.PyRagClient;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchHit;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchRequest;
|
||||
import com.superbiz.agent.client.PyRagClient.PyRagSearchResponse;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link KnowledgeSearchPort} 的 py-rag 远端实现（RAG 模块抽离后的唯一检索后端）。
|
||||
*
|
||||
* <p>原进程内链路（VectorKnowledgeSearchAdapter → VectorSearchService → MilvusHybridKnowledgeStore）
|
||||
* 已由 py-rag 服务端接管：hybrid 融合、BM25、rerank、chunk 去重、判级阈值全部下沉。
|
||||
* Java 侧只做请求映射与命中结构归一化，不碰检索算法。</p>
|
||||
*
|
||||
* <h3>映射约定</h3>
|
||||
* <ul>
|
||||
* <li>mode：{@link KnowledgeSearchMode#DENSE} → {@code semantic}，{@link KnowledgeSearchMode#HYBRID} → {@code hybrid}</li>
|
||||
* <li>retrieve_k = return_n = topK：返回 topK 条精排后命中，chunk 去重/截断仍由
|
||||
* {@code KnowledgeEvidencePostProcessor} 统一负责，故 max_chunks_per_document 同步放大避免服务端预截断</li>
|
||||
* <li>category：{@code categoryFilter} 透传；null = 不过滤；kb_scope 不传，由服务端部署配置决定</li>
|
||||
* <li>score：py-rag rerank 绝对相关分（[0,1]，越大越好），scoreLabel =
|
||||
* {@link RetrievalScoreLabels#RERANK}（quality 原样采用，不做 L2/rank 归一化）</li>
|
||||
* <li>evidence_key：{@code docId#chunk-N}，与 EvidenceGuard 验真约定一致；
|
||||
* {@code evidence_status=no_evidence} 时服务端保证 hits=[]，按"无知识"正常返回</li>
|
||||
* </ul>
|
||||
*/
|
||||
@Component
|
||||
public class PyRagKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
||||
|
||||
private static final String CHUNK_MARK = "#chunk-";
|
||||
|
||||
private final PyRagClient pyRagClient;
|
||||
|
||||
public PyRagKnowledgeSearchAdapter(PyRagClient pyRagClient) {
|
||||
this.pyRagClient = pyRagClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
||||
PyRagSearchResponse response = pyRagClient.search(toPyRagRequest(request));
|
||||
if (response == null || response.hits() == null || response.hits().isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<KnowledgeSearchHit> hits = new ArrayList<>(response.hits().size());
|
||||
for (int i = 0; i < response.hits().size(); i++) {
|
||||
hits.add(toHit(response.hits().get(i), i + 1));
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
/** 请求映射：topK 同时作为召回宽度与返回条数，服务端不预截断 chunk。 */
|
||||
private PyRagSearchRequest toPyRagRequest(KnowledgeSearchRequest request) {
|
||||
return new PyRagSearchRequest(
|
||||
request.query(),
|
||||
request.mode() == KnowledgeSearchMode.HYBRID ? "hybrid" : "semantic",
|
||||
request.topK(),
|
||||
request.topK(),
|
||||
request.topK(),
|
||||
blankToNull(request.categoryFilter()),
|
||||
null);
|
||||
}
|
||||
|
||||
private KnowledgeSearchHit toHit(PyRagSearchHit hit, int originalRank) {
|
||||
String docId = blankToNull(hit.documentId());
|
||||
Integer chunkIndex = parseChunkIndex(hit.evidenceKey());
|
||||
String evidenceKey = EvidenceIdentity.firstNonBlank(
|
||||
hit.evidenceKey(),
|
||||
EvidenceIdentity.evidenceKey(docId, chunkIndex, null, originalRank));
|
||||
Double score = hit.qualityScore() == null ? 0.0 : hit.qualityScore();
|
||||
return new KnowledgeSearchHit(
|
||||
firstNonBlank(evidenceKey, docId, "rank:" + originalRank),
|
||||
hit.excerpt(),
|
||||
score,
|
||||
score,
|
||||
RetrievalScoreLabels.RERANK,
|
||||
null,
|
||||
Map.of(),
|
||||
docId,
|
||||
chunkIndex,
|
||||
evidenceKey,
|
||||
hit.source(),
|
||||
hit.title(),
|
||||
hit.breadcrumb(),
|
||||
originalRank,
|
||||
null);
|
||||
}
|
||||
|
||||
/** evidence_key 形如 {@code docId#chunk-N}，解析末尾 chunk 序号；不符返回 null。 */
|
||||
private Integer parseChunkIndex(String evidenceKey) {
|
||||
String key = blankToNull(evidenceKey);
|
||||
if (key == null) {
|
||||
return null;
|
||||
}
|
||||
int mark = key.lastIndexOf(CHUNK_MARK);
|
||||
if (mark < 0) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return Integer.valueOf(key.substring(mark + CHUNK_MARK.length()).trim());
|
||||
} catch (NumberFormatException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String blankToNull(String value) {
|
||||
return value == null || value.isBlank() ? null : value.trim();
|
||||
}
|
||||
|
||||
private String firstNonBlank(String... values) {
|
||||
return EvidenceIdentity.firstNonBlank(values);
|
||||
}
|
||||
}
|
||||
@@ -3,8 +3,13 @@ package com.superbiz.agent.service.retrieval;
|
||||
/**
|
||||
* 检索结果一级 {@code scoreLabel} 约定。
|
||||
*
|
||||
* <p>只区分两种检索形态（与 {@code retrieval.search.mode} 对齐），
|
||||
* 不再使用 {@code bm25_only_*} 等作为正式一级 label。</p>
|
||||
* <p>三种检索形态：</p>
|
||||
* <ul>
|
||||
* <li>{@link #DENSE} —— 单路向量 ANN（L2 距离，越小越好）</li>
|
||||
* <li>{@link #HYBRID} —— dense + BM25 + RRF 融合（质量主要看 rank）</li>
|
||||
* <li>{@link #RERANK} —— py-rag 服务端 rerank 绝对相关分（[0,1]，越大越好）；
|
||||
* RAG 模块抽离后的线上主路径</li>
|
||||
* </ul>
|
||||
*/
|
||||
public final class RetrievalScoreLabels {
|
||||
|
||||
@@ -14,11 +19,14 @@ public final class RetrievalScoreLabels {
|
||||
/** hybrid（dense+BM25+RRF）：{@code score}/raw 为融合侧信号；质量分主要看 rank。 */
|
||||
public static final String HYBRID = "hybrid";
|
||||
|
||||
/** py-rag 服务端 rerank 绝对分：{@code score} 即归一化质量（[0,1]，越大越好）。 */
|
||||
public static final String RERANK = "rerank";
|
||||
|
||||
private RetrievalScoreLabels() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 将历史/别名 label 归一到 {@link #DENSE} 或 {@link #HYBRID}。
|
||||
* 将历史/别名 label 归一到 {@link #DENSE}、{@link #HYBRID} 或 {@link #RERANK}。
|
||||
* 未知或空 → dense（保守，按 L2 解释失败时 quality 偏低）。
|
||||
*/
|
||||
public static String canonicalize(String scoreLabel) {
|
||||
@@ -29,12 +37,19 @@ public final class RetrievalScoreLabels {
|
||||
return switch (label) {
|
||||
case DENSE, "l2_distance", "l2" -> DENSE;
|
||||
case HYBRID, "rrf_fused", "rrf", "bm25_only_no_dense", "bm25_only" -> HYBRID;
|
||||
default -> label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
||||
? HYBRID
|
||||
: DENSE;
|
||||
case RERANK, "rerank_score", "quality_score" -> RERANK;
|
||||
default -> label.contains("rerank") || label.contains("quality")
|
||||
? RERANK
|
||||
: label.contains("hybrid") || label.contains("rrf") || label.contains("bm25")
|
||||
? HYBRID
|
||||
: DENSE;
|
||||
};
|
||||
}
|
||||
|
||||
public static boolean isRerank(String scoreLabel) {
|
||||
return RERANK.equals(canonicalize(scoreLabel));
|
||||
}
|
||||
|
||||
public static boolean isHybrid(String scoreLabel) {
|
||||
return HYBRID.equals(canonicalize(scoreLabel));
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ package com.superbiz.agent.service.retrieval;
|
||||
* <p>后处理排序仍按 {@code originalRank}；本类只负责质量闸门 / relevance 用分。</p>
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@link RetrievalScoreLabels#RERANK}：py-rag 服务端 rerank 绝对分，
|
||||
* {@code score} 已归一化，原样 clamp 到 [0,1]（RAG 抽离后的主路径）</li>
|
||||
* <li>{@link RetrievalScoreLabels#DENSE}：{@code score} = L2 → {@code 1 - clamp(l2)/maxL2}</li>
|
||||
* <li>{@link RetrievalScoreLabels#HYBRID}：优先用可选 {@code denseDistance} 做绝对质量
|
||||
* （恢复 L0 filter low-quality 等闸门）；无 dense 时回退 rank 映射</li>
|
||||
@@ -17,8 +19,8 @@ public final class RetrievalScoreNormalizer {
|
||||
}
|
||||
|
||||
/**
|
||||
* @param scoreLabel {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
||||
* @param score 引擎主分：dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
|
||||
* @param scoreLabel {@link RetrievalScoreLabels#RERANK} / {@link RetrievalScoreLabels#DENSE} / {@link RetrievalScoreLabels#HYBRID}
|
||||
* @param score 引擎主分：rerank=绝对相关分[0,1]；dense=L2；hybrid=融合分（hybrid 质量不依赖其量纲）
|
||||
* @param originalRank 检索名次（1-based）
|
||||
* @param batchSize 本轮候选数（rank 回退映射用）
|
||||
* @param maxL2Distance L2 上界
|
||||
@@ -31,6 +33,12 @@ public final class RetrievalScoreNormalizer {
|
||||
double maxL2Distance,
|
||||
Double denseDistance) {
|
||||
String label = RetrievalScoreLabels.canonicalize(scoreLabel);
|
||||
if (RetrievalScoreLabels.RERANK.equals(label)) {
|
||||
if (score == null) {
|
||||
return 0.0;
|
||||
}
|
||||
return Math.max(0.0, Math.min(1.0, score));
|
||||
}
|
||||
if (RetrievalScoreLabels.HYBRID.equals(label)) {
|
||||
if (denseDistance != null) {
|
||||
return l2ToQuality(denseDistance, maxL2Distance);
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* Reciprocal Rank Fusion 工具：把多路检索的排名列表融合成一个分数排序。
|
||||
*
|
||||
* <pre>
|
||||
* RRF_w(d) = Σ w_i / (k + rank_i(d))
|
||||
* </pre>
|
||||
*
|
||||
* <p>只依赖排名不依赖原始分数——屏蔽跨路分数尺度不可比的问题；
|
||||
* 每路可加权（w <= 0 时按 1.0 等权），k 是平滑参数（默认 60，可配）。
|
||||
*/
|
||||
public final class RrfFusion {
|
||||
|
||||
private RrfFusion() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 融合多路排名：对每路的每个 item 累加 w/(k+rank)，按总分降序输出。
|
||||
*
|
||||
* @param paths 多路排名（每路带 name / items / weight）
|
||||
* @param rrfK 平滑参数 k（至少 1）
|
||||
* @param identityFn 跨路识别同一 item 的身份函数（如 evidenceKey）
|
||||
* @return 融合后排序（含每路排名明细）
|
||||
*/
|
||||
public static <T> List<Scored<T>> fuse(List<RankedPath<T>> paths,
|
||||
int rrfK,
|
||||
Function<T, String> identityFn) {
|
||||
if (paths == null || paths.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
int k = Math.max(1, rrfK);
|
||||
Map<String, Acc<T>> acc = new LinkedHashMap<>();
|
||||
for (RankedPath<T> path : paths) {
|
||||
if (path == null || path.items() == null || path.items().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
double weight = path.weight() <= 0 ? 1.0 : path.weight();
|
||||
List<T> items = path.items();
|
||||
for (int i = 0; i < items.size(); i++) {
|
||||
T item = items.get(i);
|
||||
if (item == null) {
|
||||
continue;
|
||||
}
|
||||
String id = identityFn.apply(item);
|
||||
if (id == null || id.isBlank()) {
|
||||
continue;
|
||||
}
|
||||
int rank = i + 1;
|
||||
double contrib = weight / (k + rank); // 排名越前贡献越大
|
||||
Acc<T> bucket = acc.computeIfAbsent(id, ignored -> new Acc<>(item));
|
||||
bucket.score += contrib;
|
||||
bucket.ranks.put(path.name(), rank);
|
||||
// Prefer first-seen item payload; callers should put preferred path first if needed.
|
||||
}
|
||||
}
|
||||
List<Scored<T>> scored = new ArrayList<>(acc.size());
|
||||
for (Map.Entry<String, Acc<T>> entry : acc.entrySet()) {
|
||||
Acc<T> value = entry.getValue();
|
||||
scored.add(new Scored<>(entry.getKey(), value.item, value.score, Map.copyOf(value.ranks)));
|
||||
}
|
||||
// 总分降序（两路共识的靠前），同分按身份稳定排序
|
||||
scored.sort(Comparator
|
||||
.comparingDouble((Scored<T> s) -> s.rrfScore()).reversed()
|
||||
.thenComparing(Scored::identity));
|
||||
return scored;
|
||||
}
|
||||
|
||||
/** 一路检索结果：name（路名）+ items（按排名顺序）+ weight（可选加权，≤0 视为等权）。 */
|
||||
public record RankedPath<T>(String name, List<T> items, double weight) {
|
||||
public RankedPath {
|
||||
Objects.requireNonNull(name, "name");
|
||||
items = items == null ? List.of() : List.copyOf(items);
|
||||
}
|
||||
}
|
||||
|
||||
/** 融合后的单个 item：identity + 原始 item + rrfScore + 每路排名明细。 */
|
||||
public record Scored<T>(String identity, T item, double rrfScore, Map<String, Integer> ranks) {
|
||||
}
|
||||
|
||||
/** 跨路累加器：同一 identity 的 item 累加 RRF 分并记录各路排名。 */
|
||||
private static final class Acc<T> {
|
||||
private final T item;
|
||||
private double score;
|
||||
private final Map<String, Integer> ranks = new HashMap<>();
|
||||
|
||||
private Acc(T item) {
|
||||
this.item = item;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package com.superbiz.agent.service.retrieval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* {@link KnowledgeSearchPort} 适配器：把向量检索结果映射为带 evidenceKey 的命中结构。
|
||||
*
|
||||
* <p>委托 {@link VectorSearchService}（背后仅 {@code MilvusHybridKnowledgeStore}）：
|
||||
* dense 或 dense+BM25 hybrid 由配置 {@code retrieval.search.mode} 选择。
|
||||
* 本类负责 metadata 解析、docId/chunk 身份与 evidenceKey，不碰 SDK。</p>
|
||||
*/
|
||||
@Component
|
||||
public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
|
||||
|
||||
private final VectorSearchService vectorSearchService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
|
||||
this.vectorSearchService = vectorSearchService;
|
||||
this.objectMapper = objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
|
||||
// Mode is global on VectorSearchService; request.mode is advisory for future per-call overrides.
|
||||
List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(
|
||||
request.query(),
|
||||
request.topK(),
|
||||
request.categoryFilter());
|
||||
return toHits(results);
|
||||
}
|
||||
|
||||
private List<KnowledgeSearchHit> toHits(List<VectorSearchService.SearchResult> results) {
|
||||
if (results == null || results.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
List<KnowledgeSearchHit> hits = new ArrayList<>(results.size());
|
||||
for (int i = 0; i < results.size(); i++) {
|
||||
hits.add(toHit(results.get(i), i + 1));
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
|
||||
Map<String, String> metadata = parseMetadata(result.getMetadata());
|
||||
String docId = EvidenceIdentity.extractDocId(
|
||||
metadata,
|
||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "source"));
|
||||
Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
|
||||
String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
|
||||
String source = EvidenceIdentity.firstNonBlank(
|
||||
EvidenceIdentity.metadataValue(metadata, "_source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "source"),
|
||||
EvidenceIdentity.metadataValue(metadata, "filePath"),
|
||||
docId,
|
||||
result.getId());
|
||||
return new KnowledgeSearchHit(
|
||||
result.getId(),
|
||||
result.getContent(),
|
||||
(double) result.getScore(),
|
||||
result.getRawScore(),
|
||||
result.getScoreLabel(),
|
||||
result.getMetadata(),
|
||||
metadata,
|
||||
docId,
|
||||
chunkIndex,
|
||||
evidenceKey,
|
||||
source,
|
||||
EvidenceIdentity.metadataValue(metadata, "title"),
|
||||
EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
|
||||
originalRank,
|
||||
result.getDenseDistance()
|
||||
);
|
||||
}
|
||||
|
||||
private Map<String, String> parseMetadata(String metadata) {
|
||||
if (metadata == null || metadata.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
try {
|
||||
Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
|
||||
Map<String, String> parsed = new LinkedHashMap<>();
|
||||
for (Map.Entry<?, ?> entry : raw.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
} catch (Exception ignored) {
|
||||
return Map.of();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import io.milvus.client.MilvusServiceClient;
|
||||
import io.milvus.param.ConnectParam;
|
||||
import io.milvus.param.R;
|
||||
import io.milvus.param.RpcStatus;
|
||||
import io.milvus.param.collection.DropCollectionParam;
|
||||
import io.milvus.param.collection.HasCollectionParam;
|
||||
|
||||
/**
|
||||
* 删除 Milvus Collection 的工具类
|
||||
* 用于重建 Collection 时清理旧数据
|
||||
*/
|
||||
public class DropCollection {
|
||||
|
||||
public static void main(String[] args) {
|
||||
MilvusServiceClient client = null;
|
||||
|
||||
try {
|
||||
// 连接到 Milvus
|
||||
System.out.println("正在连接到 Milvus localhost:19530...");
|
||||
client = new MilvusServiceClient(
|
||||
ConnectParam.newBuilder()
|
||||
.withHost("localhost")
|
||||
.withPort(19530)
|
||||
.build()
|
||||
);
|
||||
System.out.println("✓ 连接成功");
|
||||
|
||||
String collectionName = "biz";
|
||||
|
||||
// 检查 Collection 是否存在
|
||||
R<Boolean> hasResponse = client.hasCollection(
|
||||
HasCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build()
|
||||
);
|
||||
|
||||
if (hasResponse.getData()) {
|
||||
System.out.println("发现 Collection: " + collectionName);
|
||||
System.out.println("正在删除...");
|
||||
|
||||
// 删除 Collection
|
||||
R<RpcStatus> dropResponse = client.dropCollection(
|
||||
DropCollectionParam.newBuilder()
|
||||
.withCollectionName(collectionName)
|
||||
.build()
|
||||
);
|
||||
|
||||
if (dropResponse.getStatus() == 0) {
|
||||
System.out.println("✓ Collection 已成功删除");
|
||||
System.out.println("\n请重启 Spring Boot 应用，它会自动创建新的 FloatVector Collection");
|
||||
} else {
|
||||
System.err.println("✗ 删除失败: " + dropResponse.getMessage());
|
||||
}
|
||||
} else {
|
||||
System.out.println("Collection '" + collectionName + "' 不存在");
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
System.err.println("错误: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import com.superbiz.agent.dto.RetrievalTrace;
|
||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -29,10 +28,9 @@ import java.util.Map;
|
||||
*
|
||||
* <h3>主链路</h3>
|
||||
* <pre>
|
||||
* query
|
||||
* -> KnowledgeQueryTransformer
|
||||
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k)
|
||||
* -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
|
||||
* query（原始句直传；L0 query 理解已下沉 py-rag 服务端）
|
||||
* -> KnowledgeDocumentRetriever (via KnowledgeSearchPort → py-rag, retrieve-k)
|
||||
* -> KnowledgeEvidencePostProcessor (qualityScore / chunk dedup / caps / return-n)
|
||||
* -> [optional] unfiltered retry
|
||||
* -> KnowledgeContextPacker
|
||||
* -> LookupResultAssembler
|
||||
@@ -62,9 +60,6 @@ public class LookupKnowledgeTool {
|
||||
|
||||
private int retrieveK = 20;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeQueryTransformer queryTransformer;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDocumentRetriever documentRetriever;
|
||||
|
||||
@@ -98,10 +93,10 @@ public class LookupKnowledgeTool {
|
||||
*
|
||||
* <p>流程（模块化三段）：
|
||||
* <ol>
|
||||
* <li>检索前：QueryTransformer.transform → KnowledgeQuery（分类过滤/域/关键词）；</li>
|
||||
* <li>检索：DocumentRetriever.retrieve（FILTERED 或 UNFILTERED，retrieveK 候选）；</li>
|
||||
* <li>检索前：原始 query 直传（L0 domain 分析已下沉 py-rag，categoryFilter 恒为 null）；</li>
|
||||
* <li>检索：DocumentRetriever.retrieve（UNFILTERED，retrieveK 候选，py-rag 服务端融合+精排）；</li>
|
||||
* <li>检索后：PostProcessor.process（qualityScore/去重/判级）；</li>
|
||||
* <li>低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查；</li>
|
||||
* <li>低质量降级：带分类过滤结果低质 → 去掉过滤、用原始 query 重查（L0 移除后保留兜底语义）；</li>
|
||||
* <li>打包 + 组装：ContextPacker.pack → LookupResultAssembler.assemble → LookupResult。</li>
|
||||
* </ol>
|
||||
*
|
||||
@@ -113,12 +108,17 @@ public class LookupKnowledgeTool {
|
||||
log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
|
||||
log.info("----------------------------------------");
|
||||
|
||||
// ── 检索前：查询理解（L0）──
|
||||
KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
|
||||
log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
|
||||
knowledgeQuery.getCategoryFilter(),
|
||||
knowledgeQuery.getDomainHints().size(),
|
||||
knowledgeQuery.getMatchedKeywords().size());
|
||||
// ── 检索前：原始 query 直传（L0 已下沉 py-rag，不做 Java 侧 category 收窄）──
|
||||
String normalized = query == null ? "" : query.trim();
|
||||
KnowledgeQuery knowledgeQuery = KnowledgeQuery.builder()
|
||||
.originalQuery(normalized)
|
||||
.rewrittenQuery(normalized)
|
||||
.domainHints(List.of())
|
||||
.matchedKeywords(List.of())
|
||||
.entities(List.of())
|
||||
.l0Titles(List.of())
|
||||
.l0MatchCount(0)
|
||||
.build();
|
||||
|
||||
List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
|
||||
String fallbackReason = null;
|
||||
|
||||
@@ -15,30 +15,28 @@ file:
|
||||
knowledge:
|
||||
base-path: knowledge_base/
|
||||
|
||||
milvus:
|
||||
host: in03-4a578da0f27ce9d.serverless.aws-eu-central-1.cloud.zilliz.com
|
||||
port: 443
|
||||
username: ""
|
||||
password: ""
|
||||
database: db_4a578da0f27ce9d
|
||||
timeout: 10000
|
||||
token: ${MILVUS_TOKEN}
|
||||
secure: true
|
||||
vector-dim: 1024 # BGE-M3 = 1024，换模型时同步改
|
||||
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
|
||||
collection: biz
|
||||
# =====================================================
|
||||
# py-rag 知识服务接入
|
||||
# =====================================================
|
||||
# RAG 检索与文档入库均由 py-rag 服务承担（契约见 py-rag 仓库 docs/Java接入文档.md）：
|
||||
# 检索 /api/v1/search，入库 /api/v1/documents:ingest，全量重建 /api/v1/collections:rebuild。
|
||||
pyrag:
|
||||
base-url: ${PYRAG_BASE_URL:http://localhost:8000}
|
||||
connect-timeout-ms: 3000
|
||||
search-read-timeout-ms: 5000 # 正常 300–800ms（含 rerank 外呼）
|
||||
ingest-read-timeout-ms: 30000 # 正常 1–5s
|
||||
default-read-timeout-ms: 10000
|
||||
|
||||
# =====================================================
|
||||
# 模型路由配置
|
||||
# =====================================================
|
||||
# 通过关键字匹配 Bean，切换模型只改这里 + 对应 api-key
|
||||
# Chat: deepseek | openai | ollama | ...
|
||||
# Embedding: siliconflow | openai | ollama | dashscope | ...
|
||||
# Chat: deepseek | openai | ollama | ...
|
||||
# （Embedding 已随 RAG 抽离至 py-rag 服务端）
|
||||
# =====================================================
|
||||
|
||||
model-routing:
|
||||
chat: deepseek
|
||||
embedding: siliconflow
|
||||
|
||||
spring:
|
||||
config:
|
||||
@@ -102,30 +100,6 @@ spring:
|
||||
retry:
|
||||
max-attempts: 1
|
||||
|
||||
vectorstore:
|
||||
type: milvus
|
||||
milvus:
|
||||
initialize-schema: false
|
||||
database-name: ${milvus.database}
|
||||
collection-name: biz
|
||||
embedding-dimension: ${milvus.vector-dim}
|
||||
index-type: IVF_FLAT
|
||||
metric-type: L2
|
||||
index-parameters: '{"nlist":128}'
|
||||
id-field-name: id
|
||||
auto-id: false
|
||||
content-field-name: content
|
||||
metadata-field-name: metadata
|
||||
embedding-field-name: vector
|
||||
client:
|
||||
host: ${milvus.host}
|
||||
port: ${milvus.port}
|
||||
token: ${milvus.token}
|
||||
username: ${milvus.username}
|
||||
password: ${milvus.password}
|
||||
secure: ${milvus.secure}
|
||||
connect-timeout-ms: ${milvus.timeout}
|
||||
|
||||
# --- Chat: DeepSeek (原生) ---
|
||||
deepseek:
|
||||
api-key: ${DEEPSEEK_API_KEY}
|
||||
@@ -134,53 +108,27 @@ spring:
|
||||
options:
|
||||
model: deepseek-v4-flash
|
||||
|
||||
# --- OpenAI 模块供 SiliconFlow Embedding 复用 ---
|
||||
openai:
|
||||
api-key: unused
|
||||
|
||||
# Spring AI MCP 客户端配置
|
||||
mcp:
|
||||
client:
|
||||
enabled: false
|
||||
|
||||
# --- Embedding: SiliconFlow BGE-M3 ---
|
||||
siliconflow:
|
||||
api-key: ${SILICONFLOW_API_KEY}
|
||||
base-url: https://api.siliconflow.cn
|
||||
embedding:
|
||||
model: BAAI/bge-m3
|
||||
|
||||
# 文档分片配置
|
||||
document:
|
||||
chunk:
|
||||
max-size: 800
|
||||
overlap: 100
|
||||
|
||||
# RAG 配置
|
||||
rag:
|
||||
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
sidecar:
|
||||
spring-ai:
|
||||
enabled: false
|
||||
content-preview-limit: 300
|
||||
|
||||
# 检索配置
|
||||
# 知识主路径：Milvus Java SDK v2（MilvusHybridKnowledgeStore），非 Spring AI VectorStore starter。
|
||||
# 原因：starter（含 2.0.0）仅 dense similarity，无 hybridSearch / BM25 Function / RRFRanker。
|
||||
# 已移除 legacy sdk/spring/auto 多后端路由。
|
||||
# 知识主路径：py-rag 知识服务（PyRagKnowledgeSearchAdapter → /api/v1/search）。
|
||||
# 服务端负责 dense+BM25 融合、rerank（BGE-Reranker）与判级；Java 侧只做请求映射与后处理。
|
||||
retrieval:
|
||||
kb-scope: "" # 非空则过滤 metadata.kb_scope；空=不过滤
|
||||
search:
|
||||
# hybrid=线上主路径；dense=同库对照/评测/排障（非第二套线上策略）。见 mvp/architecture/rag-knowledge-retrieval-architecture.md §6.0
|
||||
mode: hybrid # dense=单路L2对照 | hybrid=dense+服务端BM25+RRF
|
||||
hybrid:
|
||||
rrf-k: 60 # RRF 平滑参数 k，score=Σ 1/(k+rank)
|
||||
# hybrid=线上主路径；dense 为对照/排障（映射 py-rag mode：hybrid→hybrid，dense→semantic）
|
||||
mode: hybrid
|
||||
normalization:
|
||||
max-l2-distance: 2.0 # dense quality：L2 上界（单位向量 ≈ 2.0）
|
||||
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（hybrid 为序数分，见架构 §6）
|
||||
highly-relevant-threshold: 0.75 # qualityScore >= 0.75 → PRECISE（与 py-rag 判级阈值一致）
|
||||
reference-threshold: 0.5 # qualityScore >= 0.5 → REFERENCE；低于则低质/可 unfiltered retry
|
||||
|
||||
# Prometheus 配置
|
||||
|
||||
Reference in New Issue
Block a user