feat(phase1): 支持按类别过滤的文档检索
功能增强:
- VectorIndexService 自动提取文档类别
- 文件索引:从路径提取(如 aiops-docs/api/ → "api")
- 上传文档:默认类别 "upload"
- metadata.category 字段存储类别信息
- VectorSearchService 支持类别过滤
- searchSimilarDocuments(query, topK): 原方法,不过滤
- searchSimilarDocuments(query, topK, category): 新方法,按类别过滤
- 使用 Milvus expr 过滤:metadata["category"] == "xxx"
使用场景:
- 目录结构:
aiops-docs/
├── api/ → category="api"
├── domain/ → category="domain"
└── troubleshoot/ → category="troubleshoot"
- 检索示例:
// 只检索 API 文档
searchSimilarDocuments("Redis接口", 5, "api")
// 只检索领域知识
searchSimilarDocuments("缓存原理", 5, "domain")
// 全量检索
searchSimilarDocuments("问题诊断", 5, null)
编译验证:BUILD SUCCESS
This commit is contained in:
@@ -268,6 +268,9 @@ public class VectorIndexService {
|
||||
metadata.put("title", chunk.getTitle());
|
||||
}
|
||||
|
||||
// 默认类别:上传文档
|
||||
metadata.put("category", "upload");
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@@ -336,23 +339,61 @@ public class VectorIndexService {
|
||||
if (dotIndex > 0) {
|
||||
extension = fileNameStr.substring(dotIndex);
|
||||
}
|
||||
|
||||
|
||||
metadata.put("_source", normalizedPath);
|
||||
metadata.put("_extension", extension);
|
||||
metadata.put("_file_name", fileNameStr);
|
||||
|
||||
|
||||
// 提取类别(从文件路径中提取目录名)
|
||||
String category = extractCategory(normalizedPath);
|
||||
if (category != null && !category.isEmpty()) {
|
||||
metadata.put("category", category);
|
||||
}
|
||||
|
||||
// 分片信息
|
||||
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||
metadata.put("totalChunks", totalChunks);
|
||||
|
||||
|
||||
// 标题信息
|
||||
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
|
||||
metadata.put("title", chunk.getTitle());
|
||||
}
|
||||
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从文件路径中提取类别
|
||||
* 例如:aiops-docs/api/redis-api.md → "api"
|
||||
*/
|
||||
private String extractCategory(String filePath) {
|
||||
try {
|
||||
// 标准化路径分隔符
|
||||
String normalized = filePath.replace("\\", "/");
|
||||
|
||||
// 查找 aiops-docs/ 后的第一级目录
|
||||
int docsIndex = normalized.indexOf("aiops-docs/");
|
||||
if (docsIndex >= 0) {
|
||||
String afterDocs = normalized.substring(docsIndex + "aiops-docs/".length());
|
||||
int slashIndex = afterDocs.indexOf("/");
|
||||
if (slashIndex > 0) {
|
||||
return afterDocs.substring(0, slashIndex);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有 aiops-docs,返回第一级目录
|
||||
int firstSlash = normalized.indexOf("/");
|
||||
if (firstSlash > 0) {
|
||||
return normalized.substring(0, firstSlash);
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
logger.warn("提取类别失败,路径: {}", filePath, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 插入向量到 Milvus
|
||||
*/
|
||||
|
||||
@@ -34,29 +34,49 @@ public class VectorSearchService {
|
||||
|
||||
/**
|
||||
* 搜索相似文档
|
||||
*
|
||||
*
|
||||
* @param query 查询文本
|
||||
* @param topK 返回最相似的K个结果
|
||||
* @return 搜索结果列表
|
||||
*/
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK) {
|
||||
return searchSimilarDocuments(query, topK, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 搜索相似文档(支持类别过滤)
|
||||
*
|
||||
* @param query 查询文本
|
||||
* @param topK 返回最相似的K个结果
|
||||
* @param category 类别过滤(可选,null 表示不过滤)
|
||||
* @return 搜索结果列表
|
||||
*/
|
||||
public List<SearchResult> searchSimilarDocuments(String query, int topK, String category) {
|
||||
try {
|
||||
logger.info("开始搜索相似文档, 查询: {}, topK: {}", query, topK);
|
||||
logger.info("开始搜索相似文档, 查询: {}, topK: {}, 类别: {}", query, topK, category);
|
||||
|
||||
// 1. 将查询文本向量化
|
||||
List<Float> queryVector = embeddingService.generateQueryVector(query);
|
||||
logger.debug("查询向量生成成功, 维度: {}", queryVector.size());
|
||||
|
||||
// 2. 构建搜索参数
|
||||
SearchParam searchParam = SearchParam.newBuilder()
|
||||
SearchParam.Builder searchParamBuilder = SearchParam.newBuilder()
|
||||
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||
.withVectorFieldName("vector")
|
||||
.withVectors(Collections.singletonList(queryVector))
|
||||
.withTopK(topK)
|
||||
.withMetricType(io.milvus.param.MetricType.L2)
|
||||
.withOutFields(List.of("id", "content", "metadata"))
|
||||
.withParams("{\"nprobe\":10}")
|
||||
.build();
|
||||
.withParams("{\"nprobe\":10}");
|
||||
|
||||
// 添加类别过滤
|
||||
if (category != null && !category.trim().isEmpty()) {
|
||||
String expr = String.format("metadata[\"category\"] == \"%s\"", category);
|
||||
searchParamBuilder.withExpr(expr);
|
||||
logger.info("添加类别过滤: {}", expr);
|
||||
}
|
||||
|
||||
SearchParam searchParam = searchParamBuilder.build();
|
||||
|
||||
// 3. 执行搜索
|
||||
R<SearchResults> searchResponse = milvusClient.search(searchParam);
|
||||
@@ -74,13 +94,13 @@ public class VectorSearchService {
|
||||
result.setId((String) wrapper.getIDScore(0).get(i).get("id"));
|
||||
result.setContent((String) wrapper.getFieldData("content", 0).get(i));
|
||||
result.setScore(wrapper.getIDScore(0).get(i).getScore());
|
||||
|
||||
|
||||
// 解析 metadata
|
||||
Object metadataObj = wrapper.getFieldData("metadata", 0).get(i);
|
||||
if (metadataObj != null) {
|
||||
result.setMetadata(metadataObj.toString());
|
||||
}
|
||||
|
||||
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user