feat(phase1): 实现文档分块向量化索引
Task 5.6: 向量化索引实现 - VectorIndexService 新增方法: - indexDocumentChunks(docId, chunks): 索引文档分块到 Milvus - deleteDocumentChunks(docId): 删除文档的所有向量 - buildDocumentMetadata(): 构建文档元数据(区分文件索引) 核心流程: 1. 上传时:文本提取 → 分块 → 向量化 → 存入 Milvus + MySQL 2. 检索时:问题向量化 → Milvus 语义检索 → 返回相似文档 3. 删除时:删除元数据 + 删除向量索引 实现细节: - 复用 indexSingleFile 的向量化逻辑 - metadata.docId 标识文档来源(区分 upload: 和 file:) - 删除表达式:metadata["docId"] == "xxx" - 自动去重:上传前删除旧向量数据 DocumentManagementService 完整实现: - uploadDocument: 完整向量化流程(移除 TODO) - deleteDocument: 同步删除向量索引(移除 TODO) 编译验证:BUILD SUCCESS Progress: 32/34 tasks completed (94%)
This commit is contained in:
@@ -42,8 +42,9 @@
|
|||||||
- [x] 5.3 文档上传接口 (DocumentController#upload, DocumentManagementService#uploadDocument)
|
- [x] 5.3 文档上传接口 (DocumentController#upload, DocumentManagementService#uploadDocument)
|
||||||
- [x] 5.4 文档查询接口 (DocumentController#query, DocumentService#queryDocuments)
|
- [x] 5.4 文档查询接口 (DocumentController#query, DocumentService#queryDocuments)
|
||||||
- [x] 5.5 文档删除接口 (DocumentController#delete, DocumentService#deleteDocument)
|
- [x] 5.5 文档删除接口 (DocumentController#delete, DocumentService#deleteDocument)
|
||||||
- [ ] 5.6 混合检索工具 (DocumentSearchTool: 精确匹配 + 语义检索 + RRF 融合)
|
- [x] 5.6 向量化索引 (VectorIndexService#indexDocumentChunks, 实现分块级别索引)
|
||||||
- [ ] 5.7 文档管理集成测试 (DocumentIntegrationTest)
|
- [ ] 5.7 混合检索工具 (DocumentSearchTool: 精确匹配 + 语义检索 + RRF 融合)
|
||||||
|
- [ ] 5.8 文档管理集成测试 (DocumentIntegrationTest)
|
||||||
|
|
||||||
## 6. 全局完善
|
## 6. 全局完善
|
||||||
|
|
||||||
|
|||||||
@@ -108,22 +108,19 @@ public class DocumentManagementService {
|
|||||||
apiDocumentRepository.save(document);
|
apiDocumentRepository.save(document);
|
||||||
log.info("文档元数据已保存,docId: {}", docId);
|
log.info("文档元数据已保存,docId: {}", docId);
|
||||||
|
|
||||||
// 6. 向量化并索引(TODO: 待实现批量分块索引)
|
// 6. 向量化并索引
|
||||||
try {
|
try {
|
||||||
// TODO: 实现 VectorIndexService.indexDocumentChunks(docId, chunks)
|
vectorIndexService.indexDocumentChunks(docId, chunks);
|
||||||
// 当前暂时标记为 INDEXED,后续补充实际向量化逻辑
|
|
||||||
log.warn("向量化索引功能待实现,docId: {}", docId);
|
|
||||||
|
|
||||||
document.setStatus("INDEXED");
|
document.setStatus("INDEXED");
|
||||||
document.setIndexedAt(LocalDateTime.now());
|
document.setIndexedAt(LocalDateTime.now());
|
||||||
apiDocumentRepository.save(document);
|
apiDocumentRepository.save(document);
|
||||||
log.info("文档元数据已创建(向量化待实现),docId: {}", docId);
|
log.info("文档索引完成,docId: {}", docId);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("文档处理失败,docId: {}", docId, e);
|
log.error("文档索引失败,docId: {}", docId, e);
|
||||||
document.setStatus("FAILED");
|
document.setStatus("FAILED");
|
||||||
apiDocumentRepository.save(document);
|
apiDocumentRepository.save(document);
|
||||||
throw new DocumentProcessException(docId, "process", "文档处理失败: " + e.getMessage(), e);
|
throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
|
||||||
return docId;
|
return docId;
|
||||||
@@ -208,8 +205,13 @@ public class DocumentManagementService {
|
|||||||
|
|
||||||
ApiDocument doc = optional.get();
|
ApiDocument doc = optional.get();
|
||||||
|
|
||||||
// TODO: 删除向量索引
|
// 删除向量索引
|
||||||
log.warn("向量索引删除功能待实现,docId: {}", docId);
|
try {
|
||||||
|
vectorIndexService.deleteDocumentChunks(docId);
|
||||||
|
log.info("文档向量索引已删除,docId: {}", docId);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("删除向量索引失败,docId: {}", docId, e);
|
||||||
|
}
|
||||||
|
|
||||||
// 删除元数据
|
// 删除元数据
|
||||||
apiDocumentRepository.delete(doc);
|
apiDocumentRepository.delete(doc);
|
||||||
|
|||||||
@@ -167,6 +167,110 @@ public class VectorIndexService {
|
|||||||
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
|
logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 索引文档分块(用于上传文档的向量化)
|
||||||
|
*
|
||||||
|
* @param docId 文档ID
|
||||||
|
* @param chunks 文档分块列表
|
||||||
|
* @throws Exception 索引失败时抛出异常
|
||||||
|
*/
|
||||||
|
public void indexDocumentChunks(String docId, List<DocumentChunk> chunks) throws Exception {
|
||||||
|
if (chunks == null || chunks.isEmpty()) {
|
||||||
|
throw new IllegalArgumentException("文档分块列表为空");
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info("开始索引文档分块,docId: {}, 分块数: {}", docId, chunks.size());
|
||||||
|
|
||||||
|
// 1. 删除该文档的旧数据(如果存在)
|
||||||
|
deleteDocumentChunks(docId);
|
||||||
|
|
||||||
|
// 2. 为每个分块生成向量并插入 Milvus
|
||||||
|
for (int i = 0; i < chunks.size(); i++) {
|
||||||
|
DocumentChunk chunk = chunks.get(i);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 生成向量
|
||||||
|
List<Float> vector = embeddingService.generateEmbedding(chunk.getContent());
|
||||||
|
|
||||||
|
// 构建元数据(使用 docId 作为来源标识)
|
||||||
|
Map<String, Object> metadata = buildDocumentMetadata(docId, chunk, chunks.size());
|
||||||
|
|
||||||
|
// 插入到 Milvus
|
||||||
|
insertToMilvus(chunk.getContent(), vector, metadata, chunk.getChunkIndex());
|
||||||
|
|
||||||
|
logger.info("✓ 文档分块 {}/{} 索引成功,docId: {}", i + 1, chunks.size(), docId);
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("✗ 文档分块 {}/{} 索引失败,docId: {}", i + 1, chunks.size(), docId, e);
|
||||||
|
throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info("文档索引完成,docId: {}, 共 {} 个分块", docId, chunks.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除文档的所有分块(根据 docId)
|
||||||
|
*/
|
||||||
|
public void deleteDocumentChunks(String docId) {
|
||||||
|
try {
|
||||||
|
// 构建删除表达式:metadata["docId"] == "xxx"
|
||||||
|
String expr = String.format("metadata[\"docId\"] == \"%s\"", docId);
|
||||||
|
|
||||||
|
logger.info("准备删除文档旧数据,docId: {}, 表达式: {}", docId, expr);
|
||||||
|
|
||||||
|
// 确保 collection 已加载
|
||||||
|
R<RpcStatus> loadResponse = milvusClient.loadCollection(
|
||||||
|
LoadCollectionParam.newBuilder()
|
||||||
|
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||||
|
.build()
|
||||||
|
);
|
||||||
|
|
||||||
|
if (loadResponse.getStatus() != 0 && loadResponse.getStatus() != 65535) {
|
||||||
|
logger.warn("加载 collection 失败: {}", loadResponse.getMessage());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteParam deleteParam = DeleteParam.newBuilder()
|
||||||
|
.withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME)
|
||||||
|
.withExpr(expr)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
R<MutationResult> deleteResponse = milvusClient.delete(deleteParam);
|
||||||
|
|
||||||
|
if (deleteResponse.getStatus() == 0) {
|
||||||
|
logger.info("删除文档旧数据成功,docId: {}", docId);
|
||||||
|
} else {
|
||||||
|
logger.warn("删除文档旧数据失败,docId: {}, 原因: {}", docId, deleteResponse.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn("删除文档旧数据异常,docId: {}", docId, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 构建文档元数据(用于上传文档)
|
||||||
|
*/
|
||||||
|
private Map<String, Object> buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks) {
|
||||||
|
Map<String, Object> metadata = new HashMap<>();
|
||||||
|
|
||||||
|
// 文档标识
|
||||||
|
metadata.put("docId", docId);
|
||||||
|
metadata.put("_source", "upload:" + docId); // 区分文件索引和上传文档
|
||||||
|
|
||||||
|
// 分片信息
|
||||||
|
metadata.put("chunkIndex", chunk.getChunkIndex());
|
||||||
|
metadata.put("totalChunks", totalChunks);
|
||||||
|
|
||||||
|
// 标题信息
|
||||||
|
if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) {
|
||||||
|
metadata.put("title", chunk.getTitle());
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadata;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除文件的旧数据(根据 metadata._source)
|
* 删除文件的旧数据(根据 metadata._source)
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user