From 4ef8d87961588595adc8b5b8768a16b942f93338 Mon Sep 17 00:00:00 2001 From: zhuyongxin Date: Tue, 23 Jun 2026 16:08:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(phase1):=20=E5=AE=9E=E7=8E=B0=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E5=88=86=E5=9D=97=E5=90=91=E9=87=8F=E5=8C=96=E7=B4=A2?= =?UTF-8?q?=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 5.6: 向量化索引实现 - VectorIndexService 新增方法： - indexDocumentChunks(docId, chunks): 索引文档分块到 Milvus - deleteDocumentChunks(docId): 删除文档的所有向量 - buildDocumentMetadata(): 构建文档元数据（区分文件索引） 核心流程： 1. 上传时：文本提取 → 分块 → 向量化 → 存入 Milvus + MySQL 2. 检索时：问题向量化 → Milvus 语义检索 → 返回相似文档 3. 删除时：删除元数据 + 删除向量索引 实现细节： - 复用 indexSingleFile 的向量化逻辑 - metadata.docId 标识文档来源（区分 upload: 和 file:） - 删除表达式：metadata["docId"] == "xxx" - 自动去重：上传前删除旧向量数据 DocumentManagementService 完整实现： - uploadDocument: 完整向量化流程（移除 TODO） - deleteDocument: 同步删除向量索引（移除 TODO） 编译验证：BUILD SUCCESS Progress: 32/34 tasks completed (94%) --- .../changes/phase-1-infrastructure/tasks.md | 5 +- .../service/DocumentManagementService.java | 22 ++-- .../agent/service/VectorIndexService.java | 104 ++++++++++++++++++ 3 files changed, 119 insertions(+), 12 deletions(-) diff --git a/openspec/changes/phase-1-infrastructure/tasks.md b/openspec/changes/phase-1-infrastructure/tasks.md index cfccc8f..daa9874 100644 --- a/openspec/changes/phase-1-infrastructure/tasks.md +++ b/openspec/changes/phase-1-infrastructure/tasks.md @@ -42,8 +42,9 @@ - [x] 5.3 文档上传接口 (DocumentController#upload, DocumentManagementService#uploadDocument) - [x] 5.4 文档查询接口 (DocumentController#query, DocumentService#queryDocuments) - [x] 5.5 文档删除接口 (DocumentController#delete, DocumentService#deleteDocument) -- [ ] 5.6 混合检索工具 (DocumentSearchTool: 精确匹配 + 语义检索 + RRF 融合) -- [ ] 5.7 文档管理集成测试 (DocumentIntegrationTest) +- [x] 5.6 向量化索引 (VectorIndexService#indexDocumentChunks, 实现分块级别索引) +- [ ] 5.7 混合检索工具 (DocumentSearchTool: 精确匹配 + 语义检索 + RRF 融合) +- [ ] 5.8 文档管理集成测试 (DocumentIntegrationTest) ## 6. 全局完善 diff --git a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java index 630ce2b..5af4ae1 100644 --- a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java +++ b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java @@ -108,22 +108,19 @@ public class DocumentManagementService { apiDocumentRepository.save(document); log.info("文档元数据已保存，docId: {}", docId); - // 6. 向量化并索引（TODO: 待实现批量分块索引） + // 6. 向量化并索引 try { - // TODO: 实现 VectorIndexService.indexDocumentChunks(docId, chunks) - // 当前暂时标记为 INDEXED，后续补充实际向量化逻辑 - log.warn("向量化索引功能待实现，docId: {}", docId); - + vectorIndexService.indexDocumentChunks(docId, chunks); document.setStatus("INDEXED"); document.setIndexedAt(LocalDateTime.now()); apiDocumentRepository.save(document); - log.info("文档元数据已创建（向量化待实现），docId: {}", docId); + log.info("文档索引完成，docId: {}", docId); } catch (Exception e) { - log.error("文档处理失败，docId: {}", docId, e); + log.error("文档索引失败，docId: {}", docId, e); document.setStatus("FAILED"); apiDocumentRepository.save(document); - throw new DocumentProcessException(docId, "process", "文档处理失败: " + e.getMessage(), e); + throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e); } return docId; @@ -208,8 +205,13 @@ public class DocumentManagementService { ApiDocument doc = optional.get(); - // TODO: 删除向量索引 - log.warn("向量索引删除功能待实现，docId: {}", docId); + // 删除向量索引 + try { + vectorIndexService.deleteDocumentChunks(docId); + log.info("文档向量索引已删除，docId: {}", docId); + } catch (Exception e) { + log.warn("删除向量索引失败，docId: {}", docId, e); + } // 删除元数据 apiDocumentRepository.delete(doc); diff --git a/src/main/java/com/superbiz/agent/service/VectorIndexService.java b/src/main/java/com/superbiz/agent/service/VectorIndexService.java index 3b5b936..e2ae9b4 100644 --- a/src/main/java/com/superbiz/agent/service/VectorIndexService.java +++ b/src/main/java/com/superbiz/agent/service/VectorIndexService.java @@ -167,6 +167,110 @@ public class VectorIndexService { logger.info("文件索引完成: {}, 共 {} 个分片", filePath, chunks.size()); } + /** + * 索引文档分块（用于上传文档的向量化） + * + * @param docId 文档ID + * @param chunks 文档分块列表 + * @throws Exception 索引失败时抛出异常 + */ + public void indexDocumentChunks(String docId, List chunks) throws Exception { + if (chunks == null || chunks.isEmpty()) { + throw new IllegalArgumentException("文档分块列表为空"); + } + + logger.info("开始索引文档分块，docId: {}, 分块数: {}", docId, chunks.size()); + + // 1. 删除该文档的旧数据（如果存在） + deleteDocumentChunks(docId); + + // 2. 为每个分块生成向量并插入 Milvus + for (int i = 0; i < chunks.size(); i++) { + DocumentChunk chunk = chunks.get(i); + + try { + // 生成向量 + List vector = embeddingService.generateEmbedding(chunk.getContent()); + + // 构建元数据（使用 docId 作为来源标识） + Map metadata = buildDocumentMetadata(docId, chunk, chunks.size()); + + // 插入到 Milvus + insertToMilvus(chunk.getContent(), vector, metadata, chunk.getChunkIndex()); + + logger.info("✓ 文档分块 {}/{} 索引成功，docId: {}", i + 1, chunks.size(), docId); + + } catch (Exception e) { + logger.error("✗ 文档分块 {}/{} 索引失败，docId: {}", i + 1, chunks.size(), docId, e); + throw new RuntimeException("文档分块索引失败: " + e.getMessage(), e); + } + } + + logger.info("文档索引完成，docId: {}, 共 {} 个分块", docId, chunks.size()); + } + + /** + * 删除文档的所有分块（根据 docId） + */ + public void deleteDocumentChunks(String docId) { + try { + // 构建删除表达式：metadata["docId"] == "xxx" + String expr = String.format("metadata[\"docId\"] == \"%s\"", docId); + + logger.info("准备删除文档旧数据，docId: {}, 表达式: {}", docId, expr); + + // 确保 collection 已加载 + R loadResponse = milvusClient.loadCollection( + LoadCollectionParam.newBuilder() + .withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME) + .build() + ); + + if (loadResponse.getStatus() != 0 && loadResponse.getStatus() != 65535) { + logger.warn("加载 collection 失败: {}", loadResponse.getMessage()); + return; + } + + DeleteParam deleteParam = DeleteParam.newBuilder() + .withCollectionName(MilvusConstants.MILVUS_COLLECTION_NAME) + .withExpr(expr) + .build(); + + R deleteResponse = milvusClient.delete(deleteParam); + + if (deleteResponse.getStatus() == 0) { + logger.info("删除文档旧数据成功，docId: {}", docId); + } else { + logger.warn("删除文档旧数据失败，docId: {}, 原因: {}", docId, deleteResponse.getMessage()); + } + + } catch (Exception e) { + logger.warn("删除文档旧数据异常，docId: {}", docId, e); + } + } + + /** + * 构建文档元数据（用于上传文档） + */ + private Map buildDocumentMetadata(String docId, DocumentChunk chunk, int totalChunks) { + Map metadata = new HashMap<>(); + + // 文档标识 + metadata.put("docId", docId); + metadata.put("_source", "upload:" + docId); // 区分文件索引和上传文档 + + // 分片信息 + metadata.put("chunkIndex", chunk.getChunkIndex()); + metadata.put("totalChunks", totalChunks); + + // 标题信息 + if (chunk.getTitle() != null && !chunk.getTitle().isEmpty()) { + metadata.put("title", chunk.getTitle()); + } + + return metadata; + } + /** * 删除文件的旧数据（根据 metadata._source） */