feat(knowledge): 完整实现知识库初始化 - 包含 Milvus 向量索引
## 核心改动
在上一版本基础上,补充完整的 Milvus (L1) 向量索引功能。
### 新增依赖注入
```java
@Autowired
private DocumentChunkService documentChunkService;
@Autowired
private VectorIndexService vectorIndexService;
@Autowired
private VectorEmbeddingService vectorEmbeddingService;
```
### 完整的数据流
```
knowledge_base/*.md
↓ 1. 扫描 & 解析 frontmatter
↓ 2. 保存到 MySQL (api_document)
↓ 3. 提取正文 & 文档分块
↓ 4. 生成向量并索引到 Milvus
↓ 5. 加入 L0 内存索引
完成 (L0 + L1 双层索引)
```
### 关键代码
```java
// 1. 提取正文(去除 frontmatter)
String body = extractBody(content);
// 2. 文档分块
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
// 3. 上传到 Milvus
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
// 4. 更新状态
document.setStatus("INDEXED");
document.setChunkCount(chunks.size());
```
### 错误处理
- Milvus 索引失败时:
- 更新文档状态为 FAILED
- 记录错误信息到 error_message 字段
- 继续处理下一个文档(不中断整个流程)
### 响应示例
```json
{
"success": true,
"scanned": 6,
"inserted": 6,
"failed": 0,
"details": {
"api/payment-errors.md": "导入成功(L0+L1)"
}
}
```
### 数据库字段
新增:
- `chunk_count`:分块数量
- `error_message`:错误信息(失败时)
## 验证步骤
```bash
# 1. 启动应用(确保 Milvus 已运行)
mvn spring-boot:run
# 2. 初始化知识库
curl -X POST http://localhost:9900/api/knowledge/init
# 3. 验证结果
# - MySQL: 检查 api_document 表
# - Milvus: 检查 knowledge_base_collection
# - L0: 日志显示"知识库索引加载完成,共 6 个文档"
# 4. 测试 L1 语义检索
# lookup_knowledge("支付为什么会失败")
# 应该返回 semantic_L1 结果
```
## 文档更新
- 更新使用文档,删除"暂未实现 L1"的说明
- 添加 Milvus 数据结构说明
- 添加 Milvus 相关错误处理
This commit is contained in:
@@ -4,6 +4,7 @@ import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.DocumentChunk;
|
||||
import lombok.Data;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -37,6 +38,15 @@ public class KnowledgeBaseInitService {
|
||||
@Autowired
|
||||
private FrontmatterParser frontmatterParser;
|
||||
|
||||
@Autowired
|
||||
private DocumentChunkService documentChunkService;
|
||||
|
||||
@Autowired
|
||||
private VectorIndexService vectorIndexService;
|
||||
|
||||
@Autowired
|
||||
private VectorEmbeddingService vectorEmbeddingService;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@@ -112,6 +122,35 @@ public class KnowledgeBaseInitService {
|
||||
// 保存到数据库
|
||||
ApiDocument document = saveToDatabase(relativePath, title, summary, category, content, keywords);
|
||||
|
||||
// 提取文档正文(去除 frontmatter)
|
||||
String body = extractBody(content);
|
||||
|
||||
// 文档分块
|
||||
List<DocumentChunk> chunks = documentChunkService.chunkDocument(body, relativePath);
|
||||
logger.debug("文档分块完成: {} -> {} 个 chunk", relativePath, chunks.size());
|
||||
|
||||
// 上传到 Milvus
|
||||
try {
|
||||
vectorIndexService.indexDocumentChunks(document.getDocId(), chunks, category);
|
||||
|
||||
document.setStatus("INDEXED");
|
||||
document.setChunkCount(chunks.size());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
logger.info("文档已索引到 Milvus: {} (docId={}, chunks={})",
|
||||
title, document.getDocId(), chunks.size());
|
||||
} catch (Exception e) {
|
||||
logger.error("上传到 Milvus 失败: {}", relativePath, e);
|
||||
|
||||
document.setStatus("FAILED");
|
||||
document.setErrorMessage(e.getMessage());
|
||||
apiDocumentRepository.save(document);
|
||||
|
||||
result.incrementFailed();
|
||||
result.addDetail(relativePath, "Milvus 索引失败: " + e.getMessage());
|
||||
continue; // 跳过该文档,继续处理下一个
|
||||
}
|
||||
|
||||
// 添加到 L0 内存索引
|
||||
KnowledgeEntry entry = KnowledgeEntry.builder()
|
||||
.filePath(relativePath)
|
||||
@@ -122,12 +161,9 @@ public class KnowledgeBaseInitService {
|
||||
.build();
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
|
||||
// TODO: 上传到 Milvus (L1) - 需要通过独立的索引任务完成
|
||||
// 当前版本只处理数据库入库和 L0 索引
|
||||
|
||||
result.incrementInserted();
|
||||
result.addDetail(relativePath, "导入成功(L0)");
|
||||
logger.info("文档导入成功: {} -> {} (L0 索引已更新)", relativePath, title);
|
||||
result.addDetail(relativePath, "导入成功(L0+L1)");
|
||||
logger.info("文档导入成功: {} -> {} (L0+L1 索引已更新)", relativePath, title);
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.error("处理文档失败: {}", relativePath, e);
|
||||
|
||||
Reference in New Issue
Block a user