package com.superbiz.agent.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.ApiDocument; import com.superbiz.agent.domain.enums.FaultCategory; import com.superbiz.agent.dto.DocumentChunk; import com.superbiz.agent.dto.DocumentQueryResponse; import com.superbiz.agent.dto.DocumentUploadRequest; import com.superbiz.agent.dto.Frontmatter; import com.superbiz.agent.dto.KnowledgeEntry; import com.superbiz.agent.exception.DocumentProcessException; import com.superbiz.agent.repository.ApiDocumentRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.MessageDigest; import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; /** * 文档管理服务 */ @Slf4j @Service public class DocumentManagementService { @Value("${knowledge.base-path}") private String knowledgeBasePath; @Autowired private TextExtractorService textExtractorService; @Autowired private DocumentChunkService documentChunkService; @Autowired private VectorIndexService vectorIndexService; @Autowired private ApiDocumentRepository apiDocumentRepository; @Autowired private FrontmatterParser frontmatterParser; @Autowired private KnowledgeIndexService knowledgeIndexService; @Autowired private DocumentFieldEnricher documentFieldEnricher; @Autowired private KnowledgeDomainService knowledgeDomainService; @Autowired private ObjectMapper objectMapper; /** * 上传文档 * * @param request 上传请求 * @return 文档ID */ @Transactional public String uploadDocument(DocumentUploadRequest request) { MultipartFile file = request.getFile(); String fileName = file.getOriginalFilename(); String localPath = null; long startTime = System.currentTimeMillis(); log.info("开始上传文档,文件名: {}, 大小: {} bytes", fileName, file.getSize()); try { // 1. 验证文件格式 if (!textExtractorService.isSupportedFormat(fileName)) { throw new DocumentProcessException( fileName, "upload", "不支持的文件格式,仅支持 .md 和 .txt" ); } // 2. 计算文件 hash(去重) long hashStart = System.currentTimeMillis(); String fileHash = calculateFileHash(file); log.debug("文件hash计算完成: hash={}, time={}ms", fileHash, System.currentTimeMillis() - hashStart); Optional existing = apiDocumentRepository.findByFileHash(fileHash); if (existing.isPresent()) { log.warn("文档已存在,hash: {}, docId: {}", fileHash, existing.get().getDocId()); throw new DocumentProcessException( fileName, "upload", "文档已存在,docId: " + existing.get().getDocId() ); } // 3. 提取文本 long extractStart = System.currentTimeMillis(); String text = textExtractorService.extractText(file, fileName); log.debug("文本提取完成: length={}, time={}ms", text != null ? text.length() : 0, System.currentTimeMillis() - extractStart); if (text == null || text.isBlank()) { throw new DocumentProcessException(fileName, "upload", "文档内容为空"); } // 4. 保存原始文件到本地 String category = request.getCategory(); if (category == null || category.isBlank()) { category = "default"; } long saveStart = System.currentTimeMillis(); localPath = saveToLocal(file, fileName, category); log.debug("文件保存到本地完成: path={}, time={}ms", localPath, System.currentTimeMillis() - saveStart); // 5. 解析 frontmatter long frontmatterStart = System.currentTimeMillis(); Frontmatter frontmatter = null; String bodyText = text; if (frontmatterParser.hasFrontmatter(text)) { frontmatter = frontmatterParser.parse(text); if (frontmatter != null) { // LLM 补全 covers / whenToRetrieve(已有值则跳过) bodyText = frontmatterParser.stripFrontmatter(text); documentFieldEnricher.enrich(frontmatter, bodyText, category); log.info("解析到frontmatter: title={}, keywords={}, time={}ms", frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart); } else { log.warn("frontmatter解析失败,文件名: {}", fileName); } } else { log.debug("文件不包含frontmatter: {}", fileName); } // 6. 分块 long chunkStart = System.currentTimeMillis(); List chunks = documentChunkService.chunkDocument(bodyText, fileName); if (chunks.isEmpty()) { throw new DocumentProcessException(fileName, "upload", "文档分块失败"); } log.info("文档分块完成: fileName={}, chunks={}, time={}ms", fileName, chunks.size(), System.currentTimeMillis() - chunkStart); // 7. 创建文档元数据 String docId = resolveDocumentId(frontmatter); String metadataJson = null; if (frontmatter != null) { try { metadataJson = objectMapper.writeValueAsString(frontmatter); } catch (Exception e) { log.warn("Frontmatter序列化失败", e); } } ApiDocument document = ApiDocument.builder() .docId(docId) .fileName(fileName) .filePath(localPath) .metadata(metadataJson) .faultCategory(parseFaultCategory(request.getFaultCategory())) .faultSource(request.getFaultSource()) .apiName(request.getApiName()) .version(request.getVersion()) .fileSize(file.getSize()) .fileHash(fileHash) .status("PROCESSING") .chunkCount(chunks.size()) .build(); apiDocumentRepository.save(document); log.info("文档元数据已保存: docId={}", docId); // 8. 向量化并索引 try { long vectorStart = System.currentTimeMillis(); vectorIndexService.indexDocumentChunks(docId, chunks, category, frontmatter); document.setStatus("INDEXED"); document.setIndexedAt(LocalDateTime.now()); apiDocumentRepository.save(document); log.info("文档向量索引完成: docId={}, category={}, time={}ms", docId, category, System.currentTimeMillis() - vectorStart); } catch (Exception e) { log.error("文档索引失败: docId={}", docId, e); document.setStatus("FAILED"); apiDocumentRepository.save(document); throw new DocumentProcessException(docId, "index", "向量化索引失败: " + e.getMessage(), e); } // 9. 更新 L0 索引 if (frontmatter != null) { KnowledgeEntry entry = KnowledgeEntry.builder() .filePath(localPath) .title(frontmatter.getTitle()) .keywords(frontmatter.getKeywords()) .summary(frontmatter.getSummary()) .category(category) .kbScope(frontmatter.getKbScope()) .sections(frontmatter.getSections()) .covers(frontmatter.getCovers()) .whenToRetrieve(frontmatter.getWhenToRetrieve()) .build(); knowledgeIndexService.addToIndex(entry); log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle()); } // 触发域级聚合重算 knowledgeDomainService.onDocumentChange(category); long totalTime = System.currentTimeMillis() - startTime; log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms", docId, fileName, frontmatter != null, totalTime); return docId; } catch (Exception e) { // 失败时清理本地文件 cleanupLocalFile(localPath); log.error("文档上传失败: fileName={}", fileName, e); throw e; } } /** * 计算文件 hash(MD5) */ private String calculateFileHash(MultipartFile file) { try { MessageDigest md = MessageDigest.getInstance("MD5"); byte[] digest = md.digest(file.getBytes()); StringBuilder sb = new StringBuilder(); for (byte b : digest) { sb.append(String.format("%02x", b)); } return sb.toString(); } catch (Exception e) { throw new DocumentProcessException( file.getOriginalFilename(), "hash", "计算文件 hash 失败: " + e.getMessage(), e ); } } /** * 保存文件到本地 * * @param file 上传的文件 * @param fileName 文件名 * @param category 类别 * @return 本地文件路径 */ private String saveToLocal(MultipartFile file, String fileName, String category) { try { // 1. 构建目标路径 Path baseDir = Paths.get(knowledgeBasePath).normalize(); Path categoryDir = baseDir.resolve(category).normalize(); Files.createDirectories(categoryDir); Path targetPath = categoryDir.resolve(fileName); // 2. 保存文件 file.transferTo(targetPath.toFile()); String relativePath = baseDir.relativize(targetPath.normalize()).toString().replace("\\", "/"); log.info("文件已保存到本地: {}, storedPath={}", targetPath, relativePath); return relativePath; } catch (IOException e) { throw new DocumentProcessException( fileName, "save-local", "保存文件到本地失败: " + e.getMessage(), e ); } } /** * 清理本地文件(事务回滚时调用) * * @param localPath 本地文件路径 */ private void cleanupLocalFile(String localPath) { if (localPath != null) { try { Files.deleteIfExists(resolveLocalPath(localPath)); log.info("已清理本地文件: {}", localPath); } catch (IOException e) { log.warn("清理本地文件失败: {}", localPath, e); } } } /** * 解析故障类别 */ private FaultCategory parseFaultCategory(String category) { if (category == null || category.isBlank()) { return FaultCategory.GENERAL; } try { return FaultCategory.valueOf(category.toUpperCase()); } catch (IllegalArgumentException e) { return FaultCategory.GENERAL; } } private String resolveDocumentId(Frontmatter frontmatter) { if (frontmatter != null && frontmatter.getSource() != null) { String source = frontmatter.getSource().trim(); if (!source.isEmpty() && source.length() <= 64) { return source; } } return UUID.randomUUID().toString(); } /** * 根据 docId 查询文档 */ public DocumentQueryResponse queryDocumentById(String docId) { Optional optional = apiDocumentRepository.findByDocId(docId); if (optional.isEmpty()) { throw new DocumentProcessException(docId, "query", "文档不存在"); } ApiDocument doc = optional.get(); return convertToResponse(doc); } /** * 根据状态查询文档列表(分页) */ public List queryDocumentsByStatus(String status, int page, int size) { Page documents = apiDocumentRepository.findByStatus(status, PageRequest.of(page, size)); return documents.stream() .map(this::convertToResponse) .collect(Collectors.toList()); } /** * 根据故障源查询文档列表 */ public List queryDocumentsByFaultSource(String faultSource) { List documents = apiDocumentRepository.findByFaultSource(faultSource); return documents.stream() .map(this::convertToResponse) .collect(Collectors.toList()); } /** * 删除文档 */ @Transactional public void deleteDocument(String docId) { Optional optional = apiDocumentRepository.findByDocId(docId); if (optional.isEmpty()) { throw new DocumentProcessException(docId, "delete", "文档不存在"); } ApiDocument doc = optional.get(); // 删除本地文件 if (doc.getFilePath() != null) { try { Files.deleteIfExists(resolveLocalPath(doc.getFilePath())); log.info("本地文件已删除: {}", doc.getFilePath()); } catch (IOException e) { log.warn("删除本地文件失败: {}", doc.getFilePath(), e); } } // 删除 L0 索引 if (doc.getFilePath() != null) { knowledgeIndexService.removeFromIndex(doc.getFilePath()); } // 删除向量索引 try { vectorIndexService.deleteDocumentChunks(docId); log.info("文档向量索引已删除,docId: {}", docId); } catch (Exception e) { log.warn("删除向量索引失败,docId: {}", docId, e); } // 删除元数据 apiDocumentRepository.delete(doc); log.info("文档已删除,docId: {}", docId); // 触发域级聚合重算 String category = doc.getFilePath() != null ? resolveCategory(doc.getFilePath()) : null; if (category != null) { knowledgeDomainService.onDocumentChange(category); } } /** * 转换为响应 DTO */ /** * 从 filePath 解析 category(取 knowledge_base/{category}/... 中的 category 段) */ private String resolveCategory(String filePath) { try { java.nio.file.Path p = java.nio.file.Paths.get(filePath); // filePath 形如 knowledge_base/payment/xxx.md,取倒数第二段 int nameCount = p.getNameCount(); if (nameCount >= 2) { return p.getName(nameCount - 2).toString(); } } catch (Exception ignored) {} return null; } private Path resolveLocalPath(String filePath) { Path path = Paths.get(filePath).normalize(); if (path.isAbsolute()) { return path; } Path basePath = Paths.get(knowledgeBasePath).toAbsolutePath().normalize(); Path baseName = basePath.getFileName(); if (baseName != null && path.startsWith(baseName) && basePath.getParent() != null) { return basePath.getParent().resolve(path).normalize(); } Path pathFromWorkingDir = path.toAbsolutePath().normalize(); if (pathFromWorkingDir.startsWith(basePath)) { return pathFromWorkingDir; } return basePath.resolve(path).normalize(); } /** * 转换为响应 DTO */ private DocumentQueryResponse convertToResponse(ApiDocument doc) { return DocumentQueryResponse.builder() .docId(doc.getDocId()) .fileName(doc.getFileName()) .faultCategory(doc.getFaultCategory().name()) .faultSource(doc.getFaultSource()) .apiName(doc.getApiName()) .version(doc.getVersion()) .fileSize(doc.getFileSize()) .status(doc.getStatus()) .chunkCount(doc.getChunkCount()) .indexedAt(doc.getIndexedAt()) .createdAt(doc.getCreatedAt()) .build(); } }