From bb44140901b50666a0b5f8ecb210a1e8782e870b Mon Sep 17 00:00:00 2001 From: zhuyongxin Date: Wed, 1 Jul 2026 10:47:46 +0800 Subject: [PATCH] =?UTF-8?q?feat(knowledge):=20=E4=BC=9A=E8=AF=9D=E7=BA=A7?= =?UTF-8?q?=E5=8E=BB=E9=87=8D=20+=20=E7=9F=A5=E8=AF=86=E5=9F=9F=E5=9C=B0?= =?UTF-8?q?=E5=9B=BE=E6=B3=A8=E5=85=A5=20Planner=20=E8=A7=A3=E5=86=B3=20IS?= =?UTF-8?q?S-001=20=E9=87=8D=E5=A4=8D=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RetrievedDocTracker: sessionId → Set 会话级去重，LookupKnowledgeTool Step 5 过滤已检索文档 - KnowledgeDomainService: 域级聚合，LLM 生成 when_to_retrieve，构建 knowledge map YAML - DocumentFieldEnricher: 上传时 LLM 补全 covers + whenToRetrieve（含同域文档排除上下文） - KnowledgeDomain entity + V009 迁移: 域级元数据持久化，避免重启重复 LLM 调用 - ChatService: 注入 knowledge map 到 Planner prompt，会话结束时清理去重状态 - KnowledgeIndexService: 手写 JSON 解析替换为 Jackson ObjectMapper，启动时补建缺失域记录 - chat-planner-prompt: 新增知识库检索规则（按域 when_to_retrieve 判断，每域最多一次检索） - doc-field-enricher-prompt / domain-summary-prompt: 外部化 LLM 提示词 --- .../agent/domain/entity/KnowledgeDomain.java | 51 +++++ .../com/superbiz/agent/dto/Frontmatter.java | 10 + .../superbiz/agent/dto/KnowledgeEntry.java | 10 + .../com/superbiz/agent/dto/LookupResult.java | 5 + .../repository/KnowledgeDomainRepository.java | 13 ++ .../superbiz/agent/service/ChatService.java | 18 +- .../agent/service/DocumentFieldEnricher.java | 135 +++++++++++++ .../service/DocumentManagementService.java | 38 ++++ .../agent/service/FrontmatterParser.java | 2 + .../agent/service/KnowledgeDomainService.java | 188 ++++++++++++++++++ .../agent/service/KnowledgeIndexService.java | 145 ++++---------- .../agent/tool/LookupKnowledgeTool.java | 30 +++ .../agent/tool/RetrievedDocTracker.java | 36 ++++ .../migration/V009__add_knowledge_domain.sql | 9 + .../resources/prompts/chat-planner-prompt.md | 6 + .../prompts/doc-field-enricher-prompt.md | 31 +++ .../prompts/domain-summary-prompt.md | 21 ++ 17 files changed, 642 insertions(+), 106 deletions(-) create mode 100644 src/main/java/com/superbiz/agent/domain/entity/KnowledgeDomain.java create mode 100644 src/main/java/com/superbiz/agent/repository/KnowledgeDomainRepository.java create mode 100644 src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java create mode 100644 src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java create mode 100644 src/main/java/com/superbiz/agent/tool/RetrievedDocTracker.java create mode 100644 src/main/resources/db/migration/V009__add_knowledge_domain.sql create mode 100644 src/main/resources/prompts/doc-field-enricher-prompt.md create mode 100644 src/main/resources/prompts/domain-summary-prompt.md diff --git a/src/main/java/com/superbiz/agent/domain/entity/KnowledgeDomain.java b/src/main/java/com/superbiz/agent/domain/entity/KnowledgeDomain.java new file mode 100644 index 0000000..1318f19 --- /dev/null +++ b/src/main/java/com/superbiz/agent/domain/entity/KnowledgeDomain.java @@ -0,0 +1,51 @@ +package com.superbiz.agent.domain.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "knowledge_domain") +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class KnowledgeDomain { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "domain_id", unique = true, nullable = false, length = 64) + private String domainId; + + @Column(name = "description", length = 256) + private String description; + + @Column(name = "when_to_retrieve", columnDefinition = "TEXT") + private String whenToRetrieve; + + @Column(name = "document_count", nullable = false) + private int documentCount; + + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + @Column(name = "updated_at", nullable = false) + private LocalDateTime updatedAt; + + @PrePersist + protected void onCreate() { + createdAt = LocalDateTime.now(); + updatedAt = LocalDateTime.now(); + } + + @PreUpdate + protected void onUpdate() { + updatedAt = LocalDateTime.now(); + } +} diff --git a/src/main/java/com/superbiz/agent/dto/Frontmatter.java b/src/main/java/com/superbiz/agent/dto/Frontmatter.java index 5d12553..3d02554 100644 --- a/src/main/java/com/superbiz/agent/dto/Frontmatter.java +++ b/src/main/java/com/superbiz/agent/dto/Frontmatter.java @@ -59,4 +59,14 @@ public class Frontmatter { * 最后更新日期（预留字段） */ private LocalDate lastUpdated; + + /** + * 业务场景标签，供 Planner 决策用（LLM 上传时自动生成） + */ + private List covers; + + /** + * 文档级检索时机（LLM 上传时自动生成） + */ + private String whenToRetrieve; } diff --git a/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java b/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java index d783993..2b89cf7 100644 --- a/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java +++ b/src/main/java/com/superbiz/agent/dto/KnowledgeEntry.java @@ -43,4 +43,14 @@ public class KnowledgeEntry { * 章节锚点（预留字段，MVP 不使用） */ private Map sections; + + /** + * 业务场景标签，供 Planner 决策用 + */ + private List covers; + + /** + * 文档级检索时机 + */ + private String whenToRetrieve; } diff --git a/src/main/java/com/superbiz/agent/dto/LookupResult.java b/src/main/java/com/superbiz/agent/dto/LookupResult.java index b26e415..4beb459 100644 --- a/src/main/java/com/superbiz/agent/dto/LookupResult.java +++ b/src/main/java/com/superbiz/agent/dto/LookupResult.java @@ -26,4 +26,9 @@ public class LookupResult { * 补充结果（L1 语义检索） */ private SupplementResult supplement; + + /** + * 系统消息（如去重提示） + */ + private String message; } diff --git a/src/main/java/com/superbiz/agent/repository/KnowledgeDomainRepository.java b/src/main/java/com/superbiz/agent/repository/KnowledgeDomainRepository.java new file mode 100644 index 0000000..f22f5b1 --- /dev/null +++ b/src/main/java/com/superbiz/agent/repository/KnowledgeDomainRepository.java @@ -0,0 +1,13 @@ +package com.superbiz.agent.repository; + +import com.superbiz.agent.domain.entity.KnowledgeDomain; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface KnowledgeDomainRepository extends JpaRepository { + + Optional findByDomainId(String domainId); +} diff --git a/src/main/java/com/superbiz/agent/service/ChatService.java b/src/main/java/com/superbiz/agent/service/ChatService.java index d97237b..1c5af62 100644 --- a/src/main/java/com/superbiz/agent/service/ChatService.java +++ b/src/main/java/com/superbiz/agent/service/ChatService.java @@ -16,8 +16,10 @@ import com.superbiz.agent.hook.TokenUsageHolder; import com.superbiz.agent.repository.AgentStepRepository; import com.superbiz.agent.repository.DiagnosisSessionRepository; import com.superbiz.agent.tool.LookupKnowledgeTool; +import com.superbiz.agent.tool.RetrievedDocTracker; import com.superbiz.agent.util.QuestionComplexity; import com.superbiz.agent.util.SessionContextHolder; +import com.superbiz.agent.service.KnowledgeDomainService; import jakarta.annotation.PostConstruct; import org.slf4j.Logger; @@ -79,6 +81,12 @@ public class ChatService { @Autowired private EvaluationService evaluationService; + @Autowired + private RetrievedDocTracker retrievedDocTracker; + + @Autowired + private KnowledgeDomainService knowledgeDomainService; + /** 多 Agent Chat 的 Prompt */ private String chatPlannerPrompt; private String chatExecutorPrompt; @@ -290,6 +298,7 @@ public class ChatService { diagnosisSessionRepository.save(session); throw e; } finally { + retrievedDocTracker.clearSession(sessionId); SessionContextHolder.clear(); } } @@ -383,6 +392,7 @@ public class ChatService { logger.error("多 Agent 执行失败", e); return new ChatResult("执行失败: " + e.getMessage(), sessionId); } finally { + retrievedDocTracker.clearSession(sessionId); SessionContextHolder.clear(); } } @@ -390,6 +400,13 @@ public class ChatService { private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks, List> history) { StringBuilder prompt = new StringBuilder(chatPlannerPrompt); + + // 注入 knowledge map + String knowledgeMap = knowledgeDomainService.buildKnowledgeMap(); + if (!knowledgeMap.isBlank()) { + prompt.append("\n\n## 可用知识库\n\n").append(knowledgeMap); + } + if (!history.isEmpty()) { prompt.append("\n\n--- 对话历史 ---\n"); for (Map msg : history) { @@ -402,7 +419,6 @@ public class ChatService { .description("负责拆解问题、规划步骤") .model(chatModel) .systemPrompt(prompt.toString()) - // Planner 不注入工具，只能规划不能执行 .hooks(new AgentLoggingHook(agentStepRepository, "planner")) .outputKey("planner_plan") .build(); diff --git a/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java b/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java new file mode 100644 index 0000000..927e73b --- /dev/null +++ b/src/main/java/com/superbiz/agent/service/DocumentFieldEnricher.java @@ -0,0 +1,135 @@ +package com.superbiz.agent.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.dto.Frontmatter; +import com.superbiz.agent.dto.KnowledgeEntry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PostConstruct; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 文档字段补全服务 + * 上传时调用 LLM 生成 covers 和 whenToRetrieve + */ +@Slf4j +@Service +public class DocumentFieldEnricher { + + @Autowired + private ChatModel chatModel; + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private KnowledgeIndexService knowledgeIndexService; + + private String promptTemplate; + + @PostConstruct + public void init() { + try { + promptTemplate = new String( + new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(), + StandardCharsets.UTF_8); + log.info("DocumentFieldEnricher prompt 加载成功"); + } catch (IOException e) { + log.error("加载 doc-field-enricher-prompt.md 失败", e); + throw new RuntimeException("Failed to load doc-field-enricher prompt", e); + } + } + + public void enrich(Frontmatter frontmatter, String bodyText) { + enrich(frontmatter, bodyText, null); + } + + /** + * 为 Frontmatter 补全 covers 和 whenToRetrieve + * 若已有值则跳过；LLM 失败时降级，不阻断主流程 + * + * @param frontmatter 待补全的 frontmatter + * @param bodyText 文档正文 + * @param category 文档所属域（用于查找同域其他文档） + */ + public void enrich(Frontmatter frontmatter, String bodyText, String category) { + if (frontmatter == null) return; + + boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty(); + boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank(); + + if (!needsCovers && !needsWhen) { + log.debug("covers 和 whenToRetrieve 已存在，跳过 LLM 生成"); + return; + } + + try { + String snippet = bodyText != null && bodyText.length() > 1000 + ? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : ""); + + String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category); + + String promptText = String.format(promptTemplate, + frontmatter.getTitle(), + frontmatter.getSummary(), + sameDomainDocs, + snippet); + + String response = chatModel.call(new Prompt(promptText)) + .getResult().getOutput().getText(); + + // 提取 JSON 部分（防止模型输出多余文本） + String json = extractJson(response); + JsonNode node = objectMapper.readTree(json); + + if (needsCovers && node.has("covers")) { + List covers = new ArrayList<>(); + node.get("covers").forEach(n -> covers.add(n.asText())); + frontmatter.setCovers(covers); + log.debug("LLM 生成 covers: {}", covers); + } + + if (needsWhen && node.has("whenToRetrieve")) { + frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText()); + log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve()); + } + + } catch (Exception e) { + log.warn("LLM 生成文档字段失败，降级处理: title={}", frontmatter.getTitle(), e); + if (needsCovers) frontmatter.setCovers(List.of()); + if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary()); + } + } + + private String extractJson(String text) { + if (text == null) return "{}"; + int start = text.indexOf('{'); + int end = text.lastIndexOf('}'); + if (start == -1 || end == -1 || end <= start) return "{}"; + return text.substring(start, end + 1); + } + + /** + * 构建同域其他文档标题列表（供 LLM 做排除判断） + */ + private String buildSameDomainDocs(String currentTitle, String category) { + if (category == null || category.isBlank()) return "（无同域文档信息）"; + List otherTitles = knowledgeIndexService.getAllEntries().stream() + .filter(e -> category.equals(e.getCategory())) + .map(KnowledgeEntry::getTitle) + .filter(t -> t != null && !t.equals(currentTitle)) + .collect(Collectors.toList()); + if (otherTitles.isEmpty()) return "（无同域其他文档）"; + return String.join("、", otherTitles); + } +} diff --git a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java index 895cd4c..1c44f39 100644 --- a/src/main/java/com/superbiz/agent/service/DocumentManagementService.java +++ b/src/main/java/com/superbiz/agent/service/DocumentManagementService.java @@ -58,6 +58,12 @@ public class DocumentManagementService { @Autowired private KnowledgeIndexService knowledgeIndexService; + @Autowired + private DocumentFieldEnricher documentFieldEnricher; + + @Autowired + private KnowledgeDomainService knowledgeDomainService; + @Autowired private ObjectMapper objectMapper; @@ -123,6 +129,8 @@ public class DocumentManagementService { if (frontmatterParser.hasFrontmatter(text)) { frontmatter = frontmatterParser.parse(text); if (frontmatter != null) { + // LLM 补全 covers / whenToRetrieve（已有值则跳过） + documentFieldEnricher.enrich(frontmatter, text, category); log.info("解析到frontmatter: title={}, keywords={}, time={}ms", frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart); } else { @@ -196,12 +204,17 @@ public class DocumentManagementService { .summary(frontmatter.getSummary()) .category(category) .sections(frontmatter.getSections()) + .covers(frontmatter.getCovers()) + .whenToRetrieve(frontmatter.getWhenToRetrieve()) .build(); knowledgeIndexService.addToIndex(entry); log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle()); } + // 触发域级聚合重算 + knowledgeDomainService.onDocumentChange(category); + long totalTime = System.currentTimeMillis() - startTime; log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms", docId, fileName, frontmatter != null, totalTime); @@ -367,6 +380,31 @@ public class DocumentManagementService { // 删除元数据 apiDocumentRepository.delete(doc); log.info("文档已删除，docId: {}", docId); + + // 触发域级聚合重算 + String category = doc.getFilePath() != null + ? resolveCategory(doc.getFilePath()) : null; + if (category != null) { + knowledgeDomainService.onDocumentChange(category); + } + } + + /** + * 转换为响应 DTO + */ + /** + * 从 filePath 解析 category（取 knowledge_base/{category}/... 中的 category 段） + */ + private String resolveCategory(String filePath) { + try { + java.nio.file.Path p = java.nio.file.Paths.get(filePath); + // filePath 形如 knowledge_base/payment/xxx.md，取倒数第二段 + int nameCount = p.getNameCount(); + if (nameCount >= 2) { + return p.getName(nameCount - 2).toString(); + } + } catch (Exception ignored) {} + return null; } /** diff --git a/src/main/java/com/superbiz/agent/service/FrontmatterParser.java b/src/main/java/com/superbiz/agent/service/FrontmatterParser.java index ffa87d6..9efcad6 100644 --- a/src/main/java/com/superbiz/agent/service/FrontmatterParser.java +++ b/src/main/java/com/superbiz/agent/service/FrontmatterParser.java @@ -65,6 +65,8 @@ public class FrontmatterParser { .sections((Map) map.get("sections")) .version((String) map.get("version")) .author((String) map.get("author")) + .covers((java.util.List) map.get("covers")) + .whenToRetrieve((String) map.get("when_to_retrieve")) .build(); // 4. 验证必填字段 diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java b/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java new file mode 100644 index 0000000..d5f6971 --- /dev/null +++ b/src/main/java/com/superbiz/agent/service/KnowledgeDomainService.java @@ -0,0 +1,188 @@ +package com.superbiz.agent.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.domain.entity.KnowledgeDomain; +import com.superbiz.agent.dto.KnowledgeEntry; +import com.superbiz.agent.repository.KnowledgeDomainRepository; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PostConstruct; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * 知识域服务 + * 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建 + */ +@Slf4j +@Service +public class KnowledgeDomainService { + + @Autowired + private KnowledgeDomainRepository knowledgeDomainRepository; + + @Autowired + private KnowledgeIndexService knowledgeIndexService; + + @Autowired + private ChatModel chatModel; + + @Autowired + private ObjectMapper objectMapper; + + private String domainPromptTemplate; + + @PostConstruct + public void init() { + try { + domainPromptTemplate = new String( + new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(), + StandardCharsets.UTF_8); + log.info("KnowledgeDomainService prompt 加载成功"); + } catch (IOException e) { + log.error("加载 domain-summary-prompt.md 失败", e); + throw new RuntimeException("Failed to load domain-summary prompt", e); + } + } + + /** + * 文档变更后重算指定域的 when_to_retrieve + */ + public void onDocumentChange(String category) { + if (category == null || category.isBlank()) return; + + List entries = knowledgeIndexService.getAllEntries().stream() + .filter(e -> category.equals(e.getCategory())) + .collect(Collectors.toList()); + + buildDomainSummary(category, entries); + } + + /** + * 聚合同域文档，调用 LLM 生成域级摘要，写入 DB + */ + public void buildDomainSummary(String category, List entries) { + if (entries.isEmpty()) { + knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> { + d.setDocumentCount(0); + knowledgeDomainRepository.save(d); + }); + return; + } + + // 构建文档列表描述 + StringBuilder docList = new StringBuilder(); + for (KnowledgeEntry entry : entries) { + docList.append("- 文档：").append(entry.getTitle()).append("\n"); + if (entry.getWhenToRetrieve() != null) { + docList.append(" 适用场景：").append(entry.getWhenToRetrieve()).append("\n"); + } + if (entry.getCovers() != null && !entry.getCovers().isEmpty()) { + docList.append(" 覆盖：").append(String.join("、", entry.getCovers())).append("\n"); + } + } + + String description = entries.stream() + .map(KnowledgeEntry::getSummary) + .filter(s -> s != null && !s.isBlank()) + .findFirst().orElse(category); + + String whenToRetrieve = null; + try { + String otherDomainsInfo = buildOtherDomainsInfo(category); + String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo); + whenToRetrieve = chatModel.call(new Prompt(promptText)) + .getResult().getOutput().getText(); + log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve); + } catch (Exception e) { + log.warn("LLM 生成域级 when_to_retrieve 失败，保留旧值: domain={}", category, e); + Optional existing = knowledgeDomainRepository.findByDomainId(category); + whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse(""); + } + + KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category) + .orElse(KnowledgeDomain.builder().domainId(category).build()); + + domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description); + domain.setWhenToRetrieve(whenToRetrieve); + domain.setDocumentCount(entries.size()); + knowledgeDomainRepository.save(domain); + } + + /** + * 构建注入 Planner 的 knowledge map YAML 文本 + */ + public String buildKnowledgeMap() { + List domains = knowledgeDomainRepository.findAll(); + if (domains.isEmpty()) return ""; + + List allEntries = knowledgeIndexService.getAllEntries(); + Map> byCategory = allEntries.stream() + .filter(e -> e.getCategory() != null) + .collect(Collectors.groupingBy(KnowledgeEntry::getCategory)); + + StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n"); + + for (KnowledgeDomain domain : domains) { + yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n"); + if (domain.getDescription() != null) { + yaml.append(" description: \"").append(domain.getDescription()).append("\"\n"); + } + if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) { + yaml.append(" when_to_retrieve: \"") + .append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n"); + } + yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n"); + + List domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of()); + if (!domainEntries.isEmpty()) { + yaml.append(" documents:\n"); + for (KnowledgeEntry entry : domainEntries) { + yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n"); + if (entry.getCovers() != null && !entry.getCovers().isEmpty()) { + yaml.append(" covers: ").append(entry.getCovers()).append("\n"); + } + } + } + } + + return yaml.toString(); + } + + /** + * 构建其他域的摘要信息（用于 LLM 域级 prompt 的边界判断） + * 优先使用其他域的 when_to_retrieve（边界信号），而非 description + */ + private String buildOtherDomainsInfo(String currentCategory) { + List allDomains = knowledgeDomainRepository.findAll(); + StringBuilder sb = new StringBuilder(); + for (KnowledgeDomain d : allDomains) { + if (d.getDomainId().equals(currentCategory)) continue; + sb.append("- ").append(d.getDomainId()); + if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) { + sb.append("：").append(d.getWhenToRetrieve()); + } else if (d.getDescription() != null && !d.getDescription().isBlank()) { + sb.append("（").append(d.getDescription()).append("）"); + } + sb.append("\n"); + } + // 如果 DB 里还没有其他域的记录（首次启动），从 L0 索引补充 + if (sb.isEmpty()) { + knowledgeIndexService.getAllEntries().stream() + .map(KnowledgeEntry::getCategory) + .filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory)) + .distinct() + .forEach(c -> sb.append("- ").append(c).append("\n")); + } + return sb.isEmpty() ? "（无其他域信息）" : sb.toString(); + } +} diff --git a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java b/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java index a97270b..de99940 100644 --- a/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java +++ b/src/main/java/com/superbiz/agent/service/KnowledgeIndexService.java @@ -1,12 +1,15 @@ package com.superbiz.agent.service; +import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.ApiDocument; -import com.superbiz.agent.repository.ApiDocumentRepository; import com.superbiz.agent.dto.Frontmatter; import com.superbiz.agent.dto.KnowledgeEntry; +import com.superbiz.agent.repository.ApiDocumentRepository; +import com.superbiz.agent.repository.KnowledgeDomainRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import jakarta.annotation.PostConstruct; @@ -14,12 +17,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * 知识库索引服务 @@ -35,26 +35,28 @@ public class KnowledgeIndexService { @Autowired private ApiDocumentRepository apiDocumentRepository; - /** - * 内存索引（线程安全） - */ + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private KnowledgeDomainRepository knowledgeDomainRepository; + + @Lazy + @Autowired + private KnowledgeDomainService knowledgeDomainService; + private final List knowledgeIndex = new CopyOnWriteArrayList<>(); - /** - * 启动时从数据库加载索引 - */ @PostConstruct public void loadIndex() { log.info("开始从数据库加载知识库索引"); try { - // 从数据库读取所有已索引的文档 List documents = apiDocumentRepository.findAll(); int loaded = 0; for (ApiDocument doc : documents) { try { - // 从 metadata JSON 中提取信息 KnowledgeEntry entry = parseDocumentToEntry(doc); if (entry != null) { knowledgeIndex.add(entry); @@ -67,34 +69,39 @@ public class KnowledgeIndexService { log.info("知识库索引加载完成，共 {} 个文档", loaded); + // 检查各域是否有 knowledge_domain 记录，无则触发生成 + knowledgeIndex.stream() + .map(KnowledgeEntry::getCategory) + .filter(c -> c != null && !c.isBlank()) + .distinct() + .forEach(category -> { + if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) { + log.info("域 {} 无 knowledge_domain 记录，触发生成", category); + knowledgeDomainService.onDocumentChange(category); + } + }); + } catch (Exception e) { log.error("知识库索引加载失败", e); } } - /** - * 将 ApiDocument 转换为 KnowledgeEntry - */ private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) { if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) { return null; } try { - // 简单的 JSON 解析 - String metadata = doc.getMetadata(); - - String title = extractJsonValue(metadata, "title"); - String summary = extractJsonValue(metadata, "summary"); - String category = extractJsonValue(metadata, "category"); - List keywords = extractJsonArray(metadata, "keywords"); + Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class); return KnowledgeEntry.builder() .filePath(doc.getFilePath()) - .title(title != null ? title : doc.getApiName()) - .keywords(keywords) - .summary(summary) - .category(category) + .title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName()) + .keywords(frontmatter.getKeywords()) + .summary(frontmatter.getSummary()) + .category(frontmatter.getCategory()) + .covers(frontmatter.getCovers()) + .whenToRetrieve(frontmatter.getWhenToRetrieve()) .build(); } catch (Exception e) { @@ -103,54 +110,6 @@ public class KnowledgeIndexService { } } - /** - * 从 JSON 字符串中提取值 - */ - private String extractJsonValue(String json, String key) { - String pattern = "\"" + key + "\":\""; - int startIndex = json.indexOf(pattern); - if (startIndex == -1) { - return null; - } - - startIndex += pattern.length(); - int endIndex = json.indexOf("\"", startIndex); - if (endIndex == -1) { - return null; - } - - return json.substring(startIndex, endIndex); - } - - /** - * 从 JSON 字符串中提取数组 - */ - private List extractJsonArray(String json, String key) { - String pattern = "\"" + key + "\":["; - int startIndex = json.indexOf(pattern); - if (startIndex == -1) { - return Collections.emptyList(); - } - - startIndex += pattern.length(); - int endIndex = json.indexOf("]", startIndex); - if (endIndex == -1) { - return Collections.emptyList(); - } - - String arrayContent = json.substring(startIndex, endIndex); - return Arrays.stream(arrayContent.split(",")) - .map(s -> s.trim().replaceAll("^\"|\"$", "")) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList()); - } - - /** - * L0 精确匹配 - * - * @param query 查询关键词 - * @return 匹配的文档列表 - */ public List exactMatch(String query) { long startTime = System.currentTimeMillis(); @@ -172,13 +131,6 @@ public class KnowledgeIndexService { return results; } - /** - * 关键词匹配逻辑（不区分大小写） - * - * @param entry 索引条目 - * @param query 查询关键词（小写） - * @return true 如果匹配 - */ private boolean matchesKeywords(KnowledgeEntry entry, String query) { if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) { return false; @@ -186,7 +138,6 @@ public class KnowledgeIndexService { for (String keyword : entry.getKeywords()) { String keywordLower = keyword.toLowerCase(); - // query 包含 keyword 或 keyword 包含 query if (query.contains(keywordLower) || keywordLower.contains(query)) { return true; } @@ -195,16 +146,8 @@ public class KnowledgeIndexService { return false; } - /** - * 读取文档内容 - * - * @param filePath 文件相对路径（如 api/payment-errors.md） - * @param maxChars 最大字符数 - * @return 文档内容（前 maxChars 字符），失败返回 null - */ public String readDocument(String filePath, int maxChars) { try { - // 拼接完整路径：knowledge_base + 相对路径 Path fullPath = Paths.get(knowledgeBasePath, filePath); String content = Files.readString(fullPath); @@ -220,32 +163,24 @@ public class KnowledgeIndexService { } } - /** - * 添加文档到索引（上传时调用） - * - * @param entry 知识库条目 - */ public void addToIndex(KnowledgeEntry entry) { knowledgeIndex.add(entry); log.debug("文档已添加到 L0 索引: title={}", entry.getTitle()); } - /** - * 从索引中移除文档（删除时调用） - * - * @param filePath 文件路径 - */ public void removeFromIndex(String filePath) { knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath)); log.debug("文档已从 L0 索引移除: {}", filePath); } - /** - * 获取索引大小 - * - * @return 索引中的文档数量 - */ public int getIndexSize() { return knowledgeIndex.size(); } + + /** + * 获取所有索引条目（供域聚合使用） + */ + public List getAllEntries() { + return List.copyOf(knowledgeIndex); + } } diff --git a/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java b/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java index 17e4440..734c72c 100644 --- a/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java +++ b/src/main/java/com/superbiz/agent/tool/LookupKnowledgeTool.java @@ -31,6 +31,9 @@ public class LookupKnowledgeTool { @Autowired private ToolInvocationRepository toolInvocationRepository; + @Autowired + private RetrievedDocTracker retrievedDocTracker; + /** * 查询知识库文档 * @@ -98,6 +101,23 @@ public class LookupKnowledgeTool { // Step 4: 组装结果 LookupResult result = buildResult(l0Matches, l1Results, highConfidence); + // Step 5: session 级去重过滤 + String sessionId = SessionContextHolder.getSessionId(); + if (sessionId != null && result.isFound()) { + String docKey = extractDocKey(result); + if (docKey != null && retrievedDocTracker.isAlreadyRetrieved(sessionId, docKey)) { + log.info("[去重] 文档已在本会话中检索过，跳过: {}", docKey); + saveToolInvocation(query, l0Matches, l1Results, highConfidence, startTime, result); + return LookupResult.builder() + .found(false) + .message("文档已在本会话中检索过，无需重复召回: " + docKey) + .build(); + } + if (docKey != null) { + retrievedDocTracker.markRetrieved(sessionId, docKey); + } + } + // 记录结构化结果摘要（替代原始 MD 内容预览） long totalTime = System.currentTimeMillis() - startTime; log.info("----------------------------------------"); @@ -430,4 +450,14 @@ public class LookupKnowledgeTool { return "(无有效内容)"; } + + private String extractDocKey(LookupResult result) { + if (result.getPrimary() != null && result.getPrimary().getSource() != null) { + return result.getPrimary().getSource(); + } + if (result.getSupplement() != null && result.getSupplement().getSource() != null) { + return result.getSupplement().getSource(); + } + return null; + } } diff --git a/src/main/java/com/superbiz/agent/tool/RetrievedDocTracker.java b/src/main/java/com/superbiz/agent/tool/RetrievedDocTracker.java new file mode 100644 index 0000000..58b3132 --- /dev/null +++ b/src/main/java/com/superbiz/agent/tool/RetrievedDocTracker.java @@ -0,0 +1,36 @@ +package com.superbiz.agent.tool; + +import org.springframework.stereotype.Component; + +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * session 级已召回文档追踪器 + * 防止同一 session 内重复召回相同文档 + */ +@Component +public class RetrievedDocTracker { + + // key: sessionId, value: 已召回文档的 filePath 集合 + private final ConcurrentHashMap> retrieved = new ConcurrentHashMap<>(); + + public boolean isAlreadyRetrieved(String sessionId, String filePath) { + if (sessionId == null || filePath == null) return false; + Set docs = retrieved.get(sessionId); + return docs != null && docs.contains(filePath); + } + + public void markRetrieved(String sessionId, String filePath) { + if (sessionId == null || filePath == null) return; + retrieved.computeIfAbsent(sessionId, + k -> Collections.newSetFromMap(new ConcurrentHashMap<>())) + .add(filePath); + } + + public void clearSession(String sessionId) { + if (sessionId == null) return; + retrieved.remove(sessionId); + } +} diff --git a/src/main/resources/db/migration/V009__add_knowledge_domain.sql b/src/main/resources/db/migration/V009__add_knowledge_domain.sql new file mode 100644 index 0000000..382251f --- /dev/null +++ b/src/main/resources/db/migration/V009__add_knowledge_domain.sql @@ -0,0 +1,9 @@ +CREATE TABLE knowledge_domain ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + domain_id VARCHAR(64) NOT NULL UNIQUE COMMENT 'category 值，如 payment/infrastructure', + description VARCHAR(256) COMMENT '域描述，聚合自文档 summary', + when_to_retrieve TEXT COMMENT '域级检索时机，LLM 聚合生成', + document_count INT NOT NULL DEFAULT 0 COMMENT '该域当前文档数', + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='知识域元数据，存储域级检索策略'; diff --git a/src/main/resources/prompts/chat-planner-prompt.md b/src/main/resources/prompts/chat-planner-prompt.md index 66752f2..0ffa0b1 100644 --- a/src/main/resources/prompts/chat-planner-prompt.md +++ b/src/main/resources/prompts/chat-planner-prompt.md @@ -18,3 +18,9 @@ - 每个步骤应该是一个可以独立执行的任务 - 步骤要具体可操作，不要模糊 - 如果问题需要查知识库，明确在步骤中说明要查什么 + +## 知识库检索规则 +- 制定步骤前，先查看下方 `available_knowledge_domains`（如果存在） +- 根据每个域的 `when_to_retrieve` 判断是否需要检索该域 +- 每个域最多安排一次检索步骤；已覆盖的域不要重复安排 +- 如果用户问题与某个域无关，不要安排对该域的检索 diff --git a/src/main/resources/prompts/doc-field-enricher-prompt.md b/src/main/resources/prompts/doc-field-enricher-prompt.md new file mode 100644 index 0000000..53d3f89 --- /dev/null +++ b/src/main/resources/prompts/doc-field-enricher-prompt.md @@ -0,0 +1,31 @@ +你是知识库文档标注助手。为文档生成两个检索辅助字段。 + +## 生成规则 +1. covers：该文档覆盖的业务场景（3-5 个简洁中文短语） +2. whenToRetrieve：用一句话描述何时检索此文档，不超过 40 字 + - 必须包含正向场景（什么问题查本文档） + - 必须包含反向排除（什么问题容易误判但不应查本文档） + - 格式："正向场景；不包含反向排除词" + +## Few-shot 示例 +文档标题：支付失败排查手册 +同域其他文档：退款处理指南 +→ {"covers": ["支付超时", "扣款无回调", "支付网关报错"], "whenToRetrieve": "支付失败/超时/无回调时检索；不含退款对账问题"} + +文档标题：退款处理指南 +同域其他文档：支付失败排查手册 +→ {"covers": ["退款未到账", "退款状态异常", "退款被拒"], "whenToRetrieve": "退款异常/未到账时检索；不含支付失败问题"} + +文档标题：MySQL 连接池配置 +同域其他文档：Redis 缓存配置指南、Flyway 数据库迁移 +→ {"covers": ["连接池耗尽", "数据库OOM", "慢查询"], "whenToRetrieve": "连接池/数据库性能问题时检索；不含Redis缓存或迁移问题"} + +## 待分析文档 +文档标题：%s +文档摘要：%s +同域其他文档：%s +文档内容节选： +%s + +请严格返回 JSON 格式，不要有其他内容： +{"covers": ["场景1", "场景2", ...], "whenToRetrieve": "描述"} diff --git a/src/main/resources/prompts/domain-summary-prompt.md b/src/main/resources/prompts/domain-summary-prompt.md new file mode 100644 index 0000000..91bd4a6 --- /dev/null +++ b/src/main/resources/prompts/domain-summary-prompt.md @@ -0,0 +1,21 @@ +你是知识库域级路由设计助手。根据域内文档和系统其他域信息，生成一条域级检索指引。 + +## 当前域：%s + +## 域内文档 +%s + +## 系统其他域（用于判断边界） +%s + +## 生成要求 +1. 用一句话描述"什么场景下 Planner 应该选择检索这个域" +2. 必须覆盖该域所有文档的适用场景，做合理抽象（不要简单拼接） +3. 必须包含明确的边界判断： + - 正向：什么问题属于这个域 + - 反向：什么问题容易被误判为属于这个域，但实际应该检索其他域 +4. 反向边界必须引用其他域的 domain_id，格式："X类问题查{domain_id}" +5. 不超过 60 字 +6. 语言要让 Planner 能做"检索/不检索"的二分判断 + +只返回纯文本指引，不要 JSON 或 markdown。