feat(knowledge): 会话级去重 + 知识域地图注入 Planner 解决 ISS-001 重复检索

- RetrievedDocTracker: sessionId → Set<filePath> 会话级去重,LookupKnowledgeTool Step 5 过滤已检索文档
- KnowledgeDomainService: 域级聚合,LLM 生成 when_to_retrieve,构建 knowledge map YAML
- DocumentFieldEnricher: 上传时 LLM 补全 covers + whenToRetrieve(含同域文档排除上下文)
- KnowledgeDomain entity + V009 迁移: 域级元数据持久化,避免重启重复 LLM 调用
- ChatService: 注入 knowledge map 到 Planner prompt,会话结束时清理去重状态
- KnowledgeIndexService: 手写 JSON 解析替换为 Jackson ObjectMapper,启动时补建缺失域记录
- chat-planner-prompt: 新增知识库检索规则(按域 when_to_retrieve 判断,每域最多一次检索)
- doc-field-enricher-prompt / domain-summary-prompt: 外部化 LLM 提示词
This commit is contained in:
zhuyongxin
2026-07-01 10:47:46 +08:00
parent 2a796da490
commit bb44140901
17 changed files with 642 additions and 106 deletions
@@ -0,0 +1,51 @@
package com.superbiz.agent.domain.entity;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Entity
@Table(name = "knowledge_domain")
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class KnowledgeDomain {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(name = "domain_id", unique = true, nullable = false, length = 64)
private String domainId;
@Column(name = "description", length = 256)
private String description;
@Column(name = "when_to_retrieve", columnDefinition = "TEXT")
private String whenToRetrieve;
@Column(name = "document_count", nullable = false)
private int documentCount;
@Column(name = "created_at", nullable = false, updatable = false)
private LocalDateTime createdAt;
@Column(name = "updated_at", nullable = false)
private LocalDateTime updatedAt;
@PrePersist
protected void onCreate() {
createdAt = LocalDateTime.now();
updatedAt = LocalDateTime.now();
}
@PreUpdate
protected void onUpdate() {
updatedAt = LocalDateTime.now();
}
}
@@ -59,4 +59,14 @@ public class Frontmatter {
* 最后更新日期(预留字段) * 最后更新日期(预留字段)
*/ */
private LocalDate lastUpdated; private LocalDate lastUpdated;
/**
* 业务场景标签,供 Planner 决策用(LLM 上传时自动生成)
*/
private List<String> covers;
/**
* 文档级检索时机(LLM 上传时自动生成)
*/
private String whenToRetrieve;
} }
@@ -43,4 +43,14 @@ public class KnowledgeEntry {
* 章节锚点(预留字段,MVP 不使用) * 章节锚点(预留字段,MVP 不使用)
*/ */
private Map<String, String> sections; private Map<String, String> sections;
/**
* 业务场景标签,供 Planner 决策用
*/
private List<String> covers;
/**
* 文档级检索时机
*/
private String whenToRetrieve;
} }
@@ -26,4 +26,9 @@ public class LookupResult {
* 补充结果(L1 语义检索) * 补充结果(L1 语义检索)
*/ */
private SupplementResult supplement; private SupplementResult supplement;
/**
* 系统消息(如去重提示)
*/
private String message;
} }
@@ -0,0 +1,13 @@
package com.superbiz.agent.repository;
import com.superbiz.agent.domain.entity.KnowledgeDomain;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface KnowledgeDomainRepository extends JpaRepository<KnowledgeDomain, Long> {
Optional<KnowledgeDomain> findByDomainId(String domainId);
}
@@ -16,8 +16,10 @@ import com.superbiz.agent.hook.TokenUsageHolder;
import com.superbiz.agent.repository.AgentStepRepository; import com.superbiz.agent.repository.AgentStepRepository;
import com.superbiz.agent.repository.DiagnosisSessionRepository; import com.superbiz.agent.repository.DiagnosisSessionRepository;
import com.superbiz.agent.tool.LookupKnowledgeTool; import com.superbiz.agent.tool.LookupKnowledgeTool;
import com.superbiz.agent.tool.RetrievedDocTracker;
import com.superbiz.agent.util.QuestionComplexity; import com.superbiz.agent.util.QuestionComplexity;
import com.superbiz.agent.util.SessionContextHolder; import com.superbiz.agent.util.SessionContextHolder;
import com.superbiz.agent.service.KnowledgeDomainService;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import org.slf4j.Logger; import org.slf4j.Logger;
@@ -79,6 +81,12 @@ public class ChatService {
@Autowired @Autowired
private EvaluationService evaluationService; private EvaluationService evaluationService;
@Autowired
private RetrievedDocTracker retrievedDocTracker;
@Autowired
private KnowledgeDomainService knowledgeDomainService;
/** 多 Agent Chat 的 Prompt */ /** 多 Agent Chat 的 Prompt */
private String chatPlannerPrompt; private String chatPlannerPrompt;
private String chatExecutorPrompt; private String chatExecutorPrompt;
@@ -290,6 +298,7 @@ public class ChatService {
diagnosisSessionRepository.save(session); diagnosisSessionRepository.save(session);
throw e; throw e;
} finally { } finally {
retrievedDocTracker.clearSession(sessionId);
SessionContextHolder.clear(); SessionContextHolder.clear();
} }
} }
@@ -383,6 +392,7 @@ public class ChatService {
logger.error("多 Agent 执行失败", e); logger.error("多 Agent 执行失败", e);
return new ChatResult("执行失败: " + e.getMessage(), sessionId); return new ChatResult("执行失败: " + e.getMessage(), sessionId);
} finally { } finally {
retrievedDocTracker.clearSession(sessionId);
SessionContextHolder.clear(); SessionContextHolder.clear();
} }
} }
@@ -390,6 +400,13 @@ public class ChatService {
private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks, private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
List<Map<String, String>> history) { List<Map<String, String>> history) {
StringBuilder prompt = new StringBuilder(chatPlannerPrompt); StringBuilder prompt = new StringBuilder(chatPlannerPrompt);
// 注入 knowledge map
String knowledgeMap = knowledgeDomainService.buildKnowledgeMap();
if (!knowledgeMap.isBlank()) {
prompt.append("\n\n## 可用知识库\n\n").append(knowledgeMap);
}
if (!history.isEmpty()) { if (!history.isEmpty()) {
prompt.append("\n\n--- 对话历史 ---\n"); prompt.append("\n\n--- 对话历史 ---\n");
for (Map<String, String> msg : history) { for (Map<String, String> msg : history) {
@@ -402,7 +419,6 @@ public class ChatService {
.description("负责拆解问题、规划步骤") .description("负责拆解问题、规划步骤")
.model(chatModel) .model(chatModel)
.systemPrompt(prompt.toString()) .systemPrompt(prompt.toString())
// Planner 不注入工具,只能规划不能执行
.hooks(new AgentLoggingHook(agentStepRepository, "planner")) .hooks(new AgentLoggingHook(agentStepRepository, "planner"))
.outputKey("planner_plan") .outputKey("planner_plan")
.build(); .build();
@@ -0,0 +1,135 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.KnowledgeEntry;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* 文档字段补全服务
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
*/
@Slf4j
@Service
public class DocumentFieldEnricher {
@Autowired
private ChatModel chatModel;
@Autowired
private ObjectMapper objectMapper;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
private String promptTemplate;
@PostConstruct
public void init() {
try {
promptTemplate = new String(
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
log.info("DocumentFieldEnricher prompt 加载成功");
} catch (IOException e) {
log.error("加载 doc-field-enricher-prompt.md 失败", e);
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
}
}
public void enrich(Frontmatter frontmatter, String bodyText) {
enrich(frontmatter, bodyText, null);
}
/**
* 为 Frontmatter 补全 covers 和 whenToRetrieve
* 若已有值则跳过;LLM 失败时降级,不阻断主流程
*
* @param frontmatter 待补全的 frontmatter
* @param bodyText 文档正文
* @param category 文档所属域(用于查找同域其他文档)
*/
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
if (frontmatter == null) return;
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
if (!needsCovers && !needsWhen) {
log.debug("covers 和 whenToRetrieve 已存在,跳过 LLM 生成");
return;
}
try {
String snippet = bodyText != null && bodyText.length() > 1000
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
String promptText = String.format(promptTemplate,
frontmatter.getTitle(),
frontmatter.getSummary(),
sameDomainDocs,
snippet);
String response = chatModel.call(new Prompt(promptText))
.getResult().getOutput().getText();
// 提取 JSON 部分(防止模型输出多余文本)
String json = extractJson(response);
JsonNode node = objectMapper.readTree(json);
if (needsCovers && node.has("covers")) {
List<String> covers = new ArrayList<>();
node.get("covers").forEach(n -> covers.add(n.asText()));
frontmatter.setCovers(covers);
log.debug("LLM 生成 covers: {}", covers);
}
if (needsWhen && node.has("whenToRetrieve")) {
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
}
} catch (Exception e) {
log.warn("LLM 生成文档字段失败,降级处理: title={}", frontmatter.getTitle(), e);
if (needsCovers) frontmatter.setCovers(List.of());
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
}
}
private String extractJson(String text) {
if (text == null) return "{}";
int start = text.indexOf('{');
int end = text.lastIndexOf('}');
if (start == -1 || end == -1 || end <= start) return "{}";
return text.substring(start, end + 1);
}
/**
* 构建同域其他文档标题列表(供 LLM 做排除判断)
*/
private String buildSameDomainDocs(String currentTitle, String category) {
if (category == null || category.isBlank()) return "(无同域文档信息)";
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
.filter(e -> category.equals(e.getCategory()))
.map(KnowledgeEntry::getTitle)
.filter(t -> t != null && !t.equals(currentTitle))
.collect(Collectors.toList());
if (otherTitles.isEmpty()) return "(无同域其他文档)";
return String.join("、", otherTitles);
}
}
@@ -58,6 +58,12 @@ public class DocumentManagementService {
@Autowired @Autowired
private KnowledgeIndexService knowledgeIndexService; private KnowledgeIndexService knowledgeIndexService;
@Autowired
private DocumentFieldEnricher documentFieldEnricher;
@Autowired
private KnowledgeDomainService knowledgeDomainService;
@Autowired @Autowired
private ObjectMapper objectMapper; private ObjectMapper objectMapper;
@@ -123,6 +129,8 @@ public class DocumentManagementService {
if (frontmatterParser.hasFrontmatter(text)) { if (frontmatterParser.hasFrontmatter(text)) {
frontmatter = frontmatterParser.parse(text); frontmatter = frontmatterParser.parse(text);
if (frontmatter != null) { if (frontmatter != null) {
// LLM 补全 covers / whenToRetrieve(已有值则跳过)
documentFieldEnricher.enrich(frontmatter, text, category);
log.info("解析到frontmatter: title={}, keywords={}, time={}ms", log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart); frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
} else { } else {
@@ -196,12 +204,17 @@ public class DocumentManagementService {
.summary(frontmatter.getSummary()) .summary(frontmatter.getSummary())
.category(category) .category(category)
.sections(frontmatter.getSections()) .sections(frontmatter.getSections())
.covers(frontmatter.getCovers())
.whenToRetrieve(frontmatter.getWhenToRetrieve())
.build(); .build();
knowledgeIndexService.addToIndex(entry); knowledgeIndexService.addToIndex(entry);
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle()); log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
} }
// 触发域级聚合重算
knowledgeDomainService.onDocumentChange(category);
long totalTime = System.currentTimeMillis() - startTime; long totalTime = System.currentTimeMillis() - startTime;
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms", log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
docId, fileName, frontmatter != null, totalTime); docId, fileName, frontmatter != null, totalTime);
@@ -367,6 +380,31 @@ public class DocumentManagementService {
// 删除元数据 // 删除元数据
apiDocumentRepository.delete(doc); apiDocumentRepository.delete(doc);
log.info("文档已删除,docId: {}", docId); log.info("文档已删除,docId: {}", docId);
// 触发域级聚合重算
String category = doc.getFilePath() != null
? resolveCategory(doc.getFilePath()) : null;
if (category != null) {
knowledgeDomainService.onDocumentChange(category);
}
}
/**
* 转换为响应 DTO
*/
/**
* 从 filePath 解析 category(取 knowledge_base/{category}/... 中的 category 段)
*/
private String resolveCategory(String filePath) {
try {
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
// filePath 形如 knowledge_base/payment/xxx.md,取倒数第二段
int nameCount = p.getNameCount();
if (nameCount >= 2) {
return p.getName(nameCount - 2).toString();
}
} catch (Exception ignored) {}
return null;
} }
/** /**
@@ -65,6 +65,8 @@ public class FrontmatterParser {
.sections((Map<String, String>) map.get("sections")) .sections((Map<String, String>) map.get("sections"))
.version((String) map.get("version")) .version((String) map.get("version"))
.author((String) map.get("author")) .author((String) map.get("author"))
.covers((java.util.List<String>) map.get("covers"))
.whenToRetrieve((String) map.get("when_to_retrieve"))
.build(); .build();
// 4. 验证必填字段 // 4. 验证必填字段
@@ -0,0 +1,188 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.KnowledgeDomain;
import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.repository.KnowledgeDomainRepository;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* 知识域服务
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
*/
@Slf4j
@Service
public class KnowledgeDomainService {
@Autowired
private KnowledgeDomainRepository knowledgeDomainRepository;
@Autowired
private KnowledgeIndexService knowledgeIndexService;
@Autowired
private ChatModel chatModel;
@Autowired
private ObjectMapper objectMapper;
private String domainPromptTemplate;
@PostConstruct
public void init() {
try {
domainPromptTemplate = new String(
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
StandardCharsets.UTF_8);
log.info("KnowledgeDomainService prompt 加载成功");
} catch (IOException e) {
log.error("加载 domain-summary-prompt.md 失败", e);
throw new RuntimeException("Failed to load domain-summary prompt", e);
}
}
/**
* 文档变更后重算指定域的 when_to_retrieve
*/
public void onDocumentChange(String category) {
if (category == null || category.isBlank()) return;
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
.filter(e -> category.equals(e.getCategory()))
.collect(Collectors.toList());
buildDomainSummary(category, entries);
}
/**
* 聚合同域文档,调用 LLM 生成域级摘要,写入 DB
*/
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
if (entries.isEmpty()) {
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
d.setDocumentCount(0);
knowledgeDomainRepository.save(d);
});
return;
}
// 构建文档列表描述
StringBuilder docList = new StringBuilder();
for (KnowledgeEntry entry : entries) {
docList.append("- 文档:").append(entry.getTitle()).append("\n");
if (entry.getWhenToRetrieve() != null) {
docList.append(" 适用场景:").append(entry.getWhenToRetrieve()).append("\n");
}
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
docList.append(" 覆盖:").append(String.join("、", entry.getCovers())).append("\n");
}
}
String description = entries.stream()
.map(KnowledgeEntry::getSummary)
.filter(s -> s != null && !s.isBlank())
.findFirst().orElse(category);
String whenToRetrieve = null;
try {
String otherDomainsInfo = buildOtherDomainsInfo(category);
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
whenToRetrieve = chatModel.call(new Prompt(promptText))
.getResult().getOutput().getText();
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
} catch (Exception e) {
log.warn("LLM 生成域级 when_to_retrieve 失败,保留旧值: domain={}", category, e);
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
}
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
.orElse(KnowledgeDomain.builder().domainId(category).build());
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
domain.setWhenToRetrieve(whenToRetrieve);
domain.setDocumentCount(entries.size());
knowledgeDomainRepository.save(domain);
}
/**
* 构建注入 Planner 的 knowledge map YAML 文本
*/
public String buildKnowledgeMap() {
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
if (domains.isEmpty()) return "";
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
.filter(e -> e.getCategory() != null)
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
for (KnowledgeDomain domain : domains) {
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
if (domain.getDescription() != null) {
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
}
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
yaml.append(" when_to_retrieve: \"")
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
}
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
if (!domainEntries.isEmpty()) {
yaml.append(" documents:\n");
for (KnowledgeEntry entry : domainEntries) {
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
}
}
}
}
return yaml.toString();
}
/**
* 构建其他域的摘要信息(用于 LLM 域级 prompt 的边界判断)
* 优先使用其他域的 when_to_retrieve(边界信号),而非 description
*/
private String buildOtherDomainsInfo(String currentCategory) {
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
StringBuilder sb = new StringBuilder();
for (KnowledgeDomain d : allDomains) {
if (d.getDomainId().equals(currentCategory)) continue;
sb.append("- ").append(d.getDomainId());
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
sb.append(":").append(d.getWhenToRetrieve());
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
sb.append("(").append(d.getDescription()).append(")");
}
sb.append("\n");
}
// 如果 DB 里还没有其他域的记录(首次启动),从 L0 索引补充
if (sb.isEmpty()) {
knowledgeIndexService.getAllEntries().stream()
.map(KnowledgeEntry::getCategory)
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
.distinct()
.forEach(c -> sb.append("- ").append(c).append("\n"));
}
return sb.isEmpty() ? "(无其他域信息)" : sb.toString();
}
}
@@ -1,12 +1,15 @@
package com.superbiz.agent.service; package com.superbiz.agent.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ApiDocument; import com.superbiz.agent.domain.entity.ApiDocument;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.dto.Frontmatter; import com.superbiz.agent.dto.Frontmatter;
import com.superbiz.agent.dto.KnowledgeEntry; import com.superbiz.agent.dto.KnowledgeEntry;
import com.superbiz.agent.repository.ApiDocumentRepository;
import com.superbiz.agent.repository.KnowledgeDomainRepository;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
@@ -14,12 +17,9 @@ import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* 知识库索引服务 * 知识库索引服务
@@ -35,26 +35,28 @@ public class KnowledgeIndexService {
@Autowired @Autowired
private ApiDocumentRepository apiDocumentRepository; private ApiDocumentRepository apiDocumentRepository;
/** @Autowired
* 内存索引(线程安全) private ObjectMapper objectMapper;
*/
@Autowired
private KnowledgeDomainRepository knowledgeDomainRepository;
@Lazy
@Autowired
private KnowledgeDomainService knowledgeDomainService;
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>(); private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
/**
* 启动时从数据库加载索引
*/
@PostConstruct @PostConstruct
public void loadIndex() { public void loadIndex() {
log.info("开始从数据库加载知识库索引"); log.info("开始从数据库加载知识库索引");
try { try {
// 从数据库读取所有已索引的文档
List<ApiDocument> documents = apiDocumentRepository.findAll(); List<ApiDocument> documents = apiDocumentRepository.findAll();
int loaded = 0; int loaded = 0;
for (ApiDocument doc : documents) { for (ApiDocument doc : documents) {
try { try {
// 从 metadata JSON 中提取信息
KnowledgeEntry entry = parseDocumentToEntry(doc); KnowledgeEntry entry = parseDocumentToEntry(doc);
if (entry != null) { if (entry != null) {
knowledgeIndex.add(entry); knowledgeIndex.add(entry);
@@ -67,34 +69,39 @@ public class KnowledgeIndexService {
log.info("知识库索引加载完成,共 {} 个文档", loaded); log.info("知识库索引加载完成,共 {} 个文档", loaded);
// 检查各域是否有 knowledge_domain 记录,无则触发生成
knowledgeIndex.stream()
.map(KnowledgeEntry::getCategory)
.filter(c -> c != null && !c.isBlank())
.distinct()
.forEach(category -> {
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
log.info("域 {} 无 knowledge_domain 记录,触发生成", category);
knowledgeDomainService.onDocumentChange(category);
}
});
} catch (Exception e) { } catch (Exception e) {
log.error("知识库索引加载失败", e); log.error("知识库索引加载失败", e);
} }
} }
/**
* 将 ApiDocument 转换为 KnowledgeEntry
*/
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) { private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) { if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
return null; return null;
} }
try { try {
// 简单的 JSON 解析 Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
String metadata = doc.getMetadata();
String title = extractJsonValue(metadata, "title");
String summary = extractJsonValue(metadata, "summary");
String category = extractJsonValue(metadata, "category");
List<String> keywords = extractJsonArray(metadata, "keywords");
return KnowledgeEntry.builder() return KnowledgeEntry.builder()
.filePath(doc.getFilePath()) .filePath(doc.getFilePath())
.title(title != null ? title : doc.getApiName()) .title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
.keywords(keywords) .keywords(frontmatter.getKeywords())
.summary(summary) .summary(frontmatter.getSummary())
.category(category) .category(frontmatter.getCategory())
.covers(frontmatter.getCovers())
.whenToRetrieve(frontmatter.getWhenToRetrieve())
.build(); .build();
} catch (Exception e) { } catch (Exception e) {
@@ -103,54 +110,6 @@ public class KnowledgeIndexService {
} }
} }
/**
* 从 JSON 字符串中提取值
*/
private String extractJsonValue(String json, String key) {
String pattern = "\"" + key + "\":\"";
int startIndex = json.indexOf(pattern);
if (startIndex == -1) {
return null;
}
startIndex += pattern.length();
int endIndex = json.indexOf("\"", startIndex);
if (endIndex == -1) {
return null;
}
return json.substring(startIndex, endIndex);
}
/**
* 从 JSON 字符串中提取数组
*/
private List<String> extractJsonArray(String json, String key) {
String pattern = "\"" + key + "\":[";
int startIndex = json.indexOf(pattern);
if (startIndex == -1) {
return Collections.emptyList();
}
startIndex += pattern.length();
int endIndex = json.indexOf("]", startIndex);
if (endIndex == -1) {
return Collections.emptyList();
}
String arrayContent = json.substring(startIndex, endIndex);
return Arrays.stream(arrayContent.split(","))
.map(s -> s.trim().replaceAll("^\"|\"$", ""))
.filter(s -> !s.isEmpty())
.collect(Collectors.toList());
}
/**
* L0 精确匹配
*
* @param query 查询关键词
* @return 匹配的文档列表
*/
public List<KnowledgeEntry> exactMatch(String query) { public List<KnowledgeEntry> exactMatch(String query) {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
@@ -172,13 +131,6 @@ public class KnowledgeIndexService {
return results; return results;
} }
/**
* 关键词匹配逻辑(不区分大小写)
*
* @param entry 索引条目
* @param query 查询关键词(小写)
* @return true 如果匹配
*/
private boolean matchesKeywords(KnowledgeEntry entry, String query) { private boolean matchesKeywords(KnowledgeEntry entry, String query) {
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) { if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
return false; return false;
@@ -186,7 +138,6 @@ public class KnowledgeIndexService {
for (String keyword : entry.getKeywords()) { for (String keyword : entry.getKeywords()) {
String keywordLower = keyword.toLowerCase(); String keywordLower = keyword.toLowerCase();
// query 包含 keyword 或 keyword 包含 query
if (query.contains(keywordLower) || keywordLower.contains(query)) { if (query.contains(keywordLower) || keywordLower.contains(query)) {
return true; return true;
} }
@@ -195,16 +146,8 @@ public class KnowledgeIndexService {
return false; return false;
} }
/**
* 读取文档内容
*
* @param filePath 文件相对路径(如 api/payment-errors.md)
* @param maxChars 最大字符数
* @return 文档内容(前 maxChars 字符),失败返回 null
*/
public String readDocument(String filePath, int maxChars) { public String readDocument(String filePath, int maxChars) {
try { try {
// 拼接完整路径:knowledge_base + 相对路径
Path fullPath = Paths.get(knowledgeBasePath, filePath); Path fullPath = Paths.get(knowledgeBasePath, filePath);
String content = Files.readString(fullPath); String content = Files.readString(fullPath);
@@ -220,32 +163,24 @@ public class KnowledgeIndexService {
} }
} }
/**
* 添加文档到索引(上传时调用)
*
* @param entry 知识库条目
*/
public void addToIndex(KnowledgeEntry entry) { public void addToIndex(KnowledgeEntry entry) {
knowledgeIndex.add(entry); knowledgeIndex.add(entry);
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle()); log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
} }
/**
* 从索引中移除文档(删除时调用)
*
* @param filePath 文件路径
*/
public void removeFromIndex(String filePath) { public void removeFromIndex(String filePath) {
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath)); knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
log.debug("文档已从 L0 索引移除: {}", filePath); log.debug("文档已从 L0 索引移除: {}", filePath);
} }
/**
* 获取索引大小
*
* @return 索引中的文档数量
*/
public int getIndexSize() { public int getIndexSize() {
return knowledgeIndex.size(); return knowledgeIndex.size();
} }
/**
* 获取所有索引条目(供域聚合使用)
*/
public List<KnowledgeEntry> getAllEntries() {
return List.copyOf(knowledgeIndex);
}
} }
@@ -31,6 +31,9 @@ public class LookupKnowledgeTool {
@Autowired @Autowired
private ToolInvocationRepository toolInvocationRepository; private ToolInvocationRepository toolInvocationRepository;
@Autowired
private RetrievedDocTracker retrievedDocTracker;
/** /**
* 查询知识库文档 * 查询知识库文档
* *
@@ -98,6 +101,23 @@ public class LookupKnowledgeTool {
// Step 4: 组装结果 // Step 4: 组装结果
LookupResult result = buildResult(l0Matches, l1Results, highConfidence); LookupResult result = buildResult(l0Matches, l1Results, highConfidence);
// Step 5: session 级去重过滤
String sessionId = SessionContextHolder.getSessionId();
if (sessionId != null && result.isFound()) {
String docKey = extractDocKey(result);
if (docKey != null && retrievedDocTracker.isAlreadyRetrieved(sessionId, docKey)) {
log.info("[去重] 文档已在本会话中检索过,跳过: {}", docKey);
saveToolInvocation(query, l0Matches, l1Results, highConfidence, startTime, result);
return LookupResult.builder()
.found(false)
.message("文档已在本会话中检索过,无需重复召回: " + docKey)
.build();
}
if (docKey != null) {
retrievedDocTracker.markRetrieved(sessionId, docKey);
}
}
// 记录结构化结果摘要(替代原始 MD 内容预览) // 记录结构化结果摘要(替代原始 MD 内容预览)
long totalTime = System.currentTimeMillis() - startTime; long totalTime = System.currentTimeMillis() - startTime;
log.info("----------------------------------------"); log.info("----------------------------------------");
@@ -430,4 +450,14 @@ public class LookupKnowledgeTool {
return "(无有效内容)"; return "(无有效内容)";
} }
private String extractDocKey(LookupResult result) {
if (result.getPrimary() != null && result.getPrimary().getSource() != null) {
return result.getPrimary().getSource();
}
if (result.getSupplement() != null && result.getSupplement().getSource() != null) {
return result.getSupplement().getSource();
}
return null;
}
} }
@@ -0,0 +1,36 @@
package com.superbiz.agent.tool;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* session 级已召回文档追踪器
* 防止同一 session 内重复召回相同文档
*/
@Component
public class RetrievedDocTracker {
// key: sessionId, value: 已召回文档的 filePath 集合
private final ConcurrentHashMap<String, Set<String>> retrieved = new ConcurrentHashMap<>();
public boolean isAlreadyRetrieved(String sessionId, String filePath) {
if (sessionId == null || filePath == null) return false;
Set<String> docs = retrieved.get(sessionId);
return docs != null && docs.contains(filePath);
}
public void markRetrieved(String sessionId, String filePath) {
if (sessionId == null || filePath == null) return;
retrieved.computeIfAbsent(sessionId,
k -> Collections.newSetFromMap(new ConcurrentHashMap<>()))
.add(filePath);
}
public void clearSession(String sessionId) {
if (sessionId == null) return;
retrieved.remove(sessionId);
}
}
@@ -0,0 +1,9 @@
CREATE TABLE knowledge_domain (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id VARCHAR(64) NOT NULL UNIQUE COMMENT 'category 值,如 payment/infrastructure',
description VARCHAR(256) COMMENT '域描述,聚合自文档 summary',
when_to_retrieve TEXT COMMENT '域级检索时机,LLM 聚合生成',
document_count INT NOT NULL DEFAULT 0 COMMENT '该域当前文档数',
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='知识域元数据,存储域级检索策略';
@@ -18,3 +18,9 @@
- 每个步骤应该是一个可以独立执行的任务 - 每个步骤应该是一个可以独立执行的任务
- 步骤要具体可操作,不要模糊 - 步骤要具体可操作,不要模糊
- 如果问题需要查知识库,明确在步骤中说明要查什么 - 如果问题需要查知识库,明确在步骤中说明要查什么
## 知识库检索规则
- 制定步骤前,先查看下方 `available_knowledge_domains`(如果存在)
- 根据每个域的 `when_to_retrieve` 判断是否需要检索该域
- 每个域最多安排一次检索步骤;已覆盖的域不要重复安排
- 如果用户问题与某个域无关,不要安排对该域的检索
@@ -0,0 +1,31 @@
你是知识库文档标注助手。为文档生成两个检索辅助字段。
## 生成规则
1. covers:该文档覆盖的业务场景(3-5 个简洁中文短语)
2. whenToRetrieve:用一句话描述何时检索此文档,不超过 40 字
- 必须包含正向场景(什么问题查本文档)
- 必须包含反向排除(什么问题容易误判但不应查本文档)
- 格式:"正向场景;不包含反向排除词"
## Few-shot 示例
文档标题:支付失败排查手册
同域其他文档:退款处理指南
→ {"covers": ["支付超时", "扣款无回调", "支付网关报错"], "whenToRetrieve": "支付失败/超时/无回调时检索;不含退款对账问题"}
文档标题:退款处理指南
同域其他文档:支付失败排查手册
→ {"covers": ["退款未到账", "退款状态异常", "退款被拒"], "whenToRetrieve": "退款异常/未到账时检索;不含支付失败问题"}
文档标题:MySQL 连接池配置
同域其他文档:Redis 缓存配置指南、Flyway 数据库迁移
→ {"covers": ["连接池耗尽", "数据库OOM", "慢查询"], "whenToRetrieve": "连接池/数据库性能问题时检索;不含Redis缓存或迁移问题"}
## 待分析文档
文档标题:%s
文档摘要:%s
同域其他文档:%s
文档内容节选:
%s
请严格返回 JSON 格式,不要有其他内容:
{"covers": ["场景1", "场景2", ...], "whenToRetrieve": "描述"}
@@ -0,0 +1,21 @@
你是知识库域级路由设计助手。根据域内文档和系统其他域信息,生成一条域级检索指引。
## 当前域:%s
## 域内文档
%s
## 系统其他域(用于判断边界)
%s
## 生成要求
1. 用一句话描述"什么场景下 Planner 应该选择检索这个域"
2. 必须覆盖该域所有文档的适用场景,做合理抽象(不要简单拼接)
3. 必须包含明确的边界判断:
- 正向:什么问题属于这个域
- 反向:什么问题容易被误判为属于这个域,但实际应该检索其他域
4. 反向边界必须引用其他域的 domain_id,格式:"X类问题查{domain_id}"
5. 不超过 60 字
6. 语言要让 Planner 能做"检索/不检索"的二分判断
只返回纯文本指引,不要 JSON 或 markdown。