feat(knowledge): 会话级去重 + 知识域地图注入 Planner 解决 ISS-001 重复检索
- RetrievedDocTracker: sessionId → Set<filePath> 会话级去重，LookupKnowledgeTool Step 5 过滤已检索文档 - KnowledgeDomainService: 域级聚合，LLM 生成 when_to_retrieve，构建 knowledge map YAML - DocumentFieldEnricher: 上传时 LLM 补全 covers + whenToRetrieve（含同域文档排除上下文） - KnowledgeDomain entity + V009 迁移: 域级元数据持久化，避免重启重复 LLM 调用 - ChatService: 注入 knowledge map 到 Planner prompt，会话结束时清理去重状态 - KnowledgeIndexService: 手写 JSON 解析替换为 Jackson ObjectMapper，启动时补建缺失域记录 - chat-planner-prompt: 新增知识库检索规则（按域 when_to_retrieve 判断，每域最多一次检索） - doc-field-enricher-prompt / domain-summary-prompt: 外部化 LLM 提示词
This commit is contained in:
zhuyongxin
@@ -0,0 +1,51 @@
|
||||
package com.superbiz.agent.domain.entity;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
@Entity
|
||||
@Table(name = "knowledge_domain")
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class KnowledgeDomain {
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.IDENTITY)
|
||||
private Long id;
|
||||
|
||||
@Column(name = "domain_id", unique = true, nullable = false, length = 64)
|
||||
private String domainId;
|
||||
|
||||
@Column(name = "description", length = 256)
|
||||
private String description;
|
||||
|
||||
@Column(name = "when_to_retrieve", columnDefinition = "TEXT")
|
||||
private String whenToRetrieve;
|
||||
|
||||
@Column(name = "document_count", nullable = false)
|
||||
private int documentCount;
|
||||
|
||||
@Column(name = "created_at", nullable = false, updatable = false)
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
@Column(name = "updated_at", nullable = false)
|
||||
private LocalDateTime updatedAt;
|
||||
|
||||
@PrePersist
|
||||
protected void onCreate() {
|
||||
createdAt = LocalDateTime.now();
|
||||
updatedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
@PreUpdate
|
||||
protected void onUpdate() {
|
||||
updatedAt = LocalDateTime.now();
|
||||
}
|
||||
}
|
||||
@@ -59,4 +59,14 @@ public class Frontmatter {
|
||||
* 最后更新日期（预留字段）
|
||||
*/
|
||||
private LocalDate lastUpdated;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用（LLM 上传时自动生成）
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机（LLM 上传时自动生成）
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
|
||||
@@ -43,4 +43,14 @@ public class KnowledgeEntry {
|
||||
* 章节锚点（预留字段，MVP 不使用）
|
||||
*/
|
||||
private Map<String, String> sections;
|
||||
|
||||
/**
|
||||
* 业务场景标签，供 Planner 决策用
|
||||
*/
|
||||
private List<String> covers;
|
||||
|
||||
/**
|
||||
* 文档级检索时机
|
||||
*/
|
||||
private String whenToRetrieve;
|
||||
}
|
||||
|
||||
@@ -26,4 +26,9 @@ public class LookupResult {
|
||||
* 补充结果（L1 语义检索）
|
||||
*/
|
||||
private SupplementResult supplement;
|
||||
|
||||
/**
|
||||
* 系统消息（如去重提示）
|
||||
*/
|
||||
private String message;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.superbiz.agent.repository;
|
||||
|
||||
import com.superbiz.agent.domain.entity.KnowledgeDomain;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
@Repository
|
||||
public interface KnowledgeDomainRepository extends JpaRepository<KnowledgeDomain, Long> {
|
||||
|
||||
Optional<KnowledgeDomain> findByDomainId(String domainId);
|
||||
}
|
||||
@@ -16,8 +16,10 @@ import com.superbiz.agent.hook.TokenUsageHolder;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisSessionRepository;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.tool.RetrievedDocTracker;
|
||||
import com.superbiz.agent.util.QuestionComplexity;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import com.superbiz.agent.service.KnowledgeDomainService;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import org.slf4j.Logger;
|
||||
@@ -79,6 +81,12 @@ public class ChatService {
|
||||
@Autowired
|
||||
private EvaluationService evaluationService;
|
||||
|
||||
@Autowired
|
||||
private RetrievedDocTracker retrievedDocTracker;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
/** 多 Agent Chat 的 Prompt */
|
||||
private String chatPlannerPrompt;
|
||||
private String chatExecutorPrompt;
|
||||
@@ -290,6 +298,7 @@ public class ChatService {
|
||||
diagnosisSessionRepository.save(session);
|
||||
throw e;
|
||||
} finally {
|
||||
retrievedDocTracker.clearSession(sessionId);
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
}
|
||||
@@ -383,6 +392,7 @@ public class ChatService {
|
||||
logger.error("多 Agent 执行失败", e);
|
||||
return new ChatResult("执行失败: " + e.getMessage(), sessionId);
|
||||
} finally {
|
||||
retrievedDocTracker.clearSession(sessionId);
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
}
|
||||
@@ -390,6 +400,13 @@ public class ChatService {
|
||||
private ReactAgent buildChatPlannerAgent(ChatModel chatModel, ToolCallback[] toolCallbacks,
|
||||
List<Map<String, String>> history) {
|
||||
StringBuilder prompt = new StringBuilder(chatPlannerPrompt);
|
||||
|
||||
// 注入 knowledge map
|
||||
String knowledgeMap = knowledgeDomainService.buildKnowledgeMap();
|
||||
if (!knowledgeMap.isBlank()) {
|
||||
prompt.append("\n\n## 可用知识库\n\n").append(knowledgeMap);
|
||||
}
|
||||
|
||||
if (!history.isEmpty()) {
|
||||
prompt.append("\n\n--- 对话历史 ---\n");
|
||||
for (Map<String, String> msg : history) {
|
||||
@@ -402,7 +419,6 @@ public class ChatService {
|
||||
.description("负责拆解问题、规划步骤")
|
||||
.model(chatModel)
|
||||
.systemPrompt(prompt.toString())
|
||||
// Planner 不注入工具，只能规划不能执行
|
||||
.hooks(new AgentLoggingHook(agentStepRepository, "planner"))
|
||||
.outputKey("planner_plan")
|
||||
.build();
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 文档字段补全服务
|
||||
* 上传时调用 LLM 生成 covers 和 whenToRetrieve
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class DocumentFieldEnricher {
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
private String promptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
promptTemplate = new String(
|
||||
new ClassPathResource("prompts/doc-field-enricher-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("DocumentFieldEnricher prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 doc-field-enricher-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load doc-field-enricher prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
public void enrich(Frontmatter frontmatter, String bodyText) {
|
||||
enrich(frontmatter, bodyText, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* 为 Frontmatter 补全 covers 和 whenToRetrieve
|
||||
* 若已有值则跳过；LLM 失败时降级，不阻断主流程
|
||||
*
|
||||
* @param frontmatter 待补全的 frontmatter
|
||||
* @param bodyText 文档正文
|
||||
* @param category 文档所属域（用于查找同域其他文档）
|
||||
*/
|
||||
public void enrich(Frontmatter frontmatter, String bodyText, String category) {
|
||||
if (frontmatter == null) return;
|
||||
|
||||
boolean needsCovers = frontmatter.getCovers() == null || frontmatter.getCovers().isEmpty();
|
||||
boolean needsWhen = frontmatter.getWhenToRetrieve() == null || frontmatter.getWhenToRetrieve().isBlank();
|
||||
|
||||
if (!needsCovers && !needsWhen) {
|
||||
log.debug("covers 和 whenToRetrieve 已存在，跳过 LLM 生成");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String snippet = bodyText != null && bodyText.length() > 1000
|
||||
? bodyText.substring(0, 1000) : (bodyText != null ? bodyText : "");
|
||||
|
||||
String sameDomainDocs = buildSameDomainDocs(frontmatter.getTitle(), category);
|
||||
|
||||
String promptText = String.format(promptTemplate,
|
||||
frontmatter.getTitle(),
|
||||
frontmatter.getSummary(),
|
||||
sameDomainDocs,
|
||||
snippet);
|
||||
|
||||
String response = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
|
||||
// 提取 JSON 部分（防止模型输出多余文本）
|
||||
String json = extractJson(response);
|
||||
JsonNode node = objectMapper.readTree(json);
|
||||
|
||||
if (needsCovers && node.has("covers")) {
|
||||
List<String> covers = new ArrayList<>();
|
||||
node.get("covers").forEach(n -> covers.add(n.asText()));
|
||||
frontmatter.setCovers(covers);
|
||||
log.debug("LLM 生成 covers: {}", covers);
|
||||
}
|
||||
|
||||
if (needsWhen && node.has("whenToRetrieve")) {
|
||||
frontmatter.setWhenToRetrieve(node.get("whenToRetrieve").asText());
|
||||
log.debug("LLM 生成 whenToRetrieve: {}", frontmatter.getWhenToRetrieve());
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成文档字段失败，降级处理: title={}", frontmatter.getTitle(), e);
|
||||
if (needsCovers) frontmatter.setCovers(List.of());
|
||||
if (needsWhen) frontmatter.setWhenToRetrieve(frontmatter.getSummary());
|
||||
}
|
||||
}
|
||||
|
||||
private String extractJson(String text) {
|
||||
if (text == null) return "{}";
|
||||
int start = text.indexOf('{');
|
||||
int end = text.lastIndexOf('}');
|
||||
if (start == -1 || end == -1 || end <= start) return "{}";
|
||||
return text.substring(start, end + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建同域其他文档标题列表（供 LLM 做排除判断）
|
||||
*/
|
||||
private String buildSameDomainDocs(String currentTitle, String category) {
|
||||
if (category == null || category.isBlank()) return "（无同域文档信息）";
|
||||
List<String> otherTitles = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.map(KnowledgeEntry::getTitle)
|
||||
.filter(t -> t != null && !t.equals(currentTitle))
|
||||
.collect(Collectors.toList());
|
||||
if (otherTitles.isEmpty()) return "（无同域其他文档）";
|
||||
return String.join("、", otherTitles);
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,12 @@ public class DocumentManagementService {
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private DocumentFieldEnricher documentFieldEnricher;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@@ -123,6 +129,8 @@ public class DocumentManagementService {
|
||||
if (frontmatterParser.hasFrontmatter(text)) {
|
||||
frontmatter = frontmatterParser.parse(text);
|
||||
if (frontmatter != null) {
|
||||
// LLM 补全 covers / whenToRetrieve（已有值则跳过）
|
||||
documentFieldEnricher.enrich(frontmatter, text, category);
|
||||
log.info("解析到frontmatter: title={}, keywords={}, time={}ms",
|
||||
frontmatter.getTitle(), frontmatter.getKeywords(), System.currentTimeMillis() - frontmatterStart);
|
||||
} else {
|
||||
@@ -196,12 +204,17 @@ public class DocumentManagementService {
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(category)
|
||||
.sections(frontmatter.getSections())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
knowledgeIndexService.addToIndex(entry);
|
||||
log.info("文档已加入L0索引: docId={}, title={}", docId, frontmatter.getTitle());
|
||||
}
|
||||
|
||||
// 触发域级聚合重算
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
log.info("文档上传完成: docId={}, fileName={}, hasFrontmatter={}, totalTime={}ms",
|
||||
docId, fileName, frontmatter != null, totalTime);
|
||||
@@ -367,6 +380,31 @@ public class DocumentManagementService {
|
||||
// 删除元数据
|
||||
apiDocumentRepository.delete(doc);
|
||||
log.info("文档已删除，docId: {}", docId);
|
||||
|
||||
// 触发域级聚合重算
|
||||
String category = doc.getFilePath() != null
|
||||
? resolveCategory(doc.getFilePath()) : null;
|
||||
if (category != null) {
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 转换为响应 DTO
|
||||
*/
|
||||
/**
|
||||
* 从 filePath 解析 category（取 knowledge_base/{category}/... 中的 category 段）
|
||||
*/
|
||||
private String resolveCategory(String filePath) {
|
||||
try {
|
||||
java.nio.file.Path p = java.nio.file.Paths.get(filePath);
|
||||
// filePath 形如 knowledge_base/payment/xxx.md，取倒数第二段
|
||||
int nameCount = p.getNameCount();
|
||||
if (nameCount >= 2) {
|
||||
return p.getName(nameCount - 2).toString();
|
||||
}
|
||||
} catch (Exception ignored) {}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -65,6 +65,8 @@ public class FrontmatterParser {
|
||||
.sections((Map<String, String>) map.get("sections"))
|
||||
.version((String) map.get("version"))
|
||||
.author((String) map.get("author"))
|
||||
.covers((java.util.List<String>) map.get("covers"))
|
||||
.whenToRetrieve((String) map.get("when_to_retrieve"))
|
||||
.build();
|
||||
|
||||
// 4. 验证必填字段
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.KnowledgeDomain;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* 知识域服务
|
||||
* 负责域级聚合、LLM 生成域级 when_to_retrieve 以及 knowledge map 构建
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class KnowledgeDomainService {
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeIndexService knowledgeIndexService;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
private String domainPromptTemplate;
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
try {
|
||||
domainPromptTemplate = new String(
|
||||
new ClassPathResource("prompts/domain-summary-prompt.md").getInputStream().readAllBytes(),
|
||||
StandardCharsets.UTF_8);
|
||||
log.info("KnowledgeDomainService prompt 加载成功");
|
||||
} catch (IOException e) {
|
||||
log.error("加载 domain-summary-prompt.md 失败", e);
|
||||
throw new RuntimeException("Failed to load domain-summary prompt", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文档变更后重算指定域的 when_to_retrieve
|
||||
*/
|
||||
public void onDocumentChange(String category) {
|
||||
if (category == null || category.isBlank()) return;
|
||||
|
||||
List<KnowledgeEntry> entries = knowledgeIndexService.getAllEntries().stream()
|
||||
.filter(e -> category.equals(e.getCategory()))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
buildDomainSummary(category, entries);
|
||||
}
|
||||
|
||||
/**
|
||||
* 聚合同域文档，调用 LLM 生成域级摘要，写入 DB
|
||||
*/
|
||||
public void buildDomainSummary(String category, List<KnowledgeEntry> entries) {
|
||||
if (entries.isEmpty()) {
|
||||
knowledgeDomainRepository.findByDomainId(category).ifPresent(d -> {
|
||||
d.setDocumentCount(0);
|
||||
knowledgeDomainRepository.save(d);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 构建文档列表描述
|
||||
StringBuilder docList = new StringBuilder();
|
||||
for (KnowledgeEntry entry : entries) {
|
||||
docList.append("- 文档：").append(entry.getTitle()).append("\n");
|
||||
if (entry.getWhenToRetrieve() != null) {
|
||||
docList.append(" 适用场景：").append(entry.getWhenToRetrieve()).append("\n");
|
||||
}
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
docList.append(" 覆盖：").append(String.join("、", entry.getCovers())).append("\n");
|
||||
}
|
||||
}
|
||||
|
||||
String description = entries.stream()
|
||||
.map(KnowledgeEntry::getSummary)
|
||||
.filter(s -> s != null && !s.isBlank())
|
||||
.findFirst().orElse(category);
|
||||
|
||||
String whenToRetrieve = null;
|
||||
try {
|
||||
String otherDomainsInfo = buildOtherDomainsInfo(category);
|
||||
String promptText = String.format(domainPromptTemplate, category, docList, otherDomainsInfo);
|
||||
whenToRetrieve = chatModel.call(new Prompt(promptText))
|
||||
.getResult().getOutput().getText();
|
||||
log.info("LLM 生成域级 when_to_retrieve: domain={}, result={}", category, whenToRetrieve);
|
||||
} catch (Exception e) {
|
||||
log.warn("LLM 生成域级 when_to_retrieve 失败，保留旧值: domain={}", category, e);
|
||||
Optional<KnowledgeDomain> existing = knowledgeDomainRepository.findByDomainId(category);
|
||||
whenToRetrieve = existing.map(KnowledgeDomain::getWhenToRetrieve).orElse("");
|
||||
}
|
||||
|
||||
KnowledgeDomain domain = knowledgeDomainRepository.findByDomainId(category)
|
||||
.orElse(KnowledgeDomain.builder().domainId(category).build());
|
||||
|
||||
domain.setDescription(description.length() > 255 ? description.substring(0, 255) : description);
|
||||
domain.setWhenToRetrieve(whenToRetrieve);
|
||||
domain.setDocumentCount(entries.size());
|
||||
knowledgeDomainRepository.save(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建注入 Planner 的 knowledge map YAML 文本
|
||||
*/
|
||||
public String buildKnowledgeMap() {
|
||||
List<KnowledgeDomain> domains = knowledgeDomainRepository.findAll();
|
||||
if (domains.isEmpty()) return "";
|
||||
|
||||
List<KnowledgeEntry> allEntries = knowledgeIndexService.getAllEntries();
|
||||
Map<String, List<KnowledgeEntry>> byCategory = allEntries.stream()
|
||||
.filter(e -> e.getCategory() != null)
|
||||
.collect(Collectors.groupingBy(KnowledgeEntry::getCategory));
|
||||
|
||||
StringBuilder yaml = new StringBuilder("available_knowledge_domains:\n");
|
||||
|
||||
for (KnowledgeDomain domain : domains) {
|
||||
yaml.append(" - domain_id: \"").append(domain.getDomainId()).append("\"\n");
|
||||
if (domain.getDescription() != null) {
|
||||
yaml.append(" description: \"").append(domain.getDescription()).append("\"\n");
|
||||
}
|
||||
if (domain.getWhenToRetrieve() != null && !domain.getWhenToRetrieve().isBlank()) {
|
||||
yaml.append(" when_to_retrieve: \"")
|
||||
.append(domain.getWhenToRetrieve().replace("\"", "'")).append("\"\n");
|
||||
}
|
||||
yaml.append(" document_count: ").append(domain.getDocumentCount()).append("\n");
|
||||
|
||||
List<KnowledgeEntry> domainEntries = byCategory.getOrDefault(domain.getDomainId(), List.of());
|
||||
if (!domainEntries.isEmpty()) {
|
||||
yaml.append(" documents:\n");
|
||||
for (KnowledgeEntry entry : domainEntries) {
|
||||
yaml.append(" - title: \"").append(entry.getTitle()).append("\"\n");
|
||||
if (entry.getCovers() != null && !entry.getCovers().isEmpty()) {
|
||||
yaml.append(" covers: ").append(entry.getCovers()).append("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建其他域的摘要信息（用于 LLM 域级 prompt 的边界判断）
|
||||
* 优先使用其他域的 when_to_retrieve（边界信号），而非 description
|
||||
*/
|
||||
private String buildOtherDomainsInfo(String currentCategory) {
|
||||
List<KnowledgeDomain> allDomains = knowledgeDomainRepository.findAll();
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (KnowledgeDomain d : allDomains) {
|
||||
if (d.getDomainId().equals(currentCategory)) continue;
|
||||
sb.append("- ").append(d.getDomainId());
|
||||
if (d.getWhenToRetrieve() != null && !d.getWhenToRetrieve().isBlank()) {
|
||||
sb.append("：").append(d.getWhenToRetrieve());
|
||||
} else if (d.getDescription() != null && !d.getDescription().isBlank()) {
|
||||
sb.append("（").append(d.getDescription()).append("）");
|
||||
}
|
||||
sb.append("\n");
|
||||
}
|
||||
// 如果 DB 里还没有其他域的记录（首次启动），从 L0 索引补充
|
||||
if (sb.isEmpty()) {
|
||||
knowledgeIndexService.getAllEntries().stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank() && !c.equals(currentCategory))
|
||||
.distinct()
|
||||
.forEach(c -> sb.append("- ").append(c).append("\n"));
|
||||
}
|
||||
return sb.isEmpty() ? "（无其他域信息）" : sb.toString();
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ApiDocument;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.dto.Frontmatter;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.repository.ApiDocumentRepository;
|
||||
import com.superbiz.agent.repository.KnowledgeDomainRepository;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
@@ -14,12 +17,9 @@ import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* 知识库索引服务
|
||||
@@ -35,26 +35,28 @@ public class KnowledgeIndexService {
|
||||
@Autowired
|
||||
private ApiDocumentRepository apiDocumentRepository;
|
||||
|
||||
/**
|
||||
* 内存索引（线程安全）
|
||||
*/
|
||||
@Autowired
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@Autowired
|
||||
private KnowledgeDomainRepository knowledgeDomainRepository;
|
||||
|
||||
@Lazy
|
||||
@Autowired
|
||||
private KnowledgeDomainService knowledgeDomainService;
|
||||
|
||||
private final List<KnowledgeEntry> knowledgeIndex = new CopyOnWriteArrayList<>();
|
||||
|
||||
/**
|
||||
* 启动时从数据库加载索引
|
||||
*/
|
||||
@PostConstruct
|
||||
public void loadIndex() {
|
||||
log.info("开始从数据库加载知识库索引");
|
||||
|
||||
try {
|
||||
// 从数据库读取所有已索引的文档
|
||||
List<ApiDocument> documents = apiDocumentRepository.findAll();
|
||||
|
||||
int loaded = 0;
|
||||
for (ApiDocument doc : documents) {
|
||||
try {
|
||||
// 从 metadata JSON 中提取信息
|
||||
KnowledgeEntry entry = parseDocumentToEntry(doc);
|
||||
if (entry != null) {
|
||||
knowledgeIndex.add(entry);
|
||||
@@ -67,34 +69,39 @@ public class KnowledgeIndexService {
|
||||
|
||||
log.info("知识库索引加载完成，共 {} 个文档", loaded);
|
||||
|
||||
// 检查各域是否有 knowledge_domain 记录，无则触发生成
|
||||
knowledgeIndex.stream()
|
||||
.map(KnowledgeEntry::getCategory)
|
||||
.filter(c -> c != null && !c.isBlank())
|
||||
.distinct()
|
||||
.forEach(category -> {
|
||||
if (knowledgeDomainRepository.findByDomainId(category).isEmpty()) {
|
||||
log.info("域 {} 无 knowledge_domain 记录，触发生成", category);
|
||||
knowledgeDomainService.onDocumentChange(category);
|
||||
}
|
||||
});
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("知识库索引加载失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 ApiDocument 转换为 KnowledgeEntry
|
||||
*/
|
||||
private KnowledgeEntry parseDocumentToEntry(ApiDocument doc) {
|
||||
if (doc.getMetadata() == null || doc.getMetadata().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// 简单的 JSON 解析
|
||||
String metadata = doc.getMetadata();
|
||||
|
||||
String title = extractJsonValue(metadata, "title");
|
||||
String summary = extractJsonValue(metadata, "summary");
|
||||
String category = extractJsonValue(metadata, "category");
|
||||
List<String> keywords = extractJsonArray(metadata, "keywords");
|
||||
Frontmatter frontmatter = objectMapper.readValue(doc.getMetadata(), Frontmatter.class);
|
||||
|
||||
return KnowledgeEntry.builder()
|
||||
.filePath(doc.getFilePath())
|
||||
.title(title != null ? title : doc.getApiName())
|
||||
.keywords(keywords)
|
||||
.summary(summary)
|
||||
.category(category)
|
||||
.title(frontmatter.getTitle() != null ? frontmatter.getTitle() : doc.getApiName())
|
||||
.keywords(frontmatter.getKeywords())
|
||||
.summary(frontmatter.getSummary())
|
||||
.category(frontmatter.getCategory())
|
||||
.covers(frontmatter.getCovers())
|
||||
.whenToRetrieve(frontmatter.getWhenToRetrieve())
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
@@ -103,54 +110,6 @@ public class KnowledgeIndexService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 JSON 字符串中提取值
|
||||
*/
|
||||
private String extractJsonValue(String json, String key) {
|
||||
String pattern = "\"" + key + "\":\"";
|
||||
int startIndex = json.indexOf(pattern);
|
||||
if (startIndex == -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
startIndex += pattern.length();
|
||||
int endIndex = json.indexOf("\"", startIndex);
|
||||
if (endIndex == -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return json.substring(startIndex, endIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 JSON 字符串中提取数组
|
||||
*/
|
||||
private List<String> extractJsonArray(String json, String key) {
|
||||
String pattern = "\"" + key + "\":[";
|
||||
int startIndex = json.indexOf(pattern);
|
||||
if (startIndex == -1) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
startIndex += pattern.length();
|
||||
int endIndex = json.indexOf("]", startIndex);
|
||||
if (endIndex == -1) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
String arrayContent = json.substring(startIndex, endIndex);
|
||||
return Arrays.stream(arrayContent.split(","))
|
||||
.map(s -> s.trim().replaceAll("^\"|\"$", ""))
|
||||
.filter(s -> !s.isEmpty())
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* L0 精确匹配
|
||||
*
|
||||
* @param query 查询关键词
|
||||
* @return 匹配的文档列表
|
||||
*/
|
||||
public List<KnowledgeEntry> exactMatch(String query) {
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
@@ -172,13 +131,6 @@ public class KnowledgeIndexService {
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关键词匹配逻辑（不区分大小写）
|
||||
*
|
||||
* @param entry 索引条目
|
||||
* @param query 查询关键词（小写）
|
||||
* @return true 如果匹配
|
||||
*/
|
||||
private boolean matchesKeywords(KnowledgeEntry entry, String query) {
|
||||
if (entry.getKeywords() == null || entry.getKeywords().isEmpty()) {
|
||||
return false;
|
||||
@@ -186,7 +138,6 @@ public class KnowledgeIndexService {
|
||||
|
||||
for (String keyword : entry.getKeywords()) {
|
||||
String keywordLower = keyword.toLowerCase();
|
||||
// query 包含 keyword 或 keyword 包含 query
|
||||
if (query.contains(keywordLower) || keywordLower.contains(query)) {
|
||||
return true;
|
||||
}
|
||||
@@ -195,16 +146,8 @@ public class KnowledgeIndexService {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 读取文档内容
|
||||
*
|
||||
* @param filePath 文件相对路径（如 api/payment-errors.md）
|
||||
* @param maxChars 最大字符数
|
||||
* @return 文档内容（前 maxChars 字符），失败返回 null
|
||||
*/
|
||||
public String readDocument(String filePath, int maxChars) {
|
||||
try {
|
||||
// 拼接完整路径：knowledge_base + 相对路径
|
||||
Path fullPath = Paths.get(knowledgeBasePath, filePath);
|
||||
String content = Files.readString(fullPath);
|
||||
|
||||
@@ -220,32 +163,24 @@ public class KnowledgeIndexService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加文档到索引（上传时调用）
|
||||
*
|
||||
* @param entry 知识库条目
|
||||
*/
|
||||
public void addToIndex(KnowledgeEntry entry) {
|
||||
knowledgeIndex.add(entry);
|
||||
log.debug("文档已添加到 L0 索引: title={}", entry.getTitle());
|
||||
}
|
||||
|
||||
/**
|
||||
* 从索引中移除文档（删除时调用）
|
||||
*
|
||||
* @param filePath 文件路径
|
||||
*/
|
||||
public void removeFromIndex(String filePath) {
|
||||
knowledgeIndex.removeIf(e -> e.getFilePath().equals(filePath));
|
||||
log.debug("文档已从 L0 索引移除: {}", filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取索引大小
|
||||
*
|
||||
* @return 索引中的文档数量
|
||||
*/
|
||||
public int getIndexSize() {
|
||||
return knowledgeIndex.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有索引条目（供域聚合使用）
|
||||
*/
|
||||
public List<KnowledgeEntry> getAllEntries() {
|
||||
return List.copyOf(knowledgeIndex);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,9 @@ public class LookupKnowledgeTool {
|
||||
@Autowired
|
||||
private ToolInvocationRepository toolInvocationRepository;
|
||||
|
||||
@Autowired
|
||||
private RetrievedDocTracker retrievedDocTracker;
|
||||
|
||||
/**
|
||||
* 查询知识库文档
|
||||
*
|
||||
@@ -98,6 +101,23 @@ public class LookupKnowledgeTool {
|
||||
// Step 4: 组装结果
|
||||
LookupResult result = buildResult(l0Matches, l1Results, highConfidence);
|
||||
|
||||
// Step 5: session 级去重过滤
|
||||
String sessionId = SessionContextHolder.getSessionId();
|
||||
if (sessionId != null && result.isFound()) {
|
||||
String docKey = extractDocKey(result);
|
||||
if (docKey != null && retrievedDocTracker.isAlreadyRetrieved(sessionId, docKey)) {
|
||||
log.info("[去重] 文档已在本会话中检索过，跳过: {}", docKey);
|
||||
saveToolInvocation(query, l0Matches, l1Results, highConfidence, startTime, result);
|
||||
return LookupResult.builder()
|
||||
.found(false)
|
||||
.message("文档已在本会话中检索过，无需重复召回: " + docKey)
|
||||
.build();
|
||||
}
|
||||
if (docKey != null) {
|
||||
retrievedDocTracker.markRetrieved(sessionId, docKey);
|
||||
}
|
||||
}
|
||||
|
||||
// 记录结构化结果摘要（替代原始 MD 内容预览）
|
||||
long totalTime = System.currentTimeMillis() - startTime;
|
||||
log.info("----------------------------------------");
|
||||
@@ -430,4 +450,14 @@ public class LookupKnowledgeTool {
|
||||
|
||||
return "(无有效内容)";
|
||||
}
|
||||
|
||||
private String extractDocKey(LookupResult result) {
|
||||
if (result.getPrimary() != null && result.getPrimary().getSource() != null) {
|
||||
return result.getPrimary().getSource();
|
||||
}
|
||||
if (result.getSupplement() != null && result.getSupplement().getSource() != null) {
|
||||
return result.getSupplement().getSource();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.superbiz.agent.tool;
|
||||
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* session 级已召回文档追踪器
|
||||
* 防止同一 session 内重复召回相同文档
|
||||
*/
|
||||
@Component
|
||||
public class RetrievedDocTracker {
|
||||
|
||||
// key: sessionId, value: 已召回文档的 filePath 集合
|
||||
private final ConcurrentHashMap<String, Set<String>> retrieved = new ConcurrentHashMap<>();
|
||||
|
||||
public boolean isAlreadyRetrieved(String sessionId, String filePath) {
|
||||
if (sessionId == null || filePath == null) return false;
|
||||
Set<String> docs = retrieved.get(sessionId);
|
||||
return docs != null && docs.contains(filePath);
|
||||
}
|
||||
|
||||
public void markRetrieved(String sessionId, String filePath) {
|
||||
if (sessionId == null || filePath == null) return;
|
||||
retrieved.computeIfAbsent(sessionId,
|
||||
k -> Collections.newSetFromMap(new ConcurrentHashMap<>()))
|
||||
.add(filePath);
|
||||
}
|
||||
|
||||
public void clearSession(String sessionId) {
|
||||
if (sessionId == null) return;
|
||||
retrieved.remove(sessionId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE knowledge_domain (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
domain_id VARCHAR(64) NOT NULL UNIQUE COMMENT 'category 值，如 payment/infrastructure',
|
||||
description VARCHAR(256) COMMENT '域描述，聚合自文档 summary',
|
||||
when_to_retrieve TEXT COMMENT '域级检索时机，LLM 聚合生成',
|
||||
document_count INT NOT NULL DEFAULT 0 COMMENT '该域当前文档数',
|
||||
created_at DATETIME NOT NULL,
|
||||
updated_at DATETIME NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='知识域元数据，存储域级检索策略';
|
||||
@@ -18,3 +18,9 @@
|
||||
- 每个步骤应该是一个可以独立执行的任务
|
||||
- 步骤要具体可操作，不要模糊
|
||||
- 如果问题需要查知识库，明确在步骤中说明要查什么
|
||||
|
||||
## 知识库检索规则
|
||||
- 制定步骤前，先查看下方 `available_knowledge_domains`（如果存在）
|
||||
- 根据每个域的 `when_to_retrieve` 判断是否需要检索该域
|
||||
- 每个域最多安排一次检索步骤；已覆盖的域不要重复安排
|
||||
- 如果用户问题与某个域无关，不要安排对该域的检索
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
你是知识库文档标注助手。为文档生成两个检索辅助字段。
|
||||
|
||||
## 生成规则
|
||||
1. covers：该文档覆盖的业务场景（3-5 个简洁中文短语）
|
||||
2. whenToRetrieve：用一句话描述何时检索此文档，不超过 40 字
|
||||
- 必须包含正向场景（什么问题查本文档）
|
||||
- 必须包含反向排除（什么问题容易误判但不应查本文档）
|
||||
- 格式："正向场景；不包含反向排除词"
|
||||
|
||||
## Few-shot 示例
|
||||
文档标题：支付失败排查手册
|
||||
同域其他文档：退款处理指南
|
||||
→ {"covers": ["支付超时", "扣款无回调", "支付网关报错"], "whenToRetrieve": "支付失败/超时/无回调时检索；不含退款对账问题"}
|
||||
|
||||
文档标题：退款处理指南
|
||||
同域其他文档：支付失败排查手册
|
||||
→ {"covers": ["退款未到账", "退款状态异常", "退款被拒"], "whenToRetrieve": "退款异常/未到账时检索；不含支付失败问题"}
|
||||
|
||||
文档标题：MySQL 连接池配置
|
||||
同域其他文档：Redis 缓存配置指南、Flyway 数据库迁移
|
||||
→ {"covers": ["连接池耗尽", "数据库OOM", "慢查询"], "whenToRetrieve": "连接池/数据库性能问题时检索；不含Redis缓存或迁移问题"}
|
||||
|
||||
## 待分析文档
|
||||
文档标题：%s
|
||||
文档摘要：%s
|
||||
同域其他文档：%s
|
||||
文档内容节选：
|
||||
%s
|
||||
|
||||
请严格返回 JSON 格式，不要有其他内容：
|
||||
{"covers": ["场景1", "场景2", ...], "whenToRetrieve": "描述"}
|
||||
@@ -0,0 +1,21 @@
|
||||
你是知识库域级路由设计助手。根据域内文档和系统其他域信息，生成一条域级检索指引。
|
||||
|
||||
## 当前域：%s
|
||||
|
||||
## 域内文档
|
||||
%s
|
||||
|
||||
## 系统其他域（用于判断边界）
|
||||
%s
|
||||
|
||||
## 生成要求
|
||||
1. 用一句话描述"什么场景下 Planner 应该选择检索这个域"
|
||||
2. 必须覆盖该域所有文档的适用场景，做合理抽象（不要简单拼接）
|
||||
3. 必须包含明确的边界判断：
|
||||
- 正向：什么问题属于这个域
|
||||
- 反向：什么问题容易被误判为属于这个域，但实际应该检索其他域
|
||||
4. 反向边界必须引用其他域的 domain_id，格式："X类问题查{domain_id}"
|
||||
5. 不超过 60 字
|
||||
6. 语言要让 Planner 能做"检索/不检索"的二分判断
|
||||
|
||||
只返回纯文本指引，不要 JSON 或 markdown。
|
||||
Reference in New Issue
Block a user