chore(rag): rebuild script in Python and use biz collection
Switch default knowledge collection back to biz (drop+recreate on rebuild), replace PowerShell rebuild runner with Python, skip README.md imports, and merge duplicate rag keys in application.yml.
This commit is contained in:
@@ -8,14 +8,15 @@ public class MilvusConstants {
|
||||
public static final String MILVUS_DB_NAME = "default";
|
||||
|
||||
/**
|
||||
* Legacy collection name (pre hybrid). Knowledge path uses milvus.collection (default biz_hybrid).
|
||||
* Default knowledge collection name (dense + BM25).
|
||||
* Overridable via {@code milvus.collection}.
|
||||
*/
|
||||
public static final String MILVUS_COLLECTION_NAME = "biz";
|
||||
|
||||
/**
|
||||
* Default hybrid dense+BM25 collection.
|
||||
* Alias kept for readability in hybrid-related code.
|
||||
*/
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = "biz_hybrid";
|
||||
public static final String MILVUS_HYBRID_COLLECTION_NAME = MILVUS_COLLECTION_NAME;
|
||||
|
||||
/**
|
||||
* 向量维度(豆包 embedding 模型的维度)
|
||||
|
||||
@@ -67,7 +67,7 @@ public class KnowledgeBaseController {
|
||||
* 清空 hybrid collection + MySQL api_document + L0 内存索引,
|
||||
* 再从 knowledge_base 全量重建 dense+BM25 索引。
|
||||
*
|
||||
* <p>危险操作:会删除 {@code milvus.collection}(默认 biz_hybrid)与文档元数据表数据。
|
||||
* <p>危险操作:会删除 {@code milvus.collection}(默认 {@code biz})与文档元数据表数据。
|
||||
* 需要显式 confirm=REBUILD。</p>
|
||||
*/
|
||||
@PostMapping("/rebuild-hybrid")
|
||||
|
||||
@@ -57,7 +57,7 @@ public class KnowledgeBaseInitService {
|
||||
|
||||
/**
|
||||
* Drop hybrid collection, clear MySQL api_document + L0 memory index,
|
||||
* then force-import all markdown under knowledge.base-path into biz_hybrid.
|
||||
* then force-import all markdown under knowledge.base-path into milvus.collection (default biz).
|
||||
*/
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public RebuildResult rebuildHybridFromKnowledgeBase() {
|
||||
@@ -82,6 +82,8 @@ public class KnowledgeBaseInitService {
|
||||
rebuild.setInit(init);
|
||||
rebuild.setL0IndexSizeAfterInit(knowledgeIndexService.getIndexSize());
|
||||
rebuild.setMysqlDocumentsAfterInit(apiDocumentRepository.count());
|
||||
// Success when at least one doc indexed and no hard failures.
|
||||
// README-like docs are skipped by scanner; remaining failures still mark unsuccessful.
|
||||
rebuild.setSuccess(init.getFailed() == 0 && init.getInserted() > 0);
|
||||
logger.info("Hybrid knowledge rebuild finished: success={}, inserted={}, failed={}",
|
||||
rebuild.isSuccess(), init.getInserted(), init.getFailed());
|
||||
@@ -257,7 +259,11 @@ public class KnowledgeBaseInitService {
|
||||
Files.walkFileTree(baseDir, new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
|
||||
if (file.toString().endsWith(".md")) {
|
||||
String name = file.getFileName() == null ? "" : file.getFileName().toString();
|
||||
// Import content docs only; skip README/index markdown without frontmatter.
|
||||
if (name.endsWith(".md")
|
||||
&& !name.equalsIgnoreCase("README.md")
|
||||
&& !name.equalsIgnoreCase("readme.md")) {
|
||||
files.add(file);
|
||||
}
|
||||
return FileVisitResult.CONTINUE;
|
||||
|
||||
@@ -61,8 +61,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
|
||||
private final MilvusProperties milvusProperties;
|
||||
|
||||
@Value("${milvus.collection:biz_hybrid}")
|
||||
private String collectionName = "biz_hybrid";
|
||||
@Value("${milvus.collection:biz}")
|
||||
private String collectionName = "biz";
|
||||
|
||||
@Value("${retrieval.hybrid.rrf-k:60}")
|
||||
private int rrfK = 60;
|
||||
@@ -144,8 +144,8 @@ public class MilvusHybridKnowledgeStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the hybrid collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Does not touch legacy {@code biz} collection.
|
||||
* Drop the configured knowledge collection (if present) and recreate empty dense+BM25 schema.
|
||||
* Used by knowledge rebuild scripts. Existing vectors in this collection are destroyed.
|
||||
*/
|
||||
public synchronized Map<String, Object> dropAndRecreateCollection() {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
|
||||
@@ -25,8 +25,8 @@ milvus:
|
||||
token: ${MILVUS_TOKEN}
|
||||
secure: true
|
||||
vector-dim: 1024 # BGE-M3 = 1024,换模型时同步改
|
||||
# dense+BM25 hybrid collection (requires reindex; legacy "biz" is not used by knowledge path)
|
||||
collection: biz_hybrid
|
||||
# knowledge collection (drop+recreate on rebuild; dense+BM25 schema)
|
||||
collection: biz
|
||||
|
||||
# =====================================================
|
||||
# 模型路由配置
|
||||
@@ -158,7 +158,10 @@ document:
|
||||
|
||||
# RAG 配置
|
||||
rag:
|
||||
top-k: 3 # 检索返回的最相似文档数量
|
||||
top-k: 3 # legacy fallback when retrieve-k/return-n absent
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
sidecar:
|
||||
spring-ai:
|
||||
enabled: false
|
||||
@@ -176,12 +179,6 @@ retrieval:
|
||||
highly-relevant-threshold: 0.75 # similarity >= 0.75 → HIGHLY_RELEVANT
|
||||
reference-threshold: 0.5 # similarity >= 0.5 → REFERENCE
|
||||
|
||||
# RAG 召回宽度(Delivery 1)
|
||||
rag:
|
||||
retrieve-k: 20
|
||||
return-n: 5
|
||||
max-chunks-per-document: 2
|
||||
|
||||
# Prometheus 配置
|
||||
prometheus:
|
||||
base-url: http://localhost:9090
|
||||
|
||||
@@ -23,7 +23,7 @@ class VectorSearchServiceTest {
|
||||
VectorSearchService.SearchResult expected = result("doc-1", 0.2f);
|
||||
when(store.searchDense(eq("query"), eq(List.of(0.1f, 0.2f)), eq(3), isNull()))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz_hybrid");
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
@@ -44,7 +44,7 @@ class VectorSearchServiceTest {
|
||||
VectorSearchService.SearchResult expected = result("doc-h", 0.4f);
|
||||
when(store.searchHybrid(eq("pool"), eq(List.of(0.3f)), eq(5), eq("mysql")))
|
||||
.thenReturn(List.of(expected));
|
||||
when(store.collectionName()).thenReturn("biz_hybrid");
|
||||
when(store.collectionName()).thenReturn("biz");
|
||||
|
||||
VectorSearchService service = new VectorSearchService();
|
||||
ReflectionTestUtils.setField(service, "knowledgeStore", store);
|
||||
|
||||
Reference in New Issue
Block a user