osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rag): close eval pipeline with live snapshots

Browse Source
This commit is contained in:
zhuyongxin
2026-07-06 21:39:27 +08:00
parent cf3333d607
commit ed7efc58b7
47 changed files with 2613 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval/rag-retrieval/reports/baseline.json
+95 -3
View File
@@ -1,11 +1,15 @@
{
"generatedAt": "2026-07-04T17:59:52.172759+00:00",
"generatedAt": "2026-07-06T13:37:59.726351+00:00",
"caseFile": "eval/rag-retrieval/cases/golden-cases.json",
"fixtureDir": "eval/rag-retrieval/fixtures",
"aggregate": {
"caseCount": 6,
"caseCount": 7,
"topK": 5,
"strongHitCount": 6,
"passedCount": 7,
"failedCount": 0,
"passRate": 1.0,
"lookupResultCaseCount": 7,
"strongHitCount": 7,
"mediumHitCount": 0,
"weakHitCount": 0,
"missCount": 0,
@@ -18,6 +22,7 @@
"caseId": "chat-mysql-connection-pool",
"scenario": "chat",
"query": "MySQL connection pool is exhausted. How should I diagnose it?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -31,12 +36,22 @@
"hikaricp"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"mysql-connection-pool",
"incident-diagnosis-flow"
],
"omittedSources": [],
"rerankTopSource": "mysql-connection-pool",
"failedChecks": []
},
{
"caseId": "chat-diagnosis-flow",
"scenario": "chat",
"query": "What is the standard troubleshooting flow for an application incident?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -50,12 +65,22 @@
"remediation"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"incident-diagnosis-flow",
"rag-chunk-context-reconstruction"
],
"omittedSources": [],
"rerankTopSource": "incident-diagnosis-flow",
"failedChecks": []
},
{
"caseId": "aiops-payment-latency-alert",
"scenario": "aiops",
"query": "Alert HighLatency on payment-service with p95 latency above threshold",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -69,12 +94,22 @@
"downstream dependency"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"payment-service-latency",
"mysql-connection-pool"
],
"omittedSources": [],
"rerankTopSource": "payment-service-latency",
"failedChecks": []
},
{
"caseId": "aiops-prometheus-alert-scope",
"scenario": "aiops",
"query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -87,12 +122,21 @@
"scope"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"aiops-alert-scope-control"
],
"omittedSources": [],
"rerankTopSource": "aiops-alert-scope-control",
"failedChecks": []
},
{
"caseId": "chat-rag-chunk-context",
"scenario": "chat",
"query": "If a long section is split into multiple chunks, how do we keep retrieval context?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -106,12 +150,22 @@
"breadcrumb"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"rag-chunk-context-reconstruction",
"rag-breadcrumb-embedding-gap"
],
"omittedSources": [],
"rerankTopSource": "rag-chunk-context-reconstruction",
"failedChecks": []
},
{
"caseId": "chat-l0-domain-hint",
"scenario": "chat",
"query": "Should L0 keyword matching decide the final retrieval result?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
@@ -125,6 +179,44 @@
"metadata filter"
],
"breadcrumbMatched": true,
"selectedAttempt": "FILTERED_VECTOR",
"fallbackReason": null,
"evidenceStatus": "supported",
"includedSources": [
"rag-l0-domain-entity-hint",
"rag-l0-l1-fusion-ranking"
],
"omittedSources": [],
"rerankTopSource": "rag-l0-domain-entity-hint",
"failedChecks": []
},
{
"caseId": "chat-l0-filter-fallback",
"scenario": "chat",
"query": "RAG query was over-filtered by L0 and filtered vector search returned low quality evidence. What should happen?",
"dataShape": "lookupResult",
"hitLevel": "strong",
"passed": true,
"firstExpectedRank": 1,
"topCandidates": [
"1:rag-l0-filter-fallback",
"2:rag-l0-domain-entity-hint"
],
"matchedKeywords": [
"skip the l0 filter",
"unfiltered vector retry",
"low quality"
],
"breadcrumbMatched": true,
"selectedAttempt": "UNFILTERED_VECTOR_RETRY",
"fallbackReason": "filtered_vector_low_quality",
"evidenceStatus": "supported",
"includedSources": [
"rag-l0-filter-fallback",
"rag-l0-domain-entity-hint"
],
"omittedSources": [],
"rerankTopSource": "rag-l0-filter-fallback",
"failedChecks": []
}
]
eval/rag-retrieval/reports/baseline.md
+16 -11
View File
@@ -1,16 +1,20 @@
# RAG Retrieval Baseline
Generated at: `2026-07-04T17:59:52.172759+00:00`
Generated at: `2026-07-06T13:37:59.726351+00:00`
## Aggregate
| Metric | Value |
|---|---:|
| Cases | 6 |
| Cases | 7 |
| Top K | 5 |
| Passed | 7 |
| Failed | 0 |
| Pass rate | 1.0 |
| LookupResult fixtures | 7 |
| Recall@K | 1.0 |
| Strong hit rate | 1.0 |
| Strong hits | 6 |
| Strong hits | 7 |
| Medium hits | 0 |
| Weak hits | 0 |
| Misses | 0 |
@@ -18,11 +22,12 @@ Generated at: `2026-07-04T17:59:52.172759+00:00`
## Cases
| Case | Scenario | Hit | First Expected Rank | Top Candidates | Failed Checks |
|---|---|---|---:|---|---|
| chat-mysql-connection-pool | chat | strong | 1 | 1:mysql-connection-pool<br>2:incident-diagnosis-flow | |
| chat-diagnosis-flow | chat | strong | 1 | 1:incident-diagnosis-flow<br>2:rag-chunk-context-reconstruction | |
| aiops-payment-latency-alert | aiops | strong | 1 | 1:payment-service-latency<br>2:mysql-connection-pool | |
| aiops-prometheus-alert-scope | aiops | strong | 1 | 1:aiops-alert-scope-control | |
| chat-rag-chunk-context | chat | strong | 1 | 1:rag-chunk-context-reconstruction<br>2:rag-breadcrumb-embedding-gap | |
| chat-l0-domain-hint | chat | strong | 1 | 1:rag-l0-domain-entity-hint<br>2:rag-l0-l1-fusion-ranking | |
| Case | Scenario | Pass | Hit | Attempt | Fallback | Evidence | First Expected Rank | Top Candidates | Failed Checks |
|---|---|---|---|---|---|---|---:|---|---|
| chat-mysql-connection-pool | chat | true | strong | FILTERED_VECTOR | | supported | 1 | 1:mysql-connection-pool<br>2:incident-diagnosis-flow | |
| chat-diagnosis-flow | chat | true | strong | FILTERED_VECTOR | | supported | 1 | 1:incident-diagnosis-flow<br>2:rag-chunk-context-reconstruction | |
| aiops-payment-latency-alert | aiops | true | strong | FILTERED_VECTOR | | supported | 1 | 1:payment-service-latency<br>2:mysql-connection-pool | |
| aiops-prometheus-alert-scope | aiops | true | strong | FILTERED_VECTOR | | supported | 1 | 1:aiops-alert-scope-control | |
| chat-rag-chunk-context | chat | true | strong | FILTERED_VECTOR | | supported | 1 | 1:rag-chunk-context-reconstruction<br>2:rag-breadcrumb-embedding-gap | |
| chat-l0-domain-hint | chat | true | strong | FILTERED_VECTOR | | supported | 1 | 1:rag-l0-domain-entity-hint<br>2:rag-l0-l1-fusion-ranking | |
| chat-l0-filter-fallback | chat | true | strong | UNFILTERED_VECTOR_RETRY | filtered_vector_low_quality | supported | 1 | 1:rag-l0-filter-fallback<br>2:rag-l0-domain-entity-hint | |