feat(harness,rag): dual LLM audit fields, run conclusion, and hybrid quality
Persist provider reasoning and assistant text separately on agent_reasoning_audit (DeepSeekAssistantMessage path), extract diagnosis_run.conclusion, enrich RAG tool audit (step_id/query/qualityScore), gate empty mysql tools, drop devtools, and align MVP docs after live E2E verification.
This commit is contained in:
@@ -3,7 +3,9 @@ param(
|
||||
[string]$Fixtures = "eval\rag-retrieval\fixtures",
|
||||
[string]$RetrievedAt = "",
|
||||
[string]$KbScope = "rag-eval",
|
||||
[string]$VectorStoreMode = "spring",
|
||||
# hybrid = production main path; dense = same-collection recall baseline
|
||||
[ValidateSet("hybrid", "dense")]
|
||||
[string]$SearchMode = "hybrid",
|
||||
[switch]$SkipEval
|
||||
)
|
||||
|
||||
@@ -16,7 +18,7 @@ $mavenArgs = @(
|
||||
"-Drag.snapshot.cases=$Cases",
|
||||
"-Drag.snapshot.fixtures=$Fixtures",
|
||||
"-Dretrieval.kb-scope=$KbScope",
|
||||
"-Dretrieval.vector-store.mode=$VectorStoreMode"
|
||||
"-Dretrieval.search.mode=$SearchMode"
|
||||
)
|
||||
|
||||
if ($RetrievedAt -ne "") {
|
||||
@@ -25,10 +27,16 @@ if ($RetrievedAt -ne "") {
|
||||
|
||||
$mavenArgs += "test"
|
||||
|
||||
Write-Host "Generating RAG lookupResult fixtures from real LookupKnowledgeTool..."
|
||||
Write-Host "Generating RAG lookupResult fixtures (search.mode=$SearchMode, kb-scope=$KbScope)..."
|
||||
& mvn @mavenArgs
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Fixture generation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
|
||||
if (-not $SkipEval) {
|
||||
Write-Host "Running offline RAG retrieval baseline..."
|
||||
& python scripts\eval_rag_retrieval.py --cases $Cases --fixtures $Fixtures
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Offline eval failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user