SuperBizAgent-java/eval/rag-retrieval/reports/current.json

{
  "generatedAt": "2026-07-28T06:48:56.421877+00:00",
  "caseFile": "eval/rag-retrieval/cases/golden-cases.json",
  "fixtureDir": "eval/rag-retrieval/fixtures",
  "aggregate": {
    "caseCount": 7,
    "topK": 5,
    "passedCount": 6,
    "failedCount": 1,
    "passRate": 0.8571,
    "lookupResultCaseCount": 7,
    "strongHitCount": 6,
    "mediumHitCount": 0,
    "weakHitCount": 1,
    "missCount": 0,
    "recallAtK": 0.8571,
    "strongHitRate": 0.8571,
    "averageFirstHitRank": 1.0
  },
  "results": [
    {
      "caseId": "chat-mysql-connection-pool",
      "scenario": "chat",
      "query": "MySQL connection pool is exhausted. How should I diagnose it?",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:mysql-connection-pool",
        "2:mysql-connection-pool"
      ],
      "matchedKeywords": [
        "connection pool",
        "max_connections",
        "hikaricp"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "mysql-connection-pool",
        "mysql-connection-pool"
      ],
      "omittedSources": [],
      "rerankTopSource": "mysql-connection-pool",
      "failedChecks": []
    },
    {
      "caseId": "chat-diagnosis-flow",
      "scenario": "chat",
      "query": "What is the standard troubleshooting flow for an application incident?",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:incident-diagnosis-flow",
        "2:incident-diagnosis-flow"
      ],
      "matchedKeywords": [
        "collect evidence",
        "verify",
        "remediation"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "incident-diagnosis-flow",
        "incident-diagnosis-flow"
      ],
      "omittedSources": [],
      "rerankTopSource": "incident-diagnosis-flow",
      "failedChecks": []
    },
    {
      "caseId": "aiops-payment-latency-alert",
      "scenario": "aiops",
      "query": "Alert HighLatency on payment-service with p95 latency above threshold",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:payment-service-latency",
        "2:aiops-alert-scope-control",
        "3:payment-service-latency",
        "4:aiops-alert-scope-control"
      ],
      "matchedKeywords": [
        "p95 latency",
        "payment-service",
        "downstream dependency"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "payment-service-latency",
        "aiops-alert-scope-control",
        "payment-service-latency",
        "aiops-alert-scope-control"
      ],
      "omittedSources": [],
      "rerankTopSource": "payment-service-latency",
      "failedChecks": []
    },
    {
      "caseId": "aiops-prometheus-alert-scope",
      "scenario": "aiops",
      "query": "When an AIOps request already includes alert payload, should the agent diagnose unrelated active alerts?",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:aiops-alert-scope-control",
        "2:payment-service-latency",
        "3:payment-service-latency",
        "4:aiops-alert-scope-control"
      ],
      "matchedKeywords": [
        "payload",
        "unrelated active alerts",
        "scope"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "aiops-alert-scope-control",
        "payment-service-latency",
        "payment-service-latency",
        "aiops-alert-scope-control"
      ],
      "omittedSources": [],
      "rerankTopSource": "aiops-alert-scope-control",
      "failedChecks": []
    },
    {
      "caseId": "chat-rag-chunk-context",
      "scenario": "chat",
      "query": "If a long section is split into multiple chunks, how do we keep retrieval context?",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:rag-chunk-context-reconstruction",
        "2:rag-l0-domain-entity-hint",
        "3:rag-chunk-context-reconstruction",
        "4:rag-l0-domain-entity-hint"
      ],
      "matchedKeywords": [
        "neighbor chunk",
        "same section",
        "breadcrumb"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "rag-chunk-context-reconstruction",
        "rag-l0-domain-entity-hint",
        "rag-chunk-context-reconstruction",
        "rag-l0-domain-entity-hint"
      ],
      "omittedSources": [],
      "rerankTopSource": "rag-chunk-context-reconstruction",
      "failedChecks": []
    },
    {
      "caseId": "chat-l0-domain-hint",
      "scenario": "chat",
      "query": "Should L0 keyword matching decide the final retrieval result?",
      "dataShape": "lookupResult",
      "hitLevel": "strong",
      "passed": true,
      "firstExpectedRank": 1,
      "topCandidates": [
        "1:rag-l0-domain-entity-hint",
        "2:rag-chunk-context-reconstruction",
        "3:rag-l0-domain-entity-hint",
        "4:rag-chunk-context-reconstruction"
      ],
      "matchedKeywords": [
        "domain detector",
        "entity extractor",
        "metadata filter"
      ],
      "breadcrumbMatched": true,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "rag-l0-domain-entity-hint",
        "rag-chunk-context-reconstruction",
        "rag-l0-domain-entity-hint",
        "rag-chunk-context-reconstruction"
      ],
      "omittedSources": [],
      "rerankTopSource": "rag-l0-domain-entity-hint",
      "failedChecks": []
    },
    {
      "caseId": "chat-l0-filter-fallback",
      "scenario": "chat",
      "query": "RAG query was over-filtered by L0 and filtered vector search returned low quality evidence. What should happen?",
      "dataShape": "lookupResult",
      "hitLevel": "weak",
      "passed": false,
      "firstExpectedRank": null,
      "topCandidates": [
        "1:rag-l0-filter-decoy",
        "2:rag-l0-filter-decoy"
      ],
      "matchedKeywords": [
        "low quality"
      ],
      "breadcrumbMatched": false,
      "selectedAttempt": "FILTERED_VECTOR",
      "fallbackReason": null,
      "evidenceStatus": "supported",
      "includedSources": [
        "rag-l0-filter-decoy",
        "rag-l0-filter-decoy"
      ],
      "omittedSources": [],
      "rerankTopSource": "rag-l0-filter-decoy",
      "failedChecks": [
        "expected document not found",
        "selected attempt mismatch: expected UNFILTERED_VECTOR_RETRY, got FILTERED_VECTOR",
        "fallback reason mismatch: expected one of [filtered_vector_low_quality, filtered_vector_no_evidence], got <none>",
        "rerank top source mismatch: expected rag-l0-filter-fallback, got rag-l0-filter-decoy",
        "expected context sources missing: rag-l0-filter-fallback"
      ]
    }
  ]
}