SuperBizAgent-java/mvp/eval/reports/baseline-report.json

{
  "totalCases" : 12,
  "passedCases" : 12,
  "passRate" : 1.0,
  "verdictDistribution" : {
    "PASS" : 5,
    "LOW_CONFID" : 6,
    "REJECT" : 1
  },
  "averageToolCallCount" : 1.4166666666666667,
  "averageDurationMs" : 38500.0,
  "results" : [ {
    "caseId" : "narrow-highcpu-observation",
    "title" : "Narrow HighCPU observation",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "PASS",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "query_metrics" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
    "promptAuditVersion" : null,
    "composerStatus" : "valid",
    "claimCheckCount" : 1,
    "gatekeeperRuleCount" : 1,
    "toolCallCount" : 1,
    "durationMs" : 18000
  }, {
    "caseId" : "prompt-gatekeeper-audit-closure",
    "title" : "Prompt and Gatekeeper audit closure",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "PASS",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "query_metrics" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
    "promptAuditVersion" : "chat-prompts-v1",
    "composerStatus" : "valid",
    "claimCheckCount" : 1,
    "gatekeeperRuleCount" : 2,
    "toolCallCount" : 1,
    "durationMs" : 19000
  }, {
    "caseId" : "hikari-no-evidence-negative-observation",
    "title" : "Hikari no-evidence negative observation",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "PASS",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "query_logs" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
    "promptAuditVersion" : null,
    "composerStatus" : "valid",
    "claimCheckCount" : 1,
    "gatekeeperRuleCount" : 1,
    "toolCallCount" : 1,
    "durationMs" : 21000
  }, {
    "caseId" : "payment-timeout",
    "title" : "Payment API timeout",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "PASS",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "lookup_knowledge" : true,
      "query_logs" : true,
      "query_metrics" : true
    },
    "gatekeeperStatus" : null,
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : null,
    "claimCheckCount" : null,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 3,
    "durationMs" : 42000
  }, {
    "caseId" : "mysql-pool-exhausted",
    "title" : "MySQL connection pool exhausted",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "lookup_knowledge" : true,
      "query_logs" : true
    },
    "gatekeeperStatus" : null,
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : null,
    "claimCheckCount" : null,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 2,
    "durationMs" : 51000
  }, {
    "caseId" : "redis-timeout",
    "title" : "Redis timeout",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 2,
    "requiredKeywordCount" : 2,
    "evidenceCoverage" : {
      "query_logs" : true
    },
    "gatekeeperStatus" : null,
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : null,
    "claimCheckCount" : null,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 1,
    "durationMs" : 36000
  }, {
    "caseId" : "slow-response",
    "title" : "Slow response",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "PASS",
    "matchedKeywordCount" : 2,
    "requiredKeywordCount" : 2,
    "evidenceCoverage" : {
      "query_metrics" : true,
      "query_logs" : true
    },
    "gatekeeperStatus" : null,
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : null,
    "claimCheckCount" : null,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 2,
    "durationMs" : 47000
  }, {
    "caseId" : "jvm-memory-risk",
    "title" : "JVM memory risk",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "query_metrics" : true,
      "query_logs" : true
    },
    "gatekeeperStatus" : null,
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : null,
    "claimCheckCount" : null,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 2,
    "durationMs" : 53000
  }, {
    "caseId" : "gatekeeper-fabricated-invocation",
    "title" : "Gatekeeper fabricated invocation",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "REJECT",
    "matchedKeywordCount" : 3,
    "requiredKeywordCount" : 3,
    "evidenceCoverage" : {
      "query_logs" : true
    },
    "gatekeeperStatus" : "fail",
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : "valid",
    "claimCheckCount" : 1,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 1,
    "durationMs" : 39000
  }, {
    "caseId" : "unsupported-claim-filtering",
    "title" : "Unsupported claim filtering",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 2,
    "requiredKeywordCount" : 2,
    "evidenceCoverage" : {
      "query_logs" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : "valid",
    "claimCheckCount" : 2,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 1,
    "durationMs" : 44000
  }, {
    "caseId" : "audit-metadata-low-confid",
    "title" : "Audit metadata low confidence",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 2,
    "requiredKeywordCount" : 2,
    "evidenceCoverage" : {
      "query_logs" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : "gatekeeper-rules-v1",
    "promptAuditVersion" : "chat-prompts-v1",
    "composerStatus" : "valid",
    "claimCheckCount" : 2,
    "gatekeeperRuleCount" : 2,
    "toolCallCount" : 1,
    "durationMs" : 45000
  }, {
    "caseId" : "composer-fallback-no-raw-json",
    "title" : "Composer fallback no raw JSON",
    "passed" : true,
    "failedChecks" : [ ],
    "verdict" : "LOW_CONFID",
    "matchedKeywordCount" : 2,
    "requiredKeywordCount" : 2,
    "evidenceCoverage" : {
      "query_metrics" : true
    },
    "gatekeeperStatus" : "pass",
    "gatekeeperRuleSetVersion" : null,
    "promptAuditVersion" : null,
    "composerStatus" : "composer_malformed",
    "claimCheckCount" : 2,
    "gatekeeperRuleCount" : null,
    "toolCallCount" : 1,
    "durationMs" : 47000
  } ]
}