SuperBizAgent-java/src/test/java/com/superbiz/agent/eval/DiagnosisEvalBaselineDiffTest.java

package com.superbiz.agent.eval;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

class DiagnosisEvalBaselineDiffTest {

    private final ObjectMapper objectMapper = new ObjectMapper();
    private final DiagnosisEvalBaselineDiffer differ = new DiagnosisEvalBaselineDiffer();

    @Test
    void compareReportsDetectsAggregateAndCaseRegressions() throws Exception {
        DiagnosisEvalReport baseline = readBaselineReport();
        DiagnosisEvalReport current = readBaselineReport();
        degradeRedisCase(current);

        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);

        assertTrue(diff.isHasRegression());
        assertEquals(6, diff.getRegressionCount());
        assertEquals(2, diff.getChangedCount());
        assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "passRate"));
        assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "averageToolCallCount"));
        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "passed"));
        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "verdict"));
        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "matchedKeywordCount"));
        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "evidenceCoverage.query_logs"));
        assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.LOW_CONFID"));
        assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.REJECT"));
    }

    @Test
    void compareReportsDetectsMissingAndNewCases() throws Exception {
        DiagnosisEvalReport baseline = readBaselineReport();
        DiagnosisEvalReport current = readBaselineReport();
        DiagnosisEvalResult removed = current.getResults().remove(0);
        current.getResults().add(DiagnosisEvalResult.builder()
                .caseId("new-case")
                .title("New case")
                .passed(true)
                .failedChecks(List.of())
                .verdict("PASS")
                .matchedKeywordCount(1)
                .requiredKeywordCount(1)
                .evidenceCoverage(new LinkedHashMap<>())
                .toolCallCount(1)
                .durationMs(1000)
                .build());

        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);

        assertTrue(hasItem(diff, "REGRESSION", "case", removed.getCaseId(), "casePresence"));
        assertTrue(hasItem(diff, "CHANGED", "case", "new-case", "casePresence"));
    }

    @Test
    void compareSameReportHasNoDiff() throws Exception {
        DiagnosisEvalReport baseline = readBaselineReport();

        DiagnosisEvalDiffReport diff = differ.compare(baseline, readBaselineReport());

        assertFalse(diff.isHasRegression());
        assertEquals(0, diff.getRegressionCount());
        assertTrue(diff.getItems().isEmpty());
    }

    @Test
    void writerOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
        DiagnosisEvalReport baseline = readBaselineReport();
        DiagnosisEvalReport current = readBaselineReport();
        degradeRedisCase(current);
        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
        DiagnosisEvalDiffReportWriter writer = new DiagnosisEvalDiffReportWriter(objectMapper);

        Path json = tempDir.resolve("baseline-diff.json");
        Path markdown = tempDir.resolve("baseline-diff.md");
        writer.writeJson(diff, json);
        writer.writeMarkdown(diff, markdown);

        assertTrue(Files.exists(json));
        assertTrue(Files.readString(json).contains("\"hasRegression\" : true"));
        assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Baseline Diff"));
        assertTrue(Files.readString(markdown).contains("redis-timeout"));
    }

    private DiagnosisEvalReport readBaselineReport() throws Exception {
        return objectMapper.readValue(Path.of("mvp/eval/reports/baseline-report.json").toFile(),
                DiagnosisEvalReport.class);
    }

    private void degradeRedisCase(DiagnosisEvalReport report) {
        report.setPassedCases(4);
        report.setPassRate(0.8);
        report.setAverageToolCallCount(3.0);
        report.setAverageDurationMs(45800.0);
        report.setVerdictDistribution(new LinkedHashMap<>());
        report.getVerdictDistribution().put("PASS", 2L);
        report.getVerdictDistribution().put("LOW_CONFID", 2L);
        report.getVerdictDistribution().put("REJECT", 1L);

        DiagnosisEvalResult redis = result(report, "redis-timeout");
        redis.setPassed(false);
        redis.setFailedChecks(new ArrayList<>(List.of("missing required evidence tool: query_logs")));
        redis.setVerdict("REJECT");
        redis.setMatchedKeywordCount(1);
        redis.getEvidenceCoverage().put("query_logs", false);
        redis.setToolCallCount(1);
        redis.setDurationMs(36000);
    }

    private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
        return report.getResults().stream()
                .filter(item -> caseId.equals(item.getCaseId()))
                .findFirst()
                .orElseThrow();
    }

    private boolean hasItem(DiagnosisEvalDiffReport diff,
                            String type,
                            String scope,
                            String caseId,
                            String metric) {
        return diff.getItems().stream().anyMatch(item ->
                type.equals(item.getType())
                        && scope.equals(item.getScope())
                        && java.util.Objects.equals(caseId, item.getCaseId())
                        && metric.equals(item.getMetric()));
    }
}