feat(agent): add executor evidence output contract
This commit is contained in:
@@ -101,6 +101,8 @@ public class DiagnosisTraceEvaluator {
|
||||
failedChecks.add("reject output does not use degraded template");
|
||||
}
|
||||
|
||||
failedChecks.addAll(validateExecutorStructuredOutput(trace));
|
||||
|
||||
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
|
||||
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
|
||||
|
||||
@@ -174,6 +176,32 @@ public class DiagnosisTraceEvaluator {
|
||||
return value == null ? null : String.valueOf(value);
|
||||
}
|
||||
|
||||
private List<String> validateExecutorStructuredOutput(DiagnosisTraceResponse trace) {
|
||||
Object structuredOutput = nestedValue(trace, "verifier_evaluation", "executor_structured_output");
|
||||
if (!(structuredOutput instanceof Map<?, ?> output)) {
|
||||
return List.of();
|
||||
}
|
||||
Object claims = output.get("claims");
|
||||
if (!(claims instanceof List<?> claimList)) {
|
||||
return List.of("executor structured output missing claims array");
|
||||
}
|
||||
|
||||
List<String> failedChecks = new ArrayList<>();
|
||||
for (Object item : claimList) {
|
||||
if (!(item instanceof Map<?, ?> claim)) {
|
||||
failedChecks.add("executor structured claim is not an object");
|
||||
continue;
|
||||
}
|
||||
Object claimIdValue = claim.get("claim_id");
|
||||
String claimId = claimIdValue == null ? "unknown" : String.valueOf(claimIdValue);
|
||||
Object bindings = claim.get("evidence_bindings");
|
||||
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
|
||||
failedChecks.add("executor confirmed claim missing evidence bindings: " + claimId);
|
||||
}
|
||||
}
|
||||
return failedChecks;
|
||||
}
|
||||
|
||||
private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
|
||||
if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) {
|
||||
return null;
|
||||
|
||||
Reference in New Issue
Block a user