refactor(harness): remove legacy agent architecture
This commit is contained in:
zhuyongxin
@@ -1,168 +0,0 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://localhost:9900",
|
||||
[string]$SessionId = "mvp-demo-interview-payment-timeout-001",
|
||||
[string]$RequestFile = "$PSScriptRoot/../requests/payment-timeout-chat.json",
|
||||
[string]$OutputDir = "$PSScriptRoot/../output"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Test-ServiceReachable {
|
||||
param([string]$Url)
|
||||
|
||||
try {
|
||||
$request = [System.Net.WebRequest]::Create($Url)
|
||||
$request.Method = "GET"
|
||||
$request.Timeout = 5000
|
||||
$response = $request.GetResponse()
|
||||
$response.Close()
|
||||
return $true
|
||||
} catch [System.Net.WebException] {
|
||||
if ($_.Exception.Response -ne $null) {
|
||||
$_.Exception.Response.Close()
|
||||
return $true
|
||||
}
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Get-TraceData {
|
||||
param($TraceResponse)
|
||||
|
||||
if ($TraceResponse.PSObject.Properties.Name -contains "data") {
|
||||
return $TraceResponse.data
|
||||
}
|
||||
return $TraceResponse
|
||||
}
|
||||
|
||||
function Get-SelfEvaluation {
|
||||
param($TraceData)
|
||||
|
||||
if ($null -eq $TraceData -or $null -eq $TraceData.session) {
|
||||
return $null
|
||||
}
|
||||
return $TraceData.session.selfEvaluation
|
||||
}
|
||||
|
||||
function Get-ToolNames {
|
||||
param($TraceData)
|
||||
|
||||
if ($null -eq $TraceData -or $null -eq $TraceData.toolInvocations) {
|
||||
return @()
|
||||
}
|
||||
return @($TraceData.toolInvocations | ForEach-Object { $_.toolName } | Where-Object { $_ } | Sort-Object -Unique)
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null
|
||||
|
||||
Write-Host "Running interview demo preflight..."
|
||||
Write-Host "BaseUrl: $BaseUrl"
|
||||
Write-Host "SessionId: $SessionId"
|
||||
|
||||
if (-not (Test-ServiceReachable -Url $BaseUrl)) {
|
||||
throw "Service is not reachable: $BaseUrl. Start the app with mvp-demo profile first: mvn spring-boot:run -Dspring-boot.run.profiles=mvp-demo"
|
||||
}
|
||||
|
||||
$request = Get-Content -Raw -Encoding UTF8 -Path $RequestFile | ConvertFrom-Json
|
||||
$request.Id = $SessionId
|
||||
$body = $request | ConvertTo-Json -Depth 8
|
||||
|
||||
$chatRequest = @{
|
||||
Method = "Post"
|
||||
Uri = "$BaseUrl/api/chat"
|
||||
ContentType = "application/json; charset=utf-8"
|
||||
Body = $body
|
||||
}
|
||||
$chat = Invoke-RestMethod @chatRequest
|
||||
|
||||
$chatPath = Join-Path $OutputDir "chat-response.json"
|
||||
$chat | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $chatPath
|
||||
|
||||
$runId = $chat.data.runId
|
||||
if (-not $runId) {
|
||||
throw "Chat response did not include runId; exact trace verification cannot continue."
|
||||
}
|
||||
|
||||
$traceRequest = @{
|
||||
Method = "Get"
|
||||
Uri = "$BaseUrl/api/diagnosis/$SessionId/trace?runId=$([System.Uri]::EscapeDataString($runId))"
|
||||
}
|
||||
$trace = Invoke-RestMethod @traceRequest
|
||||
|
||||
$tracePath = Join-Path $OutputDir "trace-response.json"
|
||||
$trace | ConvertTo-Json -Depth 80 | Set-Content -Encoding UTF8 -Path $tracePath
|
||||
|
||||
$feedbackBody = @{
|
||||
sessionId = $SessionId
|
||||
runId = $runId
|
||||
feedback = "useful"
|
||||
} | ConvertTo-Json
|
||||
|
||||
$feedbackRequest = @{
|
||||
Method = "Post"
|
||||
Uri = "$BaseUrl/api/feedback"
|
||||
ContentType = "application/json; charset=utf-8"
|
||||
Body = $feedbackBody
|
||||
}
|
||||
$feedback = Invoke-RestMethod @feedbackRequest
|
||||
|
||||
$feedbackPath = Join-Path $OutputDir "feedback-response.json"
|
||||
$feedback | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $feedbackPath
|
||||
|
||||
$traceData = Get-TraceData -TraceResponse $trace
|
||||
$selfEvaluation = Get-SelfEvaluation -TraceData $traceData
|
||||
$verifierEvaluation = $null
|
||||
if ($null -ne $selfEvaluation) {
|
||||
$verifierEvaluation = $selfEvaluation.verifier_evaluation
|
||||
}
|
||||
|
||||
$gatekeeperResult = $null
|
||||
$promptAudit = $null
|
||||
if ($null -ne $verifierEvaluation) {
|
||||
$gatekeeperResult = $verifierEvaluation.gatekeeper_result
|
||||
$promptAudit = $verifierEvaluation.prompt_audit
|
||||
}
|
||||
|
||||
$verdict = $null
|
||||
$gatekeeperStatus = $null
|
||||
$gatekeeperRuleSetVersion = $null
|
||||
$promptAuditVersion = $null
|
||||
if ($null -ne $verifierEvaluation) {
|
||||
$verdict = $verifierEvaluation.verdict
|
||||
}
|
||||
if ($null -ne $gatekeeperResult) {
|
||||
$gatekeeperStatus = $gatekeeperResult.status
|
||||
$gatekeeperRuleSetVersion = $gatekeeperResult.rule_set_version
|
||||
}
|
||||
if ($null -ne $promptAudit) {
|
||||
$promptAuditVersion = $promptAudit.version
|
||||
}
|
||||
$toolNames = Get-ToolNames -TraceData $traceData
|
||||
$summaryPath = Join-Path $OutputDir "interview-demo-summary.json"
|
||||
|
||||
$summary = [ordered]@{
|
||||
sessionId = $SessionId
|
||||
runId = $runId
|
||||
baseUrl = $BaseUrl
|
||||
chatSuccess = $chat.data.success
|
||||
verdict = $verdict
|
||||
gatekeeperStatus = $gatekeeperStatus
|
||||
gatekeeperRuleSetVersion = $gatekeeperRuleSetVersion
|
||||
promptAuditVersion = $promptAuditVersion
|
||||
toolNames = $toolNames
|
||||
paths = [ordered]@{
|
||||
chat = $chatPath
|
||||
trace = $tracePath
|
||||
feedback = $feedbackPath
|
||||
summary = $summaryPath
|
||||
}
|
||||
}
|
||||
|
||||
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -Path $summaryPath
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Interview demo preflight completed."
|
||||
Write-Host "Verdict: $($summary.verdict)"
|
||||
Write-Host "Gatekeeper rules: $($summary.gatekeeperRuleSetVersion)"
|
||||
Write-Host "Prompt audit: $($summary.promptAuditVersion)"
|
||||
Write-Host "Summary: $summaryPath"
|
||||
@@ -7,55 +7,122 @@ param(
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function ConvertFrom-NamedSse {
|
||||
param([Parameter(Mandatory = $true)][string]$Content)
|
||||
|
||||
$events = @()
|
||||
foreach ($frame in ($Content -split "(?:\r?\n){2,}")) {
|
||||
if ([string]::IsNullOrWhiteSpace($frame)) {
|
||||
continue
|
||||
}
|
||||
|
||||
$name = $null
|
||||
$dataLines = @()
|
||||
foreach ($line in ($frame -split "\r?\n")) {
|
||||
if ($line.StartsWith("event:")) {
|
||||
$name = $line.Substring(6).Trim()
|
||||
} elseif ($line.StartsWith("data:")) {
|
||||
$dataLines += $line.Substring(5).TrimStart()
|
||||
}
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($name) -or $dataLines.Count -eq 0) {
|
||||
throw "Invalid named SSE frame: $frame"
|
||||
}
|
||||
$rawData = $dataLines -join "`n"
|
||||
$events += [pscustomobject]@{
|
||||
name = $name
|
||||
payload = $rawData | ConvertFrom-Json
|
||||
}
|
||||
}
|
||||
return @($events)
|
||||
}
|
||||
|
||||
function Assert-ChatSseContract {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][array]$Events,
|
||||
[Parameter(Mandatory = $true)][string]$ExpectedSessionId
|
||||
)
|
||||
|
||||
if ($Events.Count -lt 3) {
|
||||
throw "Chat SSE must contain metadata, a terminal event, and done"
|
||||
}
|
||||
if ($Events[0].name -ne "metadata") {
|
||||
throw "First Chat SSE event must be metadata"
|
||||
}
|
||||
if ($Events[-1].name -ne "done") {
|
||||
throw "Last Chat SSE event must be done"
|
||||
}
|
||||
|
||||
$terminalEvents = @($Events | Where-Object { $_.name -in @("content", "failure") })
|
||||
if ($terminalEvents.Count -ne 1 -or $Events[-2].name -ne $terminalEvents[0].name) {
|
||||
throw "Chat SSE must contain exactly one content or failure immediately before done"
|
||||
}
|
||||
|
||||
$allowed = @("metadata", "status", "content", "failure", "done")
|
||||
$unknown = @($Events | Where-Object { $_.name -notin $allowed })
|
||||
if ($unknown.Count -gt 0) {
|
||||
throw "Chat SSE contains unknown events: $($unknown.name -join ', ')"
|
||||
}
|
||||
$invalidMiddle = @()
|
||||
if ($Events.Count -gt 3) {
|
||||
$invalidMiddle = @($Events[1..($Events.Count - 3)] |
|
||||
Where-Object { $_.name -ne "status" })
|
||||
}
|
||||
if ($invalidMiddle.Count -gt 0) {
|
||||
throw "Only status events are allowed between metadata and the terminal event"
|
||||
}
|
||||
|
||||
$metadata = $Events[0].payload
|
||||
if ($metadata.session_id -ne $ExpectedSessionId) {
|
||||
throw "SSE session_id does not match the requested SessionId"
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace([string]$metadata.run_id)) {
|
||||
throw "SSE metadata is missing run_id"
|
||||
}
|
||||
|
||||
$outcome = [string]$Events[-1].payload.outcome
|
||||
if ($terminalEvents[0].name -eq "failure" -and $outcome -ne "FAILED") {
|
||||
throw "A failure event must end with outcome FAILED"
|
||||
}
|
||||
if ($terminalEvents[0].name -eq "content" -and $outcome -notin @("SUCCESS", "FALLBACK")) {
|
||||
throw "A content event must end with outcome SUCCESS or FALLBACK"
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null
|
||||
|
||||
$request = Get-Content -Raw -Encoding UTF8 -Path $RequestFile | ConvertFrom-Json
|
||||
$request.Id = $SessionId
|
||||
$body = $request | ConvertTo-Json -Depth 8
|
||||
|
||||
Write-Host "正在运行支付超时 Chat 诊断 Demo..."
|
||||
Write-Host "Running payment-timeout Chat E2E"
|
||||
Write-Host "BaseUrl: $BaseUrl"
|
||||
Write-Host "SessionId: $SessionId"
|
||||
|
||||
$chat = Invoke-RestMethod `
|
||||
$response = Invoke-WebRequest `
|
||||
-UseBasicParsing `
|
||||
-Method Post `
|
||||
-Uri "$BaseUrl/api/chat" `
|
||||
-Headers @{ Accept = "text/event-stream" } `
|
||||
-ContentType "application/json; charset=utf-8" `
|
||||
-Body $body
|
||||
|
||||
$chat | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -Path "$OutputDir/chat-response.json"
|
||||
Write-Host "已保存 Chat 响应: $OutputDir/chat-response.json"
|
||||
$response.Content | Set-Content -Encoding UTF8 -Path "$OutputDir/chat-sse.txt"
|
||||
$events = @(ConvertFrom-NamedSse -Content $response.Content)
|
||||
Assert-ChatSseContract -Events $events -ExpectedSessionId $SessionId
|
||||
$events | ConvertTo-Json -Depth 50 | Set-Content -Encoding UTF8 -Path "$OutputDir/chat-events.json"
|
||||
|
||||
$runId = $chat.data.runId
|
||||
if (-not $runId) {
|
||||
throw "Chat 响应缺少 runId，无法查询精确 Trace。"
|
||||
}
|
||||
$runId = [string]$events[0].payload.run_id
|
||||
Write-Host "RunId: $runId"
|
||||
Write-Host "SSE sequence: $($events.name -join ' -> ')"
|
||||
|
||||
$trace = Invoke-RestMethod `
|
||||
-Method Get `
|
||||
-Uri "$BaseUrl/api/diagnosis/$SessionId/trace?runId=$([System.Uri]::EscapeDataString($runId))"
|
||||
|
||||
$trace | ConvertTo-Json -Depth 50 | Set-Content -Encoding UTF8 -Path "$OutputDir/trace-response.json"
|
||||
Write-Host "已保存 Trace 响应: $OutputDir/trace-response.json"
|
||||
|
||||
$feedbackBody = @{
|
||||
sessionId = $SessionId
|
||||
runId = $runId
|
||||
feedback = "useful"
|
||||
} | ConvertTo-Json
|
||||
|
||||
$feedback = Invoke-RestMethod `
|
||||
-Method Post `
|
||||
-Uri "$BaseUrl/api/feedback" `
|
||||
-ContentType "application/json; charset=utf-8" `
|
||||
-Body $feedbackBody
|
||||
|
||||
$feedback | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -Path "$OutputDir/feedback-response.json"
|
||||
Write-Host "已保存反馈响应: $OutputDir/feedback-response.json"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Demo 已完成，请检查:"
|
||||
Write-Host "- mvp/demo/output/chat-response.json"
|
||||
Write-Host "- mvp/demo/output/trace-response.json"
|
||||
Write-Host "- mvp/demo/output/feedback-response.json"
|
||||
Write-Host "E2E artifacts:"
|
||||
Write-Host "- $OutputDir/chat-sse.txt"
|
||||
Write-Host "- $OutputDir/chat-events.json"
|
||||
Write-Host "- $OutputDir/trace-response.json"
|
||||
|
||||
Reference in New Issue
Block a user