weichen
583c074aa4
ci / vet-and-test (pull_request) Successful in 50s
Why: Missing Usage must stop a paid sequence before a tool terminal is executable; exact attachment names and host-owned choices must also remain reliable. Evidence: Each reported behavior had a failing regression before its fix. Full Backend vet/Go tests, live-tag budget tests, ten race replays, and six task-owned HTTP/Playwright scenarios passed with the aligned Agent-Native and Agent-Squad pins. Impact: Evaluation terminal accounting becomes durable before delivery; material fallback matches bounded filenames; ready choices use host options without redundant model fields. No new model calls or budget authorization. Refs: U3 r0.4 independent review Critical 1 and Important 3-4
530 lines
22 KiB
Go
530 lines
22 KiB
Go
//go:build live
|
|
|
|
package server
|
|
|
|
import (
|
|
bruntime "agent-system-backend/internal/runtime"
|
|
"agent-system-backend/internal/store"
|
|
core "agent-system-core"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
agentllm "agent-native/runtime/llm"
|
|
)
|
|
|
|
func TestEvaluationStopsAfterMalformedToolTerminal(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
for _, tc := range []struct {
|
|
name, arguments, trailer string
|
|
complete bool
|
|
argumentsValid bool
|
|
}{
|
|
{"invalid-json", `{"findings":`, "data: [DONE]\n\n", true, false},
|
|
{"missing-done", `{}`, "", false, true},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
var providerCalls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
args, _ := json.Marshal(tc.arguments)
|
|
fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"publish_work_checkpoint\",\"arguments\":%s}}]}}]}\n\n", args)
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":5,\"total_tokens\":17}}\n\n")
|
|
fmt.Fprint(w, tc.trailer)
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
var terminal agentllm.Event
|
|
for event := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if event.Kind == agentllm.KindMessageEnd {
|
|
terminal = event
|
|
}
|
|
}
|
|
if terminal.Kind != agentllm.KindMessageEnd || terminal.Err == nil || terminal.Usage.TotalTokens != 17 ||
|
|
terminal.StreamComplete != tc.complete || len(terminal.Message.ToolCalls) != 0 {
|
|
t.Fatalf("unsafe provider terminal: %#v", terminal)
|
|
}
|
|
blocked := false
|
|
otherRun := bruntime.WithProviderRunID(context.Background(), t.Name()+"-other-run")
|
|
for event := range stream(otherRun, "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
blocked = blocked || event.Kind == agentllm.KindError
|
|
}
|
|
if !blocked || providerCalls.Load() != 1 {
|
|
t.Fatalf("terminal failure allowed another provider call: blocked=%v calls=%d", blocked, providerCalls.Load())
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var record struct {
|
|
ReportedTokens int `json:"reportedTokens"`
|
|
UsageUnknown bool `json:"usageUnknown"`
|
|
TerminalFault bool `json:"terminalFault"`
|
|
Calls []struct {
|
|
ReportedTokens int `json:"reportedTokens"`
|
|
OutputCap int `json:"outputCap"`
|
|
FinishReason string `json:"finishReason"`
|
|
StreamComplete bool `json:"streamComplete"`
|
|
ToolArgumentBytes int `json:"toolArgumentBytes"`
|
|
ArgumentsValid bool `json:"toolArgumentsValid"`
|
|
UsagePresent bool `json:"usagePresent"`
|
|
} `json:"calls"`
|
|
}
|
|
if err = json.Unmarshal(raw, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if record.ReportedTokens != 17 || record.UsageUnknown || !record.TerminalFault || len(record.Calls) != 1 ||
|
|
record.Calls[0].ReportedTokens != 17 || record.Calls[0].OutputCap != 4000 ||
|
|
record.Calls[0].FinishReason != "tool_calls" || record.Calls[0].StreamComplete != tc.complete ||
|
|
record.Calls[0].ToolArgumentBytes == 0 || record.Calls[0].ArgumentsValid != tc.argumentsValid ||
|
|
!record.Calls[0].UsagePresent {
|
|
t.Fatalf("incomplete diagnostic or lost Usage: %+v", record)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEvaluationKeepsUsageOnTerminalFailure(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
var providerCalls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"publish_work_checkpoint\",\"arguments\":\"{}\"}}]}}],\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":5,\"total_tokens\":17}}\n\n")
|
|
fmt.Fprint(w, "data: {not-json}\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
var terminal agentllm.Event
|
|
for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if ev.Kind == agentllm.KindMessageEnd {
|
|
terminal = ev
|
|
}
|
|
}
|
|
if terminal.Err == nil || terminal.Usage.TotalTokens != 17 {
|
|
t.Fatalf("late stream failure lost returned Usage: %#v", terminal)
|
|
}
|
|
blocked := false
|
|
otherRun := bruntime.WithProviderRunID(context.Background(), t.Name()+"-other-run")
|
|
for ev := range stream(otherRun, "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
blocked = blocked || ev.Kind == agentllm.KindError
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var record struct {
|
|
ReportedTokens int `json:"reportedTokens"`
|
|
UsageUnknown bool `json:"usageUnknown"`
|
|
TerminalFault bool `json:"terminalFault"`
|
|
}
|
|
if err = json.Unmarshal(raw, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !blocked || providerCalls.Load() != 1 || record.ReportedTokens != 17 || record.UsageUnknown || !record.TerminalFault {
|
|
t.Fatalf("late terminal was retried or Usage lost: blocked=%v calls=%d record=%+v", blocked, providerCalls.Load(), record)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationStopsWhenUsageMissing(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
var providerCalls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
for range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
}
|
|
blocked := false
|
|
for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
blocked = blocked || ev.Kind == agentllm.KindError
|
|
}
|
|
if !blocked || providerCalls.Load() != 1 {
|
|
t.Fatalf("missing Usage allowed a second paid request: blocked=%v calls=%d", blocked, providerCalls.Load())
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var record struct {
|
|
UsageUnknown bool `json:"usageUnknown"`
|
|
}
|
|
if err = json.Unmarshal(raw, &record); err != nil || !record.UsageUnknown {
|
|
t.Fatalf("missing Usage was not recorded: %v %+v", err, record)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationDoesNotForwardToolTerminalBeforeUsageRecorded(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
dir, _ := newEvaluationEventStore(t)
|
|
var calls atomic.Int32
|
|
release := make(chan struct{})
|
|
provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
|
|
calls.Add(1)
|
|
out := make(chan agentllm.Event)
|
|
go func() {
|
|
defer close(out)
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant,
|
|
StopReason: agentllm.StopToolCalls, ToolCalls: []agentllm.ToolCall{{ID: "call-1", Name: "publish_work_checkpoint", Arguments: `{}`}}},
|
|
FinishReason: "tool_calls", StreamComplete: true}
|
|
<-release // keep the provider stream open after its terminal
|
|
}()
|
|
return out
|
|
}
|
|
stream := evalProviderStreamWithProvider(provider, dir)
|
|
time.AfterFunc(100*time.Millisecond, func() { close(release) })
|
|
var terminal agentllm.Event
|
|
first := stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000})
|
|
for ev := range first {
|
|
if ev.Kind == agentllm.KindMessageEnd {
|
|
terminal = ev
|
|
break
|
|
}
|
|
}
|
|
blocked := false
|
|
for ev := range stream(bruntime.WithProviderRunID(context.Background(), t.Name()+"-next"), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
blocked = blocked || ev.Kind == agentllm.KindError
|
|
}
|
|
for range first {
|
|
}
|
|
if terminal.Err == nil || len(terminal.Message.ToolCalls) != 0 || !blocked || calls.Load() != 1 {
|
|
t.Fatalf("missing Usage tool terminal escaped stop gate: terminal=%#v blocked=%v providerCalls=%d", terminal, blocked, calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestEvaluationStopsWhenProviderRecordUnavailable(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
var providerCalls atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls.Add(1)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
if err := os.Mkdir(filepath.Join(dir, "provider-records.json"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
var terminal agentllm.Event
|
|
for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if ev.Kind == agentllm.KindMessageEnd {
|
|
terminal = ev
|
|
}
|
|
}
|
|
if terminal.Usage.TotalTokens != 2 {
|
|
t.Fatalf("control provider Usage missing: %#v", terminal)
|
|
}
|
|
blocked := false
|
|
for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
blocked = blocked || ev.Kind == agentllm.KindError
|
|
}
|
|
if !blocked || providerCalls.Load() != 1 {
|
|
t.Fatalf("missing durable record allowed another request: blocked=%v calls=%d", blocked, providerCalls.Load())
|
|
}
|
|
}
|
|
|
|
func TestEvaluationGroupsChangingPromptsByActualRunID(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls++
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
ctx := bruntime.WithProviderRunID(context.Background(), "same-real-run")
|
|
for _, user := range []string{"first prompt", "later prompt changed by runtime"} {
|
|
for range stream(ctx, "fixture", []agentllm.Message{{Role: agentllm.RoleSystem, Content: "stable"}, {Role: agentllm.RoleUser, Content: user}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
}
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var record struct {
|
|
ModelRuns int `json:"modelRuns"`
|
|
Calls []struct {
|
|
RunID string `json:"runId"`
|
|
RunKey string `json:"runKey"`
|
|
Call int `json:"call"`
|
|
} `json:"calls"`
|
|
}
|
|
if err := json.Unmarshal(raw, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if providerCalls != 2 || record.ModelRuns != 1 || len(record.Calls) != 2 || record.Calls[0].RunID != "same-real-run" || record.Calls[1].RunID != "same-real-run" || record.Calls[0].RunKey != record.Calls[1].RunKey || record.Calls[0].Call != 1 || record.Calls[1].Call != 2 {
|
|
t.Fatalf("one real run split into evaluator groups: provider=%d record=%+v", providerCalls, record)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationRefusesRequestWithoutActualRunID(t *testing.T) {
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { providerCalls++ }))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
rejected := false
|
|
for event := range stream(context.Background(), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "no run identity"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
rejected = rejected || event.Kind == agentllm.KindError
|
|
}
|
|
if !rejected || providerCalls != 0 {
|
|
t.Fatalf("missing run ID reached provider: rejected=%v calls=%d", rejected, providerCalls)
|
|
}
|
|
}
|
|
|
|
func evalTestContext(t *testing.T) context.Context {
|
|
t.Helper()
|
|
return bruntime.WithProviderRunID(context.Background(), t.Name())
|
|
}
|
|
|
|
func TestMechanismSourceExtractionAfterSourceOnce(t *testing.T) {
|
|
id := "src_" + strings.Repeat("a", 24)
|
|
got := mechanismCheckpointSources("sourceId=" + id + " blockIndex=0\n上线六月。验收条件需要补齐")
|
|
if len(got) != 1 || got[0][1] != id || got[0][2] != "0" || !strings.Contains(got[0][3], "上线六月") {
|
|
t.Fatalf("inline source unavailable to mechanism fixture: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationFixtureUsesEffectiveNumericOutputCap(t *testing.T) {
|
|
settings := core.EffectiveSettings(evalFixtureSettings())
|
|
if got := settings.GetNumber("models.chat.maxOutputTokens", 0); got != 4000 {
|
|
t.Fatalf("mechanism output cap is not effective: got %v, want 4000", got)
|
|
}
|
|
if got := settings.GetNumber("runtime.llmRetries", 2); got != 0 {
|
|
t.Fatalf("fixture disabled-retry setting is not effective: got %v, want 0", got)
|
|
}
|
|
}
|
|
|
|
func newEvaluationEventStore(t *testing.T) (string, *sql.DB) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
db, err := sql.Open("sqlite", filepath.Join(dir, "agent-system.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = db.Exec(`CREATE TABLE events (conversation_id TEXT, run_id TEXT, seq INTEGER, type TEXT, payload TEXT)`); err != nil {
|
|
db.Close()
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
return dir, db
|
|
}
|
|
|
|
func TestExtraA01LimitsActualRunIDs(t *testing.T) {
|
|
_, st := newTestServer(t)
|
|
p, err := st.CreateProject(store.OwnerDefault, "allowance")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, err := st.CreateConversation(store.OwnerDefault, p.ID, "allowance", "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, kind := range []core.RunKind{core.RunConversation, core.RunSubagent} {
|
|
r := core.Run{Owner: c.Owner, ProjectID: p.ID, ConversationID: c.ID, Kind: kind, Status: core.RunRunning}
|
|
if err = st.CreateRun(&r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
calls := 0
|
|
guard := limitExtraA01Runs(st, func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
|
|
calls++
|
|
out := make(chan agentllm.Event)
|
|
close(out)
|
|
return out
|
|
})
|
|
for range guard(context.Background(), "fixture", nil, nil, agentllm.StreamOptions{}) {
|
|
}
|
|
if calls != 1 {
|
|
t.Fatal("two actual runs were blocked")
|
|
}
|
|
extra := core.Run{Owner: c.Owner, ProjectID: p.ID, ConversationID: c.ID, Kind: core.RunSubagent, Status: core.RunRunning}
|
|
if err = st.CreateRun(&extra); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for event := range guard(context.Background(), "fixture", nil, nil, agentllm.StreamOptions{}) {
|
|
if event.Err == nil {
|
|
t.Fatal("missing allowance error")
|
|
}
|
|
}
|
|
if calls != 1 {
|
|
t.Fatal("a third actual run made a paid request")
|
|
}
|
|
}
|
|
|
|
func TestFollowupA01UsesNewBoundedAuthorization(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
label string
|
|
tokens int
|
|
guard bool
|
|
}{
|
|
{"", 20000, false},
|
|
{"approved-20261013", 20000, true},
|
|
{"approved-followup-20261013", 30000, true},
|
|
{"approved-r02-20261013", 30000, true},
|
|
{"approved-r03-20261013", 30000, true},
|
|
{"unapproved", 20000, false},
|
|
} {
|
|
tokens, guard := evalA01Budget(tc.label)
|
|
if tokens != tc.tokens || guard != tc.guard {
|
|
t.Fatalf("label %q: got budget=%d guard=%v; want budget=%d guard=%v", tc.label, tokens, guard, tc.tokens, tc.guard)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEvaluationPreservesProductionOutputCap(t *testing.T) {
|
|
for _, tc := range []struct{ supplied, want int }{{780, 780}, {5000, 4000}, {0, 4000}} {
|
|
t.Run(fmt.Sprint(tc.supplied), func(t *testing.T) {
|
|
got := make(chan int, 1)
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
MaxTokens int `json:"max_tokens"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
t.Error(err)
|
|
}
|
|
got <- body.MaxTokens
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
for range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "check cap"}}, nil, agentllm.StreamOptions{MaxTokens: tc.supplied}) {
|
|
}
|
|
if actual := <-got; actual != tc.want {
|
|
t.Fatalf("evaluation overwrote production cap: got %d want %d", actual, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEvaluationStopsPaidRequestsAfterRejectedCheckpoint(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls++
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
dir, _ := newEvaluationEventStore(t)
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
for _, messages := range [][]agentllm.Message{
|
|
{{Role: agentllm.RoleTool, Name: "publish_work_checkpoint", IsError: true, Content: "阶段发现未发布：checkpoint.source: quote not in provided block"}},
|
|
{{Role: agentllm.RoleUser, Content: "another request after rejection"}},
|
|
} {
|
|
rejected := false
|
|
for event := range stream(evalTestContext(t), "fixture", messages, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if event.Kind == agentllm.KindError {
|
|
rejected = true
|
|
}
|
|
}
|
|
if !rejected {
|
|
t.Fatal("rejected checkpoint did not stop the controlled sample")
|
|
}
|
|
}
|
|
if providerCalls != 0 {
|
|
t.Fatalf("%d paid provider calls started after checkpoint rejection", providerCalls)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationStopsAcrossRunsAfterPersistedCheckpointRejection(t *testing.T) {
|
|
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
|
|
dir, db := newEvaluationEventStore(t)
|
|
_, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 1, "tool.call", `{"callId":"rejected-call","name":"publish_work_checkpoint"}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 2, "tool.result", `{"callId":"rejected-call","status":"error","summary":"json: unknown field \"$text\""}`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls++
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
rejected := false
|
|
for event := range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "unrelated foreground history"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if event.Kind == agentllm.KindError {
|
|
rejected = true
|
|
}
|
|
}
|
|
if !rejected || providerCalls != 0 {
|
|
t.Fatalf("persisted background rejection allowed foreground provider request: error=%v calls=%d", rejected, providerCalls)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationFailsClosedWhenCheckpointEventsUnavailable(t *testing.T) {
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls++
|
|
}))
|
|
defer srv.Close()
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, t.TempDir())
|
|
rejected := false
|
|
for event := range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "request without the event store"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
if event.Kind == agentllm.KindError {
|
|
rejected = true
|
|
}
|
|
}
|
|
if !rejected || providerCalls != 0 {
|
|
t.Fatalf("missing checkpoint event store allowed provider request: error=%v calls=%d", rejected, providerCalls)
|
|
}
|
|
}
|
|
|
|
func TestEvaluationMatchesCheckpointRejectionWithinItsRun(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, tool, resultRun string
|
|
}{
|
|
{"different tool", "read_file", "child"},
|
|
{"different run", "publish_work_checkpoint", "other-run"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dir, db := newEvaluationEventStore(t)
|
|
if _, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 1, "tool.call", fmt.Sprintf(`{"callId":"shared","name":%q}`, tc.tool)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", tc.resultRun, 2, "tool.result", `{"callId":"shared","status":"error"}`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
providerCalls := 0
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
providerCalls++
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
|
|
}))
|
|
defer srv.Close()
|
|
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
|
|
for range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "unrelated request"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
}
|
|
if providerCalls != 1 {
|
|
t.Fatalf("unrelated tool error blocked the provider: calls=%d", providerCalls)
|
|
}
|
|
})
|
|
}
|
|
}
|