Agent-System-Backend/internal/server/work_checkpoint_eval_budget_test.go

//go:build live

package server

import (
	bruntime "agent-system-backend/internal/runtime"
	"agent-system-backend/internal/store"
	core "agent-system-core"
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	agentllm "agent-native/runtime/llm"
)

func TestEvaluationRecordsContentFreeTerminal(t *testing.T) {
	dir, _ := newEvaluationEventStore(t)
	args := `{"secret":"PRIVATE_ARGUMENT_R05"}`
	provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		out := make(chan agentllm.Event, 2)
		out <- agentllm.Event{Kind: agentllm.KindToolCallDelta, ToolIndex: 0, ToolID: "call-1", ToolName: "PRIVATE_TOOL_R05", ArgumentsDelta: args}
		out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
			Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "PRIVATE_BODY_R05", StopReason: agentllm.StopToolCalls,
				ToolCalls: []agentllm.ToolCall{{ID: "call-1", Name: "PRIVATE_TOOL_R05", Arguments: args}}},
			Usage:        agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17, CacheReadTokens: 3},
			FinishReason: "tool_calls", StreamComplete: true, HTTPStatus: http.StatusOK,
			TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", LastFrameJSONComplete: true,
				UsagePresent: true, ToolFragments: 1, ToolArgumentBytes: len(args), ToolArgumentsValid: true},
		}
		close(out)
		return out
	}
	stream := evalProviderStreamWithProvider(provider, dir)
	for range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "PRIVATE_PROMPT_R05"}},
		[]agentllm.ToolDecl{{Name: "PRIVATE_TOOL_R05", Description: "PRIVATE_SCHEMA_R05"}}, agentllm.StreamOptions{MaxTokens: 4000}) {
	}
	path := filepath.Join(dir, "provider-records.json")
	raw, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	for _, forbidden := range []string{"PRIVATE_PROMPT_R05", "PRIVATE_ARGUMENT_R05", "PRIVATE_BODY_R05", "PRIVATE_TOOL_R05", "PRIVATE_SCHEMA_R05",
		`"messages"`, `"toolDeclarations"`, `"terminal"`} {
		if strings.Contains(string(raw), forbidden) {
			t.Fatalf("provider record retained %q", forbidden)
		}
	}
	var record struct {
		Calls []struct {
			RunID          string                        `json:"runId"`
			ReportedTokens int                           `json:"reportedTokens"`
			Usage          agentllm.Usage                `json:"usage"`
			Observation    *agentllm.TerminalObservation `json:"terminalObservation"`
		} `json:"calls"`
	}
	if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].RunID == "" ||
		record.Calls[0].ReportedTokens != 17 || record.Calls[0].Usage.CacheReadTokens != 3 ||
		record.Calls[0].Observation == nil || record.Calls[0].Observation.ReadOutcome != "done_marker" {
		t.Fatalf("content-free record lost accounting: %+v err=%v", record, err)
	}
	info, err := os.Stat(path)
	if err != nil || info.Mode().Perm() != 0600 {
		t.Fatalf("provider record permission = %v err=%v", info, err)
	}
}

func TestEvaluationRedactsProviderControlledTerminalLabels(t *testing.T) {
	dir, _ := newEvaluationEventStore(t)
	provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		out := make(chan agentllm.Event, 1)
		out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
			Message:      agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
			Usage:        agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
			FinishReason: "PRIVATE_REASON_R05", StreamComplete: true,
			TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "PRIVATE_OUTCOME_R05", UsagePresent: true},
		}
		close(out)
		return out
	}
	for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(raw), "PRIVATE_REASON_R05") || strings.Contains(string(raw), "PRIVATE_OUTCOME_R05") {
		t.Fatal("provider-controlled terminal label escaped into a content-free record")
	}
	var record struct {
		Calls []struct {
			FinishReason string                        `json:"finishReason"`
			Observation  *agentllm.TerminalObservation `json:"terminalObservation"`
		} `json:"calls"`
	}
	if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 ||
		record.Calls[0].FinishReason != "unknown" || record.Calls[0].Observation == nil ||
		record.Calls[0].Observation.ReadOutcome != "unknown" {
		t.Fatalf("unexpected sanitized terminal labels: %+v err=%v", record, err)
	}
}

func TestEvaluationRedactsErrorTerminalAndBlocksTool(t *testing.T) {
	dir, _ := newEvaluationEventStore(t)
	var calls atomic.Int32
	provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		calls.Add(1)
		out := make(chan agentllm.Event, 1)
		out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
			Message: agentllm.Message{Role: agentllm.RoleAssistant, StopReason: agentllm.StopError,
				ToolCalls: []agentllm.ToolCall{{ID: "call-1", Name: "PRIVATE_TOOL_R05", Arguments: `{"secret":"PRIVATE_ARGUMENT_R05"}`}}},
			Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
			Err:   errors.New("PRIVATE_ERROR_R05"), ErrorMessage: "PRIVATE_ERROR_R05",
			FinishReason: "tool_calls", StreamComplete: false,
			TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "read_error", UsagePresent: true,
				ToolFragments: 1, ToolArgumentBytes: 33, ToolArgumentsValid: true},
		}
		close(out)
		return out
	}
	stream := evalProviderStreamWithProvider(provider, dir)
	var terminal agentllm.Event
	for event := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if event.Kind == agentllm.KindMessageEnd {
			terminal = event
		}
	}
	if terminal.Err == nil || terminal.Usage.TotalTokens != 17 || len(terminal.Message.ToolCalls) != 0 {
		t.Fatalf("error-bearing tool call escaped the controlled terminal: %#v", terminal)
	}
	blocked := false
	for event := range stream(bruntime.WithProviderRunID(context.Background(), t.Name()+"-next"), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || event.Kind == agentllm.KindError
	}
	if !blocked || calls.Load() != 1 {
		t.Fatalf("failed terminal allowed another provider call: blocked=%v calls=%d", blocked, calls.Load())
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(raw), "PRIVATE_ERROR_R05") || strings.Contains(string(raw), "PRIVATE_ARGUMENT_R05") ||
		strings.Contains(string(raw), `"terminal"`) {
		t.Fatal("error record retained raw provider content")
	}
	var record struct {
		ReportedTokens int `json:"reportedTokens"`
		Calls          []struct {
			FailureClass string `json:"failureClass"`
		} `json:"calls"`
	}
	if err := json.Unmarshal(raw, &record); err != nil || record.ReportedTokens != 17 || len(record.Calls) != 1 || record.Calls[0].FailureClass != "terminal_error" {
		t.Fatalf("failed terminal accounting/classification missing: %+v err=%v", record, err)
	}
}

func TestEvaluationStopsAfterMalformedToolTerminal(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	for _, tc := range []struct {
		name, arguments, trailer string
		complete                 bool
		argumentsValid           bool
	}{
		{"invalid-json", `{"findings":`, "data: [DONE]\n\n", true, false},
		{"missing-done", `{}`, "", false, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var providerCalls atomic.Int32
			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				providerCalls.Add(1)
				w.Header().Set("Content-Type", "text/event-stream")
				args, _ := json.Marshal(tc.arguments)
				fmt.Fprintf(w, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"publish_work_checkpoint\",\"arguments\":%s}}]}}]}\n\n", args)
				fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":5,\"total_tokens\":17}}\n\n")
				fmt.Fprint(w, tc.trailer)
			}))
			defer srv.Close()
			dir, _ := newEvaluationEventStore(t)
			stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
			var terminal agentllm.Event
			for event := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
				if event.Kind == agentllm.KindMessageEnd {
					terminal = event
				}
			}
			if terminal.Kind != agentllm.KindMessageEnd || terminal.Err == nil || terminal.Usage.TotalTokens != 17 ||
				terminal.StreamComplete != tc.complete || len(terminal.Message.ToolCalls) != 0 {
				t.Fatalf("unsafe provider terminal: %#v", terminal)
			}
			blocked := false
			otherRun := bruntime.WithProviderRunID(context.Background(), t.Name()+"-other-run")
			for event := range stream(otherRun, "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
				blocked = blocked || event.Kind == agentllm.KindError
			}
			if !blocked || providerCalls.Load() != 1 {
				t.Fatalf("terminal failure allowed another provider call: blocked=%v calls=%d", blocked, providerCalls.Load())
			}
			raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
			if err != nil {
				t.Fatal(err)
			}
			var record struct {
				ReportedTokens int  `json:"reportedTokens"`
				UsageUnknown   bool `json:"usageUnknown"`
				TerminalFault  bool `json:"terminalFault"`
				Calls          []struct {
					ReportedTokens    int                           `json:"reportedTokens"`
					OutputCap         int                           `json:"outputCap"`
					FinishReason      string                        `json:"finishReason"`
					StreamComplete    bool                          `json:"streamComplete"`
					ToolArgumentBytes int                           `json:"toolArgumentBytes"`
					ArgumentsValid    bool                          `json:"toolArgumentsValid"`
					UsagePresent      bool                          `json:"usagePresent"`
					Usage             agentllm.Usage                `json:"usage"`
					HTTPStatus        int                           `json:"httpStatus"`
					Observation       *agentllm.TerminalObservation `json:"terminalObservation"`
					FailureClass      string                        `json:"failureClass"`
				} `json:"calls"`
			}
			if err = json.Unmarshal(raw, &record); err != nil {
				t.Fatal(err)
			}
			wantOutcome := "done_marker"
			if !tc.complete {
				wantOutcome = "clean_eof"
			}
			if record.ReportedTokens != 17 || record.UsageUnknown || !record.TerminalFault || len(record.Calls) != 1 ||
				record.Calls[0].ReportedTokens != 17 || record.Calls[0].OutputCap != 4000 ||
				record.Calls[0].FinishReason != "tool_calls" || record.Calls[0].StreamComplete != tc.complete ||
				record.Calls[0].ToolArgumentBytes == 0 || record.Calls[0].ArgumentsValid != tc.argumentsValid ||
				!record.Calls[0].UsagePresent || record.Calls[0].Usage.TotalTokens != 17 ||
				record.Calls[0].HTTPStatus != http.StatusOK || record.Calls[0].FailureClass != "terminal_error" ||
				record.Calls[0].Observation == nil || record.Calls[0].Observation.ReadOutcome != wantOutcome ||
				record.Calls[0].Observation.ToolArgumentsValid != tc.argumentsValid {
				t.Fatalf("incomplete diagnostic or lost Usage: %+v", record)
			}
		})
	}
}

func TestEvaluationTransportObservationSanitized(t *testing.T) {
	const maxTail, maxBytes = 10000, int64(32*1024*1024 + 1)
	const maxFragments, maxArgumentBytes = 10000, 4*1024*1024 + 1
	for _, tc := range []struct {
		name          string
		input         agentllm.TerminalObservation
		wantTail      int
		wantBytes     int64
		wantFragments int
		wantArgBytes  int
		wantKind      string
		wantFields    [3]string
	}{
		{"negative-tail-oversized-body", agentllm.TerminalObservation{ReadOutcome: "done_marker",
			TailEventKind: "PRIVATE_TAIL_R06", TailEventCount: -5, ToolFragments: -3, ToolArgumentBytes: 1 << 30,
			BodyBoundary: &agentllm.BodyBoundaryObservation{ProtocolClass: "PRIVATE_HEADER_R06", FramingClass: "PRIVATE_FRAMING_R06",
				LengthRelation: "PRIVATE_LENGTH_R06", BytesRead: 1 << 40, DeclaredLengthKnown: true, BytesSaturated: true, EOFSeen: true, Uncompressed: true}},
			0, maxBytes, 0, maxArgumentBytes, "unknown", [3]string{"other", "unknown", "unknown"}},
		{"oversized-tail-negative-body", agentllm.TerminalObservation{ReadOutcome: "done_marker",
			TailEventKind: "PRIVATE_TAIL_R06", TailEventCount: 1 << 20, ToolFragments: 1 << 20, ToolArgumentBytes: -9,
			BodyBoundary: &agentllm.BodyBoundaryObservation{ProtocolClass: "PRIVATE_HEADER_R06", FramingClass: "PRIVATE_FRAMING_R06",
				LengthRelation: "PRIVATE_LENGTH_R06", BytesRead: -7, DeclaredLengthKnown: true, BytesSaturated: true, EOFSeen: true, Uncompressed: true}},
			maxTail, 0, maxFragments, 0, "unknown", [3]string{"other", "unknown", "unknown"}},
		{"known-values-kept", agentllm.TerminalObservation{ReadOutcome: "done_marker",
			TailEventKind: "comment", TailEventCount: 3, ToolFragments: 2, ToolArgumentBytes: 64,
			BodyBoundary: &agentllm.BodyBoundaryObservation{ProtocolClass: "http2", FramingClass: "chunked",
				LengthRelation: "match", BytesRead: 128, DeclaredLengthKnown: true, BytesSaturated: true, EOFSeen: true, Uncompressed: true}},
			3, 128, 2, 64, "comment", [3]string{"http2", "chunked", "match"}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			input := tc.input
			inputBody := *input.BodyBoundary
			input.BodyBoundary = &inputBody
			clean := evalTerminalObservation(&input)
			if clean == nil || clean.BodyBoundary == nil || clean.BodyBoundary == input.BodyBoundary {
				t.Fatalf("observation was not deep-copied: %+v", clean)
			}
			body := clean.BodyBoundary
			if clean.ReadOutcome != "done_marker" || clean.TailEventKind != tc.wantKind || clean.TailEventCount != tc.wantTail ||
				clean.ToolFragments != tc.wantFragments || clean.ToolArgumentBytes != tc.wantArgBytes ||
				body.ProtocolClass != tc.wantFields[0] || body.FramingClass != tc.wantFields[1] || body.LengthRelation != tc.wantFields[2] ||
				body.BytesRead != tc.wantBytes || !body.DeclaredLengthKnown || !body.BytesSaturated || !body.EOFSeen || !body.Uncompressed {
				t.Fatalf("unbounded observation: %+v body=%+v", clean, body)
			}
			input.BodyBoundary.ProtocolClass = "PRIVATE_LATE_R06"
			if body.ProtocolClass == "PRIVATE_LATE_R06" {
				t.Fatal("sanitized observation shares the provider body boundary")
			}

			dir, _ := newEvaluationEventStore(t)
			observed := tc.input
			observedBody := *observed.BodyBoundary
			observed.BodyBoundary = &observedBody
			provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
				out := make(chan agentllm.Event, 1)
				out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
					Message:      agentllm.Message{Role: agentllm.RoleAssistant, Content: "PRIVATE_BODY_R06", StopReason: agentllm.StopEnd},
					Usage:        agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
					FinishReason: "stop", StreamComplete: true, HTTPStatus: http.StatusOK, TerminalObservation: &observed,
				}
				close(out)
				return out
			}
			for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture",
				[]agentllm.Message{{Role: agentllm.RoleUser, Content: "PRIVATE_PROMPT_R06"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
			}
			raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
			if err != nil {
				t.Fatal(err)
			}
			for _, forbidden := range []string{"PRIVATE_TAIL_R06", "PRIVATE_HEADER_R06", "PRIVATE_FRAMING_R06", "PRIVATE_LENGTH_R06",
				"PRIVATE_BODY_R06", "PRIVATE_PROMPT_R06", `"messages"`, `"terminal"`} {
				if strings.Contains(string(raw), forbidden) {
					t.Fatalf("provider record retained %q", forbidden)
				}
			}
			var record struct {
				Calls []struct {
					RunID       string                        `json:"runId"`
					Usage       agentllm.Usage                `json:"usage"`
					Observation *agentllm.TerminalObservation `json:"terminalObservation"`
				} `json:"calls"`
			}
			if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].RunID != t.Name() ||
				record.Calls[0].Usage.TotalTokens != 17 || record.Calls[0].Observation == nil || record.Calls[0].Observation.BodyBoundary == nil {
				t.Fatalf("sanitized record lost accounting: %s err=%v", raw, err)
			}
			got := record.Calls[0].Observation
			if got.TailEventKind != tc.wantKind || got.TailEventCount != tc.wantTail || got.ToolFragments != tc.wantFragments ||
				got.ToolArgumentBytes != tc.wantArgBytes || got.BodyBoundary.ProtocolClass != tc.wantFields[0] ||
				got.BodyBoundary.FramingClass != tc.wantFields[1] || got.BodyBoundary.LengthRelation != tc.wantFields[2] ||
				got.BodyBoundary.BytesRead != tc.wantBytes {
				t.Fatalf("unbounded persisted observation: %s", raw)
			}
		})
	}
}

func TestEvaluationTransportEvidenceStopsAfterUnmarkedToolEOF(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	dir, _ := newEvaluationEventStore(t)
	var providerCalls atomic.Int32
	provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		providerCalls.Add(1)
		out := make(chan agentllm.Event, 2)
		out <- agentllm.Event{Kind: agentllm.KindToolCallDelta, ToolIndex: 0, ToolID: "call_1", ToolName: "publish_work_checkpoint", ArgumentsDelta: `{}`}
		out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
			Message: agentllm.Message{Role: agentllm.RoleAssistant, StopReason: agentllm.StopError},
			Usage:   agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
			Err:     errors.New("PRIVATE_ERROR_R06"), ErrorMessage: "PRIVATE_ERROR_R06",
			FinishReason: "tool_calls", StreamComplete: false, HTTPStatus: http.StatusOK,
			TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "clean_eof", LastFrameJSONComplete: true, UsagePresent: true,
				ToolFragments: 1, ToolArgumentBytes: 2, ToolArgumentsValid: true, TailEventKind: "json_frame", TailEventCount: 2,
				BodyBoundary: &agentllm.BodyBoundaryObservation{ProtocolClass: "http1", FramingClass: "chunked", BytesRead: 256,
					EOFSeen: true, LengthRelation: "unknown"}},
		}
		close(out)
		return out
	}
	stream := evalProviderStreamWithProvider(provider, dir)
	var terminal agentllm.Event
	for event := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if event.Kind == agentllm.KindMessageEnd {
			terminal = event
		}
	}
	if terminal.Kind != agentllm.KindMessageEnd || terminal.Err == nil || terminal.StreamComplete ||
		terminal.Usage.TotalTokens != 17 || len(terminal.Message.ToolCalls) != 0 {
		t.Fatalf("unmarked tool EOF escaped the controlled terminal: %#v", terminal)
	}
	blocked := false
	for event := range stream(bruntime.WithProviderRunID(context.Background(), t.Name()+"-next"), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || event.Kind == agentllm.KindError
	}
	if !blocked || providerCalls.Load() != 1 {
		t.Fatalf("transport EOF allowed another provider call: blocked=%v calls=%d", blocked, providerCalls.Load())
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(raw), "PRIVATE_ERROR_R06") || strings.Contains(string(raw), `"terminal"`) {
		t.Fatal("transport evidence record retained raw provider content")
	}
	var record struct {
		ReportedTokens int  `json:"reportedTokens"`
		TerminalFault  bool `json:"terminalFault"`
		Calls          []struct {
			Usage          agentllm.Usage                `json:"usage"`
			StreamComplete bool                          `json:"streamComplete"`
			FailureClass   string                        `json:"failureClass"`
			Observation    *agentllm.TerminalObservation `json:"terminalObservation"`
		} `json:"calls"`
	}
	if err := json.Unmarshal(raw, &record); err != nil || record.ReportedTokens != 17 || !record.TerminalFault || len(record.Calls) != 1 ||
		record.Calls[0].Usage.TotalTokens != 17 || record.Calls[0].StreamComplete || record.Calls[0].FailureClass != "terminal_error" {
		t.Fatalf("transport EOF accounting missing: %s err=%v", raw, err)
	}
	got := record.Calls[0].Observation
	if got == nil || got.ReadOutcome != "clean_eof" || got.TailEventKind != "json_frame" || got.TailEventCount != 2 ||
		got.BodyBoundary == nil || !got.BodyBoundary.EOFSeen || got.BodyBoundary.FramingClass != "chunked" {
		t.Fatalf("transport evidence missing from failed terminal record: %s", raw)
	}
}

func TestEvaluationKeepsUsageOnTerminalFailure(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	var providerCalls atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls.Add(1)
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"publish_work_checkpoint\",\"arguments\":\"{}\"}}]}}],\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":5,\"total_tokens\":17}}\n\n")
		fmt.Fprint(w, "data: {not-json}\n\n")
	}))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	var terminal agentllm.Event
	for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if ev.Kind == agentllm.KindMessageEnd {
			terminal = ev
		}
	}
	if terminal.Err == nil || terminal.Usage.TotalTokens != 17 {
		t.Fatalf("late stream failure lost returned Usage: %#v", terminal)
	}
	blocked := false
	otherRun := bruntime.WithProviderRunID(context.Background(), t.Name()+"-other-run")
	for ev := range stream(otherRun, "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || ev.Kind == agentllm.KindError
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	var record struct {
		ReportedTokens int  `json:"reportedTokens"`
		UsageUnknown   bool `json:"usageUnknown"`
		TerminalFault  bool `json:"terminalFault"`
	}
	if err = json.Unmarshal(raw, &record); err != nil {
		t.Fatal(err)
	}
	if !blocked || providerCalls.Load() != 1 || record.ReportedTokens != 17 || record.UsageUnknown || !record.TerminalFault {
		t.Fatalf("late terminal was retried or Usage lost: blocked=%v calls=%d record=%+v", blocked, providerCalls.Load(), record)
	}
}

func TestEvaluationStopsWhenUsageMissing(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	var providerCalls atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls.Add(1)
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n")
	}))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	for range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
	}
	blocked := false
	for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || ev.Kind == agentllm.KindError
	}
	if !blocked || providerCalls.Load() != 1 {
		t.Fatalf("missing Usage allowed a second paid request: blocked=%v calls=%d", blocked, providerCalls.Load())
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	var record struct {
		UsageUnknown bool `json:"usageUnknown"`
	}
	if err = json.Unmarshal(raw, &record); err != nil || !record.UsageUnknown {
		t.Fatalf("missing Usage was not recorded: %v %+v", err, record)
	}
}

func TestEvaluationDoesNotForwardToolTerminalBeforeUsageRecorded(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	dir, _ := newEvaluationEventStore(t)
	var calls atomic.Int32
	release := make(chan struct{})
	provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		calls.Add(1)
		out := make(chan agentllm.Event)
		go func() {
			defer close(out)
			out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant,
				StopReason: agentllm.StopToolCalls, ToolCalls: []agentllm.ToolCall{{ID: "call-1", Name: "publish_work_checkpoint", Arguments: `{}`}}},
				FinishReason: "tool_calls", StreamComplete: true}
			<-release // keep the provider stream open after its terminal
		}()
		return out
	}
	stream := evalProviderStreamWithProvider(provider, dir)
	time.AfterFunc(100*time.Millisecond, func() { close(release) })
	var terminal agentllm.Event
	first := stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000})
	for ev := range first {
		if ev.Kind == agentllm.KindMessageEnd {
			terminal = ev
			break
		}
	}
	blocked := false
	for ev := range stream(bruntime.WithProviderRunID(context.Background(), t.Name()+"-next"), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || ev.Kind == agentllm.KindError
	}
	for range first {
	}
	if terminal.Err == nil || len(terminal.Message.ToolCalls) != 0 || !blocked || calls.Load() != 1 {
		t.Fatalf("missing Usage tool terminal escaped stop gate: terminal=%#v blocked=%v providerCalls=%d", terminal, blocked, calls.Load())
	}
}

func TestEvaluationStopsWhenProviderRecordUnavailable(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	var providerCalls atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls.Add(1)
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
	}))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	if err := os.Mkdir(filepath.Join(dir, "provider-records.json"), 0700); err != nil {
		t.Fatal(err)
	}
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	var terminal agentllm.Event
	for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if ev.Kind == agentllm.KindMessageEnd {
			terminal = ev
		}
	}
	if terminal.Usage.TotalTokens != 2 {
		t.Fatalf("control provider Usage missing: %#v", terminal)
	}
	blocked := false
	for ev := range stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		blocked = blocked || ev.Kind == agentllm.KindError
	}
	if !blocked || providerCalls.Load() != 1 {
		t.Fatalf("missing durable record allowed another request: blocked=%v calls=%d", blocked, providerCalls.Load())
	}
}

func TestEvaluationGroupsChangingPromptsByActualRunID(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	providerCalls := 0
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls++
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
	}))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	ctx := bruntime.WithProviderRunID(context.Background(), "same-real-run")
	for _, user := range []string{"first prompt", "later prompt changed by runtime"} {
		for range stream(ctx, "fixture", []agentllm.Message{{Role: agentllm.RoleSystem, Content: "stable"}, {Role: agentllm.RoleUser, Content: user}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		}
	}
	raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
	if err != nil {
		t.Fatal(err)
	}
	var record struct {
		ModelRuns int `json:"modelRuns"`
		Calls     []struct {
			RunID  string `json:"runId"`
			RunKey string `json:"runKey"`
			Call   int    `json:"call"`
		} `json:"calls"`
	}
	if err := json.Unmarshal(raw, &record); err != nil {
		t.Fatal(err)
	}
	if providerCalls != 2 || record.ModelRuns != 1 || len(record.Calls) != 2 || record.Calls[0].RunID != "same-real-run" || record.Calls[1].RunID != "same-real-run" || record.Calls[0].RunKey != record.Calls[1].RunKey || record.Calls[0].Call != 1 || record.Calls[1].Call != 2 {
		t.Fatalf("one real run split into evaluator groups: provider=%d record=%+v", providerCalls, record)
	}
}

func TestEvaluationRefusesRequestWithoutActualRunID(t *testing.T) {
	providerCalls := 0
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { providerCalls++ }))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	rejected := false
	for event := range stream(context.Background(), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "no run identity"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		rejected = rejected || event.Kind == agentllm.KindError
	}
	if !rejected || providerCalls != 0 {
		t.Fatalf("missing run ID reached provider: rejected=%v calls=%d", rejected, providerCalls)
	}
}

func evalTestContext(t *testing.T) context.Context {
	t.Helper()
	return bruntime.WithProviderRunID(context.Background(), t.Name())
}

func TestMechanismSourceExtractionAfterSourceOnce(t *testing.T) {
	id := "src_" + strings.Repeat("a", 24)
	got := mechanismCheckpointSources("sourceId=" + id + " blockIndex=0\n上线六月。验收条件需要补齐")
	if len(got) != 1 || got[0][1] != id || got[0][2] != "0" || !strings.Contains(got[0][3], "上线六月") {
		t.Fatalf("inline source unavailable to mechanism fixture: %v", got)
	}
}

func TestEvaluationFixtureUsesEffectiveNumericOutputCap(t *testing.T) {
	settings := core.EffectiveSettings(evalFixtureSettings())
	if got := settings.GetNumber("models.chat.maxOutputTokens", 0); got != 4000 {
		t.Fatalf("mechanism output cap is not effective: got %v, want 4000", got)
	}
	if got := settings.GetNumber("runtime.llmRetries", 2); got != 0 {
		t.Fatalf("fixture disabled-retry setting is not effective: got %v, want 0", got)
	}
}

func newEvaluationEventStore(t *testing.T) (string, *sql.DB) {
	t.Helper()
	dir := t.TempDir()
	db, err := sql.Open("sqlite", filepath.Join(dir, "agent-system.db"))
	if err != nil {
		t.Fatal(err)
	}
	if _, err = db.Exec(`CREATE TABLE events (conversation_id TEXT, run_id TEXT, seq INTEGER, type TEXT, payload TEXT)`); err != nil {
		db.Close()
		t.Fatal(err)
	}
	t.Cleanup(func() { db.Close() })
	return dir, db
}

func TestExtraA01LimitsActualRunIDs(t *testing.T) {
	_, st := newTestServer(t)
	p, err := st.CreateProject(store.OwnerDefault, "allowance")
	if err != nil {
		t.Fatal(err)
	}
	c, err := st.CreateConversation(store.OwnerDefault, p.ID, "allowance", "")
	if err != nil {
		t.Fatal(err)
	}
	for _, kind := range []core.RunKind{core.RunConversation, core.RunSubagent} {
		r := core.Run{Owner: c.Owner, ProjectID: p.ID, ConversationID: c.ID, Kind: kind, Status: core.RunRunning}
		if err = st.CreateRun(&r); err != nil {
			t.Fatal(err)
		}
	}
	calls := 0
	guard := limitExtraA01Runs(st, func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
		calls++
		out := make(chan agentllm.Event)
		close(out)
		return out
	})
	for range guard(context.Background(), "fixture", nil, nil, agentllm.StreamOptions{}) {
	}
	if calls != 1 {
		t.Fatal("two actual runs were blocked")
	}
	extra := core.Run{Owner: c.Owner, ProjectID: p.ID, ConversationID: c.ID, Kind: core.RunSubagent, Status: core.RunRunning}
	if err = st.CreateRun(&extra); err != nil {
		t.Fatal(err)
	}
	for event := range guard(context.Background(), "fixture", nil, nil, agentllm.StreamOptions{}) {
		if event.Err == nil {
			t.Fatal("missing allowance error")
		}
	}
	if calls != 1 {
		t.Fatal("a third actual run made a paid request")
	}
}

func TestFollowupA01UsesNewBoundedAuthorization(t *testing.T) {
	for _, tc := range []struct {
		label  string
		tokens int
		guard  bool
	}{
		{"", 20000, false},
		{"approved-20261013", 20000, true},
		{"approved-followup-20261013", 30000, true},
		{"approved-r02-20261013", 30000, true},
		{"approved-r03-20261013", 30000, true},
		{"approved-r04-20261014", 30000, true},
		{"unapproved", 20000, false},
	} {
		tokens, guard := evalA01Budget(tc.label)
		if tokens != tc.tokens || guard != tc.guard {
			t.Fatalf("label %q: got budget=%d guard=%v; want budget=%d guard=%v", tc.label, tokens, guard, tc.tokens, tc.guard)
		}
	}
}

func TestEvaluationPreservesProductionOutputCap(t *testing.T) {
	for _, tc := range []struct{ supplied, want int }{{780, 780}, {5000, 4000}, {0, 4000}} {
		t.Run(fmt.Sprint(tc.supplied), func(t *testing.T) {
			got := make(chan int, 1)
			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				var body struct {
					MaxTokens int `json:"max_tokens"`
				}
				if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
					t.Error(err)
				}
				got <- body.MaxTokens
				w.Header().Set("Content-Type", "text/event-stream")
				fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
			}))
			defer srv.Close()
			dir, _ := newEvaluationEventStore(t)
			stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
			for range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "check cap"}}, nil, agentllm.StreamOptions{MaxTokens: tc.supplied}) {
			}
			if actual := <-got; actual != tc.want {
				t.Fatalf("evaluation overwrote production cap: got %d want %d", actual, tc.want)
			}
		})
	}
}

func TestEvaluationStopsPaidRequestsAfterRejectedCheckpoint(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	providerCalls := 0
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls++
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
	}))
	defer srv.Close()
	dir, _ := newEvaluationEventStore(t)
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	for _, messages := range [][]agentllm.Message{
		{{Role: agentllm.RoleTool, Name: "publish_work_checkpoint", IsError: true, Content: "阶段发现未发布：checkpoint.source: quote not in provided block"}},
		{{Role: agentllm.RoleUser, Content: "another request after rejection"}},
	} {
		rejected := false
		for event := range stream(evalTestContext(t), "fixture", messages, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
			if event.Kind == agentllm.KindError {
				rejected = true
			}
		}
		if !rejected {
			t.Fatal("rejected checkpoint did not stop the controlled sample")
		}
	}
	if providerCalls != 0 {
		t.Fatalf("%d paid provider calls started after checkpoint rejection", providerCalls)
	}
}

func TestEvaluationStopsAcrossRunsAfterPersistedCheckpointRejection(t *testing.T) {
	t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
	dir, db := newEvaluationEventStore(t)
	_, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 1, "tool.call", `{"callId":"rejected-call","name":"publish_work_checkpoint"}`)
	if err != nil {
		t.Fatal(err)
	}
	_, err = db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 2, "tool.result", `{"callId":"rejected-call","status":"error","summary":"json: unknown field \"$text\""}`)
	if err != nil {
		t.Fatal(err)
	}
	providerCalls := 0
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls++
		w.Header().Set("Content-Type", "text/event-stream")
		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
	}))
	defer srv.Close()
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
	rejected := false
	for event := range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "unrelated foreground history"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if event.Kind == agentllm.KindError {
			rejected = true
		}
	}
	if !rejected || providerCalls != 0 {
		t.Fatalf("persisted background rejection allowed foreground provider request: error=%v calls=%d", rejected, providerCalls)
	}
}

func TestEvaluationFailsClosedWhenCheckpointEventsUnavailable(t *testing.T) {
	providerCalls := 0
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		providerCalls++
	}))
	defer srv.Close()
	stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, t.TempDir())
	rejected := false
	for event := range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "request without the event store"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		if event.Kind == agentllm.KindError {
			rejected = true
		}
	}
	if !rejected || providerCalls != 0 {
		t.Fatalf("missing checkpoint event store allowed provider request: error=%v calls=%d", rejected, providerCalls)
	}
}

func TestEvaluationMatchesCheckpointRejectionWithinItsRun(t *testing.T) {
	for _, tc := range []struct {
		name, tool, resultRun string
	}{
		{"different tool", "read_file", "child"},
		{"different run", "publish_work_checkpoint", "other-run"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			dir, db := newEvaluationEventStore(t)
			if _, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", "child", 1, "tool.call", fmt.Sprintf(`{"callId":"shared","name":%q}`, tc.tool)); err != nil {
				t.Fatal(err)
			}
			if _, err := db.Exec(`INSERT INTO events(conversation_id,run_id,seq,type,payload) VALUES(?,?,?,?,?)`, "background", tc.resultRun, 2, "tool.result", `{"callId":"shared","status":"error"}`); err != nil {
				t.Fatal(err)
			}
			providerCalls := 0
			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				providerCalls++
				w.Header().Set("Content-Type", "text/event-stream")
				fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}\n\ndata: [DONE]\n\n")
			}))
			defer srv.Close()
			stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, dir)
			for range stream(evalTestContext(t), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "unrelated request"}}, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
			}
			if providerCalls != 1 {
				t.Fatalf("unrelated tool error blocked the provider: calls=%d", providerCalls)
			}
		})
	}
}