Files
Agent-System-Backend/internal/runtime/route_contract_eof_test.go
T
weichen 80faf3edbd Resolve squad routes from the run owner and project
Why: F13 - the squad factory read the default account with an empty
project, pairing the run user's model name with another account's
endpoint and credentials; direct and squad paths disagreed on
reasoning split and route contract.
Evidence: app/runtime/squadadapter tests pass (verify-u10b.json);
run-scoped snapshot, project override, profile override and
fail-closed cases added with synthetic keys.
Impact: squad execution now uses the same effective-config rules as
the direct path; default setups unchanged.
Refs: feature/agent-stack-audit-remediation-b1 U10
2026-10-21 16:15:03 +00:00

438 lines
19 KiB
Go

package runtime
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
const (
routeEOFModel = "fixture-model"
routeEOFTool = "make_report"
routeEOFRejection = "tool call stream ended before [DONE]"
)
var routeEOFParams = json.RawMessage(`{"type":"object","required":["title"],"properties":{"title":{"type":"string"}}}`)
func routeEOFContractSettings(model, base, digest string) map[string]any {
return map[string]any{"model": model, "baseUrl": base, "identitySha256": digest}
}
func routeEOFEff(chat map[string]any, contract map[string]any) core.EffectiveSettings {
models := map[string]any{"chat": chat}
if contract != nil {
models["routeContractEOF"] = contract
}
return core.EffectiveSettings{"models": models}
}
func routeEOFMatching(model, base string) map[string]any {
return routeEOFContractSettings(model, base, agentllm.RouteContractIdentity(model, base))
}
func TestRouteContractDefaultOff(t *testing.T) {
route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "https://api.example.com/v1", Key: "k"}
if c := OpenRouteClient(route, core.EffectiveSettings{}); c.RouteContract != nil {
t.Fatalf("no settings must leave RouteContract nil, got %+v", c.RouteContract)
}
full := routeEOFMatching(routeEOFModel, "https://api.example.com/v1")
for _, key := range []string{"model", "baseUrl", "identitySha256"} {
partial := map[string]any{}
for k, v := range full {
partial[k] = v
}
partial[key] = ""
if got := routeContractFor(routeEOFEff(nil, partial), route); got != nil {
t.Fatalf("empty %s must turn the contract off, got %+v", key, got)
}
}
if got := routeContractFor(routeEOFEff(nil, full), route); got == nil {
t.Fatal("complete matching configuration must attach the contract")
}
}
func TestRouteContractForRequiresExactRoute(t *testing.T) {
const base = "https://api.example.com/v1"
matching := routeEOFMatching(routeEOFModel, base)
cases := []struct {
name string
route ResolvedModel
contract map[string]any
want bool
}{
{"match", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base}, matching, true},
{"route base normalizes to configured", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "HTTPS://API.Example.com/v1/"}, matching, true},
{"kind is ignored", ResolvedModel{Kind: "lightweight", Model: routeEOFModel, Base: base}, matching, true},
{"model mismatch", ResolvedModel{Model: "fixture-model-2", Base: base}, matching, false},
{"model case mismatch", ResolvedModel{Model: "Fixture-Model", Base: base}, matching, false},
{"base host mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://other.example.com/v1"}, matching, false},
{"base path mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://api.example.com/v2"}, matching, false},
{"route base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, matching, false},
{"route base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, matching, false},
{"route base fragment", ResolvedModel{Model: routeEOFModel, Base: base + "#f"}, matching, false},
{"configured base trailing slash", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, base+"/"), false},
{"configured base uppercase host", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, "https://API.example.com/v1"), false},
{"configured base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, routeEOFMatching(routeEOFModel, "http://api.example.com/v1"), false},
{"configured base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, routeEOFMatching(routeEOFModel, base+"?x=1"), false},
{"digest mismatch", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, agentllm.RouteContractIdentity(routeEOFModel, base+"/")), false},
{"digest uppercase", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, strings.ToUpper(agentllm.RouteContractIdentity(routeEOFModel, base))), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := routeContractFor(routeEOFEff(nil, tc.contract), tc.route)
if (got != nil) != tc.want {
t.Fatalf("routeContractFor = %+v, want attached=%v", got, tc.want)
}
if got != nil {
want := agentllm.RouteContractEOF{Model: routeEOFModel, BaseURL: base, IdentitySHA256: agentllm.RouteContractIdentity(routeEOFModel, base)}
if *got != want {
t.Fatalf("contract = %+v, want %+v", *got, want)
}
}
})
}
}
func TestRouteContractEachResolvedRouteDecidesAlone(t *testing.T) {
const base = "https://api.example.com/v1"
chat := map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"}
prof := core.AgentProfile{}
eff := routeEOFEff(chat, routeEOFMatching(routeEOFModel, base))
eff["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"}
if routeContractFor(eff, ResolveChatModel(prof, eff)) == nil {
t.Fatal("chat route matching the configuration must attach")
}
vision := ResolveVisionModel(prof, eff, true)
if vision.Model != "vision-model" || routeContractFor(eff, vision) != nil {
t.Fatalf("vision re-route must decide on its own model, route=%+v", vision)
}
if light := ResolveLightweightModel(prof, eff); light.Model != routeEOFModel || routeContractFor(eff, light) == nil {
t.Fatalf("lightweight falling back to the chat route must decide on that resolved route, route=%+v", light)
}
eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": "light-model"}
if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil {
t.Fatalf("lightweight route must not inherit the chat contract, route=%+v", light)
}
eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": routeEOFModel, "baseUrl": "https://light.example.com/v1"}
if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil {
t.Fatalf("lightweight route on another base must not inherit the chat contract, route=%+v", light)
}
visionOnly := routeEOFEff(chat, routeEOFMatching("vision-model", base))
visionOnly["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"}
if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, true)) == nil {
t.Fatal("vision route matching the configuration must attach")
}
if routeContractFor(visionOnly, ResolveChatModel(prof, visionOnly)) != nil {
t.Fatal("chat route must not inherit a vision-model contract")
}
if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, false)) != nil {
t.Fatal("a turn without images keeps the chat route and must stay off")
}
override := core.AgentProfile{Model: core.ModelRoute{Chat: "profile-model"}}
if routeContractFor(eff, ResolveChatModel(override, eff)) != nil {
t.Fatal("profile model override changes the resolved route and must not reuse the contract")
}
}
// routeEOFServer answers a first request with a tool-call stream that ends
// without [DONE]; requests that already carry a tool result get a normal
// text reply terminated by [DONE].
func routeEOFServer(t *testing.T, requests *atomic.Int32) *httptest.Server {
t.Helper()
frame := func(v any) string {
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return "data: " + string(raw) + "\n\n"
}
usage := frame(map[string]any{"choices": []any{}, "usage": map[string]any{"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}})
toolStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{
map[string]any{"index": 0, "id": "call_eof_1", "function": map[string]any{"name": routeEOFTool, "arguments": `{"title":`}},
}}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{
map[string]any{"index": 0, "function": map[string]any{"arguments": `"weekly"}`}},
}}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "tool_calls"}}}) +
usage
textStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"content": "已完成。"}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "stop"}}}) +
usage + "data: [DONE]\n\n"
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
var body struct {
Messages []struct {
Role string `json:"role"`
} `json:"messages"`
}
raw, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(raw, &body)
stream := toolStream
for _, m := range body.Messages {
if m.Role == "tool" {
stream = textStream
}
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, stream)
w.(http.Flusher).Flush()
}))
server.EnableHTTP2 = true
server.StartTLS()
t.Cleanup(server.Close)
return server
}
func routeEOFHTTPClient(t *testing.T, server *httptest.Server) *http.Client {
t.Helper()
base := server.Client().Transport.(*http.Transport)
transport := &http.Transport{TLSClientConfig: base.TLSClientConfig.Clone(), ForceAttemptHTTP2: true}
t.Cleanup(transport.CloseIdleConnections)
return &http.Client{Transport: transport}
}
func routeEOFClient(t *testing.T, server *httptest.Server, route ResolvedModel, eff core.EffectiveSettings) *agentllm.Client {
t.Helper()
c := OpenRouteClient(route, eff)
c.HTTPClient = routeEOFHTTPClient(t, server)
return c
}
func TestRouteContractEOFReleasesThroughBackendWrappers(t *testing.T) {
var requests atomic.Int32
server := routeEOFServer(t, &requests)
base := server.URL + "/v1"
route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base, Key: "k", KeySource: "settings"}
tools := []agentllm.ToolDecl{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}}
maxTokens := 100000
run := func(t *testing.T, eff core.EffectiveSettings) []agentllm.Event {
t.Helper()
client := routeEOFClient(t, server, route, eff)
stream := annotateKeyFailure(client.Stream, route)
stream = GuardProjectStream(&fakeRunCtx{ctx: context.Background()}, "", stream)
stream = bindProviderRunID(stream, "run_t")
stream = budgetedStream(stream, core.RunBudget{MaxTokens: &maxTokens}, 0)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
var events []agentllm.Event
for ev := range stream(ctx, routeEOFModel, []agentllm.Message{{Role: agentllm.RoleUser, Content: "hi"}}, tools, agentllm.StreamOptions{}) {
if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError {
events = append(events, ev)
}
}
return events
}
t.Run("matching configuration releases one call", func(t *testing.T) {
eff := routeEOFEff(nil, routeEOFMatching(routeEOFModel, base))
if routeContractFor(eff, route) == nil {
t.Fatal("fixture configuration must match the resolved route")
}
events := run(t, eff)
if len(events) != 1 || events[0].Kind != agentllm.KindMessageEnd || events[0].Err != nil {
t.Fatalf("terminal events = %+v, want one clean message_end", events)
}
end := events[0]
want := agentllm.ToolCall{ID: "call_eof_1", Name: routeEOFTool, Arguments: `{"title":"weekly"}`}
if len(end.Message.ToolCalls) != 1 || end.Message.ToolCalls[0] != want {
t.Fatalf("released calls = %+v, want exactly %+v", end.Message.ToolCalls, want)
}
if end.StreamComplete {
t.Fatal("stream without [DONE] must keep StreamComplete false")
}
if end.CompletionBasis != agentllm.CompletionRouteContractEOF {
t.Fatalf("CompletionBasis = %q, want %q", end.CompletionBasis, agentllm.CompletionRouteContractEOF)
}
if end.TerminalObservation == nil || end.TerminalObservation.CompletionBasis != agentllm.CompletionRouteContractEOF {
t.Fatalf("TerminalObservation lost the completion basis: %+v", end.TerminalObservation)
}
})
t.Run("removed configuration restores rejection", func(t *testing.T) {
events := run(t, routeEOFEff(nil, nil))
if len(events) != 1 || events[0].Err == nil || len(events[0].Message.ToolCalls) != 0 {
t.Fatalf("terminal events = %+v, want one failed terminal without released calls", events)
}
if !strings.Contains(events[0].Err.Error(), routeEOFRejection) {
t.Fatalf("error = %q, want %q", events[0].Err.Error(), routeEOFRejection)
}
if events[0].CompletionBasis != "" {
t.Fatalf("rejected stream must not carry a completion basis: %q", events[0].CompletionBasis)
}
})
}
type routeEOFRunCtx struct {
*fakeRunCtx
models map[string]any
}
func (r *routeEOFRunCtx) Settings() core.EffectiveSettings {
eff := r.fakeRunCtx.Settings()
eff["models"] = r.models
return eff
}
func (r *routeEOFRunCtx) Tools() []core.ToolDesc {
return []core.ToolDesc{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}}
}
func TestRouteContractEOFReleasesOneToolCallInLoop(t *testing.T) {
run := func(t *testing.T, contract bool) (*routeEOFRunCtx, []agentllm.Event, int32, error) {
t.Helper()
var requests atomic.Int32
server := routeEOFServer(t, &requests)
base := server.URL + "/v1"
models := map[string]any{"chat": map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"}}
if contract {
models["routeContractEOF"] = routeEOFMatching(routeEOFModel, base)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)
rc := &routeEOFRunCtx{
fakeRunCtx: &fakeRunCtx{ctx: ctx, runtimeExtra: map[string]any{"llmRetries": 0}},
models: models,
}
eff := rc.Settings()
route := ResolveVisionModel(rc.Profile(), eff, false)
client := routeEOFClient(t, server, route, eff)
if (client.RouteContract != nil) != contract {
t.Fatalf("RouteContract = %+v, want attached=%v", client.RouteContract, contract)
}
var terminals []agentllm.Event
probe := func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
out := make(chan agentllm.Event)
go func() {
defer close(out)
for ev := range client.Stream(ctx, model, msgs, tools, opts) {
if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError {
terminals = append(terminals, ev)
}
select {
case out <- ev:
case <-ctx.Done():
return
}
}
}()
return out
}
err := (&LLMRunner{Stream: probe}).Execute(rc)
return rc, terminals, requests.Load(), err
}
t.Run("matching configuration", func(t *testing.T) {
rc, terminals, requests, err := run(t, true)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if rc.toolRuns != 1 {
t.Fatalf("tool runs = %d, want exactly 1", rc.toolRuns)
}
if requests != 2 || len(terminals) != 2 {
t.Fatalf("requests = %d terminals = %d, want 2 each", requests, len(terminals))
}
if terminals[0].CompletionBasis != agentllm.CompletionRouteContractEOF || len(terminals[0].Message.ToolCalls) != 1 {
t.Fatalf("first turn = basis %q calls %d, want route_contract_eof with one call", terminals[0].CompletionBasis, len(terminals[0].Message.ToolCalls))
}
if terminals[1].CompletionBasis != agentllm.CompletionDoneMarker {
t.Fatalf("second turn basis = %q, want %q", terminals[1].CompletionBasis, agentllm.CompletionDoneMarker)
}
})
t.Run("without configuration", func(t *testing.T) {
rc, terminals, _, _ := run(t, false)
if rc.toolRuns != 0 {
t.Fatalf("tool runs = %d, want 0 when the contract is off", rc.toolRuns)
}
rejected := false
for _, ev := range terminals {
if len(ev.Message.ToolCalls) > 0 {
t.Fatalf("tool call released without contract: %+v", ev.Message.ToolCalls)
}
if ev.Err != nil && strings.Contains(ev.Err.Error(), routeEOFRejection) {
rejected = true
}
}
if !rejected {
t.Fatalf("terminals = %+v, want the %q rejection", terminals, routeEOFRejection)
}
})
}
// 小队路径的路由来自同一份运行快照:契约对得上的路由与直连一样放行一次
// 工具调用;画像把模型名改掉之后,契约不再跟随,放行也一并消失。
func TestRouteContractEOFForRunScopedRoute(t *testing.T) {
var requests atomic.Int32
server := routeEOFServer(t, &requests)
base := server.URL + "/v1"
models := map[string]any{
"chat": map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"},
"routeContractEOF": routeEOFMatching(routeEOFModel, base),
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)
decls := []agentllm.ToolDecl{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}}
run := func(t *testing.T, prof core.AgentProfile) (*agentllm.Client, []agentllm.Event) {
t.Helper()
rc := &runSnapCtx{
fakeRunCtx: &fakeRunCtx{ctx: ctx, prof: prof},
eff: core.EffectiveSettings{"models": models},
}
route := RunChatRoute(rc)
client := routeEOFClient(t, server, route, rc.Settings())
stream := GuardProjectStream(rc, "", annotateKeyFailure(client.Stream, route))
var terminal []agentllm.Event
for ev := range stream(ctx, route.Model, []agentllm.Message{{Role: agentllm.RoleUser, Content: "hi"}}, decls, agentllm.StreamOptions{}) {
if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError {
terminal = append(terminal, ev)
}
}
return client, terminal
}
t.Run("route matching the configuration releases one call", func(t *testing.T) {
client, terminal := run(t, core.AgentProfile{Owner: "alice"})
if client.RouteContract == nil {
t.Fatal("run-scoped route must carry the contract, as on the direct path")
}
if len(terminal) != 1 || terminal[0].Kind != agentllm.KindMessageEnd || terminal[0].Err != nil {
t.Fatalf("terminal events = %+v, want one clean message_end", terminal)
}
if len(terminal[0].Message.ToolCalls) != 1 || terminal[0].Message.ToolCalls[0].Name != routeEOFTool {
t.Fatalf("released calls = %+v, want exactly one %s", terminal[0].Message.ToolCalls, routeEOFTool)
}
if terminal[0].CompletionBasis != agentllm.CompletionRouteContractEOF {
t.Fatalf("completion basis = %q, want %q", terminal[0].CompletionBasis, agentllm.CompletionRouteContractEOF)
}
})
t.Run("profile override drops the contract and the release", func(t *testing.T) {
client, terminal := run(t, core.AgentProfile{Owner: "alice", Model: core.ModelRoute{Chat: "profile-model"}})
if client.RouteContract != nil {
t.Fatalf("overridden route kept the contract: %+v", client.RouteContract)
}
if len(terminal) != 1 || terminal[0].Err == nil || !strings.Contains(terminal[0].Err.Error(), routeEOFRejection) {
t.Fatalf("terminal events = %+v, want the %q rejection", terminal, routeEOFRejection)
}
if len(terminal[0].Message.ToolCalls) != 0 {
t.Fatalf("tool call released without the contract: %+v", terminal[0].Message.ToolCalls)
}
})
}