Attach the O2 route contract to an exactly configured model route #224

Merged
weichen merged 2 commits from feature/fast-slow-o2-i2-route-contract into main 2026-10-20 08:06:49 +00:00
11 changed files with 759 additions and 14 deletions
+32 -1
View File
@@ -252,7 +252,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
if stream == nil {
// 重试经库默认(WP6:runtime.llmRetries → Config.RetryAttempts 映射),
// 此处不再显式包装(防双重)。
client := openLLMClient(route.Base, route.Key, route.Model)
client := openRouteClient(route, eff)
stream = client.Stream
}
stream = annotateKeyFailure(stream, route)
@@ -601,6 +601,37 @@ func openLLMClient(base, key, model string) *agentllm.Client {
return c
}
// openRouteClient 按一次解析出的路由构造客户端,并为该路由独立判定 RouteContract。
func openRouteClient(route ResolvedModel, eff core.EffectiveSettings) *agentllm.Client {
c := openLLMClient(route.Base, route.Key, route.Model)
c.RouteContract = routeContractFor(eff, route)
return c
}
// routeContractFor 只看本客户端实际解析出的 model 与 base,不看 route.Kind,
// 也不读画像:三项系统设置任一为空、模型名不逐字相等、规范化后的端点与配置
// 不逐字相等(配置本身须已是规范形态)或身份摘要不符,都返回 nil(关闭)。
func routeContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF {
model := eff.GetString("models.routeContractEOF.model", "")
base := eff.GetString("models.routeContractEOF.baseUrl", "")
digest := eff.GetString("models.routeContractEOF.identitySha256", "")
if model == "" || base == "" || digest == "" || route.Model != model {
return nil
}
if normalized, ok := agentllm.NormalizeRouteBaseURL(route.Base); !ok || normalized != base {
return nil
}
if agentllm.RouteContractIdentity(model, base) != digest {
return nil
}
return &agentllm.RouteContractEOF{Model: model, BaseURL: base, IdentitySHA256: digest}
}
// RouteContractFor 供评测等组合根按与生产完全相同的规则判定 RouteContract。
func RouteContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF {
return routeContractFor(eff, route)
}
// SubLLMRunner 子代理执行器:按内置 profile 跑一轮带工具的内核循环。
type SubLLMRunner struct {
// Prompts 提示词库(nil = 使用内联回退文案与只读回退 spec)。
+375
View File
@@ -0,0 +1,375 @@
package runtime
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
const (
routeEOFModel = "fixture-model"
routeEOFTool = "make_report"
routeEOFRejection = "tool call stream ended before [DONE]"
)
var routeEOFParams = json.RawMessage(`{"type":"object","required":["title"],"properties":{"title":{"type":"string"}}}`)
func routeEOFContractSettings(model, base, digest string) map[string]any {
return map[string]any{"model": model, "baseUrl": base, "identitySha256": digest}
}
func routeEOFEff(chat map[string]any, contract map[string]any) core.EffectiveSettings {
models := map[string]any{"chat": chat}
if contract != nil {
models["routeContractEOF"] = contract
}
return core.EffectiveSettings{"models": models}
}
func routeEOFMatching(model, base string) map[string]any {
return routeEOFContractSettings(model, base, agentllm.RouteContractIdentity(model, base))
}
func TestRouteContractDefaultOff(t *testing.T) {
route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "https://api.example.com/v1", Key: "k"}
if c := openRouteClient(route, core.EffectiveSettings{}); c.RouteContract != nil {
t.Fatalf("no settings must leave RouteContract nil, got %+v", c.RouteContract)
}
full := routeEOFMatching(routeEOFModel, "https://api.example.com/v1")
for _, key := range []string{"model", "baseUrl", "identitySha256"} {
partial := map[string]any{}
for k, v := range full {
partial[k] = v
}
partial[key] = ""
if got := routeContractFor(routeEOFEff(nil, partial), route); got != nil {
t.Fatalf("empty %s must turn the contract off, got %+v", key, got)
}
}
if got := routeContractFor(routeEOFEff(nil, full), route); got == nil {
t.Fatal("complete matching configuration must attach the contract")
}
}
func TestRouteContractForRequiresExactRoute(t *testing.T) {
const base = "https://api.example.com/v1"
matching := routeEOFMatching(routeEOFModel, base)
cases := []struct {
name string
route ResolvedModel
contract map[string]any
want bool
}{
{"match", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base}, matching, true},
{"route base normalizes to configured", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "HTTPS://API.Example.com/v1/"}, matching, true},
{"kind is ignored", ResolvedModel{Kind: "lightweight", Model: routeEOFModel, Base: base}, matching, true},
{"model mismatch", ResolvedModel{Model: "fixture-model-2", Base: base}, matching, false},
{"model case mismatch", ResolvedModel{Model: "Fixture-Model", Base: base}, matching, false},
{"base host mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://other.example.com/v1"}, matching, false},
{"base path mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://api.example.com/v2"}, matching, false},
{"route base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, matching, false},
{"route base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, matching, false},
{"route base fragment", ResolvedModel{Model: routeEOFModel, Base: base + "#f"}, matching, false},
{"configured base trailing slash", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, base+"/"), false},
{"configured base uppercase host", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, "https://API.example.com/v1"), false},
{"configured base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, routeEOFMatching(routeEOFModel, "http://api.example.com/v1"), false},
{"configured base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, routeEOFMatching(routeEOFModel, base+"?x=1"), false},
{"digest mismatch", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, agentllm.RouteContractIdentity(routeEOFModel, base+"/")), false},
{"digest uppercase", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, strings.ToUpper(agentllm.RouteContractIdentity(routeEOFModel, base))), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := routeContractFor(routeEOFEff(nil, tc.contract), tc.route)
if (got != nil) != tc.want {
t.Fatalf("routeContractFor = %+v, want attached=%v", got, tc.want)
}
if got != nil {
want := agentllm.RouteContractEOF{Model: routeEOFModel, BaseURL: base, IdentitySHA256: agentllm.RouteContractIdentity(routeEOFModel, base)}
if *got != want {
t.Fatalf("contract = %+v, want %+v", *got, want)
}
}
})
}
}
func TestRouteContractEachResolvedRouteDecidesAlone(t *testing.T) {
const base = "https://api.example.com/v1"
chat := map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"}
prof := core.AgentProfile{}
eff := routeEOFEff(chat, routeEOFMatching(routeEOFModel, base))
eff["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"}
if routeContractFor(eff, ResolveChatModel(prof, eff)) == nil {
t.Fatal("chat route matching the configuration must attach")
}
vision := ResolveVisionModel(prof, eff, true)
if vision.Model != "vision-model" || routeContractFor(eff, vision) != nil {
t.Fatalf("vision re-route must decide on its own model, route=%+v", vision)
}
if light := ResolveLightweightModel(prof, eff); light.Model != routeEOFModel || routeContractFor(eff, light) == nil {
t.Fatalf("lightweight falling back to the chat route must decide on that resolved route, route=%+v", light)
}
eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": "light-model"}
if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil {
t.Fatalf("lightweight route must not inherit the chat contract, route=%+v", light)
}
eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": routeEOFModel, "baseUrl": "https://light.example.com/v1"}
if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil {
t.Fatalf("lightweight route on another base must not inherit the chat contract, route=%+v", light)
}
visionOnly := routeEOFEff(chat, routeEOFMatching("vision-model", base))
visionOnly["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"}
if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, true)) == nil {
t.Fatal("vision route matching the configuration must attach")
}
if routeContractFor(visionOnly, ResolveChatModel(prof, visionOnly)) != nil {
t.Fatal("chat route must not inherit a vision-model contract")
}
if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, false)) != nil {
t.Fatal("a turn without images keeps the chat route and must stay off")
}
override := core.AgentProfile{Model: core.ModelRoute{Chat: "profile-model"}}
if routeContractFor(eff, ResolveChatModel(override, eff)) != nil {
t.Fatal("profile model override changes the resolved route and must not reuse the contract")
}
}
// routeEOFServer answers a first request with a tool-call stream that ends
// without [DONE]; requests that already carry a tool result get a normal
// text reply terminated by [DONE].
func routeEOFServer(t *testing.T, requests *atomic.Int32) *httptest.Server {
t.Helper()
frame := func(v any) string {
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return "data: " + string(raw) + "\n\n"
}
usage := frame(map[string]any{"choices": []any{}, "usage": map[string]any{"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}})
toolStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{
map[string]any{"index": 0, "id": "call_eof_1", "function": map[string]any{"name": routeEOFTool, "arguments": `{"title":`}},
}}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{
map[string]any{"index": 0, "function": map[string]any{"arguments": `"weekly"}`}},
}}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "tool_calls"}}}) +
usage
textStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"content": "已完成。"}}}}) +
frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "stop"}}}) +
usage + "data: [DONE]\n\n"
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests.Add(1)
var body struct {
Messages []struct {
Role string `json:"role"`
} `json:"messages"`
}
raw, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(raw, &body)
stream := toolStream
for _, m := range body.Messages {
if m.Role == "tool" {
stream = textStream
}
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, stream)
w.(http.Flusher).Flush()
}))
server.EnableHTTP2 = true
server.StartTLS()
t.Cleanup(server.Close)
return server
}
func routeEOFHTTPClient(t *testing.T, server *httptest.Server) *http.Client {
t.Helper()
base := server.Client().Transport.(*http.Transport)
transport := &http.Transport{TLSClientConfig: base.TLSClientConfig.Clone(), ForceAttemptHTTP2: true}
t.Cleanup(transport.CloseIdleConnections)
return &http.Client{Transport: transport}
}
func routeEOFClient(t *testing.T, server *httptest.Server, route ResolvedModel, eff core.EffectiveSettings) *agentllm.Client {
t.Helper()
c := openRouteClient(route, eff)
c.HTTPClient = routeEOFHTTPClient(t, server)
return c
}
func TestRouteContractEOFReleasesThroughBackendWrappers(t *testing.T) {
var requests atomic.Int32
server := routeEOFServer(t, &requests)
base := server.URL + "/v1"
route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base, Key: "k", KeySource: "settings"}
tools := []agentllm.ToolDecl{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}}
maxTokens := 100000
run := func(t *testing.T, eff core.EffectiveSettings) []agentllm.Event {
t.Helper()
client := routeEOFClient(t, server, route, eff)
stream := annotateKeyFailure(client.Stream, route)
stream = GuardProjectStream(&fakeRunCtx{ctx: context.Background()}, "", stream)
stream = bindProviderRunID(stream, "run_t")
stream = budgetedStream(stream, core.RunBudget{MaxTokens: &maxTokens}, 0)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
var events []agentllm.Event
for ev := range stream(ctx, routeEOFModel, []agentllm.Message{{Role: agentllm.RoleUser, Content: "hi"}}, tools, agentllm.StreamOptions{}) {
if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError {
events = append(events, ev)
}
}
return events
}
t.Run("matching configuration releases one call", func(t *testing.T) {
eff := routeEOFEff(nil, routeEOFMatching(routeEOFModel, base))
if routeContractFor(eff, route) == nil {
t.Fatal("fixture configuration must match the resolved route")
}
events := run(t, eff)
if len(events) != 1 || events[0].Kind != agentllm.KindMessageEnd || events[0].Err != nil {
t.Fatalf("terminal events = %+v, want one clean message_end", events)
}
end := events[0]
want := agentllm.ToolCall{ID: "call_eof_1", Name: routeEOFTool, Arguments: `{"title":"weekly"}`}
if len(end.Message.ToolCalls) != 1 || end.Message.ToolCalls[0] != want {
t.Fatalf("released calls = %+v, want exactly %+v", end.Message.ToolCalls, want)
}
if end.StreamComplete {
t.Fatal("stream without [DONE] must keep StreamComplete false")
}
if end.CompletionBasis != agentllm.CompletionRouteContractEOF {
t.Fatalf("CompletionBasis = %q, want %q", end.CompletionBasis, agentllm.CompletionRouteContractEOF)
}
if end.TerminalObservation == nil || end.TerminalObservation.CompletionBasis != agentllm.CompletionRouteContractEOF {
t.Fatalf("TerminalObservation lost the completion basis: %+v", end.TerminalObservation)
}
})
t.Run("removed configuration restores rejection", func(t *testing.T) {
events := run(t, routeEOFEff(nil, nil))
if len(events) != 1 || events[0].Err == nil || len(events[0].Message.ToolCalls) != 0 {
t.Fatalf("terminal events = %+v, want one failed terminal without released calls", events)
}
if !strings.Contains(events[0].Err.Error(), routeEOFRejection) {
t.Fatalf("error = %q, want %q", events[0].Err.Error(), routeEOFRejection)
}
if events[0].CompletionBasis != "" {
t.Fatalf("rejected stream must not carry a completion basis: %q", events[0].CompletionBasis)
}
})
}
type routeEOFRunCtx struct {
*fakeRunCtx
models map[string]any
}
func (r *routeEOFRunCtx) Settings() core.EffectiveSettings {
eff := r.fakeRunCtx.Settings()
eff["models"] = r.models
return eff
}
func (r *routeEOFRunCtx) Tools() []core.ToolDesc {
return []core.ToolDesc{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}}
}
func TestRouteContractEOFReleasesOneToolCallInLoop(t *testing.T) {
run := func(t *testing.T, contract bool) (*routeEOFRunCtx, []agentllm.Event, int32, error) {
t.Helper()
var requests atomic.Int32
server := routeEOFServer(t, &requests)
base := server.URL + "/v1"
models := map[string]any{"chat": map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"}}
if contract {
models["routeContractEOF"] = routeEOFMatching(routeEOFModel, base)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)
rc := &routeEOFRunCtx{
fakeRunCtx: &fakeRunCtx{ctx: ctx, runtimeExtra: map[string]any{"llmRetries": 0}},
models: models,
}
eff := rc.Settings()
route := ResolveVisionModel(rc.Profile(), eff, false)
client := routeEOFClient(t, server, route, eff)
if (client.RouteContract != nil) != contract {
t.Fatalf("RouteContract = %+v, want attached=%v", client.RouteContract, contract)
}
var terminals []agentllm.Event
probe := func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
out := make(chan agentllm.Event)
go func() {
defer close(out)
for ev := range client.Stream(ctx, model, msgs, tools, opts) {
if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError {
terminals = append(terminals, ev)
}
select {
case out <- ev:
case <-ctx.Done():
return
}
}
}()
return out
}
err := (&LLMRunner{Stream: probe}).Execute(rc)
return rc, terminals, requests.Load(), err
}
t.Run("matching configuration", func(t *testing.T) {
rc, terminals, requests, err := run(t, true)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if rc.toolRuns != 1 {
t.Fatalf("tool runs = %d, want exactly 1", rc.toolRuns)
}
if requests != 2 || len(terminals) != 2 {
t.Fatalf("requests = %d terminals = %d, want 2 each", requests, len(terminals))
}
if terminals[0].CompletionBasis != agentllm.CompletionRouteContractEOF || len(terminals[0].Message.ToolCalls) != 1 {
t.Fatalf("first turn = basis %q calls %d, want route_contract_eof with one call", terminals[0].CompletionBasis, len(terminals[0].Message.ToolCalls))
}
if terminals[1].CompletionBasis != agentllm.CompletionDoneMarker {
t.Fatalf("second turn basis = %q, want %q", terminals[1].CompletionBasis, agentllm.CompletionDoneMarker)
}
})
t.Run("without configuration", func(t *testing.T) {
rc, terminals, _, _ := run(t, false)
if rc.toolRuns != 0 {
t.Fatalf("tool runs = %d, want 0 when the contract is off", rc.toolRuns)
}
rejected := false
for _, ev := range terminals {
if len(ev.Message.ToolCalls) > 0 {
t.Fatalf("tool call released without contract: %+v", ev.Message.ToolCalls)
}
if ev.Err != nil && strings.Contains(ev.Err.Error(), routeEOFRejection) {
rejected = true
}
}
if !rejected {
t.Fatalf("terminals = %+v, want the %q rejection", terminals, routeEOFRejection)
}
})
}
+1 -1
View File
@@ -143,7 +143,7 @@ func newModelRouter(rc RunCtx) modelRouter {
if route.Model == "" || route.Base == "" {
return modelRouter{}
}
client := openLLMClient(route.Base, route.Key, route.Model)
client := openRouteClient(route, rc.Settings())
return modelRouter{stream: GuardProjectStream(rc, "", client.Stream), model: route.Model}
}
+1 -1
View File
@@ -80,7 +80,7 @@ func (l *SubLLMRunner) Execute(rc RunCtx) error {
if route.Model == "" || route.Base == "" {
return fmt.Errorf("llm-subagent: 模型未配置(当前解析:model=%q base=%q)", route.Model, route.Base)
}
client := openLLMClient(route.Base, route.Key, route.Model)
client := openRouteClient(route, rc.Settings())
stream = client.Stream
}
stream = GuardProjectStream(rc, subagentUserText(rc), stream)
+1 -1
View File
@@ -58,7 +58,7 @@ func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence,
if route.Model == "" || route.Base == "" {
return gates.Verdict{}, fmt.Errorf("验证器调用失败")
}
client := openLLMClient(route.Base, route.Key, route.Model)
client := openRouteClient(route, rc.Settings())
// 验证器不经内核循环:显式接重试(WP6 同映射)与 5xx 回退链。
stream = agentllm.WithFallback(
agentllm.WithRetry(client.Stream, retryAttemptsOf(rc.Settings()), nil),
+1 -1
View File
@@ -81,7 +81,7 @@ func judgeStreamFor(rc RunCtx) agentllm.StreamFunc {
if route.Model == "" || route.Base == "" {
return nil
}
client := openLLMClient(route.Base, route.Key, route.Model)
client := openRouteClient(route, rc.Settings())
model := route.Model
base := func(ctx context.Context, _ string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
return client.Stream(ctx, model, msgs, tools, opts)
@@ -0,0 +1,170 @@
//go:build live
package server
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
agentllm "agent-native/runtime/llm"
"agent-system-backend/internal/settings"
"agent-system-backend/internal/store"
)
const (
evalRouteContractModel = "fixture-model"
evalRouteContractBase = "https://api.example.com/v1"
)
func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) {
cases := []struct {
basis, check string
wantBasis, wantCheck string
}{
{"done_marker", "", "done_marker", ""},
{"route_contract_eof", "", "route_contract_eof", ""},
{"", "C1", "", "C1"},
{"", "C9", "", "C9"},
{"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"},
{"Done_Marker", "C10", "unknown", "unknown"},
{"", "c3", "", "unknown"},
{"", "", "", ""},
}
for _, tc := range cases {
got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check})
if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck {
t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck)
}
}
}
func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) {
for _, tc := range []struct{ basis, want string }{
{"done_marker", "done_marker"},
{"route_contract_eof", "route_contract_eof"},
{"PRIVATE_BASIS_I2B", "unknown"},
{"", ""},
} {
dir, _ := newEvaluationEventStore(t)
provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
out := make(chan agentllm.Event, 1)
out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
FinishReason: "stop",
StreamComplete: true,
CompletionBasis: tc.basis,
TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true,
CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"},
}
close(out)
return out
}
for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
}
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "PRIVATE_") {
t.Fatalf("provider-controlled completion label escaped into record: %s", raw)
}
var record struct {
Calls []struct {
CompletionBasis *string `json:"completionBasis"`
Observation *agentllm.TerminalObservation `json:"terminalObservation"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil {
t.Fatalf("unexpected record: %s err=%v", raw, err)
}
call := record.Calls[0]
if tc.want == "" {
if call.CompletionBasis != nil {
t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis)
}
} else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want {
t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want)
}
if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" {
t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want)
}
}
}
func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) {
t.Helper()
route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"}
if contract != nil {
route["routeContractEOF"] = contract
}
models, ok := evalModelSettings(route)
if !ok {
t.Fatal("route file rejected")
}
if _, leaked := route["routeContractEOF"]; leaked {
t.Fatal("routeContractEOF left in the chat route")
}
config := evalFixtureSettings()
config["models"] = models
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
svc := settings.NewService(st)
svc.SetSystemOverride(config)
return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route
}
func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) {
digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase)
matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest}
client, _ := evalRouteContractClient(t, matching)
if client.RouteContract == nil {
t.Fatal("matching effective settings must attach the route contract")
}
if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) {
t.Fatalf("unexpected route contract %+v", *client.RouteContract)
}
if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" {
t.Fatalf("client route changed: base=%q", client.BaseURL)
}
if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil {
t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract)
}
for _, key := range []string{"model", "baseUrl", "identitySha256"} {
partial := map[string]any{}
for k, v := range matching {
if k != key {
partial[k] = v
}
}
if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil {
t.Fatalf("missing %s must leave RouteContract nil", key)
}
}
for name, contract := range map[string]map[string]any{
"other model": {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)},
"other base": {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")},
"digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)},
} {
if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil {
t.Fatalf("%s must leave RouteContract nil", name)
}
}
}
func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) {
for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} {
route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract}
if _, ok := evalModelSettings(route); ok {
t.Fatalf("malformed routeContractEOF %v accepted", contract)
}
}
}
+69 -8
View File
@@ -204,9 +204,13 @@ func TestServeWorkCheckpointEval(t *testing.T) {
if json.Unmarshal(raw, &route) != nil {
t.Fatal("bad route file")
}
models, ok := evalModelSettings(route)
if !ok {
t.Fatal("bad route file")
}
route["maxOutputTokens"] = float64(4000)
route["fallbacks"] = ""
config["models"] = map[string]any{"chat": route}
config["models"] = models
modelRoute = route
}
// The isolated fixture owns its startup configuration; managed model/mock
@@ -241,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) {
front := &runtime.LLMRunner{Prompts: promptSvc}
sub := &runtime.SubLLMRunner{Prompts: promptSvc}
if mode == "model" {
stream := evalProviderStream(modelRoute, dir)
stream := evalProviderStreamWithSettings(modelRoute, settingSvc.Effective(store.OwnerDefault, ""), dir)
if guardedA01 {
stream = limitExtraA01Runs(st, stream)
}
@@ -425,12 +429,50 @@ func evalRejectedCheckpointPersisted(dir string) (bool, error) {
return false, nil
}
func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
// evalModelSettings moves the protected route file's optional routeContractEOF
// object out of the chat route and into the system-layer key production reads.
func evalModelSettings(route map[string]any) (map[string]any, bool) {
models := map[string]any{"chat": route}
raw, present := route["routeContractEOF"]
if !present {
return models, true
}
delete(route, "routeContractEOF")
contract, ok := raw.(map[string]any)
if !ok {
return nil, false
}
clean := map[string]any{}
for _, key := range []string{"model", "baseUrl", "identitySha256"} {
value, present := contract[key]
if !present {
continue
}
text, ok := value.(string)
if !ok {
return nil, false
}
clean[key] = text
}
models["routeContractEOF"] = clean
return models, true
}
func evalProviderClient(route map[string]any, eff core.EffectiveSettings) *agentllm.Client {
base, _ := route["baseUrl"].(string)
key, _ := route["apiKey"].(string)
model, _ := route["model"].(string)
provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream
return evalProviderStreamWithProvider(provider, dir)
client := &agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}
client.RouteContract = runtime.RouteContractFor(eff, runtime.ResolvedModel{Model: model, Base: base, Key: key})
return client
}
func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
return evalProviderStreamWithSettings(route, core.EffectiveSettings{}, dir)
}
func evalProviderStreamWithSettings(route map[string]any, eff core.EffectiveSettings, dir string) agentllm.StreamFunc {
return evalProviderStreamWithProvider(evalProviderClient(route, eff).Stream, dir)
}
// evalCallRecord contains only accounting and terminal facts. Historical
@@ -447,6 +489,7 @@ type evalCallRecord struct {
HTTPStatus int `json:"httpStatus"`
FinishReason string `json:"finishReason"`
StreamComplete bool `json:"streamComplete"`
CompletionBasis string `json:"completionBasis,omitempty"`
TerminalObservation *agentllm.TerminalObservation `json:"terminalObservation,omitempty"`
ToolArgumentBytes int `json:"toolArgumentBytes"`
ToolArgumentsValid bool `json:"toolArgumentsValid"`
@@ -479,6 +522,21 @@ func evalLabel(value, fallback string, allowed ...string) string {
return fallback
}
// Absent completion facts stay absent; never record free text.
func evalCompletionBasis(value string) string {
if value == "" {
return ""
}
return evalLabel(value, "unknown", "done_marker", "route_contract_eof")
}
func evalContractCheck(value string) string {
if value == "" {
return ""
}
return evalLabel(value, "unknown", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9")
}
func clampCount(value, limit int64) int64 {
if value < 0 {
return 0
@@ -503,7 +561,9 @@ func evalTerminalObservation(observation *agentllm.TerminalObservation) *agentll
ToolArgumentsValid: observation.ToolArgumentsValid,
TailEventKind: evalLabel(observation.TailEventKind, "unknown",
"done_literal", "json_frame", "comment", "empty_data", "partial_frame", "unknown_non_json", "unknown"),
TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)),
TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)),
CompletionBasis: evalCompletionBasis(observation.CompletionBasis),
ContractCheck: evalContractCheck(observation.ContractCheck),
}
if body := observation.BodyBoundary; body != nil {
clean.BodyBoundary = &agentllm.BodyBoundaryObservation{
@@ -667,12 +727,13 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag
if terminal != nil && fault {
terminal.Message.ToolCalls = nil
}
finishReason, streamComplete := "", false
finishReason, streamComplete, completionBasis := "", false, ""
var returnedUsage agentllm.Usage
var terminalObservation *agentllm.TerminalObservation
httpStatus := 0
if terminal != nil {
finishReason, streamComplete = evalFinishReasonClass(terminal.FinishReason), terminal.StreamComplete
completionBasis = evalCompletionBasis(terminal.CompletionBasis)
returnedUsage = terminal.Usage
terminalObservation = evalTerminalObservation(terminal.TerminalObservation)
httpStatus = terminal.HTTPStatus
@@ -702,7 +763,7 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag
record := evalCallRecord{RunID: runID, RunKey: runKey, Call: a.Calls, Model: m,
OutputCap: opts.MaxTokens, ReportedTokens: usage, UsagePresent: usage > 0,
Usage: returnedUsage, HTTPStatus: httpStatus, FinishReason: finishReason,
StreamComplete: streamComplete, TerminalObservation: terminalObservation,
StreamComplete: streamComplete, CompletionBasis: completionBasis, TerminalObservation: terminalObservation,
ToolArgumentBytes: argumentBytes, ToolArgumentsValid: argumentsValid,
FailureClass: failureClass}
records = append(records, record)
+3
View File
@@ -101,6 +101,9 @@ func Schema() []core.SettingDesc {
{Key: "models.chat.contextWindow", Group: "models", Label: "主模型上下文窗口(token)", Type: core.SettingNumber, Default: 128000, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "模型真实窗口的 token 数。运行内清理旧工具结果的阈值由此换算(此前按 32000 字符硬编码,长材料任务第一请求即触发清理且被清内容找不回)"},
{Key: "models.chat.outputReserve", Group: "models", Label: "窗口输出预留(token)", Type: core.SettingNumber, Default: 16384, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "清理阈值里为模型输出预留的 token 数"},
{Key: "models.chat.contextBuffer", Group: "models", Label: "窗口余量(token)", Type: core.SettingNumber, Default: 4096, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "避免贴着上限才清理的余量"},
{Key: "models.routeContractEOF.model", Group: "models", Label: "路由契约 EOF:模型名", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "允许缺 [DONE] 的工具调用流按 HTTP 边界放行的模型名。三项任一为空即关闭;只对模型名与 Base URL 都精确一致的路由生效"},
{Key: "models.routeContractEOF.baseUrl", Group: "models", Label: "路由契约 EOF:Base URL", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "规范化的 https 端点(小写主机、无末尾 /、无查询串)。三项任一为空即关闭;只对模型名与 Base URL 都精确一致的路由生效"},
{Key: "models.routeContractEOF.identitySha256", Group: "models", Label: "路由契约 EOF:身份摘要", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "sha256(\"openai-chat\\n\"+模型名+\"\\n\"+Base URL) 的小写十六进制,不符即关闭。三项任一为空即关闭;只对精确一致的模型名与 Base URL 生效"},
{Key: "models.lightweight.baseUrl", Group: "models", Label: "轻量模型 Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "标题生成/澄清等轻任务;空则回落主模型"},
{Key: "models.lightweight.apiKey", Group: "models", Label: "轻量模型 API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem},
{Key: "models.lightweight.model", Group: "models", Label: "轻量模型名称", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem},
@@ -0,0 +1,105 @@
package settings
import (
"errors"
"path/filepath"
"slices"
"testing"
"agent-system-backend/internal/store"
core "agent-system-core"
)
var routeContractKeys = []string{
"models.routeContractEOF.model",
"models.routeContractEOF.baseUrl",
"models.routeContractEOF.identitySha256",
}
func TestRouteContractSettingsDeclaredAsSystemManaged(t *testing.T) {
declared := map[string]core.SettingDesc{}
for _, desc := range Schema() {
declared[desc.Key] = desc
}
managed := NewService(nil).ManagedSettingPaths()
for _, key := range routeContractKeys {
desc, ok := declared[key]
if !ok {
t.Fatalf("schema missing %s", key)
}
if desc.Group != "models" || desc.Type != core.SettingString || desc.Default != "" ||
desc.Scope != core.SettingScopeSystem || desc.Effect != core.EffectNewRun || desc.Label == "" || desc.Description == "" {
t.Fatalf("%s declared as %+v", key, desc)
}
if !slices.Contains(managed, key) {
t.Fatalf("%s is not a managed path", key)
}
}
}
func TestRouteContractSettingsIgnoreUserAndProjectLayers(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "settings.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
svc := NewService(st)
for _, key := range routeContractKeys {
var managed *ErrManagedPath
if err := svc.MergeUser("local", map[string]any{key: "user-value"}); !errors.As(err, &managed) {
t.Fatalf("user write of %s = %v, want ErrManagedPath", key, err)
}
if err := svc.MergeProject("local", "p1", map[string]any{key: "project-value"}); !errors.As(err, &managed) {
t.Fatalf("project write of %s = %v, want ErrManagedPath", key, err)
}
}
if eff := svc.Effective("local", "p1"); eff.GetString(routeContractKeys[0], "") != "" {
t.Fatalf("default must be off, got %#v", eff)
}
legacy := map[string]any{"models": map[string]any{"routeContractEOF": map[string]any{
"model": "m", "baseUrl": "https://api.example.com/v1", "identitySha256": "abc",
}}}
if err := st.MergeSettingsScope("local", ScopeUser, legacy); err != nil {
t.Fatal(err)
}
if err := st.MergeSettingsScope("local", ScopeProjectPrefix+"p1", legacy); err != nil {
t.Fatal(err)
}
eff := svc.Effective("local", "p1")
for _, key := range routeContractKeys {
if got := eff.GetString(key, ""); got != "" {
t.Fatalf("stored user/project %s leaked into effective settings: %q", key, got)
}
}
user, err := svc.UserScope("local")
if err != nil {
t.Fatal(err)
}
project, err := svc.ProjectScope("local", "p1")
if err != nil {
t.Fatal(err)
}
for _, scope := range []core.EffectiveSettings{user, project} {
for _, key := range routeContractKeys {
if got := scope.GetString(key, ""); got != "" {
t.Fatalf("stripped scope still echoes %s=%q", key, got)
}
}
}
if err := svc.MergeSystem(map[string]any{
routeContractKeys[0]: "sys-model",
routeContractKeys[1]: "https://sys.example.com/v1",
routeContractKeys[2]: "sys-digest",
}); err != nil {
t.Fatal(err)
}
eff = svc.Effective("local", "p1")
for key, want := range map[string]string{routeContractKeys[0]: "sys-model", routeContractKeys[1]: "https://sys.example.com/v1", routeContractKeys[2]: "sys-digest"} {
if got := eff.GetString(key, ""); got != want {
t.Fatalf("system %s = %q, want %q", key, got, want)
}
}
}