diff --git a/deps/agent-native b/deps/agent-native index 28f3ea7..2fc66be 160000 --- a/deps/agent-native +++ b/deps/agent-native @@ -1 +1 @@ -Subproject commit 28f3ea767447e089be546fcd54d0e721798f9e85 +Subproject commit 2fc66be65201232ea38ba64d457efad5565d869d diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go index 94b2cec..6a7c17c 100644 --- a/internal/runtime/llm.go +++ b/internal/runtime/llm.go @@ -252,7 +252,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error { if stream == nil { // 重试经库默认(WP6:runtime.llmRetries → Config.RetryAttempts 映射), // 此处不再显式包装(防双重)。 - client := openLLMClient(route.Base, route.Key, route.Model) + client := openRouteClient(route, eff) stream = client.Stream } stream = annotateKeyFailure(stream, route) @@ -601,6 +601,37 @@ func openLLMClient(base, key, model string) *agentllm.Client { return c } +// openRouteClient 按一次解析出的路由构造客户端,并为该路由独立判定 RouteContract。 +func openRouteClient(route ResolvedModel, eff core.EffectiveSettings) *agentllm.Client { + c := openLLMClient(route.Base, route.Key, route.Model) + c.RouteContract = routeContractFor(eff, route) + return c +} + +// routeContractFor 只看本客户端实际解析出的 model 与 base,不看 route.Kind, +// 也不读画像:三项系统设置任一为空、模型名不逐字相等、规范化后的端点与配置 +// 不逐字相等(配置本身须已是规范形态)或身份摘要不符,都返回 nil(关闭)。 +func routeContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF { + model := eff.GetString("models.routeContractEOF.model", "") + base := eff.GetString("models.routeContractEOF.baseUrl", "") + digest := eff.GetString("models.routeContractEOF.identitySha256", "") + if model == "" || base == "" || digest == "" || route.Model != model { + return nil + } + if normalized, ok := agentllm.NormalizeRouteBaseURL(route.Base); !ok || normalized != base { + return nil + } + if agentllm.RouteContractIdentity(model, base) != digest { + return nil + } + return &agentllm.RouteContractEOF{Model: model, BaseURL: base, IdentitySHA256: digest} +} + +// RouteContractFor 供评测等组合根按与生产完全相同的规则判定 RouteContract。 +func RouteContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF { + return routeContractFor(eff, route) +} + // SubLLMRunner 子代理执行器:按内置 profile 跑一轮带工具的内核循环。 type SubLLMRunner struct { // Prompts 提示词库(nil = 使用内联回退文案与只读回退 spec)。 diff --git a/internal/runtime/route_contract_eof_test.go b/internal/runtime/route_contract_eof_test.go new file mode 100644 index 0000000..ba824b6 --- /dev/null +++ b/internal/runtime/route_contract_eof_test.go @@ -0,0 +1,375 @@ +package runtime + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + agentllm "agent-native/runtime/llm" + core "agent-system-core" +) + +const ( + routeEOFModel = "fixture-model" + routeEOFTool = "make_report" + routeEOFRejection = "tool call stream ended before [DONE]" +) + +var routeEOFParams = json.RawMessage(`{"type":"object","required":["title"],"properties":{"title":{"type":"string"}}}`) + +func routeEOFContractSettings(model, base, digest string) map[string]any { + return map[string]any{"model": model, "baseUrl": base, "identitySha256": digest} +} + +func routeEOFEff(chat map[string]any, contract map[string]any) core.EffectiveSettings { + models := map[string]any{"chat": chat} + if contract != nil { + models["routeContractEOF"] = contract + } + return core.EffectiveSettings{"models": models} +} + +func routeEOFMatching(model, base string) map[string]any { + return routeEOFContractSettings(model, base, agentllm.RouteContractIdentity(model, base)) +} + +func TestRouteContractDefaultOff(t *testing.T) { + route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "https://api.example.com/v1", Key: "k"} + if c := openRouteClient(route, core.EffectiveSettings{}); c.RouteContract != nil { + t.Fatalf("no settings must leave RouteContract nil, got %+v", c.RouteContract) + } + full := routeEOFMatching(routeEOFModel, "https://api.example.com/v1") + for _, key := range []string{"model", "baseUrl", "identitySha256"} { + partial := map[string]any{} + for k, v := range full { + partial[k] = v + } + partial[key] = "" + if got := routeContractFor(routeEOFEff(nil, partial), route); got != nil { + t.Fatalf("empty %s must turn the contract off, got %+v", key, got) + } + } + if got := routeContractFor(routeEOFEff(nil, full), route); got == nil { + t.Fatal("complete matching configuration must attach the contract") + } +} + +func TestRouteContractForRequiresExactRoute(t *testing.T) { + const base = "https://api.example.com/v1" + matching := routeEOFMatching(routeEOFModel, base) + cases := []struct { + name string + route ResolvedModel + contract map[string]any + want bool + }{ + {"match", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base}, matching, true}, + {"route base normalizes to configured", ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: "HTTPS://API.Example.com/v1/"}, matching, true}, + {"kind is ignored", ResolvedModel{Kind: "lightweight", Model: routeEOFModel, Base: base}, matching, true}, + {"model mismatch", ResolvedModel{Model: "fixture-model-2", Base: base}, matching, false}, + {"model case mismatch", ResolvedModel{Model: "Fixture-Model", Base: base}, matching, false}, + {"base host mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://other.example.com/v1"}, matching, false}, + {"base path mismatch", ResolvedModel{Model: routeEOFModel, Base: "https://api.example.com/v2"}, matching, false}, + {"route base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, matching, false}, + {"route base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, matching, false}, + {"route base fragment", ResolvedModel{Model: routeEOFModel, Base: base + "#f"}, matching, false}, + {"configured base trailing slash", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, base+"/"), false}, + {"configured base uppercase host", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFMatching(routeEOFModel, "https://API.example.com/v1"), false}, + {"configured base http", ResolvedModel{Model: routeEOFModel, Base: "http://api.example.com/v1"}, routeEOFMatching(routeEOFModel, "http://api.example.com/v1"), false}, + {"configured base query", ResolvedModel{Model: routeEOFModel, Base: base + "?x=1"}, routeEOFMatching(routeEOFModel, base+"?x=1"), false}, + {"digest mismatch", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, agentllm.RouteContractIdentity(routeEOFModel, base+"/")), false}, + {"digest uppercase", ResolvedModel{Model: routeEOFModel, Base: base}, routeEOFContractSettings(routeEOFModel, base, strings.ToUpper(agentllm.RouteContractIdentity(routeEOFModel, base))), false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := routeContractFor(routeEOFEff(nil, tc.contract), tc.route) + if (got != nil) != tc.want { + t.Fatalf("routeContractFor = %+v, want attached=%v", got, tc.want) + } + if got != nil { + want := agentllm.RouteContractEOF{Model: routeEOFModel, BaseURL: base, IdentitySHA256: agentllm.RouteContractIdentity(routeEOFModel, base)} + if *got != want { + t.Fatalf("contract = %+v, want %+v", *got, want) + } + } + }) + } +} + +func TestRouteContractEachResolvedRouteDecidesAlone(t *testing.T) { + const base = "https://api.example.com/v1" + chat := map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"} + prof := core.AgentProfile{} + + eff := routeEOFEff(chat, routeEOFMatching(routeEOFModel, base)) + eff["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"} + if routeContractFor(eff, ResolveChatModel(prof, eff)) == nil { + t.Fatal("chat route matching the configuration must attach") + } + vision := ResolveVisionModel(prof, eff, true) + if vision.Model != "vision-model" || routeContractFor(eff, vision) != nil { + t.Fatalf("vision re-route must decide on its own model, route=%+v", vision) + } + if light := ResolveLightweightModel(prof, eff); light.Model != routeEOFModel || routeContractFor(eff, light) == nil { + t.Fatalf("lightweight falling back to the chat route must decide on that resolved route, route=%+v", light) + } + + eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": "light-model"} + if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil { + t.Fatalf("lightweight route must not inherit the chat contract, route=%+v", light) + } + eff["models"].(map[string]any)["lightweight"] = map[string]any{"model": routeEOFModel, "baseUrl": "https://light.example.com/v1"} + if light := ResolveLightweightModel(prof, eff); routeContractFor(eff, light) != nil { + t.Fatalf("lightweight route on another base must not inherit the chat contract, route=%+v", light) + } + + visionOnly := routeEOFEff(chat, routeEOFMatching("vision-model", base)) + visionOnly["models"].(map[string]any)["vision"] = map[string]any{"model": "vision-model", "capable": "vision-model"} + if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, true)) == nil { + t.Fatal("vision route matching the configuration must attach") + } + if routeContractFor(visionOnly, ResolveChatModel(prof, visionOnly)) != nil { + t.Fatal("chat route must not inherit a vision-model contract") + } + if routeContractFor(visionOnly, ResolveVisionModel(prof, visionOnly, false)) != nil { + t.Fatal("a turn without images keeps the chat route and must stay off") + } + + override := core.AgentProfile{Model: core.ModelRoute{Chat: "profile-model"}} + if routeContractFor(eff, ResolveChatModel(override, eff)) != nil { + t.Fatal("profile model override changes the resolved route and must not reuse the contract") + } +} + +// routeEOFServer answers a first request with a tool-call stream that ends +// without [DONE]; requests that already carry a tool result get a normal +// text reply terminated by [DONE]. +func routeEOFServer(t *testing.T, requests *atomic.Int32) *httptest.Server { + t.Helper() + frame := func(v any) string { + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return "data: " + string(raw) + "\n\n" + } + usage := frame(map[string]any{"choices": []any{}, "usage": map[string]any{"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}}) + toolStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{ + map[string]any{"index": 0, "id": "call_eof_1", "function": map[string]any{"name": routeEOFTool, "arguments": `{"title":`}}, + }}}}}) + + frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"tool_calls": []any{ + map[string]any{"index": 0, "function": map[string]any{"arguments": `"weekly"}`}}, + }}}}}) + + frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "tool_calls"}}}) + + usage + textStream := frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{"content": "已完成。"}}}}) + + frame(map[string]any{"choices": []any{map[string]any{"delta": map[string]any{}, "finish_reason": "stop"}}}) + + usage + "data: [DONE]\n\n" + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + var body struct { + Messages []struct { + Role string `json:"role"` + } `json:"messages"` + } + raw, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(raw, &body) + stream := toolStream + for _, m := range body.Messages { + if m.Role == "tool" { + stream = textStream + } + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, stream) + w.(http.Flusher).Flush() + })) + server.EnableHTTP2 = true + server.StartTLS() + t.Cleanup(server.Close) + return server +} + +func routeEOFHTTPClient(t *testing.T, server *httptest.Server) *http.Client { + t.Helper() + base := server.Client().Transport.(*http.Transport) + transport := &http.Transport{TLSClientConfig: base.TLSClientConfig.Clone(), ForceAttemptHTTP2: true} + t.Cleanup(transport.CloseIdleConnections) + return &http.Client{Transport: transport} +} + +func routeEOFClient(t *testing.T, server *httptest.Server, route ResolvedModel, eff core.EffectiveSettings) *agentllm.Client { + t.Helper() + c := openRouteClient(route, eff) + c.HTTPClient = routeEOFHTTPClient(t, server) + return c +} + +func TestRouteContractEOFReleasesThroughBackendWrappers(t *testing.T) { + var requests atomic.Int32 + server := routeEOFServer(t, &requests) + base := server.URL + "/v1" + route := ResolvedModel{Kind: "chat", Model: routeEOFModel, Base: base, Key: "k", KeySource: "settings"} + tools := []agentllm.ToolDecl{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}} + maxTokens := 100000 + + run := func(t *testing.T, eff core.EffectiveSettings) []agentllm.Event { + t.Helper() + client := routeEOFClient(t, server, route, eff) + stream := annotateKeyFailure(client.Stream, route) + stream = GuardProjectStream(&fakeRunCtx{ctx: context.Background()}, "", stream) + stream = bindProviderRunID(stream, "run_t") + stream = budgetedStream(stream, core.RunBudget{MaxTokens: &maxTokens}, 0) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + var events []agentllm.Event + for ev := range stream(ctx, routeEOFModel, []agentllm.Message{{Role: agentllm.RoleUser, Content: "hi"}}, tools, agentllm.StreamOptions{}) { + if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError { + events = append(events, ev) + } + } + return events + } + + t.Run("matching configuration releases one call", func(t *testing.T) { + eff := routeEOFEff(nil, routeEOFMatching(routeEOFModel, base)) + if routeContractFor(eff, route) == nil { + t.Fatal("fixture configuration must match the resolved route") + } + events := run(t, eff) + if len(events) != 1 || events[0].Kind != agentllm.KindMessageEnd || events[0].Err != nil { + t.Fatalf("terminal events = %+v, want one clean message_end", events) + } + end := events[0] + want := agentllm.ToolCall{ID: "call_eof_1", Name: routeEOFTool, Arguments: `{"title":"weekly"}`} + if len(end.Message.ToolCalls) != 1 || end.Message.ToolCalls[0] != want { + t.Fatalf("released calls = %+v, want exactly %+v", end.Message.ToolCalls, want) + } + if end.StreamComplete { + t.Fatal("stream without [DONE] must keep StreamComplete false") + } + if end.CompletionBasis != agentllm.CompletionRouteContractEOF { + t.Fatalf("CompletionBasis = %q, want %q", end.CompletionBasis, agentllm.CompletionRouteContractEOF) + } + if end.TerminalObservation == nil || end.TerminalObservation.CompletionBasis != agentllm.CompletionRouteContractEOF { + t.Fatalf("TerminalObservation lost the completion basis: %+v", end.TerminalObservation) + } + }) + + t.Run("removed configuration restores rejection", func(t *testing.T) { + events := run(t, routeEOFEff(nil, nil)) + if len(events) != 1 || events[0].Err == nil || len(events[0].Message.ToolCalls) != 0 { + t.Fatalf("terminal events = %+v, want one failed terminal without released calls", events) + } + if !strings.Contains(events[0].Err.Error(), routeEOFRejection) { + t.Fatalf("error = %q, want %q", events[0].Err.Error(), routeEOFRejection) + } + if events[0].CompletionBasis != "" { + t.Fatalf("rejected stream must not carry a completion basis: %q", events[0].CompletionBasis) + } + }) +} + +type routeEOFRunCtx struct { + *fakeRunCtx + models map[string]any +} + +func (r *routeEOFRunCtx) Settings() core.EffectiveSettings { + eff := r.fakeRunCtx.Settings() + eff["models"] = r.models + return eff +} + +func (r *routeEOFRunCtx) Tools() []core.ToolDesc { + return []core.ToolDesc{{Name: routeEOFTool, Description: "report", Parameters: routeEOFParams}} +} + +func TestRouteContractEOFReleasesOneToolCallInLoop(t *testing.T) { + run := func(t *testing.T, contract bool) (*routeEOFRunCtx, []agentllm.Event, int32, error) { + t.Helper() + var requests atomic.Int32 + server := routeEOFServer(t, &requests) + base := server.URL + "/v1" + models := map[string]any{"chat": map[string]any{"model": routeEOFModel, "baseUrl": base, "apiKey": "k"}} + if contract { + models["routeContractEOF"] = routeEOFMatching(routeEOFModel, base) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + rc := &routeEOFRunCtx{ + fakeRunCtx: &fakeRunCtx{ctx: ctx, runtimeExtra: map[string]any{"llmRetries": 0}}, + models: models, + } + eff := rc.Settings() + route := ResolveVisionModel(rc.Profile(), eff, false) + client := routeEOFClient(t, server, route, eff) + if (client.RouteContract != nil) != contract { + t.Fatalf("RouteContract = %+v, want attached=%v", client.RouteContract, contract) + } + var terminals []agentllm.Event + probe := func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + out := make(chan agentllm.Event) + go func() { + defer close(out) + for ev := range client.Stream(ctx, model, msgs, tools, opts) { + if ev.Kind == agentllm.KindMessageEnd || ev.Kind == agentllm.KindError { + terminals = append(terminals, ev) + } + select { + case out <- ev: + case <-ctx.Done(): + return + } + } + }() + return out + } + err := (&LLMRunner{Stream: probe}).Execute(rc) + return rc, terminals, requests.Load(), err + } + + t.Run("matching configuration", func(t *testing.T) { + rc, terminals, requests, err := run(t, true) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if rc.toolRuns != 1 { + t.Fatalf("tool runs = %d, want exactly 1", rc.toolRuns) + } + if requests != 2 || len(terminals) != 2 { + t.Fatalf("requests = %d terminals = %d, want 2 each", requests, len(terminals)) + } + if terminals[0].CompletionBasis != agentllm.CompletionRouteContractEOF || len(terminals[0].Message.ToolCalls) != 1 { + t.Fatalf("first turn = basis %q calls %d, want route_contract_eof with one call", terminals[0].CompletionBasis, len(terminals[0].Message.ToolCalls)) + } + if terminals[1].CompletionBasis != agentllm.CompletionDoneMarker { + t.Fatalf("second turn basis = %q, want %q", terminals[1].CompletionBasis, agentllm.CompletionDoneMarker) + } + }) + + t.Run("without configuration", func(t *testing.T) { + rc, terminals, _, _ := run(t, false) + if rc.toolRuns != 0 { + t.Fatalf("tool runs = %d, want 0 when the contract is off", rc.toolRuns) + } + rejected := false + for _, ev := range terminals { + if len(ev.Message.ToolCalls) > 0 { + t.Fatalf("tool call released without contract: %+v", ev.Message.ToolCalls) + } + if ev.Err != nil && strings.Contains(ev.Err.Error(), routeEOFRejection) { + rejected = true + } + } + if !rejected { + t.Fatalf("terminals = %+v, want the %q rejection", terminals, routeEOFRejection) + } + }) +} diff --git a/internal/runtime/squad_runner.go b/internal/runtime/squad_runner.go index bc789e5..7bbcb82 100644 --- a/internal/runtime/squad_runner.go +++ b/internal/runtime/squad_runner.go @@ -143,7 +143,7 @@ func newModelRouter(rc RunCtx) modelRouter { if route.Model == "" || route.Base == "" { return modelRouter{} } - client := openLLMClient(route.Base, route.Key, route.Model) + client := openRouteClient(route, rc.Settings()) return modelRouter{stream: GuardProjectStream(rc, "", client.Stream), model: route.Model} } diff --git a/internal/runtime/sub_loop.go b/internal/runtime/sub_loop.go index 50948d2..e0ea13d 100644 --- a/internal/runtime/sub_loop.go +++ b/internal/runtime/sub_loop.go @@ -80,7 +80,7 @@ func (l *SubLLMRunner) Execute(rc RunCtx) error { if route.Model == "" || route.Base == "" { return fmt.Errorf("llm-subagent: 模型未配置(当前解析:model=%q base=%q)", route.Model, route.Base) } - client := openLLMClient(route.Base, route.Key, route.Model) + client := openRouteClient(route, rc.Settings()) stream = client.Stream } stream = GuardProjectStream(rc, subagentUserText(rc), stream) diff --git a/internal/runtime/verify.go b/internal/runtime/verify.go index 1b7ff06..635a09b 100644 --- a/internal/runtime/verify.go +++ b/internal/runtime/verify.go @@ -58,7 +58,7 @@ func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence, if route.Model == "" || route.Base == "" { return gates.Verdict{}, fmt.Errorf("验证器调用失败") } - client := openLLMClient(route.Base, route.Key, route.Model) + client := openRouteClient(route, rc.Settings()) // 验证器不经内核循环:显式接重试(WP6 同映射)与 5xx 回退链。 stream = agentllm.WithFallback( agentllm.WithRetry(client.Stream, retryAttemptsOf(rc.Settings()), nil), diff --git a/internal/runtime/workflow_compiled.go b/internal/runtime/workflow_compiled.go index ac56ec4..59de1b9 100644 --- a/internal/runtime/workflow_compiled.go +++ b/internal/runtime/workflow_compiled.go @@ -81,7 +81,7 @@ func judgeStreamFor(rc RunCtx) agentllm.StreamFunc { if route.Model == "" || route.Base == "" { return nil } - client := openLLMClient(route.Base, route.Key, route.Model) + client := openRouteClient(route, rc.Settings()) model := route.Model base := func(ctx context.Context, _ string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { return client.Stream(ctx, model, msgs, tools, opts) diff --git a/internal/server/work_checkpoint_eval_route_contract_test.go b/internal/server/work_checkpoint_eval_route_contract_test.go new file mode 100644 index 0000000..f1b2fae --- /dev/null +++ b/internal/server/work_checkpoint_eval_route_contract_test.go @@ -0,0 +1,170 @@ +//go:build live + +package server + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + agentllm "agent-native/runtime/llm" + "agent-system-backend/internal/settings" + "agent-system-backend/internal/store" +) + +const ( + evalRouteContractModel = "fixture-model" + evalRouteContractBase = "https://api.example.com/v1" +) + +func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) { + cases := []struct { + basis, check string + wantBasis, wantCheck string + }{ + {"done_marker", "", "done_marker", ""}, + {"route_contract_eof", "", "route_contract_eof", ""}, + {"", "C1", "", "C1"}, + {"", "C9", "", "C9"}, + {"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"}, + {"Done_Marker", "C10", "unknown", "unknown"}, + {"", "c3", "", "unknown"}, + {"", "", "", ""}, + } + for _, tc := range cases { + got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check}) + if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck { + t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck) + } + } +} + +func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) { + for _, tc := range []struct{ basis, want string }{ + {"done_marker", "done_marker"}, + {"route_contract_eof", "route_contract_eof"}, + {"PRIVATE_BASIS_I2B", "unknown"}, + {"", ""}, + } { + dir, _ := newEvaluationEventStore(t) + provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event { + out := make(chan agentllm.Event, 1) + out <- agentllm.Event{Kind: agentllm.KindMessageEnd, + Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd}, + Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17}, + FinishReason: "stop", + StreamComplete: true, + CompletionBasis: tc.basis, + TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true, + CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"}, + } + close(out) + return out + } + for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) { + } + raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "PRIVATE_") { + t.Fatalf("provider-controlled completion label escaped into record: %s", raw) + } + var record struct { + Calls []struct { + CompletionBasis *string `json:"completionBasis"` + Observation *agentllm.TerminalObservation `json:"terminalObservation"` + } `json:"calls"` + } + if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil { + t.Fatalf("unexpected record: %s err=%v", raw, err) + } + call := record.Calls[0] + if tc.want == "" { + if call.CompletionBasis != nil { + t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis) + } + } else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want { + t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want) + } + if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" { + t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want) + } + } +} + +func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) { + t.Helper() + route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"} + if contract != nil { + route["routeContractEOF"] = contract + } + models, ok := evalModelSettings(route) + if !ok { + t.Fatal("route file rejected") + } + if _, leaked := route["routeContractEOF"]; leaked { + t.Fatal("routeContractEOF left in the chat route") + } + config := evalFixtureSettings() + config["models"] = models + st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + svc := settings.NewService(st) + svc.SetSystemOverride(config) + return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route +} + +func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) { + digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase) + matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest} + client, _ := evalRouteContractClient(t, matching) + if client.RouteContract == nil { + t.Fatal("matching effective settings must attach the route contract") + } + if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) { + t.Fatalf("unexpected route contract %+v", *client.RouteContract) + } + if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" { + t.Fatalf("client route changed: base=%q", client.BaseURL) + } + + if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil { + t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract) + } + for _, key := range []string{"model", "baseUrl", "identitySha256"} { + partial := map[string]any{} + for k, v := range matching { + if k != key { + partial[k] = v + } + } + if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil { + t.Fatalf("missing %s must leave RouteContract nil", key) + } + } + for name, contract := range map[string]map[string]any{ + "other model": {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)}, + "other base": {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")}, + "digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)}, + } { + if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil { + t.Fatalf("%s must leave RouteContract nil", name) + } + } +} + +func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) { + for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} { + route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract} + if _, ok := evalModelSettings(route); ok { + t.Fatalf("malformed routeContractEOF %v accepted", contract) + } + } +} diff --git a/internal/server/work_checkpoint_eval_test.go b/internal/server/work_checkpoint_eval_test.go index 6c2b227..24c4274 100644 --- a/internal/server/work_checkpoint_eval_test.go +++ b/internal/server/work_checkpoint_eval_test.go @@ -204,9 +204,13 @@ func TestServeWorkCheckpointEval(t *testing.T) { if json.Unmarshal(raw, &route) != nil { t.Fatal("bad route file") } + models, ok := evalModelSettings(route) + if !ok { + t.Fatal("bad route file") + } route["maxOutputTokens"] = float64(4000) route["fallbacks"] = "" - config["models"] = map[string]any{"chat": route} + config["models"] = models modelRoute = route } // The isolated fixture owns its startup configuration; managed model/mock @@ -241,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) { front := &runtime.LLMRunner{Prompts: promptSvc} sub := &runtime.SubLLMRunner{Prompts: promptSvc} if mode == "model" { - stream := evalProviderStream(modelRoute, dir) + stream := evalProviderStreamWithSettings(modelRoute, settingSvc.Effective(store.OwnerDefault, ""), dir) if guardedA01 { stream = limitExtraA01Runs(st, stream) } @@ -425,12 +429,50 @@ func evalRejectedCheckpointPersisted(dir string) (bool, error) { return false, nil } -func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc { +// evalModelSettings moves the protected route file's optional routeContractEOF +// object out of the chat route and into the system-layer key production reads. +func evalModelSettings(route map[string]any) (map[string]any, bool) { + models := map[string]any{"chat": route} + raw, present := route["routeContractEOF"] + if !present { + return models, true + } + delete(route, "routeContractEOF") + contract, ok := raw.(map[string]any) + if !ok { + return nil, false + } + clean := map[string]any{} + for _, key := range []string{"model", "baseUrl", "identitySha256"} { + value, present := contract[key] + if !present { + continue + } + text, ok := value.(string) + if !ok { + return nil, false + } + clean[key] = text + } + models["routeContractEOF"] = clean + return models, true +} + +func evalProviderClient(route map[string]any, eff core.EffectiveSettings) *agentllm.Client { base, _ := route["baseUrl"].(string) key, _ := route["apiKey"].(string) model, _ := route["model"].(string) - provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream - return evalProviderStreamWithProvider(provider, dir) + client := &agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")} + client.RouteContract = runtime.RouteContractFor(eff, runtime.ResolvedModel{Model: model, Base: base, Key: key}) + return client +} + +func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc { + return evalProviderStreamWithSettings(route, core.EffectiveSettings{}, dir) +} + +func evalProviderStreamWithSettings(route map[string]any, eff core.EffectiveSettings, dir string) agentllm.StreamFunc { + return evalProviderStreamWithProvider(evalProviderClient(route, eff).Stream, dir) } // evalCallRecord contains only accounting and terminal facts. Historical @@ -447,6 +489,7 @@ type evalCallRecord struct { HTTPStatus int `json:"httpStatus"` FinishReason string `json:"finishReason"` StreamComplete bool `json:"streamComplete"` + CompletionBasis string `json:"completionBasis,omitempty"` TerminalObservation *agentllm.TerminalObservation `json:"terminalObservation,omitempty"` ToolArgumentBytes int `json:"toolArgumentBytes"` ToolArgumentsValid bool `json:"toolArgumentsValid"` @@ -479,6 +522,21 @@ func evalLabel(value, fallback string, allowed ...string) string { return fallback } +// Absent completion facts stay absent; never record free text. +func evalCompletionBasis(value string) string { + if value == "" { + return "" + } + return evalLabel(value, "unknown", "done_marker", "route_contract_eof") +} + +func evalContractCheck(value string) string { + if value == "" { + return "" + } + return evalLabel(value, "unknown", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9") +} + func clampCount(value, limit int64) int64 { if value < 0 { return 0 @@ -503,7 +561,9 @@ func evalTerminalObservation(observation *agentllm.TerminalObservation) *agentll ToolArgumentsValid: observation.ToolArgumentsValid, TailEventKind: evalLabel(observation.TailEventKind, "unknown", "done_literal", "json_frame", "comment", "empty_data", "partial_frame", "unknown_non_json", "unknown"), - TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)), + TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)), + CompletionBasis: evalCompletionBasis(observation.CompletionBasis), + ContractCheck: evalContractCheck(observation.ContractCheck), } if body := observation.BodyBoundary; body != nil { clean.BodyBoundary = &agentllm.BodyBoundaryObservation{ @@ -667,12 +727,13 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag if terminal != nil && fault { terminal.Message.ToolCalls = nil } - finishReason, streamComplete := "", false + finishReason, streamComplete, completionBasis := "", false, "" var returnedUsage agentllm.Usage var terminalObservation *agentllm.TerminalObservation httpStatus := 0 if terminal != nil { finishReason, streamComplete = evalFinishReasonClass(terminal.FinishReason), terminal.StreamComplete + completionBasis = evalCompletionBasis(terminal.CompletionBasis) returnedUsage = terminal.Usage terminalObservation = evalTerminalObservation(terminal.TerminalObservation) httpStatus = terminal.HTTPStatus @@ -702,7 +763,7 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag record := evalCallRecord{RunID: runID, RunKey: runKey, Call: a.Calls, Model: m, OutputCap: opts.MaxTokens, ReportedTokens: usage, UsagePresent: usage > 0, Usage: returnedUsage, HTTPStatus: httpStatus, FinishReason: finishReason, - StreamComplete: streamComplete, TerminalObservation: terminalObservation, + StreamComplete: streamComplete, CompletionBasis: completionBasis, TerminalObservation: terminalObservation, ToolArgumentBytes: argumentBytes, ToolArgumentsValid: argumentsValid, FailureClass: failureClass} records = append(records, record) diff --git a/internal/settings/settings.go b/internal/settings/settings.go index cb2491c..635de9e 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -101,6 +101,9 @@ func Schema() []core.SettingDesc { {Key: "models.chat.contextWindow", Group: "models", Label: "主模型上下文窗口(token)", Type: core.SettingNumber, Default: 128000, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "模型真实窗口的 token 数。运行内清理旧工具结果的阈值由此换算(此前按 32000 字符硬编码,长材料任务第一请求即触发清理且被清内容找不回)"}, {Key: "models.chat.outputReserve", Group: "models", Label: "窗口输出预留(token)", Type: core.SettingNumber, Default: 16384, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "清理阈值里为模型输出预留的 token 数"}, {Key: "models.chat.contextBuffer", Group: "models", Label: "窗口余量(token)", Type: core.SettingNumber, Default: 4096, Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "避免贴着上限才清理的余量"}, + {Key: "models.routeContractEOF.model", Group: "models", Label: "路由契约 EOF:模型名", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "允许缺 [DONE] 的工具调用流按 HTTP 边界放行的模型名。三项任一为空即关闭;只对模型名与 Base URL 都精确一致的路由生效"}, + {Key: "models.routeContractEOF.baseUrl", Group: "models", Label: "路由契约 EOF:Base URL", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "规范化的 https 端点(小写主机、无末尾 /、无查询串)。三项任一为空即关闭;只对模型名与 Base URL 都精确一致的路由生效"}, + {Key: "models.routeContractEOF.identitySha256", Group: "models", Label: "路由契约 EOF:身份摘要", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem, Description: "sha256(\"openai-chat\\n\"+模型名+\"\\n\"+Base URL) 的小写十六进制,不符即关闭。三项任一为空即关闭;只对精确一致的模型名与 Base URL 生效"}, {Key: "models.lightweight.baseUrl", Group: "models", Label: "轻量模型 Base URL", Type: core.SettingString, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem, Description: "标题生成/澄清等轻任务;空则回落主模型"}, {Key: "models.lightweight.apiKey", Group: "models", Label: "轻量模型 API Key", Type: core.SettingSecret, Default: "", Effect: core.EffectHot, Scope: core.SettingScopeSystem}, {Key: "models.lightweight.model", Group: "models", Label: "轻量模型名称", Type: core.SettingString, Default: "", Effect: core.EffectNewRun, Scope: core.SettingScopeSystem}, diff --git a/internal/settings/settings_route_contract_test.go b/internal/settings/settings_route_contract_test.go new file mode 100644 index 0000000..46c89b3 --- /dev/null +++ b/internal/settings/settings_route_contract_test.go @@ -0,0 +1,105 @@ +package settings + +import ( + "errors" + "path/filepath" + "slices" + "testing" + + "agent-system-backend/internal/store" + core "agent-system-core" +) + +var routeContractKeys = []string{ + "models.routeContractEOF.model", + "models.routeContractEOF.baseUrl", + "models.routeContractEOF.identitySha256", +} + +func TestRouteContractSettingsDeclaredAsSystemManaged(t *testing.T) { + declared := map[string]core.SettingDesc{} + for _, desc := range Schema() { + declared[desc.Key] = desc + } + managed := NewService(nil).ManagedSettingPaths() + for _, key := range routeContractKeys { + desc, ok := declared[key] + if !ok { + t.Fatalf("schema missing %s", key) + } + if desc.Group != "models" || desc.Type != core.SettingString || desc.Default != "" || + desc.Scope != core.SettingScopeSystem || desc.Effect != core.EffectNewRun || desc.Label == "" || desc.Description == "" { + t.Fatalf("%s declared as %+v", key, desc) + } + if !slices.Contains(managed, key) { + t.Fatalf("%s is not a managed path", key) + } + } +} + +func TestRouteContractSettingsIgnoreUserAndProjectLayers(t *testing.T) { + st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "settings.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + svc := NewService(st) + + for _, key := range routeContractKeys { + var managed *ErrManagedPath + if err := svc.MergeUser("local", map[string]any{key: "user-value"}); !errors.As(err, &managed) { + t.Fatalf("user write of %s = %v, want ErrManagedPath", key, err) + } + if err := svc.MergeProject("local", "p1", map[string]any{key: "project-value"}); !errors.As(err, &managed) { + t.Fatalf("project write of %s = %v, want ErrManagedPath", key, err) + } + } + if eff := svc.Effective("local", "p1"); eff.GetString(routeContractKeys[0], "") != "" { + t.Fatalf("default must be off, got %#v", eff) + } + + legacy := map[string]any{"models": map[string]any{"routeContractEOF": map[string]any{ + "model": "m", "baseUrl": "https://api.example.com/v1", "identitySha256": "abc", + }}} + if err := st.MergeSettingsScope("local", ScopeUser, legacy); err != nil { + t.Fatal(err) + } + if err := st.MergeSettingsScope("local", ScopeProjectPrefix+"p1", legacy); err != nil { + t.Fatal(err) + } + eff := svc.Effective("local", "p1") + for _, key := range routeContractKeys { + if got := eff.GetString(key, ""); got != "" { + t.Fatalf("stored user/project %s leaked into effective settings: %q", key, got) + } + } + user, err := svc.UserScope("local") + if err != nil { + t.Fatal(err) + } + project, err := svc.ProjectScope("local", "p1") + if err != nil { + t.Fatal(err) + } + for _, scope := range []core.EffectiveSettings{user, project} { + for _, key := range routeContractKeys { + if got := scope.GetString(key, ""); got != "" { + t.Fatalf("stripped scope still echoes %s=%q", key, got) + } + } + } + + if err := svc.MergeSystem(map[string]any{ + routeContractKeys[0]: "sys-model", + routeContractKeys[1]: "https://sys.example.com/v1", + routeContractKeys[2]: "sys-digest", + }); err != nil { + t.Fatal(err) + } + eff = svc.Effective("local", "p1") + for key, want := range map[string]string{routeContractKeys[0]: "sys-model", routeContractKeys[1]: "https://sys.example.com/v1", routeContractKeys[2]: "sys-digest"} { + if got := eff.GetString(key, ""); got != want { + t.Fatalf("system %s = %q, want %q", key, got, want) + } + } +}