diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go index f77bde0..6a7c17c 100644 --- a/internal/runtime/llm.go +++ b/internal/runtime/llm.go @@ -627,6 +627,11 @@ func routeContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm return &agentllm.RouteContractEOF{Model: model, BaseURL: base, IdentitySHA256: digest} } +// RouteContractFor 供评测等组合根按与生产完全相同的规则判定 RouteContract。 +func RouteContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF { + return routeContractFor(eff, route) +} + // SubLLMRunner 子代理执行器：按内置 profile 跑一轮带工具的内核循环。 type SubLLMRunner struct { // Prompts 提示词库（nil = 使用内联回退文案与只读回退 spec）。 diff --git a/internal/server/work_checkpoint_eval_route_contract_test.go b/internal/server/work_checkpoint_eval_route_contract_test.go new file mode 100644 index 0000000..f1b2fae --- /dev/null +++ b/internal/server/work_checkpoint_eval_route_contract_test.go @@ -0,0 +1,170 @@ +//go:build live + +package server + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + agentllm "agent-native/runtime/llm" + "agent-system-backend/internal/settings" + "agent-system-backend/internal/store" +) + +const ( + evalRouteContractModel = "fixture-model" + evalRouteContractBase = "https://api.example.com/v1" +) + +func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) { + cases := []struct { + basis, check string + wantBasis, wantCheck string + }{ + {"done_marker", "", "done_marker", ""}, + {"route_contract_eof", "", "route_contract_eof", ""}, + {"", "C1", "", "C1"}, + {"", "C9", "", "C9"}, + {"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"}, + {"Done_Marker", "C10", "unknown", "unknown"}, + {"", "c3", "", "unknown"}, + {"", "", "", ""}, + } + for _, tc := range cases { + got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check}) + if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck { + t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck) + } + } +} + +func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) { + for _, tc := range []struct{ basis, want string }{ + {"done_marker", "done_marker"}, + {"route_contract_eof", "route_contract_eof"}, + {"PRIVATE_BASIS_I2B", "unknown"}, + {"", ""}, + } { + dir, _ := newEvaluationEventStore(t) + provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event { + out := make(chan agentllm.Event, 1) + out <- agentllm.Event{Kind: agentllm.KindMessageEnd, + Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd}, + Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17}, + FinishReason: "stop", + StreamComplete: true, + CompletionBasis: tc.basis, + TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true, + CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"}, + } + close(out) + return out + } + for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) { + } + raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "PRIVATE_") { + t.Fatalf("provider-controlled completion label escaped into record: %s", raw) + } + var record struct { + Calls []struct { + CompletionBasis *string `json:"completionBasis"` + Observation *agentllm.TerminalObservation `json:"terminalObservation"` + } `json:"calls"` + } + if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil { + t.Fatalf("unexpected record: %s err=%v", raw, err) + } + call := record.Calls[0] + if tc.want == "" { + if call.CompletionBasis != nil { + t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis) + } + } else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want { + t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want) + } + if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" { + t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want) + } + } +} + +func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) { + t.Helper() + route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"} + if contract != nil { + route["routeContractEOF"] = contract + } + models, ok := evalModelSettings(route) + if !ok { + t.Fatal("route file rejected") + } + if _, leaked := route["routeContractEOF"]; leaked { + t.Fatal("routeContractEOF left in the chat route") + } + config := evalFixtureSettings() + config["models"] = models + st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + svc := settings.NewService(st) + svc.SetSystemOverride(config) + return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route +} + +func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) { + digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase) + matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest} + client, _ := evalRouteContractClient(t, matching) + if client.RouteContract == nil { + t.Fatal("matching effective settings must attach the route contract") + } + if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) { + t.Fatalf("unexpected route contract %+v", *client.RouteContract) + } + if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" { + t.Fatalf("client route changed: base=%q", client.BaseURL) + } + + if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil { + t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract) + } + for _, key := range []string{"model", "baseUrl", "identitySha256"} { + partial := map[string]any{} + for k, v := range matching { + if k != key { + partial[k] = v + } + } + if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil { + t.Fatalf("missing %s must leave RouteContract nil", key) + } + } + for name, contract := range map[string]map[string]any{ + "other model": {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)}, + "other base": {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")}, + "digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)}, + } { + if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil { + t.Fatalf("%s must leave RouteContract nil", name) + } + } +} + +func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) { + for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} { + route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract} + if _, ok := evalModelSettings(route); ok { + t.Fatalf("malformed routeContractEOF %v accepted", contract) + } + } +} diff --git a/internal/server/work_checkpoint_eval_test.go b/internal/server/work_checkpoint_eval_test.go index 6c2b227..24c4274 100644 --- a/internal/server/work_checkpoint_eval_test.go +++ b/internal/server/work_checkpoint_eval_test.go @@ -204,9 +204,13 @@ func TestServeWorkCheckpointEval(t *testing.T) { if json.Unmarshal(raw, &route) != nil { t.Fatal("bad route file") } + models, ok := evalModelSettings(route) + if !ok { + t.Fatal("bad route file") + } route["maxOutputTokens"] = float64(4000) route["fallbacks"] = "" - config["models"] = map[string]any{"chat": route} + config["models"] = models modelRoute = route } // The isolated fixture owns its startup configuration; managed model/mock @@ -241,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) { front := &runtime.LLMRunner{Prompts: promptSvc} sub := &runtime.SubLLMRunner{Prompts: promptSvc} if mode == "model" { - stream := evalProviderStream(modelRoute, dir) + stream := evalProviderStreamWithSettings(modelRoute, settingSvc.Effective(store.OwnerDefault, ""), dir) if guardedA01 { stream = limitExtraA01Runs(st, stream) } @@ -425,12 +429,50 @@ func evalRejectedCheckpointPersisted(dir string) (bool, error) { return false, nil } -func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc { +// evalModelSettings moves the protected route file's optional routeContractEOF +// object out of the chat route and into the system-layer key production reads. +func evalModelSettings(route map[string]any) (map[string]any, bool) { + models := map[string]any{"chat": route} + raw, present := route["routeContractEOF"] + if !present { + return models, true + } + delete(route, "routeContractEOF") + contract, ok := raw.(map[string]any) + if !ok { + return nil, false + } + clean := map[string]any{} + for _, key := range []string{"model", "baseUrl", "identitySha256"} { + value, present := contract[key] + if !present { + continue + } + text, ok := value.(string) + if !ok { + return nil, false + } + clean[key] = text + } + models["routeContractEOF"] = clean + return models, true +} + +func evalProviderClient(route map[string]any, eff core.EffectiveSettings) *agentllm.Client { base, _ := route["baseUrl"].(string) key, _ := route["apiKey"].(string) model, _ := route["model"].(string) - provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream - return evalProviderStreamWithProvider(provider, dir) + client := &agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")} + client.RouteContract = runtime.RouteContractFor(eff, runtime.ResolvedModel{Model: model, Base: base, Key: key}) + return client +} + +func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc { + return evalProviderStreamWithSettings(route, core.EffectiveSettings{}, dir) +} + +func evalProviderStreamWithSettings(route map[string]any, eff core.EffectiveSettings, dir string) agentllm.StreamFunc { + return evalProviderStreamWithProvider(evalProviderClient(route, eff).Stream, dir) } // evalCallRecord contains only accounting and terminal facts. Historical @@ -447,6 +489,7 @@ type evalCallRecord struct { HTTPStatus int `json:"httpStatus"` FinishReason string `json:"finishReason"` StreamComplete bool `json:"streamComplete"` + CompletionBasis string `json:"completionBasis,omitempty"` TerminalObservation *agentllm.TerminalObservation `json:"terminalObservation,omitempty"` ToolArgumentBytes int `json:"toolArgumentBytes"` ToolArgumentsValid bool `json:"toolArgumentsValid"` @@ -479,6 +522,21 @@ func evalLabel(value, fallback string, allowed ...string) string { return fallback } +// Absent completion facts stay absent; never record free text. +func evalCompletionBasis(value string) string { + if value == "" { + return "" + } + return evalLabel(value, "unknown", "done_marker", "route_contract_eof") +} + +func evalContractCheck(value string) string { + if value == "" { + return "" + } + return evalLabel(value, "unknown", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9") +} + func clampCount(value, limit int64) int64 { if value < 0 { return 0 @@ -503,7 +561,9 @@ func evalTerminalObservation(observation *agentllm.TerminalObservation) *agentll ToolArgumentsValid: observation.ToolArgumentsValid, TailEventKind: evalLabel(observation.TailEventKind, "unknown", "done_literal", "json_frame", "comment", "empty_data", "partial_frame", "unknown_non_json", "unknown"), - TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)), + TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)), + CompletionBasis: evalCompletionBasis(observation.CompletionBasis), + ContractCheck: evalContractCheck(observation.ContractCheck), } if body := observation.BodyBoundary; body != nil { clean.BodyBoundary = &agentllm.BodyBoundaryObservation{ @@ -667,12 +727,13 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag if terminal != nil && fault { terminal.Message.ToolCalls = nil } - finishReason, streamComplete := "", false + finishReason, streamComplete, completionBasis := "", false, "" var returnedUsage agentllm.Usage var terminalObservation *agentllm.TerminalObservation httpStatus := 0 if terminal != nil { finishReason, streamComplete = evalFinishReasonClass(terminal.FinishReason), terminal.StreamComplete + completionBasis = evalCompletionBasis(terminal.CompletionBasis) returnedUsage = terminal.Usage terminalObservation = evalTerminalObservation(terminal.TerminalObservation) httpStatus = terminal.HTTPStatus @@ -702,7 +763,7 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag record := evalCallRecord{RunID: runID, RunKey: runKey, Call: a.Calls, Model: m, OutputCap: opts.MaxTokens, ReportedTokens: usage, UsagePresent: usage > 0, Usage: returnedUsage, HTTPStatus: httpStatus, FinishReason: finishReason, - StreamComplete: streamComplete, TerminalObservation: terminalObservation, + StreamComplete: streamComplete, CompletionBasis: completionBasis, TerminalObservation: terminalObservation, ToolArgumentBytes: argumentBytes, ToolArgumentsValid: argumentsValid, FailureClass: failureClass} records = append(records, record)