From ee65604d831caed768f6c798cda6122acb33f13d Mon Sep 17 00:00:00 2001 From: weichen Date: Tue, 13 Oct 2026 04:16:31 +0000 Subject: [PATCH] Keep material records literal and preserve production evaluation limits Why: Authentic quotes do not establish an unstated approval, and the evaluation wrapper must honor tighter production output limits. Evidence: Approval-overstatement and 780-token-cap regressions failed first; complete Backend suite and fixture cap checks now pass. Impact: Render material records from trusted labels and validated quotations, reserve interpretation for explicit finding kinds, bump recipe identity and cap the fixture without overriding smaller limits. Refs: Approved R05; independent model assessment; original failed samples preserved --- internal/runmgr/material_sources.go | 2 +- internal/runmgr/task_checkpoint.go | 16 ++++++++ internal/runmgr/task_checkpoint_test.go | 21 ++++++++++ internal/runtime/work_checkpoint.go | 2 +- .../work_checkpoint_eval_budget_test.go | 40 +++++++++++++++++++ internal/server/work_checkpoint_eval_test.go | 4 +- 6 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 internal/server/work_checkpoint_eval_budget_test.go diff --git a/internal/runmgr/material_sources.go b/internal/runmgr/material_sources.go index a7240ed..10d760c 100644 --- a/internal/runmgr/material_sources.go +++ b/internal/runmgr/material_sources.go @@ -121,7 +121,7 @@ func (c *runCtx) materialRevision() string { if perDoc <= 0 { perDoc = materialPerDocDef } - recipe := "material-recipe.v5" + recipe := "material-recipe.v6" if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil { if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil { recipe += "\n" + body diff --git a/internal/runmgr/task_checkpoint.go b/internal/runmgr/task_checkpoint.go index c3f3b41..217438c 100644 --- a/internal/runmgr/task_checkpoint.go +++ b/internal/runmgr/task_checkpoint.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "strings" "time" ) @@ -51,6 +52,21 @@ func (c *runCtx) PublishWorkCheckpoint(candidate core.WorkCheckpointCandidate) e witnesses[src.ArtifactID] = store.ArtifactWitness{ArtifactID: src.ArtifactID, UpdatedAt: a.UpdatedAt.Format(time.RFC3339Nano)} next.Sources = append(next.Sources, src) } + if f.Kind == "material_record" { + // A quotation proves what the source says, not an unstated approval + // or model paraphrase. Keep interpretations in their own finding kinds. + quotes := make([]string, 0, len(next.Sources)) + seen := map[string]bool{} + for _, src := range next.Sources { + line := src.Label + "：「" + src.Quote + "」" + if !seen[line] { + quotes = append(quotes, line) + seen[line] = true + } + } + next.Text = strings.Join(quotes, "\n") + next.Impact, next.Assumptions = "", "" + } findings = append(findings, next) } card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID) diff --git a/internal/runmgr/task_checkpoint_test.go b/internal/runmgr/task_checkpoint_test.go index 63501c9..240c847 100644 --- a/internal/runmgr/task_checkpoint_test.go +++ b/internal/runmgr/task_checkpoint_test.go @@ -41,6 +41,25 @@ func validCheckpoint(c *runCtx, id string) core.WorkCheckpointCandidate { } return core.WorkCheckpointCandidate{CheckpointID: id, PhaseID: "crosscheck", Focus: "核对决定", Remaining: "尚在核对", NextStep: "继续整理", Findings: []core.WorkFindingCandidate{{FindingKey: "date", Text: "材料记载上线六月", Kind: "material_record", Impact: "安排交付需要采用该日期", Sources: []core.TaskSourceQuote{{SourceID: src.ID, BlockIndex: src.BlockIndex, Quote: "上线六月"}}}}} } + +func TestMaterialRecordCannotAddAnUnstatedApproval(t *testing.T) { + c := checkpointEnvironment(t) + cp := validCheckpoint(c, "approval") + cp.Findings[0].Text = "上线六月已经批准，无需再次确认" + cp.Findings[0].Impact = "批准状态已经明确" + cp.Findings[0].Assumptions = "审批已完成" + if err := c.PublishWorkCheckpoint(cp); err != nil { + t.Fatal(err) + } + card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID) + if err != nil { + t.Fatal(err) + } + f := card.Task.Findings[0] + if strings.Contains(f.Text, "批准") || f.Impact != "" || f.Assumptions != "" || !strings.Contains(f.Text, "上线六月") { + t.Fatalf("model interpretation was published as a material record: %+v", f) + } +} func TestTaskCheckpointPersistentIdempotentAndCorrected(t *testing.T) { c := checkpointEnvironment(t) cp := validCheckpoint(c, "1") @@ -148,6 +167,8 @@ func TestTaskCheckpointCorrectionWithdrawsOldClaimEvenWhenSnapshotIsFull(t *test cp = validCheckpoint(c, "large-correction") cp.Findings[0].Corrects = "date" cp.Findings[0].Text = strings.Repeat("y", 11000) + cp.Findings[0].Kind = "analysis" + cp.Findings[0].Assumptions = "来源尚需复核，撤回旧结论" if e := c.PublishWorkCheckpoint(cp); e != nil { t.Fatal("known wrong claim must be withdrawn", e) } diff --git a/internal/runtime/work_checkpoint.go b/internal/runtime/work_checkpoint.go index 4b45234..98a5351 100644 --- a/internal/runtime/work_checkpoint.go +++ b/internal/runtime/work_checkpoint.go @@ -38,7 +38,7 @@ func checkpointEnabled(rc RunCtx) bool { return true } func checkpointDecl() agentllm.ToolDecl { - return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段；分析判断写明前提。长材料在形成阶段结果时发布，不额外调用摘要模型；不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮，也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)} + return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段；material_record由宿主展示经核验的原文引用，不采用模型转述或影响判断；归纳、推算或判断用analysis/provisional并写明assumptions。长材料在形成阶段结果时发布，不额外调用摘要模型；不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮，也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)} } func publishCheckpoint(rc RunCtx, args string) agenttools.Result { if !checkpointEnabled(rc) { diff --git a/internal/server/work_checkpoint_eval_budget_test.go b/internal/server/work_checkpoint_eval_budget_test.go new file mode 100644 index 0000000..558ad70 --- /dev/null +++ b/internal/server/work_checkpoint_eval_budget_test.go @@ -0,0 +1,40 @@ +//go:build live + +package server + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + agentllm "agent-native/runtime/llm" +) + +func TestEvaluationPreservesProductionOutputCap(t *testing.T) { + for _, tc := range []struct{ supplied, want int }{{780, 780}, {5000, 4000}, {0, 4000}} { + t.Run(fmt.Sprint(tc.supplied), func(t *testing.T) { + got := make(chan int, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + MaxTokens int `json:"max_tokens"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + got <- body.MaxTokens + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n") + })) + defer srv.Close() + stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, t.TempDir()) + for range stream(context.Background(), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "check cap"}}, nil, agentllm.StreamOptions{MaxTokens: tc.supplied}) { + } + if actual := <-got; actual != tc.want { + t.Fatalf("evaluation overwrote production cap: got %d want %d", actual, tc.want) + } + }) + } +} diff --git a/internal/server/work_checkpoint_eval_test.go b/internal/server/work_checkpoint_eval_test.go index a8e12dd..ed1cef2 100644 --- a/internal/server/work_checkpoint_eval_test.go +++ b/internal/server/work_checkpoint_eval_test.go @@ -290,7 +290,9 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc { a.Calls++ groups[runKey] = a mu.Unlock() - opts.MaxTokens = 4000 + if opts.MaxTokens <= 0 || opts.MaxTokens > 4000 { + opts.MaxTokens = 4000 + } reqCtx, cancel := context.WithDeadline(ctx, a.Started.Add(300*time.Second)) in := provider(reqCtx, m, msgs, decls, opts) out := make(chan agentllm.Event)