Keep material records literal and preserve production evaluation limits

Why: Authentic quotes do not establish an unstated approval, and the evaluation wrapper must honor tighter production output limits.

Evidence: Approval-overstatement and 780-token-cap regressions failed first; complete Backend suite and fixture cap checks now pass.

Impact: Render material records from trusted labels and validated quotations, reserve interpretation for explicit finding kinds, bump recipe identity and cap the fixture without overriding smaller limits.

Refs: Approved R05; independent model assessment; original failed samples preserved
This commit is contained in:
2026-10-13 04:16:31 +00:00
parent f6b536029a
commit ee65604d83
6 changed files with 82 additions and 3 deletions
+1 -1
View File
@@ -121,7 +121,7 @@ func (c *runCtx) materialRevision() string {
if perDoc <= 0 {
perDoc = materialPerDocDef
}
recipe := "material-recipe.v5"
recipe := "material-recipe.v6"
if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil {
if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil {
recipe += "\n" + body
+16
View File
@@ -7,6 +7,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"strings"
"time"
)
@@ -51,6 +52,21 @@ func (c *runCtx) PublishWorkCheckpoint(candidate core.WorkCheckpointCandidate) e
witnesses[src.ArtifactID] = store.ArtifactWitness{ArtifactID: src.ArtifactID, UpdatedAt: a.UpdatedAt.Format(time.RFC3339Nano)}
next.Sources = append(next.Sources, src)
}
if f.Kind == "material_record" {
// A quotation proves what the source says, not an unstated approval
// or model paraphrase. Keep interpretations in their own finding kinds.
quotes := make([]string, 0, len(next.Sources))
seen := map[string]bool{}
for _, src := range next.Sources {
line := src.Label + ":「" + src.Quote + "」"
if !seen[line] {
quotes = append(quotes, line)
seen[line] = true
}
}
next.Text = strings.Join(quotes, "\n")
next.Impact, next.Assumptions = "", ""
}
findings = append(findings, next)
}
card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID)
+21
View File
@@ -41,6 +41,25 @@ func validCheckpoint(c *runCtx, id string) core.WorkCheckpointCandidate {
}
return core.WorkCheckpointCandidate{CheckpointID: id, PhaseID: "crosscheck", Focus: "核对决定", Remaining: "尚在核对", NextStep: "继续整理", Findings: []core.WorkFindingCandidate{{FindingKey: "date", Text: "材料记载上线六月", Kind: "material_record", Impact: "安排交付需要采用该日期", Sources: []core.TaskSourceQuote{{SourceID: src.ID, BlockIndex: src.BlockIndex, Quote: "上线六月"}}}}}
}
func TestMaterialRecordCannotAddAnUnstatedApproval(t *testing.T) {
c := checkpointEnvironment(t)
cp := validCheckpoint(c, "approval")
cp.Findings[0].Text = "上线六月已经批准,无需再次确认"
cp.Findings[0].Impact = "批准状态已经明确"
cp.Findings[0].Assumptions = "审批已完成"
if err := c.PublishWorkCheckpoint(cp); err != nil {
t.Fatal(err)
}
card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID)
if err != nil {
t.Fatal(err)
}
f := card.Task.Findings[0]
if strings.Contains(f.Text, "批准") || f.Impact != "" || f.Assumptions != "" || !strings.Contains(f.Text, "上线六月") {
t.Fatalf("model interpretation was published as a material record: %+v", f)
}
}
func TestTaskCheckpointPersistentIdempotentAndCorrected(t *testing.T) {
c := checkpointEnvironment(t)
cp := validCheckpoint(c, "1")
@@ -148,6 +167,8 @@ func TestTaskCheckpointCorrectionWithdrawsOldClaimEvenWhenSnapshotIsFull(t *test
cp = validCheckpoint(c, "large-correction")
cp.Findings[0].Corrects = "date"
cp.Findings[0].Text = strings.Repeat("y", 11000)
cp.Findings[0].Kind = "analysis"
cp.Findings[0].Assumptions = "来源尚需复核,撤回旧结论"
if e := c.PublishWorkCheckpoint(cp); e != nil {
t.Fatal("known wrong claim must be withdrawn", e)
}
+1 -1
View File
@@ -38,7 +38,7 @@ func checkpointEnabled(rc RunCtx) bool {
return true
}
func checkpointDecl() agentllm.ToolDecl {
return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段;分析判断写明前提。长材料在形成阶段结果时发布,不额外调用摘要模型;不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮,也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)}
return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段;material_record由宿主展示经核验的原文引用,不采用模型转述或影响判断;归纳、推算或判断用analysis/provisional并写明assumptions。长材料在形成阶段结果时发布,不额外调用摘要模型;不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮,也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)}
}
func publishCheckpoint(rc RunCtx, args string) agenttools.Result {
if !checkpointEnabled(rc) {
@@ -0,0 +1,40 @@
//go:build live
package server
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
agentllm "agent-native/runtime/llm"
)
func TestEvaluationPreservesProductionOutputCap(t *testing.T) {
for _, tc := range []struct{ supplied, want int }{{780, 780}, {5000, 4000}, {0, 4000}} {
t.Run(fmt.Sprint(tc.supplied), func(t *testing.T) {
got := make(chan int, 1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
MaxTokens int `json:"max_tokens"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
got <- body.MaxTokens
w.Header().Set("Content-Type", "text/event-stream")
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
}))
defer srv.Close()
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, t.TempDir())
for range stream(context.Background(), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "check cap"}}, nil, agentllm.StreamOptions{MaxTokens: tc.supplied}) {
}
if actual := <-got; actual != tc.want {
t.Fatalf("evaluation overwrote production cap: got %d want %d", actual, tc.want)
}
})
}
}
@@ -290,7 +290,9 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
a.Calls++
groups[runKey] = a
mu.Unlock()
if opts.MaxTokens <= 0 || opts.MaxTokens > 4000 {
opts.MaxTokens = 4000
}
reqCtx, cancel := context.WithDeadline(ctx, a.Started.Add(300*time.Second))
in := provider(reqCtx, m, msgs, decls, opts)
out := make(chan agentllm.Event)