Keep material records literal and preserve production evaluation limits
Why: Authentic quotes do not establish an unstated approval, and the evaluation wrapper must honor tighter production output limits. Evidence: Approval-overstatement and 780-token-cap regressions failed first; complete Backend suite and fixture cap checks now pass. Impact: Render material records from trusted labels and validated quotations, reserve interpretation for explicit finding kinds, bump recipe identity and cap the fixture without overriding smaller limits. Refs: Approved R05; independent model assessment; original failed samples preserved
This commit is contained in:
@@ -121,7 +121,7 @@ func (c *runCtx) materialRevision() string {
|
||||
if perDoc <= 0 {
|
||||
perDoc = materialPerDocDef
|
||||
}
|
||||
recipe := "material-recipe.v5"
|
||||
recipe := "material-recipe.v6"
|
||||
if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil {
|
||||
if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil {
|
||||
recipe += "\n" + body
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -51,6 +52,21 @@ func (c *runCtx) PublishWorkCheckpoint(candidate core.WorkCheckpointCandidate) e
|
||||
witnesses[src.ArtifactID] = store.ArtifactWitness{ArtifactID: src.ArtifactID, UpdatedAt: a.UpdatedAt.Format(time.RFC3339Nano)}
|
||||
next.Sources = append(next.Sources, src)
|
||||
}
|
||||
if f.Kind == "material_record" {
|
||||
// A quotation proves what the source says, not an unstated approval
|
||||
// or model paraphrase. Keep interpretations in their own finding kinds.
|
||||
quotes := make([]string, 0, len(next.Sources))
|
||||
seen := map[string]bool{}
|
||||
for _, src := range next.Sources {
|
||||
line := src.Label + ":「" + src.Quote + "」"
|
||||
if !seen[line] {
|
||||
quotes = append(quotes, line)
|
||||
seen[line] = true
|
||||
}
|
||||
}
|
||||
next.Text = strings.Join(quotes, "\n")
|
||||
next.Impact, next.Assumptions = "", ""
|
||||
}
|
||||
findings = append(findings, next)
|
||||
}
|
||||
card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID)
|
||||
|
||||
@@ -41,6 +41,25 @@ func validCheckpoint(c *runCtx, id string) core.WorkCheckpointCandidate {
|
||||
}
|
||||
return core.WorkCheckpointCandidate{CheckpointID: id, PhaseID: "crosscheck", Focus: "核对决定", Remaining: "尚在核对", NextStep: "继续整理", Findings: []core.WorkFindingCandidate{{FindingKey: "date", Text: "材料记载上线六月", Kind: "material_record", Impact: "安排交付需要采用该日期", Sources: []core.TaskSourceQuote{{SourceID: src.ID, BlockIndex: src.BlockIndex, Quote: "上线六月"}}}}}
|
||||
}
|
||||
|
||||
func TestMaterialRecordCannotAddAnUnstatedApproval(t *testing.T) {
|
||||
c := checkpointEnvironment(t)
|
||||
cp := validCheckpoint(c, "approval")
|
||||
cp.Findings[0].Text = "上线六月已经批准,无需再次确认"
|
||||
cp.Findings[0].Impact = "批准状态已经明确"
|
||||
cp.Findings[0].Assumptions = "审批已完成"
|
||||
if err := c.PublishWorkCheckpoint(cp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := card.Task.Findings[0]
|
||||
if strings.Contains(f.Text, "批准") || f.Impact != "" || f.Assumptions != "" || !strings.Contains(f.Text, "上线六月") {
|
||||
t.Fatalf("model interpretation was published as a material record: %+v", f)
|
||||
}
|
||||
}
|
||||
func TestTaskCheckpointPersistentIdempotentAndCorrected(t *testing.T) {
|
||||
c := checkpointEnvironment(t)
|
||||
cp := validCheckpoint(c, "1")
|
||||
@@ -148,6 +167,8 @@ func TestTaskCheckpointCorrectionWithdrawsOldClaimEvenWhenSnapshotIsFull(t *test
|
||||
cp = validCheckpoint(c, "large-correction")
|
||||
cp.Findings[0].Corrects = "date"
|
||||
cp.Findings[0].Text = strings.Repeat("y", 11000)
|
||||
cp.Findings[0].Kind = "analysis"
|
||||
cp.Findings[0].Assumptions = "来源尚需复核,撤回旧结论"
|
||||
if e := c.PublishWorkCheckpoint(cp); e != nil {
|
||||
t.Fatal("known wrong claim must be withdrawn", e)
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ func checkpointEnabled(rc RunCtx) bool {
|
||||
return true
|
||||
}
|
||||
func checkpointDecl() agentllm.ToolDecl {
|
||||
return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段;分析判断写明前提。长材料在形成阶段结果时发布,不额外调用摘要模型;不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮,也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)}
|
||||
return agentllm.ToolDecl{Name: workCheckpointTool, Description: "将本次材料整理中已形成的有用发现及时保存到同一任务卡。只引用已提供的 sourceId/blockIndex 和原文片段;material_record由宿主展示经核验的原文引用,不采用模型转述或影响判断;归纳、推算或判断用analysis/provisional并写明assumptions。长材料在形成阶段结果时发布,不额外调用摘要模型;不要为了进度凑发现。普通发布最多6次。需要更正以前发现时用 corrects 填其 findingKey。此工具不创建确认按钮,也不表示暂停执行。", Parameters: json.RawMessage(`{"type":"object","additionalProperties":false,"required":["checkpointId","phaseId","findings"],"properties":{"checkpointId":{"type":"string"},"phaseId":{"type":"string","enum":["reading","crosscheck","summarizing","draft-ready"]},"focus":{"type":"string"},"remaining":{"type":"string"},"nextStep":{"type":"string"},"findings":{"type":"array","minItems":1,"maxItems":6,"items":{"type":"object","additionalProperties":false,"required":["findingKey","text","kind","sources"],"properties":{"findingKey":{"type":"string"},"text":{"type":"string"},"impact":{"type":"string"},"kind":{"type":"string","enum":["material_record","analysis","provisional","needs_user"]},"assumptions":{"type":"string"},"corrects":{"type":"string"},"sources":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["sourceId","blockIndex","quote"],"properties":{"sourceId":{"type":"string"},"blockIndex":{"type":"integer","minimum":0},"quote":{"type":"string"}}}}}}}}}`)}
|
||||
}
|
||||
func publishCheckpoint(rc RunCtx, args string) agenttools.Result {
|
||||
if !checkpointEnabled(rc) {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//go:build live
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
)
|
||||
|
||||
func TestEvaluationPreservesProductionOutputCap(t *testing.T) {
|
||||
for _, tc := range []struct{ supplied, want int }{{780, 780}, {5000, 4000}, {0, 4000}} {
|
||||
t.Run(fmt.Sprint(tc.supplied), func(t *testing.T) {
|
||||
got := make(chan int, 1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
got <- body.MaxTokens
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\ndata: [DONE]\n\n")
|
||||
}))
|
||||
defer srv.Close()
|
||||
stream := evalProviderStream(map[string]any{"baseUrl": srv.URL, "apiKey": "fixture", "model": "fixture"}, t.TempDir())
|
||||
for range stream(context.Background(), "fixture", []agentllm.Message{{Role: agentllm.RoleUser, Content: "check cap"}}, nil, agentllm.StreamOptions{MaxTokens: tc.supplied}) {
|
||||
}
|
||||
if actual := <-got; actual != tc.want {
|
||||
t.Fatalf("evaluation overwrote production cap: got %d want %d", actual, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -290,7 +290,9 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
|
||||
a.Calls++
|
||||
groups[runKey] = a
|
||||
mu.Unlock()
|
||||
if opts.MaxTokens <= 0 || opts.MaxTokens > 4000 {
|
||||
opts.MaxTokens = 4000
|
||||
}
|
||||
reqCtx, cancel := context.WithDeadline(ctx, a.Started.Add(300*time.Second))
|
||||
in := provider(reqCtx, m, msgs, decls, opts)
|
||||
out := make(chan agentllm.Event)
|
||||
|
||||
Reference in New Issue
Block a user