From 18aa99cb95b50eab72b6132ae006278b9df78201 Mon Sep 17 00:00:00 2001 From: weichen Date: Sat, 10 Oct 2026 07:48:03 +0000 Subject: [PATCH] Keep the final reply clean when checks continue or the verifier is skipped Why: real run run_b520bf2e3037 showed the user-facing reply being displaced by internal bookkeeping - the fact-check correction round's closing text plus the same-source verifier skip notice became the visible answer while the actual summary was folded into the process timeline (frontend splits at the last process part). Same tail pollution reproduced on every same-model run since. Evidence: go test ./... -count=1 green (32 pkgs); new regression tests cover the demotion of the pre-check summary and the skip notice becoming a check part; three old-contract tests updated to assert notices land as process checks instead of final text. Impact: stop() fact-check branch now demotes the standing summary (aligned with the gate and cap branches) and asks the model for a fresh full reply; verifier errors record as process checks (pass-state for same-source skip, fail-state for real errors) instead of StopDecision.Notice text; the cap notice likewise becomes a failed check line. Refs: Dev task feature/final-reply-pollution-fix; diagnosis in .task/feature-dual-speed-tasks.md lineage and session analysis 2026-10-10. --- internal/runtime/final_reply_test.go | 97 ++++++++++++++++++++++++++++ internal/runtime/llm_direct.go | 22 +++++-- internal/runtime/llm_verify_test.go | 8 ++- internal/runtime/wp5_ab_test.go | 9 ++- 4 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 internal/runtime/final_reply_test.go diff --git a/internal/runtime/final_reply_test.go b/internal/runtime/final_reply_test.go new file mode 100644 index 0000000..6bf8e53 --- /dev/null +++ b/internal/runtime/final_reply_test.go @@ -0,0 +1,97 @@ +package runtime + +import ( + "context" + "strings" + "testing" + + core "agent-system-core" +) + +// 问题一回归（run_b520bf2e3037）：事实核查修正轮触发时，现场那段正式总结必须 +// 降级进过程时间线；修正轮的收尾才是最终答复——否则前端按「最后一个过程部件」 +// 切分后，用户只看到修正流水账，看不到原总结。 +func TestFactCheckDemotesPriorSummary(t *testing.T) { + rc := &fakeRunCtx{ + ctx: context.Background(), + prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}, + verify: true, + execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "报告.md"}, + execArtifactContent: []byte("# 报告\n\n准确率 95%。"), + } + turns := padTurns( + textDeltaTurn("", toolCall("make_report", `{"content":"准确率 95%"}`)), + textDeltaTurn("已完成。## 产出物\n\n结构完整的用户总结。"), + textDeltaTurn("已修正并重新登记。"), + ) + r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()} + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + var summary *core.Part + visible := []string{} + sawFactCheck := false + for i := range rc.assistant { + p := rc.assistant[i] + if p.Type == core.PartText && strings.Contains(p.Text, "结构完整的用户总结") { + summary = &rc.assistant[i] + } + if p.Type == core.PartText && !p.Process { + visible = append(visible, p.Text) + } + if p.Check != nil && !p.Check.Passed && strings.Contains(p.Check.Reason, "事实核查") { + sawFactCheck = true + } + } + if summary == nil { + t.Fatal("prior summary text missing") + } + if !summary.Process { + t.Fatalf("prior summary not demoted to process: %+v", *summary) + } + if !sawFactCheck { + t.Fatal("fact check note missing") + } + if len(visible) != 1 || visible[0] != "已修正并重新登记。" { + t.Fatalf("visible final texts = %v, want only the continuation closing", visible) + } +} + +// 同源跳过提示不再作为用户可见正文追加（原先经 StopDecision.Notice 顶掉正式总结）， +// 改记为通过态检查行留在过程时间线。 +func TestVerifierSkipIsCheckNotFinalText(t *testing.T) { + rc := &fakeRunCtx{ + ctx: context.Background(), + prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}, // 轻量回落=主模型 → 同源跳过 + verify: true, + } + turns := padTurns(textDeltaTurn("总结完成。")) + r := &LLMRunner{Stream: scriptStream(turns, nil)} // 无 VerifyStream + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + skipCheck, noticeText := false, false + visible := []string{} + for _, p := range rc.assistant { + if p.Check != nil && p.Check.Passed && strings.Contains(p.Check.Reason, "同源模型验证") { + skipCheck = true + } + if p.Type == core.PartText { + if strings.Contains(p.Text, verifierUnavailableNotice) { + noticeText = true + } + if !p.Process { + visible = append(visible, p.Text) + } + } + } + if !skipCheck { + t.Fatal("same-source skip not recorded as a passing check") + } + if noticeText { + t.Fatal("skip notice leaked into message text parts") + } + if len(visible) != 1 || visible[0] != "总结完成。" { + t.Fatalf("visible final texts = %v", visible) + } +} diff --git a/internal/runtime/llm_direct.go b/internal/runtime/llm_direct.go index 1ba3f70..5dabe90 100644 --- a/internal/runtime/llm_direct.go +++ b/internal/runtime/llm_direct.go @@ -537,15 +537,26 @@ func (st *directRun) stop(last agentllm.Message, info nruntime.StopInfo) nruntim if len(pairs) > 0 { if tokens := UnverifiedFacts(concatArtifactText(pairs), st.materialCorpus(), factExemptions()); len(tokens) > 0 { st.factChecked = true + // 与交付物门禁/续写上限分支对齐：检查未通过时，现场那段总结归入 + // 过程时间线——修正轮之后的收尾才是最终答复（实测 run_b520bf2e3037 + // 未降级导致好总结被前端切分折叠，用户只看到修正流水账）。 + st.demoteLastText() st.addVerifyNote("事实核查：" + factCheckNotice(tokens)) - return nruntime.StopDecision{Continue: true, Reason: systemCheckReason("终轮事实核查发现以下内容在材料中找不到出处，请修正后重新生成/登记交付物。\n" + factCheckNotice(tokens))} + return nruntime.StopDecision{Continue: true, Reason: systemCheckReason("终轮事实核查发现以下内容在材料中找不到出处，请修正后重新生成/登记交付物，然后重新给出更新后的完整最终答复（先结论、再产出物与要点，可注明本次修正）。\n" + factCheckNotice(tokens))} } } } verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, latest) if err != nil { - st.addVerifyNote(err.Error()) - return nruntime.StopDecision{Notice: verifierUnavailableNotice} + // 验证器没跑成不再是用户可见正文（实测 Notice 会顶掉正式总结）。 + // 同源跳过是能力说明，记通过态检查行；真失败记未通过态检查行， + // 过程时间线里都可见，最终答复只留模型自己的收尾。 + if strings.Contains(err.Error(), "跳过同源验证") { + st.addCheck(true, verifierUnavailableNotice) + } else { + st.addVerifyNote(err.Error()) + } + return nruntime.StopDecision{} } if !verdict.Passed { reason := strings.TrimSpace(verdict.Reason) @@ -640,7 +651,10 @@ func (st *directRun) continueOrNotice(info nruntime.StopInfo, reason string) nru return nruntime.StopDecision{Continue: true, Reason: systemCheckReason(reason)} } st.addVerifyNote(reason) - return nruntime.StopDecision{Notice: verifyCapNotice} + // 达到续写上限同样收进检查行（未通过态）：让用户看见检查确实没过， + // 而不是用一句系统提示替换掉模型的最终答复。 + st.addCheck(false, verifyCapNotice) + return nruntime.StopDecision{} } func (st *directRun) runTool(name, args string) agenttools.Result { diff --git a/internal/runtime/llm_verify_test.go b/internal/runtime/llm_verify_test.go index c6f474d..95e449a 100644 --- a/internal/runtime/llm_verify_test.go +++ b/internal/runtime/llm_verify_test.go @@ -214,16 +214,18 @@ func TestVerifierGarbageFailsClosed(t *testing.T) { t.Fatalf("main calls = %d, want 1", mainCalls) } texts := textParts(rc.assistant) - var kept, notice bool + var kept bool + notice := false for _, p := range texts { if p.Text == "答案是 42。" && !p.Process { kept = true } - if !p.Process && strings.Contains(p.Text, "验证器未独立配置") { + if !p.Process && strings.Contains(p.Text, "验证器") { notice = true } } - if !kept || !notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") && !hasProcessNote(rc.assistant, "未独立配置") { + // 2026-10-10 契约更新：验证器不可用不再作为正文 Notice，落成过程检查行。 + if !kept || notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") { t.Fatalf("parts = %+v", texts) } }) diff --git a/internal/runtime/wp5_ab_test.go b/internal/runtime/wp5_ab_test.go index ab4c9d5..084d500 100644 --- a/internal/runtime/wp5_ab_test.go +++ b/internal/runtime/wp5_ab_test.go @@ -45,8 +45,13 @@ func TestVerifyCapNoticeAfterThreeContinuations(t *testing.T) { } texts := textParts(rc.assistant) last := texts[len(texts)-1] - if last.Process || !strings.Contains(last.Text, "本轮回复未通过终轮检查") { - t.Fatalf("final notice = %+v, want non-process verify cap notice", last) + // 2026-10-10 契约更新：封顶提示不再作为正文 Notice；最后一次回复留在正式区， + // 封顶事实以未通过态检查行收进过程区。 + if last.Process || last.Text != "答案 v4。" { + t.Fatalf("final text = %+v, want the model's own last reply", last) + } + if !hasProcessNote(rc.assistant, "本轮回复未通过终轮检查") { + t.Fatalf("cap check note missing: %+v", rc.assistant) } // 理由经 CheckPart 进过程区：3 次续跑各记一次，达上限时再记一次（与旧行为一致）。 notes := 0