From 188c63357cccd7fe028af23621396e51be59f449 Mon Sep 17 00:00:00 2001 From: weichen Date: Thu, 1 Oct 2026 12:29:38 +0000 Subject: [PATCH] Replay tool calls natively and tighten the deliverable gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: Flattened "[调用工具 …]" history taught the model to imitate calls and invent artifact ids. A single regex correction then accepted the second false claim. Evidence: ctxasm replay tests cover native tool messages, missing-result pairing, argument compaction, demoted text, and forged-marker stripping. Runtime tests cover forged markers, extension-less claims, two corrections, and the fallback notice. Impact: Assistant history sent to the model no longer contains flattened tool calls. The direct runner corrects a false deliverable twice, then appends a notice instead of accepting the claim. The squad entry still fails closed on the same predicate. base.deliverables is now v5. Refs: feature/agent-harness-p0 Co-authored-by: Cursor --- internal/ctxasm/ctxasm.go | 202 ++++++++++++++++-- internal/ctxasm/replay_test.go | 180 ++++++++++++++++ internal/prompts/library/base-deliverables.md | 4 +- internal/runtime/llm.go | 59 +++-- internal/runtime/llm_test.go | 105 ++++++++- 5 files changed, 508 insertions(+), 42 deletions(-) create mode 100644 internal/ctxasm/replay_test.go diff --git a/internal/ctxasm/ctxasm.go b/internal/ctxasm/ctxasm.go index 3dd4065..ad34f3c 100644 --- a/internal/ctxasm/ctxasm.go +++ b/internal/ctxasm/ctxasm.go @@ -2,7 +2,9 @@ package ctxasm import ( + "encoding/json" "fmt" + "regexp" "strings" "time" "unicode/utf8" @@ -78,15 +80,15 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [ if m.ID == currentUser.ID { continue } + if m.Role == core.RoleAssistant { + msgs = append(msgs, renderAssistantTurns(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled)...) + continue + } content := renderMessage(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled) if content == "" { continue } - role := agentllm.RoleUser - if m.Role == core.RoleAssistant { - role = agentllm.RoleAssistant - } - msgs = append(msgs, agentllm.Message{Role: role, Content: content}) + msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: content}) } rep.Items = append(rep.Items, fmt.Sprintf("history(%d)", len(history)-start)) if cardsBackfilled > 0 { @@ -134,23 +136,23 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [ return msgs, rep } -// renderMessage 把消息部件渲染为模型可读文本（思考部件不回灌；工具结果给摘要； -// card 引用部件回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations）。 +// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值，保持合法 JSON。 +const maxToolArgRunes = 4000 + +// forgedMarkerLine 匹配模型在正文里模仿出来的调用/结果/工件行。回放时整行剔除， +// 避免下一轮把这种文本当成「以前就是这样调用工具」的格式。 +var forgedMarkerLine = regexp.MustCompile(`(?m)^[^\n]*\[(?:调用工具|工具结果|产出工件) [^\n]*\n?`) + +// renderMessage 把用户消息部件渲染为模型可读文本（思考部件不回灌； +// card 引用回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations）。 +// 助手消息不走这里：工具调用必须保持提供方的原生消息，见 renderAssistantTurns。 func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) string { var sb strings.Builder for _, p := range m.Parts { switch p.Type { case core.PartText: - sb.WriteString(p.Text) + sb.WriteString(stripForgedMarkers(p.Text)) sb.WriteString("\n") - case core.PartToolCall: - if p.ToolCall != nil { - fmt.Fprintf(&sb, "[调用工具 %s(%s)]\n", p.ToolCall.Name, p.ToolCall.Arguments) - } - case core.PartToolResult: - if p.ToolResult != nil { - fmt.Fprintf(&sb, "[工具结果 %s %s]\n", p.ToolResult.Status, p.ToolResult.Summary) - } case core.PartArtifactRef: sb.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled)) case core.PartHITL: @@ -164,6 +166,174 @@ func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, reso return strings.TrimSpace(sb.String()) } +// renderAssistantTurns 把一条助手消息切成提供方可接受的若干段： +// 前置文本与 tool_calls 组成一条 assistant，每个调用紧跟一条 tool 结果。 +// 没有结果的调用补一条失败占位，避免模型把中断当成成功。孤立的结果直接跳过。 +// 被守卫降级、且后面不是工具调用的过程文本不回放。 +func renderAssistantTurns(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) []agentllm.Message { + var out []agentllm.Message + var text strings.Builder + var calls []agentllm.ToolCall + results := map[string]core.ToolResultPart{} + seq := 0 + + flush := func() { + content := strings.TrimSpace(text.String()) + if content == "" && len(calls) == 0 { + text.Reset() + results = map[string]core.ToolResultPart{} + return + } + out = append(out, agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls}) + for _, c := range calls { + body := "（无结果：运行中断）" + isErr := true + if res, ok := results[c.ID]; ok { + body = res.Summary + isErr = res.Status == "error" + } + out = append(out, agentllm.Message{ + Role: agentllm.RoleTool, + ToolCallID: c.ID, + Name: c.Name, + Content: body, + IsError: isErr, + }) + } + text.Reset() + calls = nil + results = map[string]core.ToolResultPart{} + } + + for i, p := range m.Parts { + switch p.Type { + case core.PartText: + if demotedProcessText(m.Parts, i) { + continue + } + cleaned := stripForgedMarkers(p.Text) + if strings.TrimSpace(cleaned) == "" { + continue + } + if len(calls) > 0 { + flush() + } + text.WriteString(cleaned) + text.WriteString("\n") + case core.PartToolCall: + if p.ToolCall == nil { + continue + } + id := p.ToolCall.ID + if id == "" { + id = fmt.Sprintf("hist_%s_%d", m.ID, seq) + seq++ + } + calls = append(calls, agentllm.ToolCall{ + ID: id, + Name: p.ToolCall.Name, + Arguments: compactToolArguments(p.ToolCall.Arguments), + }) + case core.PartToolResult: + if p.ToolResult == nil { + continue + } + id := p.ToolResult.CallID + if id == "" { + for _, c := range calls { + if _, ok := results[c.ID]; !ok { + id = c.ID + break + } + } + } + if id == "" || !hasCall(calls, id) { + continue + } + results[id] = *p.ToolResult + case core.PartArtifactRef: + if len(calls) > 0 { + flush() + } + text.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled)) + case core.PartHITL: + if p.Hitl == nil || !p.Hitl.Resolved { + continue + } + if len(calls) > 0 { + flush() + } + fmt.Fprintf(&text, "[用户确认：%s → %s]\n", p.Hitl.Question, p.Hitl.Answer) + case core.PartCard: + if len(calls) > 0 { + flush() + } + text.WriteString(renderCardPart(p.Card, resolveCard, limitCard, rep, cardsBackfilled)) + } + } + flush() + return out +} + +func hasCall(calls []agentllm.ToolCall, id string) bool { + for _, c := range calls { + if c.ID == id { + return true + } + } + return false +} + +// demotedProcessText 是守卫降级后的谎称正文：过程文本，且后面紧跟的不是工具调用。 +// 工具轮的解说也标了过程，但它后面就是 tool_call，要保留。 +func demotedProcessText(parts []core.Part, i int) bool { + if parts[i].Type != core.PartText || !parts[i].Process { + return false + } + return i+1 >= len(parts) || parts[i+1].Type != core.PartToolCall +} + +func stripForgedMarkers(text string) string { + return strings.TrimSpace(forgedMarkerLine.ReplaceAllString(text, "")) +} + +// compactToolArguments 在参数超限时截断其中的长字符串。解析失败或截断后仍超限时， +// 换成仍可被提供方接受的占位对象，而不是一段被截断的非法 JSON。 +func compactToolArguments(raw string) string { + if utf8.RuneCountInString(raw) <= maxToolArgRunes { + return raw + } + var v any + if err := json.Unmarshal([]byte(raw), &v); err != nil { + return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw)) + } + truncateLongStrings(&v, 500) + encoded, err := json.Marshal(v) + if err != nil || utf8.RuneCountInString(string(encoded)) > maxToolArgRunes { + return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw)) + } + return string(encoded) +} + +func truncateLongStrings(v *any, keep int) { + switch x := (*v).(type) { + case map[string]any: + for k, val := range x { + truncateLongStrings(&val, keep) + x[k] = val + } + case []any: + for i := range x { + truncateLongStrings(&x[i], keep) + } + case string: + n := utf8.RuneCountInString(x) + if n > keep { + *v = string([]rune(x)[:keep]) + fmt.Sprintf("…（已省略 %d 字符）", n-keep) + } + } +} + // renderCardPart 渲染 card 部件：引用部件（cardId）回填产物卡全文（超限截断留痕）； // 无 resolver 或无此卡时留占位（可见即信号，不静默丢弃）。 func renderCardPart(cp *core.CardPart, resolveCard CardResolver, limit int, rep *Report, cardsBackfilled *int) string { diff --git a/internal/ctxasm/replay_test.go b/internal/ctxasm/replay_test.go new file mode 100644 index 0000000..0c6d731 --- /dev/null +++ b/internal/ctxasm/replay_test.go @@ -0,0 +1,180 @@ +package ctxasm + +import ( + "encoding/json" + "strings" + "testing" + "unicode/utf8" + + agentllm "agent-native/runtime/llm" + core "agent-system-core" +) + +func assistantWith(id string, parts ...core.Part) core.Message { + return core.Message{ID: id, Role: core.RoleAssistant, Parts: parts} +} + +func assembleHistory(hist []core.Message, resolve ArtifactResolver) []agentllm.Message { + msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, hist, fakeUser("继续"), nil, nil, nil, nil, nil, resolve) + if len(msgs) == 0 { + return nil + } + return msgs[:len(msgs)-1] +} + +func joinedContent(msgs []agentllm.Message) string { + var b strings.Builder + for _, m := range msgs { + b.WriteString(m.Content) + b.WriteByte('\n') + } + return b.String() +} + +func TestAssembleInputReplaysToolCallsNatively(t *testing.T) { + hist := []core.Message{assistantWith("m2", + core.Part{Type: core.PartText, Text: "好的，开始生成。"}, + core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_1", Name: "run_python", Arguments: `{"script":"print(1)"}`}}, + core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_1", Status: "ok", Summary: "完成"}}, + core.Part{Type: core.PartText, Text: "纪要已整理。"}, + core.Part{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "art_1", Name: "纪要.md", Kind: "document"}}, + )} + resolve := func(id core.ID) (core.Artifact, []byte, bool) { + if id == "art_1" { + return core.Artifact{ID: id, Name: "纪要.md"}, []byte("关键结论。"), true + } + return core.Artifact{}, nil, false + } + msgs := assembleHistory(hist, resolve) + if len(msgs) != 3 { + t.Fatalf("replay messages = %d, want assistant + tool + assistant, got %#v", len(msgs), msgs) + } + if msgs[0].Role != agentllm.RoleAssistant || len(msgs[0].ToolCalls) != 1 || msgs[0].ToolCalls[0].Name != "run_python" { + t.Fatalf("first message = %+v, want assistant tool call run_python", msgs[0]) + } + if !strings.Contains(msgs[0].Content, "好的，开始生成。") || strings.Contains(msgs[0].Content, "[调用工具") { + t.Fatalf("preamble must stay text and not flatten the call: %q", msgs[0].Content) + } + if msgs[1].Role != agentllm.RoleTool || msgs[1].ToolCallID != "call_1" || msgs[1].Name != "run_python" || msgs[1].Content != "完成" || msgs[1].IsError { + t.Fatalf("tool result = %+v, want paired ok summary", msgs[1]) + } + if msgs[2].Role != agentllm.RoleAssistant || len(msgs[2].ToolCalls) != 0 { + t.Fatalf("final segment = %+v, want plain assistant text", msgs[2]) + } + if !strings.Contains(msgs[2].Content, "纪要已整理。") || !strings.Contains(msgs[2].Content, "关键结论。") { + t.Fatalf("final segment missing reply or artifact body: %q", msgs[2].Content) + } + if strings.Contains(joinedContent(msgs), "[调用工具") || strings.Contains(joinedContent(msgs), "[工具结果") { + t.Fatalf("flattened markers leaked into replay:\n%s", joinedContent(msgs)) + } +} + +func TestAssembleInputPairsMissingToolResult(t *testing.T) { + hist := []core.Message{assistantWith("m3", + core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_x", Name: "run_python", Arguments: `{}`}}, + )} + msgs := assembleHistory(hist, nil) + if len(msgs) != 2 || msgs[1].Role != agentllm.RoleTool { + t.Fatalf("messages = %#v, want assistant plus synthetic tool result", msgs) + } + if msgs[1].ToolCallID != "call_x" || !msgs[1].IsError || !strings.Contains(msgs[1].Content, "无结果：运行中断") { + t.Fatalf("synthetic result = %+v", msgs[1]) + } +} + +func TestAssembleInputDropsOrphanToolResult(t *testing.T) { + hist := []core.Message{assistantWith("m4", + core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "ghost", Status: "ok", Summary: "不该出现"}}, + core.Part{Type: core.PartText, Text: "只有正文。"}, + )} + msgs := assembleHistory(hist, nil) + for _, m := range msgs { + if m.Role == agentllm.RoleTool || strings.Contains(m.Content, "不该出现") { + t.Fatalf("orphan result leaked: %+v", m) + } + } + if len(msgs) != 1 || !strings.Contains(msgs[0].Content, "只有正文。") { + t.Fatalf("text-only replay = %#v", msgs) + } +} + +func TestAssembleInputFillsEmptyToolCallID(t *testing.T) { + hist := []core.Message{assistantWith("m2", + core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{Name: "run_shell", Arguments: `{"cmd":"ls"}`}}, + core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{Status: "ok", Summary: "a.txt"}}, + )} + msgs := assembleHistory(hist, nil) + if len(msgs) != 2 { + t.Fatalf("messages = %d, want 2", len(msgs)) + } + id := msgs[0].ToolCalls[0].ID + if id != "hist_m2_0" || msgs[1].ToolCallID != id || msgs[1].Content != "a.txt" { + t.Fatalf("empty id pairing = call %q result %+v", id, msgs[1]) + } +} + +func TestAssembleInputCompactsLongToolArguments(t *testing.T) { + script := strings.Repeat("甲", 5000) + raw, err := json.Marshal(map[string]string{"script": script}) + if err != nil { + t.Fatal(err) + } + hist := []core.Message{assistantWith("m5", + core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_long", Name: "run_python", Arguments: string(raw)}}, + core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_long", Status: "ok", Summary: "ok"}}, + )} + msgs := assembleHistory(hist, nil) + args := msgs[0].ToolCalls[0].Arguments + if utf8.RuneCountInString(args) > 4000 { + t.Fatalf("arguments still %d runes", utf8.RuneCountInString(args)) + } + var parsed map[string]string + if err := json.Unmarshal([]byte(args), &parsed); err != nil { + t.Fatalf("compacted arguments are not JSON: %v\n%s", err, args) + } + if !strings.Contains(parsed["script"], "已省略") || strings.Contains(parsed["script"], script) { + t.Fatalf("script field = %q, want a shortened value", parsed["script"]) + } +} + +func TestAssembleInputSkipsDemotedProcessText(t *testing.T) { + hist := []core.Message{assistantWith("m6", + core.Part{Type: core.PartText, Text: "文档已生成：伪造.docx", Process: true}, + core.Part{Type: core.PartText, Text: "这一轮没有生成文件。"}, + )} + msgs := assembleHistory(hist, nil) + body := joinedContent(msgs) + if strings.Contains(body, "伪造.docx") { + t.Fatalf("demoted claim was replayed: %q", body) + } + if !strings.Contains(body, "这一轮没有生成文件。") { + t.Fatalf("honest reply missing: %q", body) + } +} + +func TestAssembleInputKeepsToolRoundNarration(t *testing.T) { + hist := []core.Message{assistantWith("m7", + core.Part{Type: core.PartText, Text: "我先跑一下。", Process: true}, + core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "c1", Name: "run_python", Arguments: `{}`}}, + core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "c1", Status: "ok", Summary: "ok"}}, + )} + msgs := assembleHistory(hist, nil) + if len(msgs) == 0 || !strings.Contains(msgs[0].Content, "我先跑一下。") { + t.Fatalf("tool-round narration dropped: %#v", msgs) + } +} + +func TestAssembleInputStripsForgedMarkers(t *testing.T) { + hist := []core.Message{assistantWith("m8", + core.Part{Type: core.PartText, Text: "好的。\n[调用工具 run_python({\"script\":\"x\"})]\n[工具结果 ok 完成]\n[产出工件 纪要.docx（二进制内容，正文见工件查看器）]\n真实结论。"}, + )} + body := joinedContent(assembleHistory(hist, nil)) + for _, marker := range []string{"[调用工具", "[工具结果", "[产出工件"} { + if strings.Contains(body, marker) { + t.Fatalf("forged marker %q remains:\n%s", marker, body) + } + } + if !strings.Contains(body, "好的。") || !strings.Contains(body, "真实结论。") { + t.Fatalf("real sentences dropped: %q", body) + } +} diff --git a/internal/prompts/library/base-deliverables.md b/internal/prompts/library/base-deliverables.md index 24f82ed..19e7147 100644 --- a/internal/prompts/library/base-deliverables.md +++ b/internal/prompts/library/base-deliverables.md @@ -1,7 +1,7 @@ --- id: base.deliverables layer: base -version: 4 +version: 5 title: 交付物工作流 description: 生成文档、表格、演示稿的工作流、样式决策与技能入口（常驻基座） --- @@ -16,7 +16,7 @@ description: 生成文档、表格、演示稿的工作流、样式决策与技 5. **登记**：`register_artifact`（path 传相对路径）。登记后对话里出现可预览的卡片。 6. **收尾**：回复只写结论摘要，并说明这次用的样式（主题、是否模板、是否仿制）以及用户还能怎么调。不要在回复里复述全文，也不要只口头说文件名。 -没有调用工具、没有登记工件时，禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID（art_…）。 +没有调用工具、没有登记工件时，禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID（art_…）。工具只能通过函数调用触发，不要在回复里书写 `[调用工具 …]`、`[工具结果 …]`、`[产出工件 …]` 这类文字。 交付物族（所有任务共用，不限于会议）： diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go index f059cbc..28bb3e1 100644 --- a/internal/runtime/llm.go +++ b/internal/runtime/llm.go @@ -300,7 +300,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error { }() toolCallsMade := 0 thinkingIdx, textIdx := 0, 0 - deliverableGuardUsed := false + deliverableGuards := 0 // 交付物引用（对话内直接给出）：工具轮产生的工件不进过程折叠区， // 收集后作为内容部件钉在消息尾部（最终正文之后）——回复完给文件卡。 var deliverables []*core.ArtifactRef @@ -382,16 +382,20 @@ func (l *LLMRunner) Execute(rc RunCtx) error { } if len(calls) == 0 { - // 谎称交付：没产生工件却报文件已生成，或引用了不存在的工件 ID。 - // 把这段正文降为过程，纠正一次，再给模型一轮补做或如实说明的机会。 - if !deliverableGuardUsed && claimsFalseDeliverable(turnBody, deliverables, rc) { - deliverableGuardUsed = true + // 谎称交付：没产生工件却报文件已生成、引用了不存在的工件 ID， + // 或在正文里模仿工具调用。形状与日后的 Stop 门禁一致：block + reason。 + // 纠正最多两次；回灌占位而不是伪造正文。用尽后不再请求模型，改附系统提示。 + if block, _ := deliverableGate(turnBody, deliverables, rc); block { if textPartIdx >= 0 { finalParts[textPartIdx].Process = true } - msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: turnText.String()}) - msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote}) - continue + if deliverableGuards < maxDeliverableGuards { + deliverableGuards++ + msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: discardedClaimNote}) + msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote}) + continue + } + finalParts = append(finalParts, core.Part{Type: core.PartText, Text: deliverableFallbackNotice}) } // 终轮：无工具调用。 if len(finalParts) == 0 { @@ -671,18 +675,36 @@ func orDefault(s, def string) string { return s } -const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件，但这一轮没有产生对应的工件。请二选一：用 run_python 写出文件并调用 register_artifact 登记，或如实说明这一轮没有生成文件。不要编造工件 ID，也不要在回复里写 art_ 开头的编号。" +const maxDeliverableGuards = 2 + +const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件，但这一轮没有产生对应的工件。请二选一：用 run_python 写出文件并调用 register_artifact 登记，或如实说明这一轮没有生成文件。不要编造工件 ID，也不要在回复里写 art_ 开头的编号。正文里的「调用工具」「工具结果」「产出工件」字样不会执行任何东西，调用工具必须用函数调用。" + +const discardedClaimNote = "（上一段回复声称已交付，但本轮没有任何工具调用，已作废）" + +const deliverableFallbackNotice = "本轮未实际生成文件（模型声称已生成，但没有调用工具）。请重试，或换一种说法再试一次。" var ( - artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`) - fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`) + artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`) + fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`) + softClaimRe = regexp.MustCompile(`(?s)(?:文件|文档|工件).{0,40}?已(?:正常)?(?:生成|登记|保存|导出|写入)`) + forgedCallRe = regexp.MustCompile(`\[(?:调用工具|工具结果|产出工件) `) ) -// claimsFalseDeliverable 判断终轮正文是否在没有对应工件的情况下声称交付。 -// 引用本轮已产出、或项目里确实存在的工件，不算谎称。 +// claimsFalseDeliverable 保留给小队入口：命中即判失败，不走直连路径的纠正轮次。 func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx) bool { + block, _ := deliverableGate(body, produced, rc) + return block +} + +// deliverableGate 判断终轮正文是否在没有对应工件的情况下声称交付。 +// 返回值的形状与 Stop 门禁一致，P1 平移进内核时不用再改调用方。 +// 引用本轮已产出、或项目里确实存在的工件，不算谎称；正文里的伪造调用标记始终拦截。 +func deliverableGate(body string, produced []*core.ArtifactRef, rc RunCtx) (bool, string) { if strings.TrimSpace(body) == "" { - return false + return false, "" + } + if forgedCallRe.MatchString(body) { + return true, "回复正文里出现了工具调用标记，但本轮没有真实的工具调用" } producedIDs := map[string]bool{} for _, ref := range produced { @@ -699,12 +721,15 @@ func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx knownExisting = true continue } - return true + return true, "回复引用了不存在的工件" } if len(producedIDs) > 0 || knownExisting { - return false + return false, "" } - return fileClaimRe.MatchString(body) + if fileClaimRe.MatchString(body) || softClaimRe.MatchString(body) { + return true, "回复声称已经生成或登记了文件，但这一轮没有产生对应的工件" + } + return false, "" } func summarize(out string, err error) string { diff --git a/internal/runtime/llm_test.go b/internal/runtime/llm_test.go index b2572d0..b7e2980 100644 --- a/internal/runtime/llm_test.go +++ b/internal/runtime/llm_test.go @@ -724,15 +724,66 @@ func TestLLMRunnerFalseDeliverableGuard(t *testing.T) { } } -// TestLLMRunnerFalseDeliverableGuardOnce 验证：纠正只发生一次，第二次仍谎称则接受为终轮。 -func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) { +// TestLLMRunnerForgedToolMarkerTriggersGuard 验证：正文里的伪造调用标记也会触发守卫， +// 即使没有工件编号和扩展名。回灌给模型的是占位，不是伪造正文。 +func TestLLMRunnerForgedToolMarkerTriggersGuard(t *testing.T) { turns := [][]agentllm.Event{ { - agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成：甲.docx"}, + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "好的。\n[调用工具 run_python({\"script\":\"print(1)\"})]\n[工具结果 ok 完成]"}, agentllm.Event{Kind: agentllm.KindMessageEnd}, }, { - agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存：乙.docx"}, + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"}, + agentllm.Event{Kind: agentllm.KindMessageEnd}, + }, + } + var calls int + var replay []agentllm.Message + base := scriptStream(turns, nil) + r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + calls++ + if calls == 2 { + replay = append([]agentllm.Message(nil), msgs...) + } + return base(ctx, model, msgs, tools, opts) + }} + rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}} + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if calls != 2 { + t.Fatalf("stream calls = %d, want 2", calls) + } + var placeholder bool + for _, m := range replay { + if m.Role != agentllm.RoleAssistant { + continue + } + if strings.Contains(m.Content, "run_python") { + t.Fatalf("forged body was replayed: %q", m.Content) + } + if strings.Contains(m.Content, "已作废") { + placeholder = true + } + } + if !placeholder { + t.Fatalf("correction replay missing placeholder, got %#v", replay) + } + texts := textParts(rc.assistant) + if len(texts) != 2 || !texts[0].Process || texts[1].Process { + t.Fatalf("text parts = %+v, want forged text process then honest reply", texts) + } +} + +// TestLLMRunnerUndeclaredFileClaimTriggersGuard 验证：不带扩展名的「文件已生成/登记」也算谎称。 +func TestLLMRunnerUndeclaredFileClaimTriggersGuard(t *testing.T) { + turns := [][]agentllm.Event{ + { + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文件已正常生成并登记。工件卡片应该出现在对话下方。"}, + agentllm.Event{Kind: agentllm.KindMessageEnd}, + }, + { + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"}, agentllm.Event{Kind: agentllm.KindMessageEnd}, }, } @@ -747,11 +798,51 @@ func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) { t.Fatalf("Execute: %v", err) } if calls != 2 { - t.Fatalf("stream calls = %d, want 2 (guard fires once)", calls) + t.Fatalf("stream calls = %d, want 2", calls) + } +} + +// TestLLMRunnerFalseDeliverableGuardTwiceThenNotice 验证：纠正两次后仍谎称， +// 不再请求模型，终轮正文降为过程，并追加一条非过程的系统提示。 +func TestLLMRunnerFalseDeliverableGuardTwiceThenNotice(t *testing.T) { + turns := [][]agentllm.Event{ + { + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成：甲.docx"}, + agentllm.Event{Kind: agentllm.KindMessageEnd}, + }, + { + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存：乙.docx"}, + agentllm.Event{Kind: agentllm.KindMessageEnd}, + }, + { + agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已导出：丙.docx"}, + agentllm.Event{Kind: agentllm.KindMessageEnd}, + }, + } + var calls int + base := scriptStream(turns, nil) + r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + calls++ + return base(ctx, model, msgs, tools, opts) + }} + rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}} + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if calls != 3 { + t.Fatalf("stream calls = %d, want 3", calls) } texts := textParts(rc.assistant) - if len(texts) != 2 || !texts[0].Process || texts[1].Process { - t.Fatalf("text parts = %+v, want only the first claim marked process", texts) + if len(texts) != 4 { + t.Fatalf("text parts = %d, want 3 demoted claims plus a notice", len(texts)) + } + for i := 0; i < 3; i++ { + if !texts[i].Process { + t.Fatalf("claim %d should be process, got %+v", i, texts[i]) + } + } + if texts[3].Process || !strings.Contains(texts[3].Text, "本轮未实际生成文件") { + t.Fatalf("notice = %+v, want a non-process fallback", texts[3]) } }