Replay tool calls natively and tighten the deliverable gate

Why: Flattened "[调用工具 …]" history taught the model to imitate calls and invent artifact ids. A single regex correction then accepted the second false claim.
Evidence: ctxasm replay tests cover native tool messages, missing-result pairing, argument compaction, demoted text, and forged-marker stripping. Runtime tests cover forged markers, extension-less claims, two corrections, and the fallback notice.
Impact: Assistant history sent to the model no longer contains flattened tool calls. The direct runner corrects a false deliverable twice, then appends a notice instead of accepting the claim. The squad entry still fails closed on the same predicate. base.deliverables is now v5.
Refs: feature/agent-harness-p0
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-10-01 12:29:38 +00:00
co-authored by Cursor
parent c7dd95137a
commit 188c63357c
5 changed files with 508 additions and 42 deletions
+186 -16
View File
@@ -2,7 +2,9 @@
package ctxasm
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
"unicode/utf8"
@@ -78,15 +80,15 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
if m.ID == currentUser.ID {
continue
}
if m.Role == core.RoleAssistant {
msgs = append(msgs, renderAssistantTurns(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled)...)
continue
}
content := renderMessage(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled)
if content == "" {
continue
}
role := agentllm.RoleUser
if m.Role == core.RoleAssistant {
role = agentllm.RoleAssistant
}
msgs = append(msgs, agentllm.Message{Role: role, Content: content})
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: content})
}
rep.Items = append(rep.Items, fmt.Sprintf("history(%d)", len(history)-start))
if cardsBackfilled > 0 {
@@ -134,23 +136,23 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
return msgs, rep
}
// renderMessage 把消息部件渲染为模型可读文本(思考部件不回灌;工具结果给摘要;
// card 引用部件回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations)。
// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值,保持合法 JSON。
const maxToolArgRunes = 4000
// forgedMarkerLine 匹配模型在正文里模仿出来的调用/结果/工件行。回放时整行剔除,
// 避免下一轮把这种文本当成「以前就是这样调用工具」的格式。
var forgedMarkerLine = regexp.MustCompile(`(?m)^[^\n]*\[(?:调用工具|工具结果|产出工件) [^\n]*\n?`)
// renderMessage 把用户消息部件渲染为模型可读文本(思考部件不回灌;
// card 引用回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations)。
// 助手消息不走这里:工具调用必须保持提供方的原生消息,见 renderAssistantTurns。
func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) string {
var sb strings.Builder
for _, p := range m.Parts {
switch p.Type {
case core.PartText:
sb.WriteString(p.Text)
sb.WriteString(stripForgedMarkers(p.Text))
sb.WriteString("\n")
case core.PartToolCall:
if p.ToolCall != nil {
fmt.Fprintf(&sb, "[调用工具 %s(%s)]\n", p.ToolCall.Name, p.ToolCall.Arguments)
}
case core.PartToolResult:
if p.ToolResult != nil {
fmt.Fprintf(&sb, "[工具结果 %s %s]\n", p.ToolResult.Status, p.ToolResult.Summary)
}
case core.PartArtifactRef:
sb.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled))
case core.PartHITL:
@@ -164,6 +166,174 @@ func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, reso
return strings.TrimSpace(sb.String())
}
// renderAssistantTurns 把一条助手消息切成提供方可接受的若干段:
// 前置文本与 tool_calls 组成一条 assistant,每个调用紧跟一条 tool 结果。
// 没有结果的调用补一条失败占位,避免模型把中断当成成功。孤立的结果直接跳过。
// 被守卫降级、且后面不是工具调用的过程文本不回放。
func renderAssistantTurns(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) []agentllm.Message {
var out []agentllm.Message
var text strings.Builder
var calls []agentllm.ToolCall
results := map[string]core.ToolResultPart{}
seq := 0
flush := func() {
content := strings.TrimSpace(text.String())
if content == "" && len(calls) == 0 {
text.Reset()
results = map[string]core.ToolResultPart{}
return
}
out = append(out, agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls})
for _, c := range calls {
body := "(无结果:运行中断)"
isErr := true
if res, ok := results[c.ID]; ok {
body = res.Summary
isErr = res.Status == "error"
}
out = append(out, agentllm.Message{
Role: agentllm.RoleTool,
ToolCallID: c.ID,
Name: c.Name,
Content: body,
IsError: isErr,
})
}
text.Reset()
calls = nil
results = map[string]core.ToolResultPart{}
}
for i, p := range m.Parts {
switch p.Type {
case core.PartText:
if demotedProcessText(m.Parts, i) {
continue
}
cleaned := stripForgedMarkers(p.Text)
if strings.TrimSpace(cleaned) == "" {
continue
}
if len(calls) > 0 {
flush()
}
text.WriteString(cleaned)
text.WriteString("\n")
case core.PartToolCall:
if p.ToolCall == nil {
continue
}
id := p.ToolCall.ID
if id == "" {
id = fmt.Sprintf("hist_%s_%d", m.ID, seq)
seq++
}
calls = append(calls, agentllm.ToolCall{
ID: id,
Name: p.ToolCall.Name,
Arguments: compactToolArguments(p.ToolCall.Arguments),
})
case core.PartToolResult:
if p.ToolResult == nil {
continue
}
id := p.ToolResult.CallID
if id == "" {
for _, c := range calls {
if _, ok := results[c.ID]; !ok {
id = c.ID
break
}
}
}
if id == "" || !hasCall(calls, id) {
continue
}
results[id] = *p.ToolResult
case core.PartArtifactRef:
if len(calls) > 0 {
flush()
}
text.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled))
case core.PartHITL:
if p.Hitl == nil || !p.Hitl.Resolved {
continue
}
if len(calls) > 0 {
flush()
}
fmt.Fprintf(&text, "[用户确认:%s → %s]\n", p.Hitl.Question, p.Hitl.Answer)
case core.PartCard:
if len(calls) > 0 {
flush()
}
text.WriteString(renderCardPart(p.Card, resolveCard, limitCard, rep, cardsBackfilled))
}
}
flush()
return out
}
func hasCall(calls []agentllm.ToolCall, id string) bool {
for _, c := range calls {
if c.ID == id {
return true
}
}
return false
}
// demotedProcessText 是守卫降级后的谎称正文:过程文本,且后面紧跟的不是工具调用。
// 工具轮的解说也标了过程,但它后面就是 tool_call,要保留。
func demotedProcessText(parts []core.Part, i int) bool {
if parts[i].Type != core.PartText || !parts[i].Process {
return false
}
return i+1 >= len(parts) || parts[i+1].Type != core.PartToolCall
}
func stripForgedMarkers(text string) string {
return strings.TrimSpace(forgedMarkerLine.ReplaceAllString(text, ""))
}
// compactToolArguments 在参数超限时截断其中的长字符串。解析失败或截断后仍超限时,
// 换成仍可被提供方接受的占位对象,而不是一段被截断的非法 JSON。
func compactToolArguments(raw string) string {
if utf8.RuneCountInString(raw) <= maxToolArgRunes {
return raw
}
var v any
if err := json.Unmarshal([]byte(raw), &v); err != nil {
return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw))
}
truncateLongStrings(&v, 500)
encoded, err := json.Marshal(v)
if err != nil || utf8.RuneCountInString(string(encoded)) > maxToolArgRunes {
return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw))
}
return string(encoded)
}
func truncateLongStrings(v *any, keep int) {
switch x := (*v).(type) {
case map[string]any:
for k, val := range x {
truncateLongStrings(&val, keep)
x[k] = val
}
case []any:
for i := range x {
truncateLongStrings(&x[i], keep)
}
case string:
n := utf8.RuneCountInString(x)
if n > keep {
*v = string([]rune(x)[:keep]) + fmt.Sprintf("…(已省略 %d 字符)", n-keep)
}
}
}
// renderCardPart 渲染 card 部件:引用部件(cardId)回填产物卡全文(超限截断留痕);
// 无 resolver 或无此卡时留占位(可见即信号,不静默丢弃)。
func renderCardPart(cp *core.CardPart, resolveCard CardResolver, limit int, rep *Report, cardsBackfilled *int) string {
+180
View File
@@ -0,0 +1,180 @@
package ctxasm
import (
"encoding/json"
"strings"
"testing"
"unicode/utf8"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
func assistantWith(id string, parts ...core.Part) core.Message {
return core.Message{ID: id, Role: core.RoleAssistant, Parts: parts}
}
func assembleHistory(hist []core.Message, resolve ArtifactResolver) []agentllm.Message {
msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, hist, fakeUser("继续"), nil, nil, nil, nil, nil, resolve)
if len(msgs) == 0 {
return nil
}
return msgs[:len(msgs)-1]
}
func joinedContent(msgs []agentllm.Message) string {
var b strings.Builder
for _, m := range msgs {
b.WriteString(m.Content)
b.WriteByte('\n')
}
return b.String()
}
func TestAssembleInputReplaysToolCallsNatively(t *testing.T) {
hist := []core.Message{assistantWith("m2",
core.Part{Type: core.PartText, Text: "好的,开始生成。"},
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_1", Name: "run_python", Arguments: `{"script":"print(1)"}`}},
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_1", Status: "ok", Summary: "完成"}},
core.Part{Type: core.PartText, Text: "纪要已整理。"},
core.Part{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "art_1", Name: "纪要.md", Kind: "document"}},
)}
resolve := func(id core.ID) (core.Artifact, []byte, bool) {
if id == "art_1" {
return core.Artifact{ID: id, Name: "纪要.md"}, []byte("关键结论。"), true
}
return core.Artifact{}, nil, false
}
msgs := assembleHistory(hist, resolve)
if len(msgs) != 3 {
t.Fatalf("replay messages = %d, want assistant + tool + assistant, got %#v", len(msgs), msgs)
}
if msgs[0].Role != agentllm.RoleAssistant || len(msgs[0].ToolCalls) != 1 || msgs[0].ToolCalls[0].Name != "run_python" {
t.Fatalf("first message = %+v, want assistant tool call run_python", msgs[0])
}
if !strings.Contains(msgs[0].Content, "好的,开始生成。") || strings.Contains(msgs[0].Content, "[调用工具") {
t.Fatalf("preamble must stay text and not flatten the call: %q", msgs[0].Content)
}
if msgs[1].Role != agentllm.RoleTool || msgs[1].ToolCallID != "call_1" || msgs[1].Name != "run_python" || msgs[1].Content != "完成" || msgs[1].IsError {
t.Fatalf("tool result = %+v, want paired ok summary", msgs[1])
}
if msgs[2].Role != agentllm.RoleAssistant || len(msgs[2].ToolCalls) != 0 {
t.Fatalf("final segment = %+v, want plain assistant text", msgs[2])
}
if !strings.Contains(msgs[2].Content, "纪要已整理。") || !strings.Contains(msgs[2].Content, "关键结论。") {
t.Fatalf("final segment missing reply or artifact body: %q", msgs[2].Content)
}
if strings.Contains(joinedContent(msgs), "[调用工具") || strings.Contains(joinedContent(msgs), "[工具结果") {
t.Fatalf("flattened markers leaked into replay:\n%s", joinedContent(msgs))
}
}
func TestAssembleInputPairsMissingToolResult(t *testing.T) {
hist := []core.Message{assistantWith("m3",
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_x", Name: "run_python", Arguments: `{}`}},
)}
msgs := assembleHistory(hist, nil)
if len(msgs) != 2 || msgs[1].Role != agentllm.RoleTool {
t.Fatalf("messages = %#v, want assistant plus synthetic tool result", msgs)
}
if msgs[1].ToolCallID != "call_x" || !msgs[1].IsError || !strings.Contains(msgs[1].Content, "无结果:运行中断") {
t.Fatalf("synthetic result = %+v", msgs[1])
}
}
func TestAssembleInputDropsOrphanToolResult(t *testing.T) {
hist := []core.Message{assistantWith("m4",
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "ghost", Status: "ok", Summary: "不该出现"}},
core.Part{Type: core.PartText, Text: "只有正文。"},
)}
msgs := assembleHistory(hist, nil)
for _, m := range msgs {
if m.Role == agentllm.RoleTool || strings.Contains(m.Content, "不该出现") {
t.Fatalf("orphan result leaked: %+v", m)
}
}
if len(msgs) != 1 || !strings.Contains(msgs[0].Content, "只有正文。") {
t.Fatalf("text-only replay = %#v", msgs)
}
}
func TestAssembleInputFillsEmptyToolCallID(t *testing.T) {
hist := []core.Message{assistantWith("m2",
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{Name: "run_shell", Arguments: `{"cmd":"ls"}`}},
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{Status: "ok", Summary: "a.txt"}},
)}
msgs := assembleHistory(hist, nil)
if len(msgs) != 2 {
t.Fatalf("messages = %d, want 2", len(msgs))
}
id := msgs[0].ToolCalls[0].ID
if id != "hist_m2_0" || msgs[1].ToolCallID != id || msgs[1].Content != "a.txt" {
t.Fatalf("empty id pairing = call %q result %+v", id, msgs[1])
}
}
func TestAssembleInputCompactsLongToolArguments(t *testing.T) {
script := strings.Repeat("甲", 5000)
raw, err := json.Marshal(map[string]string{"script": script})
if err != nil {
t.Fatal(err)
}
hist := []core.Message{assistantWith("m5",
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_long", Name: "run_python", Arguments: string(raw)}},
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_long", Status: "ok", Summary: "ok"}},
)}
msgs := assembleHistory(hist, nil)
args := msgs[0].ToolCalls[0].Arguments
if utf8.RuneCountInString(args) > 4000 {
t.Fatalf("arguments still %d runes", utf8.RuneCountInString(args))
}
var parsed map[string]string
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
t.Fatalf("compacted arguments are not JSON: %v\n%s", err, args)
}
if !strings.Contains(parsed["script"], "已省略") || strings.Contains(parsed["script"], script) {
t.Fatalf("script field = %q, want a shortened value", parsed["script"])
}
}
func TestAssembleInputSkipsDemotedProcessText(t *testing.T) {
hist := []core.Message{assistantWith("m6",
core.Part{Type: core.PartText, Text: "文档已生成:伪造.docx", Process: true},
core.Part{Type: core.PartText, Text: "这一轮没有生成文件。"},
)}
msgs := assembleHistory(hist, nil)
body := joinedContent(msgs)
if strings.Contains(body, "伪造.docx") {
t.Fatalf("demoted claim was replayed: %q", body)
}
if !strings.Contains(body, "这一轮没有生成文件。") {
t.Fatalf("honest reply missing: %q", body)
}
}
func TestAssembleInputKeepsToolRoundNarration(t *testing.T) {
hist := []core.Message{assistantWith("m7",
core.Part{Type: core.PartText, Text: "我先跑一下。", Process: true},
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "c1", Name: "run_python", Arguments: `{}`}},
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "c1", Status: "ok", Summary: "ok"}},
)}
msgs := assembleHistory(hist, nil)
if len(msgs) == 0 || !strings.Contains(msgs[0].Content, "我先跑一下。") {
t.Fatalf("tool-round narration dropped: %#v", msgs)
}
}
func TestAssembleInputStripsForgedMarkers(t *testing.T) {
hist := []core.Message{assistantWith("m8",
core.Part{Type: core.PartText, Text: "好的。\n[调用工具 run_python({\"script\":\"x\"})]\n[工具结果 ok 完成]\n[产出工件 纪要.docx(二进制内容,正文见工件查看器)]\n真实结论。"},
)}
body := joinedContent(assembleHistory(hist, nil))
for _, marker := range []string{"[调用工具", "[工具结果", "[产出工件"} {
if strings.Contains(body, marker) {
t.Fatalf("forged marker %q remains:\n%s", marker, body)
}
}
if !strings.Contains(body, "好的。") || !strings.Contains(body, "真实结论。") {
t.Fatalf("real sentences dropped: %q", body)
}
}
@@ -1,7 +1,7 @@
---
id: base.deliverables
layer: base
version: 4
version: 5
title: 交付物工作流
description: 生成文档、表格、演示稿的工作流、样式决策与技能入口(常驻基座)
---
@@ -16,7 +16,7 @@ description: 生成文档、表格、演示稿的工作流、样式决策与技
5. **登记**:`register_artifact`(path 传相对路径)。登记后对话里出现可预览的卡片。
6. **收尾**:回复只写结论摘要,并说明这次用的样式(主题、是否模板、是否仿制)以及用户还能怎么调。不要在回复里复述全文,也不要只口头说文件名。
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…)。
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…)。工具只能通过函数调用触发,不要在回复里书写 `[调用工具 …]`、`[工具结果 …]`、`[产出工件 …]` 这类文字。
交付物族(所有任务共用,不限于会议):
+42 -17
View File
@@ -300,7 +300,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
}()
toolCallsMade := 0
thinkingIdx, textIdx := 0, 0
deliverableGuardUsed := false
deliverableGuards := 0
// 交付物引用(对话内直接给出):工具轮产生的工件不进过程折叠区,
// 收集后作为内容部件钉在消息尾部(最终正文之后)——回复完给文件卡。
var deliverables []*core.ArtifactRef
@@ -382,16 +382,20 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
}
if len(calls) == 0 {
// 谎称交付:没产生工件却报文件已生成,或引用了不存在的工件 ID。
// 把这段正文降为过程,纠正一次,再给模型一轮补做或如实说明的机会。
if !deliverableGuardUsed && claimsFalseDeliverable(turnBody, deliverables, rc) {
deliverableGuardUsed = true
// 谎称交付:没产生工件却报文件已生成、引用了不存在的工件 ID,
// 或在正文里模仿工具调用。形状与日后的 Stop 门禁一致:block + reason。
// 纠正最多两次;回灌占位而不是伪造正文。用尽后不再请求模型,改附系统提示。
if block, _ := deliverableGate(turnBody, deliverables, rc); block {
if textPartIdx >= 0 {
finalParts[textPartIdx].Process = true
}
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: turnText.String()})
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
continue
if deliverableGuards < maxDeliverableGuards {
deliverableGuards++
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: discardedClaimNote})
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
continue
}
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: deliverableFallbackNotice})
}
// 终轮:无工具调用。
if len(finalParts) == 0 {
@@ -671,18 +675,36 @@ func orDefault(s, def string) string {
return s
}
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。"
const maxDeliverableGuards = 2
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。正文里的「调用工具」「工具结果」「产出工件」字样不会执行任何东西,调用工具必须用函数调用。"
const discardedClaimNote = "(上一段回复声称已交付,但本轮没有任何工具调用,已作废)"
const deliverableFallbackNotice = "本轮未实际生成文件(模型声称已生成,但没有调用工具)。请重试,或换一种说法再试一次。"
var (
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
softClaimRe = regexp.MustCompile(`(?s)(?:文件|文档|工件).{0,40}?已(?:正常)?(?:生成|登记|保存|导出|写入)`)
forgedCallRe = regexp.MustCompile(`\[(?:调用工具|工具结果|产出工件) `)
)
// claimsFalseDeliverable 判断终轮正文是否在没有对应工件的情况下声称交付。
// 引用本轮已产出、或项目里确实存在的工件,不算谎称。
// claimsFalseDeliverable 保留给小队入口:命中即判失败,不走直连路径的纠正轮次。
func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx) bool {
block, _ := deliverableGate(body, produced, rc)
return block
}
// deliverableGate 判断终轮正文是否在没有对应工件的情况下声称交付。
// 返回值的形状与 Stop 门禁一致,P1 平移进内核时不用再改调用方。
// 引用本轮已产出、或项目里确实存在的工件,不算谎称;正文里的伪造调用标记始终拦截。
func deliverableGate(body string, produced []*core.ArtifactRef, rc RunCtx) (bool, string) {
if strings.TrimSpace(body) == "" {
return false
return false, ""
}
if forgedCallRe.MatchString(body) {
return true, "回复正文里出现了工具调用标记,但本轮没有真实的工具调用"
}
producedIDs := map[string]bool{}
for _, ref := range produced {
@@ -699,12 +721,15 @@ func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx
knownExisting = true
continue
}
return true
return true, "回复引用了不存在的工件"
}
if len(producedIDs) > 0 || knownExisting {
return false
return false, ""
}
return fileClaimRe.MatchString(body)
if fileClaimRe.MatchString(body) || softClaimRe.MatchString(body) {
return true, "回复声称已经生成或登记了文件,但这一轮没有产生对应的工件"
}
return false, ""
}
func summarize(out string, err error) string {
+98 -7
View File
@@ -724,15 +724,66 @@ func TestLLMRunnerFalseDeliverableGuard(t *testing.T) {
}
}
// TestLLMRunnerFalseDeliverableGuardOnce 验证:纠正只发生一次,第二次仍谎称则接受为终轮。
func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
// TestLLMRunnerForgedToolMarkerTriggersGuard 验证:正文里的伪造调用标记也会触发守卫,
// 即使没有工件编号和扩展名。回灌给模型的是占位,不是伪造正文。
func TestLLMRunnerForgedToolMarkerTriggersGuard(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "好的。\n[调用工具 run_python({\"script\":\"print(1)\"})]\n[工具结果 ok 完成]"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var calls int
var replay []agentllm.Message
base := scriptStream(turns, nil)
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
if calls == 2 {
replay = append([]agentllm.Message(nil), msgs...)
}
return base(ctx, model, msgs, tools, opts)
}}
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
if calls != 2 {
t.Fatalf("stream calls = %d, want 2", calls)
}
var placeholder bool
for _, m := range replay {
if m.Role != agentllm.RoleAssistant {
continue
}
if strings.Contains(m.Content, "run_python") {
t.Fatalf("forged body was replayed: %q", m.Content)
}
if strings.Contains(m.Content, "已作废") {
placeholder = true
}
}
if !placeholder {
t.Fatalf("correction replay missing placeholder, got %#v", replay)
}
texts := textParts(rc.assistant)
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
t.Fatalf("text parts = %+v, want forged text process then honest reply", texts)
}
}
// TestLLMRunnerUndeclaredFileClaimTriggersGuard 验证:不带扩展名的「文件已生成/登记」也算谎称。
func TestLLMRunnerUndeclaredFileClaimTriggersGuard(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文件已正常生成并登记。工件卡片应该出现在对话下方。"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
@@ -747,11 +798,51 @@ func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
t.Fatalf("Execute: %v", err)
}
if calls != 2 {
t.Fatalf("stream calls = %d, want 2 (guard fires once)", calls)
t.Fatalf("stream calls = %d, want 2", calls)
}
}
// TestLLMRunnerFalseDeliverableGuardTwiceThenNotice 验证:纠正两次后仍谎称,
// 不再请求模型,终轮正文降为过程,并追加一条非过程的系统提示。
func TestLLMRunnerFalseDeliverableGuardTwiceThenNotice(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已导出:丙.docx"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var calls int
base := scriptStream(turns, nil)
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
return base(ctx, model, msgs, tools, opts)
}}
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
if calls != 3 {
t.Fatalf("stream calls = %d, want 3", calls)
}
texts := textParts(rc.assistant)
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
t.Fatalf("text parts = %+v, want only the first claim marked process", texts)
if len(texts) != 4 {
t.Fatalf("text parts = %d, want 3 demoted claims plus a notice", len(texts))
}
for i := 0; i < 3; i++ {
if !texts[i].Process {
t.Fatalf("claim %d should be process, got %+v", i, texts[i])
}
}
if texts[3].Process || !strings.Contains(texts[3].Text, "本轮未实际生成文件") {
t.Fatalf("notice = %+v, want a non-process fallback", texts[3])
}
}