Replay tool calls natively and tighten the deliverable gate
Why: Flattened "[调用工具 …]" history taught the model to imitate calls and invent artifact ids. A single regex correction then accepted the second false claim. Evidence: ctxasm replay tests cover native tool messages, missing-result pairing, argument compaction, demoted text, and forged-marker stripping. Runtime tests cover forged markers, extension-less claims, two corrections, and the fallback notice. Impact: Assistant history sent to the model no longer contains flattened tool calls. The direct runner corrects a false deliverable twice, then appends a notice instead of accepting the claim. The squad entry still fails closed on the same predicate. base.deliverables is now v5. Refs: feature/agent-harness-p0 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+186
-16
@@ -2,7 +2,9 @@
|
||||
package ctxasm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
@@ -78,15 +80,15 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
|
||||
if m.ID == currentUser.ID {
|
||||
continue
|
||||
}
|
||||
if m.Role == core.RoleAssistant {
|
||||
msgs = append(msgs, renderAssistantTurns(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled)...)
|
||||
continue
|
||||
}
|
||||
content := renderMessage(m, resolveCard, limitCard, resolveArtifact, rep, &cardsBackfilled, &artsBackfilled)
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
role := agentllm.RoleUser
|
||||
if m.Role == core.RoleAssistant {
|
||||
role = agentllm.RoleAssistant
|
||||
}
|
||||
msgs = append(msgs, agentllm.Message{Role: role, Content: content})
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: content})
|
||||
}
|
||||
rep.Items = append(rep.Items, fmt.Sprintf("history(%d)", len(history)-start))
|
||||
if cardsBackfilled > 0 {
|
||||
@@ -134,23 +136,23 @@ func AssembleInput(prof core.AgentProfile, eff core.EffectiveSettings, history [
|
||||
return msgs, rep
|
||||
}
|
||||
|
||||
// renderMessage 把消息部件渲染为模型可读文本(思考部件不回灌;工具结果给摘要;
|
||||
// card 引用部件回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations)。
|
||||
// maxToolArgRunes 是回放进模型的单次工具参数上限。超限时截断字符串值,保持合法 JSON。
|
||||
const maxToolArgRunes = 4000
|
||||
|
||||
// forgedMarkerLine 匹配模型在正文里模仿出来的调用/结果/工件行。回放时整行剔除,
|
||||
// 避免下一轮把这种文本当成「以前就是这样调用工具」的格式。
|
||||
var forgedMarkerLine = regexp.MustCompile(`(?m)^[^\n]*\[(?:调用工具|工具结果|产出工件) [^\n]*\n?`)
|
||||
|
||||
// renderMessage 把用户消息部件渲染为模型可读文本(思考部件不回灌;
|
||||
// card 引用回填产物卡正文、artifact_ref 回填工件正文——截断记入 rep.Truncations)。
|
||||
// 助手消息不走这里:工具调用必须保持提供方的原生消息,见 renderAssistantTurns。
|
||||
func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) string {
|
||||
var sb strings.Builder
|
||||
for _, p := range m.Parts {
|
||||
switch p.Type {
|
||||
case core.PartText:
|
||||
sb.WriteString(p.Text)
|
||||
sb.WriteString(stripForgedMarkers(p.Text))
|
||||
sb.WriteString("\n")
|
||||
case core.PartToolCall:
|
||||
if p.ToolCall != nil {
|
||||
fmt.Fprintf(&sb, "[调用工具 %s(%s)]\n", p.ToolCall.Name, p.ToolCall.Arguments)
|
||||
}
|
||||
case core.PartToolResult:
|
||||
if p.ToolResult != nil {
|
||||
fmt.Fprintf(&sb, "[工具结果 %s %s]\n", p.ToolResult.Status, p.ToolResult.Summary)
|
||||
}
|
||||
case core.PartArtifactRef:
|
||||
sb.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled))
|
||||
case core.PartHITL:
|
||||
@@ -164,6 +166,174 @@ func renderMessage(m core.Message, resolveCard CardResolver, limitCard int, reso
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// renderAssistantTurns 把一条助手消息切成提供方可接受的若干段:
|
||||
// 前置文本与 tool_calls 组成一条 assistant,每个调用紧跟一条 tool 结果。
|
||||
// 没有结果的调用补一条失败占位,避免模型把中断当成成功。孤立的结果直接跳过。
|
||||
// 被守卫降级、且后面不是工具调用的过程文本不回放。
|
||||
func renderAssistantTurns(m core.Message, resolveCard CardResolver, limitCard int, resolveArtifact ArtifactResolver, rep *Report, cardsBackfilled, artsBackfilled *int) []agentllm.Message {
|
||||
var out []agentllm.Message
|
||||
var text strings.Builder
|
||||
var calls []agentllm.ToolCall
|
||||
results := map[string]core.ToolResultPart{}
|
||||
seq := 0
|
||||
|
||||
flush := func() {
|
||||
content := strings.TrimSpace(text.String())
|
||||
if content == "" && len(calls) == 0 {
|
||||
text.Reset()
|
||||
results = map[string]core.ToolResultPart{}
|
||||
return
|
||||
}
|
||||
out = append(out, agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls})
|
||||
for _, c := range calls {
|
||||
body := "(无结果:运行中断)"
|
||||
isErr := true
|
||||
if res, ok := results[c.ID]; ok {
|
||||
body = res.Summary
|
||||
isErr = res.Status == "error"
|
||||
}
|
||||
out = append(out, agentllm.Message{
|
||||
Role: agentllm.RoleTool,
|
||||
ToolCallID: c.ID,
|
||||
Name: c.Name,
|
||||
Content: body,
|
||||
IsError: isErr,
|
||||
})
|
||||
}
|
||||
text.Reset()
|
||||
calls = nil
|
||||
results = map[string]core.ToolResultPart{}
|
||||
}
|
||||
|
||||
for i, p := range m.Parts {
|
||||
switch p.Type {
|
||||
case core.PartText:
|
||||
if demotedProcessText(m.Parts, i) {
|
||||
continue
|
||||
}
|
||||
cleaned := stripForgedMarkers(p.Text)
|
||||
if strings.TrimSpace(cleaned) == "" {
|
||||
continue
|
||||
}
|
||||
if len(calls) > 0 {
|
||||
flush()
|
||||
}
|
||||
text.WriteString(cleaned)
|
||||
text.WriteString("\n")
|
||||
case core.PartToolCall:
|
||||
if p.ToolCall == nil {
|
||||
continue
|
||||
}
|
||||
id := p.ToolCall.ID
|
||||
if id == "" {
|
||||
id = fmt.Sprintf("hist_%s_%d", m.ID, seq)
|
||||
seq++
|
||||
}
|
||||
calls = append(calls, agentllm.ToolCall{
|
||||
ID: id,
|
||||
Name: p.ToolCall.Name,
|
||||
Arguments: compactToolArguments(p.ToolCall.Arguments),
|
||||
})
|
||||
case core.PartToolResult:
|
||||
if p.ToolResult == nil {
|
||||
continue
|
||||
}
|
||||
id := p.ToolResult.CallID
|
||||
if id == "" {
|
||||
for _, c := range calls {
|
||||
if _, ok := results[c.ID]; !ok {
|
||||
id = c.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if id == "" || !hasCall(calls, id) {
|
||||
continue
|
||||
}
|
||||
results[id] = *p.ToolResult
|
||||
case core.PartArtifactRef:
|
||||
if len(calls) > 0 {
|
||||
flush()
|
||||
}
|
||||
text.WriteString(renderArtifactRef(p.Artifact, resolveArtifact, limitCard, rep, artsBackfilled))
|
||||
case core.PartHITL:
|
||||
if p.Hitl == nil || !p.Hitl.Resolved {
|
||||
continue
|
||||
}
|
||||
if len(calls) > 0 {
|
||||
flush()
|
||||
}
|
||||
fmt.Fprintf(&text, "[用户确认:%s → %s]\n", p.Hitl.Question, p.Hitl.Answer)
|
||||
case core.PartCard:
|
||||
if len(calls) > 0 {
|
||||
flush()
|
||||
}
|
||||
text.WriteString(renderCardPart(p.Card, resolveCard, limitCard, rep, cardsBackfilled))
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
func hasCall(calls []agentllm.ToolCall, id string) bool {
|
||||
for _, c := range calls {
|
||||
if c.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// demotedProcessText 是守卫降级后的谎称正文:过程文本,且后面紧跟的不是工具调用。
|
||||
// 工具轮的解说也标了过程,但它后面就是 tool_call,要保留。
|
||||
func demotedProcessText(parts []core.Part, i int) bool {
|
||||
if parts[i].Type != core.PartText || !parts[i].Process {
|
||||
return false
|
||||
}
|
||||
return i+1 >= len(parts) || parts[i+1].Type != core.PartToolCall
|
||||
}
|
||||
|
||||
func stripForgedMarkers(text string) string {
|
||||
return strings.TrimSpace(forgedMarkerLine.ReplaceAllString(text, ""))
|
||||
}
|
||||
|
||||
// compactToolArguments 在参数超限时截断其中的长字符串。解析失败或截断后仍超限时,
|
||||
// 换成仍可被提供方接受的占位对象,而不是一段被截断的非法 JSON。
|
||||
func compactToolArguments(raw string) string {
|
||||
if utf8.RuneCountInString(raw) <= maxToolArgRunes {
|
||||
return raw
|
||||
}
|
||||
var v any
|
||||
if err := json.Unmarshal([]byte(raw), &v); err != nil {
|
||||
return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw))
|
||||
}
|
||||
truncateLongStrings(&v, 500)
|
||||
encoded, err := json.Marshal(v)
|
||||
if err != nil || utf8.RuneCountInString(string(encoded)) > maxToolArgRunes {
|
||||
return fmt.Sprintf(`{"_omitted":"参数 %d 字符"}`, utf8.RuneCountInString(raw))
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
func truncateLongStrings(v *any, keep int) {
|
||||
switch x := (*v).(type) {
|
||||
case map[string]any:
|
||||
for k, val := range x {
|
||||
truncateLongStrings(&val, keep)
|
||||
x[k] = val
|
||||
}
|
||||
case []any:
|
||||
for i := range x {
|
||||
truncateLongStrings(&x[i], keep)
|
||||
}
|
||||
case string:
|
||||
n := utf8.RuneCountInString(x)
|
||||
if n > keep {
|
||||
*v = string([]rune(x)[:keep]) + fmt.Sprintf("…(已省略 %d 字符)", n-keep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// renderCardPart 渲染 card 部件:引用部件(cardId)回填产物卡全文(超限截断留痕);
|
||||
// 无 resolver 或无此卡时留占位(可见即信号,不静默丢弃)。
|
||||
func renderCardPart(cp *core.CardPart, resolveCard CardResolver, limit int, rep *Report, cardsBackfilled *int) string {
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
package ctxasm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func assistantWith(id string, parts ...core.Part) core.Message {
|
||||
return core.Message{ID: id, Role: core.RoleAssistant, Parts: parts}
|
||||
}
|
||||
|
||||
func assembleHistory(hist []core.Message, resolve ArtifactResolver) []agentllm.Message {
|
||||
msgs, _ := AssembleInput(core.AgentProfile{}, core.EffectiveSettings{}, hist, fakeUser("继续"), nil, nil, nil, nil, nil, resolve)
|
||||
if len(msgs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return msgs[:len(msgs)-1]
|
||||
}
|
||||
|
||||
func joinedContent(msgs []agentllm.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestAssembleInputReplaysToolCallsNatively(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m2",
|
||||
core.Part{Type: core.PartText, Text: "好的,开始生成。"},
|
||||
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_1", Name: "run_python", Arguments: `{"script":"print(1)"}`}},
|
||||
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_1", Status: "ok", Summary: "完成"}},
|
||||
core.Part{Type: core.PartText, Text: "纪要已整理。"},
|
||||
core.Part{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: "art_1", Name: "纪要.md", Kind: "document"}},
|
||||
)}
|
||||
resolve := func(id core.ID) (core.Artifact, []byte, bool) {
|
||||
if id == "art_1" {
|
||||
return core.Artifact{ID: id, Name: "纪要.md"}, []byte("关键结论。"), true
|
||||
}
|
||||
return core.Artifact{}, nil, false
|
||||
}
|
||||
msgs := assembleHistory(hist, resolve)
|
||||
if len(msgs) != 3 {
|
||||
t.Fatalf("replay messages = %d, want assistant + tool + assistant, got %#v", len(msgs), msgs)
|
||||
}
|
||||
if msgs[0].Role != agentllm.RoleAssistant || len(msgs[0].ToolCalls) != 1 || msgs[0].ToolCalls[0].Name != "run_python" {
|
||||
t.Fatalf("first message = %+v, want assistant tool call run_python", msgs[0])
|
||||
}
|
||||
if !strings.Contains(msgs[0].Content, "好的,开始生成。") || strings.Contains(msgs[0].Content, "[调用工具") {
|
||||
t.Fatalf("preamble must stay text and not flatten the call: %q", msgs[0].Content)
|
||||
}
|
||||
if msgs[1].Role != agentllm.RoleTool || msgs[1].ToolCallID != "call_1" || msgs[1].Name != "run_python" || msgs[1].Content != "完成" || msgs[1].IsError {
|
||||
t.Fatalf("tool result = %+v, want paired ok summary", msgs[1])
|
||||
}
|
||||
if msgs[2].Role != agentllm.RoleAssistant || len(msgs[2].ToolCalls) != 0 {
|
||||
t.Fatalf("final segment = %+v, want plain assistant text", msgs[2])
|
||||
}
|
||||
if !strings.Contains(msgs[2].Content, "纪要已整理。") || !strings.Contains(msgs[2].Content, "关键结论。") {
|
||||
t.Fatalf("final segment missing reply or artifact body: %q", msgs[2].Content)
|
||||
}
|
||||
if strings.Contains(joinedContent(msgs), "[调用工具") || strings.Contains(joinedContent(msgs), "[工具结果") {
|
||||
t.Fatalf("flattened markers leaked into replay:\n%s", joinedContent(msgs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputPairsMissingToolResult(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m3",
|
||||
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_x", Name: "run_python", Arguments: `{}`}},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
if len(msgs) != 2 || msgs[1].Role != agentllm.RoleTool {
|
||||
t.Fatalf("messages = %#v, want assistant plus synthetic tool result", msgs)
|
||||
}
|
||||
if msgs[1].ToolCallID != "call_x" || !msgs[1].IsError || !strings.Contains(msgs[1].Content, "无结果:运行中断") {
|
||||
t.Fatalf("synthetic result = %+v", msgs[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputDropsOrphanToolResult(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m4",
|
||||
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "ghost", Status: "ok", Summary: "不该出现"}},
|
||||
core.Part{Type: core.PartText, Text: "只有正文。"},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
for _, m := range msgs {
|
||||
if m.Role == agentllm.RoleTool || strings.Contains(m.Content, "不该出现") {
|
||||
t.Fatalf("orphan result leaked: %+v", m)
|
||||
}
|
||||
}
|
||||
if len(msgs) != 1 || !strings.Contains(msgs[0].Content, "只有正文。") {
|
||||
t.Fatalf("text-only replay = %#v", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputFillsEmptyToolCallID(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m2",
|
||||
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{Name: "run_shell", Arguments: `{"cmd":"ls"}`}},
|
||||
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{Status: "ok", Summary: "a.txt"}},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("messages = %d, want 2", len(msgs))
|
||||
}
|
||||
id := msgs[0].ToolCalls[0].ID
|
||||
if id != "hist_m2_0" || msgs[1].ToolCallID != id || msgs[1].Content != "a.txt" {
|
||||
t.Fatalf("empty id pairing = call %q result %+v", id, msgs[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputCompactsLongToolArguments(t *testing.T) {
|
||||
script := strings.Repeat("甲", 5000)
|
||||
raw, err := json.Marshal(map[string]string{"script": script})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hist := []core.Message{assistantWith("m5",
|
||||
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "call_long", Name: "run_python", Arguments: string(raw)}},
|
||||
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "call_long", Status: "ok", Summary: "ok"}},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
args := msgs[0].ToolCalls[0].Arguments
|
||||
if utf8.RuneCountInString(args) > 4000 {
|
||||
t.Fatalf("arguments still %d runes", utf8.RuneCountInString(args))
|
||||
}
|
||||
var parsed map[string]string
|
||||
if err := json.Unmarshal([]byte(args), &parsed); err != nil {
|
||||
t.Fatalf("compacted arguments are not JSON: %v\n%s", err, args)
|
||||
}
|
||||
if !strings.Contains(parsed["script"], "已省略") || strings.Contains(parsed["script"], script) {
|
||||
t.Fatalf("script field = %q, want a shortened value", parsed["script"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputSkipsDemotedProcessText(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m6",
|
||||
core.Part{Type: core.PartText, Text: "文档已生成:伪造.docx", Process: true},
|
||||
core.Part{Type: core.PartText, Text: "这一轮没有生成文件。"},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
body := joinedContent(msgs)
|
||||
if strings.Contains(body, "伪造.docx") {
|
||||
t.Fatalf("demoted claim was replayed: %q", body)
|
||||
}
|
||||
if !strings.Contains(body, "这一轮没有生成文件。") {
|
||||
t.Fatalf("honest reply missing: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputKeepsToolRoundNarration(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m7",
|
||||
core.Part{Type: core.PartText, Text: "我先跑一下。", Process: true},
|
||||
core.Part{Type: core.PartToolCall, ToolCall: &core.ToolCallPart{ID: "c1", Name: "run_python", Arguments: `{}`}},
|
||||
core.Part{Type: core.PartToolResult, ToolResult: &core.ToolResultPart{CallID: "c1", Status: "ok", Summary: "ok"}},
|
||||
)}
|
||||
msgs := assembleHistory(hist, nil)
|
||||
if len(msgs) == 0 || !strings.Contains(msgs[0].Content, "我先跑一下。") {
|
||||
t.Fatalf("tool-round narration dropped: %#v", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleInputStripsForgedMarkers(t *testing.T) {
|
||||
hist := []core.Message{assistantWith("m8",
|
||||
core.Part{Type: core.PartText, Text: "好的。\n[调用工具 run_python({\"script\":\"x\"})]\n[工具结果 ok 完成]\n[产出工件 纪要.docx(二进制内容,正文见工件查看器)]\n真实结论。"},
|
||||
)}
|
||||
body := joinedContent(assembleHistory(hist, nil))
|
||||
for _, marker := range []string{"[调用工具", "[工具结果", "[产出工件"} {
|
||||
if strings.Contains(body, marker) {
|
||||
t.Fatalf("forged marker %q remains:\n%s", marker, body)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(body, "好的。") || !strings.Contains(body, "真实结论。") {
|
||||
t.Fatalf("real sentences dropped: %q", body)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: base.deliverables
|
||||
layer: base
|
||||
version: 4
|
||||
version: 5
|
||||
title: 交付物工作流
|
||||
description: 生成文档、表格、演示稿的工作流、样式决策与技能入口(常驻基座)
|
||||
---
|
||||
@@ -16,7 +16,7 @@ description: 生成文档、表格、演示稿的工作流、样式决策与技
|
||||
5. **登记**:`register_artifact`(path 传相对路径)。登记后对话里出现可预览的卡片。
|
||||
6. **收尾**:回复只写结论摘要,并说明这次用的样式(主题、是否模板、是否仿制)以及用户还能怎么调。不要在回复里复述全文,也不要只口头说文件名。
|
||||
|
||||
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…)。
|
||||
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…)。工具只能通过函数调用触发,不要在回复里书写 `[调用工具 …]`、`[工具结果 …]`、`[产出工件 …]` 这类文字。
|
||||
|
||||
交付物族(所有任务共用,不限于会议):
|
||||
|
||||
|
||||
+42
-17
@@ -300,7 +300,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
}()
|
||||
toolCallsMade := 0
|
||||
thinkingIdx, textIdx := 0, 0
|
||||
deliverableGuardUsed := false
|
||||
deliverableGuards := 0
|
||||
// 交付物引用(对话内直接给出):工具轮产生的工件不进过程折叠区,
|
||||
// 收集后作为内容部件钉在消息尾部(最终正文之后)——回复完给文件卡。
|
||||
var deliverables []*core.ArtifactRef
|
||||
@@ -382,16 +382,20 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
}
|
||||
|
||||
if len(calls) == 0 {
|
||||
// 谎称交付:没产生工件却报文件已生成,或引用了不存在的工件 ID。
|
||||
// 把这段正文降为过程,纠正一次,再给模型一轮补做或如实说明的机会。
|
||||
if !deliverableGuardUsed && claimsFalseDeliverable(turnBody, deliverables, rc) {
|
||||
deliverableGuardUsed = true
|
||||
// 谎称交付:没产生工件却报文件已生成、引用了不存在的工件 ID,
|
||||
// 或在正文里模仿工具调用。形状与日后的 Stop 门禁一致:block + reason。
|
||||
// 纠正最多两次;回灌占位而不是伪造正文。用尽后不再请求模型,改附系统提示。
|
||||
if block, _ := deliverableGate(turnBody, deliverables, rc); block {
|
||||
if textPartIdx >= 0 {
|
||||
finalParts[textPartIdx].Process = true
|
||||
}
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: turnText.String()})
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
|
||||
continue
|
||||
if deliverableGuards < maxDeliverableGuards {
|
||||
deliverableGuards++
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: discardedClaimNote})
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
|
||||
continue
|
||||
}
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: deliverableFallbackNotice})
|
||||
}
|
||||
// 终轮:无工具调用。
|
||||
if len(finalParts) == 0 {
|
||||
@@ -671,18 +675,36 @@ func orDefault(s, def string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。"
|
||||
const maxDeliverableGuards = 2
|
||||
|
||||
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。正文里的「调用工具」「工具结果」「产出工件」字样不会执行任何东西,调用工具必须用函数调用。"
|
||||
|
||||
const discardedClaimNote = "(上一段回复声称已交付,但本轮没有任何工具调用,已作废)"
|
||||
|
||||
const deliverableFallbackNotice = "本轮未实际生成文件(模型声称已生成,但没有调用工具)。请重试,或换一种说法再试一次。"
|
||||
|
||||
var (
|
||||
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
|
||||
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
|
||||
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
|
||||
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
|
||||
softClaimRe = regexp.MustCompile(`(?s)(?:文件|文档|工件).{0,40}?已(?:正常)?(?:生成|登记|保存|导出|写入)`)
|
||||
forgedCallRe = regexp.MustCompile(`\[(?:调用工具|工具结果|产出工件) `)
|
||||
)
|
||||
|
||||
// claimsFalseDeliverable 判断终轮正文是否在没有对应工件的情况下声称交付。
|
||||
// 引用本轮已产出、或项目里确实存在的工件,不算谎称。
|
||||
// claimsFalseDeliverable 保留给小队入口:命中即判失败,不走直连路径的纠正轮次。
|
||||
func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx) bool {
|
||||
block, _ := deliverableGate(body, produced, rc)
|
||||
return block
|
||||
}
|
||||
|
||||
// deliverableGate 判断终轮正文是否在没有对应工件的情况下声称交付。
|
||||
// 返回值的形状与 Stop 门禁一致,P1 平移进内核时不用再改调用方。
|
||||
// 引用本轮已产出、或项目里确实存在的工件,不算谎称;正文里的伪造调用标记始终拦截。
|
||||
func deliverableGate(body string, produced []*core.ArtifactRef, rc RunCtx) (bool, string) {
|
||||
if strings.TrimSpace(body) == "" {
|
||||
return false
|
||||
return false, ""
|
||||
}
|
||||
if forgedCallRe.MatchString(body) {
|
||||
return true, "回复正文里出现了工具调用标记,但本轮没有真实的工具调用"
|
||||
}
|
||||
producedIDs := map[string]bool{}
|
||||
for _, ref := range produced {
|
||||
@@ -699,12 +721,15 @@ func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx
|
||||
knownExisting = true
|
||||
continue
|
||||
}
|
||||
return true
|
||||
return true, "回复引用了不存在的工件"
|
||||
}
|
||||
if len(producedIDs) > 0 || knownExisting {
|
||||
return false
|
||||
return false, ""
|
||||
}
|
||||
return fileClaimRe.MatchString(body)
|
||||
if fileClaimRe.MatchString(body) || softClaimRe.MatchString(body) {
|
||||
return true, "回复声称已经生成或登记了文件,但这一轮没有产生对应的工件"
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func summarize(out string, err error) string {
|
||||
|
||||
@@ -724,15 +724,66 @@ func TestLLMRunnerFalseDeliverableGuard(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerFalseDeliverableGuardOnce 验证:纠正只发生一次,第二次仍谎称则接受为终轮。
|
||||
func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
|
||||
// TestLLMRunnerForgedToolMarkerTriggersGuard 验证:正文里的伪造调用标记也会触发守卫,
|
||||
// 即使没有工件编号和扩展名。回灌给模型的是占位,不是伪造正文。
|
||||
func TestLLMRunnerForgedToolMarkerTriggersGuard(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "好的。\n[调用工具 run_python({\"script\":\"print(1)\"})]\n[工具结果 ok 完成]"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var calls int
|
||||
var replay []agentllm.Message
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
if calls == 2 {
|
||||
replay = append([]agentllm.Message(nil), msgs...)
|
||||
}
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("stream calls = %d, want 2", calls)
|
||||
}
|
||||
var placeholder bool
|
||||
for _, m := range replay {
|
||||
if m.Role != agentllm.RoleAssistant {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(m.Content, "run_python") {
|
||||
t.Fatalf("forged body was replayed: %q", m.Content)
|
||||
}
|
||||
if strings.Contains(m.Content, "已作废") {
|
||||
placeholder = true
|
||||
}
|
||||
}
|
||||
if !placeholder {
|
||||
t.Fatalf("correction replay missing placeholder, got %#v", replay)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
|
||||
t.Fatalf("text parts = %+v, want forged text process then honest reply", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerUndeclaredFileClaimTriggersGuard 验证:不带扩展名的「文件已生成/登记」也算谎称。
|
||||
func TestLLMRunnerUndeclaredFileClaimTriggersGuard(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文件已正常生成并登记。工件卡片应该出现在对话下方。"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
@@ -747,11 +798,51 @@ func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("stream calls = %d, want 2 (guard fires once)", calls)
|
||||
t.Fatalf("stream calls = %d, want 2", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerFalseDeliverableGuardTwiceThenNotice 验证:纠正两次后仍谎称,
|
||||
// 不再请求模型,终轮正文降为过程,并追加一条非过程的系统提示。
|
||||
func TestLLMRunnerFalseDeliverableGuardTwiceThenNotice(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已导出:丙.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var calls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("stream calls = %d, want 3", calls)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
|
||||
t.Fatalf("text parts = %+v, want only the first claim marked process", texts)
|
||||
if len(texts) != 4 {
|
||||
t.Fatalf("text parts = %d, want 3 demoted claims plus a notice", len(texts))
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
if !texts[i].Process {
|
||||
t.Fatalf("claim %d should be process, got %+v", i, texts[i])
|
||||
}
|
||||
}
|
||||
if texts[3].Process || !strings.Contains(texts[3].Text, "本轮未实际生成文件") {
|
||||
t.Fatalf("notice = %+v, want a non-process fallback", texts[3])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user