Record tool time and reject false file claims #37

Merged
weichen merged 4 commits from feature/chat-timeline-preview into main 2026-10-01 07:40:52 +00:00
4 changed files with 259 additions and 6 deletions
@@ -1,7 +1,7 @@
---
id: base.deliverables
layer: base
version: 1
version: 2
title: 交付物工作流
description: 生成文档/表格/演示稿等文件交付物的通用工作流与配方(常驻基座)
---
@@ -11,6 +11,8 @@ description: 生成文档/表格/演示稿等文件交付物的通用工作流
2. **登记**:调用 register_artifact登记生成文件(path 传相对路径);登记后对话内自动出现可预览的文件卡片。
3. **收尾**:对话回复只给结论摘要与要点(几句话),全文进文件,不要在回复里复述文档内容;也不要只口头说文件名——卡片就是交付。
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…):登记成功后卡片会自动出现。
配方已迁入技能库(启动时种子到数据目录 skills/,完整 Python 配方见对应 SKILL.md):
- **docx-author**:python-docx——标题层级/表格/列表真实排版。
+90 -4
View File
@@ -3,6 +3,7 @@ package runtime
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"time"
@@ -21,10 +22,27 @@ type LLMRunner struct {
Prompts *prompts.Service
// Stream 提供方边界注入点(nil = 按模型路由构造 agent-native 客户端)。
Stream agentllm.StreamFunc
// Now 注入时钟(nil = time.Now)。测试用它固定思考与工具耗时。
Now func() time.Time
}
func (l *LLMRunner) Name() string { return "llm" }
func (l *LLMRunner) now() time.Time {
if l.Now != nil {
return l.Now()
}
return time.Now()
}
// durationMs 把 [start, end] 换成毫秒。时钟未打点时返回 0。
func durationMs(start, end time.Time) float64 {
if start.IsZero() || end.Before(start) {
return 0
}
return float64(end.Sub(start).Microseconds()) / 1000
}
const (
toolDelegate = "delegate_task"
toolPlan = "update_plan"
@@ -245,6 +263,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
}
toolCallsMade := 0
thinkingIdx, textIdx := 0, 0
deliverableGuardUsed := false
// 交付物引用(对话内直接给出):工具轮产生的工件不进过程折叠区,
// 收集后作为内容部件钉在消息尾部(最终正文之后)——回复完给文件卡。
var deliverables []*core.ArtifactRef
@@ -259,6 +278,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
var turnThinking, turnText strings.Builder
var calls []agentllm.ToolCall
var usage agentllm.Usage
var thinkStart, thinkEnd, textStart, textEnd time.Time
ev := stream(rc.Context(), route.Model, msgs, decls, streamOpts)
phaseEmitted := false
for e := range ev {
@@ -268,9 +288,19 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
rc.Emit(core.EvRunPhase, core.RunPhasePayload{Phase: core.PhaseResponding})
phaseEmitted = true
}
now := l.now()
if textStart.IsZero() {
textStart = now
}
textEnd = now
turnText.WriteString(e.Text)
rc.Emit(core.EvMessageDelta, core.MessageDeltaPayload{Index: textIdx, Text: e.Text})
case agentllm.KindReasoningDelta:
now := l.now()
if thinkStart.IsZero() {
thinkStart = now
}
thinkEnd = now
turnThinking.WriteString(e.ReasoningDelta)
rc.Emit(core.EvThinkingDelta, core.ThinkingDeltaPayload{Index: thinkingIdx, Text: e.ReasoningDelta})
case agentllm.KindToolCallDelta:
@@ -288,25 +318,44 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
// 记录本轮部件。
if turnThinking.Len() > 0 {
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: turnThinking.String(), Process: true})
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: turnThinking.String(), Process: true, DurationMs: durationMs(thinkStart, thinkEnd)})
thinkingIdx++
}
turnBody := ""
textPartIdx := -1
if turnText.Len() > 0 {
// 工具轮的文本是过程解说(终态随过程折叠);终轮文本才是对用户的回复。
// 部分模型把推理以 <think> 内联混进 content 流:折离为思考部件(过程),
// 保证「最终输出 = 干净的对用户回复」契约。
body, think := splitThinkBlock(turnText.String())
if think != "" {
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: think, Process: true})
inlineDur := durationMs(thinkStart, thinkEnd)
if inlineDur == 0 {
inlineDur = durationMs(textStart, textEnd)
}
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: think, Process: true, DurationMs: inlineDur})
thinkingIdx++
}
if body != "" {
textPartIdx = len(finalParts)
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: body, Process: len(calls) > 0})
turnBody = body
textIdx++
}
}
if len(calls) == 0 {
// 谎称交付:没产生工件却报文件已生成,或引用了不存在的工件 ID。
// 把这段正文降为过程,纠正一次,再给模型一轮补做或如实说明的机会。
if !deliverableGuardUsed && claimsFalseDeliverable(turnBody, deliverables, rc) {
deliverableGuardUsed = true
if textPartIdx >= 0 {
finalParts[textPartIdx].Process = true
}
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: turnText.String()})
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
continue
}
// 终轮:无工具调用。
if len(finalParts) == 0 {
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: "(模型未返回内容)"})
@@ -329,6 +378,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
return fmt.Errorf("budget: 工具调用超过上限 %d(profile.budget.maxToolCalls)", maxTools)
}
rc.Emit(core.EvToolCall, core.ToolCallPayload{CallID: call.ID, Name: call.Name, Arguments: call.Arguments})
callStart := l.now()
var out string
var err error
@@ -484,13 +534,13 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
rc.Emit(core.EvToolResult, core.ToolResultPayload{CallID: call.ID, Status: "ok", Summary: summarize(out, err)})
}
summary := ""
summary := summarize(out, nil)
if err != nil {
summary = "error: " + err.Error()
out = summary
}
finalParts = append(finalParts,
core.Part{Type: core.PartToolCall, Process: true, ToolCall: &core.ToolCallPart{ID: call.ID, Name: call.Name, Arguments: call.Arguments}},
core.Part{Type: core.PartToolCall, Process: true, ToolCall: &core.ToolCallPart{ID: call.ID, Name: call.Name, Arguments: call.Arguments, Duration: durationMs(callStart, l.now())}},
core.Part{Type: core.PartToolResult, Process: true, ToolResult: &core.ToolResultPart{CallID: call.ID, Status: statusOf(err), Summary: summary}},
)
if artifactRef != nil {
@@ -576,6 +626,42 @@ func orDefault(s, def string) string {
return s
}
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。"
var (
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
)
// claimsFalseDeliverable 判断终轮正文是否在没有对应工件的情况下声称交付。
// 引用本轮已产出、或项目里确实存在的工件,不算谎称。
func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx) bool {
if strings.TrimSpace(body) == "" {
return false
}
producedIDs := map[string]bool{}
for _, ref := range produced {
if ref != nil && ref.ArtifactID != "" {
producedIDs[ref.ArtifactID] = true
}
}
knownExisting := false
for _, id := range artIDRe.FindAllString(body, -1) {
if producedIDs[id] {
continue
}
if _, _, ok := rc.Artifact(id); ok {
knownExisting = true
continue
}
return true
}
if len(producedIDs) > 0 || knownExisting {
return false
}
return fileClaimRe.MatchString(body)
}
func summarize(out string, err error) string {
if err != nil {
return truncateRun(err.Error(), 160)
+165
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"testing"
"time"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
@@ -620,3 +621,167 @@ func TestLLMRunnerCardScopeProject(t *testing.T) {
t.Errorf("项目级卡 ConversationID 应留空, got %q", rc.cards[0].ConversationID)
}
}
// TestLLMRunnerRecordsThinkingAndToolDuration 验证:思考部件带上推理增量的时长,
// 工具调用带上执行时长,成功的工具结果也留下摘要(刷新后仍能展开)。
func TestLLMRunnerRecordsThinkingAndToolDuration(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindReasoningDelta, ReasoningDelta: "先"},
agentllm.Event{Kind: agentllm.KindReasoningDelta, ReasoningDelta: "想想"},
agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{ToolCalls: []agentllm.ToolCall{
{ID: "c1", Name: "update_todos", Arguments: "{}"},
}}},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "完成了。"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var tick int64
r := &LLMRunner{
Stream: scriptStream(turns, nil),
Now: func() time.Time {
tick++
return time.Unix(0, tick*int64(time.Millisecond))
},
}
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
var thinking *core.Part
var call *core.Part
var result *core.Part
for i := range rc.assistant {
p := &rc.assistant[i]
switch p.Type {
case core.PartThinking:
thinking = p
case core.PartToolCall:
call = p
case core.PartToolResult:
result = p
}
}
if thinking == nil || thinking.DurationMs != 1 {
t.Fatalf("thinking durationMs = %v, want 1 (two reasoning deltas, 1ms apart)", thinking)
}
if call == nil || call.ToolCall == nil || call.ToolCall.Duration != 1 {
t.Fatalf("tool durationMs = %+v, want 1", call)
}
if result == nil || result.ToolResult == nil || result.ToolResult.Summary != "待办已更新" {
t.Fatalf("success summary = %+v, want 待办已更新", result)
}
}
func textParts(parts []core.Part) []core.Part {
var out []core.Part
for _, p := range parts {
if p.Type == core.PartText {
out = append(out, p)
}
}
return out
}
// TestLLMRunnerFalseDeliverableGuard 验证:终轮声称已生成文件但没有工件时,
// 这段正文降为过程,纠正一次;下一轮如实说明则作为最终回复。
func TestLLMRunnerFalseDeliverableGuard(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:\n\n研发智能体方案评审_冲突点分析.docx\n工件 ID:art_4f9d2c1a8b3e"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var calls int
var lastUser string
base := scriptStream(turns, nil)
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
if n := len(msgs); n > 0 && msgs[n-1].Role == agentllm.RoleUser {
lastUser = msgs[n-1].Content
}
return base(ctx, model, msgs, tools, opts)
}}
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
if calls != 2 {
t.Fatalf("stream calls = %d, want 2", calls)
}
if !strings.Contains(lastUser, "register_artifact") {
t.Fatalf("correction missing, got %q", lastUser)
}
texts := textParts(rc.assistant)
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
t.Fatalf("text parts = %+v, want claimed text process then honest reply", texts)
}
}
// TestLLMRunnerFalseDeliverableGuardOnce 验证:纠正只发生一次,第二次仍谎称则接受为终轮。
func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var calls int
base := scriptStream(turns, nil)
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
return base(ctx, model, msgs, tools, opts)
}}
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
if calls != 2 {
t.Fatalf("stream calls = %d, want 2 (guard fires once)", calls)
}
texts := textParts(rc.assistant)
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
t.Fatalf("text parts = %+v, want only the first claim marked process", texts)
}
}
// TestLLMRunnerExistingArtifactCitationNotGuarded 验证:引用项目里已有的工件不算谎称。
func TestLLMRunnerExistingArtifactCitationNotGuarded(t *testing.T) {
turns := [][]agentllm.Event{
{
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "上次已生成的报告.docx 在这里:art_aaaaaaaaaaaa"},
agentllm.Event{Kind: agentllm.KindMessageEnd},
},
}
var calls int
base := scriptStream(turns, nil)
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
return base(ctx, model, msgs, tools, opts)
}}
rc := &fakeRunCtx{
ctx: context.Background(),
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
execArtifact: &core.Artifact{ID: "art_aaaaaaaaaaaa", Name: "报告.docx", Kind: "document"},
}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
if calls != 1 {
t.Fatalf("stream calls = %d, want 1", calls)
}
texts := textParts(rc.assistant)
if len(texts) != 1 || texts[0].Process {
t.Fatalf("citation should stay the final reply, got %+v", texts)
}
}