Record tool time and reject false file claims #37
Vendored
+1
-1
Submodule deps/agent-system-core updated: 1b04a34bbc...d4375300cd
@@ -1,7 +1,7 @@
|
||||
---
|
||||
id: base.deliverables
|
||||
layer: base
|
||||
version: 1
|
||||
version: 2
|
||||
title: 交付物工作流
|
||||
description: 生成文档/表格/演示稿等文件交付物的通用工作流与配方(常驻基座)
|
||||
---
|
||||
@@ -11,6 +11,8 @@ description: 生成文档/表格/演示稿等文件交付物的通用工作流
|
||||
2. **登记**:调用 register_artifact登记生成文件(path 传相对路径);登记后对话内自动出现可预览的文件卡片。
|
||||
3. **收尾**:对话回复只给结论摘要与要点(几句话),全文进文件,不要在回复里复述文档内容;也不要只口头说文件名——卡片就是交付。
|
||||
|
||||
没有调用工具、没有登记工件时,禁止说「已生成 / 已保存 / 已导出」。回复里不要写工件 ID(art_…):登记成功后卡片会自动出现。
|
||||
|
||||
配方已迁入技能库(启动时种子到数据目录 skills/,完整 Python 配方见对应 SKILL.md):
|
||||
|
||||
- **docx-author**:python-docx——标题层级/表格/列表真实排版。
|
||||
|
||||
+90
-4
@@ -3,6 +3,7 @@ package runtime
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -21,10 +22,27 @@ type LLMRunner struct {
|
||||
Prompts *prompts.Service
|
||||
// Stream 提供方边界注入点(nil = 按模型路由构造 agent-native 客户端)。
|
||||
Stream agentllm.StreamFunc
|
||||
// Now 注入时钟(nil = time.Now)。测试用它固定思考与工具耗时。
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func (l *LLMRunner) Name() string { return "llm" }
|
||||
|
||||
func (l *LLMRunner) now() time.Time {
|
||||
if l.Now != nil {
|
||||
return l.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// durationMs 把 [start, end] 换成毫秒。时钟未打点时返回 0。
|
||||
func durationMs(start, end time.Time) float64 {
|
||||
if start.IsZero() || end.Before(start) {
|
||||
return 0
|
||||
}
|
||||
return float64(end.Sub(start).Microseconds()) / 1000
|
||||
}
|
||||
|
||||
const (
|
||||
toolDelegate = "delegate_task"
|
||||
toolPlan = "update_plan"
|
||||
@@ -245,6 +263,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
}
|
||||
toolCallsMade := 0
|
||||
thinkingIdx, textIdx := 0, 0
|
||||
deliverableGuardUsed := false
|
||||
// 交付物引用(对话内直接给出):工具轮产生的工件不进过程折叠区,
|
||||
// 收集后作为内容部件钉在消息尾部(最终正文之后)——回复完给文件卡。
|
||||
var deliverables []*core.ArtifactRef
|
||||
@@ -259,6 +278,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
var turnThinking, turnText strings.Builder
|
||||
var calls []agentllm.ToolCall
|
||||
var usage agentllm.Usage
|
||||
var thinkStart, thinkEnd, textStart, textEnd time.Time
|
||||
ev := stream(rc.Context(), route.Model, msgs, decls, streamOpts)
|
||||
phaseEmitted := false
|
||||
for e := range ev {
|
||||
@@ -268,9 +288,19 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
rc.Emit(core.EvRunPhase, core.RunPhasePayload{Phase: core.PhaseResponding})
|
||||
phaseEmitted = true
|
||||
}
|
||||
now := l.now()
|
||||
if textStart.IsZero() {
|
||||
textStart = now
|
||||
}
|
||||
textEnd = now
|
||||
turnText.WriteString(e.Text)
|
||||
rc.Emit(core.EvMessageDelta, core.MessageDeltaPayload{Index: textIdx, Text: e.Text})
|
||||
case agentllm.KindReasoningDelta:
|
||||
now := l.now()
|
||||
if thinkStart.IsZero() {
|
||||
thinkStart = now
|
||||
}
|
||||
thinkEnd = now
|
||||
turnThinking.WriteString(e.ReasoningDelta)
|
||||
rc.Emit(core.EvThinkingDelta, core.ThinkingDeltaPayload{Index: thinkingIdx, Text: e.ReasoningDelta})
|
||||
case agentllm.KindToolCallDelta:
|
||||
@@ -288,25 +318,44 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
|
||||
// 记录本轮部件。
|
||||
if turnThinking.Len() > 0 {
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: turnThinking.String(), Process: true})
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: turnThinking.String(), Process: true, DurationMs: durationMs(thinkStart, thinkEnd)})
|
||||
thinkingIdx++
|
||||
}
|
||||
turnBody := ""
|
||||
textPartIdx := -1
|
||||
if turnText.Len() > 0 {
|
||||
// 工具轮的文本是过程解说(终态随过程折叠);终轮文本才是对用户的回复。
|
||||
// 部分模型把推理以 <think> 内联混进 content 流:折离为思考部件(过程),
|
||||
// 保证「最终输出 = 干净的对用户回复」契约。
|
||||
body, think := splitThinkBlock(turnText.String())
|
||||
if think != "" {
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: think, Process: true})
|
||||
inlineDur := durationMs(thinkStart, thinkEnd)
|
||||
if inlineDur == 0 {
|
||||
inlineDur = durationMs(textStart, textEnd)
|
||||
}
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartThinking, Text: think, Process: true, DurationMs: inlineDur})
|
||||
thinkingIdx++
|
||||
}
|
||||
if body != "" {
|
||||
textPartIdx = len(finalParts)
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: body, Process: len(calls) > 0})
|
||||
turnBody = body
|
||||
textIdx++
|
||||
}
|
||||
}
|
||||
|
||||
if len(calls) == 0 {
|
||||
// 谎称交付:没产生工件却报文件已生成,或引用了不存在的工件 ID。
|
||||
// 把这段正文降为过程,纠正一次,再给模型一轮补做或如实说明的机会。
|
||||
if !deliverableGuardUsed && claimsFalseDeliverable(turnBody, deliverables, rc) {
|
||||
deliverableGuardUsed = true
|
||||
if textPartIdx >= 0 {
|
||||
finalParts[textPartIdx].Process = true
|
||||
}
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleAssistant, Content: turnText.String()})
|
||||
msgs = append(msgs, agentllm.Message{Role: agentllm.RoleUser, Content: deliverableGuardNote})
|
||||
continue
|
||||
}
|
||||
// 终轮:无工具调用。
|
||||
if len(finalParts) == 0 {
|
||||
finalParts = append(finalParts, core.Part{Type: core.PartText, Text: "(模型未返回内容)"})
|
||||
@@ -329,6 +378,7 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
return fmt.Errorf("budget: 工具调用超过上限 %d(profile.budget.maxToolCalls)", maxTools)
|
||||
}
|
||||
rc.Emit(core.EvToolCall, core.ToolCallPayload{CallID: call.ID, Name: call.Name, Arguments: call.Arguments})
|
||||
callStart := l.now()
|
||||
|
||||
var out string
|
||||
var err error
|
||||
@@ -484,13 +534,13 @@ func (l *LLMRunner) Execute(rc RunCtx) error {
|
||||
rc.Emit(core.EvToolResult, core.ToolResultPayload{CallID: call.ID, Status: "ok", Summary: summarize(out, err)})
|
||||
}
|
||||
|
||||
summary := ""
|
||||
summary := summarize(out, nil)
|
||||
if err != nil {
|
||||
summary = "error: " + err.Error()
|
||||
out = summary
|
||||
}
|
||||
finalParts = append(finalParts,
|
||||
core.Part{Type: core.PartToolCall, Process: true, ToolCall: &core.ToolCallPart{ID: call.ID, Name: call.Name, Arguments: call.Arguments}},
|
||||
core.Part{Type: core.PartToolCall, Process: true, ToolCall: &core.ToolCallPart{ID: call.ID, Name: call.Name, Arguments: call.Arguments, Duration: durationMs(callStart, l.now())}},
|
||||
core.Part{Type: core.PartToolResult, Process: true, ToolResult: &core.ToolResultPart{CallID: call.ID, Status: statusOf(err), Summary: summary}},
|
||||
)
|
||||
if artifactRef != nil {
|
||||
@@ -576,6 +626,42 @@ func orDefault(s, def string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
const deliverableGuardNote = "你的上一段回复声称已经生成或登记了文件,但这一轮没有产生对应的工件。请二选一:用 run_python 写出文件并调用 register_artifact 登记,或如实说明这一轮没有生成文件。不要编造工件 ID,也不要在回复里写 art_ 开头的编号。"
|
||||
|
||||
var (
|
||||
artIDRe = regexp.MustCompile(`art_[0-9a-f]{12}`)
|
||||
fileClaimRe = regexp.MustCompile(`(?s)(?:已生成|已保存|已导出|已写入).{0,120}?\.(?:docx|xlsx|pptx|pdf|png|jpe?g|csv|md|txt|json)\b`)
|
||||
)
|
||||
|
||||
// claimsFalseDeliverable 判断终轮正文是否在没有对应工件的情况下声称交付。
|
||||
// 引用本轮已产出、或项目里确实存在的工件,不算谎称。
|
||||
func claimsFalseDeliverable(body string, produced []*core.ArtifactRef, rc RunCtx) bool {
|
||||
if strings.TrimSpace(body) == "" {
|
||||
return false
|
||||
}
|
||||
producedIDs := map[string]bool{}
|
||||
for _, ref := range produced {
|
||||
if ref != nil && ref.ArtifactID != "" {
|
||||
producedIDs[ref.ArtifactID] = true
|
||||
}
|
||||
}
|
||||
knownExisting := false
|
||||
for _, id := range artIDRe.FindAllString(body, -1) {
|
||||
if producedIDs[id] {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := rc.Artifact(id); ok {
|
||||
knownExisting = true
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
if len(producedIDs) > 0 || knownExisting {
|
||||
return false
|
||||
}
|
||||
return fileClaimRe.MatchString(body)
|
||||
}
|
||||
|
||||
func summarize(out string, err error) string {
|
||||
if err != nil {
|
||||
return truncateRun(err.Error(), 160)
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
@@ -620,3 +621,167 @@ func TestLLMRunnerCardScopeProject(t *testing.T) {
|
||||
t.Errorf("项目级卡 ConversationID 应留空, got %q", rc.cards[0].ConversationID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerRecordsThinkingAndToolDuration 验证:思考部件带上推理增量的时长,
|
||||
// 工具调用带上执行时长,成功的工具结果也留下摘要(刷新后仍能展开)。
|
||||
func TestLLMRunnerRecordsThinkingAndToolDuration(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindReasoningDelta, ReasoningDelta: "先"},
|
||||
agentllm.Event{Kind: agentllm.KindReasoningDelta, ReasoningDelta: "想想"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{ToolCalls: []agentllm.ToolCall{
|
||||
{ID: "c1", Name: "update_todos", Arguments: "{}"},
|
||||
}}},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "完成了。"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var tick int64
|
||||
r := &LLMRunner{
|
||||
Stream: scriptStream(turns, nil),
|
||||
Now: func() time.Time {
|
||||
tick++
|
||||
return time.Unix(0, tick*int64(time.Millisecond))
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
var thinking *core.Part
|
||||
var call *core.Part
|
||||
var result *core.Part
|
||||
for i := range rc.assistant {
|
||||
p := &rc.assistant[i]
|
||||
switch p.Type {
|
||||
case core.PartThinking:
|
||||
thinking = p
|
||||
case core.PartToolCall:
|
||||
call = p
|
||||
case core.PartToolResult:
|
||||
result = p
|
||||
}
|
||||
}
|
||||
if thinking == nil || thinking.DurationMs != 1 {
|
||||
t.Fatalf("thinking durationMs = %v, want 1 (two reasoning deltas, 1ms apart)", thinking)
|
||||
}
|
||||
if call == nil || call.ToolCall == nil || call.ToolCall.Duration != 1 {
|
||||
t.Fatalf("tool durationMs = %+v, want 1", call)
|
||||
}
|
||||
if result == nil || result.ToolResult == nil || result.ToolResult.Summary != "待办已更新" {
|
||||
t.Fatalf("success summary = %+v, want 待办已更新", result)
|
||||
}
|
||||
}
|
||||
|
||||
func textParts(parts []core.Part) []core.Part {
|
||||
var out []core.Part
|
||||
for _, p := range parts {
|
||||
if p.Type == core.PartText {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestLLMRunnerFalseDeliverableGuard 验证:终轮声称已生成文件但没有工件时,
|
||||
// 这段正文降为过程,纠正一次;下一轮如实说明则作为最终回复。
|
||||
func TestLLMRunnerFalseDeliverableGuard(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:\n\n研发智能体方案评审_冲突点分析.docx\n工件 ID:art_4f9d2c1a8b3e"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "这一轮没有生成文件。"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var calls int
|
||||
var lastUser string
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
if n := len(msgs); n > 0 && msgs[n-1].Role == agentllm.RoleUser {
|
||||
lastUser = msgs[n-1].Content
|
||||
}
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("stream calls = %d, want 2", calls)
|
||||
}
|
||||
if !strings.Contains(lastUser, "register_artifact") {
|
||||
t.Fatalf("correction missing, got %q", lastUser)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
|
||||
t.Fatalf("text parts = %+v, want claimed text process then honest reply", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerFalseDeliverableGuardOnce 验证:纠正只发生一次,第二次仍谎称则接受为终轮。
|
||||
func TestLLMRunnerFalseDeliverableGuardOnce(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已生成:甲.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "文档已保存:乙.docx"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var calls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("stream calls = %d, want 2 (guard fires once)", calls)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) != 2 || !texts[0].Process || texts[1].Process {
|
||||
t.Fatalf("text parts = %+v, want only the first claim marked process", texts)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLLMRunnerExistingArtifactCitationNotGuarded 验证:引用项目里已有的工件不算谎称。
|
||||
func TestLLMRunnerExistingArtifactCitationNotGuarded(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
{
|
||||
agentllm.Event{Kind: agentllm.KindTextDelta, Text: "上次已生成的报告.docx 在这里:art_aaaaaaaaaaaa"},
|
||||
agentllm.Event{Kind: agentllm.KindMessageEnd},
|
||||
},
|
||||
}
|
||||
var calls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &fakeRunCtx{
|
||||
ctx: context.Background(),
|
||||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||||
execArtifact: &core.Artifact{ID: "art_aaaaaaaaaaaa", Name: "报告.docx", Kind: "document"},
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("stream calls = %d, want 1", calls)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) != 1 || texts[0].Process {
|
||||
t.Fatalf("citation should stay the final reply, got %+v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user