Check final replies before the direct run stops
Why: A direct run could stop on a missing file, a failed lint, or an unverified claim. The final turn now checks registered artifacts and asks a tool-less verifier, and nudges stale plans and todos. Evidence: go test ./... passed. New runtime tests cover missing and empty artifacts, docx lint errors, verifier reject-then-pass, unparseable and failed verifier calls, nil verifier tools, todo reminders, and the false-deliverable fixture. Impact: Direct runs with runtime.verify (default on for new runs) spend one extra lightweight call on every final turn. Setting it off keeps the previous stop behavior. Squad and SubLLMRunner are unchanged. The agent-native pin stays a34689c. Refs: feature/agent-harness-p2 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
---
|
||||
id: aux.verify.final
|
||||
layer: aux
|
||||
version: 1
|
||||
title: 终轮独立验证
|
||||
description: 不带工具,按证据清单判断终轮是否完成,只输出 JSON
|
||||
---
|
||||
你是独立验证器,不调用任何工具。根据下面给出的用户要求、终轮回复、本轮工具证据和工件清单,判断用户要求是否已经完成。
|
||||
|
||||
规则:
|
||||
- 只根据证据判断。回复里声称生成了文件,但工件清单里没有,passed 必须为 false。
|
||||
- 拿不准时 passed 必须为 false。
|
||||
- reason 写给复核的人看,说明通过或不通过的依据。
|
||||
- nextAction 写给主模型:不通过时给出下一步该做什么;通过时留空字符串。
|
||||
|
||||
只输出一个 JSON 对象,不要写其他文字,不要用代码围栏:
|
||||
{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"}
|
||||
@@ -99,6 +99,29 @@ func ScoreFile(path string) Report {
|
||||
return report
|
||||
}
|
||||
|
||||
// ScoreBytes 把内容落到带扩展名的临时文件,再走 ScoreFile。
|
||||
// docx 与 housestyle.lint 因此和按路径评分用同一条路径。
|
||||
func ScoreBytes(name string, content []byte) Report {
|
||||
ext := filepath.Ext(name)
|
||||
if ext == "" {
|
||||
ext = ".txt"
|
||||
}
|
||||
f, err := os.CreateTemp("", "score-*"+ext)
|
||||
if err != nil {
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
path := f.Name()
|
||||
defer os.Remove(path)
|
||||
if _, err := f.Write(content); err != nil {
|
||||
f.Close()
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
return ScoreFile(path)
|
||||
}
|
||||
|
||||
func scoreLead(text string) Dimension {
|
||||
title := firstHeading(text)
|
||||
if title == "" {
|
||||
|
||||
@@ -27,3 +27,15 @@ func TestScoreTextPrefersJudgementOverColumnHeading(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoreBytesUsesExtension(t *testing.T) {
|
||||
good := "# 上传文档应先判断相关性再入库\n\n## 先校验,再生成\n- 说话人1:上传后第一步是判断是否与项目相关。\n一句话:没有记忆库,后面的生成和追溯都没有依据。\n\n## 下一步\n- 事项:写出第一版需求。负责人:陈威。期限:周五前;说不清的写未指明。\n"
|
||||
got := ScoreBytes("note.md", []byte(good))
|
||||
if !got.Passed {
|
||||
t.Fatalf("markdown bytes should pass: %+v", got)
|
||||
}
|
||||
bad := ScoreBytes("note.txt", []byte("# 讨论过程\n\n今天大家聊了很多。\n"))
|
||||
if bad.Passed {
|
||||
t.Fatalf("column heading should fail: %+v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,6 +287,24 @@ func (c *runCtx) UpdatePlan(p core.PlanSnapshot) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Todos 读取本会话已有待办。没有记录时返回 false,供终轮提醒当作可选接口。
|
||||
func (c *runCtx) Todos() (core.TodoList, bool) {
|
||||
list, err := c.mgr.st.GetTodos(c.run.ConversationID)
|
||||
if err != nil {
|
||||
return core.TodoList{}, false
|
||||
}
|
||||
return list, true
|
||||
}
|
||||
|
||||
// Plan 读取本会话已有计划。没有记录时返回 false。
|
||||
func (c *runCtx) Plan() (core.PlanSnapshot, bool) {
|
||||
p, err := c.mgr.st.GetPlan(c.run.ConversationID)
|
||||
if err != nil {
|
||||
return core.PlanSnapshot{}, false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
|
||||
func (c *runCtx) UpdateTodos(l core.TodoList) error {
|
||||
l.ConversationID = c.run.ConversationID
|
||||
if err := c.mgr.st.SaveTodos(c.run.ConversationID, l); err != nil {
|
||||
|
||||
@@ -24,6 +24,8 @@ type LLMRunner struct {
|
||||
Stream agentllm.StreamFunc
|
||||
// Now 注入时钟(nil = time.Now)。测试用它固定思考与工具耗时。
|
||||
Now func() time.Time
|
||||
// VerifyStream 是终轮验证器的注入点(nil = 按轻量模型构造客户端)。
|
||||
VerifyStream agentllm.StreamFunc
|
||||
}
|
||||
|
||||
func (l *LLMRunner) Name() string { return "llm" }
|
||||
|
||||
@@ -40,6 +40,20 @@ type directRun struct {
|
||||
queued []agentllm.ToolCall
|
||||
inflight *toolSlot
|
||||
turnBody string
|
||||
|
||||
evidence []toolEvidence
|
||||
todos core.TodoList
|
||||
plan core.PlanSnapshot
|
||||
closedToolTurn bool
|
||||
checklistTouched bool
|
||||
toolTurnsSince int
|
||||
checklistReminders int
|
||||
}
|
||||
|
||||
type toolEvidence struct {
|
||||
Name string
|
||||
Status string
|
||||
Summary string
|
||||
}
|
||||
|
||||
type toolSlot struct {
|
||||
@@ -65,6 +79,7 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo
|
||||
|
||||
func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int) error {
|
||||
st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow}
|
||||
st.loadChecklist()
|
||||
defer func() {
|
||||
if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok {
|
||||
sink.AddUsage(st.usageTotal)
|
||||
@@ -123,6 +138,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin
|
||||
ContinueOnLength: true,
|
||||
OnEvent: st.onEvent,
|
||||
Stop: st.stop,
|
||||
Reminders: st.reminders,
|
||||
UnknownTool: func(call agentllm.ToolCall) agenttools.Result {
|
||||
return st.runTool(call.Name, call.Arguments)
|
||||
},
|
||||
@@ -192,6 +208,9 @@ func (st *directRun) onEvent(ev nruntime.Event) {
|
||||
st.finalParts = append(st.finalParts, core.Part{Type: core.PartThinking, Text: st.turnThink.String(), Process: true, DurationMs: durationMs(st.thinkStart, st.thinkEnd)})
|
||||
st.thinkingIdx++
|
||||
}
|
||||
if hasCalls {
|
||||
st.closedToolTurn = true
|
||||
}
|
||||
if st.turnText.Len() > 0 {
|
||||
body, think := splitThinkBlock(st.turnText.String())
|
||||
if think != "" {
|
||||
@@ -233,26 +252,72 @@ func (st *directRun) onEvent(ev nruntime.Event) {
|
||||
if slot.artifact != nil {
|
||||
st.deliverables = append(st.deliverables, slot.artifact)
|
||||
}
|
||||
st.evidence = append(st.evidence, toolEvidence{Name: slot.call.Name, Status: slot.status, Summary: slot.summary})
|
||||
_ = st.rc.Emit(core.EvToolResult, core.ToolResultPayload{CallID: slot.call.ID, Status: slot.status, Summary: slot.summary, Truncated: slot.truncated})
|
||||
st.inflight = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (st *directRun) stop(last agentllm.Message, info nruntime.StopInfo) nruntime.StopDecision {
|
||||
block, _ := deliverableGate(st.turnBody, st.deliverables, st.rc)
|
||||
if !block {
|
||||
if block, why := deliverableGate(st.turnBody, st.deliverables, st.rc); block {
|
||||
st.demoteLastText()
|
||||
if st.eff.GetBool("runtime.verify", false) {
|
||||
st.addVerifyNote(why)
|
||||
}
|
||||
if info.Continuations < maxDeliverableGuards {
|
||||
return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote}
|
||||
}
|
||||
return nruntime.StopDecision{Notice: deliverableFallbackNotice}
|
||||
}
|
||||
if !st.eff.GetBool("runtime.verify", false) {
|
||||
return nruntime.StopDecision{}
|
||||
}
|
||||
if reasons := checkDeliverables(st.rc, st.deliverables); len(reasons) > 0 {
|
||||
return st.continueOrNotice(info, strings.Join(reasons, ";"))
|
||||
}
|
||||
verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, st.deliverables)
|
||||
if err != nil {
|
||||
st.addVerifyNote(err.Error())
|
||||
return nruntime.StopDecision{Notice: verifierUnavailableNotice}
|
||||
}
|
||||
if !verdict.Passed {
|
||||
reason := strings.TrimSpace(verdict.Reason)
|
||||
if reason == "" {
|
||||
reason = "未说明原因"
|
||||
}
|
||||
if next := strings.TrimSpace(verdict.NextAction); next != "" {
|
||||
reason += "。下一步:" + next
|
||||
}
|
||||
return st.continueOrNotice(info, reason)
|
||||
}
|
||||
return nruntime.StopDecision{}
|
||||
}
|
||||
|
||||
func (st *directRun) demoteLastText() {
|
||||
for i := len(st.finalParts) - 1; i >= 0; i-- {
|
||||
if st.finalParts[i].Type == core.PartText {
|
||||
st.finalParts[i].Process = true
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
if info.Continuations < maxDeliverableGuards {
|
||||
return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote}
|
||||
}
|
||||
|
||||
func (st *directRun) addVerifyNote(reason string) {
|
||||
st.finalParts = append(st.finalParts, core.Part{
|
||||
Type: core.PartText,
|
||||
Process: true,
|
||||
Text: "(验证未通过:" + reason + ")",
|
||||
})
|
||||
}
|
||||
|
||||
func (st *directRun) continueOrNotice(info nruntime.StopInfo, reason string) nruntime.StopDecision {
|
||||
if info.Continuations < maxVerifyContinuations {
|
||||
st.demoteLastText()
|
||||
st.addVerifyNote(reason)
|
||||
return nruntime.StopDecision{Continue: true, Reason: reason}
|
||||
}
|
||||
return nruntime.StopDecision{Notice: deliverableFallbackNotice}
|
||||
st.addVerifyNote(reason)
|
||||
return nruntime.StopDecision{Notice: verifyCapNotice}
|
||||
}
|
||||
|
||||
func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
@@ -290,6 +355,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
var p core.PlanSnapshot
|
||||
if uerr := json.Unmarshal([]byte(call.Arguments), &p); uerr == nil {
|
||||
_ = st.rc.UpdatePlan(p)
|
||||
st.plan = p
|
||||
st.checklistTouched = true
|
||||
out = "计划已更新"
|
||||
} else {
|
||||
out = "计划参数错误: " + uerr.Error()
|
||||
@@ -298,6 +365,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
var t core.TodoList
|
||||
if uerr := json.Unmarshal([]byte(call.Arguments), &t); uerr == nil {
|
||||
_ = st.rc.UpdateTodos(t)
|
||||
st.todos = t
|
||||
st.checklistTouched = true
|
||||
out = "待办已更新"
|
||||
} else {
|
||||
out = "待办参数错误: " + uerr.Error()
|
||||
|
||||
@@ -32,6 +32,9 @@ type fakeRunCtx struct {
|
||||
execTruncated bool // ExecuteTool 返回的截断标记(门面卡触发)
|
||||
toolRuns int // ExecuteTool 被调用的次数
|
||||
execArtifactContent []byte // Artifact(id) 回填内容(ctxasm D46 断言用)
|
||||
verify bool // runtime.verify;缺省关闭,避免改变既有流调用次数
|
||||
missingArtifact bool // 工具登记了工件,但 Artifact 查不到
|
||||
emptyArtifact bool // Artifact 查到了,内容为空
|
||||
}
|
||||
|
||||
func (f *fakeRunCtx) Context() context.Context { return f.ctx }
|
||||
@@ -42,6 +45,9 @@ func (f *fakeRunCtx) Settings() core.EffectiveSettings {
|
||||
if f.cardflow {
|
||||
eff["ui"] = map[string]any{"cardflow": true}
|
||||
}
|
||||
if f.verify {
|
||||
eff["runtime"] = map[string]any{"verify": true}
|
||||
}
|
||||
return eff
|
||||
}
|
||||
func (f *fakeRunCtx) Emit(typ core.StreamEventType, payload any) error {
|
||||
@@ -83,6 +89,12 @@ func (f *fakeRunCtx) CreateCard(card core.ActivityCard) (core.ActivityCard, erro
|
||||
|
||||
func (f *fakeRunCtx) Artifact(id string) (core.Artifact, []byte, bool) {
|
||||
if f.execArtifact != nil && f.execArtifact.ID == id {
|
||||
if f.missingArtifact {
|
||||
return core.Artifact{}, nil, false
|
||||
}
|
||||
if f.emptyArtifact {
|
||||
return *f.execArtifact, nil, true
|
||||
}
|
||||
return *f.execArtifact, f.execArtifactContent, true
|
||||
}
|
||||
return core.Artifact{}, nil, false
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event {
|
||||
evs := []agentllm.Event{}
|
||||
if body != "" {
|
||||
evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body})
|
||||
}
|
||||
end := agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
if len(calls) > 0 {
|
||||
end.Message.ToolCalls = calls
|
||||
}
|
||||
return append(evs, end)
|
||||
}
|
||||
|
||||
func toolCall(name, args string) agentllm.ToolCall {
|
||||
return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args}
|
||||
}
|
||||
|
||||
func passedStream(payloads ...string) agentllm.StreamFunc {
|
||||
n := 0
|
||||
return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
body := `{"passed":true,"reason":"完成","nextAction":""}`
|
||||
if n < len(payloads) {
|
||||
body = payloads[n]
|
||||
n++
|
||||
}
|
||||
out := make(chan agentllm.Event, 2)
|
||||
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
close(out)
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
func verifyOn() core.AgentProfile {
|
||||
return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}
|
||||
}
|
||||
|
||||
func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
miss bool
|
||||
want string
|
||||
}{
|
||||
{name: "missing", miss: true, want: "不存在"},
|
||||
{name: "empty", miss: false, want: "内容为空"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
turns := padTurns(
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("写好了。"),
|
||||
)
|
||||
var calls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: passedStream(),
|
||||
}
|
||||
rc := &fakeRunCtx{
|
||||
ctx: context.Background(),
|
||||
prof: verifyOn(),
|
||||
verify: true,
|
||||
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"},
|
||||
missingArtifact: tc.miss,
|
||||
emptyArtifact: !tc.miss,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("stream calls = %d, want a continuation", calls)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, tc.want) {
|
||||
t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintErrorContinues(t *testing.T) {
|
||||
style := filepath.Join("..", "skills", "recipes", "house-style")
|
||||
abs, err := filepath.Abs(style)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PYTHONPATH", abs)
|
||||
docx := badDocx(t)
|
||||
turns := padTurns(
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("写好了。"),
|
||||
)
|
||||
var verifyCalls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: base,
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
verifyCalls++
|
||||
return passedStream()(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{
|
||||
ctx: context.Background(),
|
||||
prof: verifyOn(),
|
||||
verify: true,
|
||||
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"},
|
||||
execArtifactContent: docx,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if verifyCalls != 0 {
|
||||
t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, "版式") {
|
||||
t.Fatalf("lint reason missing: %+v", textParts(rc.assistant))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifierRejectsThenPasses(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn("答案是 42。"),
|
||||
textDeltaTurn("答案仍是 42。"),
|
||||
}
|
||||
var mainCalls int
|
||||
var models []string
|
||||
var toolArgs [][]agentllm.ToolDecl
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
mainCalls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
models = append(models, model)
|
||||
toolArgs = append(toolArgs, tools)
|
||||
body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}`
|
||||
if len(models) > 1 {
|
||||
body = `{"passed":true,"reason":"完成","nextAction":""}`
|
||||
}
|
||||
return passedStream(body)(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if mainCalls != 2 {
|
||||
t.Fatalf("main calls = %d, want 2", mainCalls)
|
||||
}
|
||||
if len(models) != 2 || models[0] != "lite-model" {
|
||||
t.Fatalf("verifier models = %v, want lite-model twice", models)
|
||||
}
|
||||
for i, tools := range toolArgs {
|
||||
if tools != nil {
|
||||
t.Fatalf("verifier call %d tools = %#v, want nil", i, tools)
|
||||
}
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) < 3 || !texts[0].Process || !strings.Contains(texts[1].Text, "验证未通过") || texts[len(texts)-1].Process {
|
||||
t.Fatalf("parts = %+v", texts)
|
||||
}
|
||||
if !strings.Contains(texts[1].Text, "补上结论") {
|
||||
t.Fatalf("nextAction missing: %+v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifierGarbageFailsClosed(t *testing.T) {
|
||||
for _, payload := range []agentllm.Event{
|
||||
{Kind: agentllm.KindTextDelta, Text: "当然通过"},
|
||||
{Kind: agentllm.KindError, ErrorMessage: "upstream"},
|
||||
} {
|
||||
t.Run(string(payload.Kind), func(t *testing.T) {
|
||||
turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")}
|
||||
var mainCalls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
mainCalls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
if tools != nil {
|
||||
t.Errorf("verifier tools = %#v, want nil", tools)
|
||||
}
|
||||
out := make(chan agentllm.Event, 2)
|
||||
out <- payload
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
close(out)
|
||||
return out
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if mainCalls != 1 {
|
||||
t.Fatalf("main calls = %d, want 1", mainCalls)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
var kept, notice bool
|
||||
for _, p := range texts {
|
||||
if p.Text == "答案是 42。" && !p.Process {
|
||||
kept = true
|
||||
}
|
||||
if !p.Process && strings.Contains(p.Text, "验证器不可用") {
|
||||
notice = true
|
||||
}
|
||||
}
|
||||
if !kept || !notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") {
|
||||
t.Fatalf("parts = %+v", texts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVerdictToleratesFence(t *testing.T) {
|
||||
got, err := parseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```")
|
||||
if err != nil || got.Passed || got.Reason != "不够" {
|
||||
t.Fatalf("verdict = %+v %v", got, err)
|
||||
}
|
||||
if _, err := parseVerdict(`{"reason":"no passed"}`); err == nil {
|
||||
t.Fatal("missing passed should fail")
|
||||
}
|
||||
}
|
||||
|
||||
type checklistFake struct {
|
||||
fakeRunCtx
|
||||
list core.TodoList
|
||||
}
|
||||
|
||||
func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true }
|
||||
|
||||
func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event {
|
||||
for len(turns) < 5 {
|
||||
turns = append(turns, textDeltaTurn("仍未修好。"))
|
||||
}
|
||||
return turns
|
||||
}
|
||||
|
||||
func TestTodoReminderFiresOnceAndResets(t *testing.T) {
|
||||
quiet := textDeltaTurn("", toolCall("make_report", "{}"))
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("完成了。"),
|
||||
}
|
||||
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
||||
if len(requests) != 4 {
|
||||
t.Fatalf("requests = %d, want 4", len(requests))
|
||||
}
|
||||
for i, req := range requests[:3] {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
t.Fatalf("request %d reminded too early: %s", i, req)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") {
|
||||
t.Fatalf("final request missing reminder: %s", requests[3])
|
||||
}
|
||||
|
||||
update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}`
|
||||
resetTurns := [][]agentllm.Event{
|
||||
textDeltaTurn("", toolCall("update_todos", update)),
|
||||
quiet, quiet, quiet,
|
||||
textDeltaTurn("完成了。"),
|
||||
}
|
||||
requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}})
|
||||
if len(requests) != 5 {
|
||||
t.Fatalf("reset requests = %d, want 5", len(requests))
|
||||
}
|
||||
for i, req := range requests[:4] {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
t.Fatalf("request %d reminded before three quiet turns: %s", i, req)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") {
|
||||
t.Fatalf("reminder did not refresh todos: %s", requests[4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTodoReminderCapsAtThree(t *testing.T) {
|
||||
var turns [][]agentllm.Event
|
||||
for i := 0; i < 12; i++ {
|
||||
turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}")))
|
||||
}
|
||||
turns = append(turns, textDeltaTurn("完成了。"))
|
||||
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
||||
n := 0
|
||||
for _, req := range requests {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 3 {
|
||||
t.Fatalf("reminders = %d, want 3", n)
|
||||
}
|
||||
}
|
||||
|
||||
func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string {
|
||||
t.Helper()
|
||||
var requests []string
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
requests = append(requests, joinContents(msgs))
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &checklistFake{
|
||||
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
||||
list: list,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range rc.assistant {
|
||||
if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") {
|
||||
t.Fatalf("reminder was persisted: %+v", p)
|
||||
}
|
||||
}
|
||||
return requests
|
||||
}
|
||||
|
||||
func joinContents(msgs []agentllm.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hasProcessNote(parts []core.Part, fragment string) bool {
|
||||
for _, p := range parts {
|
||||
if p.Type == core.PartText && p.Process && strings.Contains(p.Text, "验证未通过") && strings.Contains(p.Text, fragment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func badDocx(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.docx")
|
||||
cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")")
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("docx: %v %s", err, out)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func jsonString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestFalseDeliverableFixtures(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/false_deliverables.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Cases []struct {
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
} `json:"cases"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Cases) < 5 {
|
||||
t.Fatalf("cases = %d, want at least 5", len(doc.Cases))
|
||||
}
|
||||
for _, c := range doc.Cases {
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
block, reason := deliverableGate(c.Body, nil, &fakeRunCtx{ctx: context.Background()})
|
||||
if !block || strings.TrimSpace(reason) == "" {
|
||||
t.Fatalf("gate = %v %q", block, reason)
|
||||
}
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn(c.Body),
|
||||
textDeltaTurn("这一轮没有生成文件。"),
|
||||
}
|
||||
r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, reason) {
|
||||
t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,252 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
"agent-system-backend/internal/quality"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxVerifyContinuations 与内核 Stop 的 3 次上限对齐。
|
||||
maxVerifyContinuations = 3
|
||||
// reminderToolGap 连续这么多个工具轮没有更新计划或待办,就提醒一次。
|
||||
reminderToolGap = 3
|
||||
// maxChecklistReminders 每个 run 最多提醒这么多次。
|
||||
maxChecklistReminders = 3
|
||||
)
|
||||
|
||||
const verifyCapNotice = "本轮回复未通过终轮检查。"
|
||||
|
||||
const verifierUnavailableNotice = "本轮回复未通过独立验证(验证器不可用)"
|
||||
|
||||
const verifyFallbackPrompt = `你是独立验证器,不调用任何工具。根据用户要求、终轮回复、工具证据和工件清单判断这一轮是否已经完成。
|
||||
拿不准时 passed 必须为 false。只输出一个 JSON 对象,不要写其他文字:{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"}。`
|
||||
|
||||
type verifyVerdict struct {
|
||||
Passed bool `json:"passed"`
|
||||
Reason string `json:"reason"`
|
||||
NextAction string `json:"nextAction"`
|
||||
}
|
||||
|
||||
// checkDeliverables 确认本轮登记的工件存在且非空。
|
||||
// md、txt、docx 还要过 quality;其他类型只检查存在与非空。
|
||||
func checkDeliverables(rc RunCtx, refs []*core.ArtifactRef) []string {
|
||||
var reasons []string
|
||||
for _, ref := range refs {
|
||||
if ref == nil || ref.ArtifactID == "" {
|
||||
reasons = append(reasons, "工件引用缺少 ID")
|
||||
continue
|
||||
}
|
||||
art, content, ok := rc.Artifact(ref.ArtifactID)
|
||||
name := ref.Name
|
||||
if name == "" {
|
||||
name = string(ref.ArtifactID)
|
||||
}
|
||||
if !ok {
|
||||
reasons = append(reasons, name+" 不存在")
|
||||
continue
|
||||
}
|
||||
if art.Name != "" {
|
||||
name = art.Name
|
||||
}
|
||||
if len(content) == 0 {
|
||||
reasons = append(reasons, name+" 内容为空")
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".md", ".txt", ".docx":
|
||||
report := quality.ScoreBytes(name, content)
|
||||
if !report.Passed {
|
||||
why := strings.Join(report.Violations, ";")
|
||||
if why == "" {
|
||||
why = "评分未通过"
|
||||
}
|
||||
reasons = append(reasons, name+":"+why)
|
||||
}
|
||||
}
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence, refs []*core.ArtifactRef) (verifyVerdict, error) {
|
||||
route := ResolveLightweightModel(rc.Profile(), rc.Settings())
|
||||
stream := l.VerifyStream
|
||||
if stream == nil {
|
||||
if route.Model == "" || route.Base == "" {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器调用失败")
|
||||
}
|
||||
client := &agentllm.Client{BaseURL: route.Base, APIKey: route.Key}
|
||||
stream = client.Stream
|
||||
}
|
||||
model := route.Model
|
||||
if model == "" {
|
||||
model = "verify"
|
||||
}
|
||||
msgs := []agentllm.Message{
|
||||
{Role: agentllm.RoleSystem, Content: l.verifyPrompt(rc.Profile().Owner)},
|
||||
{Role: agentllm.RoleUser, Content: evidencePrompt(currentUserText(rc), body, evidence, refs)},
|
||||
}
|
||||
var b strings.Builder
|
||||
for ev := range stream(rc.Context(), model, msgs, nil, agentllm.StreamOptions{}) {
|
||||
switch ev.Kind {
|
||||
case agentllm.KindTextDelta:
|
||||
b.WriteString(ev.Text)
|
||||
case agentllm.KindError:
|
||||
return verifyVerdict{}, fmt.Errorf("验证器调用失败")
|
||||
case agentllm.KindMessageEnd:
|
||||
if b.Len() == 0 {
|
||||
b.WriteString(ev.Message.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
verdict, err := parseVerdict(b.String())
|
||||
if err != nil {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
return verdict, nil
|
||||
}
|
||||
|
||||
func (l *LLMRunner) verifyPrompt(owner string) string {
|
||||
if l.Prompts != nil {
|
||||
if body, ok, err := l.Prompts.ModuleBody(owner, "aux.verify.final"); err == nil && ok && strings.TrimSpace(body) != "" {
|
||||
return body
|
||||
}
|
||||
}
|
||||
return verifyFallbackPrompt
|
||||
}
|
||||
|
||||
func evidencePrompt(user, body string, evidence []toolEvidence, refs []*core.ArtifactRef) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("用户要求:\n")
|
||||
if strings.TrimSpace(user) == "" {
|
||||
b.WriteString("(无)\n")
|
||||
} else {
|
||||
b.WriteString(user)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString("\n终轮回复:\n")
|
||||
b.WriteString(body)
|
||||
b.WriteString("\n\n本轮工具:\n")
|
||||
if len(evidence) == 0 {
|
||||
b.WriteString("(无)\n")
|
||||
}
|
||||
for _, ev := range evidence {
|
||||
fmt.Fprintf(&b, "- %s %s %s\n", ev.Name, ev.Status, ev.Summary)
|
||||
}
|
||||
b.WriteString("\n本轮工件:\n")
|
||||
if len(refs) == 0 {
|
||||
b.WriteString("(无)\n")
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if ref == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "- %s %s\n", ref.ArtifactID, ref.Name)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func parseVerdict(raw string) (verifyVerdict, error) {
|
||||
s := stripFence(raw)
|
||||
start := strings.Index(s, "{")
|
||||
end := strings.LastIndex(s, "}")
|
||||
if start < 0 || end < start {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
s = s[start : end+1]
|
||||
var probe map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(s), &probe); err != nil {
|
||||
return verifyVerdict{}, err
|
||||
}
|
||||
if _, ok := probe["passed"]; !ok {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
var verdict verifyVerdict
|
||||
if err := json.Unmarshal([]byte(s), &verdict); err != nil {
|
||||
return verifyVerdict{}, err
|
||||
}
|
||||
return verdict, nil
|
||||
}
|
||||
|
||||
func stripFence(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if !strings.HasPrefix(s, "```") {
|
||||
return s
|
||||
}
|
||||
s = strings.TrimPrefix(s, "```")
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "```"))
|
||||
return s
|
||||
}
|
||||
|
||||
func (st *directRun) loadChecklist() {
|
||||
if src, ok := st.rc.(interface{ Todos() (core.TodoList, bool) }); ok {
|
||||
if list, ok := src.Todos(); ok {
|
||||
st.todos = list
|
||||
}
|
||||
}
|
||||
if src, ok := st.rc.(interface {
|
||||
Plan() (core.PlanSnapshot, bool)
|
||||
}); ok {
|
||||
if p, ok := src.Plan(); ok {
|
||||
st.plan = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (st *directRun) reminders(turn int) []string {
|
||||
if st.closedToolTurn {
|
||||
if st.checklistTouched {
|
||||
st.toolTurnsSince = 0
|
||||
} else {
|
||||
st.toolTurnsSince++
|
||||
}
|
||||
st.closedToolTurn = false
|
||||
st.checklistTouched = false
|
||||
}
|
||||
if st.toolTurnsSince < reminderToolGap || st.checklistReminders >= maxChecklistReminders {
|
||||
return nil
|
||||
}
|
||||
text := st.unfinishedReminder()
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
st.toolTurnsSince = 0
|
||||
st.checklistReminders++
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
func (st *directRun) unfinishedReminder() string {
|
||||
var lines []string
|
||||
for _, item := range st.todos.Items {
|
||||
if item.Status == "done" {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(item.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, "- 待办:"+text)
|
||||
}
|
||||
for _, step := range st.plan.Steps {
|
||||
if step.Status == "done" || step.Status == "skipped" {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(step.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, "- 计划:"+text)
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "计划和待办已连续 3 个工具轮没有更新。未完成项:\n" + strings.Join(lines, "\n") + "\n请更新计划或待办,或在回复里说明为什么先不做。"
|
||||
}
|
||||
@@ -35,6 +35,7 @@ func SystemDefaults() map[string]any {
|
||||
"entry": "direct", // direct = 现有对话循环;squad = 小队分诊入口(灰度,达标后再改默认)
|
||||
"rateLimitPerMin": 0, // 租户限流(次/分;0 = 不限)
|
||||
"redisAddr": "", // 限流共享后端(原生 Redis addr;空 = 进程内)
|
||||
"verify": true, // 终轮确定性检查与独立验证
|
||||
},
|
||||
"storage": map[string]any{
|
||||
"dataDir": "./data", // 启动注入实际值(--data)
|
||||
@@ -77,6 +78,7 @@ func Schema() []core.SettingDesc {
|
||||
{Key: "runtime.entry", Group: "runtime", Label: "对话入口", Type: core.SettingEnum, Default: "direct", Options: []string{"direct", "squad"}, Effect: core.EffectNewRun, Description: "direct 走现有对话循环。squad 由小队先分诊:简单任务单步完成,复杂任务由 leader 编排。灰度期默认 direct"},
|
||||
{Key: "runtime.rateLimitPerMin", Group: "runtime", Label: "租户限流(次/分)", Type: core.SettingNumber, Default: 0, Effect: core.EffectHot, Description: "0 = 不限;多用户建议 300–1200"},
|
||||
{Key: "runtime.redisAddr", Group: "runtime", Label: "Redis 地址(限流共享)", Type: core.SettingString, Default: "", Effect: core.EffectRestart, Description: "原生 Redis(如 127.0.0.1:6379);空 = 进程内限流(单实例)"},
|
||||
{Key: "runtime.verify", Group: "runtime", Label: "终轮结果检查", Type: core.SettingBool, Default: true, Effect: core.EffectNewRun, Description: "终轮确定性检查与独立验证"},
|
||||
{Key: "context.maxHistoryMessages", Group: "context", Label: "历史窗口条数", Type: core.SettingNumber, Default: 40, Effect: core.EffectNewRun},
|
||||
{Key: "context.maxToolResultChars", Group: "context", Label: "工具结果上限(字符)", Type: core.SettingNumber, Default: 20000, Effect: core.EffectNewRun},
|
||||
{Key: "crossSession.read", Group: "crossSession", Label: "跨会话读取", Type: core.SettingBool, Default: false, Effect: core.EffectNewRun, Description: "允许本项目会话读取同项目其他会话的对话记录(list_conversations / read_conversation 工具);会话可单独关闭读取或设为私有"},
|
||||
|
||||
Reference in New Issue
Block a user