Check final replies before the direct run stops #44
@@ -0,0 +1,17 @@
|
||||
---
|
||||
id: aux.verify.final
|
||||
layer: aux
|
||||
version: 1
|
||||
title: 终轮独立验证
|
||||
description: 不带工具,按证据清单判断终轮是否完成,只输出 JSON
|
||||
---
|
||||
你是独立验证器,不调用任何工具。根据下面给出的用户要求、终轮回复、本轮工具证据和工件清单,判断用户要求是否已经完成。
|
||||
|
||||
规则:
|
||||
- 只根据证据判断。回复里声称生成了文件,但工件清单里没有,passed 必须为 false。
|
||||
- 拿不准时 passed 必须为 false。
|
||||
- reason 写给复核的人看,说明通过或不通过的依据。
|
||||
- nextAction 写给主模型:不通过时给出下一步该做什么;通过时留空字符串。
|
||||
|
||||
只输出一个 JSON 对象,不要写其他文字,不要用代码围栏:
|
||||
{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"}
|
||||
@@ -99,6 +99,29 @@ func ScoreFile(path string) Report {
|
||||
return report
|
||||
}
|
||||
|
||||
// ScoreBytes 把内容落到带扩展名的临时文件,再走 ScoreFile。
|
||||
// docx 与 housestyle.lint 因此和按路径评分用同一条路径。
|
||||
func ScoreBytes(name string, content []byte) Report {
|
||||
ext := filepath.Ext(name)
|
||||
if ext == "" {
|
||||
ext = ".txt"
|
||||
}
|
||||
f, err := os.CreateTemp("", "score-*"+ext)
|
||||
if err != nil {
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
path := f.Name()
|
||||
defer os.Remove(path)
|
||||
if _, err := f.Write(content); err != nil {
|
||||
f.Close()
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
|
||||
}
|
||||
return ScoreFile(path)
|
||||
}
|
||||
|
||||
func scoreLead(text string) Dimension {
|
||||
title := firstHeading(text)
|
||||
if title == "" {
|
||||
|
||||
@@ -27,3 +27,15 @@ func TestScoreTextPrefersJudgementOverColumnHeading(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScoreBytesUsesExtension(t *testing.T) {
|
||||
good := "# 上传文档应先判断相关性再入库\n\n## 先校验,再生成\n- 说话人1:上传后第一步是判断是否与项目相关。\n一句话:没有记忆库,后面的生成和追溯都没有依据。\n\n## 下一步\n- 事项:写出第一版需求。负责人:陈威。期限:周五前;说不清的写未指明。\n"
|
||||
got := ScoreBytes("note.md", []byte(good))
|
||||
if !got.Passed {
|
||||
t.Fatalf("markdown bytes should pass: %+v", got)
|
||||
}
|
||||
bad := ScoreBytes("note.txt", []byte("# 讨论过程\n\n今天大家聊了很多。\n"))
|
||||
if bad.Passed {
|
||||
t.Fatalf("column heading should fail: %+v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,6 +287,24 @@ func (c *runCtx) UpdatePlan(p core.PlanSnapshot) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Todos 读取本会话已有待办。没有记录时返回 false,供终轮提醒当作可选接口。
|
||||
func (c *runCtx) Todos() (core.TodoList, bool) {
|
||||
list, err := c.mgr.st.GetTodos(c.run.ConversationID)
|
||||
if err != nil {
|
||||
return core.TodoList{}, false
|
||||
}
|
||||
return list, true
|
||||
}
|
||||
|
||||
// Plan 读取本会话已有计划。没有记录时返回 false。
|
||||
func (c *runCtx) Plan() (core.PlanSnapshot, bool) {
|
||||
p, err := c.mgr.st.GetPlan(c.run.ConversationID)
|
||||
if err != nil {
|
||||
return core.PlanSnapshot{}, false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
|
||||
func (c *runCtx) UpdateTodos(l core.TodoList) error {
|
||||
l.ConversationID = c.run.ConversationID
|
||||
if err := c.mgr.st.SaveTodos(c.run.ConversationID, l); err != nil {
|
||||
|
||||
@@ -24,6 +24,8 @@ type LLMRunner struct {
|
||||
Stream agentllm.StreamFunc
|
||||
// Now 注入时钟(nil = time.Now)。测试用它固定思考与工具耗时。
|
||||
Now func() time.Time
|
||||
// VerifyStream 是终轮验证器的注入点(nil = 按轻量模型构造客户端)。
|
||||
VerifyStream agentllm.StreamFunc
|
||||
}
|
||||
|
||||
func (l *LLMRunner) Name() string { return "llm" }
|
||||
|
||||
@@ -40,6 +40,20 @@ type directRun struct {
|
||||
queued []agentllm.ToolCall
|
||||
inflight *toolSlot
|
||||
turnBody string
|
||||
|
||||
evidence []toolEvidence
|
||||
todos core.TodoList
|
||||
plan core.PlanSnapshot
|
||||
closedToolTurn bool
|
||||
checklistTouched bool
|
||||
toolTurnsSince int
|
||||
checklistReminders int
|
||||
}
|
||||
|
||||
type toolEvidence struct {
|
||||
Name string
|
||||
Status string
|
||||
Summary string
|
||||
}
|
||||
|
||||
type toolSlot struct {
|
||||
@@ -65,6 +79,7 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo
|
||||
|
||||
func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int) error {
|
||||
st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow}
|
||||
st.loadChecklist()
|
||||
defer func() {
|
||||
if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok {
|
||||
sink.AddUsage(st.usageTotal)
|
||||
@@ -123,6 +138,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin
|
||||
ContinueOnLength: true,
|
||||
OnEvent: st.onEvent,
|
||||
Stop: st.stop,
|
||||
Reminders: st.reminders,
|
||||
UnknownTool: func(call agentllm.ToolCall) agenttools.Result {
|
||||
return st.runTool(call.Name, call.Arguments)
|
||||
},
|
||||
@@ -192,6 +208,9 @@ func (st *directRun) onEvent(ev nruntime.Event) {
|
||||
st.finalParts = append(st.finalParts, core.Part{Type: core.PartThinking, Text: st.turnThink.String(), Process: true, DurationMs: durationMs(st.thinkStart, st.thinkEnd)})
|
||||
st.thinkingIdx++
|
||||
}
|
||||
if hasCalls {
|
||||
st.closedToolTurn = true
|
||||
}
|
||||
if st.turnText.Len() > 0 {
|
||||
body, think := splitThinkBlock(st.turnText.String())
|
||||
if think != "" {
|
||||
@@ -233,26 +252,72 @@ func (st *directRun) onEvent(ev nruntime.Event) {
|
||||
if slot.artifact != nil {
|
||||
st.deliverables = append(st.deliverables, slot.artifact)
|
||||
}
|
||||
st.evidence = append(st.evidence, toolEvidence{Name: slot.call.Name, Status: slot.status, Summary: slot.summary})
|
||||
_ = st.rc.Emit(core.EvToolResult, core.ToolResultPayload{CallID: slot.call.ID, Status: slot.status, Summary: slot.summary, Truncated: slot.truncated})
|
||||
st.inflight = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (st *directRun) stop(last agentllm.Message, info nruntime.StopInfo) nruntime.StopDecision {
|
||||
block, _ := deliverableGate(st.turnBody, st.deliverables, st.rc)
|
||||
if !block {
|
||||
if block, why := deliverableGate(st.turnBody, st.deliverables, st.rc); block {
|
||||
st.demoteLastText()
|
||||
if st.eff.GetBool("runtime.verify", false) {
|
||||
st.addVerifyNote(why)
|
||||
}
|
||||
if info.Continuations < maxDeliverableGuards {
|
||||
return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote}
|
||||
}
|
||||
return nruntime.StopDecision{Notice: deliverableFallbackNotice}
|
||||
}
|
||||
if !st.eff.GetBool("runtime.verify", false) {
|
||||
return nruntime.StopDecision{}
|
||||
}
|
||||
if reasons := checkDeliverables(st.rc, st.deliverables); len(reasons) > 0 {
|
||||
return st.continueOrNotice(info, strings.Join(reasons, ";"))
|
||||
}
|
||||
verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, st.deliverables)
|
||||
if err != nil {
|
||||
st.addVerifyNote(err.Error())
|
||||
return nruntime.StopDecision{Notice: verifierUnavailableNotice}
|
||||
}
|
||||
if !verdict.Passed {
|
||||
reason := strings.TrimSpace(verdict.Reason)
|
||||
if reason == "" {
|
||||
reason = "未说明原因"
|
||||
}
|
||||
if next := strings.TrimSpace(verdict.NextAction); next != "" {
|
||||
reason += "。下一步:" + next
|
||||
}
|
||||
return st.continueOrNotice(info, reason)
|
||||
}
|
||||
return nruntime.StopDecision{}
|
||||
}
|
||||
|
||||
func (st *directRun) demoteLastText() {
|
||||
for i := len(st.finalParts) - 1; i >= 0; i-- {
|
||||
if st.finalParts[i].Type == core.PartText {
|
||||
st.finalParts[i].Process = true
|
||||
break
|
||||
return
|
||||
}
|
||||
}
|
||||
if info.Continuations < maxDeliverableGuards {
|
||||
return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote}
|
||||
}
|
||||
|
||||
func (st *directRun) addVerifyNote(reason string) {
|
||||
st.finalParts = append(st.finalParts, core.Part{
|
||||
Type: core.PartText,
|
||||
Process: true,
|
||||
Text: "(验证未通过:" + reason + ")",
|
||||
})
|
||||
}
|
||||
|
||||
func (st *directRun) continueOrNotice(info nruntime.StopInfo, reason string) nruntime.StopDecision {
|
||||
if info.Continuations < maxVerifyContinuations {
|
||||
st.demoteLastText()
|
||||
st.addVerifyNote(reason)
|
||||
return nruntime.StopDecision{Continue: true, Reason: reason}
|
||||
}
|
||||
return nruntime.StopDecision{Notice: deliverableFallbackNotice}
|
||||
st.addVerifyNote(reason)
|
||||
return nruntime.StopDecision{Notice: verifyCapNotice}
|
||||
}
|
||||
|
||||
func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
@@ -290,6 +355,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
var p core.PlanSnapshot
|
||||
if uerr := json.Unmarshal([]byte(call.Arguments), &p); uerr == nil {
|
||||
_ = st.rc.UpdatePlan(p)
|
||||
st.plan = p
|
||||
st.checklistTouched = true
|
||||
out = "计划已更新"
|
||||
} else {
|
||||
out = "计划参数错误: " + uerr.Error()
|
||||
@@ -298,6 +365,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result {
|
||||
var t core.TodoList
|
||||
if uerr := json.Unmarshal([]byte(call.Arguments), &t); uerr == nil {
|
||||
_ = st.rc.UpdateTodos(t)
|
||||
st.todos = t
|
||||
st.checklistTouched = true
|
||||
out = "待办已更新"
|
||||
} else {
|
||||
out = "待办参数错误: " + uerr.Error()
|
||||
|
||||
@@ -32,6 +32,9 @@ type fakeRunCtx struct {
|
||||
execTruncated bool // ExecuteTool 返回的截断标记(门面卡触发)
|
||||
toolRuns int // ExecuteTool 被调用的次数
|
||||
execArtifactContent []byte // Artifact(id) 回填内容(ctxasm D46 断言用)
|
||||
verify bool // runtime.verify;缺省关闭,避免改变既有流调用次数
|
||||
missingArtifact bool // 工具登记了工件,但 Artifact 查不到
|
||||
emptyArtifact bool // Artifact 查到了,内容为空
|
||||
}
|
||||
|
||||
func (f *fakeRunCtx) Context() context.Context { return f.ctx }
|
||||
@@ -42,6 +45,9 @@ func (f *fakeRunCtx) Settings() core.EffectiveSettings {
|
||||
if f.cardflow {
|
||||
eff["ui"] = map[string]any{"cardflow": true}
|
||||
}
|
||||
if f.verify {
|
||||
eff["runtime"] = map[string]any{"verify": true}
|
||||
}
|
||||
return eff
|
||||
}
|
||||
func (f *fakeRunCtx) Emit(typ core.StreamEventType, payload any) error {
|
||||
@@ -83,6 +89,12 @@ func (f *fakeRunCtx) CreateCard(card core.ActivityCard) (core.ActivityCard, erro
|
||||
|
||||
func (f *fakeRunCtx) Artifact(id string) (core.Artifact, []byte, bool) {
|
||||
if f.execArtifact != nil && f.execArtifact.ID == id {
|
||||
if f.missingArtifact {
|
||||
return core.Artifact{}, nil, false
|
||||
}
|
||||
if f.emptyArtifact {
|
||||
return *f.execArtifact, nil, true
|
||||
}
|
||||
return *f.execArtifact, f.execArtifactContent, true
|
||||
}
|
||||
return core.Artifact{}, nil, false
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event {
|
||||
evs := []agentllm.Event{}
|
||||
if body != "" {
|
||||
evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body})
|
||||
}
|
||||
end := agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
if len(calls) > 0 {
|
||||
end.Message.ToolCalls = calls
|
||||
}
|
||||
return append(evs, end)
|
||||
}
|
||||
|
||||
func toolCall(name, args string) agentllm.ToolCall {
|
||||
return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args}
|
||||
}
|
||||
|
||||
func passedStream(payloads ...string) agentllm.StreamFunc {
|
||||
n := 0
|
||||
return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
body := `{"passed":true,"reason":"完成","nextAction":""}`
|
||||
if n < len(payloads) {
|
||||
body = payloads[n]
|
||||
n++
|
||||
}
|
||||
out := make(chan agentllm.Event, 2)
|
||||
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
close(out)
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
func verifyOn() core.AgentProfile {
|
||||
return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}
|
||||
}
|
||||
|
||||
func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
miss bool
|
||||
want string
|
||||
}{
|
||||
{name: "missing", miss: true, want: "不存在"},
|
||||
{name: "empty", miss: false, want: "内容为空"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
turns := padTurns(
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("写好了。"),
|
||||
)
|
||||
var calls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
calls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: passedStream(),
|
||||
}
|
||||
rc := &fakeRunCtx{
|
||||
ctx: context.Background(),
|
||||
prof: verifyOn(),
|
||||
verify: true,
|
||||
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"},
|
||||
missingArtifact: tc.miss,
|
||||
emptyArtifact: !tc.miss,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("stream calls = %d, want a continuation", calls)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, tc.want) {
|
||||
t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLintErrorContinues(t *testing.T) {
|
||||
style := filepath.Join("..", "skills", "recipes", "house-style")
|
||||
abs, err := filepath.Abs(style)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PYTHONPATH", abs)
|
||||
docx := badDocx(t)
|
||||
turns := padTurns(
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("写好了。"),
|
||||
)
|
||||
var verifyCalls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: base,
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
verifyCalls++
|
||||
return passedStream()(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{
|
||||
ctx: context.Background(),
|
||||
prof: verifyOn(),
|
||||
verify: true,
|
||||
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"},
|
||||
execArtifactContent: docx,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if verifyCalls != 0 {
|
||||
t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, "版式") {
|
||||
t.Fatalf("lint reason missing: %+v", textParts(rc.assistant))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifierRejectsThenPasses(t *testing.T) {
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn("答案是 42。"),
|
||||
textDeltaTurn("答案仍是 42。"),
|
||||
}
|
||||
var mainCalls int
|
||||
var models []string
|
||||
var toolArgs [][]agentllm.ToolDecl
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
mainCalls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
models = append(models, model)
|
||||
toolArgs = append(toolArgs, tools)
|
||||
body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}`
|
||||
if len(models) > 1 {
|
||||
body = `{"passed":true,"reason":"完成","nextAction":""}`
|
||||
}
|
||||
return passedStream(body)(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if mainCalls != 2 {
|
||||
t.Fatalf("main calls = %d, want 2", mainCalls)
|
||||
}
|
||||
if len(models) != 2 || models[0] != "lite-model" {
|
||||
t.Fatalf("verifier models = %v, want lite-model twice", models)
|
||||
}
|
||||
for i, tools := range toolArgs {
|
||||
if tools != nil {
|
||||
t.Fatalf("verifier call %d tools = %#v, want nil", i, tools)
|
||||
}
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
if len(texts) < 3 || !texts[0].Process || !strings.Contains(texts[1].Text, "验证未通过") || texts[len(texts)-1].Process {
|
||||
t.Fatalf("parts = %+v", texts)
|
||||
}
|
||||
if !strings.Contains(texts[1].Text, "补上结论") {
|
||||
t.Fatalf("nextAction missing: %+v", texts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifierGarbageFailsClosed(t *testing.T) {
|
||||
for _, payload := range []agentllm.Event{
|
||||
{Kind: agentllm.KindTextDelta, Text: "当然通过"},
|
||||
{Kind: agentllm.KindError, ErrorMessage: "upstream"},
|
||||
} {
|
||||
t.Run(string(payload.Kind), func(t *testing.T) {
|
||||
turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")}
|
||||
var mainCalls int
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{
|
||||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
mainCalls++
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
},
|
||||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
if tools != nil {
|
||||
t.Errorf("verifier tools = %#v, want nil", tools)
|
||||
}
|
||||
out := make(chan agentllm.Event, 2)
|
||||
out <- payload
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
||||
close(out)
|
||||
return out
|
||||
},
|
||||
}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if mainCalls != 1 {
|
||||
t.Fatalf("main calls = %d, want 1", mainCalls)
|
||||
}
|
||||
texts := textParts(rc.assistant)
|
||||
var kept, notice bool
|
||||
for _, p := range texts {
|
||||
if p.Text == "答案是 42。" && !p.Process {
|
||||
kept = true
|
||||
}
|
||||
if !p.Process && strings.Contains(p.Text, "验证器不可用") {
|
||||
notice = true
|
||||
}
|
||||
}
|
||||
if !kept || !notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") {
|
||||
t.Fatalf("parts = %+v", texts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVerdictToleratesFence(t *testing.T) {
|
||||
got, err := parseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```")
|
||||
if err != nil || got.Passed || got.Reason != "不够" {
|
||||
t.Fatalf("verdict = %+v %v", got, err)
|
||||
}
|
||||
if _, err := parseVerdict(`{"reason":"no passed"}`); err == nil {
|
||||
t.Fatal("missing passed should fail")
|
||||
}
|
||||
}
|
||||
|
||||
type checklistFake struct {
|
||||
fakeRunCtx
|
||||
list core.TodoList
|
||||
}
|
||||
|
||||
func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true }
|
||||
|
||||
func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event {
|
||||
for len(turns) < 5 {
|
||||
turns = append(turns, textDeltaTurn("仍未修好。"))
|
||||
}
|
||||
return turns
|
||||
}
|
||||
|
||||
func TestTodoReminderFiresOnceAndResets(t *testing.T) {
|
||||
quiet := textDeltaTurn("", toolCall("make_report", "{}"))
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||||
textDeltaTurn("完成了。"),
|
||||
}
|
||||
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
||||
if len(requests) != 4 {
|
||||
t.Fatalf("requests = %d, want 4", len(requests))
|
||||
}
|
||||
for i, req := range requests[:3] {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
t.Fatalf("request %d reminded too early: %s", i, req)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") {
|
||||
t.Fatalf("final request missing reminder: %s", requests[3])
|
||||
}
|
||||
|
||||
update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}`
|
||||
resetTurns := [][]agentllm.Event{
|
||||
textDeltaTurn("", toolCall("update_todos", update)),
|
||||
quiet, quiet, quiet,
|
||||
textDeltaTurn("完成了。"),
|
||||
}
|
||||
requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}})
|
||||
if len(requests) != 5 {
|
||||
t.Fatalf("reset requests = %d, want 5", len(requests))
|
||||
}
|
||||
for i, req := range requests[:4] {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
t.Fatalf("request %d reminded before three quiet turns: %s", i, req)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") {
|
||||
t.Fatalf("reminder did not refresh todos: %s", requests[4])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTodoReminderCapsAtThree(t *testing.T) {
|
||||
var turns [][]agentllm.Event
|
||||
for i := 0; i < 12; i++ {
|
||||
turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}")))
|
||||
}
|
||||
turns = append(turns, textDeltaTurn("完成了。"))
|
||||
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
||||
n := 0
|
||||
for _, req := range requests {
|
||||
if strings.Contains(req, "system-reminder") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 3 {
|
||||
t.Fatalf("reminders = %d, want 3", n)
|
||||
}
|
||||
}
|
||||
|
||||
func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string {
|
||||
t.Helper()
|
||||
var requests []string
|
||||
base := scriptStream(turns, nil)
|
||||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||
requests = append(requests, joinContents(msgs))
|
||||
return base(ctx, model, msgs, tools, opts)
|
||||
}}
|
||||
rc := &checklistFake{
|
||||
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
||||
list: list,
|
||||
}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range rc.assistant {
|
||||
if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") {
|
||||
t.Fatalf("reminder was persisted: %+v", p)
|
||||
}
|
||||
}
|
||||
return requests
|
||||
}
|
||||
|
||||
func joinContents(msgs []agentllm.Message) string {
|
||||
var b strings.Builder
|
||||
for _, m := range msgs {
|
||||
b.WriteString(m.Content)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hasProcessNote(parts []core.Part, fragment string) bool {
|
||||
for _, p := range parts {
|
||||
if p.Type == core.PartText && p.Process && strings.Contains(p.Text, "验证未通过") && strings.Contains(p.Text, fragment) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func badDocx(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.docx")
|
||||
cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")")
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("docx: %v %s", err, out)
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func jsonString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestFalseDeliverableFixtures(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/false_deliverables.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Cases []struct {
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
} `json:"cases"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Cases) < 5 {
|
||||
t.Fatalf("cases = %d, want at least 5", len(doc.Cases))
|
||||
}
|
||||
for _, c := range doc.Cases {
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
block, reason := deliverableGate(c.Body, nil, &fakeRunCtx{ctx: context.Background()})
|
||||
if !block || strings.TrimSpace(reason) == "" {
|
||||
t.Fatalf("gate = %v %q", block, reason)
|
||||
}
|
||||
turns := [][]agentllm.Event{
|
||||
textDeltaTurn(c.Body),
|
||||
textDeltaTurn("这一轮没有生成文件。"),
|
||||
}
|
||||
r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()}
|
||||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||||
if err := r.Execute(rc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !hasProcessNote(rc.assistant, reason) {
|
||||
t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,252 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
agentllm "agent-native/runtime/llm"
|
||||
"agent-system-backend/internal/quality"
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxVerifyContinuations 与内核 Stop 的 3 次上限对齐。
|
||||
maxVerifyContinuations = 3
|
||||
// reminderToolGap 连续这么多个工具轮没有更新计划或待办,就提醒一次。
|
||||
reminderToolGap = 3
|
||||
// maxChecklistReminders 每个 run 最多提醒这么多次。
|
||||
maxChecklistReminders = 3
|
||||
)
|
||||
|
||||
const verifyCapNotice = "本轮回复未通过终轮检查。"
|
||||
|
||||
const verifierUnavailableNotice = "本轮回复未通过独立验证(验证器不可用)"
|
||||
|
||||
const verifyFallbackPrompt = `你是独立验证器,不调用任何工具。根据用户要求、终轮回复、工具证据和工件清单判断这一轮是否已经完成。
|
||||
拿不准时 passed 必须为 false。只输出一个 JSON 对象,不要写其他文字:{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"}。`
|
||||
|
||||
type verifyVerdict struct {
|
||||
Passed bool `json:"passed"`
|
||||
Reason string `json:"reason"`
|
||||
NextAction string `json:"nextAction"`
|
||||
}
|
||||
|
||||
// checkDeliverables 确认本轮登记的工件存在且非空。
|
||||
// md、txt、docx 还要过 quality;其他类型只检查存在与非空。
|
||||
func checkDeliverables(rc RunCtx, refs []*core.ArtifactRef) []string {
|
||||
var reasons []string
|
||||
for _, ref := range refs {
|
||||
if ref == nil || ref.ArtifactID == "" {
|
||||
reasons = append(reasons, "工件引用缺少 ID")
|
||||
continue
|
||||
}
|
||||
art, content, ok := rc.Artifact(ref.ArtifactID)
|
||||
name := ref.Name
|
||||
if name == "" {
|
||||
name = string(ref.ArtifactID)
|
||||
}
|
||||
if !ok {
|
||||
reasons = append(reasons, name+" 不存在")
|
||||
continue
|
||||
}
|
||||
if art.Name != "" {
|
||||
name = art.Name
|
||||
}
|
||||
if len(content) == 0 {
|
||||
reasons = append(reasons, name+" 内容为空")
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".md", ".txt", ".docx":
|
||||
report := quality.ScoreBytes(name, content)
|
||||
if !report.Passed {
|
||||
why := strings.Join(report.Violations, ";")
|
||||
if why == "" {
|
||||
why = "评分未通过"
|
||||
}
|
||||
reasons = append(reasons, name+":"+why)
|
||||
}
|
||||
}
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence, refs []*core.ArtifactRef) (verifyVerdict, error) {
|
||||
route := ResolveLightweightModel(rc.Profile(), rc.Settings())
|
||||
stream := l.VerifyStream
|
||||
if stream == nil {
|
||||
if route.Model == "" || route.Base == "" {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器调用失败")
|
||||
}
|
||||
client := &agentllm.Client{BaseURL: route.Base, APIKey: route.Key}
|
||||
stream = client.Stream
|
||||
}
|
||||
model := route.Model
|
||||
if model == "" {
|
||||
model = "verify"
|
||||
}
|
||||
msgs := []agentllm.Message{
|
||||
{Role: agentllm.RoleSystem, Content: l.verifyPrompt(rc.Profile().Owner)},
|
||||
{Role: agentllm.RoleUser, Content: evidencePrompt(currentUserText(rc), body, evidence, refs)},
|
||||
}
|
||||
var b strings.Builder
|
||||
for ev := range stream(rc.Context(), model, msgs, nil, agentllm.StreamOptions{}) {
|
||||
switch ev.Kind {
|
||||
case agentllm.KindTextDelta:
|
||||
b.WriteString(ev.Text)
|
||||
case agentllm.KindError:
|
||||
return verifyVerdict{}, fmt.Errorf("验证器调用失败")
|
||||
case agentllm.KindMessageEnd:
|
||||
if b.Len() == 0 {
|
||||
b.WriteString(ev.Message.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
verdict, err := parseVerdict(b.String())
|
||||
if err != nil {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
return verdict, nil
|
||||
}
|
||||
|
||||
func (l *LLMRunner) verifyPrompt(owner string) string {
|
||||
if l.Prompts != nil {
|
||||
if body, ok, err := l.Prompts.ModuleBody(owner, "aux.verify.final"); err == nil && ok && strings.TrimSpace(body) != "" {
|
||||
return body
|
||||
}
|
||||
}
|
||||
return verifyFallbackPrompt
|
||||
}
|
||||
|
||||
func evidencePrompt(user, body string, evidence []toolEvidence, refs []*core.ArtifactRef) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("用户要求:\n")
|
||||
if strings.TrimSpace(user) == "" {
|
||||
b.WriteString("(无)\n")
|
||||
} else {
|
||||
b.WriteString(user)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString("\n终轮回复:\n")
|
||||
b.WriteString(body)
|
||||
b.WriteString("\n\n本轮工具:\n")
|
||||
if len(evidence) == 0 {
|
||||
b.WriteString("(无)\n")
|
||||
}
|
||||
for _, ev := range evidence {
|
||||
fmt.Fprintf(&b, "- %s %s %s\n", ev.Name, ev.Status, ev.Summary)
|
||||
}
|
||||
b.WriteString("\n本轮工件:\n")
|
||||
if len(refs) == 0 {
|
||||
b.WriteString("(无)\n")
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if ref == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "- %s %s\n", ref.ArtifactID, ref.Name)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func parseVerdict(raw string) (verifyVerdict, error) {
|
||||
s := stripFence(raw)
|
||||
start := strings.Index(s, "{")
|
||||
end := strings.LastIndex(s, "}")
|
||||
if start < 0 || end < start {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
s = s[start : end+1]
|
||||
var probe map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(s), &probe); err != nil {
|
||||
return verifyVerdict{}, err
|
||||
}
|
||||
if _, ok := probe["passed"]; !ok {
|
||||
return verifyVerdict{}, fmt.Errorf("验证器返回无法解析")
|
||||
}
|
||||
var verdict verifyVerdict
|
||||
if err := json.Unmarshal([]byte(s), &verdict); err != nil {
|
||||
return verifyVerdict{}, err
|
||||
}
|
||||
return verdict, nil
|
||||
}
|
||||
|
||||
func stripFence(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if !strings.HasPrefix(s, "```") {
|
||||
return s
|
||||
}
|
||||
s = strings.TrimPrefix(s, "```")
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[i+1:]
|
||||
}
|
||||
s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "```"))
|
||||
return s
|
||||
}
|
||||
|
||||
func (st *directRun) loadChecklist() {
|
||||
if src, ok := st.rc.(interface{ Todos() (core.TodoList, bool) }); ok {
|
||||
if list, ok := src.Todos(); ok {
|
||||
st.todos = list
|
||||
}
|
||||
}
|
||||
if src, ok := st.rc.(interface {
|
||||
Plan() (core.PlanSnapshot, bool)
|
||||
}); ok {
|
||||
if p, ok := src.Plan(); ok {
|
||||
st.plan = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (st *directRun) reminders(turn int) []string {
|
||||
if st.closedToolTurn {
|
||||
if st.checklistTouched {
|
||||
st.toolTurnsSince = 0
|
||||
} else {
|
||||
st.toolTurnsSince++
|
||||
}
|
||||
st.closedToolTurn = false
|
||||
st.checklistTouched = false
|
||||
}
|
||||
if st.toolTurnsSince < reminderToolGap || st.checklistReminders >= maxChecklistReminders {
|
||||
return nil
|
||||
}
|
||||
text := st.unfinishedReminder()
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
st.toolTurnsSince = 0
|
||||
st.checklistReminders++
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
func (st *directRun) unfinishedReminder() string {
|
||||
var lines []string
|
||||
for _, item := range st.todos.Items {
|
||||
if item.Status == "done" {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(item.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, "- 待办:"+text)
|
||||
}
|
||||
for _, step := range st.plan.Steps {
|
||||
if step.Status == "done" || step.Status == "skipped" {
|
||||
continue
|
||||
}
|
||||
text := strings.TrimSpace(step.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, "- 计划:"+text)
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "计划和待办已连续 3 个工具轮没有更新。未完成项:\n" + strings.Join(lines, "\n") + "\n请更新计划或待办,或在回复里说明为什么先不做。"
|
||||
}
|
||||
@@ -35,6 +35,7 @@ func SystemDefaults() map[string]any {
|
||||
"entry": "direct", // direct = 现有对话循环;squad = 小队分诊入口(灰度,达标后再改默认)
|
||||
"rateLimitPerMin": 0, // 租户限流(次/分;0 = 不限)
|
||||
"redisAddr": "", // 限流共享后端(原生 Redis addr;空 = 进程内)
|
||||
"verify": true, // 终轮确定性检查与独立验证
|
||||
},
|
||||
"storage": map[string]any{
|
||||
"dataDir": "./data", // 启动注入实际值(--data)
|
||||
@@ -77,6 +78,7 @@ func Schema() []core.SettingDesc {
|
||||
{Key: "runtime.entry", Group: "runtime", Label: "对话入口", Type: core.SettingEnum, Default: "direct", Options: []string{"direct", "squad"}, Effect: core.EffectNewRun, Description: "direct 走现有对话循环。squad 由小队先分诊:简单任务单步完成,复杂任务由 leader 编排。灰度期默认 direct"},
|
||||
{Key: "runtime.rateLimitPerMin", Group: "runtime", Label: "租户限流(次/分)", Type: core.SettingNumber, Default: 0, Effect: core.EffectHot, Description: "0 = 不限;多用户建议 300–1200"},
|
||||
{Key: "runtime.redisAddr", Group: "runtime", Label: "Redis 地址(限流共享)", Type: core.SettingString, Default: "", Effect: core.EffectRestart, Description: "原生 Redis(如 127.0.0.1:6379);空 = 进程内限流(单实例)"},
|
||||
{Key: "runtime.verify", Group: "runtime", Label: "终轮结果检查", Type: core.SettingBool, Default: true, Effect: core.EffectNewRun, Description: "终轮确定性检查与独立验证"},
|
||||
{Key: "context.maxHistoryMessages", Group: "context", Label: "历史窗口条数", Type: core.SettingNumber, Default: 40, Effect: core.EffectNewRun},
|
||||
{Key: "context.maxToolResultChars", Group: "context", Label: "工具结果上限(字符)", Type: core.SettingNumber, Default: 20000, Effect: core.EffectNewRun},
|
||||
{Key: "crossSession.read", Group: "crossSession", Label: "跨会话读取", Type: core.SettingBool, Default: false, Effect: core.EffectNewRun, Description: "允许本项目会话读取同项目其他会话的对话记录(list_conversations / read_conversation 工具);会话可单独关闭读取或设为私有"},
|
||||
|
||||
Reference in New Issue
Block a user