From a3ae1b986ddb5ebb8a205861b83d6f796fd99fa2 Mon Sep 17 00:00:00 2001 From: weichen Date: Thu, 1 Oct 2026 13:54:15 +0000 Subject: [PATCH] Check final replies before the direct run stops Why: A direct run could stop on a missing file, a failed lint, or an unverified claim. The final turn now checks registered artifacts and asks a tool-less verifier, and nudges stale plans and todos. Evidence: go test ./... passed. New runtime tests cover missing and empty artifacts, docx lint errors, verifier reject-then-pass, unparseable and failed verifier calls, nil verifier tools, todo reminders, and the false-deliverable fixture. Impact: Direct runs with runtime.verify (default on for new runs) spend one extra lightweight call on every final turn. Setting it off keeps the previous stop behavior. Squad and SubLLMRunner are unchanged. The agent-native pin stays a34689c. Refs: feature/agent-harness-p2 Co-authored-by: Cursor --- internal/prompts/library/aux-verify-final.md | 17 + internal/quality/score.go | 23 + internal/quality/score_test.go | 12 + internal/runmgr/runmgr.go | 18 + internal/runtime/llm.go | 2 + internal/runtime/llm_direct.go | 81 +++- internal/runtime/llm_test.go | 12 + internal/runtime/llm_verify_test.go | 413 ++++++++++++++++++ .../runtime/testdata/false_deliverables.json | 28 ++ internal/runtime/verify.go | 252 +++++++++++ internal/settings/settings.go | 2 + 11 files changed, 854 insertions(+), 6 deletions(-) create mode 100644 internal/prompts/library/aux-verify-final.md create mode 100644 internal/runtime/llm_verify_test.go create mode 100644 internal/runtime/testdata/false_deliverables.json create mode 100644 internal/runtime/verify.go diff --git a/internal/prompts/library/aux-verify-final.md b/internal/prompts/library/aux-verify-final.md new file mode 100644 index 0000000..9c034fd --- /dev/null +++ b/internal/prompts/library/aux-verify-final.md @@ -0,0 +1,17 @@ +--- +id: aux.verify.final +layer: aux +version: 1 +title: 终轮独立验证 +description: 不带工具，按证据清单判断终轮是否完成，只输出 JSON +--- +你是独立验证器，不调用任何工具。根据下面给出的用户要求、终轮回复、本轮工具证据和工件清单，判断用户要求是否已经完成。 + +规则： +- 只根据证据判断。回复里声称生成了文件，但工件清单里没有，passed 必须为 false。 +- 拿不准时 passed 必须为 false。 +- reason 写给复核的人看，说明通过或不通过的依据。 +- nextAction 写给主模型：不通过时给出下一步该做什么；通过时留空字符串。 + +只输出一个 JSON 对象，不要写其他文字，不要用代码围栏： +{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"} diff --git a/internal/quality/score.go b/internal/quality/score.go index 46d48cc..af7ea2a 100644 --- a/internal/quality/score.go +++ b/internal/quality/score.go @@ -99,6 +99,29 @@ func ScoreFile(path string) Report { return report } +// ScoreBytes 把内容落到带扩展名的临时文件，再走 ScoreFile。 +// docx 与 housestyle.lint 因此和按路径评分用同一条路径。 +func ScoreBytes(name string, content []byte) Report { + ext := filepath.Ext(name) + if ext == "" { + ext = ".txt" + } + f, err := os.CreateTemp("", "score-*"+ext) + if err != nil { + return Report{Passed: false, Violations: []string{"无法写入临时交付物：" + err.Error()}} + } + path := f.Name() + defer os.Remove(path) + if _, err := f.Write(content); err != nil { + f.Close() + return Report{Passed: false, Violations: []string{"无法写入临时交付物：" + err.Error()}} + } + if err := f.Close(); err != nil { + return Report{Passed: false, Violations: []string{"无法写入临时交付物：" + err.Error()}} + } + return ScoreFile(path) +} + func scoreLead(text string) Dimension { title := firstHeading(text) if title == "" { diff --git a/internal/quality/score_test.go b/internal/quality/score_test.go index a9d5f7e..516cf89 100644 --- a/internal/quality/score_test.go +++ b/internal/quality/score_test.go @@ -27,3 +27,15 @@ func TestScoreTextPrefersJudgementOverColumnHeading(t *testing.T) { } } } + +func TestScoreBytesUsesExtension(t *testing.T) { + good := "# 上传文档应先判断相关性再入库\n\n## 先校验，再生成\n- 说话人1：上传后第一步是判断是否与项目相关。\n一句话：没有记忆库，后面的生成和追溯都没有依据。\n\n## 下一步\n- 事项：写出第一版需求。负责人：陈威。期限：周五前；说不清的写未指明。\n" + got := ScoreBytes("note.md", []byte(good)) + if !got.Passed { + t.Fatalf("markdown bytes should pass: %+v", got) + } + bad := ScoreBytes("note.txt", []byte("# 讨论过程\n\n今天大家聊了很多。\n")) + if bad.Passed { + t.Fatalf("column heading should fail: %+v", bad) + } +} diff --git a/internal/runmgr/runmgr.go b/internal/runmgr/runmgr.go index d25ff0d..d224211 100644 --- a/internal/runmgr/runmgr.go +++ b/internal/runmgr/runmgr.go @@ -287,6 +287,24 @@ func (c *runCtx) UpdatePlan(p core.PlanSnapshot) error { return err } +// Todos 读取本会话已有待办。没有记录时返回 false，供终轮提醒当作可选接口。 +func (c *runCtx) Todos() (core.TodoList, bool) { + list, err := c.mgr.st.GetTodos(c.run.ConversationID) + if err != nil { + return core.TodoList{}, false + } + return list, true +} + +// Plan 读取本会话已有计划。没有记录时返回 false。 +func (c *runCtx) Plan() (core.PlanSnapshot, bool) { + p, err := c.mgr.st.GetPlan(c.run.ConversationID) + if err != nil { + return core.PlanSnapshot{}, false + } + return p, true +} + func (c *runCtx) UpdateTodos(l core.TodoList) error { l.ConversationID = c.run.ConversationID if err := c.mgr.st.SaveTodos(c.run.ConversationID, l); err != nil { diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go index 919aad0..dde4753 100644 --- a/internal/runtime/llm.go +++ b/internal/runtime/llm.go @@ -24,6 +24,8 @@ type LLMRunner struct { Stream agentllm.StreamFunc // Now 注入时钟（nil = time.Now）。测试用它固定思考与工具耗时。 Now func() time.Time + // VerifyStream 是终轮验证器的注入点（nil = 按轻量模型构造客户端）。 + VerifyStream agentllm.StreamFunc } func (l *LLMRunner) Name() string { return "llm" } diff --git a/internal/runtime/llm_direct.go b/internal/runtime/llm_direct.go index 0f3c8de..03d8b4d 100644 --- a/internal/runtime/llm_direct.go +++ b/internal/runtime/llm_direct.go @@ -40,6 +40,20 @@ type directRun struct { queued []agentllm.ToolCall inflight *toolSlot turnBody string + + evidence []toolEvidence + todos core.TodoList + plan core.PlanSnapshot + closedToolTurn bool + checklistTouched bool + toolTurnsSince int + checklistReminders int +} + +type toolEvidence struct { + Name string + Status string + Summary string } type toolSlot struct { @@ -65,6 +79,7 @@ func (t directTool) Execute(ctx context.Context, args json.RawMessage) (agenttoo func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, streamOpts agentllm.StreamOptions, cardflow bool, eff core.EffectiveSettings, maxTurns, maxTools int) error { st := &directRun{l: l, rc: rc, eff: eff, cardflow: cardflow} + st.loadChecklist() defer func() { if sink, ok := rc.(interface{ AddUsage(core.Usage) }); ok { sink.AddUsage(st.usageTotal) @@ -123,6 +138,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin ContinueOnLength: true, OnEvent: st.onEvent, Stop: st.stop, + Reminders: st.reminders, UnknownTool: func(call agentllm.ToolCall) agenttools.Result { return st.runTool(call.Name, call.Arguments) }, @@ -192,6 +208,9 @@ func (st *directRun) onEvent(ev nruntime.Event) { st.finalParts = append(st.finalParts, core.Part{Type: core.PartThinking, Text: st.turnThink.String(), Process: true, DurationMs: durationMs(st.thinkStart, st.thinkEnd)}) st.thinkingIdx++ } + if hasCalls { + st.closedToolTurn = true + } if st.turnText.Len() > 0 { body, think := splitThinkBlock(st.turnText.String()) if think != "" { @@ -233,26 +252,72 @@ func (st *directRun) onEvent(ev nruntime.Event) { if slot.artifact != nil { st.deliverables = append(st.deliverables, slot.artifact) } + st.evidence = append(st.evidence, toolEvidence{Name: slot.call.Name, Status: slot.status, Summary: slot.summary}) _ = st.rc.Emit(core.EvToolResult, core.ToolResultPayload{CallID: slot.call.ID, Status: slot.status, Summary: slot.summary, Truncated: slot.truncated}) st.inflight = nil } } func (st *directRun) stop(last agentllm.Message, info nruntime.StopInfo) nruntime.StopDecision { - block, _ := deliverableGate(st.turnBody, st.deliverables, st.rc) - if !block { + if block, why := deliverableGate(st.turnBody, st.deliverables, st.rc); block { + st.demoteLastText() + if st.eff.GetBool("runtime.verify", false) { + st.addVerifyNote(why) + } + if info.Continuations < maxDeliverableGuards { + return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote} + } + return nruntime.StopDecision{Notice: deliverableFallbackNotice} + } + if !st.eff.GetBool("runtime.verify", false) { return nruntime.StopDecision{} } + if reasons := checkDeliverables(st.rc, st.deliverables); len(reasons) > 0 { + return st.continueOrNotice(info, strings.Join(reasons, "；")) + } + verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, st.deliverables) + if err != nil { + st.addVerifyNote(err.Error()) + return nruntime.StopDecision{Notice: verifierUnavailableNotice} + } + if !verdict.Passed { + reason := strings.TrimSpace(verdict.Reason) + if reason == "" { + reason = "未说明原因" + } + if next := strings.TrimSpace(verdict.NextAction); next != "" { + reason += "。下一步：" + next + } + return st.continueOrNotice(info, reason) + } + return nruntime.StopDecision{} +} + +func (st *directRun) demoteLastText() { for i := len(st.finalParts) - 1; i >= 0; i-- { if st.finalParts[i].Type == core.PartText { st.finalParts[i].Process = true - break + return } } - if info.Continuations < maxDeliverableGuards { - return nruntime.StopDecision{Continue: true, Replacement: discardedClaimNote, Reason: deliverableGuardNote} +} + +func (st *directRun) addVerifyNote(reason string) { + st.finalParts = append(st.finalParts, core.Part{ + Type: core.PartText, + Process: true, + Text: "（验证未通过：" + reason + "）", + }) +} + +func (st *directRun) continueOrNotice(info nruntime.StopInfo, reason string) nruntime.StopDecision { + if info.Continuations < maxVerifyContinuations { + st.demoteLastText() + st.addVerifyNote(reason) + return nruntime.StopDecision{Continue: true, Reason: reason} } - return nruntime.StopDecision{Notice: deliverableFallbackNotice} + st.addVerifyNote(reason) + return nruntime.StopDecision{Notice: verifyCapNotice} } func (st *directRun) runTool(name, args string) agenttools.Result { @@ -290,6 +355,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result { var p core.PlanSnapshot if uerr := json.Unmarshal([]byte(call.Arguments), &p); uerr == nil { _ = st.rc.UpdatePlan(p) + st.plan = p + st.checklistTouched = true out = "计划已更新" } else { out = "计划参数错误: " + uerr.Error() @@ -298,6 +365,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result { var t core.TodoList if uerr := json.Unmarshal([]byte(call.Arguments), &t); uerr == nil { _ = st.rc.UpdateTodos(t) + st.todos = t + st.checklistTouched = true out = "待办已更新" } else { out = "待办参数错误: " + uerr.Error() diff --git a/internal/runtime/llm_test.go b/internal/runtime/llm_test.go index 06e70ca..3bf1306 100644 --- a/internal/runtime/llm_test.go +++ b/internal/runtime/llm_test.go @@ -32,6 +32,9 @@ type fakeRunCtx struct { execTruncated bool // ExecuteTool 返回的截断标记（门面卡触发） toolRuns int // ExecuteTool 被调用的次数 execArtifactContent []byte // Artifact(id) 回填内容（ctxasm D46 断言用） + verify bool // runtime.verify；缺省关闭，避免改变既有流调用次数 + missingArtifact bool // 工具登记了工件，但 Artifact 查不到 + emptyArtifact bool // Artifact 查到了，内容为空 } func (f *fakeRunCtx) Context() context.Context { return f.ctx } @@ -42,6 +45,9 @@ func (f *fakeRunCtx) Settings() core.EffectiveSettings { if f.cardflow { eff["ui"] = map[string]any{"cardflow": true} } + if f.verify { + eff["runtime"] = map[string]any{"verify": true} + } return eff } func (f *fakeRunCtx) Emit(typ core.StreamEventType, payload any) error { @@ -83,6 +89,12 @@ func (f *fakeRunCtx) CreateCard(card core.ActivityCard) (core.ActivityCard, erro func (f *fakeRunCtx) Artifact(id string) (core.Artifact, []byte, bool) { if f.execArtifact != nil && f.execArtifact.ID == id { + if f.missingArtifact { + return core.Artifact{}, nil, false + } + if f.emptyArtifact { + return *f.execArtifact, nil, true + } return *f.execArtifact, f.execArtifactContent, true } return core.Artifact{}, nil, false diff --git a/internal/runtime/llm_verify_test.go b/internal/runtime/llm_verify_test.go new file mode 100644 index 0000000..b95d5e0 --- /dev/null +++ b/internal/runtime/llm_verify_test.go @@ -0,0 +1,413 @@ +package runtime + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + agentllm "agent-native/runtime/llm" + core "agent-system-core" +) + +func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event { + evs := []agentllm.Event{} + if body != "" { + evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}) + } + end := agentllm.Event{Kind: agentllm.KindMessageEnd} + if len(calls) > 0 { + end.Message.ToolCalls = calls + } + return append(evs, end) +} + +func toolCall(name, args string) agentllm.ToolCall { + return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args} +} + +func passedStream(payloads ...string) agentllm.StreamFunc { + n := 0 + return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + body := `{"passed":true,"reason":"完成","nextAction":""}` + if n < len(payloads) { + body = payloads[n] + n++ + } + out := make(chan agentllm.Event, 2) + out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body} + out <- agentllm.Event{Kind: agentllm.KindMessageEnd} + close(out) + return out + } +} + +func verifyOn() core.AgentProfile { + return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}} +} + +func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) { + for _, tc := range []struct { + name string + miss bool + want string + }{ + {name: "missing", miss: true, want: "不存在"}, + {name: "empty", miss: false, want: "内容为空"}, + } { + t.Run(tc.name, func(t *testing.T) { + turns := padTurns( + textDeltaTurn("", toolCall("make_report", "{}")), + textDeltaTurn("写好了。"), + ) + var calls int + base := scriptStream(turns, nil) + r := &LLMRunner{ + Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + calls++ + return base(ctx, model, msgs, tools, opts) + }, + VerifyStream: passedStream(), + } + rc := &fakeRunCtx{ + ctx: context.Background(), + prof: verifyOn(), + verify: true, + execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"}, + missingArtifact: tc.miss, + emptyArtifact: !tc.miss, + } + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if calls < 2 { + t.Fatalf("stream calls = %d, want a continuation", calls) + } + if !hasProcessNote(rc.assistant, tc.want) { + t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant)) + } + }) + } +} + +func TestLintErrorContinues(t *testing.T) { + style := filepath.Join("..", "skills", "recipes", "house-style") + abs, err := filepath.Abs(style) + if err != nil { + t.Fatal(err) + } + t.Setenv("PYTHONPATH", abs) + docx := badDocx(t) + turns := padTurns( + textDeltaTurn("", toolCall("make_report", "{}")), + textDeltaTurn("写好了。"), + ) + var verifyCalls int + base := scriptStream(turns, nil) + r := &LLMRunner{ + Stream: base, + VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + verifyCalls++ + return passedStream()(ctx, model, msgs, tools, opts) + }, + } + rc := &fakeRunCtx{ + ctx: context.Background(), + prof: verifyOn(), + verify: true, + execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"}, + execArtifactContent: docx, + } + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if verifyCalls != 0 { + t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls) + } + if !hasProcessNote(rc.assistant, "版式") { + t.Fatalf("lint reason missing: %+v", textParts(rc.assistant)) + } +} + +func TestVerifierRejectsThenPasses(t *testing.T) { + turns := [][]agentllm.Event{ + textDeltaTurn("答案是 42。"), + textDeltaTurn("答案仍是 42。"), + } + var mainCalls int + var models []string + var toolArgs [][]agentllm.ToolDecl + base := scriptStream(turns, nil) + r := &LLMRunner{ + Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + mainCalls++ + return base(ctx, model, msgs, tools, opts) + }, + VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + models = append(models, model) + toolArgs = append(toolArgs, tools) + body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}` + if len(models) > 1 { + body = `{"passed":true,"reason":"完成","nextAction":""}` + } + return passedStream(body)(ctx, model, msgs, tools, opts) + }, + } + rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if mainCalls != 2 { + t.Fatalf("main calls = %d, want 2", mainCalls) + } + if len(models) != 2 || models[0] != "lite-model" { + t.Fatalf("verifier models = %v, want lite-model twice", models) + } + for i, tools := range toolArgs { + if tools != nil { + t.Fatalf("verifier call %d tools = %#v, want nil", i, tools) + } + } + texts := textParts(rc.assistant) + if len(texts) < 3 || !texts[0].Process || !strings.Contains(texts[1].Text, "验证未通过") || texts[len(texts)-1].Process { + t.Fatalf("parts = %+v", texts) + } + if !strings.Contains(texts[1].Text, "补上结论") { + t.Fatalf("nextAction missing: %+v", texts) + } +} + +func TestVerifierGarbageFailsClosed(t *testing.T) { + for _, payload := range []agentllm.Event{ + {Kind: agentllm.KindTextDelta, Text: "当然通过"}, + {Kind: agentllm.KindError, ErrorMessage: "upstream"}, + } { + t.Run(string(payload.Kind), func(t *testing.T) { + turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")} + var mainCalls int + base := scriptStream(turns, nil) + r := &LLMRunner{ + Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + mainCalls++ + return base(ctx, model, msgs, tools, opts) + }, + VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + if tools != nil { + t.Errorf("verifier tools = %#v, want nil", tools) + } + out := make(chan agentllm.Event, 2) + out <- payload + out <- agentllm.Event{Kind: agentllm.KindMessageEnd} + close(out) + return out + }, + } + rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} + if err := r.Execute(rc); err != nil { + t.Fatalf("Execute: %v", err) + } + if mainCalls != 1 { + t.Fatalf("main calls = %d, want 1", mainCalls) + } + texts := textParts(rc.assistant) + var kept, notice bool + for _, p := range texts { + if p.Text == "答案是 42。" && !p.Process { + kept = true + } + if !p.Process && strings.Contains(p.Text, "验证器不可用") { + notice = true + } + } + if !kept || !notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") { + t.Fatalf("parts = %+v", texts) + } + }) + } +} + +func TestParseVerdictToleratesFence(t *testing.T) { + got, err := parseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```") + if err != nil || got.Passed || got.Reason != "不够" { + t.Fatalf("verdict = %+v %v", got, err) + } + if _, err := parseVerdict(`{"reason":"no passed"}`); err == nil { + t.Fatal("missing passed should fail") + } +} + +type checklistFake struct { + fakeRunCtx + list core.TodoList +} + +func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true } + +func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event { + for len(turns) < 5 { + turns = append(turns, textDeltaTurn("仍未修好。")) + } + return turns +} + +func TestTodoReminderFiresOnceAndResets(t *testing.T) { + quiet := textDeltaTurn("", toolCall("make_report", "{}")) + turns := [][]agentllm.Event{ + textDeltaTurn("", toolCall("make_report", "{}")), + textDeltaTurn("", toolCall("make_report", "{}")), + textDeltaTurn("", toolCall("make_report", "{}")), + textDeltaTurn("完成了。"), + } + requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}}) + if len(requests) != 4 { + t.Fatalf("requests = %d, want 4", len(requests)) + } + for i, req := range requests[:3] { + if strings.Contains(req, "system-reminder") { + t.Fatalf("request %d reminded too early: %s", i, req) + } + } + if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") { + t.Fatalf("final request missing reminder: %s", requests[3]) + } + + update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}` + resetTurns := [][]agentllm.Event{ + textDeltaTurn("", toolCall("update_todos", update)), + quiet, quiet, quiet, + textDeltaTurn("完成了。"), + } + requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}}) + if len(requests) != 5 { + t.Fatalf("reset requests = %d, want 5", len(requests)) + } + for i, req := range requests[:4] { + if strings.Contains(req, "system-reminder") { + t.Fatalf("request %d reminded before three quiet turns: %s", i, req) + } + } + if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") { + t.Fatalf("reminder did not refresh todos: %s", requests[4]) + } +} + +func TestTodoReminderCapsAtThree(t *testing.T) { + var turns [][]agentllm.Event + for i := 0; i < 12; i++ { + turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}"))) + } + turns = append(turns, textDeltaTurn("完成了。")) + requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}}) + n := 0 + for _, req := range requests { + if strings.Contains(req, "system-reminder") { + n++ + } + } + if n != 3 { + t.Fatalf("reminders = %d, want 3", n) + } +} + +func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string { + t.Helper() + var requests []string + base := scriptStream(turns, nil) + r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { + requests = append(requests, joinContents(msgs)) + return base(ctx, model, msgs, tools, opts) + }} + rc := &checklistFake{ + fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, + list: list, + } + if err := r.Execute(rc); err != nil { + t.Fatal(err) + } + for _, p := range rc.assistant { + if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") { + t.Fatalf("reminder was persisted: %+v", p) + } + } + return requests +} + +func joinContents(msgs []agentllm.Message) string { + var b strings.Builder + for _, m := range msgs { + b.WriteString(m.Content) + b.WriteString("\n") + } + return b.String() +} + +func hasProcessNote(parts []core.Part, fragment string) bool { + for _, p := range parts { + if p.Type == core.PartText && p.Process && strings.Contains(p.Text, "验证未通过") && strings.Contains(p.Text, fragment) { + return true + } + } + return false +} + +func badDocx(t *testing.T) []byte { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "bad.docx") + cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("docx: %v %s", err, out) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return data +} + +func jsonString(s string) string { + b, _ := json.Marshal(s) + return string(b) +} + +func TestFalseDeliverableFixtures(t *testing.T) { + raw, err := os.ReadFile("testdata/false_deliverables.json") + if err != nil { + t.Fatal(err) + } + var doc struct { + Cases []struct { + Name string `json:"name"` + Body string `json:"body"` + } `json:"cases"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Cases) < 5 { + t.Fatalf("cases = %d, want at least 5", len(doc.Cases)) + } + for _, c := range doc.Cases { + t.Run(c.Name, func(t *testing.T) { + block, reason := deliverableGate(c.Body, nil, &fakeRunCtx{ctx: context.Background()}) + if !block || strings.TrimSpace(reason) == "" { + t.Fatalf("gate = %v %q", block, reason) + } + turns := [][]agentllm.Event{ + textDeltaTurn(c.Body), + textDeltaTurn("这一轮没有生成文件。"), + } + r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()} + rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} + if err := r.Execute(rc); err != nil { + t.Fatal(err) + } + if !hasProcessNote(rc.assistant, reason) { + t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant)) + } + }) + } +} diff --git a/internal/runtime/testdata/false_deliverables.json b/internal/runtime/testdata/false_deliverables.json new file mode 100644 index 0000000..0891bb6 --- /dev/null +++ b/internal/runtime/testdata/false_deliverables.json @@ -0,0 +1,28 @@ +{ + "cases": [ + { + "name": "forged-marker", + "body": "好的。\n[调用工具 run_python({\"script\":\"print(1)\"})]\n[工具结果 ok 完成]" + }, + { + "name": "unknown-art", + "body": "工件 ID：art_7b24c0b7e2f1" + }, + { + "name": "soft-claim", + "body": "文件已正常生成并登记。" + }, + { + "name": "extension-claim", + "body": "文档已生成：甲.docx" + }, + { + "name": "turn-26-forged", + "body": "好的，我用更精简的方式重新整理会议纪要。\n[调用工具 run_python({\"script\": \"\\nfrom docx import Document\\nfrom docx.shared import Pt, RGBColor, Cm\\nfrom docx.enum.text import WD_ALIGN_PARAGRAPH\\nfrom docx.oxml.ns import qn\\n\\ndoc = Document()\\n\\n# 设置中文字体\\ndef set_cn_font(run, font='微软雅黑', size=11, bold=False):\\n run.font.name = font\\n run.font.size = Pt(size)\\n run.font.bold = bold\\n r = run._element\\n rPr = r.get_or_add_rPr()\\n rFonts = rPr.get_or_add_rFonts()\\n rFonts.set(qn('w:eastAsia'), font)\\n\\n# 标题\\ntitle = doc.add_heading('研发智能体文档处理与自举方案评审', 0)\\ntitle.alignment = WD_ALIGN_PARAGRAPH.CENTER\\nfor run in title.runs:\\n set_cn_font(run, size=18, bold=True)\\n\\n# 副标题 - 时间地点\\nsub = doc.add_paragraph('会议纪要')\\nsub.alignment = WD_ALIGN_PARAGRAPH.CENTER\\nfor run in sub.runs:\\n set_cn_font(run, size=14, bold=True)\\n\\ninfo = doc.add_paragraph()\\ninfo.alignment = WD_ALIGN_PARAGRAPH.CENTER\\ninfo.add_run('时间：2026年9月30日 | 形式：线下 + 飞书线上 | 参会：产品负责人、开发团队')\\n\\ndoc.add_paragraph()\\n\\n# ===== 核心结论 =====\\nh = doc.add_heading('核心结论', level=1)\\np = doc.add_paragraph()\\np.add_run('会议达成3项关键决议，形成6项待办事项，遗留4个未决问题待后续讨论。').bold = False\\n\\n# ===== 决议表格 =====\\ndoc.add_heading('会议决议', level=1)\\n\\nres_table = doc.add_table(rows=5, cols=3)\\nres_table.style = 'Table Grid'\\nheaders = ['序号', '决议事项', '具体内容']\\nfor i, h in enumerate(headers):\\n res_table.rows[0].cells[i].text = h\\n for run in res_table.rows[0].cells[i].paragraphs[0].runs:\\n run.bold = True\\n\\nres_data = [\\n ('1', '产品设计重构', '建立\\\\\\\"相关性判断 → 拆解标签 → 入库\\\\\\\"流程，实现实时渐进式反馈'),\\n ('2', '启动自举工作', '暂停代码开发，优先完成智能体产品用户需求文档（1.0版）'),\\n ('3', '代码管理规范', '建立每周代码review机制，由陈威、秦明负责'),\\n ('4', '智能助手接入', '将飞书智能助手加入飞书测试群，用于项目管理'),\\n]\\nfor i, row in enumerate(res_data, 1):\\n for j, val in enumerate(row):\\n res_table.rows[i].cells[j].text = val\\n\\n# ===== 核心问题 =====\\ndoc.add_paragraph()\\ndoc.add_heading('核心问题', level=1)\\n\\nissues_table = doc.add_table(rows=7, cols=2)\\nissues_table.style = 'Table Grid'\\nissues_table.rows[0].cells[0].text = '问题'\\nissues_table.rows[0].cells[1].text = '现状与期望'\\nfor run in issues_table.rows[0].cells[0].paragraphs[0].runs:\\n run.bold = True\\nfor run in issues_table.rows[0].cells[1].paragraphs[0].runs:\\n run.bold = True\\n\\nissues_data = [\\n ('意图识别缺失', '现状：上传文档后需用户补说一遍\\\\n期望：先判断相关性，再拆解打标签入库'),\\n ('未实现渐进式清晰', '现状：直接生成完整文档\\\\n期望：像画马一样逐步完善，区分\\\\\\\"记忆系统\\\\\\\"与\\\\\\\"输出内容\\\\\\\"'),\\n ('反馈延迟', '现状：小队运行需20分钟\\\\n期望：读一段总结一段，实时反馈'),\\n ('架构臃肿', '现状：三层意图识别，包了一层又一层\\\\n期望：简化分层，与体系文档对应'),\\n ('产品未自举', '现状：产品经理无需求文档就写代码\\\\n期望：先用自家产品管自己，形成最佳范式'),\\n ('代码管理混乱', '现状：AI生成几十万行代码，找不到代码\\\\n期望：建立review机制，分门别类'),\\n]\\nfor i, (k, v) in enumerate(issues_data, 1):\\n issues_table.rows[i].cells[0].text = k\\n issues_table.rows[i].cells[1].text = v\\n\\n# ===== 待办事项 =====\\ndoc.add_paragraph()\\ndoc.add_heading('待办事项', level=1)\\n\\ntodo_table = doc.add_table(rows=7, cols=4)\\ntodo_table.style = 'Table Grid'\\nheaders = ['序号', '待办', '负责人', '状态']\\nfor i, h in enumerate(headers):\\n todo_table.rows[0].cells[i].text = h\\n for run in todo_table.rows[0].cells[i].paragraphs[0].runs:\\n run.bold = True\\n\\ntodos = [\\n ('1', '停止当前开发，优先完成需求文档', '产品/陈磊', '待执行'),\\n ('2', '完成需求文档后确认设计方向', '陈磊', '待执行'),\\n ('3', '建立每周代码review机制', '陈威、秦明', '待执行'),\\n ('4', '飞书智能助手加入测试群', '未指明', '待执行'),\\n ('5', '重构文档处理流程', '开发团队', '待执行'),\\n ('6', '实现实时渐进式反馈', '开发团队', '待执行'),\\n]\\nfor i, row in enumerate(todos, 1):\\n for j, val in enumerate(row):\\n todo_table.rows[i].cells[j].text = val\\n\\n# ===== 未决问题 =====\\ndoc.add_paragraph()\\ndoc.add_heading('未决问题', level=1)\\n\\nopen_table = doc.add_table(rows=5, cols=2)\\nopen_table.style = 'Table Grid'\\nopen_table.rows[0].cells[0].text = '问题'\\nopen_table.rows[0].cells[1].text = '说明'\\nfor run in open_table.rows[0].cells[0].paragraphs[0].runs:\\n run.bold = True\\nfor run in open_table.rows[0].cells[1].paragraphs[0].runs:\\n run.bold = True\\n\\nopen_issues = [\\n ('文档切片技术方案', '领导提出切片入库，技术人员认为有实现问题，未定论'),\\n ('阶段隔离与资料复用', '系统按阶段隔离，但实际需要策划资料可多阶段参考'),\\n ('工作流与体系文档对应', '工作流与ISO 9001体系文档的追溯关系尚未建立'),\\n ('Token消耗优化', '大模型调用量大、耗时长的性能问题暂未解决'),\\n]\\nfor i, (k, v) in enumerate(open_issues, 1):\\n open_table.rows[i].cells[0].text = k\\n open_table.rows[i].cells[1].text = v\\n\\n# ===== 下一步 =====\\ndoc.add_paragraph()\\ndoc.add_heading('下一步行动', level=1)\\ndoc.add_paragraph('• 智能助手先加入测试群')\\ndoc.add_paragraph('• 代码同步优化')\\ndoc.add_paragraph('• 下午具体商量开发范式')\\n\\n# 保存\\ndoc.save('研发智能体方案评审_会议纪要_精简版.docx')\\nprint('完成')\\n\"})]\n[工具结果 ok exit=0\n完成\n]\n[调用工具 register_artifact({\"path\": \"研发智能体方案评审_会议纪要_精简版.docx\"})]\n[工具结果 ok 已登记交付物 研发智能体方案评审_会议纪要_精简版.docx（art_1a2b3c4d5e6f，38752 字节）——对话内会出现可预览的文件卡片]\n文档已生成：\n\n📄 **研发智能体方案评审_会议纪要_精简版.docx**\n- 工件 ID：`art_1a2b3c4d5e6f`\n- 大小：约 39 KB\n\n---\n\n**文档结构（精简版）：**\n\n| 章节 | 内容 |\n|------|------|\n| 核心结论 | 3项关键决议、6项待办、4个未决问题 |\n| 会议决议 | 4项决议表格（产品重构、自举、代码规范、助手接入） |\n| 核心问题 | 6个问题及现状/期望对比表 |\n| 待办事项 | 6项待办含负责人与状态 |\n| 未决问题 | 4个遗留问题 |\n| 下一步行动 | 3项具体行动 |\n\n---\n\n您可以在本会话的「工件」区域找到并下载该文件。\n[产出工件 研发智能体方案评审_会议纪要_精简版.docx（二进制内容，正文见工件查看器）]" + }, + { + "name": "turn-26-followup", + "body": "我检查了上一轮的调用结果：\n\n- `run_python` 返回 `exit=0`，输出 \"完成\"\n- `register_artifact` 返回 `\"status\": \"registered\"`\n\n文件已正常生成并登记。工件卡片应该出现在对话下方，如果没看到可能是加载问题，您可以尝试刷新页面或从工件面板查看。" + } + ] +} diff --git a/internal/runtime/verify.go b/internal/runtime/verify.go new file mode 100644 index 0000000..4e7bb00 --- /dev/null +++ b/internal/runtime/verify.go @@ -0,0 +1,252 @@ +package runtime + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + agentllm "agent-native/runtime/llm" + "agent-system-backend/internal/quality" + core "agent-system-core" +) + +const ( + // maxVerifyContinuations 与内核 Stop 的 3 次上限对齐。 + maxVerifyContinuations = 3 + // reminderToolGap 连续这么多个工具轮没有更新计划或待办，就提醒一次。 + reminderToolGap = 3 + // maxChecklistReminders 每个 run 最多提醒这么多次。 + maxChecklistReminders = 3 +) + +const verifyCapNotice = "本轮回复未通过终轮检查。" + +const verifierUnavailableNotice = "本轮回复未通过独立验证（验证器不可用）" + +const verifyFallbackPrompt = `你是独立验证器，不调用任何工具。根据用户要求、终轮回复、工具证据和工件清单判断这一轮是否已经完成。 +拿不准时 passed 必须为 false。只输出一个 JSON 对象，不要写其他文字：{"passed":false,"reason":"为什么没通过","nextAction":"主模型下一步该做什么"}。` + +type verifyVerdict struct { + Passed bool `json:"passed"` + Reason string `json:"reason"` + NextAction string `json:"nextAction"` +} + +// checkDeliverables 确认本轮登记的工件存在且非空。 +// md、txt、docx 还要过 quality；其他类型只检查存在与非空。 +func checkDeliverables(rc RunCtx, refs []*core.ArtifactRef) []string { + var reasons []string + for _, ref := range refs { + if ref == nil || ref.ArtifactID == "" { + reasons = append(reasons, "工件引用缺少 ID") + continue + } + art, content, ok := rc.Artifact(ref.ArtifactID) + name := ref.Name + if name == "" { + name = string(ref.ArtifactID) + } + if !ok { + reasons = append(reasons, name+" 不存在") + continue + } + if art.Name != "" { + name = art.Name + } + if len(content) == 0 { + reasons = append(reasons, name+" 内容为空") + continue + } + switch strings.ToLower(filepath.Ext(name)) { + case ".md", ".txt", ".docx": + report := quality.ScoreBytes(name, content) + if !report.Passed { + why := strings.Join(report.Violations, "；") + if why == "" { + why = "评分未通过" + } + reasons = append(reasons, name+"："+why) + } + } + } + return reasons +} + +func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence, refs []*core.ArtifactRef) (verifyVerdict, error) { + route := ResolveLightweightModel(rc.Profile(), rc.Settings()) + stream := l.VerifyStream + if stream == nil { + if route.Model == "" || route.Base == "" { + return verifyVerdict{}, fmt.Errorf("验证器调用失败") + } + client := &agentllm.Client{BaseURL: route.Base, APIKey: route.Key} + stream = client.Stream + } + model := route.Model + if model == "" { + model = "verify" + } + msgs := []agentllm.Message{ + {Role: agentllm.RoleSystem, Content: l.verifyPrompt(rc.Profile().Owner)}, + {Role: agentllm.RoleUser, Content: evidencePrompt(currentUserText(rc), body, evidence, refs)}, + } + var b strings.Builder + for ev := range stream(rc.Context(), model, msgs, nil, agentllm.StreamOptions{}) { + switch ev.Kind { + case agentllm.KindTextDelta: + b.WriteString(ev.Text) + case agentllm.KindError: + return verifyVerdict{}, fmt.Errorf("验证器调用失败") + case agentllm.KindMessageEnd: + if b.Len() == 0 { + b.WriteString(ev.Message.Content) + } + } + } + verdict, err := parseVerdict(b.String()) + if err != nil { + return verifyVerdict{}, fmt.Errorf("验证器返回无法解析") + } + return verdict, nil +} + +func (l *LLMRunner) verifyPrompt(owner string) string { + if l.Prompts != nil { + if body, ok, err := l.Prompts.ModuleBody(owner, "aux.verify.final"); err == nil && ok && strings.TrimSpace(body) != "" { + return body + } + } + return verifyFallbackPrompt +} + +func evidencePrompt(user, body string, evidence []toolEvidence, refs []*core.ArtifactRef) string { + var b strings.Builder + b.WriteString("用户要求：\n") + if strings.TrimSpace(user) == "" { + b.WriteString("（无）\n") + } else { + b.WriteString(user) + b.WriteString("\n") + } + b.WriteString("\n终轮回复：\n") + b.WriteString(body) + b.WriteString("\n\n本轮工具：\n") + if len(evidence) == 0 { + b.WriteString("（无）\n") + } + for _, ev := range evidence { + fmt.Fprintf(&b, "- %s %s %s\n", ev.Name, ev.Status, ev.Summary) + } + b.WriteString("\n本轮工件：\n") + if len(refs) == 0 { + b.WriteString("（无）\n") + } + for _, ref := range refs { + if ref == nil { + continue + } + fmt.Fprintf(&b, "- %s %s\n", ref.ArtifactID, ref.Name) + } + return b.String() +} + +func parseVerdict(raw string) (verifyVerdict, error) { + s := stripFence(raw) + start := strings.Index(s, "{") + end := strings.LastIndex(s, "}") + if start < 0 || end < start { + return verifyVerdict{}, fmt.Errorf("验证器返回无法解析") + } + s = s[start : end+1] + var probe map[string]json.RawMessage + if err := json.Unmarshal([]byte(s), &probe); err != nil { + return verifyVerdict{}, err + } + if _, ok := probe["passed"]; !ok { + return verifyVerdict{}, fmt.Errorf("验证器返回无法解析") + } + var verdict verifyVerdict + if err := json.Unmarshal([]byte(s), &verdict); err != nil { + return verifyVerdict{}, err + } + return verdict, nil +} + +func stripFence(s string) string { + s = strings.TrimSpace(s) + if !strings.HasPrefix(s, "```") { + return s + } + s = strings.TrimPrefix(s, "```") + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[i+1:] + } + s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "```")) + return s +} + +func (st *directRun) loadChecklist() { + if src, ok := st.rc.(interface{ Todos() (core.TodoList, bool) }); ok { + if list, ok := src.Todos(); ok { + st.todos = list + } + } + if src, ok := st.rc.(interface { + Plan() (core.PlanSnapshot, bool) + }); ok { + if p, ok := src.Plan(); ok { + st.plan = p + } + } +} + +func (st *directRun) reminders(turn int) []string { + if st.closedToolTurn { + if st.checklistTouched { + st.toolTurnsSince = 0 + } else { + st.toolTurnsSince++ + } + st.closedToolTurn = false + st.checklistTouched = false + } + if st.toolTurnsSince < reminderToolGap || st.checklistReminders >= maxChecklistReminders { + return nil + } + text := st.unfinishedReminder() + if text == "" { + return nil + } + st.toolTurnsSince = 0 + st.checklistReminders++ + return []string{text} +} + +func (st *directRun) unfinishedReminder() string { + var lines []string + for _, item := range st.todos.Items { + if item.Status == "done" { + continue + } + text := strings.TrimSpace(item.Text) + if text == "" { + continue + } + lines = append(lines, "- 待办："+text) + } + for _, step := range st.plan.Steps { + if step.Status == "done" || step.Status == "skipped" { + continue + } + text := strings.TrimSpace(step.Text) + if text == "" { + continue + } + lines = append(lines, "- 计划："+text) + } + if len(lines) == 0 { + return "" + } + return "计划和待办已连续 3 个工具轮没有更新。未完成项：\n" + strings.Join(lines, "\n") + "\n请更新计划或待办，或在回复里说明为什么先不做。" +} diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 8c55fab..f189221 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -35,6 +35,7 @@ func SystemDefaults() map[string]any { "entry": "direct", // direct = 现有对话循环；squad = 小队分诊入口（灰度，达标后再改默认） "rateLimitPerMin": 0, // 租户限流（次/分；0 = 不限） "redisAddr": "", // 限流共享后端（原生 Redis addr；空 = 进程内） + "verify": true, // 终轮确定性检查与独立验证 }, "storage": map[string]any{ "dataDir": "./data", // 启动注入实际值（--data） @@ -77,6 +78,7 @@ func Schema() []core.SettingDesc { {Key: "runtime.entry", Group: "runtime", Label: "对话入口", Type: core.SettingEnum, Default: "direct", Options: []string{"direct", "squad"}, Effect: core.EffectNewRun, Description: "direct 走现有对话循环。squad 由小队先分诊：简单任务单步完成，复杂任务由 leader 编排。灰度期默认 direct"}, {Key: "runtime.rateLimitPerMin", Group: "runtime", Label: "租户限流（次/分）", Type: core.SettingNumber, Default: 0, Effect: core.EffectHot, Description: "0 = 不限；多用户建议 300–1200"}, {Key: "runtime.redisAddr", Group: "runtime", Label: "Redis 地址（限流共享）", Type: core.SettingString, Default: "", Effect: core.EffectRestart, Description: "原生 Redis（如 127.0.0.1:6379）；空 = 进程内限流（单实例）"}, + {Key: "runtime.verify", Group: "runtime", Label: "终轮结果检查", Type: core.SettingBool, Default: true, Effect: core.EffectNewRun, Description: "终轮确定性检查与独立验证"}, {Key: "context.maxHistoryMessages", Group: "context", Label: "历史窗口条数", Type: core.SettingNumber, Default: 40, Effect: core.EffectNewRun}, {Key: "context.maxToolResultChars", Group: "context", Label: "工具结果上限（字符）", Type: core.SettingNumber, Default: 20000, Effect: core.EffectNewRun}, {Key: "crossSession.read", Group: "crossSession", Label: "跨会话读取", Type: core.SettingBool, Default: false, Effect: core.EffectNewRun, Description: "允许本项目会话读取同项目其他会话的对话记录（list_conversations / read_conversation 工具）；会话可单独关闭读取或设为私有"},