weichenandCursor
6026e248a6
Why: The critic asked the provider for a model named verify, so every check failed before it could read the step output. Evidence: go test ./internal/runtime/ -run TestJudgeUsesConfiguredModel passed. The request model is MiniMax-test when that is the configured chat model. Impact: A missing lightweight route still falls back to the chat model. Unparseable output still counts as a fail. Refs: feature/agent-harness-p4 Co-authored-by: Cursor <cursoragent@cursor.com>
348 lines
13 KiB
Go
348 lines
13 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"agent-system-backend/internal/workflow"
|
|
core "agent-system-core"
|
|
)
|
|
|
|
type wfFake struct {
|
|
ctx context.Context
|
|
launch workflow.Launch
|
|
hitl []core.HITLResponse
|
|
hitlQ []string
|
|
plans []core.PlanSnapshot
|
|
tools []core.ToolCallRequest
|
|
prompts []string
|
|
writes []string
|
|
cards []core.ActivityCard
|
|
assistant []core.Part
|
|
arts map[string]core.Artifact
|
|
byID map[string]core.WorkflowDef
|
|
spawnText string
|
|
spawnIDs []string
|
|
toolNames []string
|
|
settings core.EffectiveSettings
|
|
sandbox string
|
|
scripts map[string]string
|
|
}
|
|
|
|
func (f *wfFake) Context() context.Context {
|
|
if f.ctx == nil {
|
|
return context.Background()
|
|
}
|
|
return f.ctx
|
|
}
|
|
func (f *wfFake) Run() core.Run { return core.Run{ID: "run_wf"} }
|
|
func (f *wfFake) Profile() core.AgentProfile { return core.AgentProfile{} }
|
|
func (f *wfFake) Settings() core.EffectiveSettings {
|
|
if f.settings == nil {
|
|
return core.EffectiveSettings{}
|
|
}
|
|
return f.settings
|
|
}
|
|
func (f *wfFake) ConversationSandbox() string { return f.sandbox }
|
|
func (f *wfFake) Plan() (core.PlanSnapshot, bool) {
|
|
if len(f.plans) == 0 {
|
|
return core.PlanSnapshot{}, false
|
|
}
|
|
p := f.plans[len(f.plans)-1]
|
|
p.Steps = append([]core.PlanStep(nil), p.Steps...)
|
|
return p, true
|
|
}
|
|
func (f *wfFake) WorkflowScript(id, path string) (string, bool) {
|
|
body, ok := f.scripts[workflow.ScriptKey(id, path)]
|
|
return body, ok
|
|
}
|
|
func (f *wfFake) Emit(core.StreamEventType, any) error { return nil }
|
|
func (f *wfFake) SetAssistantParts(parts []core.Part) error {
|
|
f.assistant = parts
|
|
return nil
|
|
}
|
|
func (f *wfFake) AskHITL(card core.HITLCard) (core.HITLResponse, error) {
|
|
f.hitlQ = append(f.hitlQ, card.Question)
|
|
if len(f.hitl) == 0 {
|
|
return core.HITLResponse{}, context.Canceled
|
|
}
|
|
resp := f.hitl[0]
|
|
f.hitl = f.hitl[1:]
|
|
return resp, nil
|
|
}
|
|
func (f *wfFake) SpawnSubagent(name, task, hint string) (core.AgentResultPayload, error) {
|
|
return f.SpawnAgent("agent.sub.general", name, task, hint)
|
|
}
|
|
func (f *wfFake) SpawnAgent(agentID, name, task, hint string) (core.AgentResultPayload, error) {
|
|
f.prompts = append(f.prompts, agentID+"|"+task)
|
|
f.spawnIDs = append(f.spawnIDs, agentID)
|
|
text := f.spawnText
|
|
if text == "" {
|
|
text = "正文 art_0123456789ab"
|
|
}
|
|
return core.AgentResultPayload{Text: text, Status: core.RunCompleted}, nil
|
|
}
|
|
func (f *wfFake) UpdatePlan(p core.PlanSnapshot) error {
|
|
p.Steps = append([]core.PlanStep(nil), p.Steps...)
|
|
f.plans = append(f.plans, p)
|
|
return nil
|
|
}
|
|
func (f *wfFake) UpdateTodos(core.TodoList) error { return nil }
|
|
func (f *wfFake) WriteArtifact(name, kind, mime string, content []byte) (core.Artifact, error) {
|
|
f.writes = append(f.writes, name+"="+string(content))
|
|
art := core.Artifact{ID: "art_aaaaaaaaaaaa", Name: name, Kind: kind}
|
|
if f.arts == nil {
|
|
f.arts = map[string]core.Artifact{}
|
|
}
|
|
f.arts[string(art.ID)] = art
|
|
return art, nil
|
|
}
|
|
func (f *wfFake) Retrieve(core.RetrievalQuery) ([]core.RetrievalHit, error) { return nil, nil }
|
|
func (f *wfFake) ExecuteTool(call core.ToolCallRequest) core.ToolCallResult {
|
|
f.tools = append(f.tools, call)
|
|
if call.Name == "make_file" {
|
|
art := core.Artifact{ID: "art_bbbbbbbbbbbb", Name: "made.md"}
|
|
if f.arts == nil {
|
|
f.arts = map[string]core.Artifact{}
|
|
}
|
|
f.arts[string(art.ID)] = art
|
|
return core.ToolCallResult{Status: "ok", Output: "wrote", Artifact: &art}
|
|
}
|
|
return core.ToolCallResult{Status: "ok", Output: "12:00"}
|
|
}
|
|
func (f *wfFake) History() []core.Message { return nil }
|
|
func (f *wfFake) Attachments() []core.ParsedDocument { return nil }
|
|
func (f *wfFake) Tools() []core.ToolDesc {
|
|
var out []core.ToolDesc
|
|
for _, name := range f.toolNames {
|
|
out = append(out, core.ToolDesc{Name: name})
|
|
}
|
|
return out
|
|
}
|
|
func (f *wfFake) Subtask() string { return "" }
|
|
func (f *wfFake) CreateCard(card core.ActivityCard) (core.ActivityCard, error) {
|
|
card.ID = "card_1"
|
|
f.cards = append(f.cards, card)
|
|
return card, nil
|
|
}
|
|
func (f *wfFake) SetCardStatus(string, core.CardStatus, string, string) error { return nil }
|
|
func (f *wfFake) SubmitCard(card core.ActivityCard) (core.ActivityCard, error) {
|
|
return card, nil
|
|
}
|
|
func (f *wfFake) BeginStreamCard(card core.ActivityCard) (core.ActivityCard, error) {
|
|
return card, nil
|
|
}
|
|
func (f *wfFake) AppendStreamCard(string) error { return nil }
|
|
func (f *wfFake) CommitStreamCard() (core.ActivityCard, error) { return core.ActivityCard{}, nil }
|
|
func (f *wfFake) Card(string) *core.ActivityCard { return nil }
|
|
func (f *wfFake) Artifact(id string) (core.Artifact, []byte, bool) {
|
|
if f.arts == nil {
|
|
f.arts = map[string]core.Artifact{}
|
|
}
|
|
if id == "art_0123456789ab" {
|
|
return core.Artifact{ID: core.ID(id), Name: "note.md"}, []byte("x"), true
|
|
}
|
|
art, ok := f.arts[id]
|
|
return art, nil, ok
|
|
}
|
|
func (f *wfFake) SetContextObservation(core.ContextObservation) {}
|
|
func (f *wfFake) Workflow() (workflow.Launch, bool) { return f.launch, true }
|
|
func (f *wfFake) WorkflowByID(id string) (core.WorkflowDef, bool) {
|
|
def, ok := f.byID[id]
|
|
return def, ok
|
|
}
|
|
|
|
func TestWorkflowRunsFiveKindsAndAsksMissingSlot(t *testing.T) {
|
|
f := &wfFake{hitl: []core.HITLResponse{{FreeText: "核能"}, {ChoiceID: "short"}}}
|
|
f.launch = workflow.Launch{
|
|
Missing: []workflow.SlotQuestion{{Key: "topic", Question: "主题是什么？"}},
|
|
Def: core.WorkflowDef{ID: "demo.brief", Title: "简报", Steps: []core.WorkflowStep{
|
|
{ID: "len", Title: "篇幅", Kind: "hitl", Hitl: &core.HITLCardTemplate{Question: "篇幅？", Kind: core.HITLKindChoice, Choices: []core.HITLChoice{{ID: "short", Label: "短"}}}},
|
|
{ID: "draft", Title: "起草", Kind: "prompt", Agent: "research", Prompt: "写 {{slots.topic}}"},
|
|
{ID: "now", Title: "时间", Kind: "tool", Tool: "time_now", Args: []byte(`{"q":"{{slots.topic}}"}`)},
|
|
{ID: "save", Title: "存稿", Kind: "artifact", Artifact: "brief.md", Prompt: "{{steps.draft.output}}"},
|
|
{ID: "note", Title: "通知", Kind: "card", Card: &core.ActivityCardTemplate{Kind: core.CardKindResult, Title: "好了"}},
|
|
}},
|
|
}
|
|
if err := (WorkflowRunner{}).Execute(f); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.hitlQ[0] != "主题是什么？" || f.hitlQ[1] != "篇幅？" {
|
|
t.Fatalf("questions %v", f.hitlQ)
|
|
}
|
|
if len(f.prompts) != 1 || !strings.Contains(f.prompts[0], "agent.sub.research|写 核能") {
|
|
t.Fatalf("prompt %v", f.prompts)
|
|
}
|
|
if len(f.tools) != 1 || string(f.tools[0].Arguments) == "" || !strings.Contains(string(f.tools[0].Arguments), "核能") {
|
|
t.Fatalf("tool %#v", f.tools)
|
|
}
|
|
if len(f.writes) != 1 || f.writes[0] != "brief.md=正文 art_0123456789ab" {
|
|
t.Fatalf("writes %v", f.writes)
|
|
}
|
|
if len(f.cards) != 1 || f.cards[0].Title != "好了" {
|
|
t.Fatalf("cards %+v", f.cards)
|
|
}
|
|
var sawProgress, sawDone bool
|
|
for _, p := range f.plans {
|
|
for _, s := range p.Steps {
|
|
if s.ID == "draft" && s.Status == "in_progress" {
|
|
sawProgress = true
|
|
}
|
|
if s.ID == "note" && s.Status == "done" {
|
|
sawDone = true
|
|
}
|
|
}
|
|
}
|
|
if !sawProgress || !sawDone {
|
|
t.Fatal("plan did not move from in_progress to done")
|
|
}
|
|
text := f.assistant[0].Text
|
|
if !strings.Contains(text, "art_0123456789ab") || !strings.Contains(text, "art_aaaaaaaaaaaa") {
|
|
t.Fatalf("summary %s", text)
|
|
}
|
|
}
|
|
|
|
func TestCriticRetriesThenStops(t *testing.T) {
|
|
f := &wfFake{}
|
|
f.launch = workflow.Launch{Def: core.WorkflowDef{
|
|
ID: "a", Title: "验收", MaxRetries: 6,
|
|
Steps: []core.WorkflowStep{{
|
|
ID: "draft", Title: "起草", Kind: "prompt", Prompt: "写",
|
|
Critic: &core.StepCritic{Criteria: "必须点明主题", MaxRetries: 1},
|
|
}},
|
|
}}
|
|
runner := WorkflowRunner{JudgeStream: passedStream(
|
|
`{"passed":false,"reason":"没点明主题","nextAction":"补上主题"}`,
|
|
`{"passed":true,"reason":"可以","nextAction":""}`,
|
|
)}
|
|
if err := runner.Execute(f); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(f.prompts) != 2 || !strings.Contains(f.prompts[1], "没点明主题") || !strings.Contains(f.prompts[1], "补上主题") {
|
|
t.Fatalf("prompts %v", f.prompts)
|
|
}
|
|
if !strings.Contains(f.assistant[0].Text, "已丢弃") || !strings.Contains(f.assistant[0].Text, "没点明主题") {
|
|
t.Fatalf("summary %s", f.assistant[0].Text)
|
|
}
|
|
|
|
f2 := &wfFake{}
|
|
f2.launch = f.launch
|
|
bad := WorkflowRunner{JudgeStream: passedStream("不是 json", "不是 json")}
|
|
err := bad.Execute(f2)
|
|
if err == nil || len(f2.prompts) != 2 {
|
|
t.Fatalf("err %v prompts %d", err, len(f2.prompts))
|
|
}
|
|
if !strings.Contains(f2.assistant[0].Text, "无法解析") {
|
|
t.Fatalf("summary %s", f2.assistant[0].Text)
|
|
}
|
|
|
|
f3 := &wfFake{}
|
|
f3.launch = workflow.Launch{Def: core.WorkflowDef{
|
|
ID: "a", Title: "验收", MaxRetries: 1,
|
|
Steps: []core.WorkflowStep{{
|
|
ID: "draft", Title: "起草", Kind: "prompt", Prompt: "写",
|
|
Critic: &core.StepCritic{Criteria: "要过", MaxRetries: 3},
|
|
}},
|
|
}}
|
|
fail := `{"passed":false,"reason":"不行","nextAction":"重写"}`
|
|
stop := WorkflowRunner{JudgeStream: passedStream(fail, fail, fail)}
|
|
if err := stop.Execute(f3); err == nil || len(f3.prompts) != 2 {
|
|
t.Fatalf("whole cap err %v attempts %d", err, len(f3.prompts))
|
|
}
|
|
}
|
|
|
|
func TestRunWorkflowToolUsesRegisteredDefinition(t *testing.T) {
|
|
f := &wfFake{hitl: []core.HITLResponse{{ChoiceID: "yes"}}}
|
|
f.byID = map[string]core.WorkflowDef{
|
|
"demo.brief": {ID: "demo.brief", Title: "简报", Steps: []core.WorkflowStep{
|
|
{ID: "ask", Title: "确认", Kind: "hitl", Hitl: &core.HITLCardTemplate{Question: "可以吗？", Kind: core.HITLKindConfirm}},
|
|
}},
|
|
}
|
|
st := &directRun{rc: f}
|
|
out := st.runRegisteredWorkflow(`{"workflow":"demo.brief","inputs":{"topic":"核能"}}`)
|
|
if !strings.Contains(out, "完成") || f.hitlQ[0] != "可以吗？" {
|
|
t.Fatalf("out %s questions %v", out, f.hitlQ)
|
|
}
|
|
if got := st.runRegisteredWorkflow(`{"workflow":"missing"}`); !strings.Contains(got, "未注册") {
|
|
t.Fatalf("%s", got)
|
|
}
|
|
if got := st.runRegisteredWorkflow(`{"goal":"做一件事","workflow":"demo.brief"}`); !strings.Contains(got, "只能") {
|
|
t.Fatalf("%s", got)
|
|
}
|
|
}
|
|
|
|
func TestJudgeUsesConfiguredModel(t *testing.T) {
|
|
var got string
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body map[string]any
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
got, _ = body["model"].(string)
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
_, _ = io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"{\\\"passed\\\":true,\\\"reason\\\":\\\"ok\\\",\\\"nextAction\\\":\\\"\\\"}\"}}]}\n\n")
|
|
_, _ = io.WriteString(w, "data: [DONE]\n\n")
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
f := &wfFake{
|
|
settings: core.EffectiveSettings{"models": map[string]any{"chat": map[string]any{
|
|
"baseUrl": srv.URL, "apiKey": "test-key", "model": "MiniMax-test",
|
|
}}},
|
|
launch: workflow.Launch{Def: core.WorkflowDef{
|
|
ID: "c", Title: "验收",
|
|
Steps: []core.WorkflowStep{{
|
|
ID: "draft", Title: "起草", Kind: "prompt", Prompt: "写",
|
|
Critic: &core.StepCritic{Criteria: "要过", MaxRetries: 1},
|
|
}},
|
|
}},
|
|
}
|
|
if err := (WorkflowRunner{}).Execute(f); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got != "MiniMax-test" {
|
|
t.Fatalf("model %q", got)
|
|
}
|
|
}
|
|
|
|
func TestPlannedWorkflowNeedsConfirm(t *testing.T) {
|
|
plan := `{"id":"plan.brief","title":"简报","steps":[{"id":"ask","title":"确认","kind":"hitl","hitl":{"question":"可以吗？","kind":"confirm","allowFreeInput":false}}]}`
|
|
f := &wfFake{
|
|
spawnText: plan,
|
|
hitl: []core.HITLResponse{{ChoiceID: "yes"}, {ChoiceID: "ok"}},
|
|
}
|
|
st := &directRun{rc: f}
|
|
out := st.runRegisteredWorkflow(`{"goal":"写一份简报"}`)
|
|
if f.spawnIDs[0] != "agent.sub.workflow-plan" || !strings.Contains(f.hitlQ[0], "按这个计划执行") {
|
|
t.Fatalf("spawn %v questions %v", f.spawnIDs, f.hitlQ)
|
|
}
|
|
if !strings.Contains(out, "完成") || f.hitlQ[1] != "可以吗？" {
|
|
t.Fatalf("out %s questions %v", out, f.hitlQ)
|
|
}
|
|
|
|
f2 := &wfFake{spawnText: plan, hitl: []core.HITLResponse{{ChoiceID: "no"}}}
|
|
out = (&directRun{rc: f2}).runRegisteredWorkflow(`{"goal":"写一份简报"}`)
|
|
if out != "未确认，未执行" || len(f2.hitlQ) != 1 {
|
|
t.Fatalf("out %s questions %v", out, f2.hitlQ)
|
|
}
|
|
|
|
var steps []string
|
|
for i := 0; i < 9; i++ {
|
|
steps = append(steps, fmt.Sprintf(`{"id":"s%d","title":"步","kind":"hitl"}`, i))
|
|
}
|
|
f3 := &wfFake{spawnText: `{"id":"too.big","title":"太多","steps":[` + strings.Join(steps, ",") + `]}`}
|
|
out = (&directRun{rc: f3}).runRegisteredWorkflow(`{"goal":"做很多事"}`)
|
|
if !strings.Contains(out, "超出上限") || len(f3.hitlQ) != 0 {
|
|
t.Fatalf("out %s questions %v", out, f3.hitlQ)
|
|
}
|
|
|
|
f4 := &wfFake{spawnText: `{"id":"bad.script","title":"脚本","steps":[{"id":"run","title":"跑","kind":"script","script":"a.py"}]}`}
|
|
out = (&directRun{rc: f4}).runRegisteredWorkflow(`{"goal":"跑脚本"}`)
|
|
if !strings.Contains(out, "script") || len(f4.hitlQ) != 0 {
|
|
t.Fatalf("out %s questions %v", out, f4.hitlQ)
|
|
}
|
|
}
|