diff --git a/internal/server/work_checkpoint_eval_test.go b/internal/server/work_checkpoint_eval_test.go index f85b78f..2adae1c 100644 --- a/internal/server/work_checkpoint_eval_test.go +++ b/internal/server/work_checkpoint_eval_test.go @@ -36,10 +36,13 @@ import ( core "agent-system-core" ) -// evalForeground excludes the existing automatic notice model turn in BOTH -// baseline and candidate. It is not evidence for notice quality; normal notice -// behavior is covered separately by the production regression suite. -type evalForeground struct{ delegate runtime.Runner } +// Paid baseline/candidate comparisons exclude the automatic notice model turn +// equally. Only the task-owned mechanism browser fixture enables it to check +// the user-visible choice and failure paths without provider requests. +type evalForeground struct { + delegate runtime.Runner + allowNotice bool // only the task-owned, zero-paid browser mechanism enables this +} func (e evalForeground) Name() string { return "evaluation-foreground" } @@ -56,7 +59,7 @@ func evalA01Budget(label string) (int, bool) { func (e evalForeground) Execute(rc runtime.RunCtx) error { history := rc.History() - if len(history) > 0 && history[len(history)-1].Role == core.RoleSystem { + if !e.allowNotice && len(history) > 0 && history[len(history)-1].Role == core.RoleSystem { return rc.SetAssistantParts([]core.Part{{Type: core.PartText, Text: "后台结果已回投任务卡(评测不再调用通知模型)"}}) } return e.delegate.Execute(rc) @@ -65,26 +68,53 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess out := make(chan agentllm.Event, 2) defer close(out) checkpoint := false + notice := false + choiceReady := false + sourceText := "" + toolReplies := 0 for _, d := range decls { if d.Name == "publish_work_checkpoint" { checkpoint = true } } + for _, m := range msgs { + if m.Role == agentllm.RoleUser { + sourceText += m.Content + } + if m.Role == agentllm.RoleTool { + toolReplies++ + } + if strings.HasPrefix(strings.TrimSpace(m.Content), "") && strings.Contains(m.Content, "后台子代理「") { + notice = true + choiceReady = strings.Contains(m.Content, runtime.MaterialChoiceReadyMarker) + } + } + if notice { + content := "后台整理状态已回投工作卡;请按卡片中的结果继续。" + var calls []agentllm.ToolCall + if choiceReady { + // Deliberately supply unrelated options: the runtime must replace them + // with the host-extracted, source-bound options from the notice. + calls = []agentllm.ToolCall{{ID: "notice-choice", Name: "confirm_choice", Arguments: `{"question":"模型提出的交付物?","options":[{"id":"fake-a","label":"虚构报告","description":"无来源","impact":"虚构"},{"id":"fake-b","label":"虚构方案","description":"无来源","impact":"虚构"}]}`}} + } + reason := agentllm.StopEnd + if len(calls) > 0 { + reason = agentllm.StopToolCalls + } + out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls, StopReason: reason}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}} + return out + } content := "已收到材料,正在后台整理;请结合阶段发现继续工作。" var calls []agentllm.ToolCall if checkpoint { - sourceText := "" - count := 0 - for _, m := range msgs { - if m.Role == agentllm.RoleUser { - sourceText += m.Content - } - if m.Role == agentllm.RoleTool { - count++ - } - } + count := toolReplies matches := mechanismCheckpointSources(sourceText) matches = filterCheckpointSources(matches) + if (count == 0 && strings.Contains(sourceText, "[fixture:zero-failure]")) || + (count == 1 && strings.Contains(sourceText, "[fixture:partial-failure]")) { + out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, StopReason: agentllm.StopEnd}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}, Err: fmt.Errorf("task-owned mechanism failure after %d checkpoint(s)", count)} + return out + } if count < 2 && len(matches) > 0 { var block int fmt.Sscan(matches[0][2], &block) @@ -108,7 +138,14 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess } else { content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。" } - time.Sleep(200 * time.Millisecond) + if strings.Contains(sourceText, "[fixture:slow-stages]") { + time.Sleep(1500 * time.Millisecond) + } else { + time.Sleep(200 * time.Millisecond) + } + } + if toolReplies >= 2 && strings.Contains(sourceText, "[fixture:choice]") { + content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。\n\n## 建议后续处理\n1. **会议摘要**\n - 依据:材料记载上线六月。\n2. **验收清单**\n - 依据:验收条件需要补齐。\n3. **联调计划**\n - 依据:三项需求依赖外部接口。" } if len(calls) == 0 { out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: content} @@ -208,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) { front.Stream = evalMechanismStream sub.Stream = evalMechanismStream } - mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{front}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{}) + mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{delegate: front, allowNotice: mode == "mechanism" && os.Getenv("WORK_CHECKPOINT_MECHANISM_NOTICES") == "1"}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{}) mgr.SetWorkspaceRoot(filepath.Join(dir, "workspaces")) mgr.SetInstanceID("checkpoint-eval") st.SetInstanceID("checkpoint-eval")