Exercise material notice outcomes in the browser fixture
ci / vet-and-test (pull_request) Successful in 1m3s

Why: The task-owned mechanism must reproduce stage progress, partial and empty failures, and source-bound choices without a paid provider call.

Evidence: Backend vet/full Go tests and live-tag terminal tests passed; six task-owned HTTP/Playwright cases passed with this fixture.

Impact: Evaluation fixture only; model comparison still disables notice turns unless the explicit mechanism browser flag is set.

Refs: U3 r0.4 Task 6
This commit is contained in:
2026-10-14 03:34:27 +00:00
parent 8ea9a6e788
commit 726e4a6d2e
+54 -17
View File
@@ -36,10 +36,13 @@ import (
core "agent-system-core"
)
// evalForeground excludes the existing automatic notice model turn in BOTH
// baseline and candidate. It is not evidence for notice quality; normal notice
// behavior is covered separately by the production regression suite.
type evalForeground struct{ delegate runtime.Runner }
// Paid baseline/candidate comparisons exclude the automatic notice model turn
// equally. Only the task-owned mechanism browser fixture enables it to check
// the user-visible choice and failure paths without provider requests.
type evalForeground struct {
delegate runtime.Runner
allowNotice bool // only the task-owned, zero-paid browser mechanism enables this
}
func (e evalForeground) Name() string { return "evaluation-foreground" }
@@ -56,7 +59,7 @@ func evalA01Budget(label string) (int, bool) {
func (e evalForeground) Execute(rc runtime.RunCtx) error {
history := rc.History()
if len(history) > 0 && history[len(history)-1].Role == core.RoleSystem {
if !e.allowNotice && len(history) > 0 && history[len(history)-1].Role == core.RoleSystem {
return rc.SetAssistantParts([]core.Part{{Type: core.PartText, Text: "后台结果已回投任务卡(评测不再调用通知模型)"}})
}
return e.delegate.Execute(rc)
@@ -65,26 +68,53 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
out := make(chan agentllm.Event, 2)
defer close(out)
checkpoint := false
notice := false
choiceReady := false
sourceText := ""
toolReplies := 0
for _, d := range decls {
if d.Name == "publish_work_checkpoint" {
checkpoint = true
}
}
for _, m := range msgs {
if m.Role == agentllm.RoleUser {
sourceText += m.Content
}
if m.Role == agentllm.RoleTool {
toolReplies++
}
if strings.HasPrefix(strings.TrimSpace(m.Content), "<system-reminder>") && strings.Contains(m.Content, "后台子代理「") {
notice = true
choiceReady = strings.Contains(m.Content, runtime.MaterialChoiceReadyMarker)
}
}
if notice {
content := "后台整理状态已回投工作卡;请按卡片中的结果继续。"
var calls []agentllm.ToolCall
if choiceReady {
// Deliberately supply unrelated options: the runtime must replace them
// with the host-extracted, source-bound options from the notice.
calls = []agentllm.ToolCall{{ID: "notice-choice", Name: "confirm_choice", Arguments: `{"question":"模型提出的交付物?","options":[{"id":"fake-a","label":"虚构报告","description":"无来源","impact":"虚构"},{"id":"fake-b","label":"虚构方案","description":"无来源","impact":"虚构"}]}`}}
}
reason := agentllm.StopEnd
if len(calls) > 0 {
reason = agentllm.StopToolCalls
}
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls, StopReason: reason}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}}
return out
}
content := "已收到材料,正在后台整理;请结合阶段发现继续工作。"
var calls []agentllm.ToolCall
if checkpoint {
sourceText := ""
count := 0
for _, m := range msgs {
if m.Role == agentllm.RoleUser {
sourceText += m.Content
}
if m.Role == agentllm.RoleTool {
count++
}
}
count := toolReplies
matches := mechanismCheckpointSources(sourceText)
matches = filterCheckpointSources(matches)
if (count == 0 && strings.Contains(sourceText, "[fixture:zero-failure]")) ||
(count == 1 && strings.Contains(sourceText, "[fixture:partial-failure]")) {
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, StopReason: agentllm.StopEnd}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}, Err: fmt.Errorf("task-owned mechanism failure after %d checkpoint(s)", count)}
return out
}
if count < 2 && len(matches) > 0 {
var block int
fmt.Sscan(matches[0][2], &block)
@@ -108,7 +138,14 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
} else {
content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。"
}
time.Sleep(200 * time.Millisecond)
if strings.Contains(sourceText, "[fixture:slow-stages]") {
time.Sleep(1500 * time.Millisecond)
} else {
time.Sleep(200 * time.Millisecond)
}
}
if toolReplies >= 2 && strings.Contains(sourceText, "[fixture:choice]") {
content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。\n\n## 建议后续处理\n1. **会议摘要**\n - 依据:材料记载上线六月。\n2. **验收清单**\n - 依据:验收条件需要补齐。\n3. **联调计划**\n - 依据:三项需求依赖外部接口。"
}
if len(calls) == 0 {
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: content}
@@ -208,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) {
front.Stream = evalMechanismStream
sub.Stream = evalMechanismStream
}
mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{front}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{})
mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{delegate: front, allowNotice: mode == "mechanism" && os.Getenv("WORK_CHECKPOINT_MECHANISM_NOTICES") == "1"}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{})
mgr.SetWorkspaceRoot(filepath.Join(dir, "workspaces"))
mgr.SetInstanceID("checkpoint-eval")
st.SetInstanceID("checkpoint-eval")