Exercise material notice outcomes in the browser fixture
ci / vet-and-test (pull_request) Successful in 1m3s
ci / vet-and-test (pull_request) Successful in 1m3s
Why: The task-owned mechanism must reproduce stage progress, partial and empty failures, and source-bound choices without a paid provider call. Evidence: Backend vet/full Go tests and live-tag terminal tests passed; six task-owned HTTP/Playwright cases passed with this fixture. Impact: Evaluation fixture only; model comparison still disables notice turns unless the explicit mechanism browser flag is set. Refs: U3 r0.4 Task 6
This commit is contained in:
@@ -36,10 +36,13 @@ import (
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
// evalForeground excludes the existing automatic notice model turn in BOTH
|
||||
// baseline and candidate. It is not evidence for notice quality; normal notice
|
||||
// behavior is covered separately by the production regression suite.
|
||||
type evalForeground struct{ delegate runtime.Runner }
|
||||
// Paid baseline/candidate comparisons exclude the automatic notice model turn
|
||||
// equally. Only the task-owned mechanism browser fixture enables it to check
|
||||
// the user-visible choice and failure paths without provider requests.
|
||||
type evalForeground struct {
|
||||
delegate runtime.Runner
|
||||
allowNotice bool // only the task-owned, zero-paid browser mechanism enables this
|
||||
}
|
||||
|
||||
func (e evalForeground) Name() string { return "evaluation-foreground" }
|
||||
|
||||
@@ -56,7 +59,7 @@ func evalA01Budget(label string) (int, bool) {
|
||||
|
||||
func (e evalForeground) Execute(rc runtime.RunCtx) error {
|
||||
history := rc.History()
|
||||
if len(history) > 0 && history[len(history)-1].Role == core.RoleSystem {
|
||||
if !e.allowNotice && len(history) > 0 && history[len(history)-1].Role == core.RoleSystem {
|
||||
return rc.SetAssistantParts([]core.Part{{Type: core.PartText, Text: "后台结果已回投任务卡(评测不再调用通知模型)"}})
|
||||
}
|
||||
return e.delegate.Execute(rc)
|
||||
@@ -65,26 +68,53 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
|
||||
out := make(chan agentllm.Event, 2)
|
||||
defer close(out)
|
||||
checkpoint := false
|
||||
notice := false
|
||||
choiceReady := false
|
||||
sourceText := ""
|
||||
toolReplies := 0
|
||||
for _, d := range decls {
|
||||
if d.Name == "publish_work_checkpoint" {
|
||||
checkpoint = true
|
||||
}
|
||||
}
|
||||
for _, m := range msgs {
|
||||
if m.Role == agentllm.RoleUser {
|
||||
sourceText += m.Content
|
||||
}
|
||||
if m.Role == agentllm.RoleTool {
|
||||
toolReplies++
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSpace(m.Content), "<system-reminder>") && strings.Contains(m.Content, "后台子代理「") {
|
||||
notice = true
|
||||
choiceReady = strings.Contains(m.Content, runtime.MaterialChoiceReadyMarker)
|
||||
}
|
||||
}
|
||||
if notice {
|
||||
content := "后台整理状态已回投工作卡;请按卡片中的结果继续。"
|
||||
var calls []agentllm.ToolCall
|
||||
if choiceReady {
|
||||
// Deliberately supply unrelated options: the runtime must replace them
|
||||
// with the host-extracted, source-bound options from the notice.
|
||||
calls = []agentllm.ToolCall{{ID: "notice-choice", Name: "confirm_choice", Arguments: `{"question":"模型提出的交付物?","options":[{"id":"fake-a","label":"虚构报告","description":"无来源","impact":"虚构"},{"id":"fake-b","label":"虚构方案","description":"无来源","impact":"虚构"}]}`}}
|
||||
}
|
||||
reason := agentllm.StopEnd
|
||||
if len(calls) > 0 {
|
||||
reason = agentllm.StopToolCalls
|
||||
}
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls, StopReason: reason}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}}
|
||||
return out
|
||||
}
|
||||
content := "已收到材料,正在后台整理;请结合阶段发现继续工作。"
|
||||
var calls []agentllm.ToolCall
|
||||
if checkpoint {
|
||||
sourceText := ""
|
||||
count := 0
|
||||
for _, m := range msgs {
|
||||
if m.Role == agentllm.RoleUser {
|
||||
sourceText += m.Content
|
||||
}
|
||||
if m.Role == agentllm.RoleTool {
|
||||
count++
|
||||
}
|
||||
}
|
||||
count := toolReplies
|
||||
matches := mechanismCheckpointSources(sourceText)
|
||||
matches = filterCheckpointSources(matches)
|
||||
if (count == 0 && strings.Contains(sourceText, "[fixture:zero-failure]")) ||
|
||||
(count == 1 && strings.Contains(sourceText, "[fixture:partial-failure]")) {
|
||||
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, StopReason: agentllm.StopEnd}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}, Err: fmt.Errorf("task-owned mechanism failure after %d checkpoint(s)", count)}
|
||||
return out
|
||||
}
|
||||
if count < 2 && len(matches) > 0 {
|
||||
var block int
|
||||
fmt.Sscan(matches[0][2], &block)
|
||||
@@ -108,7 +138,14 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
|
||||
} else {
|
||||
content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。"
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
if strings.Contains(sourceText, "[fixture:slow-stages]") {
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
} else {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if toolReplies >= 2 && strings.Contains(sourceText, "[fixture:choice]") {
|
||||
content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。\n\n## 建议后续处理\n1. **会议摘要**\n - 依据:材料记载上线六月。\n2. **验收清单**\n - 依据:验收条件需要补齐。\n3. **联调计划**\n - 依据:三项需求依赖外部接口。"
|
||||
}
|
||||
if len(calls) == 0 {
|
||||
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: content}
|
||||
@@ -208,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) {
|
||||
front.Stream = evalMechanismStream
|
||||
sub.Stream = evalMechanismStream
|
||||
}
|
||||
mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{front}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{})
|
||||
mgr := runmgr.New(st, b, registry, prof, settingSvc, evalForeground{delegate: front, allowNotice: mode == "mechanism" && os.Getenv("WORK_CHECKPOINT_MECHANISM_NOTICES") == "1"}, sub, &runtime.MockRunner{}, &runtime.MockSubagent{})
|
||||
mgr.SetWorkspaceRoot(filepath.Join(dir, "workspaces"))
|
||||
mgr.SetInstanceID("checkpoint-eval")
|
||||
st.SetInstanceID("checkpoint-eval")
|
||||
|
||||
Reference in New Issue
Block a user