Switch the R4 failure trigger to a sliding window #133
@@ -11,34 +11,25 @@ import (
|
||||
|
||||
// ---- E4:R4 升级提醒 ----
|
||||
|
||||
// 两连败后 reminders 注入换路指令;成功一次即重置、可再次触发。
|
||||
// 滑动窗口判据:近 6 次结果中 3 次失败(可与成功交错)触发换路指令;
|
||||
// 每 run 一次;2 次失败不触发。
|
||||
func TestEscalationOnConsecutiveToolFailures(t *testing.T) {
|
||||
f := &fakeRunCtx{ctx: context.Background(), prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}}, execStatus: "error", execOutput: "boom"}
|
||||
r := &LLMRunner{Stream: scriptStream(padTurns(
|
||||
textDeltaTurn("", toolCall("run_python", `{"code":"1"}`), toolCall("run_python", `{"code":"2"}`)),
|
||||
textDeltaTurn("换路了。"),
|
||||
), nil)}
|
||||
if err := r.Execute(f); err != nil {
|
||||
t.Fatal(err)
|
||||
st := &directRun{l: &LLMRunner{}, inflight: map[string]*toolSlot{}}
|
||||
st.noteFailureOutcome("ok")
|
||||
st.noteFailureOutcome("error")
|
||||
st.noteFailureOutcome("ok")
|
||||
st.noteFailureOutcome("error")
|
||||
if got := st.escalationReminders(); len(got) != 0 {
|
||||
t.Fatalf("two scattered failures must not escalate: %v", got)
|
||||
}
|
||||
// 完整 Execute 冒烟(真实事件路径跑通)+ 状态机断言(onEvent 终点计数
|
||||
// → escalationReminders 文案)。
|
||||
st2 := &directRun{l: &LLMRunner{}, inflight: map[string]*toolSlot{}}
|
||||
st2.noteFailureOutcome("error")
|
||||
if got := st2.escalationReminders(); len(got) != 0 {
|
||||
t.Fatalf("single failure must not escalate: %v", got)
|
||||
}
|
||||
st2.noteFailureOutcome("error")
|
||||
got := st2.escalationReminders()
|
||||
st.noteFailureOutcome("ok")
|
||||
st.noteFailureOutcome("error") // 窗口内第 3 败(交错分布)
|
||||
got := st.escalationReminders()
|
||||
if len(got) != 1 || !strings.Contains(got[0], "停止原样重试") {
|
||||
t.Fatalf("two consecutive failures must escalate: %v", got)
|
||||
t.Fatalf("three failures in the window must escalate: %v", got)
|
||||
}
|
||||
if again := st2.escalationReminders(); len(again) != 0 {
|
||||
t.Fatalf("escalation must be once per streak: %v", again)
|
||||
}
|
||||
st2.noteFailureOutcome("ok")
|
||||
if st2.consecToolFailures != 0 || st2.failStreakNoted {
|
||||
t.Fatal("success must reset the failure streak")
|
||||
if again := st.escalationReminders(); len(again) != 0 {
|
||||
t.Fatalf("escalation must fire once per run: %v", again)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -271,8 +271,17 @@ func isEvidenceRead(name string, args json.RawMessage) bool {
|
||||
return strings.HasPrefix(p, "inputs/") || strings.HasPrefix(p, "/inputs/")
|
||||
}
|
||||
|
||||
// escalationFailureThreshold 是连续工具失败的换路提醒阈值(R4)。
|
||||
const escalationFailureThreshold = 2
|
||||
// R4 换路提醒的滑动窗口判据:最近 escalationWindow 次工具结果里出现
|
||||
// escalationFailureThreshold 次失败即触发(每 run 一次)。用窗口而不是
|
||||
// 「连续 N 次」——实测失败多与成功交错(20 失败散布在 108 次调用里,
|
||||
// 连续计数从未达 2,首轮 eval 生产路径零触发),滑动窗口才抓得住
|
||||
// 「整体在原地打转」的形态。
|
||||
const (
|
||||
escalationWindow = 6
|
||||
escalationFailureThreshold = 3
|
||||
// 检索空命中阈值独立取 2:检索单次成本高,两次落空即是换维度信号。
|
||||
escalationZeroHitThreshold = 2
|
||||
)
|
||||
|
||||
// tagSource 把一次成功的证据工具输出登记进账本,返回给模型看的来源标记
|
||||
// (如 src_3|knowledge_search)。并行执行下由 toolMu 串行化计数与追加。
|
||||
@@ -324,33 +333,42 @@ func containsAny(s string, subs []string) bool {
|
||||
func (st *directRun) escalationReminders() []string {
|
||||
var out []string
|
||||
st.toolMu.Lock()
|
||||
fails, failNoted := st.consecToolFailures, st.failStreakNoted
|
||||
fails, failNoted := countFails(st.recentOutcomes), st.failWindowNoted
|
||||
zero, zeroNoted := st.zeroHitSearches, st.zeroHitNoted
|
||||
if fails >= escalationFailureThreshold && !failNoted {
|
||||
st.failStreakNoted = true
|
||||
st.failWindowNoted = true
|
||||
}
|
||||
if zero >= escalationFailureThreshold && !zeroNoted {
|
||||
if zero >= escalationZeroHitThreshold && !zeroNoted {
|
||||
st.zeroHitNoted = true
|
||||
}
|
||||
st.toolMu.Unlock()
|
||||
if fails >= escalationFailureThreshold && !failNoted {
|
||||
out = append(out, fmt.Sprintf("工具已连续失败 %d 次:停止原样重试。逐条重读最近一次错误信息,改参数、换工具或换路径再试;确认无法绕过时,如实在答复里说明卡点与已尝试的手段,不要编造结果。", fails))
|
||||
out = append(out, fmt.Sprintf("工具最近集中失败(近 %d 次调用中多次报错):停止原样重试。逐条重读最近一次错误信息,改参数、换工具或换路径再试;确认无法绕过时,如实在答复里说明卡点与已尝试的手段,不要编造结果。", fails))
|
||||
}
|
||||
if zero >= escalationFailureThreshold && !zeroNoted {
|
||||
if zero >= escalationZeroHitThreshold && !zeroNoted {
|
||||
out = append(out, "检索已连续多次无结果:换关键词/换检索维度再试一次;仍无结果就如实写「材料中无此信息」,不要凭印象补数字。")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// noteFailureOutcome 供 onEvent 在工具结束时同步成败计数(并行下事件乱序,
|
||||
// 以 runToolCtx 已写入槽位的 status 为准)。
|
||||
// noteFailureOutcome 供 onEvent 在工具结束时记录成败(并行下事件乱序,
|
||||
// 以 runToolCtx 已写入槽位的 status 为准)。维护最近 escalationWindow 次
|
||||
// 结果的滑动窗口。
|
||||
func (st *directRun) noteFailureOutcome(status string) {
|
||||
st.toolMu.Lock()
|
||||
defer st.toolMu.Unlock()
|
||||
if status == "error" {
|
||||
st.consecToolFailures++
|
||||
} else {
|
||||
st.consecToolFailures = 0
|
||||
st.failStreakNoted = false // 新失败链可再触发提醒
|
||||
st.recentOutcomes = append(st.recentOutcomes, status == "error")
|
||||
if len(st.recentOutcomes) > escalationWindow {
|
||||
st.recentOutcomes = st.recentOutcomes[len(st.recentOutcomes)-escalationWindow:]
|
||||
}
|
||||
}
|
||||
|
||||
func countFails(outcomes []bool) int {
|
||||
n := 0
|
||||
for _, failed := range outcomes {
|
||||
if failed {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -65,10 +65,10 @@ type directRun struct {
|
||||
// E4(R4 升级规则):harness 侧触发的换路提醒——不依赖模型自觉发现
|
||||
// 自己在原地打转。成败计数在 onEvent 终点统一维护(noteFailureOutcome),
|
||||
// 达阈值注入一次换路指令;检索连续空结果同理(noteEvidenceCall)。
|
||||
consecToolFailures int
|
||||
failStreakNoted bool
|
||||
zeroHitSearches int
|
||||
zeroHitNoted bool
|
||||
recentOutcomes []bool // E4(R4):最近 N 次工具成败(true=失败),滑动窗口
|
||||
failWindowNoted bool // E4(R4):窗口判据已提醒(每 run 一次)
|
||||
zeroHitSearches int
|
||||
zeroHitNoted bool
|
||||
// E4(结构化 source_id 账本):证据类工具的成功结果登记为带 ID 的
|
||||
// 条目,工具输出尾注【src_N|工具】标记——数字溯源的「材料/来源」
|
||||
// 从此有可指地址。
|
||||
|
||||
Reference in New Issue
Block a user