Files
Agent-System-Backend/internal/runmgr/recovery_hold_test.go
T
weichen 5cd67be982
ci / vet-and-test (pull_request) Successful in 41s
Close request atomicity: locked message append, card versions, atomic rate limit, recovery holds (U3 slice 1)
Why: the approved plan U3 (KTD3/KTD7/KTD10 plus minimal recovery isolation) requires concurrent submits to not silently overwrite, unknown-side-effect runs to never auto-resume, and the shared rate limiter to degrade conservatively instead of failing open.

Evidence: full backend suite green twice-relevant paths against real PostgreSQL 18.6 and Redis (isolated instance) and on SQLite; new tests - card_version_test (concurrent single winner), recovery_hold_test + recovery_holds_test (skip/release/fail-closed CRUD), ratelimit atomic-window and degraded-fallback tests.

Impact: AppendMessageWithRequest now allocates seq under the conversation advisory lock in one transaction; activity_cards gain version (v19) with SaveCardVersioned/GetCardVersion/CardVersion and cardsvc user mutations (status/pin/action/resolve-by-ref) return 409-mapped conflicts while system mirrors keep blind writes; recovery_holds (v20) with admin list/release endpoints and fail-closed resume skip; Redis limiter uses one Lua INCR+PEXPIRE script and falls back to a per-instance limit with a Degraded counter.

Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U3 KTD3/KTD7/KTD10; R3/R6/R7/R8.
2026-10-11 04:37:50 +00:00

123 lines
4.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package runmgr
import (
"os"
"path/filepath"
"testing"
"time"
"agent-system-backend/internal/store"
core "agent-system-core"
)
// 协作存储 U3:恢复隔离(recovery hold)。未知外部副作用的 run 不得被启动
// 续跑自动重派;人工核对后显式解除才恢复。判定失败同样跳过(fail-closed)。
// hold 中的子 run:续跑跳过;解除后续跑恢复。
func TestRecoveryHoldBlocksResumeUntilReleased(t *testing.T) {
runner := &gateRunner{parts: []core.Part{{Type: core.PartText, Text: "续跑结果"}}}
c, st := newSpawnEnv(t, runner)
c.mgr.runner = noopRunner{}
ws := t.TempDir()
c.mgr.SetWorkspaceRoot(ws)
parentRun := c.run
if err := st.CreateRun(&parentRun); err != nil {
t.Fatal(err)
}
child := core.Run{
ID: store.NewID("run"), ParentRunID: c.run.ID, ConversationID: c.run.ConversationID,
ProjectID: c.run.ProjectID, Owner: c.run.Owner, Kind: core.RunSubagent, ProfileID: c.run.ProfileID,
Status: core.RunRunning, CreatedAt: time.Now().UTC(),
}
if err := st.CreateRun(&child); err != nil {
t.Fatal(err)
}
c.mgr.writeSubagentRequest(child, "agent.sub.explore", "检索", "原始任务", "原始提示", true)
if _, err := st.SealInterrupted(child.ID); err != nil {
t.Fatal(err)
}
before, err := st.SubagentRunIDs(c.run.ConversationID)
if err != nil {
t.Fatal(err)
}
// run 级 hold:跳过。
if err := st.SetRecoveryHold(child.ID, child.Owner, string(child.ConversationID), "shell 命令超时,副作用结果未知"); err != nil {
t.Fatal(err)
}
c.mgr.ResumeInterruptedChildren()
ids, err := st.SubagentRunIDs(c.run.ConversationID)
if err != nil {
t.Fatal(err)
}
if len(ids) != len(before) {
t.Fatalf("hold 未解除时续跑应跳过:before=%v after=%v", before, ids)
}
// resumed.json 不得写出(下次解除后仍可续跑)。
dir := c.mgr.subagentDir(child.Owner, child.ConversationID, child.ID)
if _, err := os.Stat(filepath.Join(dir, "resumed.json")); !os.IsNotExist(err) {
t.Fatalf("hold 跳过不应写 resumed.json:%v", err)
}
holds, err := st.ListOpenRecoveryHolds()
if err != nil || len(holds) != 1 || holds[0].RunID != child.ID {
t.Fatalf("open holds=%v err=%v", holds, err)
}
// 解除(人工核对后):续跑恢复。
if err := st.ReleaseRecoveryHold(child.ID); err != nil {
t.Fatal(err)
}
c.mgr.ResumeInterruptedChildren()
ids, err = st.SubagentRunIDs(c.run.ConversationID)
if err != nil {
t.Fatal(err)
}
if len(ids) != len(before)+1 {
t.Fatalf("解除后续跑应新增子 run:before=%v after=%v", before, ids)
}
if held, herr := st.HasOpenRecoveryHold(child.ID, string(child.ConversationID)); held || herr != nil {
t.Fatalf("解除后不应再有 hold:held=%v err=%v", held, herr)
}
}
// 会话级 hold:同会话的其他中断子 run 一并跳过(副作用疑点未核对前,
// 整个会话的自动重派都冻结)。
func TestRecoveryHoldConversationScope(t *testing.T) {
runner := &gateRunner{parts: []core.Part{{Type: core.PartText, Text: "续跑结果"}}}
c, st := newSpawnEnv(t, runner)
c.mgr.runner = noopRunner{}
ws := t.TempDir()
c.mgr.SetWorkspaceRoot(ws)
parentRun := c.run
if err := st.CreateRun(&parentRun); err != nil {
t.Fatal(err)
}
child := core.Run{
ID: store.NewID("run"), ParentRunID: c.run.ID, ConversationID: c.run.ConversationID,
ProjectID: c.run.ProjectID, Owner: c.run.Owner, Kind: core.RunSubagent, ProfileID: c.run.ProfileID,
Status: core.RunRunning, CreatedAt: time.Now().UTC(),
}
if err := st.CreateRun(&child); err != nil {
t.Fatal(err)
}
c.mgr.writeSubagentRequest(child, "agent.sub.explore", "检索", "原始任务", "原始提示", true)
if _, err := st.SealInterrupted(child.ID); err != nil {
t.Fatal(err)
}
before, _ := st.SubagentRunIDs(c.run.ConversationID)
// hold 挂在「另一个」run 上、但同会话。
other := string(c.run.ConversationID)
if err := st.SetRecoveryHold("run_other", c.run.Owner, other, "会话存在未知副作用,整体冻结"); err != nil {
t.Fatal(err)
}
c.mgr.ResumeInterruptedChildren()
ids, _ := st.SubagentRunIDs(c.run.ConversationID)
if len(ids) != len(before) {
t.Fatalf("会话级 hold 应跳过续跑:before=%v after=%v", before, ids)
}
}