weichen
23f161bfb3
Why: F05 - files-presence could promote failures to success, recovery marked crashed steps completed on files alone or reran them blindly, and retries minted a fresh ExecutionID each attempt. Evidence: squad tests pass incl. executor result contract and recovery tables (verify-u3b.json); (nil,nil), canceled and side-effect-possible semantics handled; interrupted steps without a trusted terminal now hold for reconciliation instead of auto-advancing. Impact: retries reuse the step ExecutionID and shared deadline; no auto-replay of unverified external actions. Refs: feature/agent-stack-audit-remediation-b1 U3B
263 lines
9.2 KiB
Go
263 lines
9.2 KiB
Go
package squad
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"os"
|
||
"path/filepath"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
)
|
||
|
||
// jsonForTest 包装 json.Marshal 让测试里构型参数不拉锁。是仅为占用 import 走
|
||
// 公共路径，不污染生产代码。
|
||
func jsonForTest(v any) ([]byte, error) { return json.Marshal(v) }
|
||
|
||
// perStepRecordingBackend 按子工作区归因计数（steps 的 role→tasks/NN_role 布局）。
|
||
type perStepRecordingBackend struct {
|
||
mu sync.Mutex
|
||
calls map[string]int
|
||
}
|
||
|
||
func (b *perStepRecordingBackend) Name() string { return "recording" }
|
||
|
||
func (b *perStepRecordingBackend) Execute(ctx context.Context, req ExecRequest) (*ExecResult, error) {
|
||
b.mu.Lock()
|
||
if b.calls == nil {
|
||
b.calls = map[string]int{}
|
||
}
|
||
b.calls[filepath.Base(req.WorkDir)]++
|
||
b.mu.Unlock()
|
||
out := map[string][]byte{}
|
||
for _, t := range req.Targets {
|
||
out[t] = []byte(goodReport)
|
||
}
|
||
if _, err := MaterializeTargets(req.WorkDir, req.Targets, out); err != nil {
|
||
return &ExecResult{Success: false, ErrorReason: err.Error()}, nil
|
||
}
|
||
return &ExecResult{Success: true, Logs: "ok"}, nil
|
||
}
|
||
|
||
func (b *perStepRecordingBackend) countOf(sub string) int {
|
||
b.mu.Lock()
|
||
defer b.mu.Unlock()
|
||
return b.calls[sub]
|
||
}
|
||
|
||
func newAttemptTestEngine(t *testing.T, backend *perStepRecordingBackend, dir string) *SquadEngine {
|
||
t.Helper()
|
||
reg := NewRoleRegistry()
|
||
for _, role := range []string{"researcher", "writer", "reviewer"} {
|
||
if err := reg.Register(RoleSpec{Role: role, Backend: "recording"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
backends := NewBackendResolver()
|
||
backends.RegisterInstance("recording", backend)
|
||
eng, err := NewEngine(Options{
|
||
DataDir: dir, Registry: reg, Planner: mockScriptPlanner(baseSteps()...),
|
||
Backends: backends, Sink: NopSink{},
|
||
})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return eng
|
||
}
|
||
|
||
// flipToInterrupted 把该步骤手动置为 running，同时清除 trusted terminal
|
||
// record（模拟「崩溃于执行中」状态）。清除让该路径能够测试 F05 走的
|
||
// 「无 record」分支。
|
||
func flipToInterrupted(t *testing.T, workDir string, stepIdx int) {
|
||
t.Helper()
|
||
st, err := loadRunState(workDir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
st.Meta.Status = string(RunExecuting)
|
||
st.Meta.CompletedAt = nil
|
||
wf := st.Workflow()
|
||
wf.Status = "running"
|
||
wf.Steps[stepIdx].Status = "running"
|
||
wf.Steps[stepIdx].Attempts = 1
|
||
st.setWorkflow(wf)
|
||
if err := st.save(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// 同时清除该步骤的 trusted terminal record（模拟未达记录写入点崩溃）。
|
||
terminals := loadStepTerminalsForWorkDir(workDir)
|
||
if terminals != nil {
|
||
stepID := st.Plan.Workflow.Steps[stepIdx].ID
|
||
if _, ok := terminals[stepID]; ok {
|
||
delete(terminals, stepID)
|
||
data, _ := jsonForTest(terminals)
|
||
_ = os.WriteFile(filepath.Join(workDir, "step_terminals.json"), data, 0o644)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestAttemptIDFormat：attempt id 稳定落盘且形如 runID/stepID/N（Q1 WP3）。
|
||
func TestAttemptIDFormat(t *testing.T) {
|
||
backend := &perStepRecordingBackend{}
|
||
dir := t.TempDir()
|
||
eng := newAttemptTestEngine(t, backend, dir)
|
||
res, err := eng.Run(context.Background(), Goal{Text: "格式"}, RunOptions{})
|
||
if err != nil || res.Status != RunCompleted {
|
||
t.Fatalf("run: %v %+v", err, res)
|
||
}
|
||
if res.Plan == nil {
|
||
t.Fatal("no plan")
|
||
}
|
||
for _, s := range res.Plan.Steps {
|
||
want := res.RunID + "/" + s.ID + "/1"
|
||
if s.AttemptID != want {
|
||
t.Fatalf("step %s attempt id = %q, want %q", s.ID, s.AttemptID, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestKilledAtSuperstepBoundarySideEffectsOnce：杀进程于 superstep 边界后恢复，
|
||
// 已完成步骤不重新执行（执行次数=1）。
|
||
//
|
||
// F05/U3b：running 崩溃、文件齐全、却无 trusted terminal record（U3b 前被视为
|
||
// “副作用真发生”自动完成；U3b 后仅记录 step.Status=needs_reconciliation，
|
||
// 后台不再自动 rerun/完成。本测试验证该迁变后的语义不变量：
|
||
//
|
||
// - running 且无 record：需要 reconciliation，不隐默 rerun；
|
||
// - file presence 不替 trusted record 完成步骤。
|
||
// - 若事先写入 trusted record（手工合成），即便文件缺失也恢复为 completed。
|
||
func TestKilledAtSuperstepBoundarySideEffectsOnce(t *testing.T) {
|
||
backend := &perStepRecordingBackend{}
|
||
dir := t.TempDir()
|
||
eng := newAttemptTestEngine(t, backend, dir)
|
||
res, err := eng.Run(context.Background(), Goal{Text: "先跑完再中断", HitlMode: HitlAuto}, RunOptions{})
|
||
if err != nil || res.Status != RunCompleted {
|
||
t.Fatalf("setup run: %v %+v", err, res)
|
||
}
|
||
before := map[string]int{}
|
||
st, err := loadRunState(res.WorkDir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, s := range st.Plan.Workflow.Steps {
|
||
before[s.ID] = backend.countOf(filepath.Base(filepath.FromSlash(s.SubWorkspace)))
|
||
}
|
||
|
||
// 第二步停在 running（产物其实在盘上——首次运行真实写出的）。
|
||
flipToInterrupted(t, res.WorkDir, 1)
|
||
|
||
eng2 := newAttemptTestEngine(t, backend, dir)
|
||
res, err = eng2.Resume(context.Background(), res.RunID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// F05：需 reconciliation（auto 模式），不等于 completed。
|
||
if res.Status != RunWaitingReconciliation {
|
||
t.Fatalf("want waiting_reconciliation (auto mode), got %s", res.Status)
|
||
}
|
||
if res.Plan == nil || len(res.Plan.Steps) == 0 {
|
||
t.Fatal("missing plan")
|
||
}
|
||
var recon int
|
||
for _, s := range res.Plan.Steps {
|
||
if s.ID == "s2" && s.Status == StepStatusNeedsReconciliation {
|
||
recon++
|
||
}
|
||
}
|
||
if recon != 1 {
|
||
t.Fatalf("step s2 not in needs_reconciliation (got plan=%+v)", res.Plan.Steps)
|
||
}
|
||
// 重要：重复执行的次数不应发生隐默 rerun（本测试用 auto 模式运行走到 reconciliation）。
|
||
for _, s := range res.Plan.Steps {
|
||
got := backend.countOf(filepath.Base(filepath.FromSlash(s.SubWorkspace)))
|
||
if got != before[s.ID] {
|
||
t.Fatalf("step %s executed %d times after kill (before=%d)——running 步骤被隐默 rerun", s.ID, got, before[s.ID])
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestTrustedRecordCompletesMissingFiles：trusted terminal record 存在 + 文件缺失
|
||
// → 恢复为 completed（files 仍作诊断，state 是权威）。
|
||
func TestTrustedRecordCompletesMissingFiles(t *testing.T) {
|
||
backend := &perStepRecordingBackend{}
|
||
dir := t.TempDir()
|
||
eng := newAttemptTestEngine(t, backend, dir)
|
||
res, err := eng.Run(context.Background(), Goal{Text: "trusted record 完成", HitlMode: HitlAuto}, RunOptions{})
|
||
if err != nil || res.Status != RunCompleted {
|
||
t.Fatalf("setup run: %v %+v", err, res)
|
||
}
|
||
// 制造「第二步走 running 但 trusted record 已存在」的场景：先写入 record，再
|
||
// flip 到 running + 删 file。
|
||
st, err := loadRunState(res.WorkDir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sub2 := filepath.FromSlash(st.Plan.Workflow.Steps[1].SubWorkspace)
|
||
flipToInterrupted(t, res.WorkDir, 1)
|
||
if err := os.Remove(filepath.Join(res.WorkDir, sub2, "output.md")); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// 手工写入 trusted record（这能覆盖「后端状态已落盘但文件被外部删」该返回
|
||
// 已完成的合法语义）。
|
||
terminals := map[string]stepTerminalRecord{
|
||
"s2": {StepID: "s2", AttemptID: st.Plan.Workflow.Steps[1].AttemptID, Terminal: "completed", SideEffects: string(ExecSideEffectsNone), At: time.Now()},
|
||
}
|
||
data, _ := jsonForTest(terminals)
|
||
if err := os.WriteFile(filepath.Join(res.WorkDir, "step_terminals.json"), data, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
before := backend.countOf(filepath.Base(sub2))
|
||
eng2 := newAttemptTestEngine(t, backend, dir)
|
||
res, err = eng2.Resume(context.Background(), res.RunID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Status != RunCompleted {
|
||
t.Fatalf("want completed via trusted record, got %s", res.Status)
|
||
}
|
||
if got := backend.countOf(filepath.Base(sub2)); got != before {
|
||
t.Fatalf("trusted-record step reran (before=%d after=%d)——record 需被信任为 completed", before, got)
|
||
}
|
||
}
|
||
|
||
// TestInterruptedMissingProductsEntersReconciliation：F05/U3b：running 崩溃 + 无
|
||
// trusted record + 文件缺失 → auto 模式不再隐默 rerun，进入 waiting_reconciliation。
|
||
func TestInterruptedMissingProductsEntersReconciliation(t *testing.T) {
|
||
backend := &perStepRecordingBackend{}
|
||
dir := t.TempDir()
|
||
eng := newAttemptTestEngine(t, backend, dir)
|
||
res, err := eng.Run(context.Background(), Goal{Text: "先跑完再中断", HitlMode: HitlAuto}, RunOptions{})
|
||
if err != nil || res.Status != RunCompleted {
|
||
t.Fatalf("setup run: %v %+v", err, res)
|
||
}
|
||
flipToInterrupted(t, res.WorkDir, 1)
|
||
st, err := loadRunState(res.WorkDir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sub := filepath.FromSlash(st.Plan.Workflow.Steps[1].SubWorkspace)
|
||
if err := os.Remove(filepath.Join(res.WorkDir, sub, "output.md")); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
before := backend.countOf(filepath.Base(sub))
|
||
eng2 := newAttemptTestEngine(t, backend, dir)
|
||
res, err = eng2.Resume(context.Background(), res.RunID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res.Status != RunWaitingReconciliation {
|
||
t.Fatalf("want waiting_reconciliation, got %s", res.Status)
|
||
}
|
||
// 后续 Resume 仍旧是 reconciliation，不再 rerun。
|
||
res2, err := eng2.Resume(context.Background(), res.RunID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if res2.Status != RunWaitingReconciliation {
|
||
t.Fatalf("second resume shouldn't rerun, got %s", res2.Status)
|
||
}
|
||
if got := backend.countOf(filepath.Base(sub)); got != before {
|
||
t.Fatalf("missing-files step reran (before=%d after=%d)——record 缺失不应 rerun", before, got)
|
||
}
|
||
}
|