diff --git a/.doc/ops/CURRENT.md b/.doc/ops/CURRENT.md index 6c0ee311..5909601f 100644 --- a/.doc/ops/CURRENT.md +++ b/.doc/ops/CURRENT.md @@ -5,14 +5,16 @@ ## 焦点 -1. **Wave 0 done**:FEATURE-MATRIX 已合入 `dev`;下一步 Wave 1(IAM fail-closed + race) -2. **Wave 1 / backlog**:IAM fail-closed + race flake;多意图 U9–U12、FAV、PI 叙事、软件自举 -3. **运维**:夜间 CI `RUN_PI_E2E=1`;`promote-to-test` 避免 origin rebase;promotion receipt(Wave 2B) +1. **Wave 1**:IAM fail-closed(`authz`)+ TempDir race(`ci`) +2. **首发后 backlog**:多意图 U9–U12、FAV、PI 叙事、软件自举 +3. **运维**:夜间 CI `RUN_PI_E2E=1`;promotion receipt(Wave 2B) ## 并行认领 | owner | task | branch | worktree | mutex | status | |-------|------|--------|----------|-------|--------| +| Cursor | 2026-09-16-wave1-race-tempdir-cleanup | fix/race-tempdir-cleanup | - | ci | done | +| Cursor | 2026-09-16-wave1-iam-fail-closed | fix/iam-store-fail-closed | - | authz | planned | | Cursor | 2026-09-16-feature-matrix-wave0-calibration | feature/feature-matrix-production-readiness-plan | .worktrees/feature-matrix-production-readiness-plan | architecture-docs | done | | Codex | 2026-09-16-feature-matrix-production-readiness-plan | feature/feature-matrix-production-readiness-plan | .worktrees/feature-matrix-production-readiness-plan | architecture-docs | done | | Cursor-Auto | 2026-09-16-matrix-deep-test-promote-test | feature/matrix-deep-test-promote-test | - | infra | done | @@ -97,7 +99,7 @@ | holder | phase | since | scope | queue | |--------|-------|-------|-------|-------| -| - | idle | - | - | - | +| Cursor | merging | 2026-09-16 | Wave1 PU5A race TempDir merge | - | **phase**: `idle` | `merging` | `smoke` | `e2e` diff --git a/.doc/ops/short-term/2026-09-16-wave1-iam-fail-closed.md b/.doc/ops/short-term/2026-09-16-wave1-iam-fail-closed.md new file mode 100644 index 00000000..4ddd55dd --- /dev/null +++ b/.doc/ops/short-term/2026-09-16-wave1-iam-fail-closed.md @@ -0,0 +1,45 @@ +--- +title: "Wave 1 PU3 IAM store fail-closed" +status: planned +created: 2026-09-16 +updated: 2026-09-16 +owner: Cursor +branch: fix/iam-store-fail-closed +worktree: "-" +mutex: authz +--- + +# Wave 1 PU3 IAM store fail-closed + +## 目标 + +禁止非显式 opt-in 环境回退 MemoryStore;main/空值/缺表时启动 fail-closed。 + +## 范围 + +- `auth.NewStores` 双门:仅 `dev|test` + `AllowInMemory` +- characterization + 负例测试 +- 更新 FEATURE-MATRIX IAM 行关闭条件进度(fail-closed 子集) + +## 非目标 + +- 完整全路由 inventory 大重写(可增量;本波先关 MemoryStore 旁路) +- 真实 PI 补证(Wave 2) + +## 步骤 + +1. [ ] 从 dev 开分支并认领 +2. [ ] 先写负例测试再改 NewStores +3. [ ] 跑 auth/server 相关测试 +4. [ ] 文档与提交 + +## 验收标准 + +- [ ] main/空 env/缺表不得 MemoryStore 继续服务 +- [ ] 仅 dev/test + ALLOW_IN_MEMORY 可用内存 fixture +- [ ] 授权用户 happy path 不变 + +## 相关链接 + +- Remediation PU3: `../../remediation/2026-09-16-feature-matrix-production-readiness-plan.md` +- Wave 0 IAM 只读: `./2026-09-16-feature-matrix-wave0-calibration.md` diff --git a/.doc/ops/short-term/2026-09-16-wave1-race-tempdir-cleanup.md b/.doc/ops/short-term/2026-09-16-wave1-race-tempdir-cleanup.md new file mode 100644 index 00000000..095f2e9c --- /dev/null +++ b/.doc/ops/short-term/2026-09-16-wave1-race-tempdir-cleanup.md @@ -0,0 +1,48 @@ +--- +title: "Wave 1 PU5A TempDir race flake 修复" +status: done +created: 2026-09-16 +updated: 2026-09-16 +owner: Cursor +branch: fix/race-tempdir-cleanup +worktree: "-" +mutex: ci +--- + +# Wave 1 PU5A TempDir race flake 修复 + +## 目标 + +关闭全仓 `-race` 下 TempDir `RemoveAll` 与异步 taskrun worker 的竞态,证明 worker 在清理前已退出。 + +## 范围 + +- `taskrun.Manager` 增加 WaitRunDone / WaitIdle +- `newTestServer` cleanup 在 TempDir 删除前 drain workers +- 阻塞类用例显式 WaitRunDone(如需要) +- 多轮 `-race` 验证与 test-report + +## 非目标 + +- 夜间 PI / promotion receipt(PU5B) +- IAM fail-closed(PU3,独立分支) + +## 步骤 + +1. [x] 认领 CURRENT +2. [x] 实现 WaitIdle + newTestServer cleanup +3. [x] 修补已知阻塞用例 +4. [x] `go test -race` 多轮验证并写报告 +5. [x] 提交 + +## 验收标准 + +- [x] 相关包/全仓 race 多轮无 TempDir RemoveAll flake +- [x] 不用 skip/ignore 假绿 +- [x] 报告归档 `.doc/test-reports/` + +## 相关链接 + +- Remediation PU5A: `../../remediation/2026-09-16-feature-matrix-production-readiness-plan.md` +- Report: `../../test-reports/2026-09-16-wave1-race-tempdir-cleanup-test-report.md` +- Handoff: `../handoffs/2026-09-16-feature-matrix-wave0-calibration.md` diff --git a/.doc/test-reports/2026-09-16-wave1-race-tempdir-cleanup-test-report.md b/.doc/test-reports/2026-09-16-wave1-race-tempdir-cleanup-test-report.md new file mode 100644 index 00000000..21bf3d8b --- /dev/null +++ b/.doc/test-reports/2026-09-16-wave1-race-tempdir-cleanup-test-report.md @@ -0,0 +1,62 @@ +# Wave 1 PU5A TempDir race flake 测试报告 + +## 一、 测试背景 (Background) + +全仓 `go test -race` 曾出现 `t.TempDir()` 清理时 `RemoveAll: directory not empty`,根因是 `taskrun.Manager` 异步 worker 在测试返回后仍写 `DataDir`。已知基建债,不挡首发;本波按 continuity 计划 PU5A 独立关闭。 + +## 二、 测试目标 (Test Objectives) + +- Worker 在 TempDir 删除前退出(`WaitIdle` / `WaitRunDone`) +- `newTestServer` 与 `Server.Shutdown` 统一 drain +- 多轮 `-race` 无 TempDir flake;不用 skip/ignore 假绿 + +## 三、 测试方案与环境 (Test Plan & Environment) + +- OS: linux;Go race detector +- 变更:`internal/taskrun/manager.go` Wait API;`server_test.go` cleanup;`server.go` Shutdown;幂等 SSE 用例显式 wait +- 层级:单元(WaitIdle)+ server 包 race ×3(跳过真实 PI E2E)+ short race ×2(taskrun/server/auth) + +## 四、 预期结果 (Expected Results) + +- `TestManager_WaitIdle_WaitsForWorker` PASS +- `TestCreateTaskRun_PreActivationAcceptedNot409` / 幂等与 async 相关用例 race 下 PASS +- 输出中无 `TempDir` / `RemoveAll` / `directory not empty` / `DATA RACE` + +## 五、 实际测试结果与判定 (Actual Results & Evaluation) + +### 用例 1:WaitIdle 单元 + +- **测试输入 (Input)**:`go test -race -count=1 ./internal/taskrun/ -run TestManager_WaitIdle_WaitsForWorker -v` +- **实际输出 (Actual Output)**:`--- PASS: TestManager_WaitIdle_WaitsForWorker (0.05s)` +- **评价判定 (Evaluation / Verdict)**:PASS +- **性能与耗时 (Duration & Latency)**:~1s 包级 + +### 用例 2:server 包 race ×3(排除真实 PI) + +- **测试输入 (Input)**:`go test -race -count=3 ./internal/server/ -skip 'TestPiAgent_'` +- **实际输出 (Actual Output)**:`ok chat-agent-system/internal/server 357.605s`(全文见 assets) +- **评价判定 (Evaluation / Verdict)**:PASS(无 TempDir/DATA RACE 标记) +- **性能与耗时 (Duration & Latency)**:~358s + +### 用例 3:taskrun/server/auth short race ×2 + +- **测试输入 (Input)**:`go test -race -short -count=2 ./internal/taskrun/ ./internal/server/ ./internal/auth/` +- **实际输出 (Actual Output)**:三包均 `ok`(见 assets) +- **评价判定 (Evaluation / Verdict)**:PASS +- **性能与耗时 (Duration & Latency)**:~240s + +备注:未 skip 的 `TestPiAgent_FullLifecycle_*` 在本机曾因真实 runner `typed result rejected` 失败,属外部 PI 路径,与 TempDir flake 无关;PR CI 默认不跑真实 PI。 + +## 六、 自动化测试套件汇总 (Test Suite Execution) + +| 命令 | 结果 | +|------|------| +| `go test -race -count=3 ./internal/server/ -skip 'TestPiAgent_'` | PASS | +| `go test -race -short -count=2 ./internal/taskrun/ ./internal/server/ ./internal/auth/` | PASS | + +产物:`.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/` + +## 七、 测试结论与后续优化建议 (Conclusion & Next Steps) + +- PU5A TempDir race flake **关闭**(server/taskrun/auth 多轮 race 无该签名) +- 下一步:Wave 1 PU3 IAM fail-closed;PU5B 夜间 PI / receipt 另立 diff --git a/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/server-race-count3-skip-pi.txt b/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/server-race-count3-skip-pi.txt new file mode 100644 index 00000000..ea3d4423 --- /dev/null +++ b/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/server-race-count3-skip-pi.txt @@ -0,0 +1 @@ +ok chat-agent-system/internal/server 357.605s diff --git a/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/taskrun-server-auth-race-short-count2.txt b/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/taskrun-server-auth-race-short-count2.txt new file mode 100644 index 00000000..39bd5ee0 --- /dev/null +++ b/.doc/test-reports/assets/2026-09-16-wave1-race-tempdir-cleanup/taskrun-server-auth-race-short-count2.txt @@ -0,0 +1,3 @@ +ok chat-agent-system/internal/taskrun 1.455s +ok chat-agent-system/internal/server 239.214s +ok chat-agent-system/internal/auth 20.428s diff --git a/internal/server/server.go b/internal/server/server.go index 6226816a..370f9674 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -372,6 +372,9 @@ func (s *Server) Start() error { // Shutdown 优雅停机 func (s *Server) Shutdown(ctx context.Context) error { + if s.taskRuns != nil { + _ = s.taskRuns.WaitIdle(ctx) + } if s.rt != nil { _ = s.rt.Stop(ctx) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 9beafd75..b0922b4b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strings" "testing" + "time" "chat-agent-system/internal/bizdev" "chat-agent-system/internal/chat" @@ -31,7 +32,16 @@ func newTestServer(t testing.TB) *Server { AdminPassword: "admin-test-pass", } svc := bizdev.MustNewService(t, cfg) - return NewServer(cfg, svc) + srv := NewServer(cfg, svc) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if mgr := srv.TaskRuns(); mgr != nil { + _ = mgr.WaitIdle(ctx) + } + _ = srv.Shutdown(ctx) + }) + return srv } func TestServer_Health(t *testing.T) { diff --git a/internal/server/task_run_idempotent_sse_test.go b/internal/server/task_run_idempotent_sse_test.go index b4d40448..40bbcfdf 100644 --- a/internal/server/task_run_idempotent_sse_test.go +++ b/internal/server/task_run_idempotent_sse_test.go @@ -26,7 +26,17 @@ func TestCreateTaskRun_SameTurnIdempotentReturnsExistingDespiteBusyLease(t *test // 慢执行器:保持 run 在内存中可被 FindByIdempotencyKey 命中 block := make(chan struct{}) - t.Cleanup(func() { close(block) }) + var firstRunID string + t.Cleanup(func() { + close(block) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if firstRunID != "" { + _ = srv.taskRuns.WaitRunDone(ctx, firstRunID) + } else if srv.taskRuns != nil { + _ = srv.taskRuns.WaitIdle(ctx) + } + }) srv.taskRuns.SetExecutor( func(ctx context.Context, _ string, _ bizdev.TaskRunRequest, _ *stream.Streamer) (*bizdev.TaskRunResult, error) { select { @@ -70,6 +80,7 @@ func TestCreateTaskRun_SameTurnIdempotentReturnsExistingDespiteBusyLease(t *test if first.RunID == "" { t.Fatal("missing run_id") } + firstRunID = first.RunID hist1, err := srv.ctxMgr.GetChatHistory(ctx, "sess-idem-busy", 50, 0) if err != nil { diff --git a/internal/taskrun/manager.go b/internal/taskrun/manager.go index 21d86eee..4df35073 100644 --- a/internal/taskrun/manager.go +++ b/internal/taskrun/manager.go @@ -675,6 +675,49 @@ func (m *Manager) Cancel(runID string) error { return fmt.Errorf("run not found") } +// WaitRunDone 等待指定 run 的 worker 退出(runWorker 关闭 ar.done)。 +// 若 run 从未启动或已从内存表移除,立即返回 nil。 +func (m *Manager) WaitRunDone(ctx context.Context, runID string) error { + if m == nil || runID == "" { + return nil + } + m.mu.RLock() + ar, ok := m.runs[runID] + m.mu.RUnlock() + if !ok || ar == nil || ar.done == nil { + return nil + } + select { + case <-ar.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// WaitIdle 等待当前内存中全部 active run 的 worker 退出。 +func (m *Manager) WaitIdle(ctx context.Context) error { + if m == nil { + return nil + } + m.mu.RLock() + dones := make([]<-chan struct{}, 0, len(m.runs)) + for _, ar := range m.runs { + if ar != nil && ar.done != nil { + dones = append(dones, ar.done) + } + } + m.mu.RUnlock() + for _, done := range dones { + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + func (m *Manager) heartbeatLoop(ctx context.Context, runID string) { ticker := time.NewTicker(defaultHeartbeatPeriod) defer ticker.Stop() diff --git a/internal/taskrun/wait_idle_test.go b/internal/taskrun/wait_idle_test.go new file mode 100644 index 00000000..8bfd4ae5 --- /dev/null +++ b/internal/taskrun/wait_idle_test.go @@ -0,0 +1,62 @@ +package taskrun + +import ( + "context" + "testing" + "time" + + "chat-agent-system/internal/bizdev" + "chat-agent-system/internal/stream" +) + +func TestManager_WaitIdle_WaitsForWorker(t *testing.T) { + m := NewManager(nil, nil) + block := make(chan struct{}) + m.SetExecutor( + func(ctx context.Context, _ string, _ bizdev.TaskRunRequest, _ *stream.Streamer) (*bizdev.TaskRunResult, error) { + select { + case <-ctx.Done(): + return &bizdev.TaskRunResult{Success: false}, ctx.Err() + case <-block: + return &bizdev.TaskRunResult{Success: true}, nil + } + }, + nil, + nil, + ) + + rec := &RunRecord{ + RunID: "run-wait-idle", + SessionID: "sess", + UserID: 1, + Status: "pending", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + if _, err := m.CreateAndStart(context.Background(), rec.RunID, rec, bizdev.TaskRunRequest{}, bizdev.StageRunRequest{}, ""); err != nil { + t.Fatalf("CreateAndStart: %v", err) + } + + idleDone := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + idleDone <- m.WaitIdle(ctx) + }() + + select { + case err := <-idleDone: + t.Fatalf("WaitIdle returned before worker released: %v", err) + case <-time.After(50 * time.Millisecond): + } + + close(block) + select { + case err := <-idleDone: + if err != nil { + t.Fatalf("WaitIdle: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("WaitIdle timed out after releasing worker") + } +}