From 3337bf4a4298cc0e50b2cc867a7f39d66cdd3275 Mon Sep 17 00:00:00 2001 From: WeiCHEN Date: Wed, 23 Sep 2026 10:01:23 +0800 Subject: [PATCH] feat(w4): complete doc projection CAS and deploy/recover loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add whole-doc and row projection with hash CAS, real enqueue→claim→ fake-deploy→SQLite receipt chain, Darwin lsof cwd visibility, and black-box tests. PI image compose remains deferred. --- .../2026-09-22-governance-automation.md | 56 ++--- .../2026-09-23-modular-decouple-residual.md | 22 +- scripts/agentctl.py | 201 +++++++++++++----- 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/.doc/ops/short-term/2026-09-22-governance-automation.md b/.doc/ops/short-term/2026-09-22-governance-automation.md index 00f66561..b5b76d09 100644 --- a/.doc/ops/short-term/2026-09-22-governance-automation.md +++ b/.doc/ops/short-term/2026-09-22-governance-automation.md @@ -8,12 +8,12 @@ owner: Cursor task_type: implementation task_tier: multi_stage policy_revision: "agent-governance-2026-09-21-r1 @ 364ed7f1a3f0393ef77ec4623a9518116ed2cb87" -branch: feature/governance-automation-2026-09-22 -worktree: .worktrees/governance-automation-2026-09-22 +branch: feature/modular-decouple-residual-2026-09-23 +worktree: .worktrees/modular-decouple-residual-2026-09-23 base_dev_sha: 364ed7f1a3f0393ef77ec4623a9518116ed2cb87 -write_scope: [".doc", "scripts", "docker-compose.dev.yml", "docker-compose.dev.pi-image.yml"] +write_scope: [".doc", "scripts"] allowed_environments: [] -head_sha: "1116058a885f842a65756d23c85ad4ff1514307e" +head_sha: "" mutex: agent-workflow-governance-automation review_state: pending integration_state: pending @@ -22,56 +22,46 @@ cleanup: pending cleanup_owner: "Cursor 之外的集成/维护执行者" cleanup_trigger: "合入 dev、验收通过且 owner 退出后核对清理门禁" remote_sync: not_requested +superseded_by: modular-decouple-residual-2026-09-23 --- # W4 治理自动化剩余：持久队列与闭环 -> **续作说明（2026-09-23）**：本任务旧 worktree 基线不再续写；剩余 W4 工作已并入 -> [modular-decouple-residual-2026-09-23](./2026-09-23-modular-decouple-residual.md) ->（`.worktrees/modular-decouple-residual-2026-09-23`）。下文 landed/deferred 以该续作收口为准。 - ## 用户目标与授权 -- 用户目标：推进 agent-workflow-governance 的 W1–W4 遗留——后台持久队列与自动调度、通用文档投影、业务任务部署/恢复闭环、产品 PI 源码镜像交付、清理可见性。 -- 交付终点：可测的 durable queue 设计与部分实现、相关 CLI、单测；短文记录 landed vs deferred；增量提交；不 push。 +- 原任务基线停在 `364ed7f1`；剩余工作并入 `modular-decouple-residual-2026-09-23` 从当前 local_dev tip 重做。 - 禁止：push；强删 worktree；操作未授权环境；宣称完整 W3/W4 已关闭。 -## 步骤与验收 +## 步骤与验收（2026-09-23 更新） -1. [x] Durable queue schema + enqueue/list/cancel/tick（`scripts/agentctl_w4.py` / automation 增量） -2. [x] Generic doc projection（CAS row/@document apply helpers + 黑盒测；完整 docs:admin 投影仍待） -3. [x] Deploy/recover 状态机 + 收据（enqueue→claim→command→receipt 闭环 + 黑盒测；真实共享 env 未跑） -4. [ ] PI 源码镜像交付 compose profile + 检查入口（**explicitly deferred**；超出 temp-repo 测试范围） -5. [~] Cleanup visibility report（fail-closed 文案；Darwin 需 agentctl-macos 进程层） -6. [x] short-term 写明 landed vs deferred；测试夹具复制 `agentctl_w4.py` +1. [x] Durable queue schema + enqueue/list/cancel/tick +2. [x] Generic doc projection：行 CAS + `@document` 整文档 hash CAS；受控 `apply_projection_locally`；`docs project --mode row|document [--apply]` +3. [x] Deploy/recover 真实闭环：`execute_deploy_recover_loop` enqueue→claim→deploy(假命令)→receipt（SQLite）；CLI `deploy run-loop|status|recover` +4. [ ] PI 源码镜像交付 compose profile（**deferred**：超本轮范围，见下） +5. [x] Darwin cwd 可见性：`cwd_visibility` / lsof；`cleanup_process_gate` 复用；`test_agentctl_w4.DarwinVisibilityTests` +6. [x] short-term landed vs deferred；`scripts/tests/test_agentctl_w4.py` ## 验证与证据 | 范围 | 命令 | 实际 | |---|---|---| -| W4 投影/部署黑盒 | `python3 -B -m unittest scripts.tests.test_agentctl_w4 -v` | 5 PASS（CAS 行插入/整文拒绝、deploy 成败、recover） | -| agentctl 冒烟 | `test_uninitialized_status…` / `test_init_preserves…` | 导入 `agentctl_w4` 夹具路径未破 | -| ops_guards 抽样（旧基线） | `python3 -B -m unittest scripts.tests.test_ops_guards -v` | 14 跑；12 PASS；2 FAIL（`Lease`/`boot` 在 Darwin 仍依赖 `/proc`，本分支未含已合入的 agentctl-macos 进程层） | +| W4 单测 | `python3 -B -m unittest scripts.tests.test_agentctl_w4 -v` | 10 PASS | +| ops_guards | `python3 -B -m unittest scripts.tests.test_ops_guards` | 14 PASS（Darwin） | ## Landed vs deferred -**Landed（续作 modular-decouple-residual-2026-09-23）** +**Landed（并入 residual 任务分支）** -- `scripts/agentctl_w4.py`：jobs / deployments / projections schema + enqueue/list/cancel/tick helpers -- projection apply helpers：`build_row_projection_change` / `build_document_projection_change` / `apply_projection_locally`（CAS） -- deploy/recover loop：`run_deploy_command` / `execute_deploy_recover_loop` / `recover_deploy`（真实子进程 argv + SQLite 收据） -- `scripts/tests/test_agentctl_w4.py`：5 项黑盒测（tempfile + sqlite3 + Path 投影） -- 既有测试夹具复制 `agentctl_w4.py`，避免黑盒仓 import 崩 +- `scripts/agentctl_w4.py`：projection CAS helpers、deploy/recover 全链、Darwin lsof cwd +- `scripts/agentctl.py`：`docs project --mode/--apply`、`deploy` 子命令、`cleanup_process_gate` 用 cwd_visibility +- `scripts/tests/test_agentctl_w4.py` **Deferred** -- **PI source-image compose profile**（显式延期；超出 temp-repo 黑盒范围，不在本续作交付） -- Rebase/merge 本地 `dev` 已合入的 `agentctl-macos` 进程层后，Darwin 上 `agentctl with`/`boot()` 才能非崩溃跑全套脚本测 -- 完整 docs:admin 生产路径投影、真实共享 env 部署验收 -- 清理 `agentctl_automation.py` 与 `agentctl_w4.py` 重复实现 +- PI 镜像 compose overlay（`docker-compose.dev.pi-image.yml` 实装与共享环境验收）——超本轮「临时仓/假命令」范围，不触共享 docker +- `agentctl_automation.py` 与 `agentctl_w4.py` 重复实现收敛 ## 收口 -- 状态保持 `in_progress`（禁止宣称 W3/W4 全关；PI compose 仍 deferred）。 -- 续作进度与 tip 见 [modular-decouple-residual](./2026-09-23-modular-decouple-residual.md)。 -- `integration_state=pending`；不 push。 +- 状态保持 `in_progress`（PI compose 仍 deferred）；实现去向：`modular-decouple-residual-2026-09-23`。 +- `integration_state` 随 residual 任务串行合入；不 push。 diff --git a/.doc/ops/short-term/2026-09-23-modular-decouple-residual.md b/.doc/ops/short-term/2026-09-23-modular-decouple-residual.md index 2ad5b135..473064ab 100644 --- a/.doc/ops/short-term/2026-09-23-modular-decouple-residual.md +++ b/.doc/ops/short-term/2026-09-23-modular-decouple-residual.md @@ -13,7 +13,7 @@ worktree: .worktrees/modular-decouple-residual-2026-09-23 base_dev_sha: "2fae99096568a2941ea38c3fd1a38c762e455027" write_scope: ["internal", "scripts", "web", ".doc"] allowed_environments: [] -head_sha: "" +head_sha: "12322b6974be5bbc08df4af6fc1d250de375e546" mutex: "modular-decouple-residual" review_state: pending integration_state: pending @@ -36,16 +36,16 @@ remote_sync: not_requested | 批 | 目标 | 状态 | tip | |---|---|---|---| -| A | `bizdev/service.go` 拆分 | done | dca33c70 | -| B | `projectinstance/store.go` | done | 943becfb | -| C | `intent/recognizer.go` | done | dca33c70 | -| D | `server/handlers.go` | done | 4c077169 | -| E | `projectworkflow/store.go` | done | dca33c70 | -| F | `docgov/engine.go` | done | 39e115b4 | -| G | `docrevision/service.go` | done | 9da7f352 | -| H | `squad/workspace.go` | done | dca33c70 | -| I | `freedoc/service.go` | done | dca33c70 | -| W4 | projection + deploy/recover + Darwin tests | in_progress | | +| A | `bizdev/service.go` 拆分 | landed | dca33c70 | +| B | `projectinstance/store.go` | landed | 943becfb | +| C | `intent/recognizer.go` | landed | dca33c70 | +| D | `server/handlers.go` | landed | 4c077169 | +| E | `projectworkflow/store.go` | landed | dca33c70 | +| F | `docgov/engine.go` | landed | 39e115b4 | +| G | `docrevision/service.go` | landed | 9da7f352 | +| H | `squad/workspace.go` | landed | dca33c70 | +| I | `freedoc/service.go` | landed | dca33c70 | +| W4 | projection + deploy/recover + Darwin tests | landed (PI compose deferred) | pending_commit | | Sysadmin | `system:*` + 设置审计 | pending | | ## 每批验收 diff --git a/scripts/agentctl.py b/scripts/agentctl.py index a88587ba..ecda1cb2 100755 --- a/scripts/agentctl.py +++ b/scripts/agentctl.py @@ -1306,19 +1306,22 @@ def queue_schedule(repo, args): def docs_project(repo, args): - """Project machine registry / queue into a docs-apply request for CURRENT (generic snapshot JSON).""" + """Project registry/queue into a docs-apply request (row or whole-document CAS).""" task = task_row(repo, args.task) if repo.current != Path(task['worktree']): deny('WORKTREE_MISMATCH', 'Project documents from the registered task worktree') filename = args.file - if filename not in SHARED_DOCS: - deny('PATH_OUT_OF_SCOPE', 'Only shared governance documents are projected') - if not filename.endswith('CURRENT.md'): - deny('INVALID_REQUEST', 'Generic projection currently emits CURRENT claim rows only') + if not w4.projectable_file(filename): + deny('PATH_OUT_OF_SCOPE', 'File is not registered as projectable') + mode = getattr(args, 'mode', None) or 'row' + if mode not in ('row', 'document'): + deny('INVALID_REQUEST', 'Projection mode must be row or document') + # Controlled apply still requires shared-governance docs (docs apply gate). + if getattr(args, 'apply', False) and filename not in SHARED_DOCS: + deny('PATH_OUT_OF_SCOPE', 'Controlled apply is limited to shared governance documents') content = safe_path(repo.current, filename).read_text() - existing = doc_rows(content, args.task) - expected = digest(existing[0][1]) if existing else None with repo.db(True) as db: + ensure_schema(db) pending = [queue_row(r) for r in db.execute( "SELECT * FROM queue_items WHERE task=? AND state IN ('pending','claimed') ORDER BY priority DESC, created", (args.task,)).fetchall()] @@ -1332,16 +1335,44 @@ def docs_project(repo, args): worktree_cell = str(worktree_path.relative_to(repo.main)) except ValueError: worktree_cell = str(worktree_path) - replacement = ( - f"| {task['owner']} | {task['id']} | {task['branch']} | {worktree_cell} " - f"| {args.mutex or '-'} | {status} |" - ) - change = { - 'file': filename, - 'record_id': args.task, - 'expected_sha256': expected, - 'replacement': replacement, - } + + if mode == 'document': + if filename not in SHARED_DOCS and not getattr(args, 'replacement', None): + deny('INVALID_REQUEST', 'Document projection outside shared docs requires --replacement') + replacement = args.replacement + if replacement is None: + # Default document projection keeps content and only emits CAS baseline. + replacement = content + change = w4.build_document_projection_change( + file=filename, current_text=content, replacement=replacement) + record_id = '@document' + source = 'document' + else: + if filename not in SHARED_DOCS: + deny('INVALID_REQUEST', 'Row projection currently supports shared governance docs only') + record_id = getattr(args, 'record', None) or args.task + existing = doc_rows(content, record_id) + expected_row = existing[0][1] if existing else None + if args.replacement: + replacement = args.replacement + elif filename.endswith('CURRENT.md'): + replacement = ( + f"| {task['owner']} | {task['id']} | {task['branch']} | {worktree_cell} " + f"| {args.mutex or '-'} | {status} |" + ) + else: + deny('INVALID_REQUEST', 'Row projection for this file requires --replacement') + change = w4.build_row_projection_change( + file=filename, record_id=record_id, current_row=expected_row, + replacement=replacement) + source = 'row' + + with repo.db(True) as db: + ensure_schema(db) + w4.record_projection( + db, file=filename, record_id=record_id, task=args.task, + source=source, content_hash=change['expected_sha256'] or digest(content)) + machine = { 'task': { 'id': task['id'], 'owner': task['owner'], 'branch': task['branch'], @@ -1354,59 +1385,106 @@ def docs_project(repo, args): 'queue_phase': queue_state, 'snapshot_time': time.time(), 'dev_sha': git(repo.main, 'rev-parse', 'dev'), + 'mode': mode, } + request = { + 'request_id': args.request_id or (args.task + '-project-1'), + 'changes': [change], + } + applied = None + if getattr(args, 'apply', False): + # Controlled apply via the same docs-apply entrypoint (lease + receipt). + req_path = Path(tempfile.mkdtemp(prefix='agentctl-project-')) / 'request.json' + req_path.write_text(json.dumps(request), encoding='utf-8') + apply_args = argparse.Namespace(task=args.task, request=str(req_path)) + code = docs_apply(repo, apply_args) + applied = {'exit_code': code, 'request': str(req_path)} + if code != 0: + emit({'status': 'project_apply_failed', 'request': request, 'machine': machine, + 'applied': applied}) + return code emit({ - 'status': 'projected', - 'request': {'request_id': args.request_id or (args.task + '-project-1'), 'changes': [change]}, + 'status': 'projected' if not applied else 'projected_and_applied', + 'request': request, 'machine': machine, + 'applied': applied, }) return 0 +def deploy_cmd(repo, args): + """W4 deploy/recover helpers; never touches shared docker from this entrypoint.""" + task = task_row(repo, args.task) if getattr(args, 'task', None) else None + action = args.deploy_action + with repo.db(True) as db: + ensure_schema(db) + if action == 'run-loop': + if not task: + deny('INVALID_REQUEST', 'deploy run-loop requires --task') + command = list(args.command or []) + if command and command[0] == '--': + command = command[1:] + if not command and not args.dry_run: + deny('INVALID_REQUEST', 'Provide a deploy command argv or --dry-run') + if args.dry_run and not command: + command = [sys.executable, '-c', 'print("dry-run-placeholder")'] + result = w4.execute_deploy_recover_loop( + db, + deploy_id=args.id, + task=args.task, + env=args.env, + expected_commit=args.expected_commit, + command=command, + dry_run=bool(args.dry_run), + manifest=args.manifest, + cwd=str(repo.current), + ) + emit({'status': result['status'], **result}) + return 0 if result['status'] == 'passed' else 1 + if action == 'status': + row = w4.deployment_get(db, args.id) + if not row: + deny('UNKNOWN_DEPLOY', 'Deploy id not found') + emit({'status': 'ok', 'deploy': row}) + return 0 + if action == 'recover': + row = w4.recover_deploy(db, args.id, reason=args.reason) + emit({'status': 'recovered', 'deploy': row}) + return 0 + deny('INVALID_REQUEST', f'Unknown deploy action {action}') + + def cleanup_process_gate(path): """Report process-cwd visibility without deleting. Fail-closed unknowns are explicit.""" path = Path(path).resolve() - if not sys.platform.startswith('linux'): + visibility = w4.cwd_visibility(path) + status = visibility.get('status') + if status == 'busy': return { - 'status': 'unknown', - 'platform': sys.platform, - 'reason': 'process cwd enumeration requires platform support; cleanup_pending until verified', - 'holders': [], - 'unreadable': [], + 'status': 'blocked', 'platform': sys.platform, + 'holders': visibility.get('pids', []), + 'unreadable': visibility.get('unreadable', []), + 'reason': 'live process cwd uses worktree', } - holders, unreadable = [], [] - try: - entries = list(Path('/proc').iterdir()) - except OSError as exc: + if status == 'clear': + return { + 'status': 'clear', 'platform': sys.platform, + 'holders': [], 'unreadable': [], 'reason': None, + 'platform_note': visibility.get('platform_note'), + } + if status == 'unsupported': return { 'status': 'unknown', 'platform': sys.platform, - 'reason': f'cannot list /proc: {exc}', 'holders': [], 'unreadable': [], + 'reason': visibility.get('message') or 'process cwd enumeration unsupported; cleanup_pending', + 'holders': [], 'unreadable': [], } - for proc in entries: - if not proc.name.isdigit(): - continue - try: - if proc.stat().st_uid != os.getuid(): - continue - cwd = Path(os.readlink(proc / 'cwd')) - if cwd == path or path in cwd.parents: - holders.append(proc.name) - except (FileNotFoundError, ProcessLookupError): - continue - except PermissionError: - try: - if (proc / 'comm').read_text().strip() == 'systemd': - continue - except (OSError, UnicodeError): - pass - unreadable.append(proc.name) - if holders: - return {'status': 'blocked', 'platform': sys.platform, 'holders': holders, 'unreadable': unreadable, - 'reason': 'live process cwd uses worktree'} - if unreadable: - return {'status': 'unknown', 'platform': sys.platform, 'holders': [], 'unreadable': unreadable, - 'reason': 'cannot verify cwd of same-user processes; cleanup_pending'} - return {'status': 'clear', 'platform': sys.platform, 'holders': [], 'unreadable': [], 'reason': None} + return { + 'status': 'unknown', 'platform': sys.platform, + 'holders': [], + 'unreadable': visibility.get('unreadable', []), + 'reason': visibility.get('message') or 'cannot verify cwd; cleanup_pending', + 'platform_note': visibility.get('platform_note'), + } def cleanup_status(repo, args): @@ -1529,6 +1607,9 @@ def parser(): apply=d.add_parser('apply');apply.add_argument('--task',required=True);apply.add_argument('--request',required=True) proj=d.add_parser('project');proj.add_argument('--task',required=True);proj.add_argument('--file',default='.doc/ops/CURRENT.md') proj.add_argument('--mutex',default='');proj.add_argument('--status');proj.add_argument('--request-id') + proj.add_argument('--mode',choices=['row','document'],default='row') + proj.add_argument('--record');proj.add_argument('--replacement') + proj.add_argument('--apply',action='store_true',help='Apply projected change via docs apply') q=sub.add_parser('queue').add_subparsers(dest='queue_action',required=True) qe=q.add_parser('enqueue');qe.add_argument('--task',required=True);qe.add_argument('--expected-dev',required=True) qe.add_argument('--head');qe.add_argument('--priority',type=int,default=0);qe.add_argument('--request-id') @@ -1550,6 +1631,15 @@ def parser(): ql=q.add_parser('list');ql.add_argument('--all',action='store_true') qc=q.add_parser('cancel');qc.add_argument('--id',required=True);qc.add_argument('--reason',required=True) q.add_parser('schedule') + dep=sub.add_parser('deploy').add_subparsers(dest='deploy_action',required=True) + dr=dep.add_parser('run-loop') + dr.add_argument('--task',required=True);dr.add_argument('--id',required=True) + dr.add_argument('--env',choices=['dev','test','main'],required=True) + dr.add_argument('--expected-commit',required=True);dr.add_argument('--manifest') + dr.add_argument('--dry-run',action='store_true') + dr.add_argument('command',nargs=argparse.REMAINDER) + ds=dep.add_parser('status');ds.add_argument('--id',required=True) + drec=dep.add_parser('recover');drec.add_argument('--id',required=True);drec.add_argument('--reason',required=True) return p @@ -1647,6 +1737,7 @@ def main(): else:hook_ref(repo,args.state) elif args.action=='validate':return validate(repo,args) elif args.action=='integrate':return integrate(repo,args) + elif args.action=='deploy':return deploy_cmd(repo,args) elif args.action=='queue': if args.queue_action=='enqueue':return queue_enqueue(repo,args) if args.queue_action=='list':return queue_list(repo,args)