Merge authoritative main into the card-close branch
This commit is contained in:
@@ -373,3 +373,7 @@ jobs:
|
||||
AGENT_PG_TEST_DSN: postgres://ci_agent:ci_job_local_only@127.0.0.1:5432/agent_test?sslmode=disable
|
||||
AGENT_REDIS_TEST_ADDR: 127.0.0.1:6399
|
||||
run: make test-backend-pg
|
||||
- name: Two-process HTTP acceptance over shared PostgreSQL (U7, required)
|
||||
env:
|
||||
AGENT_PG_TEST_DSN: postgres://ci_agent:ci_job_local_only@127.0.0.1:5432/agent_test?sslmode=disable
|
||||
run: make test-multi-instance
|
||||
|
||||
@@ -6,6 +6,7 @@ __pycache__/
|
||||
/data-test/
|
||||
.zcode/plans/
|
||||
runtime/
|
||||
runtime-backups/
|
||||
.svc.env
|
||||
|
||||
# 运行时环境文件（含密钥，不入库；svc-envfile 任务约定）
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -13,7 +13,7 @@ PLUGINS := plugins
|
||||
COMBO := core=$(shell git -C $(CORE) rev-parse --short HEAD 2>/dev/null)+backend=$(shell git -C $(BACKEND) rev-parse --short HEAD 2>/dev/null)+frontend=$(shell git -C $(FRONTEND) rev-parse --short HEAD 2>/dev/null)+ui=$(shell git -C $(UI) rev-parse --short HEAD 2>/dev/null)+agentcmd=$(shell git -C deps/agent-command rev-parse --short HEAD 2>/dev/null)+documents=$(shell git -C modules/documents rev-parse --short HEAD 2>/dev/null)+native=$(shell git -C deps/agent-native rev-parse --short HEAD 2>/dev/null)+squad=$(shell git -C deps/agent-squad rev-parse --short HEAD 2>/dev/null)+intent=$(shell git -C deps/agent-intent rev-parse --short HEAD 2>/dev/null)
|
||||
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null)+$(COMBO)
|
||||
|
||||
.PHONY: help sync status build frontend svc svc-list svc-info svc-commits svc-history svc-update svc-recover svc-watchdog test-backend test-backend-pg test-frontend test-ui test-core test-agentcmd test-pins test-libs dev pin clean setup precheck task-start task-sync task-close task-finish facts-verify evidence version
|
||||
.PHONY: help sync status build frontend svc svc-list svc-info svc-commits svc-history svc-update svc-recover svc-watchdog test-backend test-backend-pg test-multi-instance test-frontend test-ui test-core test-agentcmd test-pins test-libs dev pin clean setup precheck task-start task-sync task-close task-finish facts-verify evidence version
|
||||
|
||||
version:
|
||||
@echo "agent-system $(VERSION)"
|
||||
@@ -112,6 +112,14 @@ test-backend-pg:
|
||||
@test -n "$${AGENT_REDIS_TEST_ADDR}" || { echo "test-backend-pg: AGENT_REDIS_TEST_ADDR 未设置（真实 Redis 必需，不静默跳过）"; exit 1; }
|
||||
cd $(BACKEND) && go test -count=1 ./...
|
||||
|
||||
# 双进程 HTTP 级验收（协作存储方案 U7）：两个真实 serve 进程共享隔离 PG。
|
||||
# 缺环境时明确失败而非静默跳过；见 tests/integration/multi-instance/README.md。
|
||||
test-multi-instance:
|
||||
@test -n "$${AGENT_PG_TEST_DSN}" || { echo "test-multi-instance: AGENT_PG_TEST_DSN 未设置（真实 PostgreSQL 必需，不静默跳过）"; exit 1; }
|
||||
mkdir -p tests/integration/multi-instance/.build
|
||||
cd $(BACKEND) && CGO_ENABLED=0 go build -o ../../tests/integration/multi-instance/.build/agent-system ./cmd/agent-system
|
||||
cd tests/integration/multi-instance && AGENT_BIN=./.build/agent-system go test -count=1 -timeout 15m .
|
||||
|
||||
eval-deliverables:
|
||||
cd $(BACKEND) && go test ./eval/ ./internal/quality/ -count=1
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# U1 数据归属清单 v0.1（R3"所有数据"的枚举基线）
|
||||
|
||||
状态：设计候选，随 U1 评审；配套设计见 [u1-identity-manifest.md](u1-identity-manifest.md)。主记录：方案 r2 第 3 节 R3 与第 6 节（提案在 feature/service-restart-recovery-plan 分支待合入，路径 `modules/documents/docs/proposals/2026-10-12-service-restart-data-recovery.md`）。标注规则：`已核实`＝SELF 源码观察（基线 Dev `89174a0` / Backend 同指针）；`待核对`＝实施 I2 时必须落定，未落定不得进入备份/恢复范围声明。
|
||||
本清单只回答"数据在哪里、谁是权威、怎么恢复"；耐久性参数（fsync/WAL 同步档）与备份窗口属 U6。
|
||||
|
||||
| # | 数据类别 | 权威位置 | 存储/机制 | 恢复方法 | 状态 |
|
||||
|---|---|---|---|---|---|
|
||||
| 1 | 团队业务库（账号/项目/成员/消息/文档/卡片/Surface 修订/设置/插件授权/run/step/命令/通知/审批/取消/额度/恢复 hold/事件） | PG 团队库（dbref 指向实例+库） | `modules/backend/internal/store/migrate.go` 表族（v17 pending_notices、run_commands、surface_revisions、recovery_holds、execution 等已核实存在） | PITR/pg_dump 逻辑备份（U6）；身份由 U1 DB marker 锚定 | 部分已核实，全表清单待核对 |
|
||||
| 2 | 个人档库（同 schema） | `<DataDir>/agent-system.db` | SQLite（app.go:196 已核实） | 在线一致性备份接口（现有）+ WAL 同档复制禁止 | 已核实 |
|
||||
| 3 | 工件对象（内容文件） | `<DataDir>/artifacts`（PG 档；app.go:179 已核实） | blob 临时写+rename（KTD11 对象化目标） | 文件+DB 元数据配对恢复，hash 对账 | 已核实路径，对象语义待 U6 |
|
||||
| 4 | 上传/输入附件 | `<DataDir>` 下上传子目录 | 同上 | 同上 | 路径待核对 |
|
||||
| 5 | 文档大内容/版本 | DB 行 +（如启用）内容文件 | documents 域 | DB 恢复+文件对账 | 待核对 |
|
||||
| 6 | 模型转录（Native Session） | 各工作区 `<sandbox>/.agentx/transcript.jsonl` | llm_direct.go:245 起播种/重建（已核实） | 随工作区恢复；revision 不符可由消息库重播种（已核实的 reseed 机制） | 已核实 |
|
||||
| 7 | 子代理派生请求 | 工作区内派生请求文件（writeSubagentRequest 已核实存在） | 落盘凭据 | 随工作区恢复 | 机制已核实，路径清单待核对 |
|
||||
| 8 | 执行工作区/overlay/checkpoint | 受管 workspace 目录 + filestate | KTD7 账本对象 | 工作区对账+副作用记录 | 待核对 |
|
||||
| 9 | 任务 Git 现场（worktree/未提交文件） | 各任务 worktree | git | 现场+基准 SHA 对账（U6/U7） | 待核对 |
|
||||
| 10 | 插件自有存储 | skillsRoot 与 sidecar dir（host_v2.go:18/24 已核实机制）；各启用插件自有 DB/目录 | 插件自定义 | 插件恢复适配器（r2 §3：必需插件不可恢复则阻塞全系统签收） | 机制已核实，逐插件清单待核对 |
|
||||
| 11 | RDM 等下游模块存储 | 模块自有目录/库 | 模块自定义 | 同上，纳入兼容清单后覆盖"所有数据"承诺 | 待核对 |
|
||||
| 12 | 配置与机密 | `runtime/<name>/svc.env`（迁移后 manifest.json+dsn_ref）、`.svc/<name>.json`、后端模型密钥（部署库） | 文件/表 | 受限配置备份（U6）；机密独立可恢复 | svc.env 已核实；密钥存储待核对 |
|
||||
| 13 | 派生状态（可重建，不作权威） | 检索/向量索引（collab U6 向量域）、Redis 缓存（如启用）、`runtime/<name>/history.jsonl` 事件史 | 索引/日志 | 从权威源重建（r2 §5.2 索引行）；Redis 可清空 | 部分待核对 |
|
||||
| 14 | 程序/资产组合 | bin 制品、Web 资产、插件二进制 | 不可变组合 | 组合 manifest+摘要（U1 §3 program 字段、U6 备份） | 已核实（deploy source registry） |
|
||||
|
||||
## 清单纪律
|
||||
|
||||
- 进入 U6 备份范围与 U7 签收的每一行，实施增量必须把该行"待核对"清零并附证据；无法证明可恢复的必需项按 r2 §3 阻塞全系统签收。
|
||||
- 新增插件/下游模块上线时必须同步登记本清单（守护测试在 U7 范围评估）。
|
||||
@@ -0,0 +1,84 @@
|
||||
# U1 实例与数据身份 manifest 合同（设计 v0.1）
|
||||
|
||||
状态：已批准（2026-10-12 用户“按建议办”，四未决项按 §8 建议定案），进入实施 I2。主记录：服务重启与全量数据恢复方案 r2（`modules/documents/docs/proposals/2026-10-12-service-restart-data-recovery.md` §7 U1 与 KTD1/KTD2；该提案落在本仓 feature/service-restart-recovery-plan 分支待推送合入，合入后即为上述路径）；本任务卡 `feature/recovery-u1-instance-identity`。授权：用户批准 r2（2026-10-12）并指示启动 U1；实施（改 `scripts/svc.sh` 与测试）待本设计评审通过。
|
||||
基线：Dev `89174a0`（含 svc-autorecover 合入：autostart 期望态、`recover`、watchdog、`.start.lock`）。事实来源 SELF·源码观察；本文所有"已核实"均可在该基线复核。
|
||||
|
||||
## 1. 目标与不变量
|
||||
|
||||
持久配置是启动权威，PID 与临时环境只是观察/覆盖。五条不变量对应 U1 验收：
|
||||
|
||||
- **I1 连续性**：连续裸启停（无参数）使用同一 DB 身份与数据/工件/工作区根。
|
||||
- **I2 非漂移**：只改端口/监听/程序版本/调用 cwd，不改变 DB、数据根与插件组合。
|
||||
- **I3 原子性**：配置提交是整份原子操作，任意 kill 窗口只可能读到完整旧版或完整新版。
|
||||
- **I4 阻塞不重建**：必需配置缺失、DB 身份不匹配或数据根 marker 丢失时拒绝启动，绝不新建空库/空目录掩盖。
|
||||
- **I5 机密引用化**：DSN/凭据不进 manifest 正文与日志，只存受保护文件引用。
|
||||
|
||||
## 2. 现状（基线事实）
|
||||
|
||||
- 每实例配置位于 `runtime/<name>/svc.env`（扁平 `key=value`，键含 kind/dbref/dsn/auth/exec/addr/autostart/config/source 等），`svc_get`/`svc_set` 逐键读写（已核实）。
|
||||
- `resolve_db_config()`（svc.sh ≈1373）统一解析 dbref/DSN/config，裸启停复用绑定，绑定失效拒绝启动（已核实）。
|
||||
- desired state `autostart` 已持久化于 svc.env 并被 recover/watchdog 消费（U2 首切片，PR #396/#401，已核实）。
|
||||
- 生命周期互斥 `.start.lock` 覆盖 start/stop/restart/deploy/update/recover（已核实）。
|
||||
- 缺口：逐键更新无原子性（kill 窗口可留中间态）；参数重建路径可漂移自定义 data/plugins；无配置 revision 与 CAS；数据根/DB 无身份 marker（"看起来是同一目录"≠同一身份）；首次 init 与常规 start 无入口区分；`dsn=` 明文存 svc.env。
|
||||
|
||||
## 3. manifest v1 结构
|
||||
|
||||
文件：`runtime/<name>/manifest.json`（权威）。svc.env 迁移后由 manifest 派生生成、仅供未改造消费方只读（见 §6）。
|
||||
|
||||
| 字段组 | 字段 | 说明 |
|
||||
|---|---|---|
|
||||
| 元 | `schema`(=1)、`revision`(int，单调)、`updated_at` | 整份 CAS 的版本轴 |
|
||||
| 身份 | `instance_id`(稳定，创建后不变)、`service_group`、`created_at` | R2 的"实例 ID" |
|
||||
| 期望态 | `desired`: `running`\|`stopped` | 吸收 autostart；U2 watchdog/recover 改读此字段（映射：autostart=1⇔running） |
|
||||
| 程序 | `program`: `{source, exec, digest}` | deploy source registry 的组合锚点 |
|
||||
| 数据库 | `db`: `{mode, dbref, dsn_ref, auth}` | `dsn_ref` 指向 0600 密钥文件路径，正文无 DSN |
|
||||
| 路径 | `paths`: `{data_root, artifacts_root, workspaces_root, plugins_dir}` | 绝对路径；I2 要求变更端口/程序不动这些 |
|
||||
| 监听 | `listen`: `{addr, port}` | 显式覆盖白名单字段 |
|
||||
| 配置 | `config`: `{file}` | 现 `.svc/<name>.json` 引用 |
|
||||
| 依赖 | `deps`: `{required:[], optional:[]}` | U2 依赖图输入 |
|
||||
| 恢复 | `recovery`: `{on_boot, watchdog}` | 策略位，U2/U7 消费 |
|
||||
| 身份锚 | `identity`: `{db_marker:{expected_instance_id}, data_marker:{path}}` | §5 校验目标 |
|
||||
| 机密 | `secrets`: `[{path, keys[], perms}]` | 引用清单，供 U6 备份范围 |
|
||||
|
||||
**v1.0 实施映射（I2，2026-10-12）**：`scripts/svc_manifest.py` 以「`identity` 锚点（data_root/db_mode/db_ref，冻结）+ `kv` 权威键值映射（svc.env 全量键，派生来源）+ `desired` 期望态」三件套承载上表全部不变量；§3 的完整 typed 分区（program/listen/config/deps/recovery/secrets）随 U2 依赖图与 U6 备份合同落地时扩展，扩展只增字段不改语义。`dsn_ref` 机密迁出同属 U6。
|
||||
|
||||
## 4. 写路径与并发
|
||||
|
||||
- 所有 manifest 写与 start/stop/restart/deploy/update/recover 在该实例的 `.start.lock` 内串行（复用 U2 已建锁，不新造锁）。
|
||||
- 提交流程：整份读改 → 校验 → 写 `manifest.json.new` → fsync 文件 → `rename` 覆盖 → fsync 目录 →（派生 svc.env 同法）。revision 在 candidate 内 +1；rename 前复核在位 revision，不一致则丢弃 candidate 重读重试（防御性 CAS）。
|
||||
- 禁止对 manifest 逐键写；迁移期兼容层 `svc_set` 语义改为"整份读-改-写"。
|
||||
|
||||
## 5. 解析顺序与启动校验
|
||||
|
||||
- 解析：CLI 显式覆盖（仅 `listen.*`、`program.source` 等白名单字段）> manifest > 环境变量（迁移后仅作显式 override 记录，不再隐式成为下次事实）。
|
||||
- 启动校验（全部通过才 exec）：
|
||||
1. schema/revision 合法且 revision ≥ 在位值；
|
||||
2. **DB marker**：PG 专用表 `agent_system_identity(instance_id, dbref, schema_version, created_at)`；SQLite 同表。首建仅发生在 `svc init`；常规启动要求 manifest.instance_id 与 marker 匹配，缺失/不匹配即阻塞（I4，对齐 resolve_db_config 现有"绑定失效拒绝启动"）。
|
||||
3. **数据根 marker**：`<data_root>/.agentx/data-marker.json`（instance_id + roots 清单摘要）；不匹配/丢失阻塞。
|
||||
4. `dsn_ref` 指向文件存在且权限 ≤0600。
|
||||
- 显式入口：`svc init`（首次：创建 manifest+双 marker，要求目标为空）；`svc restore`（灾难恢复：新目标装配 manifest+marker，走 U6 验证合同）。`start` 永不隐式 init。
|
||||
|
||||
## 6. 一次性迁移
|
||||
|
||||
lock → 读全部 svc.env 键 → 构造 manifest v1(rev=1) → §4 原子写 → 原 svc.env 更名 `svc.env.legacy`（只读留存）→ 派生新 svc.env。kill 窗口语义：只存在 legacy（未迁）或完整 v1+legacy（已迁），两者并存时以 manifest 为准并幂等补齐派生；无第三态。
|
||||
|
||||
## 7. 现状刻画回归（实施 I2 的前置测试计划）
|
||||
|
||||
`tests/test_svc_identity.py`，桩二进制+临时目录模式沿 `tests/test_svc_recover.py`：
|
||||
|
||||
| # | 场景 | 判据 |
|
||||
|---|---|---|
|
||||
| T1 | 裸 start→stop→start×2 | 同一 DB 标识与数据根（断言 svc.env/迁移后 manifest 不变、marker 不重建） |
|
||||
| T2 | 只改端口再启停 | dbref/data/plugins 组合逐字节不变 |
|
||||
| T3 | 单独 data 覆盖 | 按合同显式生效或整体拒绝，无半套路径 |
|
||||
| T4 | 迁移窗口注入 kill（写 .new 后 rename 前） | 下次启动只读到完整旧或完整新 |
|
||||
| T5 | 缺凭据/错 dbref/删 marker | 拒绝启动，退出码非 0，不产生新库/新 marker |
|
||||
| T6 | 更换 exec 路径与 cwd 调用 svc | 身份字段不变（对齐 r2 U1 验收新增条目） |
|
||||
|
||||
真实 PG 专项（marker SQL、跨库身份）单列隔离用例，沿用 test_svc_recover 的桩 http/健康探测 + 真实 pg 实例环境标记；缺环境时 NOT_RUN。
|
||||
|
||||
## 8. 验收映射与未决
|
||||
|
||||
r2 U1 验收五条 ↔ 本文：连续裸启停（§5/I1）、只改端口（I2/§3 paths）、data 覆盖（§5 白名单+T3）、迁移 kill（§6/T4）、缺凭据错库丢 marker（§5.2-4/T5）；外加程序/cwd（T6）。
|
||||
|
||||
已定案（2026-10-12“按建议办”）：(1) `instance_id` 生成规则建议 `svc-<group>-<随机12>` 创建即冻结；(2) dsn 明文迁 `dsn_ref` 的备份影响（U6 范围）；(3) manifest 与 `.svc/<name>.json` 应用配置的边界——本设计仅引用不合并；(4) Redis 等 optional 依赖在 deps 中的表达粒度。
|
||||
@@ -0,0 +1,59 @@
|
||||
# 双实例放行前检查单（协作存储 U7）
|
||||
|
||||
协作存储方案 §5 U7 的放行出口要求：**单实例团队、双进程、双主机分别出具
|
||||
结果；CI 身份/链接/SHA 或独立 REPLAY 可核验才用于放行，本地 SELF 只作辅助。**
|
||||
本检查单列出每一放行条件、其证据来源与当前状态；未全部满足前，第二实例
|
||||
不得承接写请求、双主机部署不得开工。
|
||||
|
||||
状态图例：✅ 已满足（证据可核验）｜⏳ 入口就绪、待执行窗口｜❌ 未开始。
|
||||
|
||||
## A. 正确性与故障矩阵（代码与 CI 可覆盖）
|
||||
|
||||
| # | 条件 | 证据（命令/测试） | 状态 |
|
||||
|---|---|---|---|
|
||||
| A1 | 单实例全链路（mock）：认证/项目/会话/SSE/检索/卡片 | backend `go test ./...`（CI job `test`，SQLite 档 35+ 包） | ✅ CI 绿 |
|
||||
| A2 | 真 PG/Redis 全量 | `make test-backend-pg`（CI job `test-pg`，AGENT_PG_TEST_DSN 必跑） | ✅ CI 绿 |
|
||||
| A3 | 磁盘满清楚拒绝、零残留、可恢复 | backend `TestUploadRejectsCleanlyWhenObjectDirUnwritable`、`TestArtifactOverwriteRejectsCleanlyWhenObjectDirUnwritable` | ✅ CI 绿 |
|
||||
| A4 | 恢复隔离（未知副作用不自动重跑） | backend `recovery_hold_test.go`（U3） | ✅ CI 绿 |
|
||||
| A5 | 库级多实例语义（争抢/围栏/命令/跨实例 resolve 不落卡） | backend `runmgr/multi_instance_test.go`（真库门控） | ✅ CI 绿 |
|
||||
| A6 | **双进程 HTTP 级验收**（跨实例读/SSE/409/HITL/取消/崩溃恢复/重启不变量） | `make test-multi-instance`（CI job `test-pg`；本地 `tests/integration/multi-instance/README.md`） | ✅ CI 绿 |
|
||||
| A7 | 权限矩阵重放（撤权零泄漏：请求/SSE/检索/下载/工具） | backend `resource_authz_test`、`shared_*_test`、`sse_gap_test`、`pgvector_test`（U3/U6） | ✅ CI 绿 |
|
||||
|
||||
## B. 容量与负载合同（需要实测窗口，本地入口已就绪）
|
||||
|
||||
| # | 条件 | 证据（命令） | 状态 |
|
||||
|---|---|---|---|
|
||||
| B1 | 冒烟压测通过（入口可复核、不变量零违反） | `tests/load/collaboration/run_load.sh --profile smoke` | ✅ 已跑通（SELF，2026-10-12：读 P95<1ms、SSE P95≈18ms、对账 0 违反） |
|
||||
| B2 | 阶梯负载 10→30→100 用户，读 P95≤300ms、P99≤1s、非预期失败≤0.1% | 同上 `--profile ladder`（k6 需安装：`brew install k6`） | ⏳ 待窗口 |
|
||||
| B3 | 60 分钟稳定负载 + 2 小时混合长稳 | `--profile sustained60m` / `--profile sustained2h` | ⏳ 待窗口（约 3h 机时） |
|
||||
| B4 | SSE 持久事件可见延迟 P95≤1s | 同上套件自动执行（sse_probe，断言即退出码） | ✅ 冒烟档已证机制；正式窗口随 B3 |
|
||||
| B5 | 数据量档（100 万消息/10 万知识分块/多项目交叉授权） | sustained 长跑自然积累 + seed 扩展（未实现 seed 工具，标注） | ❌ seed 工具未实施 |
|
||||
| B6 | 数据库基线（pgbench 只测 DB 能力，不替代业务验收） | `pgbench` 对隔离实例（只读参考） | ❌ 未执行 |
|
||||
| B7 | 容量数字绑定组件 SHA/配置/数据集归档 | results/<ts>/ 目录 + 报告引用 SHA | ⏳ 随 B2/B3 产出 |
|
||||
|
||||
## C. 双进程与双主机放行（分布式出口）
|
||||
|
||||
| # | 条件 | 证据 | 状态 |
|
||||
|---|---|---|---|
|
||||
| C1 | 双进程共享 PG：正确性矩阵全过（A6） | `make test-multi-instance` | ✅ |
|
||||
| C2 | 双进程**负载**下互相不拖累（热点会话/命令时延） | 双栈压测（run_load.sh 尚为单实例档，需扩展第二实例参数） | ❌ 入口未扩展 |
|
||||
| C3 | 双主机（跨机网络）验证：同套验收在两台主机执行 | A6 的 AGENT_BIN/DSN 指向远端；需用户指定第二主机 | ❌ 待资源 |
|
||||
| C4 | 故障演练：PG 断连/Redis 断连恢复（RTO≤15min） | backend ratelimit/recovery 测试 + 运维手册演练记录 | ⏳ 代码级 ✅；运维演练待窗口 |
|
||||
| C5 | 备份恢复演练（RPO≤5min/RTO≤60min，pg_dump+物理备份） | U4 迁移演练已过；定期备份恢复演练属运维，待排期 | ⏳ |
|
||||
|
||||
## 放行结论（当前）
|
||||
|
||||
- **单实例团队档**：A1–A7 全绿 + B1 冒烟通过。正式容量结论待 B2/B3 窗口；
|
||||
在此之前按「已验收功能、未承诺容量」运营。
|
||||
- **双进程档**：正确性已证（A5/A6/C1），负载与运维演练（C2/C4）未过，
|
||||
**不放行第二实例承接写请求**。
|
||||
- **双主机档**：未开始（C3），不放行。
|
||||
|
||||
## 已知边界与诚实声明
|
||||
|
||||
- U7 期间经双进程验收发现并修复三个跨实例缺陷（详见
|
||||
`tests/integration/multi-instance/README.md`）：HITL resolve 落卡短路命令、
|
||||
cancel 命令空会话 ID、CancelRun 假成功。修复均已入 CI 回归。
|
||||
- k6 首次冒烟使用临时二进制（v2.3.0）验证后删除；正式窗口前按
|
||||
`brew install k6` 装固定位。
|
||||
- B5 大数据量 seed 与 C2 双实例压测入口是下一步实施项，本检查单不替代。
|
||||
+26
@@ -583,6 +583,32 @@ scripts/svc.sh db backups pg1 # 列表（时间 / 大小 / 文
|
||||
- 完成后 `pg_restore --list` 自检归档有效性，失败即删除该文件并记 `backup fail`；
|
||||
- 备份**只增不删**；随 `rm --purge` 一并删除。
|
||||
|
||||
### 8.1.1 agent 数据目录快照（SQLite 一致性备份）
|
||||
|
||||
```bash
|
||||
scripts/svc.sh dev backup # 快照 data/ + svc.env + history.jsonl
|
||||
scripts/svc.sh dev backup --keep 3 # 备份后只保留最近 3 份
|
||||
scripts/svc.sh dev backup --out /mnt/bk/dev1 # 自定义快照目录（不参与自动清理）
|
||||
scripts/svc.sh dev backups # 快照列表（时间 / 大小 / 文件数）
|
||||
scripts/svc.sh dev backups --prune --keep 3 # 手动清理旧快照
|
||||
```
|
||||
|
||||
- 位置：`runtime-backups/<name>/<UTC时间戳>/`——**在 runtime/ 之外**。2026-10-11
|
||||
发生过 `runtime/` 被整体误删的事故（SQLite 聊天/设置数据全失，且无任何备份）；
|
||||
放在服务目录内的备份会跟着一起消失，因此快照根独立于 runtime/；
|
||||
- 一致性：每个 `*.db` 经 `sqlite3 .backup` 导出（并入 WAL 内容，服务运行中也可
|
||||
安全备份）并做 `PRAGMA integrity_check` 自检；`-wal`/`-shm` 不入快照。无 sqlite3
|
||||
时**拒绝备份**，不回退裸拷贝（裸拷会漏 WAL 得到撕裂副本）；
|
||||
- MANIFEST：快照内 `MANIFEST`（600）记录服务、来源版本与逐文件 sha256，恢复时
|
||||
可校验完整性。非 SQLite 的 `*.db` 文件（无文件头）保留原样副本并告警；
|
||||
- 保留策略：默认保留最近 7 份（`--keep N` 调整；只清理符合时间戳命名的快照目录，
|
||||
不动 `--out` 指定的自定义目录）；`rm <name> --purge` 会连带删除
|
||||
`runtime-backups/<name>/`，`rm`（不带 --purge）保留快照；
|
||||
- 边界：本命令是**数据面快照**，不含 restore（恢复到新目标、manifest 合同与
|
||||
外部效果对账属恢复方案 U6，见
|
||||
`modules/documents/docs/proposals/2026-10-12-service-restart-data-recovery.md`）；
|
||||
postgres 实例请用 `db backup`，static 服务无数据目录。
|
||||
|
||||
### 8.2 恢复
|
||||
|
||||
```bash
|
||||
|
||||
+1
-1
Submodule modules/backend updated: e49f165d0b...52d75fd931
+1
-1
Submodule modules/documents updated: da59ccff04...bd5fb18519
+1
-1
Submodule modules/frontend updated: 7acfef0a1f...d01a3ebd81
+282
-15
@@ -7,6 +7,8 @@
|
||||
# scripts/svc.sh <name> [action] [参数...] 服务级：deploy(默认)|start|stop|restart|status|logs [N]
|
||||
# ｜db [pg名|--clear]（agent 数据库绑定/切换）
|
||||
# ｜config [file|--clear]（agent 配置文件更新/生效）
|
||||
# ｜backup [--out D] [--keep N]（agent 数据目录快照）
|
||||
# ｜backups [--prune]（快照列表 / 按保留数清理）
|
||||
# scripts/svc.sh list [--tag T] 服务总览（status 等价）
|
||||
# scripts/svc.sh info [name] [--tag T] [--reveal] 综合报告（只读）：服务/数据库/配置快照/最近事件
|
||||
# scripts/svc.sh set <name> [--port N --apply] [--tag T] 变更登记（换端口/标签）
|
||||
@@ -26,6 +28,7 @@
|
||||
# scripts/svc.sh recover [name] [--quiet] 自恢复：把 autostart=1 但已死的服务重新拉起
|
||||
# （崩溃/OOM 被杀/重启后由 watchdog 定时调用）
|
||||
# scripts/svc.sh watchdog install|remove|status 自恢复看护：systemd 用户级 timer（每分钟 recover）
|
||||
# scripts/svc.sh <name> init 显式建立/查看实例身份（manifest+数据 marker；U1）
|
||||
# scripts/svc.sh help [主题] 分主题详细帮助
|
||||
#
|
||||
# ════════════════════ 部署来源（agent/static 的 deploy）════════════════════
|
||||
@@ -58,6 +61,7 @@
|
||||
# db backup：--db X --out F db restore：--file F --db X --force
|
||||
# db clone：--port P --host H --password PW --tag T --online
|
||||
# db copy：--db X --force set：--port N --host H --apply --tag T
|
||||
# backup：--out DIR --keep N backups：--prune --keep N（agent 数据目录快照）
|
||||
#
|
||||
# ════════════════════ 配置优先级 ════════════════════
|
||||
# 命令行 > --config/--env 文件 > svc.env 持久化（exec=/addr=/dbref=/dsn=/db=）
|
||||
@@ -76,6 +80,9 @@
|
||||
# - runtime/<name>/：bin|htdocs|data|pgdata|plugins|backups + svc.pid +
|
||||
# svc.log|pg.log + svc.env（登记+持久化配置，600）+ history.jsonl（事件流）
|
||||
# + .start.lock（生命周期互斥锁，recover 与 deploy/手动操作防竞态）
|
||||
# - runtime-backups/<name>/<UTC时间戳>/：agent 数据目录快照（svc.env+history+data；
|
||||
# *.db 经 sqlite3 .backup 一致性导出并自检，MANIFEST 记逐文件 sha256。放 runtime/
|
||||
# 之外——runtime 被整体误删时快照仍在，rm --purge 随服务一并删除）
|
||||
# - kind：agent（后端单二进制，内嵌前端）/ static（前端静态服务）/ postgres（数据库）；
|
||||
# 无 kind= 的旧登记自动视为 agent；start/restart 使用环境内产物
|
||||
#
|
||||
@@ -88,6 +95,7 @@
|
||||
# scripts/svc.sh ui-qa deploy --kind static --dir ../some/dist
|
||||
# scripts/svc.sh set pg1 --port 8895 --apply # 换端口并重启生效
|
||||
# scripts/svc.sh db backup pg1 && scripts/svc.sh db backups pg1
|
||||
# scripts/svc.sh dev backup && scripts/svc.sh dev backups # agent 数据目录快照 + 列表
|
||||
# scripts/svc.sh db clone pg1 pg2 --port 8896 # 物理克隆一套到新端口
|
||||
# scripts/svc.sh dev db pg1 --db dev # 把 dev 服务绑定到 pg1/dev 库（自动重启）
|
||||
# scripts/svc.sh dev db pg2 --no-restart # 切换但不重启（下次 start 生效）
|
||||
@@ -104,7 +112,7 @@
|
||||
# make svc ENV=dev ARGS="--env .env.dev --port 9000"
|
||||
#
|
||||
# ════════════════════ 安全约束 ════════════════════
|
||||
# - 只写 runtime/<name>/、runtime/.worktrees/ 与独立部署源仓（SVC_SRC_DIR，默认
|
||||
# - 只写 runtime/<name>/、runtime/.worktrees/、runtime-backups/<name>/ 与独立部署源仓（SVC_SRC_DIR，默认
|
||||
# ~/.cache/agent-system-svc/src，可随时删除重建）；不改主检出 git 状态、不动他人现场；
|
||||
# --from local 是唯一写主检出构建产物（bin/、node_modules/、dist/，均 gitignore）
|
||||
# 的路径，dev@sha/release/static 构建全部在隔离 worktree（自部署源仓物化）；
|
||||
@@ -112,8 +120,10 @@
|
||||
# pg_ctl -m fast 优雅停机，SIGKILL 不用于 postmaster；
|
||||
# - 端口被非本脚本进程占用时报错并给出占用者，不抢不杀；
|
||||
# - svc.env/备份文件权限 600；DSN 默认掩码密码（--reveal 才显示）；
|
||||
# - 备份/克隆只增不删；restore/copy 覆盖非空库必须显式 --force（drop 重建再恢复）；
|
||||
# 备份文件在 runtime/<name>/backups/，随 rm --purge 一并删除；
|
||||
# - 备份/克隆只增不删（backup --keep / backups --prune 的显式保留策略除外）；
|
||||
# restore/copy 覆盖非空库必须显式 --force（drop 重建再恢复）；
|
||||
# pg 备份在 runtime/<name>/backups/，agent 快照在 runtime-backups/<name>/（在
|
||||
# runtime/ 之外，防整删事故连带），均随 rm --purge 一并删除；
|
||||
# - 物理克隆期间源实例短暂停止（一致性要求），完成后自动复启；--online 不停源。
|
||||
#
|
||||
# 完整文档：docs/svc.md
|
||||
@@ -158,6 +168,8 @@ svc.sh —— 本机服务管理：后端(agent) / 前端(static) / 数据库(po
|
||||
scripts/svc.sh <name> start|stop|restart|logs [N] 生命周期与日志
|
||||
scripts/svc.sh <name> status 服务详情（版本/来源/DSN 等）
|
||||
scripts/svc.sh <name> db [pg名|--clear] agent 数据库绑定/切换（缺省查看）
|
||||
scripts/svc.sh <name> backup [--out D] [--keep N] agent 数据目录快照（SQLite 一致性备份）
|
||||
scripts/svc.sh <name> backups [--prune] 快照列表 / 按保留数清理
|
||||
scripts/svc.sh <name> config [file|--clear] agent 配置文件更新/即时生效（缺省查看）
|
||||
scripts/svc.sh db create pg1 --port 8890 建数据库
|
||||
scripts/svc.sh db dsn|backup|backups|restore|clone|copy|ls|set pg1 ... 数据库运维
|
||||
@@ -216,6 +228,12 @@ update deploy stop start restart logs help info version。
|
||||
history.jsonl 事件流（append-only，一行一个 JSON）
|
||||
.start.lock 生命周期互斥锁（recover 与 deploy/手动 start 防竞态）
|
||||
|
||||
runtime-backups/<name>/<UTC时间戳>/
|
||||
agent 数据目录快照：svc.env + history.jsonl + data/（*.db 经 sqlite3 .backup
|
||||
导出一致性副本并入 WAL 内容，integrity_check 自检；-wal/-shm 不入快照；
|
||||
MANIFEST 记逐文件 sha256）。位于 runtime/ 之外——runtime 被整体误删时快照
|
||||
仍在；rm --purge 随服务一并删除。
|
||||
|
||||
svc.env 字段：
|
||||
kind agent|static|postgres（旧登记无此键视为 agent）
|
||||
env 服务名
|
||||
@@ -244,7 +262,7 @@ history.jsonl 事件（scripts/svc.sh history 查看）：
|
||||
recover ok|fail（watchdog/手动 recover 拉回死亡服务）
|
||||
stop ok|fail
|
||||
set 登记变更（addr 旧 → 新）
|
||||
backup ok|fail（文件与大小）
|
||||
backup ok|fail（pg：库/文件/大小；agent：快照目录/大小）
|
||||
restore ok|fail（库、文件、表数）
|
||||
clone ok|fail（物理/在线）
|
||||
copy ok|fail
|
||||
@@ -339,6 +357,10 @@ EOF
|
||||
scripts/svc.sh rm <name> 停止并移除登记（目录/数据/历史保留）
|
||||
scripts/svc.sh rm <name> --purge 连 runtime/<name>/ 整个目录一起删除
|
||||
|
||||
数据目录快照（agent；SQLite 一致性导出，详见 docs/svc.md）：
|
||||
scripts/svc.sh <name> backup [--out DIR] [--keep N] 快照到 runtime-backups/<name>/
|
||||
scripts/svc.sh <name> backups [--prune] [--keep N] 列表 / 按保留数清理旧快照
|
||||
|
||||
自动恢复（崩溃 / OOM 被杀 / 机器重启后自动拉回；详见 help recovery）：
|
||||
scripts/svc.sh recover [name] [--quiet] 把 autostart=1 但已死的服务重新拉起
|
||||
scripts/svc.sh watchdog install systemd 用户 timer 每分钟执行 recover
|
||||
@@ -668,7 +690,7 @@ addr_compose() {
|
||||
}
|
||||
|
||||
# ---- 名称与端口 ----
|
||||
RESERVED_ENVS=" status list add set db rm create dsn commits history update deploy stop start restart logs help info version recover watchdog "
|
||||
RESERVED_ENVS=" status list add set db rm create dsn commits history update deploy stop start restart logs help info version recover watchdog backup backups "
|
||||
|
||||
# valid_env NAME：服务名合法性（小写字母开头、a-z0-9-、≤16 位；保留字不可用）。
|
||||
valid_env() {
|
||||
@@ -815,6 +837,17 @@ record_state() {
|
||||
[ $# -gt 0 ] && printf '%s\n' "$@"
|
||||
} > "$tmp"
|
||||
chmod 600 "$tmp"
|
||||
# U1 身份：manifest 治理下的整份提交（bulk-set 走 temp→fsync→rename 原子链，
|
||||
# revision 递增，svc.env 由 manifest 派生）；失败保留旧配置，不产生半写态。
|
||||
if manifest_ok; then
|
||||
if python3 "$MANIFEST_PY" "$ENV_DIR" bulk-set < "$tmp"; then
|
||||
rm -f "$tmp"
|
||||
return 0
|
||||
fi
|
||||
c_err "[$ENV_NAME] manifest 提交失败，保留旧配置（详见上方错误）"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
mv -f "$tmp" "$ENV_DIR/svc.env"
|
||||
}
|
||||
|
||||
@@ -882,10 +915,55 @@ svc_unlock() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# svc_mark_autostart / svc_clear_autostart：期望运行态登记（成功 start 置 1，
|
||||
# 显式 stop 移除；进程自行死亡不动——recover 据此判断是否拉回）。
|
||||
svc_mark_autostart() { svc_set_key autostart 1 >/dev/null 2>&1 || true; }
|
||||
svc_clear_autostart() { svc_del_key autostart 2>/dev/null || true; }
|
||||
# ---- U1 实例身份 manifest（设计：docs/recovery/u1-identity-manifest.md，已批准）----
|
||||
# manifest.json 为启动权威（identity 锚点冻结数据根/数据库身份；kv 为权威键值；
|
||||
# desired 为期望态），svc.env 降为派生只读视图，未迁移实例沿用旧路径。
|
||||
MANIFEST_PY="$ROOT/scripts/svc_manifest.py"
|
||||
manifest_ok() {
|
||||
[ -f "$ENV_DIR/manifest.json" ] && command -v python3 >/dev/null 2>&1 && [ -f "$MANIFEST_PY" ]
|
||||
}
|
||||
# svc_manifest_migrate：锁内一次性迁移（幂等）。已有数据视为"领养"——按当前数据根
|
||||
# 建立 marker；数据根不存在或携带他实例 marker 时拒绝（不新建空系统掩盖丢数据），
|
||||
# 失败保持 legacy 治理继续运行。
|
||||
svc_manifest_migrate() {
|
||||
[ -f "$ENV_DIR/svc.env" ] && [ ! -f "$ENV_DIR/manifest.json" ] || return 0
|
||||
command -v python3 >/dev/null 2>&1 && [ -f "$MANIFEST_PY" ] || {
|
||||
c_warn "[$ENV_NAME] 跳过 manifest 迁移：需 python3 与 scripts/svc_manifest.py"
|
||||
return 0
|
||||
}
|
||||
local out rc=0
|
||||
out="$(python3 "$MANIFEST_PY" "$ENV_DIR" migrate 2>&1)" || rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
c_ok "[$ENV_NAME] manifest：$(printf '%s' "$out" | tail -1)"
|
||||
return 0
|
||||
fi
|
||||
if [ "$rc" -eq 4 ]; then
|
||||
# 数据根携带他实例 marker：身份治理冲突（如 manifest 丢失后试图重建）——
|
||||
# 中止而非降级 legacy（I4：丢身份是事件，禁止新建空系统掩盖）。
|
||||
c_err "[$ENV_NAME] 身份治理冲突，中止：$(printf '%s' "$out" | tail -1)"
|
||||
return 1
|
||||
fi
|
||||
c_err "[$ENV_NAME] manifest 迁移失败（保持旧治理）：$(printf '%s' "$out" | tail -1)"
|
||||
return 0
|
||||
}
|
||||
# svc_manifest_verify：启动前身份校验——数据 marker 存在且 instance_id 匹配；
|
||||
# 生效 --data / 数据库绑定与冻结锚点一致。失败即拒绝启动（KTD1/KTD2，I4）。
|
||||
svc_manifest_verify() {
|
||||
manifest_ok || return 0
|
||||
local out
|
||||
if ! out="$(python3 "$MANIFEST_PY" "$ENV_DIR" verify \
|
||||
--data "$ENV_DATA" --db-mode "${DB_MODE:-}" --db-ref "${DB_REF:-}" 2>&1)"; then
|
||||
c_err "[$ENV_NAME] 身份校验失败，拒绝启动：$(printf '%s' "$out" | tail -1)"
|
||||
c_err " 数据根/数据库绑定是冻结身份；换根/重绑须显式迁移或登记命令（docs/recovery/u1-identity-manifest.md §5）"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# svc_mark_autostart / svc_clear_autostart：期望运行态登记（成功 start 置 running，
|
||||
# 显式 stop 置 stopped；进程自行死亡不动——recover 据 svc.env 派生的 autostart 判断）。
|
||||
svc_mark_autostart() { manifest_ok && python3 "$MANIFEST_PY" "$ENV_DIR" set-desired running >/dev/null 2>&1 && return 0; svc_set_key autostart 1 >/dev/null 2>&1 || true; }
|
||||
svc_clear_autostart() { manifest_ok && python3 "$MANIFEST_PY" "$ENV_DIR" set-desired stopped >/dev/null 2>&1 && return 0; svc_del_key autostart 2>/dev/null || true; }
|
||||
|
||||
check_port_free() {
|
||||
local owner; owner="$(port_owner)"
|
||||
@@ -1832,6 +1910,9 @@ agent_do_start() {
|
||||
hosts="$(env_var "$ENV_NAME" ALLOWED_HOSTS)"
|
||||
[ -n "$insecure" ] && guard_env+=("AGENT_SYSTEM_INSECURE_LISTEN=$insecure")
|
||||
[ -n "$hosts" ] && guard_env+=("AGENT_SYSTEM_ALLOWED_HOSTS=$hosts")
|
||||
# U1 身份校验：manifest 治理下，数据 marker 与冻结锚点不符即拒绝启动（在一切
|
||||
# 副作用之前；登记不变更）。
|
||||
svc_manifest_verify || { hist_event start fail "" "manifest 身份校验失败"; return 1; }
|
||||
# 预检（快速拦截）：非回环地址且认证判定为关时提前拒绝，报错可操作——
|
||||
# 不等二进制 listen guard 以"进程即退"暴露；二进制守卫仍是权威。
|
||||
# 认证判定依据统一解析的最终有效 DSN 与 auth（含环境变量与配置文件来源），
|
||||
@@ -2261,10 +2342,14 @@ check_port_available() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# svc_set_key KEY VALUE：改写当前服务 svc.env 的单键（先读后写 + 临时文件原子替换，
|
||||
# 避免重定向截断与半写）。
|
||||
# svc_set_key KEY VALUE：改写当前服务 svc.env 的单键。manifest 治理下经 manifest
|
||||
# 整份提交（原子 + revision）；legacy 路径保持先读后写 + 临时文件原子替换。
|
||||
svc_set_key() {
|
||||
local key="$1" val="$2" rest tmp
|
||||
if manifest_ok; then
|
||||
python3 "$MANIFEST_PY" "$ENV_DIR" set "$key" "$val" && return 0
|
||||
c_warn "[$ENV_NAME] manifest set 失败，回退 legacy 单键写"
|
||||
fi
|
||||
[ -f "$ENV_DIR/svc.env" ] || { c_err "svc.env 不存在：$ENV_DIR"; return 1; }
|
||||
rest="$(grep -v "^$key=" "$ENV_DIR/svc.env" || true)"
|
||||
tmp="$ENV_DIR/svc.env.tmp.$$"
|
||||
@@ -2274,9 +2359,14 @@ svc_set_key() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# svc_del_key KEY：从当前服务 svc.env 删除单键（键不存在则无操作）。
|
||||
# svc_del_key KEY：从当前服务 svc.env 删除单键（键不存在则无操作）。manifest 治理下
|
||||
# 同样经 manifest 整份提交。
|
||||
svc_del_key() {
|
||||
local key="$1"
|
||||
if manifest_ok; then
|
||||
python3 "$MANIFEST_PY" "$ENV_DIR" del "$key" && return 0
|
||||
c_warn "[$ENV_NAME] manifest del 失败，回退 legacy 单键写"
|
||||
fi
|
||||
[ -f "$ENV_DIR/svc.env" ] || return 0
|
||||
{ grep -v "^$key=" "$ENV_DIR/svc.env" || true; } > "$ENV_DIR/svc.env.tmp"
|
||||
mv "$ENV_DIR/svc.env.tmp" "$ENV_DIR/svc.env"
|
||||
@@ -2871,6 +2961,150 @@ EOF
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---- agent 数据目录快照（SQLite 一致性备份；2026-10-12）----
|
||||
# 快照根在 runtime-backups/<name>/（runtime/ 之外）：2026-10-11 runtime/ 被整体
|
||||
# 误删的事故里，放在 runtime/<name>/backups/ 的任何东西都会一起消失。每个 *.db
|
||||
# 经 sqlite3 .backup 导出一致性副本（并入 WAL 内容）并 integrity_check 自检，
|
||||
# 不裸拷 .db（约束见 modules/documents/docs/proposals/
|
||||
# 2026-10-12-service-restart-data-recovery.md §6）；-wal/-shm 不入快照。
|
||||
BACKUP_KEEP_DEFAULT=7
|
||||
BACKUP_TS_GLOB='[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z'
|
||||
|
||||
agent_backup_root() { printf '%s/runtime-backups/%s' "$ROOT" "$ENV_NAME"; }
|
||||
|
||||
agent_backup_guard() { # 公共前置：agent 已登记（backups 列表也走这里）
|
||||
if [ "$SVC_KIND" = postgres ]; then
|
||||
c_err "[$ENV_NAME] postgres 用 db backup（pg_dump）；backup 是 agent 数据目录快照"
|
||||
exit 2
|
||||
elif [ "$SVC_KIND" != agent ]; then
|
||||
c_err "[$ENV_NAME] kind=$SVC_KIND 无数据目录，不适用 backup"
|
||||
exit 2
|
||||
fi
|
||||
[ -f "$ENV_DIR/svc.env" ] || { c_err "[$ENV_NAME] 未登记（无 svc.env）；先 deploy"; exit 2; }
|
||||
}
|
||||
|
||||
agent_backup_snapshot() { # $1=dest：svc.env + history.jsonl + data/（*.db 一致性化）
|
||||
local dest="$1" rel
|
||||
cp -p "$ENV_DIR/svc.env" "$dest/svc.env" || return 1
|
||||
[ -f "$ENV_DIR/history.jsonl" ] && cp -p "$ENV_DIR/history.jsonl" "$dest/history.jsonl"
|
||||
mkdir -p "$dest/data" || return 1
|
||||
# 先整树拷贝（非 db 内容的原样副本），再对 *.db 用一致性导出覆盖：cp 得到的
|
||||
# 撕裂 db/-wal 副本在下一步被丢弃替换，不影响最终一致性。
|
||||
cp -a "$ENV_DATA/." "$dest/data/" || return 1
|
||||
while IFS= read -r rel; do
|
||||
[ -f "$ENV_DATA/$rel" ] || continue
|
||||
case "$rel" in
|
||||
*"'"*|*'"'*)
|
||||
if head -c 15 "$ENV_DATA/$rel" | grep -qa '^SQLite format 3$'; then
|
||||
c_err "[$ENV_NAME] 路径含引号，无法走 sqlite 一致性导出：$rel"
|
||||
return 1
|
||||
fi
|
||||
continue ;;
|
||||
esac
|
||||
if sqlite3 "$ENV_DATA/$rel" ".backup '$dest/data/$rel.bk'" 2>/dev/null \
|
||||
&& [ "$(sqlite3 "$dest/data/$rel.bk" 'PRAGMA integrity_check;' 2>/dev/null)" = ok ]; then
|
||||
mv -f "$dest/data/$rel.bk" "$dest/data/$rel" || return 1
|
||||
chmod 600 "$dest/data/$rel"
|
||||
rm -f "$dest/data/$rel-wal" "$dest/data/$rel-shm" # .backup 已并入 WAL 内容
|
||||
else
|
||||
rm -f "$dest/data/$rel.bk" # 失败尝试的半成品不留在快照里
|
||||
if head -c 15 "$ENV_DATA/$rel" | grep -qa '^SQLite format 3$'; then
|
||||
c_err "[$ENV_NAME] SQLite 一致性备份/自检失败：$rel"
|
||||
return 1
|
||||
fi
|
||||
c_warn "[$ENV_NAME] $rel 不是 SQLite 库（无文件头），保留原样副本"
|
||||
fi
|
||||
done < <(cd "$dest/data" 2>/dev/null && find . -type f -name '*.db' | sort)
|
||||
return 0
|
||||
}
|
||||
|
||||
agent_backup_manifest() { # $1=dest：快照自描述（600）——逐文件 sha256 供恢复时校验
|
||||
local dest="$1"
|
||||
{
|
||||
printf 'service=%s\nkind=%s\ncreated=%s\nsource=%s\nversion=%s\ndata_dir=%s\n' \
|
||||
"$ENV_NAME" "$SVC_KIND" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
"$(svc_get source)" "$(svc_get version)" "$ENV_DATA"
|
||||
( cd "$dest" && find . -type f ! -name MANIFEST -print0 | sort -z | xargs -0 -r sha256sum )
|
||||
} > "$dest/MANIFEST"
|
||||
chmod 600 "$dest/MANIFEST"
|
||||
}
|
||||
|
||||
agent_backup_prune() { # $1=root $2=keep：只清理符合时间戳命名的快照目录（最新在前保留 keep 份）
|
||||
local root="$1" keep="$2" n=0 d
|
||||
[ -d "$root" ] || return 0
|
||||
while IFS= read -r d; do
|
||||
n=$((n + 1))
|
||||
[ "$n" -le "$keep" ] && continue
|
||||
rm -rf "$root/$d"
|
||||
c_info "[$ENV_NAME] 清理旧快照：$d（保留最近 $keep 份）"
|
||||
done < <(cd "$root" && find . -maxdepth 1 -mindepth 1 -type d -name "$BACKUP_TS_GLOB" | sort -r)
|
||||
return 0
|
||||
}
|
||||
|
||||
svc_agent_backup() { # [--out DIR] [--keep N]
|
||||
local out="" keep="$BACKUP_KEEP_DEFAULT"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--out) need_value "$@"; out="$2"; shift 2 ;;
|
||||
--keep) need_value "$@"; keep="$2"; shift 2
|
||||
case "$keep" in ''|*[!0-9]*) c_err "--keep 须为非负整数"; exit 2 ;; esac ;;
|
||||
*) c_err "用法：scripts/svc.sh <name> backup [--out DIR] [--keep N]（未知参数：$1）"; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
agent_backup_guard
|
||||
[ -d "$ENV_DATA" ] || { c_err "[$ENV_NAME] 数据目录不存在：$ENV_DATA"; exit 2; }
|
||||
command -v sqlite3 >/dev/null 2>&1 \
|
||||
|| { c_err "缺少 sqlite3——一致性备份必需，不回退裸拷贝（会漏 WAL 得到撕裂副本）"; exit 1; }
|
||||
local ts dest size
|
||||
ts="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
dest="${out:-$(agent_backup_root)/$ts}"
|
||||
case "$dest" in /*) ;; *) dest="$PWD/$dest" ;; esac
|
||||
[ ! -e "$dest" ] || { c_err "备份目标已存在：$dest"; exit 1; }
|
||||
mkdir -p "$dest" || { c_err "无法创建备份目录：$dest"; exit 1; }
|
||||
if ! agent_backup_snapshot "$dest" || ! agent_backup_manifest "$dest"; then
|
||||
rm -rf "$dest"
|
||||
hist_event backup fail "$(read_pid)" "dir=$dest"
|
||||
c_err "[$ENV_NAME] 备份失败，半成品已清理：$dest"
|
||||
exit 1
|
||||
fi
|
||||
chmod 700 "$dest" 2>/dev/null || true
|
||||
size="$(du -sh "$dest" 2>/dev/null | cut -f1)"
|
||||
hist_event backup ok "$(read_pid)" "dir=$dest size=$size"
|
||||
[ -n "$out" ] || agent_backup_prune "$(agent_backup_root)" "$keep"
|
||||
c_ok "[$ENV_NAME] 数据目录快照完成：$dest（$size；svc.env+history+data，db 已一致性自检）"
|
||||
}
|
||||
|
||||
svc_agent_backups() { # [--prune] [--keep N]
|
||||
local prune=0 keep="$BACKUP_KEEP_DEFAULT"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--prune) prune=1; shift ;;
|
||||
--keep) need_value "$@"; keep="$2"; shift 2
|
||||
case "$keep" in ''|*[!0-9]*) c_err "--keep 须为非负整数"; exit 2 ;; esac ;;
|
||||
*) c_err "用法：scripts/svc.sh <name> backups [--prune] [--keep N]（未知参数：$1）"; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
agent_backup_guard
|
||||
local root n=0
|
||||
root="$(agent_backup_root)"
|
||||
if [ -d "$root" ]; then
|
||||
while IFS= read -r _; do n=$((n + 1)); done \
|
||||
< <(cd "$root" && find . -maxdepth 1 -mindepth 1 -type d -name "$BACKUP_TS_GLOB")
|
||||
fi
|
||||
if [ "$n" -eq 0 ]; then
|
||||
c_info "[$ENV_NAME] 暂无快照（root=$root；scripts/svc.sh $ENV_NAME backup）"
|
||||
return 0
|
||||
fi
|
||||
c_info "[$ENV_NAME] 快照列表（$root，最新在前）"
|
||||
( cd "$root" && find . -maxdepth 1 -mindepth 1 -type d -name "$BACKUP_TS_GLOB" | sort -r ) \
|
||||
| while IFS= read -r d; do
|
||||
printf ' %s %6s %s\n' "${d#./}" "$(du -sh "$root/$d" | cut -f1)" \
|
||||
"$(find "$root/$d/data" -type f 2>/dev/null | wc -l) 个文件"
|
||||
done
|
||||
if [ "$prune" = 1 ]; then agent_backup_prune "$root" "$keep"; fi
|
||||
return 0
|
||||
}
|
||||
|
||||
cmd_set() {
|
||||
local name="${1:-}"
|
||||
[ -n "$name" ] || { c_err "用法：scripts/svc.sh set <name> [--port N] [--host H] [--apply] [--tag T...]"; exit 2; }
|
||||
@@ -3009,6 +3243,11 @@ cmd_rm() {
|
||||
rm -f "$ENV_DIR/svc.env" "$ENV_DIR/svc.pid"
|
||||
if [ "$CLI_PURGE" = 1 ]; then
|
||||
rm -rf "${ENV_DIR:?}"
|
||||
local bkroot="$ROOT/runtime-backups/$name"
|
||||
if [ -d "$bkroot" ]; then
|
||||
rm -rf "$bkroot"
|
||||
c_ok "[$ENV_NAME] 数据目录快照已删除：$bkroot"
|
||||
fi
|
||||
c_ok "[$ENV_NAME] 已注销并删除整个目录：$ENV_DIR（含数据与历史）"
|
||||
else
|
||||
c_ok "[$ENV_NAME] 已注销登记；目录保留（数据/历史在 $ENV_DIR；彻底删除用 --purge）"
|
||||
@@ -3885,12 +4124,13 @@ main() {
|
||||
# 动作词可省略（省略 = deploy）；flags 可直接跟随服务名。
|
||||
local action="deploy"
|
||||
case "${1:-}" in
|
||||
deploy|stop|start|restart|status|logs|db|config) action="$1"; shift ;;
|
||||
deploy|stop|start|restart|status|logs|db|config|backup|backups) action="$1"; shift ;;
|
||||
esac
|
||||
if [ "$action" = logs ] && [ "${1:-}" != "" ] && [[ "$1" =~ ^[0-9]+$ ]]; then
|
||||
CLI_LOGS_N="$1"; shift
|
||||
fi
|
||||
# db / config 子命令：自带参数解析（位置参数不走 parse_args；未知 flags 由其自行拒绝）。
|
||||
# db / config / backup / backups 子命令：自带参数解析（位置参数不走 parse_args；
|
||||
# 未知 flags 由其自行拒绝）。
|
||||
if [ "$action" = db ] || [ "$action" = config ]; then
|
||||
env_config "$ENV_NAME"
|
||||
case "$action" in
|
||||
@@ -3899,6 +4139,14 @@ main() {
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
if [ "$action" = backup ] || [ "$action" = backups ]; then
|
||||
env_config "$ENV_NAME"
|
||||
case "$action" in
|
||||
backup) svc_agent_backup "$@" || exit $? ;;
|
||||
backups) svc_agent_backups "$@" || exit $? ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
parse_args "$@"
|
||||
if { [ -n "$CLI_FROM" ] || [ -n "$CLI_PICK" ] || [ "$CLI_DRYRUN" = 1 ] || [ -n "$CLI_DIR" ]; } && [ "$action" != deploy ]; then
|
||||
c_err "--from/--pick/--dry-run/--dir 仅用于 deploy 动作"
|
||||
@@ -3908,7 +4156,12 @@ main() {
|
||||
# 生命周期动作持 .start.lock（wait 模式；进程退出自动释放）——deploy 全程含
|
||||
# 构建窗口，防 recover/并发 start 在停机-换产物间隙抢跑。只读动作不加锁。
|
||||
case "$action" in
|
||||
deploy|stop|start|restart) svc_lock wait || { c_err "[$ENV_NAME] 无法获取生命周期锁"; exit 1; } ;;
|
||||
deploy|stop|start|restart|init) svc_lock wait || { c_err "[$ENV_NAME] 无法获取生命周期锁"; exit 1; } ;;
|
||||
esac
|
||||
# U1 身份：锁内一次性迁移（幂等；含既有数据领养 marker）。归属冲突（rc=4）中止；
|
||||
# 其余失败保持 legacy 治理继续运行。
|
||||
case "$action" in
|
||||
start|restart|deploy|init) svc_manifest_migrate || exit 1 ;;
|
||||
esac
|
||||
case "$action" in
|
||||
deploy)
|
||||
@@ -3920,6 +4173,20 @@ main() {
|
||||
stop) do_stop ;;
|
||||
start) do_start ;;
|
||||
restart) do_restart ;;
|
||||
init)
|
||||
# 显式建立/查看身份（manifest+数据 marker）。迁移钩子已先行：有 svc.env 无
|
||||
# manifest 时 migrate 已建好，此处只报告；无 svc.env 引导走首次 start。
|
||||
command -v python3 >/dev/null 2>&1 && [ -f "$MANIFEST_PY" ] || {
|
||||
c_err "[$ENV_NAME] init 需要 python3 与 scripts/svc_manifest.py"; exit 2; }
|
||||
if [ -f "$ENV_DIR/manifest.json" ]; then
|
||||
c_ok "[$ENV_NAME] 已在 manifest 治理下（init 幂等）："
|
||||
elif [ -f "$ENV_DIR/svc.env" ]; then
|
||||
python3 "$MANIFEST_PY" "$ENV_DIR" init \
|
||||
|| { c_err "[$ENV_NAME] init 失败（数据根冲突等，见上）"; exit 1; }
|
||||
else
|
||||
c_err "[$ENV_NAME] 尚无 svc.env：先用 start 完成首次登记；init 用于显式化已有实例"; exit 2
|
||||
fi
|
||||
python3 "$MANIFEST_PY" "$ENV_DIR" status ;;
|
||||
status) env_status_detail ;;
|
||||
logs) [ -f "$ENV_LOG" ] && tail -n "${CLI_LOGS_N:-50}" -f "$ENV_LOG" || c_warn "[$ENV_NAME] 暂无日志" ;;
|
||||
esac
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""svc 实例身份 manifest（U1）：manifest.json 为启动权威，svc.env 为派生只读视图。
|
||||
|
||||
设计合同：docs/recovery/u1-identity-manifest.md（已批准 2026-10-12）。
|
||||
落地说明：v1.0 以「identity 锚点（数据根/数据库身份，冻结）+ kv 权威键值映射 +
|
||||
desired 期望态」承载 KTD1/KTD2 的全部不变量；完整 typed 分区随 U2/U6 演进。
|
||||
|
||||
用法: svc_manifest.py ENV_DIR <migrate|init|status|set|del|bulk-set|set-desired|verify> [参数]
|
||||
|
||||
不变量（对应设计 §1）：
|
||||
I3 原子性 —— 整份 temp→fsync→rename→dirsync，任意时刻只有完整旧版或新版。
|
||||
I4 阻塞 —— manifest 存在时，数据 marker 缺失/身份不符即拒绝，不新建空系统。
|
||||
I5 机密 —— 本文件不新增任何机密；kv 沿袭 svc.env 既有内容（DSN 迁出属 U6）。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
|
||||
SCHEMA = 1
|
||||
MANIFEST = "manifest.json"
|
||||
DERIVED = "svc.env"
|
||||
MARKER_REL = os.path.join(".agentx", "data-marker.json")
|
||||
PROTECTED = ("db", "dbref", "dsn", "conffile") # 显式持久化写入可重锚；启动期漂移由 verify 拦截
|
||||
|
||||
|
||||
def die(msg, code=1):
|
||||
print("[manifest] %s" % msg, file=sys.stderr)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def now_iso():
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
|
||||
def atomic_write(path, text, mode=0o600):
|
||||
d = os.path.dirname(path) or "."
|
||||
os.makedirs(d, exist_ok=True)
|
||||
tmp = path + ".new"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.chmod(tmp, mode)
|
||||
os.replace(tmp, path)
|
||||
dfd = os.open(d, os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(dfd)
|
||||
finally:
|
||||
os.close(dfd)
|
||||
|
||||
|
||||
def read_kv(env_dir):
|
||||
kv = {}
|
||||
path = os.path.join(env_dir, DERIVED)
|
||||
if os.path.exists(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.rstrip("\n")
|
||||
if line and "=" in line and not line.startswith("#"):
|
||||
k, v = line.split("=", 1)
|
||||
kv[k] = v
|
||||
return kv
|
||||
|
||||
|
||||
def load(env_dir):
|
||||
path = os.path.join(env_dir, MANIFEST)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
m = json.load(f)
|
||||
except Exception as e: # noqa: BLE001 —— 损坏必须阻塞，不许猜测重建
|
||||
die("manifest 损坏（拒绝猜测，人工处置）：%s: %s" % (path, e), 2)
|
||||
if m.get("schema") != SCHEMA:
|
||||
die("manifest schema 不受支持：%s（当前 %s）" % (m.get("schema"), SCHEMA), 2)
|
||||
for field in ("revision", "instance_id", "identity", "kv", "desired"):
|
||||
if field not in m:
|
||||
die("manifest 缺少必需字段 %s（拒绝猜测）" % field, 2)
|
||||
return m
|
||||
|
||||
|
||||
def derive_text(m):
|
||||
kv = dict(m["kv"])
|
||||
if m["desired"] == "running":
|
||||
kv["autostart"] = "1"
|
||||
else:
|
||||
kv.pop("autostart", None)
|
||||
return "".join("%s=%s\n" % (k, v) for k, v in kv.items())
|
||||
|
||||
|
||||
def save(env_dir, m):
|
||||
atomic_write(os.path.join(env_dir, MANIFEST),
|
||||
json.dumps(m, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
|
||||
atomic_write(os.path.join(env_dir, DERIVED), derive_text(m))
|
||||
|
||||
|
||||
def data_root_of(kv, env_dir):
|
||||
kind = kv.get("kind", "agent")
|
||||
if kind == "static":
|
||||
return os.path.abspath(env_dir) # static 无数据目录语义，锚定实例目录本身
|
||||
parts = (kv.get("exec") or "").split()
|
||||
for i, w in enumerate(parts):
|
||||
if w == "--data" and i + 1 < len(parts):
|
||||
return os.path.abspath(parts[i + 1])
|
||||
return os.path.abspath(os.path.join(env_dir, "pgdata" if kind == "postgres" else "data"))
|
||||
|
||||
|
||||
def db_anchor_of(kv):
|
||||
mode = kv.get("db", "")
|
||||
if mode == "pg":
|
||||
return "pg", kv.get("dbref", "")
|
||||
if mode == "sqlite":
|
||||
return "sqlite", kv.get("dsn", "") or "default"
|
||||
return "", ""
|
||||
|
||||
|
||||
def marker_path(data_root):
|
||||
return os.path.join(data_root, MARKER_REL)
|
||||
|
||||
|
||||
def write_marker(data_root, instance_id):
|
||||
atomic_write(marker_path(data_root), json.dumps({
|
||||
"schema": 1,
|
||||
"instance_id": instance_id,
|
||||
"data_root": data_root,
|
||||
"created_at": now_iso(),
|
||||
}, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
def read_marker(data_root):
|
||||
path = marker_path(data_root)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception: # noqa: BLE001 —— marker 损坏等同身份不可证
|
||||
return {"schema": -1}
|
||||
|
||||
|
||||
def reanchor(env_dir, m):
|
||||
"""按显式持久化写入重算身份锚点；数据根变化时在新根补写 marker（目录存在才写）。"""
|
||||
ident = dict(m["identity"])
|
||||
ident["data_root"] = data_root_of(m["kv"], env_dir)
|
||||
ident["db_mode"], ident["db_ref"] = db_anchor_of(m["kv"])
|
||||
if ident != m["identity"]:
|
||||
if os.path.isdir(ident["data_root"]):
|
||||
write_marker(ident["data_root"], m["instance_id"])
|
||||
m["identity"] = ident
|
||||
return m
|
||||
|
||||
|
||||
def build(env_dir, kv, allow_create):
|
||||
kind = kv.get("kind", "agent")
|
||||
name = kv.get("env") or os.path.basename(os.path.abspath(env_dir))
|
||||
data_root = data_root_of(kv, env_dir)
|
||||
if not os.path.isdir(data_root):
|
||||
if not allow_create:
|
||||
die("数据根不存在且非显式 init，拒绝建立身份（防新建空系统掩盖丢数据）：%s" % data_root, 3)
|
||||
os.makedirs(data_root, exist_ok=True)
|
||||
existing = read_marker(data_root)
|
||||
if existing is not None:
|
||||
if existing.get("instance_id") != kv.get("instance_id"):
|
||||
die("数据根已携带其他实例 marker（instance_id=%s）：数据归属冲突，拒绝迁移/初始化，人工处置"
|
||||
% existing.get("instance_id"), 4)
|
||||
instance_id = existing["instance_id"]
|
||||
else:
|
||||
instance_id = kv.get("instance_id") or "svc-%s-%s" % (name, secrets.token_hex(6))
|
||||
write_marker(data_root, instance_id)
|
||||
db_mode, db_ref = db_anchor_of(kv)
|
||||
m = {
|
||||
"schema": SCHEMA,
|
||||
"revision": 1,
|
||||
"instance_id": instance_id,
|
||||
"service_group": name,
|
||||
"created_at": now_iso(),
|
||||
"desired": "running" if kv.get("autostart") == "1" else "stopped",
|
||||
"identity": {"data_root": data_root, "db_mode": db_mode, "db_ref": db_ref},
|
||||
"kv": {k: v for k, v in kv.items() if k != "autostart"},
|
||||
}
|
||||
save(env_dir, m)
|
||||
return m
|
||||
|
||||
|
||||
def cmd_migrate(env_dir):
|
||||
if load(env_dir) is not None:
|
||||
print("manifest 已存在，幂等跳过")
|
||||
return 0
|
||||
if not os.path.exists(os.path.join(env_dir, DERIVED)):
|
||||
die("svc.env 不存在：先完成首次登记（start）或使用 init", 3)
|
||||
m = build(env_dir, read_kv(env_dir), allow_create=False)
|
||||
print("已迁移 rev=1 instance_id=%s data=%s" % (m["instance_id"], m["identity"]["data_root"]))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_init(env_dir):
|
||||
if load(env_dir) is not None:
|
||||
die("manifest 已存在（init 仅用于显式建立；重建需人工处置）", 3)
|
||||
kv = read_kv(env_dir)
|
||||
m = build(env_dir, kv, allow_create=True)
|
||||
print("已建立 rev=1 instance_id=%s data=%s" % (m["instance_id"], m["identity"]["data_root"]))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(env_dir):
|
||||
m = load(env_dir)
|
||||
if m is None:
|
||||
print(json.dumps({"exists": False}, ensure_ascii=False))
|
||||
return 0
|
||||
print(json.dumps({
|
||||
"exists": True, "revision": m["revision"], "instance_id": m["instance_id"],
|
||||
"desired": m["desired"], "identity": m["identity"],
|
||||
}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def mutate(env_dir, fn):
|
||||
m = load(env_dir)
|
||||
if m is None:
|
||||
die("manifest 不存在：先 migrate/init", 3)
|
||||
before = json.dumps(m["identity"], sort_keys=True)
|
||||
fn(m)
|
||||
m["revision"] = int(m["revision"]) + 1
|
||||
m = reanchor(env_dir, m)
|
||||
save(env_dir, m)
|
||||
if before != json.dumps(m["identity"], sort_keys=True):
|
||||
print("[manifest] 身份锚点已随显式持久化写入更新（data=%s db=%s/%s）"
|
||||
% (m["identity"]["data_root"], m["identity"]["db_mode"], m["identity"]["db_ref"]))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_set(env_dir, key, value):
|
||||
return mutate(env_dir, lambda m: m["kv"].__setitem__(key, value))
|
||||
|
||||
|
||||
def cmd_del(env_dir, key):
|
||||
def drop(m):
|
||||
m["kv"].pop(key, None)
|
||||
return mutate(env_dir, drop)
|
||||
|
||||
|
||||
def cmd_bulk_set(env_dir):
|
||||
kv = {}
|
||||
for line in sys.stdin.read().splitlines():
|
||||
if line and "=" in line and not line.startswith("#"):
|
||||
k, v = line.split("=", 1)
|
||||
kv[k] = v
|
||||
kv.pop("autostart", None) # desired 独占期望态
|
||||
|
||||
def replace(m):
|
||||
m["kv"] = kv
|
||||
return mutate(env_dir, replace)
|
||||
|
||||
|
||||
def cmd_set_desired(env_dir, state):
|
||||
if state not in ("running", "stopped"):
|
||||
die("desired 取值仅 running|stopped", 2)
|
||||
return mutate(env_dir, lambda m: m.__setitem__("desired", state))
|
||||
|
||||
|
||||
def cmd_verify(env_dir, data_root=None, db_mode=None, db_ref=None):
|
||||
m = load(env_dir)
|
||||
if m is None:
|
||||
die("manifest 缺失：本实例已进入 manifest 治理，缺失即事件（先 init/人工处置，禁止静默重建）", 4)
|
||||
ident = m["identity"]
|
||||
marker = read_marker(ident["data_root"])
|
||||
if marker is None or marker.get("schema") == -1:
|
||||
die("数据 marker 缺失/损坏：data=%s（数据归属不可证即拒绝启动，防新建空库掩盖）"
|
||||
% marker_path(ident["data_root"]), 5)
|
||||
if marker.get("instance_id") != m["instance_id"]:
|
||||
die("数据 marker 与 manifest 身份不符：marker=%s manifest=%s（数据被换过或身份被改，人工处置）"
|
||||
% (marker.get("instance_id"), m["instance_id"]), 5)
|
||||
if data_root is not None and os.path.abspath(data_root) != ident["data_root"]:
|
||||
die("数据根漂移：生效 %s ≠ 锚定 %s；数据根是冻结身份，换根须显式迁移（docs/recovery/u1-identity-manifest.md §5）"
|
||||
% (os.path.abspath(data_root), ident["data_root"]), 6)
|
||||
if db_mode and ident["db_mode"] and db_mode != ident["db_mode"]:
|
||||
die("数据库模式漂移：生效 %s ≠ 锚定 %s" % (db_mode, ident["db_mode"]), 6)
|
||||
if db_mode and ident["db_mode"] and db_ref is not None and (db_ref or "") != ident["db_ref"]:
|
||||
die("数据库绑定漂移：生效 %s ≠ 锚定 %s（重绑须经显式登记命令）" % (db_ref or "<空>", ident["db_ref"]), 6)
|
||||
print("身份校验通过 instance_id=%s data=%s db=%s/%s"
|
||||
% (m["instance_id"], ident["data_root"], ident["db_mode"] or "-", ident["db_ref"] or "-"))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 3:
|
||||
die("用法: svc_manifest.py ENV_DIR <migrate|init|status|set|del|bulk-set|set-desired|verify> [参数]", 2)
|
||||
env_dir, cmd = argv[1], argv[2]
|
||||
args = argv[3:]
|
||||
if cmd == "migrate":
|
||||
return cmd_migrate(env_dir)
|
||||
if cmd == "init":
|
||||
return cmd_init(env_dir)
|
||||
if cmd == "status":
|
||||
return cmd_status(env_dir)
|
||||
if cmd == "set" and len(args) == 2:
|
||||
return cmd_set(env_dir, args[0], args[1])
|
||||
if cmd == "del" and len(args) == 1:
|
||||
return cmd_del(env_dir, args[0])
|
||||
if cmd == "bulk-set":
|
||||
return cmd_bulk_set(env_dir)
|
||||
if cmd == "set-desired" and len(args) == 1:
|
||||
return cmd_set_desired(env_dir, args[0])
|
||||
if cmd == "verify":
|
||||
data_root = db_mode = db_ref = None
|
||||
it = iter(args)
|
||||
for a in it:
|
||||
if a == "--data":
|
||||
data_root = next(it, None)
|
||||
elif a == "--db-mode":
|
||||
db_mode = next(it, None)
|
||||
elif a == "--db-ref":
|
||||
db_ref = next(it, None)
|
||||
return cmd_verify(env_dir, data_root, db_mode, db_ref)
|
||||
die("未知命令或参数不全：%s %s" % (cmd, " ".join(args)), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -0,0 +1,2 @@
|
||||
/.build/
|
||||
/data/
|
||||
@@ -0,0 +1,45 @@
|
||||
# 双进程 HTTP 级验收（协作存储 U7）
|
||||
|
||||
两个**真实 backend serve 进程**共享同一 PostgreSQL 的黑盒验收，覆盖方案 §7
|
||||
最小验收矩阵的进程级行（库级语义已在 backend `internal/runmgr/multi_instance_test.go`
|
||||
覆盖；本目录验证 HTTP/SSE/进程生命周期层面的同一套不变量）：
|
||||
|
||||
| 子场景 | 断言（矩阵行） |
|
||||
|---|---|
|
||||
| `cross_read` | A 建的项目/会话在 B 立即可见（共享权威） |
|
||||
| `cross_event_sse` | A 写入的事件在 B 的 SSE 可见（R5 跨实例传播） |
|
||||
| `same_session_conflict_then_cross_hitl` | 同会话并发提交明确 409 不误写；B 跨实例 resolve A 的 HITL 经命令总线到达执行者（R3/R4） |
|
||||
| `cross_cancel` | B 跨实例取消 A 的 run：命令持久、唯一消费、run 收口 cancelled（R4） |
|
||||
| `owner_crash_takeover` | 执行实例被 SIGKILL：租约期内 B 明确 409 不误抢；死实例不拖累其他会话；原实例重启对账收口遗留 run 后恢复推进（R4/R7/R8） |
|
||||
| `restart_invariants` | 实例重启后消息俱在、SSE 从 0 重放 seq 连续无洞、写入可推进（R5/R7） |
|
||||
|
||||
## 运行
|
||||
|
||||
```bash
|
||||
# 1) 隔离 PostgreSQL（示例：svc.sh 独立实例；已有实例则直接给 DSN）
|
||||
scripts/svc.sh db create mi7 --port 18890
|
||||
scripts/svc.sh mi7 start
|
||||
DSN="$(scripts/svc.sh db dsn mi7 --reveal)"
|
||||
|
||||
# 2) 构建 + 运行（Make 目标自动构建二进制并注入）
|
||||
make test-multi-instance AGENT_PG_TEST_DSN="$DSN?sslmode=disable"
|
||||
|
||||
# 3) 回收
|
||||
scripts/svc.sh mi7 stop && scripts/svc.sh db rm mi7
|
||||
```
|
||||
|
||||
门控：`AGENT_PG_TEST_DSN`（真实 PostgreSQL，CI test-pg job 必跑不跳过）与
|
||||
`AGENT_BIN`（由 Make 目标构建注入）。测试自建临时数据库、自起进程并在
|
||||
退出时回收；不打任何生产服务。
|
||||
|
||||
## 已实测发现并修复的缺陷（历史）
|
||||
|
||||
本验收在 U7 落地时抓出三个跨实例真实缺陷（均已有回归测试）：
|
||||
|
||||
1. **HITL resolve 落卡短路命令路径**：非执行实例先把卡标记 resolved，
|
||||
命令 fallback 的 `!card.Resolved` 判定被跳过，执行者永远等不到答案
|
||||
（backend `TestCrossInstanceResolveDoesNotSettleCard`）。
|
||||
2. **跨实例 cancel 命令 conversation_id 为空**：消费侧按执行权 JOIN
|
||||
匹配，空串永远无法送达。
|
||||
3. **CancelRun 假成功**：非执行实例无句柄时 `CancelRun` 返回 nil，HTTP
|
||||
层 200 受理但 run 永远停不下（命令路径被短路）。
|
||||
@@ -0,0 +1,13 @@
|
||||
module agent-system-dev/tests/integration/multi-instance
|
||||
|
||||
go 1.27
|
||||
|
||||
require github.com/jackc/pgx/v5 v5.9.2
|
||||
|
||||
require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
golang.org/x/sync v0.17.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
|
||||
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,620 @@
|
||||
// 协作存储 U7：双进程（真实 backend serve ×2 共享同一 PG）黑盒验收。
|
||||
// 对应方案 §7 最小验收矩阵的进程级行：
|
||||
// - 两实例同会话不同请求 → 仅一个主执行，另一个明确 409（不误写）
|
||||
// - 在 B 提交 A 的 HITL / 取消 → 命令持久、唯一消费、到达执行者
|
||||
// - 执行实例被杀 → 租约期内不误抢（明确 409），过期后另一实例受控接管
|
||||
// - 实例重启 → 数据不变量保持（消息在、事件 seq 连续无洞）
|
||||
// - 跨实例读一致与事件传播（A 写 → B 的 SSE 可见）
|
||||
//
|
||||
// 资源边界：测试自建临时数据库、自起进程、自回收（方案 §5 U7 的隔离测试
|
||||
// 入口要求）。门控：AGENT_PG_TEST_DSN（真 PG；CI test-pg job 必跑）与
|
||||
// AGENT_BIN（backend 二进制；由 make test-multi-instance 构建并传入）。
|
||||
// 不打任何生产服务。
|
||||
package multiinstance
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
const (
|
||||
aliceOwner = "mi_alice"
|
||||
alicePass0 = "mi-alice-pass-0"
|
||||
alicePass = "mi-alice-pass-1"
|
||||
httpWait = 15 * time.Second
|
||||
)
|
||||
|
||||
type instance struct {
|
||||
name string
|
||||
base string
|
||||
cmd *exec.Cmd
|
||||
log *lockedBuffer
|
||||
}
|
||||
|
||||
type lockedBuffer struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (b *lockedBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.Write(p)
|
||||
}
|
||||
|
||||
func (b *lockedBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
type hitlCard struct{ id, runID string }
|
||||
|
||||
// userClient 持 alice 在两实例的会话（A：api；B：apiB/streamB）。
|
||||
// 会话表在共享 PG，同账号可在两实例同时登录。
|
||||
type userClient struct {
|
||||
api *http.Client // alice@A 普通请求（带超时）
|
||||
apiB *http.Client // alice@B 普通请求
|
||||
streamB *http.Client // alice@B 流式（SSE：无整体超时，chan 背压控制）
|
||||
}
|
||||
|
||||
type env struct {
|
||||
t *testing.T
|
||||
dbName string
|
||||
dbDSN string
|
||||
artsDir string
|
||||
bin string
|
||||
a, b *instance
|
||||
uc *userClient
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("AGENT_PG_TEST_DSN") == "" {
|
||||
fmt.Println("SKIP: AGENT_PG_TEST_DSN 未设置（双进程验收由 CI test-pg job 必跑覆盖）")
|
||||
return
|
||||
}
|
||||
if os.Getenv("AGENT_BIN") == "" {
|
||||
fmt.Println("SKIP: AGENT_BIN 未设置（由 make test-multi-instance 构建并传入 agent-system 二进制路径）")
|
||||
return
|
||||
}
|
||||
// 子进程用独立 cwd（避免 data/ 落测试目录），二进制路径必须先转绝对。
|
||||
bin, err := filepath.Abs(os.Getenv("AGENT_BIN"))
|
||||
if err != nil {
|
||||
fmt.Println("AGENT_BIN abs:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Setenv("AGENT_BIN", bin)
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func TestTwoInstancesOverSharedPG(t *testing.T) {
|
||||
e := newEnv(t)
|
||||
e.startPair()
|
||||
|
||||
uc := e.loginAlice(t)
|
||||
e.uc = uc
|
||||
projID, convID := e.setupConversation(t, uc)
|
||||
|
||||
t.Run("cross_read", func(t *testing.T) {
|
||||
// A 建的资源在 B 立即可见（共享权威，无实例本地视图缺口）。
|
||||
var detail map[string]any
|
||||
e.do(t, uc.apiB, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
if conv, _ := detail["conversation"].(map[string]any); conv == nil || conv["id"] != convID {
|
||||
t.Fatalf("B cannot read conversation created via A: %v", detail)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cross_event_sse", func(t *testing.T) {
|
||||
lastSeq := e.lastSeq(t, uc.apiB, convID)
|
||||
events := e.subscribe(t, uc.streamB, convID, lastSeq)
|
||||
e.postMessage(t, uc.api, convID, "hello from A")
|
||||
select {
|
||||
case ev, ok := <-events:
|
||||
if !ok {
|
||||
t.Fatal("SSE stream closed before any event")
|
||||
}
|
||||
if ev.seq <= lastSeq {
|
||||
t.Fatalf("stale event replayed: seq %d <= since %d", ev.seq, lastSeq)
|
||||
}
|
||||
case <-time.After(8 * time.Second):
|
||||
t.Fatal("event written via A not visible on B SSE within 8s")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("same_session_conflict_then_cross_hitl", func(t *testing.T) {
|
||||
// A 发 #hitl：mock runner 挂起 run（waiting_hitl），会话被 A 的执行租约占用。
|
||||
e.postMessage(t, uc.api, convID, "#hitl")
|
||||
card := e.waitActiveHITL(t, uc.apiB, convID)
|
||||
|
||||
// B 同会话并发提交 → 明确 409，而不是排队后误写或静默接受。
|
||||
code, body := e.raw(t, uc.apiB, "POST", "/api/conversations/"+convID+"/messages", map[string]any{"text": "from B while A holds"})
|
||||
if code != http.StatusConflict {
|
||||
t.Fatalf("concurrent submit on busy session: status %d (want 409), body %.200s", code, body)
|
||||
}
|
||||
|
||||
// B 跨实例 resolve A 的 HITL → 持久命令转交执行者（202 via command；
|
||||
// 执行实例本地 resolve 才是 200），run 随后走完。
|
||||
code, body = e.raw(t, uc.apiB, "POST", "/api/hitl/"+card.id+"/resolve", map[string]any{"choiceId": "md"})
|
||||
if code != http.StatusOK && code != http.StatusAccepted {
|
||||
t.Fatalf("cross-instance resolve: status %d, body %.200s", code, body)
|
||||
}
|
||||
e.waitRunStatus(t, uc.apiB, card.runID, "completed")
|
||||
// run 收口后会话释放：B 可推进。
|
||||
e.postMessage(t, uc.apiB, convID, "B proceeds after release")
|
||||
})
|
||||
|
||||
t.Run("cross_cancel", func(t *testing.T) {
|
||||
e.postMessage(t, uc.api, convID, "#hitl")
|
||||
card := e.waitActiveHITL(t, uc.apiB, convID)
|
||||
// 跨实例取消经命令总线受理（202 via command；执行实例本地取消为 200）。
|
||||
code, body := e.raw(t, uc.apiB, "POST", "/api/runs/"+card.runID+"/cancel", nil)
|
||||
if code != http.StatusOK && code != http.StatusAccepted {
|
||||
t.Fatalf("cross-instance cancel: status %d, body %.200s", code, body)
|
||||
}
|
||||
e.waitRunStatus(t, uc.apiB, card.runID, "cancelled")
|
||||
e.postMessage(t, uc.apiB, convID, "B proceeds after cancel")
|
||||
})
|
||||
|
||||
t.Run("owner_crash_takeover", func(t *testing.T) {
|
||||
e.postMessage(t, uc.api, convID, "#hitl")
|
||||
_ = e.waitActiveHITL(t, uc.api, convID)
|
||||
|
||||
e.killNine(t, e.a)
|
||||
|
||||
// 会话执行租约（15 分钟档）期内不误抢：B 对该会话的提交得到明确
|
||||
// 409（有界失败，不是写坏或挂死）；过期接管语义由库级围栏测试覆盖
|
||||
// （backend runmgr multi_instance_test），进程级不空等 15 分钟。
|
||||
code, body := e.raw(t, uc.apiB, "POST", "/api/conversations/"+convID+"/messages", map[string]any{"text": "too early"})
|
||||
if code != http.StatusConflict {
|
||||
t.Fatalf("submit inside dead owner's lease: status %d (want 409), body %.200s", code, body)
|
||||
}
|
||||
// 死实例不拖累其他会话：B 在新会话上正常推进（公平有界，R8）。
|
||||
var conv2 map[string]any
|
||||
e.do(t, uc.apiB, "POST", "/api/projects/"+projID+"/conversations", map[string]any{"title": "不受拖累"}, &conv2)
|
||||
e.do(t, uc.apiB, "POST", "/api/conversations/"+conv2["id"].(string)+"/messages", map[string]any{"text": "B unaffected"}, nil)
|
||||
// 整机失效恢复剧本：原实例以同一身份重启，启动对账把遗留 run 收口
|
||||
// （interrupted），随后原实例重新占用会话并推进。
|
||||
e.a = e.startInstance(t, e.a.name, strings.TrimPrefix(e.a.base, "http://"))
|
||||
var detail map[string]any
|
||||
deadline := time.Now().Add(httpWait)
|
||||
for time.Now().Before(deadline) {
|
||||
e.do(t, uc.api, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
if runs, ok := detail["runs"].([]any); ok && len(runs) > 0 {
|
||||
last, _ := runs[len(runs)-1].(map[string]any)
|
||||
if last["status"] == "interrupted" {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
e.do(t, uc.api, "POST", "/api/conversations/"+convID+"/messages", map[string]any{"text": "A resumes after restart"}, nil)
|
||||
})
|
||||
|
||||
t.Run("restart_invariants", func(t *testing.T) {
|
||||
e.killNine(t, e.b)
|
||||
e.b = e.startInstance(t, e.b.name, strings.TrimPrefix(e.b.base, "http://"))
|
||||
|
||||
var detail map[string]any
|
||||
e.do(t, uc.apiB, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
msgs, _ := detail["messages"].([]any)
|
||||
if len(msgs) < 5 {
|
||||
t.Fatalf("messages lost after restart: %d", len(msgs))
|
||||
}
|
||||
lastSeq := e.lastSeq(t, uc.apiB, convID)
|
||||
if lastSeq == 0 {
|
||||
t.Fatal("event seq lost after restart")
|
||||
}
|
||||
// SSE 从 0 全量重放：seq 严格连续，无洞（R5 不变量在进程失效后的视图）。
|
||||
events := e.subscribe(t, uc.streamB, convID, 0)
|
||||
seen := int64(0)
|
||||
deadline := time.After(15 * time.Second)
|
||||
for seen < lastSeq {
|
||||
select {
|
||||
case ev, ok := <-events:
|
||||
if !ok {
|
||||
t.Fatalf("replay stream closed early at seq %d (want up to %d)", seen, lastSeq)
|
||||
}
|
||||
if ev.seq != seen+1 {
|
||||
t.Fatalf("replay gap: got seq %d after %d", ev.seq, seen)
|
||||
}
|
||||
seen = ev.seq
|
||||
case <-deadline:
|
||||
t.Fatalf("replay stalled at seq %d / %d", seen, lastSeq)
|
||||
}
|
||||
}
|
||||
// 重启后写入仍可推进。
|
||||
e.postMessage(t, uc.apiB, convID, "after restart")
|
||||
})
|
||||
}
|
||||
|
||||
// ---- 环境与进程 ----
|
||||
|
||||
func newEnv(t *testing.T) *env {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("AGENT_PG_TEST_DSN")
|
||||
admin, err := sql.Open("pgx", dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dbName := "mi_" + strconv.Itoa(os.Getpid()) + "_" + fmt.Sprint(time.Now().UnixNano()%1e6)
|
||||
if _, err := admin.Exec("CREATE DATABASE " + dbName); err != nil {
|
||||
admin.Close()
|
||||
t.Fatalf("create test database: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = admin.Exec("DROP DATABASE IF EXISTS " + dbName + " WITH (FORCE)")
|
||||
admin.Close()
|
||||
})
|
||||
testDSN, err := swapDB(dsn, dbName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
return &env{t: t, dbName: dbName, dbDSN: testDSN, artsDir: filepath.Join(root, "artifacts"), bin: os.Getenv("AGENT_BIN")}
|
||||
}
|
||||
|
||||
// swapDB 替换 DSN 的库名，保留其余参数（sslmode 等）。
|
||||
func swapDB(dsn, dbName string) (string, error) {
|
||||
u, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.Path = "/" + dbName
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func freePort(t *testing.T) string {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer l.Close()
|
||||
_, port, _ := net.SplitHostPort(l.Addr().String())
|
||||
return port
|
||||
}
|
||||
|
||||
func (e *env) startPair() {
|
||||
e.t.Helper()
|
||||
e.a = e.startInstance(e.t, "miA", "127.0.0.1:"+freePort(e.t))
|
||||
e.b = e.startInstance(e.t, "miB", "127.0.0.1:"+freePort(e.t))
|
||||
}
|
||||
|
||||
func (e *env) startInstance(t *testing.T, name, addr string) *instance {
|
||||
t.Helper()
|
||||
cmd := exec.Command(e.bin, "serve",
|
||||
"--dsn", e.dbDSN,
|
||||
"--addr", addr,
|
||||
"--tier", "team",
|
||||
"--auth", "on",
|
||||
"--mock",
|
||||
"--instance", name,
|
||||
"--artifacts", e.artsDir,
|
||||
"--disable-exec-tools",
|
||||
)
|
||||
log := &lockedBuffer{}
|
||||
cmd.Stdout, cmd.Stderr = log, log
|
||||
// 独立 cwd：serve 会向 ./data 写 skills/SQLite 等，不能落在测试目录。
|
||||
cmd.Dir = t.TempDir()
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("start %s: %v", name, err)
|
||||
}
|
||||
inst := &instance{name: name, base: "http://" + addr, cmd: cmd, log: log}
|
||||
t.Cleanup(func() { e.stopInstance(inst) })
|
||||
e.waitHealthy(t, inst)
|
||||
return inst
|
||||
}
|
||||
|
||||
func (e *env) waitHealthy(t *testing.T, inst *instance) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := http.Get(inst.base + "/api/health")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
// auth=on 时 health 同样要求鉴权：401 = 服务已就绪并正确应答挑战。
|
||||
if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusUnauthorized {
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("instance %s not healthy in 60s; log:\n%s", inst.name, inst.log.String())
|
||||
}
|
||||
|
||||
func (e *env) stopInstance(inst *instance) {
|
||||
if inst.cmd.Process == nil || inst.cmd.ProcessState != nil {
|
||||
return
|
||||
}
|
||||
_ = inst.cmd.Process.Signal(syscall.SIGTERM)
|
||||
done := make(chan struct{})
|
||||
go func() { _ = inst.cmd.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
_ = inst.cmd.Process.Kill()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func (e *env) killNine(t *testing.T, inst *instance) {
|
||||
t.Helper()
|
||||
if inst.cmd.Process == nil {
|
||||
t.Fatal("no process to kill")
|
||||
}
|
||||
if err := inst.cmd.Process.Signal(syscall.SIGKILL); err != nil {
|
||||
t.Fatalf("SIGKILL %s: %v", inst.name, err)
|
||||
}
|
||||
_ = inst.cmd.Wait()
|
||||
}
|
||||
|
||||
// ---- HTTP 客户端 ----
|
||||
|
||||
// loginAlice 引导 alice 账号并发放 API key（CLI 直连测试库；key 存共享 PG，
|
||||
// 对两实例同时有效——程序化验收的首选凭证，SSE 亦可用 ?key= 兜底）。
|
||||
func (e *env) loginAlice(t *testing.T) *userClient {
|
||||
t.Helper()
|
||||
e.cli(t, "user", "create", "--dsn", e.dbDSN, "--owner", aliceOwner, "--name", "Alice", "--password", alicePass)
|
||||
key := e.cliOut(t, "apikey", "create", "--dsn", e.dbDSN, "--owner", aliceOwner, "--name", "mi-acceptance")
|
||||
uc := &userClient{
|
||||
api: &http.Client{Timeout: httpWait, Transport: bearerRT{key: key}},
|
||||
apiB: &http.Client{Timeout: httpWait, Transport: bearerRT{key: key}},
|
||||
streamB: &http.Client{Transport: bearerRT{key: key}}, // SSE 无整体超时；chan 背压控制
|
||||
}
|
||||
// --mock 只写 OwnerDefault 的设置；alice 是独立租户，须显式开 mock
|
||||
// （selectConversationRunner 按 EffectiveSettings 分流），否则 run 走
|
||||
// LLMRunner 且无模型配置直接失败。
|
||||
e.do(t, uc.api, "PUT", "/api/settings/user", map[string]any{"runtime": map[string]any{"mock": true}}, nil)
|
||||
return uc
|
||||
}
|
||||
|
||||
// bearerRT 为所有请求附加 Authorization: Bearer <key>。
|
||||
type bearerRT struct{ key string }
|
||||
|
||||
func (b bearerRT) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
r2 := r.Clone(r.Context())
|
||||
r2.Header.Set("Authorization", "Bearer "+b.key)
|
||||
return http.DefaultTransport.RoundTrip(r2)
|
||||
}
|
||||
|
||||
// cliOut 运行 CLI 并取 stdout 首行（apikey create 的明文仅此一次输出）。
|
||||
func (e *env) cliOut(t *testing.T, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command(e.bin, args...)
|
||||
cmd.Dir = t.TempDir()
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Fatalf("cli %v…: %v\n%s", args[:2], err, stderr.String())
|
||||
}
|
||||
line := strings.TrimSpace(stdout.String())
|
||||
if line == "" {
|
||||
t.Fatalf("cli %v…: empty stdout", args[:2])
|
||||
}
|
||||
return strings.Fields(line)[0]
|
||||
}
|
||||
|
||||
func (e *env) cli(t *testing.T, args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command(e.bin, args...)
|
||||
cmd.Dir = t.TempDir()
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("cli %v…: %v\n%s", args[:2], err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *env) do(t *testing.T, c *http.Client, method, path string, body any, out any) {
|
||||
t.Helper()
|
||||
code, raw := e.raw(t, c, method, path, body)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("%s %s: status %d, body %.300s", method, path, code, raw)
|
||||
}
|
||||
if out != nil {
|
||||
if err := json.Unmarshal([]byte(raw), out); err != nil {
|
||||
t.Fatalf("%s %s: decode %v", method, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// raw 以 JSON body 发起请求，返回状态码与响应体文本。请求地址取
|
||||
// client 所绑定实例（由调用方选择 api / apiB / streamB 及对应 base）。
|
||||
func (e *env) raw(t *testing.T, c *http.Client, method, path string, body any) (int, string) {
|
||||
t.Helper()
|
||||
return e.rawBase(t, c, e.baseFor(c), method, path, body)
|
||||
}
|
||||
|
||||
func (e *env) rawBase(t *testing.T, c *http.Client, base, method, path string, body any) (int, string) {
|
||||
t.Helper()
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
buf, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(buf)
|
||||
}
|
||||
req, err := http.NewRequest(method, base+path, rd)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, path, err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", method, path, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
out, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
return resp.StatusCode, string(out)
|
||||
}
|
||||
|
||||
// baseFor 按 client 身份路由实例：B 侧（apiB/streamB）→ B；其余 → A。
|
||||
func (e *env) baseFor(c *http.Client) string {
|
||||
if e.uc != nil && (c == e.uc.apiB || c == e.uc.streamB) {
|
||||
return e.b.base
|
||||
}
|
||||
return e.a.base
|
||||
}
|
||||
|
||||
// ---- 场景辅助 ----
|
||||
|
||||
type sseEvent struct {
|
||||
seq int64
|
||||
event string
|
||||
data string
|
||||
}
|
||||
|
||||
func (e *env) setupConversation(t *testing.T, uc *userClient) (string, string) {
|
||||
t.Helper()
|
||||
var proj map[string]any
|
||||
e.do(t, uc.api, "POST", "/api/projects", map[string]any{"name": "mi-验收"}, &proj)
|
||||
projID, _ := proj["id"].(string)
|
||||
var conv map[string]any
|
||||
e.do(t, uc.api, "POST", "/api/projects/"+projID+"/conversations", map[string]any{"title": "双进程"}, &conv)
|
||||
convID, _ := conv["id"].(string)
|
||||
return projID, convID
|
||||
}
|
||||
|
||||
// postMessage 先等会话空闲（上一 run 终态），再提交消息——#hitl 需要一个
|
||||
// 可占用的空闲会话；普通消息同样避免撞上上一 run 的收口窗口。
|
||||
func (e *env) postMessage(t *testing.T, c *http.Client, convID, text string) {
|
||||
t.Helper()
|
||||
e.waitConversationIdle(t, c, convID, 20*time.Second)
|
||||
e.do(t, c, "POST", "/api/conversations/"+convID+"/messages", map[string]any{"text": text}, nil)
|
||||
}
|
||||
|
||||
func (e *env) waitConversationIdle(t *testing.T, c *http.Client, convID string, within time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(within)
|
||||
for time.Now().Before(deadline) {
|
||||
var detail map[string]any
|
||||
e.do(t, c, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
busy := false
|
||||
if runs, ok := detail["runs"].([]any); ok {
|
||||
for _, r := range runs {
|
||||
m, _ := r.(map[string]any)
|
||||
switch m["status"] {
|
||||
case "pending", "running", "waiting_hitl":
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
return
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("conversation %s still busy after %v", convID, within)
|
||||
}
|
||||
|
||||
func (e *env) lastSeq(t *testing.T, c *http.Client, convID string) int64 {
|
||||
t.Helper()
|
||||
var detail map[string]any
|
||||
e.do(t, c, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
n, _ := detail["lastSeq"].(float64)
|
||||
return int64(n)
|
||||
}
|
||||
|
||||
// subscribe 在 B 实例打开会话 SSE（since 起），帧 seq > since 才投递；
|
||||
// 返回通道在流结束/出错时关闭。调用方通过关闭底层响应取消。
|
||||
func (e *env) subscribe(t *testing.T, c *http.Client, convID string, since int64) <-chan sseEvent {
|
||||
t.Helper()
|
||||
u := e.b.base + "/api/events?conversationId=" + convID + "&since=" + strconv.FormatInt(since, 10)
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe SSE: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
t.Fatalf("subscribe SSE: status %d", resp.StatusCode)
|
||||
}
|
||||
ch := make(chan sseEvent, 1024)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
defer resp.Body.Close()
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4<<20)
|
||||
var seq int64
|
||||
var event, data string
|
||||
flush := func() {
|
||||
if seq > since {
|
||||
// 阻塞发送：restart 场景依赖全量重放断言，不允许丢帧；
|
||||
// 消费停滞时读取 goroutine 挂起（进程随测试退出回收）。
|
||||
ch <- sseEvent{seq: seq, event: event, data: data}
|
||||
}
|
||||
seq, event, data = 0, "", ""
|
||||
}
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
switch {
|
||||
case line == "":
|
||||
if seq > 0 {
|
||||
flush()
|
||||
}
|
||||
case strings.HasPrefix(line, "id: "):
|
||||
seq, _ = strconv.ParseInt(strings.TrimPrefix(line, "id: "), 10, 64)
|
||||
case strings.HasPrefix(line, "event: "):
|
||||
event = strings.TrimPrefix(line, "event: ")
|
||||
case strings.HasPrefix(line, "data: "):
|
||||
data = strings.TrimPrefix(line, "data: ")
|
||||
}
|
||||
}
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
func (e *env) waitActiveHITL(t *testing.T, c *http.Client, convID string) hitlCard {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
var detail map[string]any
|
||||
e.do(t, c, "GET", "/api/conversations/"+convID, nil, &detail)
|
||||
if cards, ok := detail["activeHitl"].([]any); ok && len(cards) > 0 {
|
||||
m, _ := cards[0].(map[string]any)
|
||||
id, _ := m["id"].(string)
|
||||
runID, _ := m["runId"].(string)
|
||||
if id != "" && runID != "" {
|
||||
return hitlCard{id: id, runID: runID}
|
||||
}
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("no active HITL card in 15s; logA:\n%s", e.a.log.String())
|
||||
return hitlCard{}
|
||||
}
|
||||
|
||||
func (e *env) waitRunStatus(t *testing.T, c *http.Client, runID, status string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(httpWait)
|
||||
for time.Now().Before(deadline) {
|
||||
var out map[string]any
|
||||
e.do(t, c, "GET", "/api/runs/"+runID, nil, &out)
|
||||
run, _ := out["run"].(map[string]any)
|
||||
if s, _ := run["status"].(string); s == status {
|
||||
return
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("run %s not %s within %v", runID, status, httpWait)
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
/.build/
|
||||
/results/
|
||||
@@ -0,0 +1,50 @@
|
||||
# 协作存储压测套件（U7）
|
||||
|
||||
方案 §3 负载合同 / §5 U7 的业务压测入口。**只打隔离栈**（自建 PG 实例 +
|
||||
mock 模式 backend），绝不打生产服务；模拟模型（mock）用于容量基线，真实
|
||||
模型的全链路验证按方案另行小流量执行。
|
||||
|
||||
## 组成
|
||||
|
||||
| 文件 | 职责 |
|
||||
|---|---|
|
||||
| `run_load.sh` | 编排入口：起隔离栈 → 引导账号 → k6 阶梯 → SSE 探针 → 不变量对账 → 回收 |
|
||||
| `k6/collab_api.js` | 非模型普通 API 的混合读写负载（登录/项目/会话/卡片/设置/消息，requestId 幂等重放抽样；预期 409/429 单列计数） |
|
||||
| `sse_probe/` | SSE 持久事件可见延迟探针（Go）：k6 核心不支持 SSE 流式读取，P95≤1s 指标由此承担，含 seq 单调性断言 |
|
||||
| `results/<ts>/` | 每次运行的原始输出：`k6-summary.json`、`k6-output.txt`、`sse-probe.json`、`invariants.txt`、`app.log` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
# 冒烟（~2 分钟，无需 k6——缺失时自动跳过 k6 段并给出安装指引）
|
||||
tests/load/collaboration/run_load.sh --profile smoke
|
||||
|
||||
# 阶梯（10→30→100 VU，~12 分钟；k6: brew install k6）
|
||||
tests/load/collaboration/run_load.sh --profile ladder --users 30
|
||||
|
||||
# 长稳（60 分钟 / 2 小时混合，负载合同的两个正式窗口）
|
||||
tests/load/collaboration/run_load.sh --profile sustained60m
|
||||
tests/load/collaboration/run_load.sh --profile sustained2h
|
||||
|
||||
# 复用已有隔离 PG；保留栈便于复查
|
||||
tests/load/collaboration/run_load.sh --dsn 'postgresql://…/agent' --keep
|
||||
```
|
||||
|
||||
## 阈值与口径（负载合同，待实测校准）
|
||||
|
||||
- 非模型普通 API：读路径 P95≤300ms、P99≤1s（k6 threshold 按 `read` 组单列）。
|
||||
- 非预期失败率≤0.1%（`http_req_failed<0.001`）；预期拥塞（会话忙 409 /
|
||||
队列满 429）记入 `expected_contention` 计数器，**不**计入失败。
|
||||
- SSE 持久事件可见延迟 P95≤1s + seq 零乱序（探针退出码即断言）。
|
||||
- 数据不变量零违反：events 每会话 seq 无重复（SQL 对账，违反即 exit 1）；
|
||||
各表计数与孤儿 run 报告落 `invariants.txt`。
|
||||
|
||||
## 边界（诚实声明）
|
||||
|
||||
- 首期为**单实例**压测（mock 模式、执行原语关闭）；分布式限流窗口与双实例
|
||||
放行指标在双栈就位后扩展（`--dsn` 复用 + 第二实例待加）。
|
||||
- 容量数字（连接等待、CPU/内存/IO、pgbench 数据库基线）须绑定组件 SHA、
|
||||
配置与数据集记录在结果目录；`pgbench` 只测数据库能力，不替代本套业务
|
||||
验收（方案 §5 U7）。
|
||||
- 正式 60min/2h 窗口的执行与双主机验证需要用户指定资源与时间；本仓提供
|
||||
的是可复核入口，不预填容量结论。
|
||||
@@ -0,0 +1,137 @@
|
||||
// 协作存储 U7 压测（方案 §3 负载合同 / §5 U7）：k6 覆盖非模型普通 API
|
||||
// 的吞吐与延迟阶梯；SSE 可见延迟 P95 由 Go 探针（../sse_probe）承担——
|
||||
// k6 核心不支持 SSE 流式读取，两者分工记录在同一份结果目录。
|
||||
//
|
||||
// 目标实例：run_load.sh 起的隔离栈（mock 模式 backend + 隔离 PG），绝不打
|
||||
// 生产服务。预期限流/冲突（409/429）单列计数，不计入失败。
|
||||
//
|
||||
// 阶梯与环境变量（由 run_load.sh 注入）：
|
||||
// BASE_URL 目标基址（默认 http://127.0.0.1:18980）
|
||||
// ADMIN_* 引导管理员（run_load.sh 预建；setup 里登录后建压测用户）
|
||||
// USERS 压测用户数（默认 30；每 VU 绑定一个用户）
|
||||
// PROFILE smoke | ladder | sustained60m | sustained2h（默认 smoke）
|
||||
//
|
||||
// 阈值（负载合同，待实测后校准）：
|
||||
// 读路径 p(95)<300ms、全量 p(99)<1s、非预期失败率<0.1%。
|
||||
|
||||
import http from 'k6/http';
|
||||
import { check, group, sleep } from 'k6';
|
||||
import { Counter, Trend } from 'k6/metrics';
|
||||
import exec from 'k6/execution';
|
||||
|
||||
const BASE = __ENV.BASE_URL || 'http://127.0.0.1:18980';
|
||||
const ADMIN_OWNER = __ENV.ADMIN_OWNER || 'load_admin';
|
||||
const ADMIN_PASS = __ENV.ADMIN_PASS || 'load-admin-pass';
|
||||
const USERS = parseInt(__ENV.USERS || '30', 10);
|
||||
const PROFILE = __ENV.PROFILE || 'smoke';
|
||||
|
||||
// 预期拥塞（负载合同要求单列，不算失败）：409 会话忙 / 429 队列满。
|
||||
const expectedContention = new Counter('expected_contention');
|
||||
const msgLatency = new Trend('message_roundtrip', true);
|
||||
|
||||
const stages = {
|
||||
smoke: [{ duration: '30s', target: 10 }],
|
||||
ladder: [
|
||||
{ duration: '2m', target: 10 },
|
||||
{ duration: '3m', target: 30 },
|
||||
{ duration: '5m', target: 100 },
|
||||
{ duration: '2m', target: 0 },
|
||||
],
|
||||
sustained60m: [
|
||||
{ duration: '2m', target: 30 },
|
||||
{ duration: '56m', target: 30 },
|
||||
{ duration: '2m', target: 0 },
|
||||
],
|
||||
sustained2h: [
|
||||
{ duration: '3m', target: 30 },
|
||||
{ duration: '114m', target: 30 },
|
||||
{ duration: '3m', target: 0 },
|
||||
],
|
||||
};
|
||||
|
||||
export const options = {
|
||||
stages: stages[PROFILE] || stages.smoke,
|
||||
thresholds: {
|
||||
// 非预期失败：2xx 之外的（409/429 已在 expected_contention 单列）。
|
||||
http_req_failed: ['rate<0.001'],
|
||||
'http_req_duration{group:::read}': ['p(95)<300', 'p(99)<1000'],
|
||||
http_req_duration: ['p(99)<1000'],
|
||||
},
|
||||
summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'],
|
||||
};
|
||||
|
||||
// ---- setup：管理员建压测用户，各自建项目与会话（数据集随用户数扩展） ----
|
||||
|
||||
export function setup() {
|
||||
// setup 上下文的 cookie jar 自动附着（k6 语义：jar 不作为请求器使用）。
|
||||
const login = http.post(`${BASE}/api/auth/login`, JSON.stringify({ owner: ADMIN_OWNER, password: ADMIN_PASS }), { headers: { 'Content-Type': 'application/json' } });
|
||||
if (login.status !== 200) throw new Error(`admin login: ${login.status} ${login.body}`);
|
||||
// 首登强制改密由 run_load.sh 预处理（CLI 建号即带终态口令），此处直接可用。
|
||||
|
||||
const users = [];
|
||||
for (let i = 0; i < USERS; i++) {
|
||||
const owner = `load_u${i}`;
|
||||
const password = `load-user-pass-${i}-Aa9`;
|
||||
const created = http.post(`${BASE}/api/admin/users`, JSON.stringify({ owner, name: `压测用户${i}`, password, isAdmin: false }), { headers: { 'Content-Type': 'application/json' } });
|
||||
// 409 = 已存在（重复压测同一栈），接受。
|
||||
if (created.status !== 200 && created.status !== 409) throw new Error(`create user ${owner}: ${created.status} ${created.body}`);
|
||||
users.push({ owner, password });
|
||||
}
|
||||
return { users };
|
||||
}
|
||||
|
||||
// ---- default：每 VU 一个用户身份，混合读写 ----
|
||||
|
||||
export default function (data) {
|
||||
const u = data.users[exec.scenario.iterationInTest % data.users.length];
|
||||
// VU 上下文的 cookie jar 自动附着；每 VU 绑定一个用户身份。
|
||||
const login = http.post(`${BASE}/api/auth/login`, JSON.stringify({ owner: u.owner, password: u.password }), { headers: { 'Content-Type': 'application/json' } });
|
||||
if (login.status !== 200) {
|
||||
check(null, { 'user login': () => false });
|
||||
return;
|
||||
}
|
||||
// 会话/项目绑定 VU（VU 内多次迭代复用同一身份的会话，形成热点会话）。
|
||||
if (!u.__ctx) {
|
||||
const proj = http.post(`${BASE}/api/projects`, JSON.stringify({ name: `压测-${u.owner}` }), { headers: { 'Content-Type': 'application/json' } });
|
||||
const conv = http.post(`${BASE}/api/projects/${proj.json().id}/conversations`, JSON.stringify({ title: '压测会话' }), { headers: { 'Content-Type': 'application/json' } });
|
||||
u.__ctx = { projectID: proj.json().id, convID: conv.json().id };
|
||||
}
|
||||
const { projectID, convID } = u.__ctx;
|
||||
|
||||
group('read', () => {
|
||||
const projs = http.get(`${BASE}/api/projects`);
|
||||
check(projs, { 'list projects 2xx': (r) => r.status === 200 });
|
||||
const detail = http.get(`${BASE}/api/conversations/${convID}`);
|
||||
check(detail, { 'conversation detail 2xx': (r) => r.status === 200 });
|
||||
const cards = http.get(`${BASE}/api/conversations/${convID}/cards`);
|
||||
check(cards, { 'cards 2xx': (r) => r.status === 200 });
|
||||
const settings = http.get(`${BASE}/api/settings`);
|
||||
check(settings, { 'settings 2xx': (r) => r.status === 200 });
|
||||
});
|
||||
|
||||
group('write', () => {
|
||||
const requestId = `req-${u.owner}-${exec.scenario.iterationInTest}`;
|
||||
const body = JSON.stringify({ text: `压测消息 ${Date.now()}`, requestId });
|
||||
const t0 = Date.now();
|
||||
const sent = http.post(`${BASE}/api/conversations/${convID}/messages`, body, { headers: { 'Content-Type': 'application/json' } });
|
||||
if (sent.status === 200) {
|
||||
msgLatency.add(Date.now() - t0);
|
||||
const id = sent.json().message && sent.json().message.id;
|
||||
check(sent, { 'message has id': () => !!id });
|
||||
// 幂等重放（矩阵行：同 requestId 返回同一逻辑结果）：抽样 10%。
|
||||
if (exec.scenario.iterationInTest % 10 === 0) {
|
||||
const replay = http.post(`${BASE}/api/conversations/${convID}/messages`, body, { headers: { 'Content-Type': 'application/json' } });
|
||||
check(replay, {
|
||||
'requestId replay same message': (r) => r.status === 200 && r.json().message && r.json().message.id === id,
|
||||
});
|
||||
}
|
||||
} else if (sent.status === 409 || sent.status === 429) {
|
||||
// 热点会话忙/队列满：预期拥塞，单列。
|
||||
expectedContention.add(1);
|
||||
} else {
|
||||
check(sent, { 'message send 2xx': () => false });
|
||||
}
|
||||
});
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
# 协作存储 U7 压测入口（方案 §3 负载合同 / §5 U7）。
|
||||
#
|
||||
# 起一个隔离栈（backend mock 单实例 + 隔离 PG）→ 预建引导账号 → k6 阶梯
|
||||
# → SSE 可见延迟探针 → 数据不变量对账 → 结果落 results/<ts>/ → 回收。
|
||||
# 绝不打生产服务：默认自建 PG 实例（svc.sh）；--dsn 复用已有隔离实例。
|
||||
#
|
||||
# 用法：
|
||||
# tests/load/collaboration/run_load.sh [--profile smoke|ladder|sustained60m|sustained2h]
|
||||
# [--users N] [--dsn <PG DSN>] [--port <app port>]
|
||||
# [--keep] [--skip-k6] [--skip-probe] [--help]
|
||||
# 依赖：bash4、go（构建 backend/探针）、psql（对账）、curl；
|
||||
# k6 可选（缺失时跳过 k6 段并给出安装指引，探针与对账照跑）。
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
HERE="$ROOT/tests/load/collaboration"
|
||||
PROFILE=smoke USERS=30 APP_PORT=18980 PG_PORT=18981 KEEP=0 SKIP_K6=0 SKIP_PROBE=0
|
||||
DSN=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--profile) PROFILE="$2"; shift 2 ;;
|
||||
--users) USERS="$2"; shift 2 ;;
|
||||
--dsn) DSN="$2"; shift 2 ;;
|
||||
--port) APP_PORT="$2"; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
--skip-k6) SKIP_K6=1; shift ;;
|
||||
--skip-probe) SKIP_PROBE=1; shift ;;
|
||||
-h|--help) sed -n '2,16p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TS="$(date +%Y%m%d-%H%M%S)"
|
||||
RESULTS="$HERE/results/$TS"
|
||||
mkdir -p "$RESULTS"
|
||||
OWNED_PG=0
|
||||
APP_PID=""
|
||||
cleanup() {
|
||||
[[ -n "$APP_PID" ]] && kill "$APP_PID" 2>/dev/null || true
|
||||
if [[ $OWNED_PG -eq 1 && $KEEP -eq 0 ]]; then
|
||||
"$ROOT/scripts/svc.sh" u7load stop >/dev/null 2>&1 || true
|
||||
"$ROOT/scripts/svc.sh" rm u7load --purge >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $KEEP -eq 1 ]]; then
|
||||
echo "[load] --keep：栈保留（PG=u7load:$PG_PORT，app=127.0.0.1:$APP_PORT，结果 $RESULTS）"
|
||||
fi
|
||||
return 0 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---- 1) PG：自建隔离实例（或复用 --dsn） ----
|
||||
if [[ -z "$DSN" ]]; then
|
||||
echo "[load] 创建隔离 PG 实例 u7load（端口 $PG_PORT）"
|
||||
"$ROOT/scripts/svc.sh" db create u7load --port "$PG_PORT" >/dev/null
|
||||
OWNED_PG=1
|
||||
"$ROOT/scripts/svc.sh" u7load start >/dev/null
|
||||
DSN="$("$ROOT/scripts/svc.sh" db dsn u7load --reveal)"
|
||||
fi
|
||||
echo "[load] PG DSN 已就绪（密码不回显）"
|
||||
|
||||
# ---- 2) 构建 backend 与探针 ----
|
||||
echo "[load] 构建 backend（worktree 当前源码）与 SSE 探针"
|
||||
BIN="$HERE/.build/agent-system"
|
||||
mkdir -p "$HERE/.build"
|
||||
( cd "$ROOT/modules/backend" && CGO_ENABLED=0 go build -o "$BIN" ./cmd/agent-system )
|
||||
( cd "$HERE/sse_probe" && go build -o "$HERE/.build/sse_probe" . )
|
||||
|
||||
# ---- 3) 压测库 + 引导账号（CLI 直连，终态口令避免首登改密分支） ----
|
||||
# DSN 规范化：库名换 u7load_app、query 补 sslmode=disable（密码不回显）。
|
||||
ADMIN_DSN="$(python3 - "$DSN" <<'PY'
|
||||
import sys
|
||||
from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode
|
||||
u = urlsplit(sys.argv[1])
|
||||
q = dict(parse_qsl(u.query))
|
||||
q.setdefault("sslmode", "disable")
|
||||
print(urlunsplit((u.scheme, u.netloc, "/" + (u.path.lstrip("/") or "postgres"), urlencode(q), "")))
|
||||
PY
|
||||
)"
|
||||
psql "$ADMIN_DSN" -v ON_ERROR_STOP=1 -c "DROP DATABASE IF EXISTS u7load_app WITH (FORCE)" -c "CREATE DATABASE u7load_app" >/dev/null
|
||||
APP_DSN="$(python3 - "$ADMIN_DSN" <<'PY'
|
||||
import sys
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
u = urlsplit(sys.argv[1])
|
||||
print(urlunsplit((u.scheme, u.netloc, "/u7load_app", u.query, "")))
|
||||
PY
|
||||
)"
|
||||
"$BIN" user create --dsn "$APP_DSN" --admin --owner load_admin --password load-admin-pass-Aa1 >/dev/null
|
||||
"$BIN" user create --dsn "$APP_DSN" --owner probe_user --password probe-user-pass-Aa9 >/dev/null
|
||||
|
||||
# ---- 4) 起被测实例（mock、单实例首期；执行原语关闭——压测不打执行面） ----
|
||||
echo "[load] 启动被测实例 127.0.0.1:$APP_PORT（mock / tier=team / auth=on）"
|
||||
APP_LOG="$RESULTS/app.log"
|
||||
"$BIN" serve --dsn "$APP_DSN" --addr "127.0.0.1:$APP_PORT" --tier team --auth on --mock \
|
||||
--instance u7load --disable-exec-tools >"$APP_LOG" 2>&1 &
|
||||
APP_PID=$!
|
||||
# auth=on 时 /api/health 同样要求鉴权：200/401 都视为已就绪（服务在正确应答）。
|
||||
app_ready() { code=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:$APP_PORT/api/health"); [[ "$code" == 200 || "$code" == 401 ]]; }
|
||||
for _ in $(seq 1 60); do app_ready && break; sleep 0.5; done
|
||||
app_ready || { echo "[load] app 未就绪，日志 $APP_LOG" >&2; exit 1; }
|
||||
|
||||
BASE="http://127.0.0.1:$APP_PORT"
|
||||
|
||||
# ---- 5) k6 阶梯 ----
|
||||
if [[ $SKIP_K6 -eq 1 ]]; then
|
||||
echo "[load] --skip-k6：跳过 k6 段"
|
||||
elif ! command -v k6 >/dev/null 2>&1; then
|
||||
cat >"$RESULTS/k6-skipped.txt" <<'EOF'
|
||||
k6 未安装：本段跳过（探针与对账照跑）。
|
||||
安装（本机 linuxbrew）：brew install k6
|
||||
安装后重跑：tests/load/collaboration/run_load.sh --profile <同参数>
|
||||
EOF
|
||||
echo "[load] k6 未安装，跳过（见 $RESULTS/k6-skipped.txt）"
|
||||
else
|
||||
echo "[load] k6 profile=$PROFILE users=$USERS"
|
||||
( cd "$HERE" && BASE_URL="$BASE" ADMIN_OWNER=load_admin ADMIN_PASS=load-admin-pass-Aa1 \
|
||||
USERS="$USERS" PROFILE="$PROFILE" \
|
||||
k6 run --summary-export "$RESULTS/k6-summary.json" k6/collab_api.js ) | tee "$RESULTS/k6-output.txt"
|
||||
fi
|
||||
|
||||
# ---- 6) SSE 可见延迟探针 ----
|
||||
if [[ $SKIP_PROBE -eq 0 ]]; then
|
||||
echo "[load] SSE 可见延迟探针（60s，10 会话）"
|
||||
"$HERE/.build/sse_probe" -base "$BASE" -dur 60s -convs 10 -out "$RESULTS/sse-probe.json"
|
||||
fi
|
||||
|
||||
# ---- 7) 数据不变量对账（方案 §3：零违反） ----
|
||||
echo "[load] 数据不变量对账"
|
||||
psql "$APP_DSN" -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT 'users' k, COUNT(*) v FROM users UNION ALL SELECT 'projects', COUNT(*) FROM projects UNION ALL SELECT 'conversations', COUNT(*) FROM conversations UNION ALL SELECT 'messages', COUNT(*) FROM messages UNION ALL SELECT 'events', COUNT(*) FROM events UNION ALL SELECT 'runs', COUNT(*) FROM runs" \
|
||||
-c "SELECT 'dup_seq_per_conv' k, COUNT(*) v FROM (SELECT conversation_id, seq FROM events GROUP BY 1,2 HAVING COUNT(*)>1) d" \
|
||||
-c "SELECT 'nonterminal_orphan_runs' k, COUNT(*) v FROM runs WHERE status IN ('pending','running','waiting_hitl')" \
|
||||
| tee "$RESULTS/invariants.txt"
|
||||
|
||||
DUP=$(psql "$APP_DSN" -tAc "SELECT COUNT(*) FROM (SELECT conversation_id, seq FROM events GROUP BY 1,2 HAVING COUNT(*)>1) d")
|
||||
if [[ "$DUP" != "0" ]]; then
|
||||
echo "[load] 对账失败：events 每会话 seq 出现重复（$DUP 组）" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[load] 对账通过：无重复 seq；孤儿 run 报告见 invariants.txt（探针正常收口应为 0）"
|
||||
|
||||
echo "[load] 完成。结果目录：$RESULTS"
|
||||
@@ -0,0 +1,3 @@
|
||||
module agent-system-dev/tests/load/collaboration/sse_probe
|
||||
|
||||
go 1.27.1
|
||||
@@ -0,0 +1,241 @@
|
||||
// 协作存储 U7：SSE 持久事件可见延迟探针（负载合同：P95≤1s）。
|
||||
// k6 核心不支持 SSE 流式读取，此项由本 Go 探针承担；与 k6 结果记录在
|
||||
// 同一目录（run_load.sh 编排）。
|
||||
//
|
||||
// 行为：并发 C 个会话各挂一个 SSE 订阅（共享登录 cookie）；写者按固定
|
||||
// 间隔轮转 POST 消息（mock 即时回合）；统计「POST 前基准 seq → 会话内
|
||||
// 新帧到达」延迟分布与 seq 单调性，输出 JSON 报告并以退出码执行
|
||||
// P95≤1s、零乱序断言（数据不变量零违反，方案 §3）。
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type sample struct {
|
||||
Conversation string `json:"conversation"`
|
||||
SendSeq int64 `json:"sendSeq"`
|
||||
LatencyMS float64 `json:"latencyMs"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Target string `json:"target"`
|
||||
Duration string `json:"duration"`
|
||||
Subscribers int `json:"subscribers"`
|
||||
Messages int `json:"messages"`
|
||||
Samples int `json:"samples"`
|
||||
P50MS float64 `json:"p50Ms"`
|
||||
P95MS float64 `json:"p95Ms"`
|
||||
P99MS float64 `json:"p99Ms"`
|
||||
MaxMS float64 `json:"maxMs"`
|
||||
OutOfOrder int `json:"outOfOrder"` // seq 乱序/回退（不变量违反）
|
||||
ThresholdMS float64 `json:"thresholdMs"`
|
||||
Pass bool `json:"pass"`
|
||||
Detail []sample `json:"samplesDetail"`
|
||||
}
|
||||
|
||||
var (
|
||||
jar, _ = cookiejar.New(nil)
|
||||
client = &http.Client{Timeout: 10 * time.Second, Jar: jar}
|
||||
streamClient = &http.Client{Jar: jar} // SSE：无整体超时，stop 控制
|
||||
pendingMu sync.Mutex
|
||||
pending = map[string]pendingSend{}
|
||||
outOfOrderMu sync.Mutex
|
||||
outOfOrderN int
|
||||
samplesMu sync.Mutex
|
||||
samples []sample
|
||||
)
|
||||
|
||||
type pendingSend struct {
|
||||
seq int64
|
||||
sent time.Time
|
||||
}
|
||||
|
||||
func main() {
|
||||
base := flag.String("base", "http://127.0.0.1:18980", "目标实例基址")
|
||||
owner := flag.String("owner", "probe_user", "探针用户")
|
||||
password := flag.String("password", "probe-user-pass-Aa9", "探针用户口令")
|
||||
convs := flag.Int("convs", 10, "并发订阅会话数")
|
||||
every := flag.Duration("every", 500*time.Millisecond, "写者间隔（轮转各会话）")
|
||||
dur := flag.Duration("dur", 60*time.Second, "采样时长")
|
||||
out := flag.String("out", "", "JSON 报告输出路径（空 = stdout）")
|
||||
flag.Parse()
|
||||
|
||||
login(*base, *owner, *password)
|
||||
|
||||
projectID := apiJSON("POST", *base, "/api/projects", map[string]any{"name": "sse-probe"})["id"].(string)
|
||||
convIDs := make([]string, *convs)
|
||||
for i := range convIDs {
|
||||
convIDs[i] = apiJSON("POST", *base, "/api/projects/"+projectID+"/conversations", map[string]any{"title": fmt.Sprintf("probe-%d", i)})["id"].(string)
|
||||
}
|
||||
|
||||
stop := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for _, cid := range convIDs {
|
||||
wg.Add(1)
|
||||
go func(cid string) {
|
||||
defer wg.Done()
|
||||
subscribe(*base, cid, stop)
|
||||
}(cid)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // 订阅就绪
|
||||
|
||||
deadline := time.Now().Add(*dur)
|
||||
sent := 0
|
||||
for i := 0; time.Now().Before(deadline); i++ {
|
||||
cid := convIDs[i%len(convIDs)]
|
||||
detail := apiJSON("GET", *base, "/api/conversations/"+cid, nil)
|
||||
seq, _ := detail["lastSeq"].(float64)
|
||||
pendingMu.Lock()
|
||||
pending[cid] = pendingSend{seq: int64(seq), sent: time.Now()}
|
||||
pendingMu.Unlock()
|
||||
apiJSON("POST", *base, "/api/conversations/"+cid+"/messages", map[string]any{"text": fmt.Sprintf("probe %d", i)})
|
||||
sent++
|
||||
time.Sleep(*every)
|
||||
}
|
||||
|
||||
time.Sleep(3 * time.Second) // 最后一批帧在途余量
|
||||
close(stop)
|
||||
wg.Wait()
|
||||
|
||||
r := buildReport(*base, *dur, *convs, sent)
|
||||
data, _ := json.MarshalIndent(r, "", " ")
|
||||
if *out != "" {
|
||||
if err := os.WriteFile(*out, data, 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write report:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
fmt.Println("report:", *out)
|
||||
} else {
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
if !r.Pass {
|
||||
fmt.Fprintf(os.Stderr, "sse probe FAIL: p95=%.1fms (≤%.0fms) samples=%d outOfOrder=%d\n", r.P95MS, r.ThresholdMS, r.Samples, r.OutOfOrder)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("sse probe PASS: p95=%.1fms samples=%d\n", r.P95MS, r.Samples)
|
||||
}
|
||||
|
||||
func login(base, owner, pass string) {
|
||||
body, _ := json.Marshal(map[string]string{"owner": owner, "password": pass})
|
||||
resp, err := client.Post(base+"/api/auth/login", "application/json", bytes.NewReader(body))
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
fmt.Fprintf(os.Stderr, "probe login: err=%v status=%v\n", err, statusOf(resp))
|
||||
os.Exit(2)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func statusOf(resp *http.Response) string {
|
||||
if resp == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return strconv.Itoa(resp.StatusCode)
|
||||
}
|
||||
|
||||
func apiJSON(method, base, path string, body any) map[string]any {
|
||||
var rd io.Reader
|
||||
_ = rd
|
||||
if body != nil {
|
||||
buf, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(buf)
|
||||
}
|
||||
req, _ := http.NewRequest(method, base+path, rd)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: %v\n", method, path, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
out := map[string]any{}
|
||||
if resp.StatusCode != 200 {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: status %d\n", method, path, resp.StatusCode)
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: decode %v\n", method, path, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// subscribe 读会话 SSE：seq 严格递增校验；帧到达时与写者基准配对成延迟样本。
|
||||
func subscribe(base, cid string, stop <-chan struct{}) {
|
||||
req, _ := http.NewRequest(http.MethodGet, base+"/api/events?conversationId="+cid, nil)
|
||||
resp, err := streamClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4<<20)
|
||||
last := int64(-1)
|
||||
for sc.Scan() {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
line := sc.Text()
|
||||
if !strings.HasPrefix(line, "id: ") {
|
||||
continue
|
||||
}
|
||||
seq, _ := strconv.ParseInt(strings.TrimPrefix(line, "id: "), 10, 64)
|
||||
arrived := time.Now()
|
||||
if last != -1 && seq <= last {
|
||||
outOfOrderMu.Lock()
|
||||
outOfOrderN++
|
||||
outOfOrderMu.Unlock()
|
||||
}
|
||||
last = seq
|
||||
pendingMu.Lock()
|
||||
p, ok := pending[cid]
|
||||
pendingMu.Unlock()
|
||||
if ok && seq > p.seq {
|
||||
pendingMu.Lock()
|
||||
delete(pending, cid)
|
||||
pendingMu.Unlock()
|
||||
samplesMu.Lock()
|
||||
samples = append(samples, sample{Conversation: cid, SendSeq: seq, LatencyMS: float64(arrived.Sub(p.sent)) / float64(time.Millisecond)})
|
||||
samplesMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildReport(base string, dur time.Duration, subs, msgs int) report {
|
||||
samplesMu.Lock()
|
||||
detail := append([]sample(nil), samples...)
|
||||
samplesMu.Unlock()
|
||||
outOfOrderMu.Lock()
|
||||
ooo := outOfOrderN
|
||||
outOfOrderMu.Unlock()
|
||||
|
||||
r := report{Target: base, Duration: dur.String(), Subscribers: subs, Messages: msgs, Samples: len(detail), OutOfOrder: ooo, ThresholdMS: 1000, Detail: detail}
|
||||
if len(detail) > 0 {
|
||||
vals := make([]float64, len(detail))
|
||||
for i, s := range detail {
|
||||
vals[i] = s.LatencyMS
|
||||
}
|
||||
sort.Float64s(vals)
|
||||
pct := func(p float64) float64 { return vals[int(float64(len(vals)-1)*p)] }
|
||||
r.P50MS, r.P95MS, r.P99MS, r.MaxMS = pct(0.50), pct(0.95), pct(0.99), vals[len(vals)-1]
|
||||
}
|
||||
r.Pass = len(detail) > 0 && r.P95MS <= r.ThresholdMS && ooo == 0
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
"""svc.sh agent 数据目录快照回归（2026-10-12）。
|
||||
|
||||
验证 backup/backups：活跃 WAL 写入者下 SQLite 一致性导出（.backup + integrity_check，
|
||||
不裸拷 .db/-wal）、快照落在 runtime/ 之外的 runtime-backups/<name>/、MANIFEST 逐文件
|
||||
sha256、--keep/--prune 保留策略只清理时间戳目录、--out 自定义目录不参与清理、
|
||||
kind 守卫（postgres 引导 db backup）、缺 sqlite3 拒绝备份、rm --purge 连带删除快照。
|
||||
全部使用临时目录 + 手工登记 svc.env（backup 不触碰 bin/进程，无需桩二进制）。
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE = Path(__file__).resolve().parents[1]
|
||||
SVC_SCRIPT = SOURCE / "scripts" / "svc.sh"
|
||||
|
||||
SQLITE3 = shutil.which("sqlite3")
|
||||
|
||||
# 活跃写入者：后台进程持有 WAL 连接（提交后不关闭），保证备份时 -wal 真实存在——
|
||||
# 裸拷 .db 会丢这部分已提交内容，.backup 不会。
|
||||
WRITER = r"""
|
||||
import sqlite3, sys, time
|
||||
db = sys.argv[1]
|
||||
c = sqlite3.connect(db)
|
||||
c.execute("pragma journal_mode=wal")
|
||||
c.execute("create table if not exists t(x)")
|
||||
c.execute("insert into t values (42)")
|
||||
c.commit()
|
||||
print("ready", flush=True)
|
||||
time.sleep(float(sys.argv[2]))
|
||||
"""
|
||||
|
||||
|
||||
@unittest.skipUnless(SQLITE3, "agent 备份依赖 sqlite3 CLI（一致性导出不回退裸拷贝）")
|
||||
class SvcAgentBackupTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.mkdtemp(prefix="svc-backup-")
|
||||
self.addCleanup(shutil.rmtree, tmp, True)
|
||||
self.root = Path(tmp)
|
||||
(self.root / "scripts").mkdir(parents=True)
|
||||
shutil.copy2(SVC_SCRIPT, self.root / "scripts" / "svc.sh")
|
||||
self.writer_proc = None
|
||||
self.env = os.environ.copy()
|
||||
self.env.update({
|
||||
"GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": "/dev/null",
|
||||
"SVC_SRC_DIR": str(self.root / "srccache"),
|
||||
"XDG_CONFIG_HOME": str(self.root / "xdg"),
|
||||
})
|
||||
for key in list(self.env):
|
||||
if key.startswith("SVC_") and key != "SVC_SRC_DIR":
|
||||
self.env.pop(key)
|
||||
for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE"):
|
||||
self.env.pop(key, None)
|
||||
|
||||
# ---- 夹具 ----
|
||||
|
||||
def svc(self, *args, check=True, env=None):
|
||||
proc = subprocess.run(
|
||||
["bash", str(self.root / "scripts" / "svc.sh"), *map(str, args)],
|
||||
cwd=self.root, env=env if env is not None else self.env, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=60)
|
||||
if check:
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout)
|
||||
return proc
|
||||
|
||||
def register_agent(self, name="t1", port=8899):
|
||||
d = self.root / "runtime" / name
|
||||
(d / "data").mkdir(parents=True, exist_ok=True)
|
||||
(d / "svc.env").write_text(
|
||||
"kind=agent\nenv=%s\npid=\naddr=127.0.0.1:%d\ntags=\n"
|
||||
"source=dev@test123\nversion=stub\nstarted=\npidcheck=\n"
|
||||
"exec=--addr 127.0.0.1:%d --data %s\n" % (name, port, port, d / "data"))
|
||||
return d
|
||||
|
||||
def seed_db(self, name="t1", live_writer=True, hold_seconds=30):
|
||||
"""建 WAL 库并提交一行；live_writer=True 时留后台连接持有 WAL。"""
|
||||
db = self.root / "runtime" / name / "data" / "agent-system.db"
|
||||
if live_writer:
|
||||
self.writer_proc = subprocess.Popen(
|
||||
["python3", "-c", WRITER, str(db), str(hold_seconds)],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
self.assertEqual(self.writer_proc.stdout.readline().strip(), "ready")
|
||||
self.addCleanup(self.stop_writer)
|
||||
else:
|
||||
c = sqlite3.connect(db)
|
||||
c.execute("pragma journal_mode=wal")
|
||||
c.execute("create table t(x)")
|
||||
c.execute("insert into t values (42)")
|
||||
c.commit()
|
||||
c.close()
|
||||
return db
|
||||
|
||||
def stop_writer(self):
|
||||
if self.writer_proc and self.writer_proc.poll() is None:
|
||||
self.writer_proc.kill()
|
||||
self.writer_proc.wait(timeout=10)
|
||||
|
||||
def snapshots(self, name="t1"):
|
||||
root = self.root / "runtime-backups" / name
|
||||
if not root.is_dir():
|
||||
return []
|
||||
return sorted(p.name for p in root.iterdir() if p.is_dir())
|
||||
|
||||
# ---- 用例 ----
|
||||
|
||||
def test_backup_live_writer_consistent_and_outside_runtime(self):
|
||||
"""活跃 WAL 写入者：快照可读、行在、integrity ok；-wal/-shm 不入快照。"""
|
||||
self.register_agent()
|
||||
db = self.seed_db()
|
||||
self.assertTrue(db.with_name("agent-system.db-wal").exists()) # 前置：WAL 真实存在
|
||||
proc = self.svc("t1", "backup")
|
||||
self.assertIn("runtime-backups/t1/", proc.stdout)
|
||||
snaps = self.snapshots()
|
||||
self.assertEqual(len(snaps), 1)
|
||||
snap = self.root / "runtime-backups" / "t1" / snaps[0]
|
||||
# 位置在 runtime/ 之外（runtime 整删不波及）
|
||||
self.assertTrue(snap.exists())
|
||||
self.assertFalse((self.root / "runtime" / "t1" / "backups").exists())
|
||||
# 内容：登记 + 数据库一致性副本
|
||||
self.assertTrue((snap / "svc.env").exists())
|
||||
bak = snap / "data" / "agent-system.db"
|
||||
self.assertTrue(bak.exists())
|
||||
self.assertFalse((snap / "data" / "agent-system.db-wal").exists())
|
||||
self.assertFalse((snap / "data" / "agent-system.db-shm").exists())
|
||||
out = subprocess.run([SQLITE3, str(bak), "select x from t; pragma integrity_check;"],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
self.assertEqual(out.stdout.split(), ["42", "ok"], out.stderr)
|
||||
|
||||
def test_backup_manifest_sha_matches(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
self.svc("t1", "backup")
|
||||
snap = self.root / "runtime-backups" / "t1" / self.snapshots()[0]
|
||||
manifest = (snap / "MANIFEST").read_text()
|
||||
self.assertIn("service=t1\n", manifest)
|
||||
self.assertIn("source=dev@test123\n", manifest)
|
||||
entries = [l.split() for l in manifest.splitlines() if " ./" in l]
|
||||
self.assertTrue(entries)
|
||||
for sha, rel in entries:
|
||||
self.assertRegex(sha, r"^[0-9a-f]{64}$")
|
||||
self.assertEqual(
|
||||
subprocess.run(["sha256sum", str(snap / rel)],
|
||||
capture_output=True, text=True).stdout.split()[0], sha)
|
||||
|
||||
def test_backup_non_sqlite_db_kept_raw_without_leftovers(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
junk = self.root / "runtime" / "t1" / "data" / "blob.db"
|
||||
junk.write_bytes(b"plain-bytes-not-a-sqlite-file")
|
||||
proc = self.svc("t1", "backup")
|
||||
self.assertIn("blob.db", proc.stdout) # 告警原样保留
|
||||
snap = self.root / "runtime-backups" / "t1" / self.snapshots()[0]
|
||||
self.assertEqual((snap / "data" / "blob.db").read_bytes(), b"plain-bytes-not-a-sqlite-file")
|
||||
self.assertFalse(list(snap.rglob("*.bk"))) # 失败尝试的半成品不残留
|
||||
|
||||
def test_backup_target_exists_refused(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
dest = self.root / "runtime-backups" / "t1" / "20260101T000000Z"
|
||||
dest.mkdir(parents=True)
|
||||
(dest / "marker").write_text("x")
|
||||
proc = self.svc("t1", "backup", "--out", dest, check=False)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout)
|
||||
self.assertIn("已存在", proc.stdout)
|
||||
self.assertTrue((dest / "marker").exists()) # 不覆盖既有内容
|
||||
|
||||
def test_backup_custom_out_not_pruned(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
outdir = self.root / "custom-bk"
|
||||
self.svc("t1", "backup", "--out", outdir)
|
||||
self.assertTrue((outdir / "data" / "agent-system.db").exists())
|
||||
# 默认 root 无快照；再走默认路径备份 --keep 0 也不应动自定义目录
|
||||
self.svc("t1", "backup", "--keep", "0")
|
||||
self.assertEqual(self.snapshots(), [])
|
||||
self.assertTrue(outdir.exists())
|
||||
|
||||
def test_retention_and_listing(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
for _ in range(3):
|
||||
self.svc("t1", "backup")
|
||||
time.sleep(1.1) # 时间戳目录名唯一（UTC 秒级）
|
||||
self.assertEqual(len(self.snapshots()), 3)
|
||||
proc = self.svc("t1", "backups")
|
||||
for name in self.snapshots():
|
||||
self.assertIn(name, proc.stdout)
|
||||
# backup --keep 2 → 剩 2 份最新
|
||||
self.svc("t1", "backup", "--keep", "2")
|
||||
self.assertEqual(len(self.snapshots()), 2)
|
||||
# backups --prune --keep 1 → 剩最新 1 份
|
||||
self.svc("t1", "backups", "--prune", "--keep", "1")
|
||||
self.assertEqual(len(self.snapshots()), 1)
|
||||
|
||||
def test_backup_event_recorded(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
self.svc("t1", "backup")
|
||||
hist = (self.root / "runtime" / "t1" / "history.jsonl").read_text()
|
||||
self.assertIn('"event":"backup"', hist)
|
||||
self.assertIn('"result":"ok"', hist)
|
||||
self.assertIn("runtime-backups/t1/", hist)
|
||||
|
||||
def test_backup_rejects_postgres_with_db_hint(self):
|
||||
d = self.root / "runtime" / "pgx"
|
||||
d.mkdir(parents=True)
|
||||
(d / "svc.env").write_text("kind=postgres\nenv=pgx\naddr=127.0.0.1:8890\n")
|
||||
proc = self.svc("pgx", "backup", check=False)
|
||||
self.assertEqual(proc.returncode, 2, proc.stdout)
|
||||
self.assertIn("db backup", proc.stdout)
|
||||
|
||||
def test_backup_requires_registration(self):
|
||||
proc = self.svc("ghost", "backup", check=False)
|
||||
self.assertEqual(proc.returncode, 2, proc.stdout)
|
||||
|
||||
def test_backup_without_sqlite3_refuses(self):
|
||||
"""PATH 无 sqlite3 → 拒绝（不回退裸拷贝）。重建与真实 PATH 等价的工具目录、
|
||||
唯独不链 sqlite3——只测 command -v 失败这一条分支，其余行为不变形。"""
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
bindir = self.root / "bin-nosqlite3"
|
||||
bindir.mkdir()
|
||||
seen = set()
|
||||
for d in self.env.get("PATH", os.environ["PATH"]).split(os.pathsep):
|
||||
if not d or not os.path.isdir(d):
|
||||
continue
|
||||
for f in os.listdir(d):
|
||||
if f == "sqlite3" or f in seen or (bindir / f).exists():
|
||||
continue
|
||||
seen.add(f)
|
||||
try:
|
||||
os.symlink(os.path.join(d, f), bindir / f)
|
||||
except OSError:
|
||||
pass
|
||||
self.assertFalse((bindir / "sqlite3").exists())
|
||||
env = dict(self.env)
|
||||
env["PATH"] = str(bindir)
|
||||
proc = self.svc("t1", "backup", check=False, env=env)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout)
|
||||
self.assertIn("sqlite3", proc.stdout)
|
||||
self.assertEqual(self.snapshots(), []) # 无半成品快照
|
||||
|
||||
def test_rm_purge_removes_snapshots(self):
|
||||
self.register_agent()
|
||||
self.seed_db(live_writer=False)
|
||||
self.svc("t1", "backup")
|
||||
self.assertEqual(len(self.snapshots()), 1)
|
||||
self.svc("rm", "t1", "--purge")
|
||||
self.assertFalse((self.root / "runtime-backups" / "t1").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,235 @@
|
||||
"""svc.sh 实例身份回归（U1 设计 §7 T1–T6，manifest 治理）。
|
||||
|
||||
覆盖：连续裸启停身份稳定（T1）、只改端口锚点不动（T2）、数据根漂移拒绝（T3）、
|
||||
迁移窗口/损坏阻塞不猜测（T4）、marker 缺失/外宾/manifest 丢失均为事件（T5）、
|
||||
cwd 无关性（T6）、原子写崩溃窗口单元级（原文件不受损）。
|
||||
桩二进制与夹具复用 tests/test_svc_recover.py。真实 PG 身份专项（DB marker SQL 层）
|
||||
NOT_RUN：需真实实例，SVC_TEST_PG=1 且具备环境时另行执行，本文件不模拟。
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE = Path(__file__).resolve().parents[1]
|
||||
SVC_SCRIPT = SOURCE / "scripts" / "svc.sh"
|
||||
|
||||
_spec = importlib.util.spec_from_file_location("tsr", SOURCE / "tests" / "test_svc_recover.py")
|
||||
tsr = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(tsr)
|
||||
|
||||
|
||||
class IdentityTests(tsr.SvcAutoRecoverTests):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
shutil.copy2(SOURCE / "scripts" / "svc_manifest.py",
|
||||
self.root / "scripts" / "svc_manifest.py")
|
||||
|
||||
# ---- 夹具 ----
|
||||
|
||||
def manifest(self, name):
|
||||
p = self.runtime(name) / "manifest.json"
|
||||
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else None
|
||||
|
||||
def marker(self, name):
|
||||
m = self.manifest(name)
|
||||
if not m:
|
||||
return None
|
||||
p = Path(m["identity"]["data_root"]) / ".agentx" / "data-marker.json"
|
||||
return json.loads(p.read_text(encoding="utf-8")) if p.exists() else None
|
||||
|
||||
def running(self, name):
|
||||
pid = self.pid_of(name)
|
||||
if not pid:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
# ---- T1 连续裸启停：同一身份与数据根 ----
|
||||
|
||||
def test_T1_bare_restart_stable_identity(self):
|
||||
self.seed_agent("id1", 9431)
|
||||
self.start_ok("id1")
|
||||
m1 = self.manifest("id1")
|
||||
self.assertIsNotNone(m1, "start 应完成 manifest 迁移")
|
||||
iid = m1["instance_id"]
|
||||
self.assertTrue(iid.startswith("svc-id1-"), iid)
|
||||
mk = self.marker("id1")
|
||||
self.assertEqual(mk["instance_id"], iid)
|
||||
data_before = m1["identity"]["data_root"]
|
||||
exec_before = m1["kv"]["exec"]
|
||||
self.svc("id1", "stop")
|
||||
self.start_ok("id1")
|
||||
m2 = self.manifest("id1")
|
||||
self.assertEqual(m2["instance_id"], iid, "裸启停不得换身份")
|
||||
self.assertEqual(m2["identity"]["data_root"], data_before)
|
||||
self.assertEqual(m2["kv"]["exec"], exec_before, "裸启停不得漂移 exec（数据/插件根）")
|
||||
self.assertGreaterEqual(m2["revision"], m1["revision"], "整份提交 revision 单调")
|
||||
self.assertTrue(self.running("id1"))
|
||||
self.svc("id1", "stop")
|
||||
|
||||
# ---- T2 只改端口：数据/DB 锚点不动 ----
|
||||
|
||||
def test_T2_port_change_keeps_anchors(self):
|
||||
self.seed_agent("id2", 9432)
|
||||
self.start_ok("id2")
|
||||
anchor = self.manifest("id2")["identity"]
|
||||
self.svc("id2", "stop")
|
||||
result = self.svc("id2", "start", "--port", "9433")
|
||||
self.assertIn("已启动", result.stdout)
|
||||
m2 = self.manifest("id2")
|
||||
self.assertEqual(m2["identity"]["data_root"], anchor["data_root"], "改端口不得动数据根")
|
||||
self.assertEqual(m2["identity"]["db_mode"], anchor["db_mode"])
|
||||
self.assertEqual(m2["identity"]["db_ref"], anchor["db_ref"])
|
||||
self.assertIn("9433", m2["kv"]["addr"])
|
||||
self.svc("id2", "stop")
|
||||
|
||||
# ---- T3 数据根漂移：拒绝启动，不新建空系统 ----
|
||||
|
||||
def test_T3_data_drift_rejected(self):
|
||||
self.seed_agent("id3", 9434)
|
||||
self.start_ok("id3")
|
||||
self.svc("id3", "stop")
|
||||
other = self.root / "other-data"
|
||||
other.mkdir()
|
||||
result = self.svc("id3", "start", "--data", str(other), check=False)
|
||||
self.assertNotEqual(result.returncode, 0, "数据根漂移必须拒绝")
|
||||
self.assertTrue(any(k in result.stdout for k in ("身份校验失败", "数据根漂移")),
|
||||
result.stdout)
|
||||
self.assertFalse(self.running("id3"), "拒绝后不得有服务进程")
|
||||
self.assertEqual(self.manifest("id3")["identity"]["data_root"],
|
||||
str(self.runtime("id3") / "data"), "锚点不得被污染")
|
||||
|
||||
# ---- T4 迁移窗口与损坏：不猜测 ----
|
||||
|
||||
def test_T4a_leftover_new_file_ignored(self):
|
||||
self.seed_agent("id4", 9435)
|
||||
# 模拟 rename 前崩溃残留：.new 半成品必须被忽略/覆盖，不影响迁移结果
|
||||
(self.runtime("id4") / "manifest.json.new").write_text("{ truncated")
|
||||
self.start_ok("id4")
|
||||
m = self.manifest("id4")
|
||||
self.assertIsNotNone(m)
|
||||
self.assertEqual(m["schema"], 1)
|
||||
self.svc("id4", "stop")
|
||||
|
||||
def test_T4b_corrupt_manifest_blocks(self):
|
||||
self.seed_agent("id5", 9436)
|
||||
self.start_ok("id5")
|
||||
self.svc("id5", "stop")
|
||||
(self.runtime("id5") / "manifest.json").write_text("{ not json")
|
||||
result = self.svc("id5", "start", check=False)
|
||||
self.assertNotEqual(result.returncode, 0, "损坏 manifest 必须阻塞而非猜测重建")
|
||||
self.assertIn("损坏", result.stdout)
|
||||
self.assertFalse(self.running("id5"))
|
||||
|
||||
# ---- T5 marker/manifest 事件：缺一即阻塞，不自动新建 ----
|
||||
|
||||
def test_T5a_missing_marker_blocks(self):
|
||||
self.seed_agent("id6", 9437)
|
||||
self.start_ok("id6")
|
||||
self.svc("id6", "stop")
|
||||
m = self.manifest("id6")
|
||||
mk = Path(m["identity"]["data_root"]) / ".agentx" / "data-marker.json"
|
||||
mk.unlink()
|
||||
result = self.svc("id6", "start", check=False)
|
||||
self.assertNotEqual(result.returncode, 0, "marker 缺失即事件")
|
||||
self.assertIn("marker", result.stdout)
|
||||
self.assertFalse(self.running("id6"))
|
||||
|
||||
def test_T5b_foreign_marker_blocks(self):
|
||||
self.seed_agent("id7", 9438)
|
||||
self.start_ok("id7")
|
||||
self.svc("id7", "stop")
|
||||
m = self.manifest("id7")
|
||||
mk = Path(m["identity"]["data_root"]) / ".agentx" / "data-marker.json"
|
||||
data = json.loads(mk.read_text(encoding="utf-8"))
|
||||
data["instance_id"] = "svc-other-zzzzzzzzzzzz"
|
||||
mk.write_text(json.dumps(data), encoding="utf-8")
|
||||
result = self.svc("id7", "start", check=False)
|
||||
self.assertNotEqual(result.returncode, 0, "marker 身份不符即事件")
|
||||
self.assertFalse(self.running("id7"))
|
||||
|
||||
def test_T5c_lost_manifest_blocks_not_rebuild(self):
|
||||
self.seed_agent("id8", 9439)
|
||||
self.start_ok("id8")
|
||||
m0 = self.manifest("id8")
|
||||
iid = m0["instance_id"]
|
||||
marker_path = Path(m0["identity"]["data_root"]) / ".agentx" / "data-marker.json"
|
||||
self.svc("id8", "stop")
|
||||
(self.runtime("id8") / "manifest.json").unlink()
|
||||
result = self.svc("id8", "start", check=False)
|
||||
# manifest 丢失后 marker 仍在：重建会撞归属冲突 → 中止（I4），不得 legacy 复活
|
||||
self.assertNotEqual(result.returncode, 0, "丢 manifest 不得静默重建/legacy 复活")
|
||||
self.assertTrue(any(k in result.stdout for k in ("归属冲突", "身份治理冲突")),
|
||||
result.stdout)
|
||||
self.assertFalse(self.running("id8"))
|
||||
# marker 保持原身份，未被新随机身份覆盖
|
||||
mk = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(mk["instance_id"], iid)
|
||||
|
||||
# ---- T6 cwd 无关：从别处调用身份不变 ----
|
||||
|
||||
def test_T6_cwd_independent_identity(self):
|
||||
self.seed_agent("id9", 9440)
|
||||
self.start_ok("id9")
|
||||
iid = self.manifest("id9")["instance_id"]
|
||||
self.svc("id9", "stop")
|
||||
elsewhere = tempfile.mkdtemp(prefix="svc-elsewhere-")
|
||||
self.addCleanup(shutil.rmtree, elsewhere, True)
|
||||
proc = subprocess.run(
|
||||
["bash", str(self.root / "scripts" / "svc.sh"), "id9", "start"],
|
||||
cwd=elsewhere, env=self.env, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=120)
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout)
|
||||
self.assertIn("已启动", proc.stdout)
|
||||
self.assertEqual(self.manifest("id9")["instance_id"], iid, "换 cwd 不得换身份")
|
||||
self.svc("id9", "stop")
|
||||
|
||||
# ---- 真实 PG 专项：显式 NOT_RUN 标注（不模拟） ----
|
||||
|
||||
def test_Z_real_pg_not_run(self):
|
||||
# DB marker（agent_system_identity 表）与跨库身份断言属真实 PG 专项，
|
||||
# 由隔离真实实例执行；本桩套件明确 NOT_RUN，不以桩冒充（设计 §7）。
|
||||
self.skipTest("真实 PG 身份专项 NOT_RUN：需真实实例（SVC_TEST_PG 环境另行执行）")
|
||||
|
||||
|
||||
class AtomicWriteTests(unittest.TestCase):
|
||||
"""崩溃窗口单元级：os.replace 失败时原文件必须完好（I3 半写不可能）。"""
|
||||
|
||||
def test_replace_failure_keeps_original(self):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"sm", SOURCE / "scripts" / "svc_manifest.py")
|
||||
sm = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(sm)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = os.path.join(d, "m.json")
|
||||
sm.atomic_write(path, '{"v": 1}\n')
|
||||
real_replace = os.replace
|
||||
calls = {"n": 0}
|
||||
|
||||
def boom(a, b):
|
||||
calls["n"] += 1
|
||||
raise OSError("simulated crash before rename completes")
|
||||
|
||||
os.replace = boom
|
||||
try:
|
||||
try:
|
||||
sm.atomic_write(path, '{"v": 2}\n')
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
os.replace = real_replace
|
||||
self.assertEqual(calls["n"], 1)
|
||||
with open(path, encoding="utf-8") as f:
|
||||
self.assertEqual(f.read(), '{"v": 1}\n', "rename 前崩溃必须保留完整旧版")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user