Add immutable release dirs with switch journal and failure rollback

Why: deploy overwrote live artifacts after stopping the old service, so
a failed start or workbench check left instances in a mixed
registration-vs-artifact state with no recovery path (2026-10-12
rdm1/rdmcand); the report's U2 requires restorable updates.
Evidence: make test-governance 250 OK; five svc suites 105 OK (rows in
the task card); new tests cover layout/pidcheck stability, rollback on
start failure with desired-state restore, build failure keeping the old
service running, interrupted-switch journal resolution, and schema-
advance blocking rollback without touching the database.
Impact: agent deploys stage immutable releases/<id>/ and switch the
current symlink atomically; deploy.journal persists the switch and is
resolved at start/restart/deploy/update/recover entries; legacy layouts
are adopted on first deploy and remain readable; release/rdm tracks
verify sha256 before staging.
Refs: .task/feature-svc-u2-release-recovery.md; user goal directive
2026-10-12; Dev #474 (U1 gate); Documents unified-deploy plan r2.
This commit is contained in:
2026-10-12 13:52:29 +00:00
parent 2076994b16
commit 241bea6ce7
4 changed files with 666 additions and 74 deletions
File diff suppressed because one or more lines are too long
+371 -63
View File
@@ -1253,7 +1253,17 @@ env_config() {
ENV_DIR="$RUNTIME/$name"
ENV_PID="$ENV_DIR/svc.pid"
ENV_LOG="$ENV_DIR/svc.log"
ENV_BIN_DIR="$ENV_DIR/bin"
# U2:releases/ 不可变版本目录 + current 原子指针。已切换实例经 current 解析
# 产物(进程 cmdline 保留未解析路径——指针切换不破坏 pidcheck 匹配,也彻底
# 消除 exe=(deleted) 覆盖孤儿);legacy 实例沿用原路径(首次 deploy 领养,
# svc_release_adopt_legacy)。
if [ -L "$ENV_DIR/current" ]; then
ENV_BIN_DIR="$ENV_DIR/current/bin"
ENV_PLUGINS="$ENV_DIR/current/plugins"
else
ENV_BIN_DIR="$ENV_DIR/bin"
ENV_PLUGINS="$ENV_DIR/plugins"
fi
ENV_BIN="$ENV_BIN_DIR/agent-system"
# 按实例重置内存态——批量 update/restart 逐服务解析时不串用上一服务。
# DB_*:防跨服务串库;SVC_PIDCHECK/EXEC/VERSION/SOURCE:防 pid_alive 与
@@ -1599,6 +1609,256 @@ install_artifacts() { # SRCBIN SRCPLUGINS:agent 产物复制进服务目录。
c_ok "[$ENV_NAME] 产物就位:$ENV_BIN(plugins:$ENV_DIR/plugins)"
}
# ---- U2:不可变版本目录(releases/)+ current 原子指针 + 切换 journal + 失败回退 ----
# 布局:releases/<id>/{bin/agent-system, plugins/, htdocs/, release.json};
# current → releases/<id>(相对符号链接,temp+rename 原子切换);deploy.journal
# 记录切换过程(staging→switched→done/rolledback/blocked),任意步骤中断后由
# svc_deploy_journal_resolve 判定并恢复实际状态。legacy 目录永不删除:旧进程
# cmdline 指向 legacy 路径,pidcheck 兼容到自然换版。
svc_release_cur() { [ -L "$ENV_DIR/current" ] && readlink "$ENV_DIR/current" || true; }
svc_release_cur_id() { local c; c="$(svc_release_cur)"; printf '%s' "${c##*/}"; }
# svc_release_adopt_legacy:legacy 布局(bin/ 直接落服务目录)→ 领养为首个
# release(幂等:已有 current 即跳过;无产物即跳过——全新部署无需领养)。
svc_release_adopt_legacy() {
[ -L "$ENV_DIR/current" ] && return 0
[ -x "$ENV_DIR/bin/agent-system" ] || return 0
# 注意:bash 5.3 set -u 下同一 local 行不得引用本行先声明变量(拆行)。
local id="legacy-$(date -u +%Y%m%dT%H%M%SZ)"
local dir="$ENV_DIR/releases/$id"
mkdir -p "$dir/bin" || return 1
cp -f "$ENV_DIR/bin/agent-system" "$dir/bin/agent-system" || return 1
if [ -d "$ENV_DIR/plugins" ]; then
cp -r "$ENV_DIR/plugins" "$dir/plugins" || return 1
else
mkdir -p "$dir/plugins"
fi
[ -d "$ENV_DIR/htdocs" ] && cp -r "$ENV_DIR/htdocs" "$dir/htdocs" || true
printf '{"id":"%s","source":"legacy","created":"%s"}\n' "$id" "$(date -u +%FT%TZ)" > "$dir/release.json"
ln -s "releases/$id" "$ENV_DIR/.current.new" && mv -Tf "$ENV_DIR/.current.new" "$ENV_DIR/current" || {
rm -f "$ENV_DIR/.current.new"; return 1; }
c_info "[$ENV_NAME] 领养 legacy 制品为首个 release:releases/$id"
}
# svc_release_id:源标签净化的版本目录名(rdm-ddc09eb-20261012T131500)。
svc_release_id() {
local label
label="$(printf '%s' "${SRC_LABEL:-manual}" | tr '@' '-' | tr -cd 'a-zA-Z0-9._-' | cut -c1-40)"
printf '%s-%s' "$label" "$(date -u +%Y%m%dT%H%M%SZ)"
}
# svc_release_stage ID SRCBIN SRCPLUGINS [HTDOCS]:候选产物入不可变版本目录。
# stdout 回显 release 目录(供门禁/探测引用)。
svc_release_stage() {
local id="$1" srcbin="$2" srcplug="$3" htdocs="${4:-}"
local dir="$ENV_DIR/releases/$id"
rm -rf "$dir"
mkdir -p "$dir/bin" || return 1
cp -f "$srcbin" "$dir/bin/agent-system" || return 1
if [ -d "$srcplug" ]; then
cp -r "$srcplug" "$dir/plugins" || return 1
else
mkdir -p "$dir/plugins"
fi
if [ -n "$htdocs" ] && [ -d "$htdocs" ]; then
cp -r "$htdocs" "$dir/htdocs" || return 1
fi
printf '{"id":"%s","source":"%s","created":"%s"}\n' \
"$id" "${SRC_LABEL:-}" "$(date -u +%FT%TZ)" > "$dir/release.json"
printf '%s' "$dir"
}
# svc_release_switch ID:原子切换 current(temp 符号链接 + rename)。
svc_release_switch() {
ln -s "releases/$1" "$ENV_DIR/.current.new" || return 1
mv -Tf "$ENV_DIR/.current.new" "$ENV_DIR/current" || {
rm -f "$ENV_DIR/.current.new"; return 1; }
}
# svc_release_gc [KEEP]:裁剪历史 release(保留最近 KEEP 个,缺省 5;current 与
# legacy 永不删除)。失败静默——目录非关键路径。
svc_release_gc() {
local keep="${1:-5}" cur id dirs=()
[ -d "$ENV_DIR/releases" ] || return 0
cur="$(svc_release_cur_id)"
while IFS= read -r d; do dirs+=("$(basename "$d")"); done \
< <(ls -1dt "$ENV_DIR/releases"/*/ 2>/dev/null)
local n=${#dirs[@]} i rmv
for ((i = keep; i < n; i++)); do
rmv="${dirs[$i]}"
[ "$rmv" = "$cur" ] && continue
case "$rmv" in legacy-*) continue ;; esac
rm -rf "$ENV_DIR/releases/$rmv"
done
return 0
}
# ---- 切换 journal(deploy.journal):python3 原子写;缺 python3 时降级为无 journal ----
svc_journal_path() { printf '%s' "$ENV_DIR/deploy.journal"; }
svc_journal_write() { # PHASE [k=v ...]
local phase="$1"; shift
command -v python3 >/dev/null 2>&1 || return 0
python3 - "$phase" "$(svc_journal_path)" "$@" <<'PYJ'
import json, os, sys, time
phase, out, args = sys.argv[1], sys.argv[2], sys.argv[3:]
j = {"phase": phase, "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
for kv in args:
if "=" in kv:
k, v = kv.split("=", 1)
j[k] = v
tmp = out + ".new"
with open(tmp, "w") as f:
f.write(json.dumps(j, ensure_ascii=False) + "\n")
os.replace(tmp, out)
PYJ
}
svc_journal_get() { # KEY → 值(journal 缺失/损坏 → 空)
local f; f="$(svc_journal_path)"
[ -f "$f" ] || return 0
python3 -c 'import json,sys
try:
d = json.load(open(sys.argv[1])); v = d.get(sys.argv[2], "")
print(v if isinstance(v, str) else json.dumps(v))
except Exception:
pass' "$f" "$1" 2>/dev/null
}
svc_journal_phase() { svc_journal_get phase; }
svc_journal_clear() { rm -f "$(svc_journal_path)"; }
# svc_db_schema_version:SQLite 档当前 schema(schema_migrations 最大 id;
# 无库/无表/pg → 空)。只读连接,绝不写库。
svc_db_schema_version() {
local db="$ENV_DATA/agent-system.db"
[ -f "$db" ] || return 0
python3 -c 'import sqlite3,sys
try:
con = sqlite3.connect("file:" + sys.argv[1] + "?mode=ro", uri=True)
print(con.execute("select max(id) from schema_migrations").fetchone()[0])
con.close()
except Exception:
pass' "$db" 2>/dev/null
}
# svc_release_schema_supported ID:release.json 的 schema_at_commit(无则空)。
svc_release_schema_supported() {
local f="$ENV_DIR/releases/$1/release.json"
[ -f "$f" ] || return 0
python3 -c 'import json,sys
try:
print(json.load(open(sys.argv[1])).get("schema_at_commit") or "")
except Exception:
pass' "$f" 2>/dev/null
}
# svc_release_commit_schema ID:成功提交时把当前库 schema 记入 release.json
# (回退时据此判定旧制品可否运行;pg/无库静默跳过)。
svc_release_commit_schema() {
local f="$ENV_DIR/releases/$1/release.json" v
v="$(svc_db_schema_version)"
[ -n "$v" ] || return 0
python3 -c 'import json,sys
try:
d = json.load(open(sys.argv[1])); d["schema_at_commit"] = sys.argv[2]
open(sys.argv[1], "w").write(json.dumps(d, ensure_ascii=False) + "\n")
except Exception:
pass' "$f" "$v" 2>/dev/null || true
}
# svc_rollback_deploy REASON:切换失败 → 回退上一完整版本并恢复期望运行态。
# schema 阻断(库 schema 超过上一 release 提交时点)不自动回退启动、绝不触碰
# 业务数据库——登记 blocked 待受控迁移(U2 约束:不能靠覆盖数据库「回滚」)。
svc_rollback_deploy() {
local reason="$1"
local prev new desired pid cur_schema prev_schema
prev="$(svc_journal_get prev)"
new="$(svc_journal_get new)"
desired="$(svc_journal_get desired)"
pid="$(read_pid)"
if [ -n "$pid" ] && pid_alive "$pid"; then
c_warn "[$ENV_NAME] 回退前停止新制品进程 pid=$pid"
proc_stop "$pid" || true
rm -f "$ENV_PID"
fi
if [ -z "$prev" ] || [ ! -d "$ENV_DIR/releases/$prev" ]; then
c_err "[$ENV_NAME] 无可回退版本(prev 缺失,$reason)——保持停止;清理:rm 前先取证"
svc_journal_write blocked reason="无 prev release:$reason"
hist_event deploy fail "$pid" "回退无 prev($reason)"
return 1
fi
cur_schema="$(svc_db_schema_version)"
prev_schema="$(svc_release_schema_supported "$prev")"
if [ -n "$cur_schema" ] && [ -n "$prev_schema" ] \
&& [ "$cur_schema" -gt "$prev_schema" ] 2>/dev/null; then
c_err "[$ENV_NAME] schema 已前进(库 v$cur_schema > 上一 release 提交 v$prev_schema):旧制品无法运行,禁止自动回退"
c_err " 不触碰业务数据库。处理:deploy 换可运行制品(受控迁移后),或人工处置;登记 blocked"
svc_journal_write blocked reason="schema v$cur_schema > prev v$prev_schema($reason)"
hist_event deploy fail "$pid" "回退阻断:schema 前进($reason)"
return 1
fi
svc_release_switch "$prev" || {
svc_journal_write blocked reason="指针回退失败:$reason"
return 1
}
c_ok "[$ENV_NAME] 已回退上一版本:releases/$prev($reason)"
# 指针已切回:重解析产物路径到回退后的 release。
ENV_BIN_DIR="$ENV_DIR/current/bin"
ENV_PLUGINS="$ENV_DIR/current/plugins"
ENV_BIN="$ENV_BIN_DIR/agent-system"
hist_event deploy rollback "$pid" "new=${new:-?} prev=$prev($reason)"
svc_journal_write rolledback "prev=$prev" "new=${new:-}" "reason=$reason"
if [ "$desired" = "running" ]; then
c_info "[$ENV_NAME] 按原期望态恢复运行(上一版本)"
# 登记一致性:清掉新部署的源标签——回退后的 record_state 须回落到 svc.env
# 里上一版的 source(否则实际跑旧制品、登记却写新标签)。
SRC_LABEL=""
SVC_SOURCE="$(svc_get source)"
agent_do_start || { svc_journal_clear; return 1; }
fi
svc_journal_clear
return 0
}
# svc_deploy_journal_resolve [ACTION]:生命周期入口处理中断的切换。
# switched/verifying:指针已切但未经完整验证 → 回退上一完整版本(无 prev 的
# 首次部署中断 → 保持停止)。blocked:拒绝除 deploy 外的动作(显式换制品即
# 解除路径)。staging:未切换,清记录(孤儿 release 目录无害,随 gc 裁剪)。
svc_deploy_journal_resolve() {
local action="${1:-}" phase
[ -f "$(svc_journal_path)" ] || return 0
phase="$(svc_journal_phase)"
case "$phase" in
""|done|rolledback)
svc_journal_clear; return 0 ;;
staging)
c_warn "[$ENV_NAME] 发现中断的暂存(staging,未切换)——忽略"
svc_journal_clear; return 0 ;;
blocked)
if [ "$action" = "deploy" ]; then
c_warn "[$ENV_NAME] 存在 blocked 切换记录,本次 deploy 尝试跨越(换可运行制品)"
return 0
fi
c_err "[$ENV_NAME] 切换被阻断($(svc_journal_get reason)):先 deploy 换可运行制品,或受控迁移后再操作"
return 1 ;;
switched|verifying)
c_warn "[$ENV_NAME] 发现中断的切换($phase,new=$(svc_journal_get new))——回退上一完整版本"
svc_rollback_deploy "中断的切换(journal=$phase)" || return 1
return 0 ;;
esac
return 0
}
# svc_deploy_recycle_wt:deploy 各失败/成功出口统一回收构建 worktree(尽力而为)。
svc_deploy_recycle_wt() {
case "${SRC_KINDSRC:-}" in
dev) [ -n "${wt:-}" ] && recycle_worktree "$SRC_SHA" ;;
rdm) [ -n "${wt:-}" ] && recycle_worktree "rdm-$SRC_SHA" "$RDM_SRC_DIR" ;;
esac
return 0
}
# materialize_dev_commit FULL SHA TARGET:隔离 worktree 物化+构建;stdout 只回显 worktree
# 路径(提示信息走 stderr)。TARGET=build(agent)| frontend(static)。
materialize_dev_commit() {
@@ -1868,33 +2128,26 @@ agent_do_deploy() {
return 1
fi
hist_event deploy begin "$(read_pid)" "from=$SRC_LABEL"
# U1/U2(构建前置):先构建/物化候选并过身份门禁,再停旧服务——构建失败或候选
# 不合格时旧服务继续运行、旧制品与登记不动(2026-10-12 rdm1/rdmcand 事故:停服
# →覆盖→启动即拒,只留「登记版本≠磁盘制品」的混合状态且无恢复路径)。
local wt RDM_UI_OK=0
# U2(不可变版本目录 + 失败回退):构建 → 暂存 release → 门禁(身份/环境 DSN)
# → 停旧 → current 原子切换 → 启动与 UI 验证 → 成功提交(schema 记录 + journal
# 清理 + release 裁剪);任一步失败回退上一完整版本并恢复期望运行态
# (svc_rollback_deploy;库 schema 已前进时阻断,绝不覆盖业务数据库)。
local wt relid staged srcbin srcplug srchtdocs="" prev_id desired
svc_release_adopt_legacy \
|| { hist_event deploy fail "$(read_pid)" "legacy 领养失败($SRC_LABEL)"; return 1; }
case "$SRC_KINDSRC" in
local)
c_info "[$ENV_NAME] 构建当前工作区(make build)——唯一写主检出构建产物的路径"
make -C "$ROOT" build || { hist_event deploy fail "$(read_pid)" "构建失败($SRC_LABEL)"; return 1; }
svc_deploy_identity_gate "$BIN" \
|| { hist_event deploy fail "$(read_pid)" "候选身份门禁拒绝($SRC_LABEL)"; return 1; }
do_stop_quiet
install_artifacts "$BIN" "$ROOT/plugins" \
|| { hist_event deploy fail "$(read_pid)" "产物复制失败"; return 1; }
;;
make -C "$ROOT" build || {
c_err "[$ENV_NAME] 构建失败($SRC_LABEL)——旧服务保持运行、制品与登记不变"
hist_event deploy fail "$(read_pid)" "构建失败($SRC_LABEL)"
return 1
}
srcbin="$BIN"; srcplug="$ROOT/plugins" ;;
dev)
wt="$(materialize_dev_commit "$SRC_FULL" "$SRC_SHA" build)" \
|| { hist_event deploy fail "$(read_pid)" "物化/构建失败($SRC_LABEL)"; return 1; }
if ! svc_deploy_identity_gate "$wt/bin/agent-system"; then
hist_event deploy fail "$(read_pid)" "候选身份门禁拒绝($SRC_LABEL)"
recycle_worktree "$SRC_SHA"
return 1
fi
do_stop_quiet
install_artifacts "$wt/bin/agent-system" "$wt/plugins" \
|| { hist_event deploy fail "$(read_pid)" "产物复制失败"; return 1; }
recycle_worktree "$SRC_SHA"
;;
srcbin="$wt/bin/agent-system"; srcplug="$wt/plugins" ;;
release)
local relbin="$REL_ROOT/build/$REL_VERSION/agent-system"
if [ ! -f "$relbin" ]; then
@@ -1903,65 +2156,98 @@ agent_do_deploy() {
hist_event deploy fail "$(read_pid)" "制品缺失($SRC_LABEL)"
return 1
fi
svc_deploy_identity_gate "$relbin" \
|| { hist_event deploy fail "$(read_pid)" "候选身份门禁拒绝($SRC_LABEL)"; return 1; }
do_stop_quiet
materialize_release || { hist_event deploy fail "$(read_pid)" "物化/校验失败($SRC_LABEL)"; return 1; }
;;
local mf="$REL_ROOT/versions/$REL_VERSION/manifest.json" want have
if [ -f "$mf" ]; then
want="$(sed -n 's/.*"agent-system": *"\([0-9a-f]\{64\}\)".*/\1/p' "$mf" | head -1)"
have="$(sha256sum "$relbin" | cut -d' ' -f1)"
if [ -n "$want" ] && [ "$want" != "$have" ]; then
c_err "[$ENV_NAME] sha256 校验失败:制品与 manifest 不符($relbin)——拒绝部署"
hist_event deploy fail "$(read_pid)" "sha256 校验失败($SRC_LABEL)"
return 1
fi
[ -n "$want" ] && c_ok "[$ENV_NAME] sha256 校验通过(manifest:$REL_VERSION)"
else
c_warn "[$ENV_NAME] 无 manifest 可校验:$mf"
fi
srcbin="$relbin"; srcplug="$REL_ROOT/build/$REL_VERSION/plugins" ;;
rdm)
wt="$(materialize_rdm_commit "$SRC_FULL" "$SRC_SHA")" \
|| { hist_event deploy fail "$(read_pid)" "物化/构建失败($SRC_LABEL)"; return 1; }
# RDM 组合插件编译进二进制(provides 无声明式资产)——插件目录按空集安装。
# RDM 组合插件编译进二进制(provides 无声明式资产)——插件目录按空集。
mkdir -p "$wt/.svc-empty-plugins"
if ! svc_deploy_identity_gate "$wt/bin/agent-system-rdm"; then
hist_event deploy fail "$(read_pid)" "候选身份门禁拒绝($SRC_LABEL)"
recycle_worktree "rdm-$SRC_SHA" "$RDM_SRC_DIR"
return 1
fi
do_stop_quiet
install_artifacts "$wt/bin/agent-system-rdm" "$wt/.svc-empty-plugins" \
|| { hist_event deploy fail "$(read_pid)" "产物复制失败"; return 1; }
# RDM 自带零构建工作台(web/)落 htdocs/;收尾自动部署伴生 ${name}web。
if [ -d "$wt/web" ]; then
rm -rf "$ENV_DIR/htdocs"
if cp -r "$wt/web" "$ENV_DIR/htdocs"; then
RDM_UI_OK=1
c_ok "[$ENV_NAME] RDM 工作台就位:$ENV_DIR/htdocs(将自动部署伴生 $(rdm_companion_name "$ENV_NAME"))"
else
c_warn "[$ENV_NAME] RDM web/ 工作台复制失败"
fi
else
c_warn "[$ENV_NAME] RDM 源无 web/ 工作台目录"
fi
recycle_worktree "rdm-$SRC_SHA" "$RDM_SRC_DIR"
;;
srcbin="$wt/bin/agent-system-rdm"; srcplug="$wt/.svc-empty-plugins"
[ -d "$wt/web" ] && srchtdocs="$wt/web" ;;
esac
# 新产物承接能力复核(环境 DSN 交付):不支持即失败——绑定保持可恢复,不降级启动。
if [ "$DB_MODE" = pg ] && [ "$DB_SOURCE" != conffile ] && ! bin_env_dsn_supported "$ENV_BIN"; then
relid="$(svc_release_id)"
staged="$(svc_release_stage "$relid" "$srcbin" "$srcplug" ${srchtdocs:+"$srchtdocs"})" \
|| { hist_event deploy fail "$(read_pid)" "产物暂存失败($SRC_LABEL)"; svc_deploy_recycle_wt; return 1; }
if ! svc_deploy_identity_gate "$staged/bin/agent-system"; then
hist_event deploy fail "$(read_pid)" "候选身份门禁拒绝($SRC_LABEL)"
rm -rf "$staged"
svc_deploy_recycle_wt
return 1
fi
# 新产物承接能力复核(环境 DSN 交付;候选上、停服前):不支持即失败——绑定
# 保持可恢复,不降级启动。
if [ "$DB_MODE" = pg ] && [ "$DB_SOURCE" != conffile ] && ! bin_env_dsn_supported "$staged/bin/agent-system"; then
c_err "[$ENV_NAME] 新产物不消费环境变量 DSN(AGENT_SYSTEM_DSN)——部署中止;旧绑定保留在 svc.env"
c_err " 恢复:换部署版本,或改用携带 dsn 的配置文件(scripts/svc.sh $ENV_NAME config <file.json>)"
hist_event deploy fail "$(read_pid)" "新产物不支持环境 DSN($SRC_LABEL)"
rm -rf "$staged"
return 1
fi
prev_id="$(svc_release_cur_id)"
desired="stopped"; pid_alive "$(read_pid)" 2>/dev/null && desired="running"
svc_journal_write staging "new=$relid" "prev=$prev_id" "desired=$desired"
do_stop_quiet
svc_release_switch "$relid" || {
svc_journal_write blocked reason="current 指针切换失败"
hist_event deploy fail "$(read_pid)" "current 指针切换失败($SRC_LABEL)"
return 1
}
svc_journal_write switched "new=$relid" "prev=$prev_id" "desired=$desired"
c_ok "[$ENV_NAME] 已切换 release:releases/$relid(prev:${prev_id:-无})"
# 指针已切:重解析产物路径(env_config 解析时 current 可能尚不存在)。
ENV_BIN_DIR="$ENV_DIR/current/bin"
ENV_PLUGINS="$ENV_DIR/current/plugins"
ENV_BIN="$ENV_BIN_DIR/agent-system"
if ! agent_do_start; then
svc_rollback_deploy "新制品启动失败($SRC_LABEL)" || true
hist_event deploy fail "$(read_pid)" "启动失败已回退($SRC_LABEL)"
return 1
fi
agent_do_start || { hist_event deploy fail "$(read_pid)" "启动失败($SRC_LABEL)"; return 1; }
case "$SRC_KINDSRC" in
rdm)
if [ "$RDM_UI_OK" != 1 ]; then
c_err "[$ENV_NAME] RDM 工作台未就位(ui-missing):缺少可伺服的 htdocs/"
hist_event deploy fail "$(read_pid)" "ui-missing(htdocs 未就位)"
return 1
fi
if ! deploy_rdm_companion_ui "$(rdm_companion_name "$ENV_NAME")" "$ENV_DIR/htdocs"; then
hist_event deploy fail "$(read_pid)" "ui-missing(伴生部署失败)"
# 顶层 htdocs → current/htdocs 符号链接:伴生经它伺服,指针切换自动换内容
# (legacy 实体目录内容已随领养保入 release,替换不丢)。
if [ -n "$srchtdocs" ]; then
if [ ! -L "$ENV_DIR/htdocs" ]; then
rm -rf "$ENV_DIR/htdocs"
ln -s current/htdocs "$ENV_DIR/htdocs"
fi
c_ok "[$ENV_NAME] RDM 工作台就位:releases/$relid/htdocs → $ENV_DIR/htdocs(伴生 $(rdm_companion_name "$ENV_NAME"))"
if ! deploy_rdm_companion_ui "$(rdm_companion_name "$ENV_NAME")" "$ENV_DIR/htdocs"; then
svc_rollback_deploy "RDM 伴生工作台部署失败(ui-missing)" || true
hist_event deploy fail "$(read_pid)" "ui-missing(伴生部署失败,已回退)"
return 1
fi
else
svc_rollback_deploy "RDM 源无 web/ 工作台(ui-missing)" || true
hist_event deploy fail "$(read_pid)" "ui-missing(源无 web/,已回退)"
return 1
fi
;;
esac
if ! assert_ui_ready; then
hist_event deploy fail "$(read_pid)" "ui-missing"
svc_rollback_deploy "UI 探活失败(ui-missing)" || true
hist_event deploy fail "$(read_pid)" "ui-missing(已回退)"
return 1
fi
hist_event deploy ok "$(read_pid)" "from=$SRC_LABEL"
svc_release_commit_schema "$relid"
svc_journal_clear
svc_release_gc
svc_deploy_recycle_wt
hist_event deploy ok "$(read_pid)" "from=$SRC_LABEL release=$relid"
return 0
}
@@ -2380,6 +2666,9 @@ agent_do_start() {
local exec_args
if ! agent_has_explicit_cfg && [ -n "$(svc_get exec)" ]; then
exec_args="$(svc_get exec)"
# U2:release 治理实例的持久化 exec 里 legacy plugins 路径跟随 current
#(指针切换后插件目录随 release 解析)。
[ -L "$ENV_DIR/current" ] && exec_args="${exec_args//$ENV_DIR\/plugins/$ENV_PLUGINS}"
c_info "[$ENV_NAME] 复用上次配置(svc.env exec=;显式参数可覆盖):$(mask_dsn "$exec_args")"
else
local args="" explicit_auth="${CLI_AUTH:-$(env_var "$ENV_NAME" AUTH)}"
@@ -2398,7 +2687,7 @@ agent_do_start() {
c_warn "[$ENV_NAME] 该产物不支持部分启动参数(旧版本二进制),已自动省略:${args:-<无>} → ${adjusted:-<无>}"
fi
args="$adjusted"
exec_args="--addr $ENV_ADDR --data $ENV_DATA --plugins ${CLI_PLUGINS:-$ENV_DIR/plugins} $args $CLI_EXTRA"
exec_args="--addr $ENV_ADDR --data $ENV_DATA --plugins ${CLI_PLUGINS:-$ENV_PLUGINS} $args $CLI_EXTRA"
exec_args="$(printf '%s' "$exec_args" | sed 's/[[:space:]]*$//')"
fi
SVC_EXEC="$exec_args"
@@ -4405,6 +4694,15 @@ cmd_update() {
failed=$((failed + 1))
continue
fi
# U2:批内同样处理中断切换(deploy 可跨越 blocked)。
if ! svc_deploy_journal_resolve deploy; then
hist_event update fail "$(read_pid)" "from=$from to=$tov(中断切换处置失败)"
c_err "[$name] 更新失败(deploy.journal 处置),其余服务继续"
svc_unlock
CLI_FROM=""
failed=$((failed + 1))
continue
fi
if agent_do_deploy; then
hist_event update ok "$(read_pid)" "from=$from to=$tov"
c_ok "[$name] 已更新:$from → $tov"
@@ -4479,6 +4777,11 @@ EOF
c_warn "[$n] 生命周期锁被占用(deploy/手动操作进行中),本轮跳过"
skipped=$((skipped + 1)); continue
fi
# U2:拉起前处置中断的切换(switched → 回退上一完整版本再按期望态启动)。
if ! svc_deploy_journal_resolve recover; then
c_warn "[$n] 存在阻断的切换记录,本轮跳过(见 deploy.journal)"
failed=$((failed + 1)); rc=1; svc_unlock; continue
fi
c_info "[$n] recover:autostart=1 且进程已死,重新拉起(复用持久化配置)"
if do_start; then
hist_event recover ok "$(read_pid)" "autostart revival"
@@ -4700,6 +5003,11 @@ main() {
case "$action" in
start|restart|deploy|init) svc_manifest_migrate || exit 1 ;;
esac
# U2:入口处理中断的切换(deploy.journal)。blocked 拒绝除 deploy 外的动作
#(显式换制品即解除路径);start/restart 触发回退判定。
case "$action" in
start|restart|deploy) svc_deploy_journal_resolve "$action" || exit 1 ;;
esac
case "$action" in
deploy)
case "$SVC_KIND" in
+6 -4
View File
@@ -796,12 +796,14 @@ class SvcDbRecoveryTests(unittest.TestCase):
env = self.rdm_env(remote)
result = self.svc("rdm1", "deploy", "--from", "rdm", "--port", "18841", env=env)
d = self.runtime("rdm1")
# 组合二进制复制改名落位、插件目录空集、工作台静态资产落 htdocs/。
self.assertTrue((d / "bin" / "agent-system").exists())
self.assertTrue((d / "bin" / "agent-system").stat().st_mode & 0o111)
# U2:组合产物入不可变 release,经 current 指针解析;插件空集、工作台随 release。
active = d / "current"
self.assertTrue(active.is_symlink(), "current 指针须就位")
self.assertTrue((active / "bin" / "agent-system").exists())
self.assertTrue((active / "bin" / "agent-system").stat().st_mode & 0o111)
self.assertEqual(self.kv("rdm1", "kind"), "agent")
self.assertTrue(self.kv("rdm1", "source").startswith("rdm@"))
self.assertEqual(list((d / "plugins").iterdir()), [])
self.assertEqual(list((active / "plugins").iterdir()), [])
self.assertIn("rdm workbench", (d / "htdocs" / "index.html").read_text())
self.assertIn("args:", self.serve_record()) # 健康通过即登记启动参数
# 伴生 static 自动部署并探活 200。
+228
View File
@@ -0,0 +1,228 @@
"""svc.sh U2 回归:不可变版本目录(releases/)+ current 原子切换 + deploy.journal
+ 失败回退(2026-10-12 统一部署方案 U2)。
覆盖:release 布局与 current 解析(pidcheck 经未解析路径跨版本稳定)、启动失败
回退上一完整版本并恢复期望态、构建失败旧服务保持运行、中断切换经 start/recover
入口回退、schema 前进阻断回退(不触碰业务库)、legacy 布局兼容。夹具复用
tests/test_svc_identity.py(IdentityTests:manifest 治理 + 源仓轨道 + noid 桩)。
"""
import importlib.util
import json
import os
import shutil
import subprocess
import unittest
from pathlib import Path
SOURCE = Path(__file__).resolve().parents[1]
_spec = importlib.util.spec_from_file_location("tsr", SOURCE / "tests" / "test_svc_recover.py")
tsr = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(tsr)
_spec2 = importlib.util.spec_from_file_location("tsi", SOURCE / "tests" / "test_svc_identity.py")
tsi = importlib.util.module_from_spec(_spec2)
_spec2.loader.exec_module(tsi)
class U2ReleaseTests(tsi.IdentityTests):
"""manifest 治理 + identity 桩 + dev 轨道源仓(make_src_repo 等)。"""
def setUp(self):
super().setUp()
# --from local 需要根为 git 仓(local@HEAD 作源标签);recover 夹具不初始化。
self.git_ok(self.root, "init", "--initial-branch=main", ".")
self.git_ok(self.root, "add", ".")
self.git_ok(self.root, "commit", "-m", "u2 fixture")
# ---- 夹具 ----
def cur_release(self, name):
c = self.runtime(name) / "current"
return c.resolve().name if c.is_symlink() else None
def active_bin(self, name):
return self.runtime(name) / "current" / "bin" / "agent-system"
def cmdline_of(self, pid):
return Path("/proc/%d/cmdline" % pid).read_bytes().replace(b"\0", b" ").decode()
def journal(self, name):
f = self.runtime(name) / "deploy.journal"
return json.loads(f.read_text()) if f.exists() else None
def die_stub(self):
"""identity 可用、serve 即退的桩(模拟新制品启动失败)。"""
p = self.root / "stub-agent-system-die"
if not p.exists():
stub = tsr.STUB
head, sep, tail = stub.partition(" serve)")
assert sep, "STUB 缺 serve 分支锚点"
_, sep2, tail2 = tail.partition(' wait "$child"\n ;;')
assert sep2, "STUB 缺 serve 分支结尾锚点"
p.write_text(head + ' serve)\n shift\n'
+ ' echo "die-stub: serve exits immediately" >&2\n'
+ " exit 1 ;;" + tail2)
p.chmod(0o755)
return p
def set_makefile(self, stub):
(self.root / "Makefile").write_text(
"build:\n\tmkdir -p bin\n\tcp %s bin/agent-system\n" % stub)
def bump_schema(self, name, version):
"""把库 schema_migrations 推进到 version(桩库无该表则建;模拟新制品
已迁移数据库)。"""
import sqlite3
db = self.runtime(name) / "data" / "agent-system.db"
con = sqlite3.connect(str(db))
con.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations"
"(id INTEGER PRIMARY KEY, name TEXT, applied_at TEXT)")
for i in (1, 2, 3, version):
con.execute(
"INSERT OR IGNORE INTO schema_migrations(id,name,applied_at)"
" VALUES(?, 'u2-test', '2026-10-12T00:00:00Z')", (i,))
con.commit()
con.close()
# ---- T1:release 布局 + 启动失败回退 + 期望态恢复 ----
def test_U2_layout_switch_and_rollback_on_start_failure(self):
self.make_src_repo() # 预克隆 srccache(本用例走 --from local,仅保环境一致)
self.seed_agent("u2a", 9461)
self.start_ok("u2a") # legacy 运行中(期望态=running)
self.assertTrue(self.running("u2a"))
# v1 部署:legacy 领养 → current 切换 → 经 current 启动
self.set_makefile(self.stub)
result = self.svc("u2a", "deploy", "--from", "local")
self.assertIn("已切换 release", result.stdout)
self.assertIn("领养 legacy", result.stdout)
cur1 = self.cur_release("u2a")
self.assertTrue(cur1 and cur1.startswith("local-"), cur1)
self.assertTrue(self.active_bin("u2a").exists())
releases = sorted((self.runtime("u2a") / "releases").iterdir())
self.assertEqual(len(releases), 2, "legacy + v1 两个 release")
pid = self.pid_of("u2a")
self.assertIn("current/bin/agent-system", self.cmdline_of(pid),
"进程 cmdline 须走 current 未解析路径(pidcheck 跨版本稳定)")
self.assertIn("current/bin", self.kv("u2a", "pidcheck") or "")
v1_bin = self.active_bin("u2a").read_bytes()
self.assertEqual(v1_bin, Path(self.stub).read_bytes(), "v1 制品 = 可用桩")
self.assertIsNone(self.journal("u2a"), "成功后 journal 清理")
# v2 部署:serve 即退 → 回退 v1 并恢复运行
self.set_makefile(self.die_stub())
result = self.svc("u2a", "deploy", "--from", "local", check=False)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("已回退上一版本", result.stdout)
self.assertTrue(self.running("u2a"), "回退须按原期望态恢复运行")
pid2 = self.pid_of("u2a")
self.assertNotEqual(pid2, pid)
self.assertIn("current/bin/agent-system", self.cmdline_of(pid2))
self.assertEqual(self.active_bin("u2a").read_bytes(), v1_bin, "回退后制品=v1")
hist = (self.runtime("u2a") / "history.jsonl").read_text()
self.assertIn('"event":"deploy"', hist)
self.assertIn("rollback", hist)
self.assertIsNone(self.journal("u2a"))
self.svc("u2a", "stop")
# ---- T2:构建失败 → 旧服务保持运行、current 不动 ----
def test_U2_build_failure_keeps_old_running(self):
self.seed_agent("u2b", 9462)
self.set_makefile(self.stub)
self.svc("u2b", "deploy", "--from", "local")
cur1 = self.cur_release("u2b")
pid = self.pid_of("u2b")
(self.root / "Makefile").write_text("build:\n\texit 1\n")
result = self.svc("u2b", "deploy", "--from", "local", check=False)
self.assertNotEqual(result.returncode, 0)
self.assertIn("构建失败", result.stdout)
self.assertTrue(self.running("u2b"), "构建失败旧服务必须保持运行")
self.assertEqual(self.pid_of("u2b"), pid)
self.assertEqual(self.cur_release("u2b"), cur1, "current 指针不动")
self.svc("u2b", "stop")
# ---- T3:中断切换(journal switched)→ start 入口回退并恢复 ----
def test_U2_journal_interrupt_resolved_on_start(self):
self.seed_agent("u2c", 9463)
self.set_makefile(self.stub)
self.svc("u2c", "deploy", "--from", "local")
v1 = self.cur_release("u2c")
self.svc("u2c", "stop")
# 伪造中断于「指针已切」:current → 假 release,journal switched
fake = self.runtime("u2c") / "releases" / "fake-interrupt"
(fake / "bin").mkdir(parents=True)
shutil.copy2(self.die_stub(), fake / "bin" / "agent-system")
(self.runtime("u2c") / "current").unlink()
(self.runtime("u2c") / "current").symlink_to("releases/fake-interrupt")
(self.runtime("u2c") / "deploy.journal").write_text(json.dumps({
"phase": "switched", "new": "fake-interrupt", "prev": v1,
"desired": "running", "ts": "2026-10-12T00:00:00Z"}))
result = self.svc("u2c", "start")
self.assertIn("中断的切换", result.stdout)
self.assertIn("已回退上一版本", result.stdout)
self.assertTrue(self.running("u2c"))
self.assertEqual(self.cur_release("u2c"), v1)
self.assertIsNone(self.journal("u2c"))
self.svc("u2c", "stop")
# ---- T4:schema 前进 → 回退阻断,不触碰业务库 ----
def test_U2_schema_advance_blocks_rollback(self):
self.seed_agent("u2d", 9464)
self.set_makefile(self.stub)
self.svc("u2d", "deploy", "--from", "local")
v1 = self.cur_release("u2d")
self.svc("u2d", "stop")
# v1 提交时点 schema=3;库被「新制品」推进到 5
rf = self.runtime("u2d") / "releases" / v1 / "release.json"
data = json.loads(rf.read_text())
data["schema_at_commit"] = "3"
rf.write_text(json.dumps(data))
self.bump_schema("u2d", 5)
fake = self.runtime("u2d") / "releases" / "fake-advance"
(fake / "bin").mkdir(parents=True)
shutil.copy2(self.die_stub(), fake / "bin" / "agent-system")
(self.runtime("u2d") / "current").unlink()
(self.runtime("u2d") / "current").symlink_to("releases/fake-advance")
(self.runtime("u2d") / "deploy.journal").write_text(json.dumps({
"phase": "switched", "new": "fake-advance", "prev": v1,
"desired": "running", "ts": "2026-10-12T00:00:00Z"}))
result = self.svc("u2d", "start", check=False)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("禁止自动回退", result.stdout)
self.assertFalse(self.running("u2d"), "阻断后保持停止")
j = self.journal("u2d")
self.assertEqual(j["phase"], "blocked")
# 业务库不被回写:schema 仍是 5
import sqlite3
con = sqlite3.connect(str(self.runtime("u2d") / "data" / "agent-system.db"))
self.assertEqual(
con.execute("select max(id) from schema_migrations").fetchone()[0], 5)
con.close()
# blocked 下普通动作被拒;deploy 是解除路径(此处仅验证拒绝语义)
result = self.svc("u2d", "restart", check=False)
self.assertNotEqual(result.returncode, 0)
self.assertIn("切换被阻断", result.stdout)
# ---- T5:legacy 布局未迁移 → 裸 start/restart 仍走原路径(兼容) ----
def test_U2_legacy_layout_bare_lifecycle_still_works(self):
self.seed_agent("u2e", 9465)
self.start_ok("u2e")
self.assertFalse((self.runtime("u2e") / "current").exists())
self.assertTrue((self.runtime("u2e") / "bin" / "agent-system").exists())
self.svc("u2e", "stop")
self.start_ok("u2e")
self.assertTrue(self.running("u2e"))
self.svc("u2e", "stop")
if __name__ == "__main__":
unittest.main()