Agent-System-Dev/tests/ops/lib.sh

#!/usr/bin/env bash
# 演练栈公共函数。由 run_fault_drill.sh / run_backup_drill.sh / self_check.sh source。
# 不打生产：名称前缀 u7drill 或 u7bk，端口 18980–18999，只杀命令行匹配的 pid。

ops_port_listening() {
  local p="$1"
  if command -v ss >/dev/null 2>&1; then
    ss -ltn 2>/dev/null | awk -v p=":$p" '$4 ~ p"$" { found=1 } END { exit !found }' && return 0
  fi
  (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null && return 0
  return 1
}

ops_require_name() {
  local name="$1"
  case "$name" in
    u7drill|u7drill-*|u7bk|u7bk-*) ;;
    *) echo "[ops] refusing instance name $name" >&2; return 2 ;;
  esac
  [[ "$name" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || {
    echo "[ops] illegal instance name $name" >&2
    return 2
  }
}

ops_require_port() {
  local port="$1"
  [[ "$port" =~ ^[0-9]+$ ]] || { echo "[ops] port must be an integer" >&2; return 2; }
  if (( port < 18980 || port > 18999 )); then
    echo "[ops] refusing port $port outside 18980-18999" >&2
    return 2
  fi
}

ops_resource_gate() {
  local root="$1" avail_kb load1
  avail_kb="$(df -Pk "$root" | awk 'NR==2 { print $4 }')"
  load1="$(awk '{ print $1 }' /proc/loadavg)"
  if (( avail_kb < 100 * 1024 * 1024 )); then
    echo "[ops] disk free ${avail_kb}KB is below 100GB; pausing" >&2
    return 3
  fi
  awk -v load1="$load1" 'BEGIN { if (load1+0 > 16) exit 1 }' || {
    echo "[ops] load average $load1 is above 16; pausing" >&2
    return 3
  }
}

# ops_kill_checked PIDFILE NEEDLE：只在 /proc/pid/cmdline 含 NEEDLE 时发信号。
ops_kill_checked() {
  local pidfile="$1" needle="$2" sig="${3:-TERM}"
  [[ -f "$pidfile" ]] || return 0
  local pid cmd
  pid="$(head -1 "$pidfile" 2>/dev/null || true)"
  [[ "$pid" =~ ^[0-9]+$ ]] || return 0
  if [[ ! -d "/proc/$pid" ]]; then
    rm -f "$pidfile"
    return 0
  fi
  cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
  if [[ "$cmd" != *"$needle"* ]]; then
    echo "[ops] refusing to signal $pid; cmdline does not contain $needle" >&2
    return 1
  fi
  kill "-$sig" "$pid" 2>/dev/null || true
}

ops_pg_create() {
  local root="$1" name="$2" port="$3"
  ops_require_name "$name" || return $?
  ops_require_port "$port" || return $?
  if ops_port_listening "$port"; then
    echo "[ops] port $port is in use; refusing before create" >&2
    return 2
  fi
  if [[ -e "$root/runtime/$name" ]]; then
    echo "[ops] runtime/$name already exists; refusing to touch it" >&2
    return 2
  fi
  "$root/scripts/svc.sh" db create "$name" --port "$port" >/dev/null
  local addr
  addr="$(sed -n 's/^addr=//p' "$root/runtime/$name/svc.env" | head -1)"
  if [[ "$addr" != "127.0.0.1:$port" ]]; then
    echo "[ops] $name addr is $addr, expected 127.0.0.1:$port" >&2
    return 1
  fi
}

ops_pg_start() {
  local root="$1" name="$2"
  "$root/scripts/svc.sh" "$name" start >/dev/null
}

ops_pg_stop() {
  local root="$1" name="$2"
  "$root/scripts/svc.sh" "$name" stop >/dev/null
}

ops_pg_kill() {
  local root="$1" name="$2"
  local pgdata="$root/runtime/$name/pgdata"
  local pidfile="$pgdata/postmaster.pid"
  [[ -f "$pidfile" ]] || { echo "[ops] no postmaster.pid for $name" >&2; return 1; }
  ops_kill_checked "$pidfile" "$pgdata" KILL
}

ops_pg_dsn() {
  local root="$1" name="$2"
  "$root/scripts/svc.sh" db dsn "$name" --reveal
}

ops_pg_remove() {
  local root="$1" name="$2"
  "$root/scripts/svc.sh" "$name" stop >/dev/null 2>&1 || true
  "$root/scripts/svc.sh" rm "$name" --purge >/dev/null 2>&1 || true
}

ops_redis_start() {
  local port="$1" dir="$2"
  ops_require_port "$port" || return $?
  if ops_port_listening "$port"; then
    echo "[ops] redis port $port is in use" >&2
    return 2
  fi
  mkdir -p "$dir"
  redis-server --bind 127.0.0.1 --port "$port" --dir "$dir" --dbfilename drill.rdb \
    --save '' --appendonly no --daemonize yes --pidfile "$dir/redis.pid" --logfile "$dir/redis.log"
}

ops_redis_kill() {
  local dir="$1" port="$2"
  ops_kill_checked "$dir/redis.pid" ":$port" KILL
}

ops_snapshot() {
  local root="$1" dest="$2"
  {
    echo "dev_head=$(git -C "$root" rev-parse HEAD)"
    echo "backend=$(git -C "$root" rev-parse HEAD:modules/backend)"
    echo "when=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
    echo "nproc=$(nproc)"
    echo "loadavg=$(cat /proc/loadavg)"
    free -h
    df -h "$root"
    echo "--- heavy processes (comm only) ---"
    ps -eo pcpu,pmem,comm --sort=-pcpu | head -12
  } >"$dest"
}