323 lines
11 KiB
Bash
Executable File
323 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 检查单 B6：隔离 PostgreSQL 上的 pgbench 参考基线。
|
||
# 只创建并回收名称以 u7bench 开头、端口落在 18980–18999 的实例。
|
||
# 不打生产服务，不把连接串或口令写进输出。
|
||
#
|
||
# 用法：
|
||
# tests/load/collaboration/run_pgbench.sh [--name u7bench] [--port 18995]
|
||
# [--scale 100] [--duration 60] [--clients 10,30,100] [--help]
|
||
# tests/load/collaboration/run_pgbench.sh --self-check
|
||
set -euo pipefail
|
||
|
||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
NAME=u7bench
|
||
PORT=18995
|
||
SCALE=100
|
||
DURATION=60
|
||
CLIENTS="10,30,100"
|
||
SELF_CHECK=0
|
||
HOLD_AFTER_START=0
|
||
|
||
usage() {
|
||
sed -n '2,9p' "$0"
|
||
}
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--name) NAME="$2"; shift 2 ;;
|
||
--port) PORT="$2"; shift 2 ;;
|
||
--scale) SCALE="$2"; shift 2 ;;
|
||
--duration) DURATION="$2"; shift 2 ;;
|
||
--clients) CLIENTS="$2"; shift 2 ;;
|
||
--self-check) SELF_CHECK=1; shift ;;
|
||
--hold-after-start) HOLD_AFTER_START=1; shift ;;
|
||
-h|--help) usage; exit 0 ;;
|
||
*) echo "[pgbench] unknown arg: $1" >&2; usage >&2; exit 2 ;;
|
||
esac
|
||
done
|
||
|
||
die() { echo "[pgbench] $*" >&2; exit 2; }
|
||
|
||
case "$NAME" in
|
||
u7bench|u7bench-*) ;;
|
||
*) die "--name must start with u7bench (refusing $NAME)" ;;
|
||
esac
|
||
[[ "$NAME" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || die "illegal instance name: $NAME"
|
||
[[ "$PORT" =~ ^[0-9]+$ ]] || die "--port must be an integer"
|
||
if (( PORT < 18980 || PORT > 18999 )); then
|
||
die "--port must be inside 18980-18999 (refusing $PORT)"
|
||
fi
|
||
[[ "$SCALE" =~ ^[0-9]+$ && "$DURATION" =~ ^[0-9]+$ ]] || die "--scale and --duration must be integers"
|
||
|
||
port_listening() {
|
||
local p="$1"
|
||
if command -v ss >/dev/null 2>&1; then
|
||
ss -ltn 2>/dev/null | awk -v p=":$p" '$4 ~ p"$" { found=1 } END { exit !found }' && return 0
|
||
fi
|
||
(exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null && return 0
|
||
return 1
|
||
}
|
||
|
||
# ---- 自检：正常 / 拒绝 / 中断回收。在创建任何实例之前跑完拒绝项。 ----
|
||
if [[ "$SELF_CHECK" -eq 1 ]]; then
|
||
echo "[pgbench] self-check: --help"
|
||
"$0" --help >/dev/null
|
||
echo "[pgbench] self-check: unknown arg exits 2"
|
||
set +e
|
||
"$0" --not-a-flag >/dev/null 2>&1
|
||
ec=$?
|
||
set -e
|
||
[[ "$ec" -eq 2 ]] || die "expected exit 2 for unknown arg, got $ec"
|
||
echo "[pgbench] self-check: name outside the prefix is refused"
|
||
set +e
|
||
"$0" --name prod --port 18995 >/dev/null 2>&1
|
||
ec=$?
|
||
set -e
|
||
[[ "$ec" -eq 2 ]] || die "expected exit 2 for --name prod, got $ec"
|
||
echo "[pgbench] self-check: busy port is refused before any instance exists"
|
||
busy=18994
|
||
python3 -m http.server "$busy" --bind 127.0.0.1 >/dev/null 2>&1 &
|
||
busy_pid=$!
|
||
for _ in $(seq 1 20); do port_listening "$busy" && break; sleep 0.1; done
|
||
set +e
|
||
"$0" --name u7bench --port "$busy" --duration 5 >/dev/null 2>&1
|
||
ec=$?
|
||
set -e
|
||
kill "$busy_pid" 2>/dev/null || true
|
||
wait "$busy_pid" 2>/dev/null || true
|
||
[[ "$ec" -eq 2 ]] || die "expected exit 2 for busy port, got $ec"
|
||
[[ ! -e "$ROOT/runtime/u7bench" ]] || die "busy-port refusal created runtime/u7bench"
|
||
echo "[pgbench] self-check: SIGINT removes the instance and the port"
|
||
# 非交互脚本里后台任务会忽略 SIGINT。打开监视模式后，子进程能接到 SIGINT。
|
||
set -m
|
||
"$0" --name u7bench --port 18995 --hold-after-start >/tmp/u7bench-hold.txt 2>&1 &
|
||
hold_pid=$!
|
||
set +m
|
||
ready=0
|
||
for _ in $(seq 1 90); do
|
||
if [[ -f "$ROOT/runtime/u7bench/hold.ready" ]]; then
|
||
ready=1
|
||
break
|
||
fi
|
||
if ! kill -0 "$hold_pid" 2>/dev/null; then
|
||
break
|
||
fi
|
||
sleep 0.5
|
||
done
|
||
[[ "$ready" -eq 1 ]] || {
|
||
echo "[pgbench] hold output:" >&2
|
||
cat /tmp/u7bench-hold.txt >&2 || true
|
||
die "hold-after-start did not become ready"
|
||
}
|
||
kill -INT -- "-$hold_pid" 2>/dev/null || kill -INT "$hold_pid" 2>/dev/null || true
|
||
set +e
|
||
wait "$hold_pid"
|
||
set -e
|
||
if port_listening 18995 || [[ -e "$ROOT/runtime/u7bench" ]]; then
|
||
die "SIGINT left port 18995 or runtime/u7bench behind"
|
||
fi
|
||
echo "[pgbench] self-check passed"
|
||
exit 0
|
||
fi
|
||
|
||
if port_listening "$PORT"; then
|
||
die "port $PORT is already in use; refusing before creating $NAME"
|
||
fi
|
||
if [[ -e "$ROOT/runtime/$NAME" ]]; then
|
||
die "runtime/$NAME already exists; refusing to touch it"
|
||
fi
|
||
|
||
command -v pgbench >/dev/null 2>&1 || die "pgbench is not installed"
|
||
|
||
# 计时窗口才受负载门槛约束，并且在创建实例之前判断。
|
||
# 生命周期自检（--hold-after-start）不测量吞吐。
|
||
resource_gate() {
|
||
local avail_kb load1
|
||
avail_kb="$(df -Pk "$ROOT" | awk 'NR==2 { print $4 }')"
|
||
load1="$(awk '{ print $1 }' /proc/loadavg)"
|
||
if (( avail_kb < 100 * 1024 * 1024 )); then
|
||
echo "[pgbench] disk free ${avail_kb}KB is below 100GB; pausing" >&2
|
||
exit 3
|
||
fi
|
||
awk -v load1="$load1" 'BEGIN { if (load1+0 > 16) exit 1 }' || {
|
||
echo "[pgbench] load average $load1 is above 16; pausing" >&2
|
||
exit 3
|
||
}
|
||
}
|
||
|
||
OWNED=0
|
||
cleanup() {
|
||
local ec=$?
|
||
trap - EXIT INT TERM
|
||
if [[ "$OWNED" -eq 1 ]]; then
|
||
"$ROOT/scripts/svc.sh" "$NAME" stop >/dev/null 2>&1 || true
|
||
"$ROOT/scripts/svc.sh" rm "$NAME" --purge >/dev/null 2>&1 || true
|
||
fi
|
||
exit "$ec"
|
||
}
|
||
trap cleanup EXIT INT TERM
|
||
|
||
if [[ "$HOLD_AFTER_START" -eq 0 ]]; then
|
||
resource_gate
|
||
fi
|
||
|
||
TS="$(date +%Y%m%d-%H%M%S)"
|
||
RESULTS="$HERE/results/pgbench-$TS"
|
||
mkdir -p "$RESULTS"
|
||
echo "[pgbench] results $RESULTS"
|
||
|
||
{
|
||
echo "dev_head=$(git -C "$ROOT" rev-parse HEAD)"
|
||
echo "backend=$(git -C "$ROOT" rev-parse HEAD:modules/backend)"
|
||
echo "when=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "nproc=$(nproc)"
|
||
echo "loadavg=$(cat /proc/loadavg)"
|
||
free -h
|
||
df -h "$ROOT"
|
||
pgbench --version
|
||
echo "--- heavy processes (comm only) ---"
|
||
ps -eo pcpu,pmem,comm --sort=-pcpu | sed -n '1,12p'
|
||
} >"$RESULTS/host.txt" 2>&1
|
||
|
||
echo "[pgbench] creating isolated instance $NAME on 127.0.0.1:$PORT"
|
||
"$ROOT/scripts/svc.sh" db create "$NAME" --port "$PORT" >"$RESULTS/svc-create.txt" 2>&1
|
||
OWNED=1
|
||
addr="$(sed -n '/^addr=/{s///p;q;}' "$ROOT/runtime/$NAME/svc.env")"
|
||
[[ "$addr" == "127.0.0.1:$PORT" ]] || {
|
||
echo "[pgbench] instance addr is $addr, expected 127.0.0.1:$PORT; aborting" >&2
|
||
exit 1
|
||
}
|
||
"$ROOT/scripts/svc.sh" "$NAME" start >"$RESULTS/svc-start.txt" 2>&1
|
||
|
||
if [[ "$HOLD_AFTER_START" -eq 1 ]]; then
|
||
echo "[pgbench] holding after start for signal test"
|
||
: >"$ROOT/runtime/$NAME/hold.ready"
|
||
sleep 3600
|
||
fi
|
||
|
||
DSN="$("$ROOT/scripts/svc.sh" db dsn "$NAME" --reveal)"
|
||
|
||
psql "$DSN" -v ON_ERROR_STOP=1 -c "SELECT version()" >"$RESULTS/pg-version.txt"
|
||
psql "$DSN" -v ON_ERROR_STOP=1 -c \
|
||
"SELECT name, setting, unit FROM pg_settings WHERE name IN ('max_connections','superuser_reserved_connections','shared_buffers','work_mem','effective_cache_size','wal_level','listen_addresses')" \
|
||
>"$RESULTS/pg-settings.txt"
|
||
|
||
max_conn="$(psql "$DSN" -tAc "SHOW max_connections")"
|
||
reserved="$(psql "$DSN" -tAc "SHOW superuser_reserved_connections")"
|
||
headroom=$(( max_conn - reserved - 2 ))
|
||
(( headroom >= 1 )) || die "no connection headroom (max=$max_conn reserved=$reserved)"
|
||
|
||
echo "[pgbench] init scale=$SCALE"
|
||
pgbench -i -s "$SCALE" -d "$DSN" >"$RESULTS/init.txt" 2>&1
|
||
psql "$DSN" -v ON_ERROR_STOP=1 -c "VACUUM ANALYZE" >"$RESULTS/vacuum.txt"
|
||
|
||
runs_tsv="$RESULTS/runs.tsv"
|
||
: >"$runs_tsv"
|
||
IFS=',' read -r -a client_list <<<"$CLIENTS"
|
||
for raw in "${client_list[@]}"; do
|
||
[[ "$raw" =~ ^[0-9]+$ ]] || die "bad client count: $raw"
|
||
clients="$raw"
|
||
capped=0
|
||
if (( clients > headroom )); then
|
||
echo "[pgbench] capping clients $clients -> $headroom (max_connections=$max_conn reserved=$reserved)"
|
||
clients="$headroom"
|
||
capped=1
|
||
fi
|
||
jobs="$clients"
|
||
ncpu="$(nproc)"
|
||
if (( jobs > ncpu )); then
|
||
jobs="$ncpu"
|
||
fi
|
||
for mode in select-only tpcb; do
|
||
echo "[pgbench] mode=$mode clients=$clients jobs=$jobs duration=${DURATION}s"
|
||
prefix="$RESULTS/${mode}-${clients}"
|
||
extra=()
|
||
if [[ "$mode" == "select-only" ]]; then
|
||
extra+=(-S)
|
||
fi
|
||
pgbench -n -c "$clients" -j "$jobs" -T "$DURATION" -M prepared -P 10 \
|
||
--sampling-rate=0.01 -l --log-prefix="$prefix" \
|
||
"${extra[@]}" -d "$DSN" | tee "$RESULTS/${mode}-${clients}.txt"
|
||
failed="$(awk '/number of failed transactions:/ { print $5; exit }' "$RESULTS/${mode}-${clients}.txt")"
|
||
tps="$(awk '/^tps = / { print $3; exit }' "$RESULTS/${mode}-${clients}.txt")"
|
||
avg="$(awk '/^latency average = / { print $4; exit }' "$RESULTS/${mode}-${clients}.txt")"
|
||
[[ -n "$failed" && "$failed" == "0" ]] || {
|
||
echo "[pgbench] failed transactions = ${failed:-missing} for $mode/$clients" >&2
|
||
exit 1
|
||
}
|
||
stats="$(python3 - "$prefix" <<'PY'
|
||
import glob, os, re, sys
|
||
prefix = sys.argv[1]
|
||
base = os.path.basename(prefix)
|
||
pat = re.compile(re.escape(base) + r"\.\d+(?:\.\d+)?$")
|
||
vals = []
|
||
for path in sorted(glob.glob(prefix + ".*")):
|
||
if not pat.match(os.path.basename(path)):
|
||
continue
|
||
with open(path) as fh:
|
||
for line in fh:
|
||
parts = line.split()
|
||
if len(parts) >= 3 and parts[2].isdigit():
|
||
vals.append(int(parts[2]))
|
||
if not vals:
|
||
sys.exit("no latency samples under " + prefix)
|
||
vals.sort()
|
||
def pct(p):
|
||
idx = int(round((p / 100.0) * (len(vals) - 1)))
|
||
return f"{vals[idx] / 1000.0:.3f}"
|
||
print(pct(50), pct(95), pct(99), len(vals))
|
||
PY
|
||
)" || exit 1
|
||
read -r p50 p95 p99 samples <<<"$stats"
|
||
awk -v p50="$p50" 'BEGIN { if (p50+0 > 60000) exit 1 }' || {
|
||
echo "[pgbench] latency sample looks unparsed (p50=${p50}ms)" >&2
|
||
exit 1
|
||
}
|
||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||
"$mode" "$clients" "$jobs" "$tps" "$failed" "$avg" "$p50" "$p95" "$p99" "$samples" \
|
||
>>"$runs_tsv"
|
||
echo "[pgbench] $mode clients=$clients tps=$tps avg=${avg}ms p50=${p50}ms p95=${p95}ms p99=${p99}ms samples=$samples capped=$capped"
|
||
load1="$(awk '{ print $1 }' /proc/loadavg)"
|
||
echo "after ${mode}-${clients} loadavg=$load1" >>"$RESULTS/host.txt"
|
||
done
|
||
done
|
||
|
||
python3 - "$RESULTS" "$runs_tsv" "$max_conn" "$reserved" "$headroom" "$SCALE" "$DURATION" <<'PY'
|
||
import json, pathlib, sys
|
||
results, tsv, max_conn, reserved, headroom, scale, duration = sys.argv[1:]
|
||
host = pathlib.Path(results, "host.txt").read_text()
|
||
def grab(key):
|
||
for line in host.splitlines():
|
||
if line.startswith(key + "="):
|
||
return line.split("=", 1)[1]
|
||
return ""
|
||
runs = []
|
||
for line in pathlib.Path(tsv).read_text().splitlines():
|
||
mode, clients, jobs, tps, failed, avg, p50, p95, p99, samples = line.split("\t")
|
||
runs.append({
|
||
"mode": mode, "clients": int(clients), "jobs": int(jobs),
|
||
"tps": float(tps), "failed": int(failed), "latency_avg_ms": float(avg),
|
||
"p50_ms": float(p50), "p95_ms": float(p95), "p99_ms": float(p99),
|
||
"samples": int(samples),
|
||
})
|
||
doc = {
|
||
"source": "SELF",
|
||
"kind": "pgbench-reference",
|
||
"note": "Database capacity only. Not a business acceptance threshold.",
|
||
"dev_head": grab("dev_head"),
|
||
"backend": grab("backend"),
|
||
"scale": int(scale),
|
||
"duration_s": int(duration),
|
||
"max_connections": int(max_conn),
|
||
"superuser_reserved_connections": int(reserved),
|
||
"client_headroom": int(headroom),
|
||
"runs": runs,
|
||
}
|
||
path = pathlib.Path(results, "pgbench-summary.json")
|
||
path.write_text(json.dumps(doc, indent=2) + "\n")
|
||
print(json.dumps(doc, indent=2))
|
||
PY
|
||
|
||
echo "[pgbench] done. summary $RESULTS/pgbench-summary.json"
|