Add U7 two-process acceptance, k6 load harness and release checklist
Why: collaboration storage U7 (proposal sections 3/5/7) requires two-process HTTP-level acceptance over shared PG, a reproducible load-test entry honoring the load contract, and an auditable pre-release checklist; capacity windows and dual-host runs need user-scheduled resources, so the deliverable here is the verifiable entry, not capacity claims. Evidence: make test green (composite); make test-multi-instance green locally against an isolated PG (five scenarios incl. crash-takeover and restart invariants); run_load.sh --profile smoke end-to-end green (SSE probe p95~18ms, zero seq duplicates, stack auto-recycled); backend pinned to feature branch with the three cross-instance fixes. Impact: new tests/integration/multi-instance (CI test-pg job step), new tests/load/collaboration (k6 + SSE probe + run_load.sh), release checklist at docs/release-multi-instance.md; no runtime behavior change in Dev repo itself. Refs: backend PR #164; task card feature/u7-load-fault-release.
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
/.build/
|
||||
/results/
|
||||
@@ -0,0 +1,50 @@
|
||||
# 协作存储压测套件(U7)
|
||||
|
||||
方案 §3 负载合同 / §5 U7 的业务压测入口。**只打隔离栈**(自建 PG 实例 +
|
||||
mock 模式 backend),绝不打生产服务;模拟模型(mock)用于容量基线,真实
|
||||
模型的全链路验证按方案另行小流量执行。
|
||||
|
||||
## 组成
|
||||
|
||||
| 文件 | 职责 |
|
||||
|---|---|
|
||||
| `run_load.sh` | 编排入口:起隔离栈 → 引导账号 → k6 阶梯 → SSE 探针 → 不变量对账 → 回收 |
|
||||
| `k6/collab_api.js` | 非模型普通 API 的混合读写负载(登录/项目/会话/卡片/设置/消息,requestId 幂等重放抽样;预期 409/429 单列计数) |
|
||||
| `sse_probe/` | SSE 持久事件可见延迟探针(Go):k6 核心不支持 SSE 流式读取,P95≤1s 指标由此承担,含 seq 单调性断言 |
|
||||
| `results/<ts>/` | 每次运行的原始输出:`k6-summary.json`、`k6-output.txt`、`sse-probe.json`、`invariants.txt`、`app.log` |
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
# 冒烟(~2 分钟,无需 k6——缺失时自动跳过 k6 段并给出安装指引)
|
||||
tests/load/collaboration/run_load.sh --profile smoke
|
||||
|
||||
# 阶梯(10→30→100 VU,~12 分钟;k6: brew install k6)
|
||||
tests/load/collaboration/run_load.sh --profile ladder --users 30
|
||||
|
||||
# 长稳(60 分钟 / 2 小时混合,负载合同的两个正式窗口)
|
||||
tests/load/collaboration/run_load.sh --profile sustained60m
|
||||
tests/load/collaboration/run_load.sh --profile sustained2h
|
||||
|
||||
# 复用已有隔离 PG;保留栈便于复查
|
||||
tests/load/collaboration/run_load.sh --dsn 'postgresql://…/agent' --keep
|
||||
```
|
||||
|
||||
## 阈值与口径(负载合同,待实测校准)
|
||||
|
||||
- 非模型普通 API:读路径 P95≤300ms、P99≤1s(k6 threshold 按 `read` 组单列)。
|
||||
- 非预期失败率≤0.1%(`http_req_failed<0.001`);预期拥塞(会话忙 409 /
|
||||
队列满 429)记入 `expected_contention` 计数器,**不**计入失败。
|
||||
- SSE 持久事件可见延迟 P95≤1s + seq 零乱序(探针退出码即断言)。
|
||||
- 数据不变量零违反:events 每会话 seq 无重复(SQL 对账,违反即 exit 1);
|
||||
各表计数与孤儿 run 报告落 `invariants.txt`。
|
||||
|
||||
## 边界(诚实声明)
|
||||
|
||||
- 首期为**单实例**压测(mock 模式、执行原语关闭);分布式限流窗口与双实例
|
||||
放行指标在双栈就位后扩展(`--dsn` 复用 + 第二实例待加)。
|
||||
- 容量数字(连接等待、CPU/内存/IO、pgbench 数据库基线)须绑定组件 SHA、
|
||||
配置与数据集记录在结果目录;`pgbench` 只测数据库能力,不替代本套业务
|
||||
验收(方案 §5 U7)。
|
||||
- 正式 60min/2h 窗口的执行与双主机验证需要用户指定资源与时间;本仓提供
|
||||
的是可复核入口,不预填容量结论。
|
||||
@@ -0,0 +1,137 @@
|
||||
// 协作存储 U7 压测(方案 §3 负载合同 / §5 U7):k6 覆盖非模型普通 API
|
||||
// 的吞吐与延迟阶梯;SSE 可见延迟 P95 由 Go 探针(../sse_probe)承担——
|
||||
// k6 核心不支持 SSE 流式读取,两者分工记录在同一份结果目录。
|
||||
//
|
||||
// 目标实例:run_load.sh 起的隔离栈(mock 模式 backend + 隔离 PG),绝不打
|
||||
// 生产服务。预期限流/冲突(409/429)单列计数,不计入失败。
|
||||
//
|
||||
// 阶梯与环境变量(由 run_load.sh 注入):
|
||||
// BASE_URL 目标基址(默认 http://127.0.0.1:18980)
|
||||
// ADMIN_* 引导管理员(run_load.sh 预建;setup 里登录后建压测用户)
|
||||
// USERS 压测用户数(默认 30;每 VU 绑定一个用户)
|
||||
// PROFILE smoke | ladder | sustained60m | sustained2h(默认 smoke)
|
||||
//
|
||||
// 阈值(负载合同,待实测后校准):
|
||||
// 读路径 p(95)<300ms、全量 p(99)<1s、非预期失败率<0.1%。
|
||||
|
||||
import http from 'k6/http';
|
||||
import { check, group, sleep } from 'k6';
|
||||
import { Counter, Trend } from 'k6/metrics';
|
||||
import exec from 'k6/execution';
|
||||
|
||||
const BASE = __ENV.BASE_URL || 'http://127.0.0.1:18980';
|
||||
const ADMIN_OWNER = __ENV.ADMIN_OWNER || 'load_admin';
|
||||
const ADMIN_PASS = __ENV.ADMIN_PASS || 'load-admin-pass';
|
||||
const USERS = parseInt(__ENV.USERS || '30', 10);
|
||||
const PROFILE = __ENV.PROFILE || 'smoke';
|
||||
|
||||
// 预期拥塞(负载合同要求单列,不算失败):409 会话忙 / 429 队列满。
|
||||
const expectedContention = new Counter('expected_contention');
|
||||
const msgLatency = new Trend('message_roundtrip', true);
|
||||
|
||||
const stages = {
|
||||
smoke: [{ duration: '30s', target: 10 }],
|
||||
ladder: [
|
||||
{ duration: '2m', target: 10 },
|
||||
{ duration: '3m', target: 30 },
|
||||
{ duration: '5m', target: 100 },
|
||||
{ duration: '2m', target: 0 },
|
||||
],
|
||||
sustained60m: [
|
||||
{ duration: '2m', target: 30 },
|
||||
{ duration: '56m', target: 30 },
|
||||
{ duration: '2m', target: 0 },
|
||||
],
|
||||
sustained2h: [
|
||||
{ duration: '3m', target: 30 },
|
||||
{ duration: '114m', target: 30 },
|
||||
{ duration: '3m', target: 0 },
|
||||
],
|
||||
};
|
||||
|
||||
export const options = {
|
||||
stages: stages[PROFILE] || stages.smoke,
|
||||
thresholds: {
|
||||
// 非预期失败:2xx 之外的(409/429 已在 expected_contention 单列)。
|
||||
http_req_failed: ['rate<0.001'],
|
||||
'http_req_duration{group:::read}': ['p(95)<300', 'p(99)<1000'],
|
||||
http_req_duration: ['p(99)<1000'],
|
||||
},
|
||||
summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'],
|
||||
};
|
||||
|
||||
// ---- setup:管理员建压测用户,各自建项目与会话(数据集随用户数扩展) ----
|
||||
|
||||
export function setup() {
|
||||
// setup 上下文的 cookie jar 自动附着(k6 语义:jar 不作为请求器使用)。
|
||||
const login = http.post(`${BASE}/api/auth/login`, JSON.stringify({ owner: ADMIN_OWNER, password: ADMIN_PASS }), { headers: { 'Content-Type': 'application/json' } });
|
||||
if (login.status !== 200) throw new Error(`admin login: ${login.status} ${login.body}`);
|
||||
// 首登强制改密由 run_load.sh 预处理(CLI 建号即带终态口令),此处直接可用。
|
||||
|
||||
const users = [];
|
||||
for (let i = 0; i < USERS; i++) {
|
||||
const owner = `load_u${i}`;
|
||||
const password = `load-user-pass-${i}-Aa9`;
|
||||
const created = http.post(`${BASE}/api/admin/users`, JSON.stringify({ owner, name: `压测用户${i}`, password, isAdmin: false }), { headers: { 'Content-Type': 'application/json' } });
|
||||
// 409 = 已存在(重复压测同一栈),接受。
|
||||
if (created.status !== 200 && created.status !== 409) throw new Error(`create user ${owner}: ${created.status} ${created.body}`);
|
||||
users.push({ owner, password });
|
||||
}
|
||||
return { users };
|
||||
}
|
||||
|
||||
// ---- default:每 VU 一个用户身份,混合读写 ----
|
||||
|
||||
export default function (data) {
|
||||
const u = data.users[exec.scenario.iterationInTest % data.users.length];
|
||||
// VU 上下文的 cookie jar 自动附着;每 VU 绑定一个用户身份。
|
||||
const login = http.post(`${BASE}/api/auth/login`, JSON.stringify({ owner: u.owner, password: u.password }), { headers: { 'Content-Type': 'application/json' } });
|
||||
if (login.status !== 200) {
|
||||
check(null, { 'user login': () => false });
|
||||
return;
|
||||
}
|
||||
// 会话/项目绑定 VU(VU 内多次迭代复用同一身份的会话,形成热点会话)。
|
||||
if (!u.__ctx) {
|
||||
const proj = http.post(`${BASE}/api/projects`, JSON.stringify({ name: `压测-${u.owner}` }), { headers: { 'Content-Type': 'application/json' } });
|
||||
const conv = http.post(`${BASE}/api/projects/${proj.json().id}/conversations`, JSON.stringify({ title: '压测会话' }), { headers: { 'Content-Type': 'application/json' } });
|
||||
u.__ctx = { projectID: proj.json().id, convID: conv.json().id };
|
||||
}
|
||||
const { projectID, convID } = u.__ctx;
|
||||
|
||||
group('read', () => {
|
||||
const projs = http.get(`${BASE}/api/projects`);
|
||||
check(projs, { 'list projects 2xx': (r) => r.status === 200 });
|
||||
const detail = http.get(`${BASE}/api/conversations/${convID}`);
|
||||
check(detail, { 'conversation detail 2xx': (r) => r.status === 200 });
|
||||
const cards = http.get(`${BASE}/api/conversations/${convID}/cards`);
|
||||
check(cards, { 'cards 2xx': (r) => r.status === 200 });
|
||||
const settings = http.get(`${BASE}/api/settings`);
|
||||
check(settings, { 'settings 2xx': (r) => r.status === 200 });
|
||||
});
|
||||
|
||||
group('write', () => {
|
||||
const requestId = `req-${u.owner}-${exec.scenario.iterationInTest}`;
|
||||
const body = JSON.stringify({ text: `压测消息 ${Date.now()}`, requestId });
|
||||
const t0 = Date.now();
|
||||
const sent = http.post(`${BASE}/api/conversations/${convID}/messages`, body, { headers: { 'Content-Type': 'application/json' } });
|
||||
if (sent.status === 200) {
|
||||
msgLatency.add(Date.now() - t0);
|
||||
const id = sent.json().message && sent.json().message.id;
|
||||
check(sent, { 'message has id': () => !!id });
|
||||
// 幂等重放(矩阵行:同 requestId 返回同一逻辑结果):抽样 10%。
|
||||
if (exec.scenario.iterationInTest % 10 === 0) {
|
||||
const replay = http.post(`${BASE}/api/conversations/${convID}/messages`, body, { headers: { 'Content-Type': 'application/json' } });
|
||||
check(replay, {
|
||||
'requestId replay same message': (r) => r.status === 200 && r.json().message && r.json().message.id === id,
|
||||
});
|
||||
}
|
||||
} else if (sent.status === 409 || sent.status === 429) {
|
||||
// 热点会话忙/队列满:预期拥塞,单列。
|
||||
expectedContention.add(1);
|
||||
} else {
|
||||
check(sent, { 'message send 2xx': () => false });
|
||||
}
|
||||
});
|
||||
|
||||
sleep(1);
|
||||
}
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
# 协作存储 U7 压测入口(方案 §3 负载合同 / §5 U7)。
|
||||
#
|
||||
# 起一个隔离栈(backend mock 单实例 + 隔离 PG)→ 预建引导账号 → k6 阶梯
|
||||
# → SSE 可见延迟探针 → 数据不变量对账 → 结果落 results/<ts>/ → 回收。
|
||||
# 绝不打生产服务:默认自建 PG 实例(svc.sh);--dsn 复用已有隔离实例。
|
||||
#
|
||||
# 用法:
|
||||
# tests/load/collaboration/run_load.sh [--profile smoke|ladder|sustained60m|sustained2h]
|
||||
# [--users N] [--dsn <PG DSN>] [--port <app port>]
|
||||
# [--keep] [--skip-k6] [--skip-probe] [--help]
|
||||
# 依赖:bash4、go(构建 backend/探针)、psql(对账)、curl;
|
||||
# k6 可选(缺失时跳过 k6 段并给出安装指引,探针与对账照跑)。
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
HERE="$ROOT/tests/load/collaboration"
|
||||
PROFILE=smoke USERS=30 APP_PORT=18980 PG_PORT=18981 KEEP=0 SKIP_K6=0 SKIP_PROBE=0
|
||||
DSN=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--profile) PROFILE="$2"; shift 2 ;;
|
||||
--users) USERS="$2"; shift 2 ;;
|
||||
--dsn) DSN="$2"; shift 2 ;;
|
||||
--port) APP_PORT="$2"; shift 2 ;;
|
||||
--keep) KEEP=1; shift ;;
|
||||
--skip-k6) SKIP_K6=1; shift ;;
|
||||
--skip-probe) SKIP_PROBE=1; shift ;;
|
||||
-h|--help) sed -n '2,16p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
TS="$(date +%Y%m%d-%H%M%S)"
|
||||
RESULTS="$HERE/results/$TS"
|
||||
mkdir -p "$RESULTS"
|
||||
OWNED_PG=0
|
||||
APP_PID=""
|
||||
cleanup() {
|
||||
[[ -n "$APP_PID" ]] && kill "$APP_PID" 2>/dev/null || true
|
||||
if [[ $OWNED_PG -eq 1 && $KEEP -eq 0 ]]; then
|
||||
"$ROOT/scripts/svc.sh" u7load stop >/dev/null 2>&1 || true
|
||||
"$ROOT/scripts/svc.sh" rm u7load --purge >/dev/null 2>&1 || true
|
||||
fi
|
||||
[[ $KEEP -eq 1 ]] && echo "[load] --keep:栈保留(PG=u7load:$PG_PORT,app=127.0.0.1:$APP_PORT,结果 $RESULTS)"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---- 1) PG:自建隔离实例(或复用 --dsn) ----
|
||||
if [[ -z "$DSN" ]]; then
|
||||
echo "[load] 创建隔离 PG 实例 u7load(端口 $PG_PORT)"
|
||||
"$ROOT/scripts/svc.sh" db create u7load --port "$PG_PORT" >/dev/null
|
||||
OWNED_PG=1
|
||||
"$ROOT/scripts/svc.sh" u7load start >/dev/null
|
||||
DSN="$("$ROOT/scripts/svc.sh" db dsn u7load --reveal)"
|
||||
fi
|
||||
echo "[load] PG DSN 已就绪(密码不回显)"
|
||||
|
||||
# ---- 2) 构建 backend 与探针 ----
|
||||
echo "[load] 构建 backend(worktree 当前源码)与 SSE 探针"
|
||||
BIN="$HERE/.build/agent-system"
|
||||
mkdir -p "$HERE/.build"
|
||||
( cd "$ROOT/modules/backend" && CGO_ENABLED=0 go build -o "$BIN" ./cmd/agent-system )
|
||||
( cd "$HERE/sse_probe" && go build -o "$HERE/.build/sse_probe" . )
|
||||
|
||||
# ---- 3) 压测库 + 引导账号(CLI 直连,终态口令避免首登改密分支) ----
|
||||
# DSN 规范化:库名换 u7load_app、query 补 sslmode=disable(密码不回显)。
|
||||
ADMIN_DSN="$(python3 - "$DSN" <<'PY'
|
||||
import sys
|
||||
from urllib.parse import urlsplit, urlunsplit, parse_qsl, urlencode
|
||||
u = urlsplit(sys.argv[1])
|
||||
q = dict(parse_qsl(u.query))
|
||||
q.setdefault("sslmode", "disable")
|
||||
print(urlunsplit((u.scheme, u.netloc, "/" + (u.path.lstrip("/") or "postgres"), urlencode(q), "")))
|
||||
PY
|
||||
)"
|
||||
psql "$ADMIN_DSN" -v ON_ERROR_STOP=1 -c "DROP DATABASE IF EXISTS u7load_app WITH (FORCE)" -c "CREATE DATABASE u7load_app" >/dev/null
|
||||
APP_DSN="$(python3 - "$ADMIN_DSN" <<'PY'
|
||||
import sys
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
u = urlsplit(sys.argv[1])
|
||||
print(urlunsplit((u.scheme, u.netloc, "/u7load_app", u.query, "")))
|
||||
PY
|
||||
)"
|
||||
"$BIN" user create --dsn "$APP_DSN" --admin --owner load_admin --password load-admin-pass-Aa1 >/dev/null
|
||||
"$BIN" user create --dsn "$APP_DSN" --owner probe_user --password probe-user-pass-Aa9 >/dev/null
|
||||
|
||||
# ---- 4) 起被测实例(mock、单实例首期;执行原语关闭——压测不打执行面) ----
|
||||
echo "[load] 启动被测实例 127.0.0.1:$APP_PORT(mock / tier=team / auth=on)"
|
||||
APP_LOG="$RESULTS/app.log"
|
||||
"$BIN" serve --dsn "$APP_DSN" --addr "127.0.0.1:$APP_PORT" --tier team --auth on --mock \
|
||||
--instance u7load --disable-exec-tools >"$APP_LOG" 2>&1 &
|
||||
APP_PID=$!
|
||||
# auth=on 时 /api/health 同样要求鉴权:200/401 都视为已就绪(服务在正确应答)。
|
||||
app_ready() { code=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:$APP_PORT/api/health"); [[ "$code" == 200 || "$code" == 401 ]]; }
|
||||
for _ in $(seq 1 60); do app_ready && break; sleep 0.5; done
|
||||
app_ready || { echo "[load] app 未就绪,日志 $APP_LOG" >&2; exit 1; }
|
||||
|
||||
BASE="http://127.0.0.1:$APP_PORT"
|
||||
|
||||
# ---- 5) k6 阶梯 ----
|
||||
if [[ $SKIP_K6 -eq 1 ]]; then
|
||||
echo "[load] --skip-k6:跳过 k6 段"
|
||||
elif ! command -v k6 >/dev/null 2>&1; then
|
||||
cat >"$RESULTS/k6-skipped.txt" <<'EOF'
|
||||
k6 未安装:本段跳过(探针与对账照跑)。
|
||||
安装(本机 linuxbrew):brew install k6
|
||||
安装后重跑:tests/load/collaboration/run_load.sh --profile <同参数>
|
||||
EOF
|
||||
echo "[load] k6 未安装,跳过(见 $RESULTS/k6-skipped.txt)"
|
||||
else
|
||||
echo "[load] k6 profile=$PROFILE users=$USERS"
|
||||
( cd "$HERE" && BASE_URL="$BASE" ADMIN_OWNER=load_admin ADMIN_PASS=load-admin-pass-Aa1 \
|
||||
USERS="$USERS" PROFILE="$PROFILE" \
|
||||
k6 run --summary-export "$RESULTS/k6-summary.json" k6/collab_api.js ) | tee "$RESULTS/k6-output.txt"
|
||||
fi
|
||||
|
||||
# ---- 6) SSE 可见延迟探针 ----
|
||||
if [[ $SKIP_PROBE -eq 0 ]]; then
|
||||
echo "[load] SSE 可见延迟探针(60s,10 会话)"
|
||||
"$HERE/.build/sse_probe" -base "$BASE" -dur 60s -convs 10 -out "$RESULTS/sse-probe.json"
|
||||
fi
|
||||
|
||||
# ---- 7) 数据不变量对账(方案 §3:零违反) ----
|
||||
echo "[load] 数据不变量对账"
|
||||
psql "$APP_DSN" -v ON_ERROR_STOP=1 \
|
||||
-c "SELECT 'users' k, COUNT(*) v FROM users UNION ALL SELECT 'projects', COUNT(*) FROM projects UNION ALL SELECT 'conversations', COUNT(*) FROM conversations UNION ALL SELECT 'messages', COUNT(*) FROM messages UNION ALL SELECT 'events', COUNT(*) FROM events UNION ALL SELECT 'runs', COUNT(*) FROM runs" \
|
||||
-c "SELECT 'dup_seq_per_conv' k, COUNT(*) v FROM (SELECT conversation_id, seq FROM events GROUP BY 1,2 HAVING COUNT(*)>1) d" \
|
||||
-c "SELECT 'nonterminal_orphan_runs' k, COUNT(*) v FROM runs WHERE status IN ('pending','running','waiting_hitl')" \
|
||||
| tee "$RESULTS/invariants.txt"
|
||||
|
||||
DUP=$(psql "$APP_DSN" -tAc "SELECT COUNT(*) FROM (SELECT conversation_id, seq FROM events GROUP BY 1,2 HAVING COUNT(*)>1) d")
|
||||
if [[ "$DUP" != "0" ]]; then
|
||||
echo "[load] 对账失败:events 每会话 seq 出现重复($DUP 组)" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[load] 对账通过:无重复 seq;孤儿 run 报告见 invariants.txt(探针正常收口应为 0)"
|
||||
|
||||
echo "[load] 完成。结果目录:$RESULTS"
|
||||
@@ -0,0 +1,3 @@
|
||||
module agent-system-dev/tests/load/collaboration/sse_probe
|
||||
|
||||
go 1.27.1
|
||||
@@ -0,0 +1,241 @@
|
||||
// 协作存储 U7:SSE 持久事件可见延迟探针(负载合同:P95≤1s)。
|
||||
// k6 核心不支持 SSE 流式读取,此项由本 Go 探针承担;与 k6 结果记录在
|
||||
// 同一目录(run_load.sh 编排)。
|
||||
//
|
||||
// 行为:并发 C 个会话各挂一个 SSE 订阅(共享登录 cookie);写者按固定
|
||||
// 间隔轮转 POST 消息(mock 即时回合);统计「POST 前基准 seq → 会话内
|
||||
// 新帧到达」延迟分布与 seq 单调性,输出 JSON 报告并以退出码执行
|
||||
// P95≤1s、零乱序断言(数据不变量零违反,方案 §3)。
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type sample struct {
|
||||
Conversation string `json:"conversation"`
|
||||
SendSeq int64 `json:"sendSeq"`
|
||||
LatencyMS float64 `json:"latencyMs"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Target string `json:"target"`
|
||||
Duration string `json:"duration"`
|
||||
Subscribers int `json:"subscribers"`
|
||||
Messages int `json:"messages"`
|
||||
Samples int `json:"samples"`
|
||||
P50MS float64 `json:"p50Ms"`
|
||||
P95MS float64 `json:"p95Ms"`
|
||||
P99MS float64 `json:"p99Ms"`
|
||||
MaxMS float64 `json:"maxMs"`
|
||||
OutOfOrder int `json:"outOfOrder"` // seq 乱序/回退(不变量违反)
|
||||
ThresholdMS float64 `json:"thresholdMs"`
|
||||
Pass bool `json:"pass"`
|
||||
Detail []sample `json:"samplesDetail"`
|
||||
}
|
||||
|
||||
var (
|
||||
jar, _ = cookiejar.New(nil)
|
||||
client = &http.Client{Timeout: 10 * time.Second, Jar: jar}
|
||||
streamClient = &http.Client{Jar: jar} // SSE:无整体超时,stop 控制
|
||||
pendingMu sync.Mutex
|
||||
pending = map[string]pendingSend{}
|
||||
outOfOrderMu sync.Mutex
|
||||
outOfOrderN int
|
||||
samplesMu sync.Mutex
|
||||
samples []sample
|
||||
)
|
||||
|
||||
type pendingSend struct {
|
||||
seq int64
|
||||
sent time.Time
|
||||
}
|
||||
|
||||
func main() {
|
||||
base := flag.String("base", "http://127.0.0.1:18980", "目标实例基址")
|
||||
owner := flag.String("owner", "probe_user", "探针用户")
|
||||
password := flag.String("password", "probe-user-pass-Aa9", "探针用户口令")
|
||||
convs := flag.Int("convs", 10, "并发订阅会话数")
|
||||
every := flag.Duration("every", 500*time.Millisecond, "写者间隔(轮转各会话)")
|
||||
dur := flag.Duration("dur", 60*time.Second, "采样时长")
|
||||
out := flag.String("out", "", "JSON 报告输出路径(空 = stdout)")
|
||||
flag.Parse()
|
||||
|
||||
login(*base, *owner, *password)
|
||||
|
||||
projectID := apiJSON("POST", *base, "/api/projects", map[string]any{"name": "sse-probe"})["id"].(string)
|
||||
convIDs := make([]string, *convs)
|
||||
for i := range convIDs {
|
||||
convIDs[i] = apiJSON("POST", *base, "/api/projects/"+projectID+"/conversations", map[string]any{"title": fmt.Sprintf("probe-%d", i)})["id"].(string)
|
||||
}
|
||||
|
||||
stop := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for _, cid := range convIDs {
|
||||
wg.Add(1)
|
||||
go func(cid string) {
|
||||
defer wg.Done()
|
||||
subscribe(*base, cid, stop)
|
||||
}(cid)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond) // 订阅就绪
|
||||
|
||||
deadline := time.Now().Add(*dur)
|
||||
sent := 0
|
||||
for i := 0; time.Now().Before(deadline); i++ {
|
||||
cid := convIDs[i%len(convIDs)]
|
||||
detail := apiJSON("GET", *base, "/api/conversations/"+cid, nil)
|
||||
seq, _ := detail["lastSeq"].(float64)
|
||||
pendingMu.Lock()
|
||||
pending[cid] = pendingSend{seq: int64(seq), sent: time.Now()}
|
||||
pendingMu.Unlock()
|
||||
apiJSON("POST", *base, "/api/conversations/"+cid+"/messages", map[string]any{"text": fmt.Sprintf("probe %d", i)})
|
||||
sent++
|
||||
time.Sleep(*every)
|
||||
}
|
||||
|
||||
time.Sleep(3 * time.Second) // 最后一批帧在途余量
|
||||
close(stop)
|
||||
wg.Wait()
|
||||
|
||||
r := buildReport(*base, *dur, *convs, sent)
|
||||
data, _ := json.MarshalIndent(r, "", " ")
|
||||
if *out != "" {
|
||||
if err := os.WriteFile(*out, data, 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write report:", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
fmt.Println("report:", *out)
|
||||
} else {
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
if !r.Pass {
|
||||
fmt.Fprintf(os.Stderr, "sse probe FAIL: p95=%.1fms (≤%.0fms) samples=%d outOfOrder=%d\n", r.P95MS, r.ThresholdMS, r.Samples, r.OutOfOrder)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("sse probe PASS: p95=%.1fms samples=%d\n", r.P95MS, r.Samples)
|
||||
}
|
||||
|
||||
func login(base, owner, pass string) {
|
||||
body, _ := json.Marshal(map[string]string{"owner": owner, "password": pass})
|
||||
resp, err := client.Post(base+"/api/auth/login", "application/json", bytes.NewReader(body))
|
||||
if err != nil || resp.StatusCode != 200 {
|
||||
fmt.Fprintf(os.Stderr, "probe login: err=%v status=%v\n", err, statusOf(resp))
|
||||
os.Exit(2)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func statusOf(resp *http.Response) string {
|
||||
if resp == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return strconv.Itoa(resp.StatusCode)
|
||||
}
|
||||
|
||||
func apiJSON(method, base, path string, body any) map[string]any {
|
||||
var rd io.Reader
|
||||
_ = rd
|
||||
if body != nil {
|
||||
buf, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(buf)
|
||||
}
|
||||
req, _ := http.NewRequest(method, base+path, rd)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: %v\n", method, path, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
out := map[string]any{}
|
||||
if resp.StatusCode != 200 {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: status %d\n", method, path, resp.StatusCode)
|
||||
os.Exit(2)
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s %s: decode %v\n", method, path, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// subscribe 读会话 SSE:seq 严格递增校验;帧到达时与写者基准配对成延迟样本。
|
||||
func subscribe(base, cid string, stop <-chan struct{}) {
|
||||
req, _ := http.NewRequest(http.MethodGet, base+"/api/events?conversationId="+cid, nil)
|
||||
resp, err := streamClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
sc := bufio.NewScanner(resp.Body)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 4<<20)
|
||||
last := int64(-1)
|
||||
for sc.Scan() {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
line := sc.Text()
|
||||
if !strings.HasPrefix(line, "id: ") {
|
||||
continue
|
||||
}
|
||||
seq, _ := strconv.ParseInt(strings.TrimPrefix(line, "id: "), 10, 64)
|
||||
arrived := time.Now()
|
||||
if last != -1 && seq <= last {
|
||||
outOfOrderMu.Lock()
|
||||
outOfOrderN++
|
||||
outOfOrderMu.Unlock()
|
||||
}
|
||||
last = seq
|
||||
pendingMu.Lock()
|
||||
p, ok := pending[cid]
|
||||
pendingMu.Unlock()
|
||||
if ok && seq > p.seq {
|
||||
pendingMu.Lock()
|
||||
delete(pending, cid)
|
||||
pendingMu.Unlock()
|
||||
samplesMu.Lock()
|
||||
samples = append(samples, sample{Conversation: cid, SendSeq: seq, LatencyMS: float64(arrived.Sub(p.sent)) / float64(time.Millisecond)})
|
||||
samplesMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildReport(base string, dur time.Duration, subs, msgs int) report {
|
||||
samplesMu.Lock()
|
||||
detail := append([]sample(nil), samples...)
|
||||
samplesMu.Unlock()
|
||||
outOfOrderMu.Lock()
|
||||
ooo := outOfOrderN
|
||||
outOfOrderMu.Unlock()
|
||||
|
||||
r := report{Target: base, Duration: dur.String(), Subscribers: subs, Messages: msgs, Samples: len(detail), OutOfOrder: ooo, ThresholdMS: 1000, Detail: detail}
|
||||
if len(detail) > 0 {
|
||||
vals := make([]float64, len(detail))
|
||||
for i, s := range detail {
|
||||
vals[i] = s.LatencyMS
|
||||
}
|
||||
sort.Float64s(vals)
|
||||
pct := func(p float64) float64 { return vals[int(float64(len(vals)-1)*p)] }
|
||||
r.P50MS, r.P95MS, r.P99MS, r.MaxMS = pct(0.50), pct(0.95), pct(0.99), vals[len(vals)-1]
|
||||
}
|
||||
r.Pass = len(detail) > 0 && r.P95MS <= r.ThresholdMS && ooo == 0
|
||||
return r
|
||||
}
|
||||
Reference in New Issue
Block a user