diff --git a/tests/ops/.gitignore b/tests/ops/.gitignore new file mode 100644 index 00000000..ec0fb096 --- /dev/null +++ b/tests/ops/.gitignore @@ -0,0 +1,2 @@ +/.build/ +/results/ diff --git a/tests/ops/README.md b/tests/ops/README.md new file mode 100644 index 00000000..2c540c49 --- /dev/null +++ b/tests/ops/README.md @@ -0,0 +1,20 @@ +# U7 运维演练(C4 / C5) + +只打隔离栈。实例名必须以 `u7drill` 或 `u7bk` 开头,端口必须落在 18980–18999。 +脚本只按自己的 pidfile 结束进程,并且先核对命令行里有本演练的数据目录或端口。 + +| 文件 | 职责 | +|---|---| +| `lib.sh` | 隔离 PG / Redis / serve 的创建、回收、端口与名称守卫 | +| `drill_driver/` | 时间线、幂等重放、SSE seq、标记写入、工件、限流突发、权限核对 | +| `run_fault_drill.sh` | C4:`pg-stop`、`pg-kill`、`redis-kill` | +| `run_backup_drill.sh` | C5:物理备份 + WAL/PITR,以及 pg_dump,恢复到新目标 | + +```bash +tests/ops/self_check.sh +tests/ops/run_fault_drill.sh --fault all +tests/ops/run_backup_drill.sh +``` + +结果在 `results//`(gitignore)。证据以 `make evidence` 记入任务卡,来源是 SELF。 +本入口不放行第二实例写流量。 diff --git a/tests/ops/drill_driver/go.mod b/tests/ops/drill_driver/go.mod new file mode 100644 index 00000000..9843c4be --- /dev/null +++ b/tests/ops/drill_driver/go.mod @@ -0,0 +1,3 @@ +module drilldriver + +go 1.22 diff --git a/tests/ops/drill_driver/holes.go b/tests/ops/drill_driver/holes.go new file mode 100644 index 00000000..fa6c4995 --- /dev/null +++ b/tests/ops/drill_driver/holes.go @@ -0,0 +1,26 @@ +package main + +// seqHoles 返回已观察到的 seq 里缺失的编号。空切片没有洞。 +func seqHoles(seqs []int64) []int64 { + if len(seqs) == 0 { + return nil + } + min, max := seqs[0], seqs[0] + seen := map[int64]bool{} + for _, s := range seqs { + seen[s] = true + if s < min { + min = s + } + if s > max { + max = s + } + } + var holes []int64 + for s := min; s <= max; s++ { + if !seen[s] { + holes = append(holes, s) + } + } + return holes +} diff --git a/tests/ops/drill_driver/holes_test.go b/tests/ops/drill_driver/holes_test.go new file mode 100644 index 00000000..0535ed9c --- /dev/null +++ b/tests/ops/drill_driver/holes_test.go @@ -0,0 +1,17 @@ +package main + +import "reflect" +import "testing" + +func TestSeqHoles(t *testing.T) { + if got := seqHoles(nil); got != nil { + t.Fatalf("empty: %#v", got) + } + if got := seqHoles([]int64{1, 2, 3}); got != nil { + t.Fatalf("contiguous: %#v", got) + } + got := seqHoles([]int64{1, 3, 4}) + if !reflect.DeepEqual(got, []int64{2}) { + t.Fatalf("hole: %#v", got) + } +} diff --git a/tests/ops/drill_driver/main.go b/tests/ops/drill_driver/main.go new file mode 100644 index 00000000..d1be7908 --- /dev/null +++ b/tests/ops/drill_driver/main.go @@ -0,0 +1,533 @@ +// 演练驱动:对隔离实例发真实 HTTP,记录每次请求的结果。 +// 连接失败和超时记为 status=0,不把它们当成业务成功。 +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "os" + "strconv" + "strings" + "time" +) + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + var err error + switch os.Args[1] { + case "prepare": + err = cmdPrepare(os.Args[2:]) + case "timeline": + err = cmdTimeline(os.Args[2:]) + case "idem-verify": + err = cmdIdem(os.Args[2:]) + case "sse-check": + err = cmdSSE(os.Args[2:]) + case "markers": + err = cmdMarkers(os.Args[2:]) + case "artifacts-put": + err = cmdArtifactsPut(os.Args[2:]) + case "artifacts-verify": + err = cmdArtifactsVerify(os.Args[2:]) + case "burst": + err = cmdBurst(os.Args[2:]) + case "perm-check": + err = cmdPerm(os.Args[2:]) + case "-h", "--help", "help": + usage() + return + default: + usage() + os.Exit(2) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "drill_driver prepare|timeline|idem-verify|sse-check|markers|artifacts-put|artifacts-verify|burst|perm-check") +} + +type session struct { + base string + client *http.Client +} + +func newSession(base, owner, password string) (*session, error) { + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + s := &session{ + base: strings.TrimRight(base, "/"), + client: &http.Client{Timeout: 15 * time.Second, Jar: jar}, + } + _, err = s.do("POST", "/api/auth/login", map[string]string{"owner": owner, "password": password}) + if err != nil { + return nil, fmt.Errorf("login %s: %w", base, err) + } + return s, nil +} + +type callResult struct { + Status int `json:"status"` + Elapsed time.Duration `json:"-"` + ElapsedMs float64 `json:"elapsedMs"` + Body json.RawMessage `json:"body,omitempty"` + Error string `json:"error,omitempty"` +} + +func (s *session) do(method, path string, body any) (callResult, error) { + var rd io.Reader + if body != nil { + buf, err := json.Marshal(body) + if err != nil { + return callResult{}, err + } + rd = bytes.NewReader(buf) + } + req, err := http.NewRequest(method, s.base+path, rd) + if err != nil { + return callResult{}, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + start := time.Now() + resp, err := s.client.Do(req) + elapsed := time.Since(start) + if err != nil { + return callResult{Elapsed: elapsed, ElapsedMs: ms(elapsed), Error: err.Error()}, nil + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + out := callResult{Status: resp.StatusCode, Elapsed: elapsed, ElapsedMs: ms(elapsed), Body: raw} + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return out, fmt.Errorf("%s %s: status %d", method, path, resp.StatusCode) + } + return out, nil +} + +func ms(d time.Duration) float64 { + return float64(d) / float64(time.Millisecond) +} + +func writeJSON(path string, v any) error { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if path == "" || path == "-" { + _, err = os.Stdout.Write(data) + return err + } + return os.WriteFile(path, data, 0o644) +} + +func cmdPrepare(args []string) error { + fs := flag.NewFlagSet("prepare", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + out := fs.String("out", "-", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + proj, err := s.do("POST", "/api/projects", map[string]any{"name": "drill"}) + if err != nil { + return err + } + var projBody struct { + ID string `json:"id"` + } + if err := json.Unmarshal(proj.Body, &projBody); err != nil { + return err + } + conv, err := s.do("POST", "/api/projects/"+projBody.ID+"/conversations", map[string]any{"title": "drill"}) + if err != nil { + return err + } + var convBody struct { + ID string `json:"id"` + } + if err := json.Unmarshal(conv.Body, &convBody); err != nil { + return err + } + return writeJSON(*out, map[string]string{"projectId": projBody.ID, "conversationId": convBody.ID}) +} + +type attempt struct { + RequestID string `json:"requestId"` + Base string `json:"base"` + Started string `json:"started"` + ElapsedMs float64 `json:"elapsedMs"` + Status int `json:"status"` + Error string `json:"error,omitempty"` + MessageID string `json:"messageId,omitempty"` +} + +func cmdTimeline(args []string) error { + fs := flag.NewFlagSet("timeline", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + conv := fs.String("conv", "", "conversation id") + every := fs.Duration("every", 500*time.Millisecond, "write interval") + dur := fs.Duration("dur", 90*time.Second, "how long to write") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + var attempts []attempt + deadline := time.Now().Add(*dur) + for i := 0; time.Now().Before(deadline); i++ { + reqID := fmt.Sprintf("tl-%d-%d", time.Now().UnixNano(), i) + started := time.Now().UTC() + res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{ + "text": "drill " + reqID, "requestId": reqID, + }) + attempts = append(attempts, attempt{ + RequestID: reqID, Base: *base, Started: started.Format(time.RFC3339Nano), + ElapsedMs: res.ElapsedMs, Status: res.Status, Error: res.Error, MessageID: messageID(res.Body), + }) + time.Sleep(*every) + } + return writeJSON(*out, map[string]any{"attempts": attempts}) +} + +func messageID(raw json.RawMessage) string { + var body struct { + Message struct { + ID string `json:"id"` + } `json:"message"` + } + _ = json.Unmarshal(raw, &body) + return body.Message.ID +} + +func cmdIdem(args []string) error { + fs := flag.NewFlagSet("idem-verify", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + conv := fs.String("conv", "", "conversation id") + in := fs.String("in", "", "timeline json") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + data, err := os.ReadFile(*in) + if err != nil { + return err + } + var doc struct { + Attempts []attempt `json:"attempts"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + var replays []attempt + for _, a := range doc.Attempts { + if a.Status >= 200 && a.Status < 300 { + continue + } + res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{ + "text": "drill " + a.RequestID, "requestId": a.RequestID, + }) + replays = append(replays, attempt{ + RequestID: a.RequestID, Base: *base, Started: time.Now().UTC().Format(time.RFC3339Nano), + ElapsedMs: res.ElapsedMs, Status: res.Status, Error: res.Error, MessageID: messageID(res.Body), + }) + } + return writeJSON(*out, map[string]any{"replays": replays}) +} + +func cmdSSE(args []string) error { + fs := flag.NewFlagSet("sse-check", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + conv := fs.String("conv", "", "conversation id") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.base+"/api/events?conversationId="+*conv, nil) + if err != nil { + return err + } + stream := &http.Client{Jar: s.client.Jar} + resp, err := stream.Do(req) + var seqs []int64 + status := 0 + errText := "" + if err != nil && resp == nil { + errText = err.Error() + } else if resp != nil { + status = resp.StatusCode + defer resp.Body.Close() + body, readErr := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if readErr != nil { + errText = readErr.Error() + } + for _, line := range strings.Split(string(body), "\n") { + if !strings.HasPrefix(line, "id: ") { + continue + } + n, convErr := strconv.ParseInt(strings.TrimPrefix(line, "id: "), 10, 64) + if convErr == nil { + seqs = append(seqs, n) + } + } + } + return writeJSON(*out, map[string]any{ + "status": status, "error": errText, "seqs": seqs, "holes": seqHoles(seqs), + }) +} + +func cmdMarkers(args []string) error { + fs := flag.NewFlagSet("markers", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + conv := fs.String("conv", "", "conversation id") + every := fs.Duration("every", 5*time.Second, "marker interval") + dur := fs.Duration("dur", 5*time.Minute, "how long to write") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + var marks []map[string]any + deadline := time.Now().Add(*dur) + for i := 0; time.Now().Before(deadline); i++ { + reqID := fmt.Sprintf("mk-%d", i) + acked := time.Now().UTC() + res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{ + "text": "marker " + reqID, "requestId": reqID, + }) + marks = append(marks, map[string]any{ + "requestId": reqID, "acked": acked.Format(time.RFC3339Nano), + "status": res.Status, "error": res.Error, "messageId": messageID(res.Body), + }) + time.Sleep(*every) + } + return writeJSON(*out, map[string]any{"markers": marks}) +} + +func cmdArtifactsPut(args []string) error { + fs := flag.NewFlagSet("artifacts-put", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + project := fs.String("project", "", "project id") + n := fs.Int("n", 5, "how many files") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + var items []map[string]any + for i := 0; i < *n; i++ { + payload := []byte(fmt.Sprintf("drill-artifact-%d-%d", i, time.Now().UnixNano())) + sum := sha256.Sum256(payload) + id, status, errText, acked := upload(s, *project, fmt.Sprintf("drill-%d.txt", i), payload) + items = append(items, map[string]any{ + "name": fmt.Sprintf("drill-%d.txt", i), "sha256": hex.EncodeToString(sum[:]), + "id": id, "status": status, "error": errText, "acked": acked.Format(time.RFC3339Nano), + }) + } + return writeJSON(*out, map[string]any{"artifacts": items}) +} + +func upload(s *session, project, name string, payload []byte) (id string, status int, errText string, acked time.Time) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + part, err := w.CreateFormFile("file", name) + if err != nil { + return "", 0, err.Error(), time.Now().UTC() + } + _, _ = part.Write(payload) + _ = w.Close() + req, err := http.NewRequest(http.MethodPost, s.base+"/api/projects/"+project+"/uploads", &buf) + if err != nil { + return "", 0, err.Error(), time.Now().UTC() + } + req.Header.Set("Content-Type", w.FormDataContentType()) + acked = time.Now().UTC() + resp, err := s.client.Do(req) + if err != nil { + return "", 0, err.Error(), acked + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + var body struct { + ID string `json:"id"` + } + _ = json.Unmarshal(raw, &body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return body.ID, resp.StatusCode, string(raw), acked + } + return body.ID, resp.StatusCode, "", acked +} + +func cmdArtifactsVerify(args []string) error { + fs := flag.NewFlagSet("artifacts-verify", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + in := fs.String("in", "", "artifacts json") + before := fs.String("before", "", "only artifacts acked before this RFC3339 time") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + data, err := os.ReadFile(*in) + if err != nil { + return err + } + var doc struct { + Artifacts []struct { + ID string `json:"id"` + SHA256 string `json:"sha256"` + Acked string `json:"acked"` + Status int `json:"status"` + } `json:"artifacts"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return err + } + cutoff, err := time.Parse(time.RFC3339Nano, *before) + if err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + missing := 0 + mismatch := 0 + checked := 0 + for _, item := range doc.Artifacts { + acked, err := time.Parse(time.RFC3339Nano, item.Acked) + if err != nil || item.Status < 200 || item.Status >= 300 || !acked.Before(cutoff) { + continue + } + checked++ + req, err := http.NewRequest(http.MethodGet, s.base+"/api/artifacts/"+item.ID+"/content", nil) + if err != nil { + missing++ + continue + } + resp, err := s.client.Do(req) + if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 { + if resp != nil { + resp.Body.Close() + } + missing++ + continue + } + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + sum := sha256.Sum256(raw) + if hex.EncodeToString(sum[:]) != item.SHA256 { + mismatch++ + } + } + return writeJSON(*out, map[string]any{"checked": checked, "missing": missing, "mismatch": mismatch}) +} + +func cmdBurst(args []string) error { + fs := flag.NewFlagSet("burst", flag.ContinueOnError) + bases := fs.String("bases", "", "comma-separated base URLs") + owner := fs.String("owner", "", "login owner") + password := fs.String("password", "", "login password") + n := fs.Int("n", 40, "requests per base") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + counts := map[string]int{} + for _, base := range strings.Split(*bases, ",") { + base = strings.TrimSpace(base) + if base == "" { + continue + } + s, err := newSession(base, *owner, *password) + if err != nil { + return err + } + for i := 0; i < *n; i++ { + res, _ := s.do("GET", "/api/projects", nil) + key := fmt.Sprintf("%s:%d", base, res.Status) + if res.Status == 0 { + key = base + ":error" + } + counts[key]++ + } + } + return writeJSON(*out, map[string]any{"counts": counts}) +} + +func cmdPerm(args []string) error { + fs := flag.NewFlagSet("perm-check", flag.ContinueOnError) + base := fs.String("base", "", "instance base URL") + owner := fs.String("owner", "", "other user") + password := fs.String("password", "", "other user password") + project := fs.String("project", "", "someone else's project") + artifact := fs.String("artifact", "", "someone else's artifact") + out := fs.String("out", "", "json output") + if err := fs.Parse(args); err != nil { + return err + } + s, err := newSession(*base, *owner, *password) + if err != nil { + return err + } + proj, _ := s.do("GET", "/api/projects/"+*project+"/conversations", nil) + art, _ := s.do("GET", "/api/artifacts/"+*artifact+"/content", nil) + denied := func(status int) bool { return status == http.StatusForbidden || status == http.StatusNotFound } + return writeJSON(*out, map[string]any{ + "projectStatus": proj.Status, "artifactStatus": art.Status, + "projectDenied": denied(proj.Status), "artifactDenied": denied(art.Status), + }) +} diff --git a/tests/ops/lib.sh b/tests/ops/lib.sh new file mode 100755 index 00000000..25de9c88 --- /dev/null +++ b/tests/ops/lib.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# 演练栈公共函数。由 run_fault_drill.sh / run_backup_drill.sh / self_check.sh source。 +# 不打生产:名称前缀 u7drill 或 u7bk,端口 18980–18999,只杀命令行匹配的 pid。 + +ops_port_listening() { + local p="$1" + if command -v ss >/dev/null 2>&1; then + ss -ltn 2>/dev/null | awk -v p=":$p" '$4 ~ p"$" { found=1 } END { exit !found }' && return 0 + fi + (exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null && return 0 + return 1 +} + +ops_require_name() { + local name="$1" + case "$name" in + u7drill|u7drill-*|u7bk|u7bk-*) ;; + *) echo "[ops] refusing instance name $name" >&2; return 2 ;; + esac + [[ "$name" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || { + echo "[ops] illegal instance name $name" >&2 + return 2 + } +} + +ops_require_port() { + local port="$1" + [[ "$port" =~ ^[0-9]+$ ]] || { echo "[ops] port must be an integer" >&2; return 2; } + if (( port < 18980 || port > 18999 )); then + echo "[ops] refusing port $port outside 18980-18999" >&2 + return 2 + fi +} + +ops_resource_gate() { + local root="$1" avail_kb load1 + avail_kb="$(df -Pk "$root" | awk 'NR==2 { print $4 }')" + load1="$(awk '{ print $1 }' /proc/loadavg)" + if (( avail_kb < 100 * 1024 * 1024 )); then + echo "[ops] disk free ${avail_kb}KB is below 100GB; pausing" >&2 + return 3 + fi + awk -v load1="$load1" 'BEGIN { if (load1+0 > 16) exit 1 }' || { + echo "[ops] load average $load1 is above 16; pausing" >&2 + return 3 + } +} + +# ops_kill_checked PIDFILE NEEDLE:只在 /proc/pid/cmdline 含 NEEDLE 时发信号。 +ops_kill_checked() { + local pidfile="$1" needle="$2" sig="${3:-TERM}" + [[ -f "$pidfile" ]] || return 0 + local pid cmd + pid="$(head -1 "$pidfile" 2>/dev/null || true)" + [[ "$pid" =~ ^[0-9]+$ ]] || return 0 + if [[ ! -d "/proc/$pid" ]]; then + rm -f "$pidfile" + return 0 + fi + cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)" + if [[ "$cmd" != *"$needle"* ]]; then + echo "[ops] refusing to signal $pid; cmdline does not contain $needle" >&2 + return 1 + fi + kill "-$sig" "$pid" 2>/dev/null || true +} + +ops_pg_create() { + local root="$1" name="$2" port="$3" + ops_require_name "$name" || return $? + ops_require_port "$port" || return $? + if ops_port_listening "$port"; then + echo "[ops] port $port is in use; refusing before create" >&2 + return 2 + fi + if [[ -e "$root/runtime/$name" ]]; then + echo "[ops] runtime/$name already exists; refusing to touch it" >&2 + return 2 + fi + "$root/scripts/svc.sh" db create "$name" --port "$port" >/dev/null + local addr + addr="$(sed -n 's/^addr=//p' "$root/runtime/$name/svc.env" | head -1)" + if [[ "$addr" != "127.0.0.1:$port" ]]; then + echo "[ops] $name addr is $addr, expected 127.0.0.1:$port" >&2 + return 1 + fi +} + +ops_pg_start() { + local root="$1" name="$2" + "$root/scripts/svc.sh" "$name" start >/dev/null +} + +ops_pg_stop() { + local root="$1" name="$2" + "$root/scripts/svc.sh" "$name" stop >/dev/null +} + +ops_pg_kill() { + local root="$1" name="$2" + local pgdata="$root/runtime/$name/pgdata" + local pidfile="$pgdata/postmaster.pid" + [[ -f "$pidfile" ]] || { echo "[ops] no postmaster.pid for $name" >&2; return 1; } + ops_kill_checked "$pidfile" "$pgdata" KILL +} + +ops_pg_dsn() { + local root="$1" name="$2" + "$root/scripts/svc.sh" db dsn "$name" --reveal +} + +ops_pg_remove() { + local root="$1" name="$2" + "$root/scripts/svc.sh" "$name" stop >/dev/null 2>&1 || true + "$root/scripts/svc.sh" rm "$name" --purge >/dev/null 2>&1 || true +} + +ops_redis_start() { + local port="$1" dir="$2" + ops_require_port "$port" || return $? + if ops_port_listening "$port"; then + echo "[ops] redis port $port is in use" >&2 + return 2 + fi + mkdir -p "$dir" + redis-server --bind 127.0.0.1 --port "$port" --dir "$dir" --dbfilename drill.rdb \ + --save '' --appendonly no --daemonize yes --pidfile "$dir/redis.pid" --logfile "$dir/redis.log" +} + +ops_redis_kill() { + local dir="$1" port="$2" + ops_kill_checked "$dir/redis.pid" ":$port" KILL +} + +ops_snapshot() { + local root="$1" dest="$2" + { + echo "dev_head=$(git -C "$root" rev-parse HEAD)" + echo "backend=$(git -C "$root" rev-parse HEAD:modules/backend)" + echo "when=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "nproc=$(nproc)" + echo "loadavg=$(cat /proc/loadavg)" + free -h + df -h "$root" + echo "--- heavy processes (comm only) ---" + ps -eo pcpu,pmem,comm --sort=-pcpu | head -12 + } >"$dest" +} diff --git a/tests/ops/self_check.sh b/tests/ops/self_check.sh new file mode 100755 index 00000000..08927fe9 --- /dev/null +++ b/tests/ops/self_check.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# lib.sh 的正常 / 拒绝 / 中断回收自检。不测量吞吐,不受 load 门槛约束。 +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=lib.sh +source "$ROOT/tests/ops/lib.sh" + +usage() { + echo "tests/ops/self_check.sh [--hold]" +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +if [[ "${1:-}" == "--hold" ]]; then + owned=0 + cleanup() { + trap - EXIT INT TERM + if [[ "$owned" -eq 1 ]]; then + ops_pg_remove "$ROOT" u7drill || true + fi + exit 0 + } + trap cleanup EXIT INT TERM + ops_pg_create "$ROOT" u7drill 18989 + owned=1 + ops_pg_start "$ROOT" u7drill + : >"$ROOT/runtime/u7drill/hold.ready" + sleep 3600 + exit 0 +fi + +if [[ $# -gt 0 ]]; then + echo "[ops] unknown arg: $1" >&2 + exit 2 +fi + +echo "[ops] self-check: --help" +"$0" --help >/dev/null + +echo "[ops] self-check: unknown arg exits 2" +set +e +"$0" --nope >/dev/null 2>&1 +ec=$? +set -e +[[ "$ec" -eq 2 ]] + +echo "[ops] self-check: foreign name and production port are refused" +set +e +ops_require_name prod +ec=$? +set -e +[[ "$ec" -eq 2 ]] +set +e +ops_require_port 8787 +ec=$? +set -e +[[ "$ec" -eq 2 ]] + +echo "[ops] self-check: busy port is refused before create" +python3 -m http.server 18988 --bind 127.0.0.1 >/dev/null 2>&1 & +busy_pid=$! +for _ in $(seq 1 20); do ops_port_listening 18988 && break; sleep 0.1; done +set +e +ops_pg_create "$ROOT" u7drill 18988 +ec=$? +set -e +kill "$busy_pid" 2>/dev/null || true +wait "$busy_pid" 2>/dev/null || true +[[ "$ec" -eq 2 ]] +[[ ! -e "$ROOT/runtime/u7drill" ]] + +echo "[ops] self-check: SIGINT removes the instance and the port" +set -m +"$0" --hold >/tmp/u7ops-hold.txt 2>&1 & +hold_pid=$! +set +m +ready=0 +for _ in $(seq 1 90); do + if [[ -f "$ROOT/runtime/u7drill/hold.ready" ]]; then + ready=1 + break + fi + if ! kill -0 "$hold_pid" 2>/dev/null; then + break + fi + sleep 0.5 +done +if [[ "$ready" -ne 1 ]]; then + cat /tmp/u7ops-hold.txt >&2 || true + echo "[ops] hold did not become ready" >&2 + exit 1 +fi +kill -INT -- "-$hold_pid" 2>/dev/null || kill -INT "$hold_pid" 2>/dev/null || true +set +e +wait "$hold_pid" +set -e +if ops_port_listening 18989 || [[ -e "$ROOT/runtime/u7drill" ]]; then + echo "[ops] SIGINT left port 18989 or runtime/u7drill behind" >&2 + exit 1 +fi +echo "[ops] self-check passed"