Add the isolated ops harness for disconnect and backup drills
Why: C4 and C5 need a shared way to start, refuse, and tear down stacks without touching production ports or processes. Evidence: tests/ops/self_check.sh passed (help, unknown arg, foreign name, production port, busy port, SIGINT cleanup). drill_driver go test passed. Impact: New tests/ops library and HTTP driver. No drill result and no release-gate change yet. Refs: docs/release-multi-instance.md C4 C5 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,2 @@
|
||||
/.build/
|
||||
/results/
|
||||
@@ -0,0 +1,20 @@
|
||||
# U7 运维演练（C4 / C5）
|
||||
|
||||
只打隔离栈。实例名必须以 `u7drill` 或 `u7bk` 开头，端口必须落在 18980–18999。
|
||||
脚本只按自己的 pidfile 结束进程，并且先核对命令行里有本演练的数据目录或端口。
|
||||
|
||||
| 文件 | 职责 |
|
||||
|---|---|
|
||||
| `lib.sh` | 隔离 PG / Redis / serve 的创建、回收、端口与名称守卫 |
|
||||
| `drill_driver/` | 时间线、幂等重放、SSE seq、标记写入、工件、限流突发、权限核对 |
|
||||
| `run_fault_drill.sh` | C4：`pg-stop`、`pg-kill`、`redis-kill` |
|
||||
| `run_backup_drill.sh` | C5：物理备份 + WAL/PITR，以及 pg_dump，恢复到新目标 |
|
||||
|
||||
```bash
|
||||
tests/ops/self_check.sh
|
||||
tests/ops/run_fault_drill.sh --fault all
|
||||
tests/ops/run_backup_drill.sh
|
||||
```
|
||||
|
||||
结果在 `results/<ts>/`（gitignore）。证据以 `make evidence` 记入任务卡，来源是 SELF。
|
||||
本入口不放行第二实例写流量。
|
||||
@@ -0,0 +1,3 @@
|
||||
module drilldriver
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
// seqHoles 返回已观察到的 seq 里缺失的编号。空切片没有洞。
|
||||
func seqHoles(seqs []int64) []int64 {
|
||||
if len(seqs) == 0 {
|
||||
return nil
|
||||
}
|
||||
min, max := seqs[0], seqs[0]
|
||||
seen := map[int64]bool{}
|
||||
for _, s := range seqs {
|
||||
seen[s] = true
|
||||
if s < min {
|
||||
min = s
|
||||
}
|
||||
if s > max {
|
||||
max = s
|
||||
}
|
||||
}
|
||||
var holes []int64
|
||||
for s := min; s <= max; s++ {
|
||||
if !seen[s] {
|
||||
holes = append(holes, s)
|
||||
}
|
||||
}
|
||||
return holes
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package main
|
||||
|
||||
import "reflect"
|
||||
import "testing"
|
||||
|
||||
func TestSeqHoles(t *testing.T) {
|
||||
if got := seqHoles(nil); got != nil {
|
||||
t.Fatalf("empty: %#v", got)
|
||||
}
|
||||
if got := seqHoles([]int64{1, 2, 3}); got != nil {
|
||||
t.Fatalf("contiguous: %#v", got)
|
||||
}
|
||||
got := seqHoles([]int64{1, 3, 4})
|
||||
if !reflect.DeepEqual(got, []int64{2}) {
|
||||
t.Fatalf("hole: %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
// 演练驱动：对隔离实例发真实 HTTP，记录每次请求的结果。
|
||||
// 连接失败和超时记为 status=0，不把它们当成业务成功。
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/cookiejar"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
var err error
|
||||
switch os.Args[1] {
|
||||
case "prepare":
|
||||
err = cmdPrepare(os.Args[2:])
|
||||
case "timeline":
|
||||
err = cmdTimeline(os.Args[2:])
|
||||
case "idem-verify":
|
||||
err = cmdIdem(os.Args[2:])
|
||||
case "sse-check":
|
||||
err = cmdSSE(os.Args[2:])
|
||||
case "markers":
|
||||
err = cmdMarkers(os.Args[2:])
|
||||
case "artifacts-put":
|
||||
err = cmdArtifactsPut(os.Args[2:])
|
||||
case "artifacts-verify":
|
||||
err = cmdArtifactsVerify(os.Args[2:])
|
||||
case "burst":
|
||||
err = cmdBurst(os.Args[2:])
|
||||
case "perm-check":
|
||||
err = cmdPerm(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
return
|
||||
default:
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintln(os.Stderr, "drill_driver prepare|timeline|idem-verify|sse-check|markers|artifacts-put|artifacts-verify|burst|perm-check")
|
||||
}
|
||||
|
||||
type session struct {
|
||||
base string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newSession(base, owner, password string) (*session, error) {
|
||||
jar, err := cookiejar.New(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &session{
|
||||
base: strings.TrimRight(base, "/"),
|
||||
client: &http.Client{Timeout: 15 * time.Second, Jar: jar},
|
||||
}
|
||||
_, err = s.do("POST", "/api/auth/login", map[string]string{"owner": owner, "password": password})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("login %s: %w", base, err)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type callResult struct {
|
||||
Status int `json:"status"`
|
||||
Elapsed time.Duration `json:"-"`
|
||||
ElapsedMs float64 `json:"elapsedMs"`
|
||||
Body json.RawMessage `json:"body,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (s *session) do(method, path string, body any) (callResult, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
buf, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return callResult{}, err
|
||||
}
|
||||
rd = bytes.NewReader(buf)
|
||||
}
|
||||
req, err := http.NewRequest(method, s.base+path, rd)
|
||||
if err != nil {
|
||||
return callResult{}, err
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
start := time.Now()
|
||||
resp, err := s.client.Do(req)
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
return callResult{Elapsed: elapsed, ElapsedMs: ms(elapsed), Error: err.Error()}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
out := callResult{Status: resp.StatusCode, Elapsed: elapsed, ElapsedMs: ms(elapsed), Body: raw}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return out, fmt.Errorf("%s %s: status %d", method, path, resp.StatusCode)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ms(d time.Duration) float64 {
|
||||
return float64(d) / float64(time.Millisecond)
|
||||
}
|
||||
|
||||
func writeJSON(path string, v any) error {
|
||||
data, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if path == "" || path == "-" {
|
||||
_, err = os.Stdout.Write(data)
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
|
||||
func cmdPrepare(args []string) error {
|
||||
fs := flag.NewFlagSet("prepare", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
out := fs.String("out", "-", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
proj, err := s.do("POST", "/api/projects", map[string]any{"name": "drill"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var projBody struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(proj.Body, &projBody); err != nil {
|
||||
return err
|
||||
}
|
||||
conv, err := s.do("POST", "/api/projects/"+projBody.ID+"/conversations", map[string]any{"title": "drill"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var convBody struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(conv.Body, &convBody); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeJSON(*out, map[string]string{"projectId": projBody.ID, "conversationId": convBody.ID})
|
||||
}
|
||||
|
||||
type attempt struct {
|
||||
RequestID string `json:"requestId"`
|
||||
Base string `json:"base"`
|
||||
Started string `json:"started"`
|
||||
ElapsedMs float64 `json:"elapsedMs"`
|
||||
Status int `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
MessageID string `json:"messageId,omitempty"`
|
||||
}
|
||||
|
||||
func cmdTimeline(args []string) error {
|
||||
fs := flag.NewFlagSet("timeline", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
conv := fs.String("conv", "", "conversation id")
|
||||
every := fs.Duration("every", 500*time.Millisecond, "write interval")
|
||||
dur := fs.Duration("dur", 90*time.Second, "how long to write")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var attempts []attempt
|
||||
deadline := time.Now().Add(*dur)
|
||||
for i := 0; time.Now().Before(deadline); i++ {
|
||||
reqID := fmt.Sprintf("tl-%d-%d", time.Now().UnixNano(), i)
|
||||
started := time.Now().UTC()
|
||||
res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{
|
||||
"text": "drill " + reqID, "requestId": reqID,
|
||||
})
|
||||
attempts = append(attempts, attempt{
|
||||
RequestID: reqID, Base: *base, Started: started.Format(time.RFC3339Nano),
|
||||
ElapsedMs: res.ElapsedMs, Status: res.Status, Error: res.Error, MessageID: messageID(res.Body),
|
||||
})
|
||||
time.Sleep(*every)
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"attempts": attempts})
|
||||
}
|
||||
|
||||
func messageID(raw json.RawMessage) string {
|
||||
var body struct {
|
||||
Message struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"message"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &body)
|
||||
return body.Message.ID
|
||||
}
|
||||
|
||||
func cmdIdem(args []string) error {
|
||||
fs := flag.NewFlagSet("idem-verify", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
conv := fs.String("conv", "", "conversation id")
|
||||
in := fs.String("in", "", "timeline json")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(*in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var doc struct {
|
||||
Attempts []attempt `json:"attempts"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var replays []attempt
|
||||
for _, a := range doc.Attempts {
|
||||
if a.Status >= 200 && a.Status < 300 {
|
||||
continue
|
||||
}
|
||||
res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{
|
||||
"text": "drill " + a.RequestID, "requestId": a.RequestID,
|
||||
})
|
||||
replays = append(replays, attempt{
|
||||
RequestID: a.RequestID, Base: *base, Started: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
ElapsedMs: res.ElapsedMs, Status: res.Status, Error: res.Error, MessageID: messageID(res.Body),
|
||||
})
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"replays": replays})
|
||||
}
|
||||
|
||||
func cmdSSE(args []string) error {
|
||||
fs := flag.NewFlagSet("sse-check", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
conv := fs.String("conv", "", "conversation id")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.base+"/api/events?conversationId="+*conv, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stream := &http.Client{Jar: s.client.Jar}
|
||||
resp, err := stream.Do(req)
|
||||
var seqs []int64
|
||||
status := 0
|
||||
errText := ""
|
||||
if err != nil && resp == nil {
|
||||
errText = err.Error()
|
||||
} else if resp != nil {
|
||||
status = resp.StatusCode
|
||||
defer resp.Body.Close()
|
||||
body, readErr := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if readErr != nil {
|
||||
errText = readErr.Error()
|
||||
}
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
if !strings.HasPrefix(line, "id: ") {
|
||||
continue
|
||||
}
|
||||
n, convErr := strconv.ParseInt(strings.TrimPrefix(line, "id: "), 10, 64)
|
||||
if convErr == nil {
|
||||
seqs = append(seqs, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
return writeJSON(*out, map[string]any{
|
||||
"status": status, "error": errText, "seqs": seqs, "holes": seqHoles(seqs),
|
||||
})
|
||||
}
|
||||
|
||||
func cmdMarkers(args []string) error {
|
||||
fs := flag.NewFlagSet("markers", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
conv := fs.String("conv", "", "conversation id")
|
||||
every := fs.Duration("every", 5*time.Second, "marker interval")
|
||||
dur := fs.Duration("dur", 5*time.Minute, "how long to write")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var marks []map[string]any
|
||||
deadline := time.Now().Add(*dur)
|
||||
for i := 0; time.Now().Before(deadline); i++ {
|
||||
reqID := fmt.Sprintf("mk-%d", i)
|
||||
acked := time.Now().UTC()
|
||||
res, _ := s.do("POST", "/api/conversations/"+*conv+"/messages", map[string]any{
|
||||
"text": "marker " + reqID, "requestId": reqID,
|
||||
})
|
||||
marks = append(marks, map[string]any{
|
||||
"requestId": reqID, "acked": acked.Format(time.RFC3339Nano),
|
||||
"status": res.Status, "error": res.Error, "messageId": messageID(res.Body),
|
||||
})
|
||||
time.Sleep(*every)
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"markers": marks})
|
||||
}
|
||||
|
||||
func cmdArtifactsPut(args []string) error {
|
||||
fs := flag.NewFlagSet("artifacts-put", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
project := fs.String("project", "", "project id")
|
||||
n := fs.Int("n", 5, "how many files")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var items []map[string]any
|
||||
for i := 0; i < *n; i++ {
|
||||
payload := []byte(fmt.Sprintf("drill-artifact-%d-%d", i, time.Now().UnixNano()))
|
||||
sum := sha256.Sum256(payload)
|
||||
id, status, errText, acked := upload(s, *project, fmt.Sprintf("drill-%d.txt", i), payload)
|
||||
items = append(items, map[string]any{
|
||||
"name": fmt.Sprintf("drill-%d.txt", i), "sha256": hex.EncodeToString(sum[:]),
|
||||
"id": id, "status": status, "error": errText, "acked": acked.Format(time.RFC3339Nano),
|
||||
})
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"artifacts": items})
|
||||
}
|
||||
|
||||
func upload(s *session, project, name string, payload []byte) (id string, status int, errText string, acked time.Time) {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
part, err := w.CreateFormFile("file", name)
|
||||
if err != nil {
|
||||
return "", 0, err.Error(), time.Now().UTC()
|
||||
}
|
||||
_, _ = part.Write(payload)
|
||||
_ = w.Close()
|
||||
req, err := http.NewRequest(http.MethodPost, s.base+"/api/projects/"+project+"/uploads", &buf)
|
||||
if err != nil {
|
||||
return "", 0, err.Error(), time.Now().UTC()
|
||||
}
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
acked = time.Now().UTC()
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return "", 0, err.Error(), acked
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return body.ID, resp.StatusCode, string(raw), acked
|
||||
}
|
||||
return body.ID, resp.StatusCode, "", acked
|
||||
}
|
||||
|
||||
func cmdArtifactsVerify(args []string) error {
|
||||
fs := flag.NewFlagSet("artifacts-verify", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
in := fs.String("in", "", "artifacts json")
|
||||
before := fs.String("before", "", "only artifacts acked before this RFC3339 time")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := os.ReadFile(*in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var doc struct {
|
||||
Artifacts []struct {
|
||||
ID string `json:"id"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Acked string `json:"acked"`
|
||||
Status int `json:"status"`
|
||||
} `json:"artifacts"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
return err
|
||||
}
|
||||
cutoff, err := time.Parse(time.RFC3339Nano, *before)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
missing := 0
|
||||
mismatch := 0
|
||||
checked := 0
|
||||
for _, item := range doc.Artifacts {
|
||||
acked, err := time.Parse(time.RFC3339Nano, item.Acked)
|
||||
if err != nil || item.Status < 200 || item.Status >= 300 || !acked.Before(cutoff) {
|
||||
continue
|
||||
}
|
||||
checked++
|
||||
req, err := http.NewRequest(http.MethodGet, s.base+"/api/artifacts/"+item.ID+"/content", nil)
|
||||
if err != nil {
|
||||
missing++
|
||||
continue
|
||||
}
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
missing++
|
||||
continue
|
||||
}
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
sum := sha256.Sum256(raw)
|
||||
if hex.EncodeToString(sum[:]) != item.SHA256 {
|
||||
mismatch++
|
||||
}
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"checked": checked, "missing": missing, "mismatch": mismatch})
|
||||
}
|
||||
|
||||
func cmdBurst(args []string) error {
|
||||
fs := flag.NewFlagSet("burst", flag.ContinueOnError)
|
||||
bases := fs.String("bases", "", "comma-separated base URLs")
|
||||
owner := fs.String("owner", "", "login owner")
|
||||
password := fs.String("password", "", "login password")
|
||||
n := fs.Int("n", 40, "requests per base")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, base := range strings.Split(*bases, ",") {
|
||||
base = strings.TrimSpace(base)
|
||||
if base == "" {
|
||||
continue
|
||||
}
|
||||
s, err := newSession(base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i < *n; i++ {
|
||||
res, _ := s.do("GET", "/api/projects", nil)
|
||||
key := fmt.Sprintf("%s:%d", base, res.Status)
|
||||
if res.Status == 0 {
|
||||
key = base + ":error"
|
||||
}
|
||||
counts[key]++
|
||||
}
|
||||
}
|
||||
return writeJSON(*out, map[string]any{"counts": counts})
|
||||
}
|
||||
|
||||
func cmdPerm(args []string) error {
|
||||
fs := flag.NewFlagSet("perm-check", flag.ContinueOnError)
|
||||
base := fs.String("base", "", "instance base URL")
|
||||
owner := fs.String("owner", "", "other user")
|
||||
password := fs.String("password", "", "other user password")
|
||||
project := fs.String("project", "", "someone else's project")
|
||||
artifact := fs.String("artifact", "", "someone else's artifact")
|
||||
out := fs.String("out", "", "json output")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
s, err := newSession(*base, *owner, *password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
proj, _ := s.do("GET", "/api/projects/"+*project+"/conversations", nil)
|
||||
art, _ := s.do("GET", "/api/artifacts/"+*artifact+"/content", nil)
|
||||
denied := func(status int) bool { return status == http.StatusForbidden || status == http.StatusNotFound }
|
||||
return writeJSON(*out, map[string]any{
|
||||
"projectStatus": proj.Status, "artifactStatus": art.Status,
|
||||
"projectDenied": denied(proj.Status), "artifactDenied": denied(art.Status),
|
||||
})
|
||||
}
|
||||
Executable
+148
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env bash
|
||||
# 演练栈公共函数。由 run_fault_drill.sh / run_backup_drill.sh / self_check.sh source。
|
||||
# 不打生产：名称前缀 u7drill 或 u7bk，端口 18980–18999，只杀命令行匹配的 pid。
|
||||
|
||||
ops_port_listening() {
|
||||
local p="$1"
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn 2>/dev/null | awk -v p=":$p" '$4 ~ p"$" { found=1 } END { exit !found }' && return 0
|
||||
fi
|
||||
(exec 3<>"/dev/tcp/127.0.0.1/$p") 2>/dev/null && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
ops_require_name() {
|
||||
local name="$1"
|
||||
case "$name" in
|
||||
u7drill|u7drill-*|u7bk|u7bk-*) ;;
|
||||
*) echo "[ops] refusing instance name $name" >&2; return 2 ;;
|
||||
esac
|
||||
[[ "$name" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || {
|
||||
echo "[ops] illegal instance name $name" >&2
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
ops_require_port() {
|
||||
local port="$1"
|
||||
[[ "$port" =~ ^[0-9]+$ ]] || { echo "[ops] port must be an integer" >&2; return 2; }
|
||||
if (( port < 18980 || port > 18999 )); then
|
||||
echo "[ops] refusing port $port outside 18980-18999" >&2
|
||||
return 2
|
||||
fi
|
||||
}
|
||||
|
||||
ops_resource_gate() {
|
||||
local root="$1" avail_kb load1
|
||||
avail_kb="$(df -Pk "$root" | awk 'NR==2 { print $4 }')"
|
||||
load1="$(awk '{ print $1 }' /proc/loadavg)"
|
||||
if (( avail_kb < 100 * 1024 * 1024 )); then
|
||||
echo "[ops] disk free ${avail_kb}KB is below 100GB; pausing" >&2
|
||||
return 3
|
||||
fi
|
||||
awk -v load1="$load1" 'BEGIN { if (load1+0 > 16) exit 1 }' || {
|
||||
echo "[ops] load average $load1 is above 16; pausing" >&2
|
||||
return 3
|
||||
}
|
||||
}
|
||||
|
||||
# ops_kill_checked PIDFILE NEEDLE：只在 /proc/pid/cmdline 含 NEEDLE 时发信号。
|
||||
ops_kill_checked() {
|
||||
local pidfile="$1" needle="$2" sig="${3:-TERM}"
|
||||
[[ -f "$pidfile" ]] || return 0
|
||||
local pid cmd
|
||||
pid="$(head -1 "$pidfile" 2>/dev/null || true)"
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] || return 0
|
||||
if [[ ! -d "/proc/$pid" ]]; then
|
||||
rm -f "$pidfile"
|
||||
return 0
|
||||
fi
|
||||
cmd="$(tr '\0' ' ' <"/proc/$pid/cmdline" 2>/dev/null || true)"
|
||||
if [[ "$cmd" != *"$needle"* ]]; then
|
||||
echo "[ops] refusing to signal $pid; cmdline does not contain $needle" >&2
|
||||
return 1
|
||||
fi
|
||||
kill "-$sig" "$pid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
ops_pg_create() {
|
||||
local root="$1" name="$2" port="$3"
|
||||
ops_require_name "$name" || return $?
|
||||
ops_require_port "$port" || return $?
|
||||
if ops_port_listening "$port"; then
|
||||
echo "[ops] port $port is in use; refusing before create" >&2
|
||||
return 2
|
||||
fi
|
||||
if [[ -e "$root/runtime/$name" ]]; then
|
||||
echo "[ops] runtime/$name already exists; refusing to touch it" >&2
|
||||
return 2
|
||||
fi
|
||||
"$root/scripts/svc.sh" db create "$name" --port "$port" >/dev/null
|
||||
local addr
|
||||
addr="$(sed -n 's/^addr=//p' "$root/runtime/$name/svc.env" | head -1)"
|
||||
if [[ "$addr" != "127.0.0.1:$port" ]]; then
|
||||
echo "[ops] $name addr is $addr, expected 127.0.0.1:$port" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
ops_pg_start() {
|
||||
local root="$1" name="$2"
|
||||
"$root/scripts/svc.sh" "$name" start >/dev/null
|
||||
}
|
||||
|
||||
ops_pg_stop() {
|
||||
local root="$1" name="$2"
|
||||
"$root/scripts/svc.sh" "$name" stop >/dev/null
|
||||
}
|
||||
|
||||
ops_pg_kill() {
|
||||
local root="$1" name="$2"
|
||||
local pgdata="$root/runtime/$name/pgdata"
|
||||
local pidfile="$pgdata/postmaster.pid"
|
||||
[[ -f "$pidfile" ]] || { echo "[ops] no postmaster.pid for $name" >&2; return 1; }
|
||||
ops_kill_checked "$pidfile" "$pgdata" KILL
|
||||
}
|
||||
|
||||
ops_pg_dsn() {
|
||||
local root="$1" name="$2"
|
||||
"$root/scripts/svc.sh" db dsn "$name" --reveal
|
||||
}
|
||||
|
||||
ops_pg_remove() {
|
||||
local root="$1" name="$2"
|
||||
"$root/scripts/svc.sh" "$name" stop >/dev/null 2>&1 || true
|
||||
"$root/scripts/svc.sh" rm "$name" --purge >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
ops_redis_start() {
|
||||
local port="$1" dir="$2"
|
||||
ops_require_port "$port" || return $?
|
||||
if ops_port_listening "$port"; then
|
||||
echo "[ops] redis port $port is in use" >&2
|
||||
return 2
|
||||
fi
|
||||
mkdir -p "$dir"
|
||||
redis-server --bind 127.0.0.1 --port "$port" --dir "$dir" --dbfilename drill.rdb \
|
||||
--save '' --appendonly no --daemonize yes --pidfile "$dir/redis.pid" --logfile "$dir/redis.log"
|
||||
}
|
||||
|
||||
ops_redis_kill() {
|
||||
local dir="$1" port="$2"
|
||||
ops_kill_checked "$dir/redis.pid" ":$port" KILL
|
||||
}
|
||||
|
||||
ops_snapshot() {
|
||||
local root="$1" dest="$2"
|
||||
{
|
||||
echo "dev_head=$(git -C "$root" rev-parse HEAD)"
|
||||
echo "backend=$(git -C "$root" rev-parse HEAD:modules/backend)"
|
||||
echo "when=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "nproc=$(nproc)"
|
||||
echo "loadavg=$(cat /proc/loadavg)"
|
||||
free -h
|
||||
df -h "$root"
|
||||
echo "--- heavy processes (comm only) ---"
|
||||
ps -eo pcpu,pmem,comm --sort=-pcpu | head -12
|
||||
} >"$dest"
|
||||
}
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib.sh 的正常 / 拒绝 / 中断回收自检。不测量吞吐，不受 load 门槛约束。
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
# shellcheck source=lib.sh
|
||||
source "$ROOT/tests/ops/lib.sh"
|
||||
|
||||
usage() {
|
||||
echo "tests/ops/self_check.sh [--hold]"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" == "--hold" ]]; then
|
||||
owned=0
|
||||
cleanup() {
|
||||
trap - EXIT INT TERM
|
||||
if [[ "$owned" -eq 1 ]]; then
|
||||
ops_pg_remove "$ROOT" u7drill || true
|
||||
fi
|
||||
exit 0
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
ops_pg_create "$ROOT" u7drill 18989
|
||||
owned=1
|
||||
ops_pg_start "$ROOT" u7drill
|
||||
: >"$ROOT/runtime/u7drill/hold.ready"
|
||||
sleep 3600
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ $# -gt 0 ]]; then
|
||||
echo "[ops] unknown arg: $1" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "[ops] self-check: --help"
|
||||
"$0" --help >/dev/null
|
||||
|
||||
echo "[ops] self-check: unknown arg exits 2"
|
||||
set +e
|
||||
"$0" --nope >/dev/null 2>&1
|
||||
ec=$?
|
||||
set -e
|
||||
[[ "$ec" -eq 2 ]]
|
||||
|
||||
echo "[ops] self-check: foreign name and production port are refused"
|
||||
set +e
|
||||
ops_require_name prod
|
||||
ec=$?
|
||||
set -e
|
||||
[[ "$ec" -eq 2 ]]
|
||||
set +e
|
||||
ops_require_port 8787
|
||||
ec=$?
|
||||
set -e
|
||||
[[ "$ec" -eq 2 ]]
|
||||
|
||||
echo "[ops] self-check: busy port is refused before create"
|
||||
python3 -m http.server 18988 --bind 127.0.0.1 >/dev/null 2>&1 &
|
||||
busy_pid=$!
|
||||
for _ in $(seq 1 20); do ops_port_listening 18988 && break; sleep 0.1; done
|
||||
set +e
|
||||
ops_pg_create "$ROOT" u7drill 18988
|
||||
ec=$?
|
||||
set -e
|
||||
kill "$busy_pid" 2>/dev/null || true
|
||||
wait "$busy_pid" 2>/dev/null || true
|
||||
[[ "$ec" -eq 2 ]]
|
||||
[[ ! -e "$ROOT/runtime/u7drill" ]]
|
||||
|
||||
echo "[ops] self-check: SIGINT removes the instance and the port"
|
||||
set -m
|
||||
"$0" --hold >/tmp/u7ops-hold.txt 2>&1 &
|
||||
hold_pid=$!
|
||||
set +m
|
||||
ready=0
|
||||
for _ in $(seq 1 90); do
|
||||
if [[ -f "$ROOT/runtime/u7drill/hold.ready" ]]; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "$hold_pid" 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
if [[ "$ready" -ne 1 ]]; then
|
||||
cat /tmp/u7ops-hold.txt >&2 || true
|
||||
echo "[ops] hold did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
kill -INT -- "-$hold_pid" 2>/dev/null || kill -INT "$hold_pid" 2>/dev/null || true
|
||||
set +e
|
||||
wait "$hold_pid"
|
||||
set -e
|
||||
if ops_port_listening 18989 || [[ -e "$ROOT/runtime/u7drill" ]]; then
|
||||
echo "[ops] SIGINT left port 18989 or runtime/u7drill behind" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[ops] self-check passed"
|
||||
Reference in New Issue
Block a user