weichen
0a7413903d
ci / vet-and-test (pull_request) Successful in 48s
Why: two-process HTTP acceptance (Dev tests/integration/multi-instance, added in the matching Dev-side change) exposed three real cross-instance defects that the U5 library-level tests bypassed via direct Manager calls, plus the U7 fault matrix needs explicit disk-full rejection evidence. Evidence: go test ./... green on SQLite and real PG (AGENT_PG_TEST_DSN); new regressions TestCrossInstanceResolveDoesNotSettleCard (runmgr, PG-gated), TestUploadRejectsCleanlyWhenObjectDirUnwritable and TestArtifactOverwriteRejectsCleanlyWhenObjectDirUnwritable (server). Impact: ResolveHITL no longer settles the card when the run executes on another instance (command-bus fallback stays available); cancelRun enqueues the command with the real conversation id and no longer reports a false local success; object-store failures now provably leave zero DB/object residue. Refs: collaboration storage proposal U7 (sections 3/5/7); task card feature/u7-load-fault-release.
218 lines
7.5 KiB
Go
218 lines
7.5 KiB
Go
package server
|
||
|
||
import (
|
||
"bytes"
|
||
"io"
|
||
"mime/multipart"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
|
||
"agent-system-backend/internal/bus"
|
||
"agent-system-backend/internal/cmdbus"
|
||
"agent-system-backend/internal/plugins"
|
||
"agent-system-backend/internal/profiles"
|
||
"agent-system-backend/internal/runmgr"
|
||
"agent-system-backend/internal/runtime"
|
||
"agent-system-backend/internal/settings"
|
||
"agent-system-backend/internal/store"
|
||
"agent-system-backend/internal/tools"
|
||
)
|
||
|
||
// 协作存储 U7 故障矩阵（§7「磁盘满/备份恢复/整机失效」行的磁盘满维度）：
|
||
// 对象目录不可写时，写入类入口必须**清楚拒绝**（明确 5xx，绝不 2xx 静默或
|
||
// 半成品落库），且恢复后不留后遗症。注入不依赖 chmod（CI 容器以 root 运行，
|
||
// 0500 目录仍可写）：把 artifacts 目录移走并在原位放置同名普通文件，
|
||
// blobstore 临时文件写入得到 ENOTDIR——与磁盘满同为 os.WriteFile 层的
|
||
// 持久性失败。
|
||
//
|
||
// 覆盖：
|
||
// 1. 上传（POST /api/projects/{id}/uploads）在对象目录失效时 5xx，
|
||
// artifacts 表零新增、对象目录零残留；恢复后同一上传成功。
|
||
// 2. 工件内容覆写（PUT /api/artifacts/{id}/content）同样拒绝，
|
||
// 且原内容保持可读（拒绝不产生半写状态）。
|
||
|
||
// breakArtifactsDir 将 <dir>/artifacts 移至 <dir>/artifacts.moved 并在原位
|
||
// 放置同名文件；返回恢复函数。
|
||
func breakArtifactsDir(t *testing.T, dataDir string) (restore func()) {
|
||
t.Helper()
|
||
dir := filepath.Join(dataDir, "artifacts")
|
||
moved := dir + ".moved"
|
||
if err := os.Rename(dir, moved); err != nil {
|
||
t.Fatalf("move artifacts dir: %v", err)
|
||
}
|
||
if err := os.WriteFile(dir, []byte("not a directory"), 0o644); err != nil {
|
||
t.Fatalf("plant file at artifacts path: %v", err)
|
||
}
|
||
return func() {
|
||
if err := os.Remove(dir); err != nil {
|
||
t.Fatalf("remove planted file: %v", err)
|
||
}
|
||
if err := os.Rename(moved, dir); err != nil {
|
||
t.Fatalf("restore artifacts dir: %v", err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// diskFullServer 自建可控 dataDir 的服务（artifacts 根 = <dataDir>/artifacts，
|
||
// 见 store.OpenSQLite 的 finishOpen 约定），其余装配与 admission/e2e 一致。
|
||
func diskFullServer(t *testing.T) (http.Handler, *store.Store, string) {
|
||
t.Helper()
|
||
dir := t.TempDir()
|
||
st, err := store.OpenSQLite(filepath.Join(dir, "test.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { st.Close() })
|
||
reg := tools.NewRegistry()
|
||
reg.RegisterBuiltin()
|
||
setSvc := settings.NewService(st)
|
||
profSvc := profiles.New(st)
|
||
profSvc.EnsureDefault()
|
||
mgr := runmgr.New(st, bus.New(), reg, profSvc, setSvc, &runtime.MockRunner{}, &runtime.MockRunner{}, &runtime.MockRunner{}, &runtime.MockSubagent{})
|
||
t.Cleanup(mgr.Shutdown)
|
||
ph := plugins.New(st, reg, setSvc, filepath.Join(dir, "plugins"))
|
||
srv := New(st, mgr, reg, profSvc, setSvc, ph)
|
||
srv.Bus = cmdbus.New(st, nil, nil)
|
||
srv.WireBuiltinCommands()
|
||
return srv.Handler(), st, dir
|
||
}
|
||
|
||
func multipartUpload(t *testing.T, h http.Handler, projectID, filename string) *httptest.ResponseRecorder {
|
||
t.Helper()
|
||
var buf bytes.Buffer
|
||
mw := multipart.NewWriter(&buf)
|
||
fw, err := mw.CreateFormFile("file", filename)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := io.Copy(fw, bytes.NewReader([]byte("disk-full probe content: "+filename))); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
mw.Close()
|
||
req := httptest.NewRequest(http.MethodPost, "/api/projects/"+projectID+"/uploads", &buf)
|
||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||
rec := httptest.NewRecorder()
|
||
h.ServeHTTP(rec, req)
|
||
return rec
|
||
}
|
||
|
||
func artifactCount(t *testing.T, st *store.Store, projectID string) (int, map[string]bool) {
|
||
t.Helper()
|
||
arts, err := st.ListArtifacts(store.OwnerDefault, projectID)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
names := map[string]bool{}
|
||
for _, a := range arts {
|
||
names[a.Name] = true
|
||
}
|
||
return len(arts), names
|
||
}
|
||
|
||
func objectsIn(t *testing.T, dir string) map[string]bool {
|
||
t.Helper()
|
||
out := map[string]bool{}
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, e := range entries {
|
||
out[e.Name()] = true
|
||
}
|
||
return out
|
||
}
|
||
|
||
func TestUploadRejectsCleanlyWhenObjectDirUnwritable(t *testing.T) {
|
||
h, st, dir := diskFullServer(t)
|
||
code, out := doJSON(t, h, "POST", "/api/projects", map[string]any{"name": "disk-full"})
|
||
if code != http.StatusOK {
|
||
t.Fatalf("create project: %d %v", code, out)
|
||
}
|
||
projID := out["id"].(string)
|
||
|
||
seed := multipartUpload(t, h, projID, "seed.txt")
|
||
if seed.Code != http.StatusOK {
|
||
t.Fatalf("seed upload: %d %.200s", seed.Code, seed.Body.String())
|
||
}
|
||
beforeCount, _ := artifactCount(t, st, projID)
|
||
beforeObjs := objectsIn(t, filepath.Join(dir, "artifacts"))
|
||
|
||
restore := breakArtifactsDir(t, dir)
|
||
rec := multipartUpload(t, h, projID, "probe-diskfull.txt")
|
||
if rec.Code < 500 {
|
||
t.Fatalf("upload under broken object dir: status %d (want explicit 5xx rejection), body %.200s", rec.Code, rec.Body.String())
|
||
}
|
||
if got, _ := artifactCount(t, st, projID); got != beforeCount {
|
||
t.Fatalf("rejected upload left DB rows: %d -> %d", beforeCount, got)
|
||
}
|
||
if got := objectsIn(t, filepath.Join(dir, "artifacts.moved")); len(got) != len(beforeObjs) {
|
||
t.Fatalf("rejected upload left object residue: %v -> %v", beforeObjs, got)
|
||
}
|
||
restore()
|
||
|
||
after := multipartUpload(t, h, projID, "probe-diskfull-recovered.txt")
|
||
if after.Code != http.StatusOK {
|
||
t.Fatalf("upload after recovery: %d %.200s", after.Code, after.Body.String())
|
||
}
|
||
if got, names := artifactCount(t, st, projID); got != beforeCount+1 || !names["probe-diskfull-recovered.txt"] {
|
||
t.Fatalf("recovered upload missing row: count %d names %v", got, names)
|
||
}
|
||
}
|
||
|
||
func TestArtifactOverwriteRejectsCleanlyWhenObjectDirUnwritable(t *testing.T) {
|
||
h, st, dir := diskFullServer(t)
|
||
code, out := doJSON(t, h, "POST", "/api/projects", map[string]any{"name": "disk-full-2"})
|
||
if code != http.StatusOK {
|
||
t.Fatalf("create project: %d %v", code, out)
|
||
}
|
||
projID := out["id"].(string)
|
||
|
||
seed := multipartUpload(t, h, projID, "base.txt")
|
||
if seed.Code != http.StatusOK {
|
||
t.Fatalf("seed upload: %d %.200s", seed.Code, seed.Body.String())
|
||
}
|
||
arts, err := st.ListArtifacts(store.OwnerDefault, projID)
|
||
if err != nil || len(arts) != 1 {
|
||
t.Fatalf("seed artifacts: %d err %v", len(arts), err)
|
||
}
|
||
artID := string(arts[0].ID)
|
||
|
||
getContent := func() string {
|
||
req := httptest.NewRequest(http.MethodGet, "/api/artifacts/"+artID+"/content", nil)
|
||
rec := httptest.NewRecorder()
|
||
h.ServeHTTP(rec, req)
|
||
if rec.Code != http.StatusOK {
|
||
t.Fatalf("get content: %d %.200s", rec.Code, rec.Body.String())
|
||
}
|
||
return rec.Body.String()
|
||
}
|
||
before := getContent()
|
||
|
||
restore := breakArtifactsDir(t, dir)
|
||
put := httptest.NewRequest(http.MethodPut, "/api/artifacts/"+artID+"/content", bytes.NewReader([]byte("half-written?")))
|
||
putRec := httptest.NewRecorder()
|
||
h.ServeHTTP(putRec, put)
|
||
if putRec.Code < 500 {
|
||
t.Fatalf("overwrite under broken object dir: status %d (want explicit 5xx)", putRec.Code)
|
||
}
|
||
restore()
|
||
|
||
// 原位文件占位已撤；拒绝期间不允许半写污染读取路径。
|
||
if got := getContent(); got != before {
|
||
t.Fatalf("rejected overwrite altered readable content: %q -> %q", before, got)
|
||
}
|
||
|
||
put2 := httptest.NewRequest(http.MethodPut, "/api/artifacts/"+artID+"/content", bytes.NewReader([]byte("recovered write")))
|
||
put2Rec := httptest.NewRecorder()
|
||
h.ServeHTTP(put2Rec, put2)
|
||
if put2Rec.Code != http.StatusOK {
|
||
t.Fatalf("overwrite after recovery: %d %.200s", put2Rec.Code, put2Rec.Body.String())
|
||
}
|
||
if got := getContent(); got != "recovered write" {
|
||
t.Fatalf("content after recovered overwrite: %q", got)
|
||
}
|
||
}
|