diff --git a/.doc/test-reports/2026-09-12-compose-mime-soak-test-report.md b/.doc/test-reports/2026-09-12-compose-mime-soak-test-report.md new file mode 100644 index 00000000..72777505 --- /dev/null +++ b/.doc/test-reports/2026-09-12-compose-mime-soak-test-report.md @@ -0,0 +1,43 @@ +# Compose Runner Token + 上传 MIME 校验 + Test Soak 测试报告 + +## 一、 测试背景 (Background) + +继续 Phase 5 / Gap P1：Compose 注入 `RUNNER_AUTH_TOKEN`、项目材料上传 MIME/魔数校验、test 环境 readiness soak 探测脚本与短循环 CI 测试。 + +## 二、 测试目标 (Test Objectives) + +1. dev/test/main Compose：`app` 与 `pi-agent` 共享 `RUNNER_AUTH_TOKEN` +2. main 环境配置校验：有 Runner URL 时必须配置 auth token +3. 上传：扩展名白名单 + 魔数/HTML/脚本拒绝 → 415 +4. test soak：`/api/v1/ready` 短循环稳定 200 + +## 三、 测试方案与环境 (Test Plan & Environment) + +- Go 1.27, PG dev 5433 +- `internal/upload` 单元测试 + `internal/server` 集成测试 + +## 四、 实际测试结果 (Actual Results) + +``` +go test ./internal/upload/... ./internal/server/... -run 'TestValidate|TestUpload|TestPhase5_Soak' +ok internal/upload 0.003s +ok internal/server 0.962s +``` + +| 用例 | 判定 | +|------|------| +| 合法 txt 上传 | PASS | +| HTML 内容拒绝 | PASS | +| 超长文件名拒绝 | PASS | +| PDF 魔数校验 | PASS | +| readiness 8 次短 soak | PASS | + +## 五、 结论 (Conclusion) + +本批 **PASS**。test 环境长期 soak 请运维执行： + +```bash +scripts/test-soak.sh http://127.0.0.1:8081 86400 60 +``` + +main 启动前需在 `.env` 设置 `RUNNER_AUTH_TOKEN`（Compose 已强制）。 diff --git a/configs/.env.test.example b/configs/.env.test.example index 78d67762..8ca77b0b 100644 --- a/configs/.env.test.example +++ b/configs/.env.test.example @@ -31,5 +31,11 @@ KB_DB_PORT=5432 # 5. Cursor / LLM Agent Runner 服务 CURSOR_RUNNER_PORT=18766 +PI_AGENT_RUNNER_PORT=18866 AGENT_BACKEND=stub AGENT_TIMEOUT=60m +# app 与 pi-agent 容器共享（test 环境建议设置非空值） +# RUNNER_AUTH_TOKEN=test-runner-shared-secret + +# 6. Phase 5 soak 探测示例 +# scripts/test-soak.sh http://127.0.0.1:8081 300 10 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index fd707ad5..52f793d7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -61,6 +61,7 @@ services: MINIMAX_PROVIDER: minimax TAVILY_API_KEYS: ${TAVILY_API_KEYS:-} TAVILY_API_KEY: ${TAVILY_API_KEY:-} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} ports: - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18865}:18865" volumes: @@ -100,6 +101,9 @@ services: POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable CURSOR_RUNNER_URL: http://cursor-agent:18765 PI_AGENT_RUNNER_URL: http://pi-agent:18865 + PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} + ALLOW_IN_MEMORY: "false" AUX_FILE_DSN: file:/var/lib/feishu/outbox_dev.db?_journal_mode=WAL&_busy_timeout=5000 ports: - "${APP_PUBLISH_HOST:-127.0.0.1}:${HTTP_PORT:-8080}:${HTTP_PORT:-8080}" diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 3ee07670..3b8b7a25 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -61,6 +61,7 @@ services: MINIMAX_PROVIDER: minimax TAVILY_API_KEYS: ${TAVILY_API_KEYS:-} TAVILY_API_KEY: ${TAVILY_API_KEY:-} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production} ports: - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18867}:18867" volumes: @@ -98,11 +99,14 @@ services: POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable CURSOR_RUNNER_URL: http://cursor-agent:18767 PI_AGENT_RUNNER_URL: http://pi-agent:18867 + PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production} + ALLOW_IN_MEMORY: "false" AUX_FILE_DSN: file:/var/lib/feishu/outbox_main.db?_journal_mode=WAL&_busy_timeout=5000 ports: - "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8000}:${HTTP_PORT:-8000}" healthcheck: - test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8000}/api/v1/health"] + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8000}/api/v1/ready"] interval: 10s timeout: 5s retries: 6 diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 87af5b96..a53a045e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -61,6 +61,7 @@ services: MINIMAX_PROVIDER: minimax TAVILY_API_KEYS: ${TAVILY_API_KEYS:-} TAVILY_API_KEY: ${TAVILY_API_KEY:-} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} ports: - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18866}:18866" volumes: @@ -98,11 +99,14 @@ services: POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable CURSOR_RUNNER_URL: http://cursor-agent:18766 PI_AGENT_RUNNER_URL: http://pi-agent:18866 + PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} + RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-} + ALLOW_IN_MEMORY: "false" AUX_FILE_DSN: file:/var/lib/feishu/outbox_test.db?_journal_mode=WAL&_busy_timeout=5000 ports: - "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8081}:${HTTP_PORT:-8081}" healthcheck: - test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8081}/api/v1/health"] + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8081}/api/v1/ready"] interval: 10s timeout: 5s retries: 6 diff --git a/internal/config/config.go b/internal/config/config.go index a70967fb..9284068c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -412,6 +412,9 @@ func validateProductionSecrets(cfg *Config) error { if cfg.AllowInMemory { return fmt.Errorf("ALLOW_IN_MEMORY must be false in %s environment", cfg.Env) } + if strings.TrimSpace(cfg.PiAgentRunnerURL) != "" && strings.TrimSpace(cfg.PiAgentRunnerAuthToken) == "" { + return fmt.Errorf("PI_AGENT_RUNNER_AUTH_TOKEN (or RUNNER_AUTH_TOKEN) must be set when PI_AGENT_RUNNER_URL is configured in %s environment", cfg.Env) + } return nil } diff --git a/internal/server/asset_handlers.go b/internal/server/asset_handlers.go index d125f2d8..680745c2 100644 --- a/internal/server/asset_handlers.go +++ b/internal/server/asset_handlers.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strconv" "strings" + + "chat-agent-system/internal/upload" ) type projectAssetResponse struct { @@ -57,9 +59,11 @@ func (s *Server) handleUploadProjectAsset(w http.ResponseWriter, r *http.Request if filename == "" { filename = "upload.bin" } - mimeType := header.Header.Get("Content-Type") - if mimeType == "" { - mimeType = mimeFromExt(filename) + declaredMIME := header.Header.Get("Content-Type") + mimeType, verr := upload.ValidateProjectMaterial(filename, declaredMIME, data) + if verr != nil { + http.Error(w, "upload rejected: "+verr.Error(), http.StatusUnsupportedMediaType) + return } cas := s.svc.CAS() diff --git a/internal/server/asset_handlers_test.go b/internal/server/asset_handlers_test.go index 7358223b..bd9355e4 100644 --- a/internal/server/asset_handlers_test.go +++ b/internal/server/asset_handlers_test.go @@ -6,6 +6,7 @@ import ( "mime/multipart" "net/http" "net/http/httptest" + "strings" "testing" "chat-agent-system/internal/bizdev" @@ -44,3 +45,51 @@ func TestUploadProjectAsset(t *testing.T) { t.Fatalf("unexpected response: %+v", out) } } + +func TestUploadProjectAsset_RejectsHTML(t *testing.T) { + cfg := &config.Config{Env: "test", DataDir: t.TempDir()} + srv := NewServer(cfg, bizdev.MustNewService(t, cfg)) + ts := httptest.NewServer(srv.Routes()) + defer ts.Close() + + var body bytes.Buffer + w := multipart.NewWriter(&body) + part, _ := w.CreateFormFile("file", "evil.txt") + _, _ = part.Write([]byte("x")) + _ = w.Close() + + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/projects/7/assets", &body) + req.Header.Set("Content-Type", w.FormDataContentType()) + withBearerAuth(req, loginTestAdmin(t, srv)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("expected 415, got %d", resp.StatusCode) + } +} + +func TestUploadProjectAsset_RejectsOversizedFilename(t *testing.T) { + cfg := &config.Config{Env: "test", DataDir: t.TempDir()} + srv := NewServer(cfg, bizdev.MustNewService(t, cfg)) + ts := httptest.NewServer(srv.Routes()) + defer ts.Close() + + longName := strings.Repeat("a", 201) + ".txt" + var body bytes.Buffer + w := multipart.NewWriter(&body) + part, _ := w.CreateFormFile("file", longName) + _, _ = part.Write([]byte("ok")) + _ = w.Close() + + req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/projects/7/assets", &body) + req.Header.Set("Content-Type", w.FormDataContentType()) + withBearerAuth(req, loginTestAdmin(t, srv)) + resp, _ := http.DefaultClient.Do(req) + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("expected 415 for long filename, got %d", resp.StatusCode) + } +} diff --git a/internal/server/phase5_soak_test.go b/internal/server/phase5_soak_test.go new file mode 100644 index 00000000..19feb417 --- /dev/null +++ b/internal/server/phase5_soak_test.go @@ -0,0 +1,33 @@ +package server_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestPhase5_SoakReadinessLoop_Short(t *testing.T) { + pool := openPGPoolWithMigrations(t) + if pool == nil { + return + } + defer pool.Close() + + handler, _ := setupPGGovernanceServer(t, pool) + const iterations = 8 + var failures int + for i := 0; i < iterations; i++ { + req := httptest.NewRequest(http.MethodGet, "/api/v1/ready", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + failures++ + t.Logf("iteration %d: ready=%d body=%s", i, rec.Code, rec.Body.String()) + } + time.Sleep(50 * time.Millisecond) + } + if failures > 0 { + t.Fatalf("readiness soak failures=%d/%d", failures, iterations) + } +} diff --git a/internal/upload/validate.go b/internal/upload/validate.go new file mode 100644 index 00000000..086e1e16 --- /dev/null +++ b/internal/upload/validate.go @@ -0,0 +1,151 @@ +package upload + +import ( + "fmt" + "net/http" + "path/filepath" + "strings" + "unicode/utf8" +) + +const maxFilenameRunes = 200 + +// 项目材料上传允许的扩展名 → 期望 MIME。 +var allowedMaterialExt = map[string]string{ + ".txt": "text/plain", + ".md": "text/plain", + ".csv": "text/plain", + ".json": "application/json", + ".xml": "application/xml", + ".yaml": "text/plain", + ".yml": "text/plain", + ".log": "text/plain", + ".pdf": "application/pdf", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", +} + +var blockedDetectedMIME = map[string]struct{}{ + "text/html": {}, + "application/javascript": {}, + "application/x-javascript": {}, +} + +// ValidateProjectMaterial 校验文件名、扩展名白名单与内容魔数/MIME。 +func ValidateProjectMaterial(filename, declaredMIME string, data []byte) (string, error) { + base := filepath.Base(strings.TrimSpace(filename)) + if base == "" || base == "." || base == ".." { + return "", fmt.Errorf("invalid filename") + } + if utf8.RuneCountInString(base) > maxFilenameRunes { + return "", fmt.Errorf("filename too long") + } + ext := strings.ToLower(filepath.Ext(base)) + expectedMIME, ok := allowedMaterialExt[ext] + if !ok { + return "", fmt.Errorf("file type not allowed: %s", ext) + } + if len(data) == 0 { + return "", fmt.Errorf("empty file") + } + if err := rejectDangerousContent(data); err != nil { + return "", err + } + if looksLikeHTML(data) { + return "", fmt.Errorf("html content rejected") + } + detected := http.DetectContentType(data) + if _, blocked := blockedDetectedMIME[detected]; blocked { + return "", fmt.Errorf("dangerous content type detected: %s", detected) + } + if err := assertExtensionMatchesContent(ext, data, detected); err != nil { + return "", err + } + declaredMIME = strings.TrimSpace(strings.Split(declaredMIME, ";")[0]) + if declaredMIME != "" && !mimeCompatible(declaredMIME, expectedMIME, detected) { + return "", fmt.Errorf("declared content type mismatch") + } + return expectedMIME, nil +} + +func looksLikeHTML(data []byte) bool { + sample := strings.ToLower(strings.TrimSpace(string(data[:min(len(data), 512)]))) + return strings.HasPrefix(sample, "= 2 && data[0] == 'M' && data[1] == 'Z' { + return fmt.Errorf("executable content rejected") + } + if len(data) >= 4 && data[0] == 0x7f && data[1] == 'E' && data[2] == 'L' && data[3] == 'F' { + return fmt.Errorf("executable content rejected") + } + head := strings.ToLower(string(data[:min(len(data), 256)])) + if strings.HasPrefix(head, "#!/") || strings.Contains(head, "= len(prefix) && string(data[:len(prefix)]) == string(prefix) +} + +func containsNullByte(data []byte) bool { + for _, b := range data { + if b == 0 { + return true + } + } + return false +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/upload/validate_test.go b/internal/upload/validate_test.go new file mode 100644 index 00000000..e29b413e --- /dev/null +++ b/internal/upload/validate_test.go @@ -0,0 +1,44 @@ +package upload + +import ( + "testing" +) + +func TestValidateProjectMaterial_AcceptsPlainText(t *testing.T) { + mime, err := ValidateProjectMaterial("spec.txt", "text/plain", []byte("hello material")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if mime != "text/plain" { + t.Fatalf("mime=%s", mime) + } +} + +func TestValidateProjectMaterial_RejectsHTML(t *testing.T) { + data := []byte("") + _, err := ValidateProjectMaterial("evil.txt", "text/html", data) + if err == nil { + t.Fatal("expected rejection for html content") + } +} + +func TestValidateProjectMaterial_RejectsBadExtension(t *testing.T) { + _, err := ValidateProjectMaterial("malware.exe", "application/octet-stream", []byte("MZ")) + if err == nil { + t.Fatal("expected rejection for exe extension") + } +} + +func TestValidateProjectMaterial_RejectsPDFMagicMismatch(t *testing.T) { + _, err := ValidateProjectMaterial("fake.pdf", "application/pdf", []byte("not a pdf")) + if err == nil { + t.Fatal("expected pdf magic rejection") + } +} + +func TestValidateProjectMaterial_AcceptsPDF(t *testing.T) { + _, err := ValidateProjectMaterial("doc.pdf", "", []byte("%PDF-1.4\n")) + if err != nil { + t.Fatalf("pdf: %v", err) + } +} diff --git a/scripts/test-soak.sh b/scripts/test-soak.sh new file mode 100755 index 00000000..0efef0c4 --- /dev/null +++ b/scripts/test-soak.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# test-soak.sh — test 环境 readiness 稳定性探测（Phase 5 soak 起步） +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8081}" +DURATION_SEC="${2:-300}" +INTERVAL_SEC="${3:-10}" + +end=$((SECONDS + DURATION_SEC)) +ok=0 +fail=0 + +echo "[soak] base=${BASE_URL} duration=${DURATION_SEC}s interval=${INTERVAL_SEC}s" + +while (( SECONDS < end )); do + if curl -fsS "${BASE_URL}/api/v1/ready" >/dev/null 2>&1; then + ok=$((ok + 1)) + echo "[soak] $(date -Is) ready OK (ok=${ok} fail=${fail})" + else + fail=$((fail + 1)) + echo "[soak] $(date -Is) ready FAIL (ok=${ok} fail=${fail})" >&2 + fi + sleep "${INTERVAL_SEC}" +done + +echo "[soak] finished ok=${ok} fail=${fail}" +if (( fail > 0 )); then + exit 1 +fi