feat(ops): compose runner auth, upload MIME validation, and test soak

Wire RUNNER_AUTH_TOKEN through dev/test/main Compose, validate uploads with extension whitelist and content sniffing, add readiness soak script and short PG loop test, and require runner token in prod when URL is set. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-13 02:50:23 +00:00
parent d7a144de26
commit 5856585d1d
12 changed files with 379 additions and 5 deletions
@@ -0,0 +1,43 @@
+# Compose Runner Token + 上传 MIME 校验 + Test Soak 测试报告
+
+## 一、 测试背景 (Background)
+
+继续 Phase 5 / Gap P1：Compose 注入 `RUNNER_AUTH_TOKEN`、项目材料上传 MIME/魔数校验、test 环境 readiness soak 探测脚本与短循环 CI 测试。
+
+## 二、 测试目标 (Test Objectives)
+
+1. dev/test/main Compose：`app` 与 `pi-agent` 共享 `RUNNER_AUTH_TOKEN`
+2. main 环境配置校验：有 Runner URL 时必须配置 auth token
+3. 上传：扩展名白名单 + 魔数/HTML/脚本拒绝 → 415
+4. test soak：`/api/v1/ready` 短循环稳定 200
+
+## 三、 测试方案与环境 (Test Plan & Environment)
+
+- Go 1.27, PG dev 5433
+- `internal/upload` 单元测试 + `internal/server` 集成测试
+
+## 四、 实际测试结果 (Actual Results)
+
+```
+go test ./internal/upload/... ./internal/server/... -run 'TestValidate|TestUpload|TestPhase5_Soak'
+ok   internal/upload   0.003s
+ok   internal/server   0.962s
+```
+
+| 用例 | 判定 |
+|------|------|
+| 合法 txt 上传 | PASS |
+| HTML 内容拒绝 | PASS |
+| 超长文件名拒绝 | PASS |
+| PDF 魔数校验 | PASS |
+| readiness 8 次短 soak | PASS |
+
+## 五、 结论 (Conclusion)
+
+本批 **PASS**。test 环境长期 soak 请运维执行：
+
+```bash
+scripts/test-soak.sh http://127.0.0.1:8081 86400 60
+```
+
+main 启动前需在 `.env` 设置 `RUNNER_AUTH_TOKEN`（Compose 已强制）。
@@ -31,5 +31,11 @@ KB_DB_PORT=5432

 # 5. Cursor / LLM Agent Runner 服务
 CURSOR_RUNNER_PORT=18766
+PI_AGENT_RUNNER_PORT=18866
 AGENT_BACKEND=stub
 AGENT_TIMEOUT=60m
+# app 与 pi-agent 容器共享（test 环境建议设置非空值）
+# RUNNER_AUTH_TOKEN=test-runner-shared-secret
+
+# 6. Phase 5 soak 探测示例
+# scripts/test-soak.sh http://127.0.0.1:8081 300 10
@@ -61,6 +61,7 @@ services:
      MINIMAX_PROVIDER: minimax
      TAVILY_API_KEYS: ${TAVILY_API_KEYS:-}
      TAVILY_API_KEY: ${TAVILY_API_KEY:-}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
    ports:
      - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18865}:18865"
    volumes:
@@ -100,6 +101,9 @@ services:
      POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable
      CURSOR_RUNNER_URL: http://cursor-agent:18765
      PI_AGENT_RUNNER_URL: http://pi-agent:18865
+      PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
+      ALLOW_IN_MEMORY: "false"
      AUX_FILE_DSN: file:/var/lib/feishu/outbox_dev.db?_journal_mode=WAL&_busy_timeout=5000
    ports:
      - "${APP_PUBLISH_HOST:-127.0.0.1}:${HTTP_PORT:-8080}:${HTTP_PORT:-8080}"
@@ -61,6 +61,7 @@ services:
      MINIMAX_PROVIDER: minimax
      TAVILY_API_KEYS: ${TAVILY_API_KEYS:-}
      TAVILY_API_KEY: ${TAVILY_API_KEY:-}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production}
    ports:
      - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18867}:18867"
    volumes:
@@ -98,11 +99,14 @@ services:
      POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable
      CURSOR_RUNNER_URL: http://cursor-agent:18767
      PI_AGENT_RUNNER_URL: http://pi-agent:18867
+      PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:?RUNNER_AUTH_TOKEN is required for main/production}
+      ALLOW_IN_MEMORY: "false"
      AUX_FILE_DSN: file:/var/lib/feishu/outbox_main.db?_journal_mode=WAL&_busy_timeout=5000
    ports:
      - "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8000}:${HTTP_PORT:-8000}"
    healthcheck:
-      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8000}/api/v1/health"]
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8000}/api/v1/ready"]
      interval: 10s
      timeout: 5s
      retries: 6
@@ -61,6 +61,7 @@ services:
      MINIMAX_PROVIDER: minimax
      TAVILY_API_KEYS: ${TAVILY_API_KEYS:-}
      TAVILY_API_KEY: ${TAVILY_API_KEY:-}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
    ports:
      - "127.0.0.1:${PI_AGENT_RUNNER_PORT:-18866}:18866"
    volumes:
@@ -98,11 +99,14 @@ services:
      POSTGRES_SHARED_KB_DSN: postgres://${KB_DB_USER:-regkb}:${KB_DB_PASSWORD:-shared_kb_secret_pass}@shuimu-kb-shared:5432/${KB_DB_NAME:-regulations}?sslmode=disable
      CURSOR_RUNNER_URL: http://cursor-agent:18766
      PI_AGENT_RUNNER_URL: http://pi-agent:18866
+      PI_AGENT_RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
+      RUNNER_AUTH_TOKEN: ${RUNNER_AUTH_TOKEN:-}
+      ALLOW_IN_MEMORY: "false"
      AUX_FILE_DSN: file:/var/lib/feishu/outbox_test.db?_journal_mode=WAL&_busy_timeout=5000
    ports:
      - "${APP_PUBLISH_HOST:-0.0.0.0}:${HTTP_PORT:-8081}:${HTTP_PORT:-8081}"
    healthcheck:
-      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8081}/api/v1/health"]
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:${HTTP_PORT:-8081}/api/v1/ready"]
      interval: 10s
      timeout: 5s
      retries: 6
@@ -412,6 +412,9 @@ func validateProductionSecrets(cfg *Config) error {
 	if cfg.AllowInMemory {
 		return fmt.Errorf("ALLOW_IN_MEMORY must be false in %s environment", cfg.Env)
 	}
+	if strings.TrimSpace(cfg.PiAgentRunnerURL) != "" && strings.TrimSpace(cfg.PiAgentRunnerAuthToken) == "" {
+		return fmt.Errorf("PI_AGENT_RUNNER_AUTH_TOKEN (or RUNNER_AUTH_TOKEN) must be set when PI_AGENT_RUNNER_URL is configured in %s environment", cfg.Env)
+	}
 	return nil
 }

@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+
+	"chat-agent-system/internal/upload"
 )

 type projectAssetResponse struct {
@@ -57,9 +59,11 @@ func (s *Server) handleUploadProjectAsset(w http.ResponseWriter, r *http.Request
 	if filename == "" {
 		filename = "upload.bin"
 	}
-	mimeType := header.Header.Get("Content-Type")
-	if mimeType == "" {
-		mimeType = mimeFromExt(filename)
+	declaredMIME := header.Header.Get("Content-Type")
+	mimeType, verr := upload.ValidateProjectMaterial(filename, declaredMIME, data)
+	if verr != nil {
+		http.Error(w, "upload rejected: "+verr.Error(), http.StatusUnsupportedMediaType)
+		return
 	}

 	cas := s.svc.CAS()
@@ -6,6 +6,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"

 	"chat-agent-system/internal/bizdev"
@@ -44,3 +45,51 @@ func TestUploadProjectAsset(t *testing.T) {
 		t.Fatalf("unexpected response: %+v", out)
 	}
 }
+
+func TestUploadProjectAsset_RejectsHTML(t *testing.T) {
+	cfg := &config.Config{Env: "test", DataDir: t.TempDir()}
+	srv := NewServer(cfg, bizdev.MustNewService(t, cfg))
+	ts := httptest.NewServer(srv.Routes())
+	defer ts.Close()
+
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	part, _ := w.CreateFormFile("file", "evil.txt")
+	_, _ = part.Write([]byte("<html><body>x</body></html>"))
+	_ = w.Close()
+
+	req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/projects/7/assets", &body)
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	withBearerAuth(req, loginTestAdmin(t, srv))
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusUnsupportedMediaType {
+		t.Fatalf("expected 415, got %d", resp.StatusCode)
+	}
+}
+
+func TestUploadProjectAsset_RejectsOversizedFilename(t *testing.T) {
+	cfg := &config.Config{Env: "test", DataDir: t.TempDir()}
+	srv := NewServer(cfg, bizdev.MustNewService(t, cfg))
+	ts := httptest.NewServer(srv.Routes())
+	defer ts.Close()
+
+	longName := strings.Repeat("a", 201) + ".txt"
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	part, _ := w.CreateFormFile("file", longName)
+	_, _ = part.Write([]byte("ok"))
+	_ = w.Close()
+
+	req, _ := http.NewRequest(http.MethodPost, ts.URL+"/api/v1/projects/7/assets", &body)
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	withBearerAuth(req, loginTestAdmin(t, srv))
+	resp, _ := http.DefaultClient.Do(req)
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusUnsupportedMediaType {
+		t.Fatalf("expected 415 for long filename, got %d", resp.StatusCode)
+	}
+}
@@ -0,0 +1,33 @@
+package server_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestPhase5_SoakReadinessLoop_Short(t *testing.T) {
+	pool := openPGPoolWithMigrations(t)
+	if pool == nil {
+		return
+	}
+	defer pool.Close()
+
+	handler, _ := setupPGGovernanceServer(t, pool)
+	const iterations = 8
+	var failures int
+	for i := 0; i < iterations; i++ {
+		req := httptest.NewRequest(http.MethodGet, "/api/v1/ready", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			failures++
+			t.Logf("iteration %d: ready=%d body=%s", i, rec.Code, rec.Body.String())
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	if failures > 0 {
+		t.Fatalf("readiness soak failures=%d/%d", failures, iterations)
+	}
+}
@@ -0,0 +1,151 @@
+package upload
+
+import (
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+)
+
+const maxFilenameRunes = 200
+
+// 项目材料上传允许的扩展名 → 期望 MIME。
+var allowedMaterialExt = map[string]string{
+	".txt":  "text/plain",
+	".md":   "text/plain",
+	".csv":  "text/plain",
+	".json": "application/json",
+	".xml":  "application/xml",
+	".yaml": "text/plain",
+	".yml":  "text/plain",
+	".log":  "text/plain",
+	".pdf":  "application/pdf",
+	".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+}
+
+var blockedDetectedMIME = map[string]struct{}{
+	"text/html":              {},
+	"application/javascript": {},
+	"application/x-javascript": {},
+}
+
+// ValidateProjectMaterial 校验文件名、扩展名白名单与内容魔数/MIME。
+func ValidateProjectMaterial(filename, declaredMIME string, data []byte) (string, error) {
+	base := filepath.Base(strings.TrimSpace(filename))
+	if base == "" || base == "." || base == ".." {
+		return "", fmt.Errorf("invalid filename")
+	}
+	if utf8.RuneCountInString(base) > maxFilenameRunes {
+		return "", fmt.Errorf("filename too long")
+	}
+	ext := strings.ToLower(filepath.Ext(base))
+	expectedMIME, ok := allowedMaterialExt[ext]
+	if !ok {
+		return "", fmt.Errorf("file type not allowed: %s", ext)
+	}
+	if len(data) == 0 {
+		return "", fmt.Errorf("empty file")
+	}
+	if err := rejectDangerousContent(data); err != nil {
+		return "", err
+	}
+	if looksLikeHTML(data) {
+		return "", fmt.Errorf("html content rejected")
+	}
+	detected := http.DetectContentType(data)
+	if _, blocked := blockedDetectedMIME[detected]; blocked {
+		return "", fmt.Errorf("dangerous content type detected: %s", detected)
+	}
+	if err := assertExtensionMatchesContent(ext, data, detected); err != nil {
+		return "", err
+	}
+	declaredMIME = strings.TrimSpace(strings.Split(declaredMIME, ";")[0])
+	if declaredMIME != "" && !mimeCompatible(declaredMIME, expectedMIME, detected) {
+		return "", fmt.Errorf("declared content type mismatch")
+	}
+	return expectedMIME, nil
+}
+
+func looksLikeHTML(data []byte) bool {
+	sample := strings.ToLower(strings.TrimSpace(string(data[:min(len(data), 512)])))
+	return strings.HasPrefix(sample, "<!doctype") ||
+		strings.HasPrefix(sample, "<html") ||
+		strings.Contains(sample, "<script") ||
+		strings.Contains(sample, "<iframe")
+}
+
+func rejectDangerousContent(data []byte) error {
+	if len(data) >= 2 && data[0] == 'M' && data[1] == 'Z' {
+		return fmt.Errorf("executable content rejected")
+	}
+	if len(data) >= 4 && data[0] == 0x7f && data[1] == 'E' && data[2] == 'L' && data[3] == 'F' {
+		return fmt.Errorf("executable content rejected")
+	}
+	head := strings.ToLower(string(data[:min(len(data), 256)]))
+	if strings.HasPrefix(head, "#!/") || strings.Contains(head, "<?php") {
+		return fmt.Errorf("script content rejected")
+	}
+	return nil
+}
+
+func assertExtensionMatchesContent(ext string, data []byte, detected string) error {
+	switch ext {
+	case ".pdf":
+		if !bytesHasPrefix(data, []byte("%PDF")) {
+			return fmt.Errorf("pdf magic mismatch")
+		}
+	case ".docx", ".xlsx":
+		if !bytesHasPrefix(data, []byte("PK\x03\x04")) {
+			return fmt.Errorf("office document magic mismatch")
+		}
+	case ".txt", ".md", ".csv", ".log", ".yaml", ".yml":
+		if containsNullByte(data[:min(len(data), 8192)]) {
+			return fmt.Errorf("text file contains binary content")
+		}
+	case ".json":
+		if !strings.HasPrefix(strings.TrimSpace(string(data[:min(len(data), 32)])), "{") &&
+			!strings.HasPrefix(strings.TrimSpace(string(data[:min(len(data), 32)])), "[") {
+			return fmt.Errorf("json magic mismatch")
+		}
+	}
+	_ = detected
+	return nil
+}
+
+func mimeCompatible(declared, expected, detected string) bool {
+	if declared == expected {
+		return true
+	}
+	if strings.HasPrefix(detected, "text/") && strings.HasPrefix(expected, "text/") {
+		return true
+	}
+	if detected == "application/zip" && (expected == allowedMaterialExt[".docx"] || expected == allowedMaterialExt[".xlsx"]) {
+		return true
+	}
+	if detected == "application/octet-stream" {
+		return declared == expected || declared == "application/octet-stream"
+	}
+	return declared == detected
+}
+
+func bytesHasPrefix(data, prefix []byte) bool {
+	return len(data) >= len(prefix) && string(data[:len(prefix)]) == string(prefix)
+}
+
+func containsNullByte(data []byte) bool {
+	for _, b := range data {
+		if b == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
@@ -0,0 +1,44 @@
+package upload
+
+import (
+	"testing"
+)
+
+func TestValidateProjectMaterial_AcceptsPlainText(t *testing.T) {
+	mime, err := ValidateProjectMaterial("spec.txt", "text/plain", []byte("hello material"))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if mime != "text/plain" {
+		t.Fatalf("mime=%s", mime)
+	}
+}
+
+func TestValidateProjectMaterial_RejectsHTML(t *testing.T) {
+	data := []byte("<html><script>alert(1)</script></html>")
+	_, err := ValidateProjectMaterial("evil.txt", "text/html", data)
+	if err == nil {
+		t.Fatal("expected rejection for html content")
+	}
+}
+
+func TestValidateProjectMaterial_RejectsBadExtension(t *testing.T) {
+	_, err := ValidateProjectMaterial("malware.exe", "application/octet-stream", []byte("MZ"))
+	if err == nil {
+		t.Fatal("expected rejection for exe extension")
+	}
+}
+
+func TestValidateProjectMaterial_RejectsPDFMagicMismatch(t *testing.T) {
+	_, err := ValidateProjectMaterial("fake.pdf", "application/pdf", []byte("not a pdf"))
+	if err == nil {
+		t.Fatal("expected pdf magic rejection")
+	}
+}
+
+func TestValidateProjectMaterial_AcceptsPDF(t *testing.T) {
+	_, err := ValidateProjectMaterial("doc.pdf", "", []byte("%PDF-1.4\n"))
+	if err != nil {
+		t.Fatalf("pdf: %v", err)
+	}
+}
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# test-soak.sh — test 环境 readiness 稳定性探测（Phase 5 soak 起步）
+set -euo pipefail
+
+BASE_URL="${1:-http://127.0.0.1:8081}"
+DURATION_SEC="${2:-300}"
+INTERVAL_SEC="${3:-10}"
+
+end=$((SECONDS + DURATION_SEC))
+ok=0
+fail=0
+
+echo "[soak] base=${BASE_URL} duration=${DURATION_SEC}s interval=${INTERVAL_SEC}s"
+
+while (( SECONDS < end )); do
+  if curl -fsS "${BASE_URL}/api/v1/ready" >/dev/null 2>&1; then
+    ok=$((ok + 1))
+    echo "[soak] $(date -Is) ready OK (ok=${ok} fail=${fail})"
+  else
+    fail=$((fail + 1))
+    echo "[soak] $(date -Is) ready FAIL (ok=${ok} fail=${fail})" >&2
+  fi
+  sleep "${INTERVAL_SEC}"
+done
+
+echo "[soak] finished ok=${ok} fail=${fail}"
+if (( fail > 0 )); then
+  exit 1
+fi