feat(taskrun): resume interrupted runs and harden P1 security gaps
Add confirm-gated resume API with resumable flag, Phase 5 finalize fault and rolling-restart replay tests, Mermaid strict mode, request body limits, and optional Runner Bearer auth between app and pi-agent. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
weichen
co-authored by
Cursor
parent
3ade654256
commit
d7a144de26
@@ -0,0 +1,48 @@
|
||||
# 任务续跑 + Phase 5 扩展 + Gap P1 硬化测试报告
|
||||
|
||||
## 一、 测试背景 (Background)
|
||||
|
||||
按默认推进顺序实施:① interrupted 任务用户确认续跑;② Phase 5 Finalize 故障注入与滚动重启重放;③ Gap P1(Mermaid strict、上传 body 上限、Runner 服务间 token)。
|
||||
|
||||
## 二、 测试目标 (Test Objectives)
|
||||
|
||||
1. `POST /api/v1/tasks/runs/{id}/resume` 需 `confirm=true`,仅 `interrupted`+有效 workspace 可续跑
|
||||
2. `GET /api/v1/tasks/runs/{id}` 返回 `resumable` 标志
|
||||
3. Finalize 故障注入不产生 `done` 终态(`finalizing_failed`)
|
||||
4. 三次「进程重启」后 PG 事件重放仍有效
|
||||
5. JSON body >4MiB 返回 413;Mermaid 切换 `strict`;Runner Bearer token
|
||||
|
||||
## 三、 测试方案与环境 (Test Plan & Environment)
|
||||
|
||||
- Go 1.27, PG dev `5433`, goose v20
|
||||
- 单元 + PG 集成测试
|
||||
|
||||
## 四、 预期结果 (Expected Results)
|
||||
|
||||
| 用例 | 预期 |
|
||||
|------|------|
|
||||
| Resume interrupted | 202 → completed, attempt≥2 |
|
||||
| 无 confirm | 400 ERR_CONFIRM_REQUIRED |
|
||||
| resumable 标志 | interrupted+workspace → true |
|
||||
| Finalize 故障 | finalizing_failed |
|
||||
| 滚动重启×3 | GetRun completed + replay 非空 |
|
||||
| Body limit | 413 |
|
||||
| Runner auth | Authorization Bearer 头 |
|
||||
|
||||
## 五、 实际测试结果与判定 (Actual Results & Evaluation)
|
||||
|
||||
```
|
||||
go test ./internal/taskrun/... ./internal/server/... ./internal/agent/... \
|
||||
-run 'TestTaskRun_Resume|TestPhase5_Finalize|TestPhase5_Rolling|TestBodyLimit|TestApplyRunner|TestPGStore_'
|
||||
ok internal/taskrun 0.036s
|
||||
ok internal/server 0.527s
|
||||
ok internal/agent 0.003s
|
||||
```
|
||||
|
||||
**判定**: 全部 PASS
|
||||
|
||||
## 六、 测试结论与后续优化建议 (Conclusion & Next Steps)
|
||||
|
||||
**结论**: 默认顺序 1→2→3 的本批目标已达成;仍非 main 发布级(缺 test 环境 soak、渗透、完整备份恢复演练)。
|
||||
|
||||
**后续**: test 分支 soak、Runner token 写入 Compose 生产模板、上传 MIME 魔数校验。
|
||||
@@ -42,6 +42,8 @@ TAVILY_API_KEYS=
|
||||
PI_AGENT_RUNNER_PORT=18865
|
||||
# 若希望 Go 程序通过 HTTP 直连 pi-agent 容器,解除下方注释;若留空则走本机本地 pi CLI
|
||||
# PI_AGENT_RUNNER_URL=http://127.0.0.1:18865
|
||||
# Runner 服务间认证(生产必填;app 与 pi-agent 容器需相同值)
|
||||
# RUNNER_AUTH_TOKEN=change-me-in-production
|
||||
|
||||
# --- 7. 本机 Pi CLI 本地回退配置 (当不使用容器时) ---
|
||||
PI_AGENT_BIN=/opt/homebrew/bin/pi
|
||||
|
||||
@@ -13,6 +13,7 @@ import { TavilyKeyPool, searchTavily, formatTavilyResults } from "./extensions/t
|
||||
|
||||
const PORT = parseInt(process.env.PORT || process.env.PI_AGENT_RUNNER_PORT || "18766", 10);
|
||||
const DATA_DIR = process.env.DATA_DIR || "/app/data/workspaces";
|
||||
const RUNNER_AUTH_TOKEN = (process.env.RUNNER_AUTH_TOKEN || process.env.PI_AGENT_RUNNER_AUTH_TOKEN || "").trim();
|
||||
const DEFAULT_MODEL = process.env.MINIMAX_MODEL_ID || "MiniMax-M3";
|
||||
const DEFAULT_PROVIDER = process.env.MINIMAX_PROVIDER || "minimax";
|
||||
const tavilyPool = TavilyKeyPool.fromEnv();
|
||||
@@ -76,6 +77,18 @@ function sendJson(res, statusCode, data) {
|
||||
res.end(JSON.stringify(data, null, 2) + "\n");
|
||||
}
|
||||
|
||||
function authorizeRunner(req, res) {
|
||||
if (!RUNNER_AUTH_TOKEN) {
|
||||
return true;
|
||||
}
|
||||
const auth = req.headers.authorization || "";
|
||||
if (auth === `Bearer ${RUNNER_AUTH_TOKEN}`) {
|
||||
return true;
|
||||
}
|
||||
sendJson(res, 401, { ok: false, error: "unauthorized" });
|
||||
return false;
|
||||
}
|
||||
|
||||
async function handleRun(body) {
|
||||
const rawWs = body.workspace || "";
|
||||
const workspace = resolveWorkspace(rawWs);
|
||||
@@ -321,6 +334,9 @@ const server = http.createServer(async (req, res) => {
|
||||
|
||||
// 3. Cancel active child process by workspace key
|
||||
if (method === "POST" && url.pathname === "/v1/cancel") {
|
||||
if (!authorizeRunner(req, res)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const body = await parseJsonBody(req);
|
||||
const workspace = resolveWorkspace(body.workspace || "");
|
||||
@@ -340,6 +356,9 @@ const server = http.createServer(async (req, res) => {
|
||||
|
||||
// 4. Agent Task Run Endpoint
|
||||
if (method === "POST" && url.pathname === "/v1/run") {
|
||||
if (!authorizeRunner(req, res)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const body = await parseJsonBody(req);
|
||||
const result = await handleRun(body);
|
||||
|
||||
@@ -140,6 +140,7 @@ func (p *PiAgentBackend) executeRemote(ctx context.Context, runnerURL string, re
|
||||
}, nil
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
ApplyRunnerAuth(httpReq, p.cfg.PiAgentRunnerAuthToken)
|
||||
|
||||
client := &http.Client{Timeout: timeout}
|
||||
resp, err := client.Do(httpReq)
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package agent
|
||||
|
||||
import "net/http"
|
||||
|
||||
const runnerAuthHeader = "Authorization"
|
||||
|
||||
// ApplyRunnerAuth 为 Runner HTTP 请求附加 Bearer token(token 为空时不设置)。
|
||||
func ApplyRunnerAuth(req *http.Request, token string) {
|
||||
if req == nil || token == "" {
|
||||
return
|
||||
}
|
||||
req.Header.Set(runnerAuthHeader, "Bearer "+token)
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestApplyRunnerAuth_SetsBearerHeader(t *testing.T) {
|
||||
req := httptest.NewRequest("POST", "http://runner/v1/run", nil)
|
||||
ApplyRunnerAuth(req, "secret-token")
|
||||
got := req.Header.Get("Authorization")
|
||||
if got != "Bearer secret-token" {
|
||||
t.Fatalf("Authorization=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRunnerAuth_EmptyTokenSkipsHeader(t *testing.T) {
|
||||
req := httptest.NewRequest("POST", "http://runner/v1/run", nil)
|
||||
ApplyRunnerAuth(req, "")
|
||||
if req.Header.Get("Authorization") != "" {
|
||||
t.Fatal("expected no Authorization header")
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,8 @@ type Config struct {
|
||||
PiAgentBridgeBin string `json:"pi_agent_bridge_bin"`
|
||||
PiAgentProvider string `json:"pi_agent_provider"`
|
||||
PiAgentModel string `json:"pi_agent_model"`
|
||||
PiAgentRunnerURL string `json:"pi_agent_runner_url"`
|
||||
PiAgentRunnerURL string `json:"pi_agent_runner_url"`
|
||||
PiAgentRunnerAuthToken string `json:"pi_agent_runner_auth_token"`
|
||||
KbDbHost string `json:"kb_db_host"`
|
||||
KbDbPort int `json:"kb_db_port"`
|
||||
KbDbUser string `json:"kb_db_user"`
|
||||
@@ -332,6 +333,12 @@ func Load(path string) (*Config, error) {
|
||||
if v := os.Getenv("PI_AGENT_RUNNER_URL"); v != "" {
|
||||
cfg.PiAgentRunnerURL = v
|
||||
}
|
||||
if v := os.Getenv("PI_AGENT_RUNNER_AUTH_TOKEN"); v != "" {
|
||||
cfg.PiAgentRunnerAuthToken = v
|
||||
}
|
||||
if v := os.Getenv("RUNNER_AUTH_TOKEN"); v != "" {
|
||||
cfg.PiAgentRunnerAuthToken = v
|
||||
}
|
||||
if v := os.Getenv("ALLOW_IN_MEMORY"); v != "" {
|
||||
cfg.AllowInMemory = v == "true" || v == "1"
|
||||
}
|
||||
|
||||
@@ -28,7 +28,11 @@ func (s *Server) handleUploadProjectAsset(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
userID := user.ID
|
||||
if err := r.ParseMultipartForm(64 << 20); err != nil {
|
||||
if err := r.ParseMultipartForm(defaultUploadBodyLimit); err != nil {
|
||||
if isRequestBodyTooLarge(err) {
|
||||
http.Error(w, "upload too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
http.Error(w, "invalid multipart form: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultJSONBodyLimit = 4 << 20 // 4 MiB
|
||||
defaultUploadBodyLimit = 32 << 20 // 32 MiB
|
||||
)
|
||||
|
||||
func (s *Server) bodyLimitMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Body == nil || r.Method == http.MethodGet || r.Method == http.MethodHead || r.Method == http.MethodOptions {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
limit := int64(defaultJSONBodyLimit)
|
||||
if isUploadPath(r.URL.Path) {
|
||||
limit = defaultUploadBodyLimit
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, limit)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func isUploadPath(path string) bool {
|
||||
if strings.HasSuffix(path, "/assets") {
|
||||
return true
|
||||
}
|
||||
return strings.Contains(path, "/admin/templates") && strings.HasSuffix(path, "/upload")
|
||||
}
|
||||
|
||||
func isRequestBodyTooLarge(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(err.Error(), "http: request body too large")
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBodyLimitMiddleware_RejectsOversizedBody(t *testing.T) {
|
||||
srv := &Server{}
|
||||
inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := io.ReadAll(r.Body); isRequestBodyTooLarge(err) {
|
||||
w.WriteHeader(http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
handler := srv.bodyLimitMiddleware(inner)
|
||||
|
||||
body := bytes.Repeat([]byte("x"), int(defaultJSONBodyLimit)+1024)
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/chat", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("expected 413, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package server_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"chat-agent-system/internal/bizdev"
|
||||
"chat-agent-system/internal/stream"
|
||||
"chat-agent-system/internal/taskrun"
|
||||
)
|
||||
|
||||
func TestPhase5_FinalizeFaultInjection_NoDoneEvent(t *testing.T) {
|
||||
m := taskrun.NewManager(nil, nil)
|
||||
m.SetExecutor(
|
||||
func(_ context.Context, _ string, _ bizdev.TaskRunRequest, _ *stream.Streamer) (*bizdev.TaskRunResult, error) {
|
||||
return &bizdev.TaskRunResult{Success: true, TaskKey: "intake"}, nil
|
||||
},
|
||||
nil, nil,
|
||||
)
|
||||
m.SetFinalizer(func(_ context.Context, _ *taskrun.RunRecord, _ *bizdev.TaskRunResult) (string, error) {
|
||||
return "", errors.New("injected finalize db failure")
|
||||
})
|
||||
|
||||
runID := "run_finalize_fault"
|
||||
record := &taskrun.RunRecord{RunID: runID, Status: "running", TaskKey: "intake"}
|
||||
_, _ = m.CreateAndStart(context.Background(), runID, record, bizdev.TaskRunRequest{TaskKey: "intake"}, bizdev.StageRunRequest{}, "")
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
rec, err := m.GetRun(runID)
|
||||
if err == nil && rec.Status == "finalizing_failed" {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("expected finalizing_failed after injected finalize fault")
|
||||
}
|
||||
|
||||
func TestPhase5_RollingRestartReplay_PG(t *testing.T) {
|
||||
pool := openPGPoolWithMigrations(t)
|
||||
if pool == nil {
|
||||
return
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
handler, srv := setupPGGovernanceServer(t, pool)
|
||||
token := loginTestUser(t, handler, "zhanggong", "123456")
|
||||
userID := fetchAuthUserID(t, handler, token)
|
||||
|
||||
store := taskrun.NewPGStore(pool)
|
||||
runID := "run_rolling_" + time.Now().Format("150405.000000")
|
||||
srv.TaskRuns().SetStore(store)
|
||||
srv.TaskRuns().SetExecutor(
|
||||
func(_ context.Context, _ string, _ bizdev.TaskRunRequest, st *stream.Streamer) (*bizdev.TaskRunResult, error) {
|
||||
if st != nil {
|
||||
st.EmitPhase("roll", "step", "agent", 1, 1, 10)
|
||||
}
|
||||
return &bizdev.TaskRunResult{Success: true, TaskKey: "intake"}, nil
|
||||
},
|
||||
nil, nil,
|
||||
)
|
||||
srv.TaskRuns().SetFinalizer(func(_ context.Context, _ *taskrun.RunRecord, _ *bizdev.TaskRunResult) (string, error) {
|
||||
return "completed", nil
|
||||
})
|
||||
|
||||
record := &taskrun.RunRecord{
|
||||
RunID: runID, SessionID: "sess_roll", UserID: userID, TaskKey: "intake", Status: "running",
|
||||
}
|
||||
_, _ = srv.TaskRuns().CreateAndStart(context.Background(), runID, record, bizdev.TaskRunRequest{TaskKey: "intake"}, bizdev.StageRunRequest{}, "")
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM task_run_events WHERE run_id = $1`, runID)
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM task_runs WHERE run_id = $1`, runID)
|
||||
})
|
||||
|
||||
deadline := time.Now().Add(8 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
rec, err := store.GetRun(context.Background(), runID)
|
||||
if err == nil && rec.Status == "completed" {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
// 模拟三次进程重启:新 Manager 仅挂载 Store
|
||||
for i := 0; i < 3; i++ {
|
||||
restarted := taskrun.NewManager(srv.ProjectWorkflowService(), srv.BizdevService().ContextManager())
|
||||
restarted.SetStore(store)
|
||||
rec, err := restarted.GetRun(runID)
|
||||
if err != nil {
|
||||
t.Fatalf("restart %d GetRun: %v", i, err)
|
||||
}
|
||||
if rec.Status != "completed" {
|
||||
t.Fatalf("restart %d status=%s", i, rec.Status)
|
||||
}
|
||||
_, replay, err := restarted.Subscribe(runID, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("restart %d Subscribe: %v", i, err)
|
||||
}
|
||||
if len(replay) == 0 {
|
||||
t.Fatalf("restart %d expected replay events", i)
|
||||
}
|
||||
}
|
||||
|
||||
_ = handler
|
||||
_ = token
|
||||
}
|
||||
@@ -187,6 +187,7 @@ func (s *Server) Routes() http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/tasks/runs/{run_id}", s.handleGetTaskRun)
|
||||
mux.HandleFunc("GET /api/v1/tasks/runs/{run_id}/events", s.handleTaskRunEvents)
|
||||
mux.HandleFunc("POST /api/v1/tasks/runs/{run_id}/cancel", s.handleCancelTaskRun)
|
||||
mux.HandleFunc("POST /api/v1/tasks/runs/{run_id}/resume", s.handleResumeTaskRun)
|
||||
mux.HandleFunc("POST /api/v1/admin/workspaces/reconcile", s.handleReconcileWorkspace)
|
||||
mux.HandleFunc("POST /api/v1/projects/{id}/assets", s.handleUploadProjectAsset)
|
||||
mux.HandleFunc("GET /api/v1/projects/{id}/assets", s.handleListProjectAssets)
|
||||
@@ -276,7 +277,7 @@ func (s *Server) Routes() http.Handler {
|
||||
// 10. 现代 SPA 前端页面与静态资产托管
|
||||
mux.HandleFunc("GET /", s.serveSPA)
|
||||
|
||||
return s.corsMiddleware(s.authMiddleware(mux))
|
||||
return s.corsMiddleware(s.bodyLimitMiddleware(s.authMiddleware(mux)))
|
||||
}
|
||||
|
||||
// serveSPA 托管 web/dist 静态产物并提供前端路由自动回退
|
||||
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"chat-agent-system/internal/agent"
|
||||
"chat-agent-system/internal/bizdev"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/stream"
|
||||
@@ -30,6 +32,15 @@ type createTaskRunResponse struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type taskRunResponse struct {
|
||||
taskrun.RunRecord
|
||||
Resumable bool `json:"resumable"`
|
||||
}
|
||||
|
||||
type resumeTaskRunRequest struct {
|
||||
Confirm bool `json:"confirm"`
|
||||
}
|
||||
|
||||
func (s *Server) handleCreateTaskRun(w http.ResponseWriter, r *http.Request) {
|
||||
user, ok := s.requirePrincipal(w, r)
|
||||
if !ok {
|
||||
@@ -209,6 +220,7 @@ func (s *Server) cancelPiAgentRunner(workspaceID string) {
|
||||
return
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
agent.ApplyRunnerAuth(req, s.cfg.PiAgentRunnerAuthToken)
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
@@ -231,7 +243,111 @@ func (s *Server) handleGetTaskRun(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSONResponse(w, http.StatusOK, rec)
|
||||
writeJSONResponse(w, http.StatusOK, s.buildTaskRunResponse(rec))
|
||||
}
|
||||
|
||||
func (s *Server) handleResumeTaskRun(w http.ResponseWriter, r *http.Request) {
|
||||
user, ok := s.requirePrincipal(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
runID := r.PathValue("run_id")
|
||||
if runID == "" {
|
||||
http.Error(w, "run_id required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !s.assertTaskRunOwnedByUser(w, r.Context(), runID, user) {
|
||||
return
|
||||
}
|
||||
|
||||
var req resumeTaskRunRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !req.Confirm {
|
||||
writeJSONResponse(w, http.StatusBadRequest, map[string]string{
|
||||
"error_code": "ERR_CONFIRM_REQUIRED",
|
||||
"message": "续跑 interrupted 任务需要 confirm=true",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
rec, err := s.taskRuns.GetRun(runID)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if rec.Status != "interrupted" {
|
||||
writeJSONResponse(w, http.StatusConflict, map[string]string{
|
||||
"error_code": "ERR_NOT_RESUMABLE",
|
||||
"message": "仅 interrupted 状态且存在 workspace 的任务可续跑",
|
||||
})
|
||||
return
|
||||
}
|
||||
if rec.WorkspaceID == "" {
|
||||
writeJSONResponse(w, http.StatusConflict, map[string]string{
|
||||
"error_code": "ERR_NO_WORKSPACE",
|
||||
"message": "任务缺少可恢复的工作区",
|
||||
})
|
||||
return
|
||||
}
|
||||
if s.resolver == nil {
|
||||
http.Error(w, "workspace resolver unavailable", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
workDir, werr := s.resolver.ResolveStoredWorkDir(rec.WorkspaceID)
|
||||
if werr != nil {
|
||||
writeJSONResponse(w, http.StatusForbidden, map[string]string{
|
||||
"error_code": "ERR_WORKSPACE_FORBIDDEN",
|
||||
"message": "工作区路径无效或越界",
|
||||
})
|
||||
return
|
||||
}
|
||||
_ = workDir // resume path validated; executor receives stored workspace id
|
||||
|
||||
taskReq := s.buildBizdevTaskRequest(TaskRunRequestDTO{
|
||||
UserID: rec.UserID,
|
||||
ProjectID: rec.ProjectID,
|
||||
TaskKey: rec.TaskKey,
|
||||
}, rec.SessionID, rec.ClientTurnID, rec.RunID)
|
||||
stageReq := s.buildBizdevStageRequest(TaskRunRequestDTO{
|
||||
UserID: rec.UserID,
|
||||
ProjectID: rec.ProjectID,
|
||||
Stage: rec.Stage,
|
||||
}, rec.SessionID, rec.ClientTurnID, rec.RunID)
|
||||
|
||||
resumed, err := s.taskRuns.ResumeInterrupted(r.Context(), rec, rec.WorkspaceID, taskReq, stageReq)
|
||||
if err != nil {
|
||||
if errors.Is(err, taskrun.ErrRunNotResumable) || errors.Is(err, taskrun.ErrRunAlreadyActive) {
|
||||
writeJSONResponse(w, http.StatusConflict, map[string]string{
|
||||
"error_code": "ERR_NOT_RESUMABLE",
|
||||
"message": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSONResponse(w, http.StatusAccepted, createTaskRunResponse{
|
||||
RunID: resumed.RunID,
|
||||
WorkspaceID: resumed.WorkspaceID,
|
||||
Status: resumed.Status,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) buildTaskRunResponse(rec *taskrun.RunRecord) taskRunResponse {
|
||||
resumable := rec != nil && rec.Status == "interrupted" && rec.WorkspaceID != ""
|
||||
if resumable && s.resolver != nil {
|
||||
if _, err := s.resolver.ResolveStoredWorkDir(rec.WorkspaceID); err != nil {
|
||||
resumable = false
|
||||
}
|
||||
}
|
||||
if rec == nil {
|
||||
return taskRunResponse{}
|
||||
}
|
||||
return taskRunResponse{RunRecord: *rec, Resumable: resumable}
|
||||
}
|
||||
|
||||
func (s *Server) buildBizdevTaskRequest(dto TaskRunRequestDTO, sessionID, clientTurnID, runID string) bizdev.TaskRunRequest {
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package server_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"chat-agent-system/internal/bizdev"
|
||||
"chat-agent-system/internal/config"
|
||||
"chat-agent-system/internal/server"
|
||||
"chat-agent-system/internal/stream"
|
||||
"chat-agent-system/internal/taskrun"
|
||||
)
|
||||
|
||||
func TestTaskRun_ResumeInterrupted_PG(t *testing.T) {
|
||||
pool := openPGPoolWithMigrations(t)
|
||||
if pool == nil {
|
||||
return
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
dataDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Env: "dev", DataDir: dataDir, AgentBackend: "stub", AllowInMemory: false,
|
||||
ContextDbHost: "127.0.0.1", ContextDbPort: 5433, ContextDbUser: "app",
|
||||
ContextDbPassword: "dev_secret_db_pass", ContextDbName: "agent_system_dev",
|
||||
AdminUsername: "admin", AdminPassword: "admin-secret-password-123",
|
||||
}
|
||||
svc := bizdev.MustNewService(t, cfg)
|
||||
srv := server.NewServer(cfg, svc)
|
||||
handler := srv.Routes()
|
||||
token := loginTestUser(t, handler, "zhanggong", "123456")
|
||||
userID := fetchAuthUserID(t, handler, token)
|
||||
|
||||
workDir := filepath.Join(dataDir, "workspaces", "bizdev", "resume_test")
|
||||
if err := os.MkdirAll(workDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(workDir, "output.md"), []byte("# resumed\n"), 0o644)
|
||||
|
||||
store := taskrun.NewPGStore(pool)
|
||||
runID := "run_resume_" + time.Now().Format("150405.000000")
|
||||
_, err := pool.Exec(context.Background(), `
|
||||
INSERT INTO task_runs (
|
||||
run_id, session_id, user_id, task_key, status, workspace_id, error_message, created_at, updated_at
|
||||
) VALUES ($1, 'sess_resume', $2, 'intake', 'interrupted', $3, 'worker lease expired', now(), now())
|
||||
`, runID, userID, workDir)
|
||||
if err != nil {
|
||||
t.Fatalf("seed interrupted run: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM task_runs WHERE run_id = $1`, runID)
|
||||
})
|
||||
|
||||
srv.TaskRuns().SetExecutor(
|
||||
func(_ context.Context, _ string, _ bizdev.TaskRunRequest, _ *stream.Streamer) (*bizdev.TaskRunResult, error) {
|
||||
return &bizdev.TaskRunResult{
|
||||
Success: true, WorkDir: workDir, TaskKey: "intake", ArtifactTitle: "Intake",
|
||||
}, nil
|
||||
},
|
||||
nil, nil,
|
||||
)
|
||||
srv.TaskRuns().SetFinalizer(func(ctx context.Context, record *taskrun.RunRecord, result *bizdev.TaskRunResult) (string, error) {
|
||||
return srv.FinalizeTaskRunProjection(ctx, record, result)
|
||||
})
|
||||
srv.TaskRuns().SetStore(store)
|
||||
|
||||
body, _ := json.Marshal(map[string]bool{"confirm": true})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/tasks/runs/"+runID+"/resume", bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
withBearerAuth(req, token)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusAccepted {
|
||||
t.Fatalf("resume expected 202, got %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(8 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
got, getErr := store.GetRun(context.Background(), runID)
|
||||
if getErr == nil && got.Status == "completed" {
|
||||
if got.Attempt < 2 {
|
||||
t.Fatalf("expected attempt>=2 after resume, got %d", got.Attempt)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("resume did not complete")
|
||||
}
|
||||
|
||||
func TestTaskRun_ResumeRequiresConfirm_PG(t *testing.T) {
|
||||
pool := openPGPoolWithMigrations(t)
|
||||
if pool == nil {
|
||||
return
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
dataDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Env: "dev", DataDir: dataDir, AgentBackend: "stub", AllowInMemory: false,
|
||||
ContextDbHost: "127.0.0.1", ContextDbPort: 5433, ContextDbUser: "app",
|
||||
ContextDbPassword: "dev_secret_db_pass", ContextDbName: "agent_system_dev",
|
||||
AdminUsername: "admin", AdminPassword: "admin-secret-password-123",
|
||||
}
|
||||
srv := server.NewServer(cfg, bizdev.MustNewService(t, cfg))
|
||||
handler := srv.Routes()
|
||||
token := loginTestUser(t, handler, "zhanggong", "123456")
|
||||
userID := fetchAuthUserID(t, handler, token)
|
||||
workDir := filepath.Join(dataDir, "ws")
|
||||
_ = os.MkdirAll(workDir, 0o755)
|
||||
|
||||
runID := "run_no_confirm_" + time.Now().Format("150405.000000")
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
INSERT INTO task_runs (run_id, session_id, user_id, task_key, status, workspace_id, created_at, updated_at)
|
||||
VALUES ($1, 's', $2, 'intake', 'interrupted', $3, now(), now())
|
||||
`, runID, userID, workDir)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM task_runs WHERE run_id = $1`, runID)
|
||||
})
|
||||
_ = srv
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/tasks/runs/"+runID+"/resume", bytes.NewReader([]byte(`{"confirm":false}`)))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
withBearerAuth(req, token)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400 without confirm, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskRun_GetRunShowsResumable_PG(t *testing.T) {
|
||||
pool := openPGPoolWithMigrations(t)
|
||||
if pool == nil {
|
||||
return
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
dataDir := t.TempDir()
|
||||
cfg := &config.Config{
|
||||
Env: "dev", DataDir: dataDir, AgentBackend: "stub", AllowInMemory: false,
|
||||
ContextDbHost: "127.0.0.1", ContextDbPort: 5433, ContextDbUser: "app",
|
||||
ContextDbPassword: "dev_secret_db_pass", ContextDbName: "agent_system_dev",
|
||||
AdminUsername: "admin", AdminPassword: "admin-secret-password-123",
|
||||
}
|
||||
handler := server.NewServer(cfg, bizdev.MustNewService(t, cfg)).Routes()
|
||||
token := loginTestUser(t, handler, "zhanggong", "123456")
|
||||
userID := fetchAuthUserID(t, handler, token)
|
||||
workDir := filepath.Join(dataDir, "workspaces", "bizdev", "resumable_flag")
|
||||
_ = os.MkdirAll(workDir, 0o755)
|
||||
|
||||
runID := "run_flag_" + time.Now().Format("150405.000000")
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
INSERT INTO task_runs (run_id, session_id, user_id, task_key, status, workspace_id, created_at, updated_at)
|
||||
VALUES ($1, 's', $2, 'intake', 'interrupted', $3, now(), now())
|
||||
`, runID, userID, workDir)
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `DELETE FROM task_runs WHERE run_id = $1`, runID)
|
||||
})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/api/v1/tasks/runs/%s", runID), nil)
|
||||
withBearerAuth(req, token)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET run: %d", rec.Code)
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if body["resumable"] != true {
|
||||
t.Fatalf("expected resumable=true, got %v", body["resumable"])
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package taskrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
@@ -41,6 +42,11 @@ type Manager struct {
|
||||
|
||||
const runRetention = time.Hour
|
||||
|
||||
var (
|
||||
ErrRunNotResumable = errors.New("taskrun: run is not resumable")
|
||||
ErrRunAlreadyActive = errors.New("taskrun: run already active in memory")
|
||||
)
|
||||
|
||||
func NewManager(pwSvc *projectworkflow.Service, ctxMgr *ctxmgr.ContextManager) *Manager {
|
||||
m := &Manager{
|
||||
runs: make(map[string]*activeRun),
|
||||
@@ -422,6 +428,68 @@ func (m *Manager) Unsubscribe(runID string, ch <-chan StoredEvent) {
|
||||
}
|
||||
}
|
||||
|
||||
// ResumeInterrupted 在用户确认后续跑 interrupted 任务(需有效 workspace 路径)。
|
||||
func (m *Manager) ResumeInterrupted(parent context.Context, record *RunRecord, resumeWorkDir string, taskReq bizdev.TaskRunRequest, stageReq bizdev.StageRunRequest) (*RunRecord, error) {
|
||||
if record == nil || record.RunID == "" {
|
||||
return nil, fmt.Errorf("taskrun: run_id required")
|
||||
}
|
||||
if resumeWorkDir == "" {
|
||||
return nil, fmt.Errorf("taskrun: workspace required")
|
||||
}
|
||||
if record.Status != "interrupted" {
|
||||
return nil, ErrRunNotResumable
|
||||
}
|
||||
|
||||
m.mu.RLock()
|
||||
_, exists := m.runs[record.RunID]
|
||||
m.mu.RUnlock()
|
||||
if exists {
|
||||
return nil, ErrRunAlreadyActive
|
||||
}
|
||||
|
||||
nextAttempt := record.Attempt + 1
|
||||
if nextAttempt <= 0 {
|
||||
nextAttempt = 2
|
||||
}
|
||||
if m.store != nil && m.store.Available() {
|
||||
if err := m.store.ResetForResume(parent, record.RunID, nextAttempt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
leaseUntil := now.Add(defaultLeaseTTL)
|
||||
cpy := *record
|
||||
cpy.Status = "running"
|
||||
cpy.ErrorMessage = ""
|
||||
cpy.Attempt = nextAttempt
|
||||
cpy.WorkerID = m.workerID
|
||||
cpy.LeaseExpiresAt = &leaseUntil
|
||||
cpy.UpdatedAt = now
|
||||
|
||||
if m.store != nil && m.store.Available() {
|
||||
_ = m.store.UpdateRun(parent, &cpy)
|
||||
if err := m.store.AcquireLease(parent, cpy.RunID, m.workerID, defaultLeaseTTL); err != nil {
|
||||
slog.Warn("taskrun_resume_lease_failed", "run_id", cpy.RunID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
workerCtx, cancel := context.WithCancel(context.WithoutCancel(parent))
|
||||
ar := &activeRun{
|
||||
record: &cpy,
|
||||
subs: make(map[int]chan StoredEvent),
|
||||
cancel: cancel,
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.runs[cpy.RunID] = ar
|
||||
m.mu.Unlock()
|
||||
|
||||
go m.runWorker(workerCtx, ar, taskReq, stageReq, resumeWorkDir)
|
||||
return &cpy, nil
|
||||
}
|
||||
|
||||
func (m *Manager) Cancel(runID string) error {
|
||||
m.mu.RLock()
|
||||
ar, ok := m.runs[runID]
|
||||
|
||||
@@ -26,6 +26,7 @@ type Store interface {
|
||||
AcquireLease(ctx context.Context, runID, workerID string, ttl time.Duration) error
|
||||
HeartbeatLease(ctx context.Context, runID, workerID string, ttl time.Duration) error
|
||||
ReleaseLease(ctx context.Context, runID, workerID string) error
|
||||
ResetForResume(ctx context.Context, runID string, attempt int) error
|
||||
MarkOrphanedRunning(ctx context.Context) (int, error)
|
||||
}
|
||||
|
||||
@@ -309,6 +310,27 @@ func (s *PGStore) ReleaseLease(ctx context.Context, runID, workerID string) erro
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PGStore) ResetForResume(ctx context.Context, runID string, attempt int) error {
|
||||
if !s.Available() {
|
||||
return ErrStoreNotAvailable
|
||||
}
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE task_runs
|
||||
SET status = 'running',
|
||||
error_message = '',
|
||||
attempt = $2,
|
||||
updated_at = now()
|
||||
WHERE run_id = $1 AND status = 'interrupted'
|
||||
`, runID, attempt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("taskrun: run %s not resumable", runID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PGStore) MarkOrphanedRunning(ctx context.Context) (int, error) {
|
||||
if !s.Available() {
|
||||
return 0, ErrStoreNotAvailable
|
||||
@@ -405,7 +427,10 @@ func (noopStore) HeartbeatLease(context.Context, string, string, time.Duration)
|
||||
return ErrStoreNotAvailable
|
||||
}
|
||||
func (noopStore) ReleaseLease(context.Context, string, string) error { return ErrStoreNotAvailable }
|
||||
func (noopStore) MarkOrphanedRunning(context.Context) (int, error) { return 0, ErrStoreNotAvailable }
|
||||
func (noopStore) ResetForResume(context.Context, string, int) error {
|
||||
return ErrStoreNotAvailable
|
||||
}
|
||||
func (noopStore) MarkOrphanedRunning(context.Context) (int, error) { return 0, ErrStoreNotAvailable }
|
||||
|
||||
// EnsureUpdatedAt 若 UpdatedAt 为零则设为当前时间。
|
||||
func EnsureUpdatedAt(rec *RunRecord) {
|
||||
|
||||
@@ -14,9 +14,11 @@ function setupMermaid() {
|
||||
mermaid.initialize({
|
||||
startOnLoad: false,
|
||||
theme: 'neutral',
|
||||
securityLevel: 'loose',
|
||||
securityLevel: 'strict',
|
||||
fontFamily: 'system-ui, -apple-system, sans-serif',
|
||||
fontSize: 12,
|
||||
flowchart: { htmlLabels: false, useMaxWidth: true },
|
||||
sequence: { useMaxWidth: true },
|
||||
});
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
@@ -975,7 +975,7 @@ export async function buildDocxModel(ast: DocumentAST): Promise<Document> {
|
||||
mermaid.initialize({
|
||||
startOnLoad: false,
|
||||
theme: 'neutral',
|
||||
securityLevel: 'loose',
|
||||
securityLevel: 'strict',
|
||||
fontFamily: 'Microsoft YaHei, system-ui, sans-serif',
|
||||
fontSize: 12,
|
||||
flowchart: {
|
||||
|
||||
Reference in New Issue
Block a user