package chat_test // 长任务治理 × 路由层 真实后端质量评估 harness。 // 运行方式(内部测试目录为 cwd): // // RUN_QUALITY_E2E=1 go test ./internal/chat/ -run TestLongTaskGovernance_QualityE2E -v -timeout 90m // // 产出存档至 .doc/test-reports/2026-09-09-longtask-governance-quality/ 供人工审阅对比: // 01-router-upgrade/ 对话初答 vs 小队交付物(同题质量对比) // 02-standard-research/ 固定 workflow 标准任务治理运行 // REPORT.md 指标汇总 import ( "context" "fmt" "os" "path/filepath" "strings" "testing" "time" "chat-agent-system/internal/agent" "chat-agent-system/internal/bizdev" "chat-agent-system/internal/chat" "chat-agent-system/internal/coordinator" "chat-agent-system/internal/config" "chat-agent-system/internal/intent" ) const qualityComplexInput = "帮我做一份掌上超声与进口竞品(如 GE Vscan、Philips Lumify)的横向对标分析报告,覆盖法规注册路径、工程架构与商务定价三个视角,要详尽成体系,并给出 V0.5 阶段的切入建议" func qualityReportDir(t *testing.T) string { dir := os.Getenv("QUALITY_REPORT_DIR") if dir == "" { dir = "../../.doc/test-reports/2026-09-09-longtask-governance-quality" } if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatalf("mkdir report dir: %v", err) } return dir } func saveQualityFile(t *testing.T, dir, name, content string) { t.Helper() if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil { t.Fatalf("write %s: %v", name, err) } } func copyQualityWorkspace(t *testing.T, srcWorkDir, dstDir string) { t.Helper() if srcWorkDir == "" { return } if err := os.MkdirAll(dstDir, 0o755); err != nil { t.Fatalf("mkdir workspace archive: %v", err) } for _, name := range []string{ "output.md", "workflow.json", "blackboard.json", "ledger.json", "trace.jsonl", "gov_audit.jsonl", "manifest.json", "workspace.json", "advisor_brief.json", } { if b, err := os.ReadFile(filepath.Join(srcWorkDir, name)); err == nil { _ = os.WriteFile(filepath.Join(dstDir, name), b, 0o644) } } // 各步骤 step_manifest 与 output tasksDir := filepath.Join(srcWorkDir, "tasks") if entries, err := os.ReadDir(tasksDir); err == nil { for _, e := range entries { if !e.IsDir() { continue } subDst := filepath.Join(dstDir, "tasks", e.Name()) _ = os.MkdirAll(subDst, 0o755) for _, f := range []string{"output.md", "step_manifest.json", "TASK.md"} { if b, err := os.ReadFile(filepath.Join(tasksDir, e.Name(), f)); err == nil { _ = os.WriteFile(filepath.Join(subDst, f), b, 0o644) } } } } } func TestLongTaskGovernance_QualityE2E(t *testing.T) { if os.Getenv("RUN_QUALITY_E2E") == "" { t.Skip("set RUN_QUALITY_E2E=1 to run the real-backend quality evaluation harness") } cfg, err := config.Load("../../.env") if err != nil { t.Fatalf("load config: %v", err) } // 关键:测试进程 cwd 为 internal/chat,而 DATA_DIR=./data/dev 为相对路径—— // 必须换算为仓库根绝对路径,否则工作区写入 internal/chat/data/... 与 // pi-agent runner 的容器挂载映射(仓库根 data/dev/workspaces ↔ /app/data/workspaces) // 错位,导致容器内 context/template ENOENT 及治理文件落盘错位。 if !filepath.IsAbs(cfg.DataDir) { repoRoot, _ := filepath.Abs(filepath.Join("..", "..")) cfg.DataDir = filepath.Join(repoRoot, cfg.DataDir) } svc := bizdev.MustNewService(t, cfg) dialogueBackend := agent.NewBackend("pi-agent", cfg) engine := coordinator.NewCoordinatorEngineWithDialogue(svc, dialogueBackend, cfg.DataDir) rec := intent.NewRecognizer() reportRoot := qualityReportDir(t) ctx := context.Background() // ========================================================================= // 场景 A:路由升级全链路(同题对比 = 升级前单轮初答 vs 升级后小队交付物) // ========================================================================= scenarioADir := filepath.Join(reportRoot, "01-router-upgrade") _ = os.MkdirAll(scenarioADir, 0o755) userCtx := &coordinator.UserProjectContext{ User: coordinator.UserInfo{ID: "1", Name: "张工", Role: "engineer", ProjectRole: "composer"}, Project: coordinator.ProjectInfo{ Name: "掌上无线超声系统", CustomerName: "华西医院微创神经外科", DeviceType: "超声软组织手术吸引设备", IntendedUse: "脑组织空化乳化与同轴吸引", TargetMarket: "中国 NMPA 三类", CurrentStage: bizdev.StageV01_Triage, }, BlackboardFacts: map[string]string{}, } session := chat.NewSession(fmt.Sprintf("quality_e2e_%d", time.Now().Unix()), userCtx, rec, engine) t.Log(">>> [A-1] 发送复杂自由任务请求,期望:即时初答 + 升级建议挂载") t1 := time.Now() resp1, err := session.SendMessage(ctx, qualityComplexInput) if err != nil { t.Fatalf("A-1 send failed: %v", err) } t.Logf("A-1 意图=%s 分支=%s 耗时=%v", resp1.IntentResult.TopIntent.Intent, resp1.Branch, time.Since(t1)) if resp1.PendingExpectation == nil || resp1.PendingExpectation.TargetAction != "launch_adhoc_task" { t.Fatalf("A-1 expected upgrade expectation attached, got %+v", resp1.PendingExpectation) } header1 := fmt.Sprintf("# 场景A-1 对话助手即时初答(升级前的用户所得)\n\n- 请求: %s\n- 意图: %s | 分支: %s\n- 耗时: %v\n- 挂载升级期望: %s\n\n---\n\n", qualityComplexInput, resp1.IntentResult.TopIntent.Intent, resp1.Branch, time.Since(t1), resp1.PendingExpectation.TargetAction) saveQualityFile(t, scenarioADir, "dialogue_first_answer.md", header1+resp1.ReplyText) a1Dur := time.Since(t1) t.Log(">>> [A-2] 用户回复「好」→ 特设小队真实执行(多步,预计 5-20 分钟)") t2 := time.Now() resp2, err := session.SendMessage(ctx, "好") if err != nil { t.Fatalf("A-2 send failed: %v", err) } adhocDur := time.Since(t2) t.Logf("A-2 意图=%s 耗时=%v 回复摘要: %s", resp2.IntentResult.TopIntent.Intent, adhocDur, firstLine(resp2.ReplyText)) if resp2.IntentResult.TopIntent.Intent != intent.IntentTriggerAdhocTask { t.Fatalf("A-2 expected TRIGGER_ADHOC_TASK, got %s (reply: %s)", resp2.IntentResult.TopIntent.Intent, resp2.ReplyText) } if resp2.CoordinationResult == nil || resp2.CoordinationResult.Status != coordinator.StatusAccepted { t.Fatalf("A-2 expected ACCEPTED, reply: %s", resp2.ReplyText) } if !strings.Contains(resp2.ReplyText, "已为您组建特设小队") { t.Fatalf("A-2 unexpected reply (task may have failed): %s", resp2.ReplyText) } header2 := fmt.Sprintf("# 场景A-2 特设小队自由任务交付(升级后的用户所得)\n\n- 意图: TRIGGER_ADHOC_TASK | 耗时: %v\n- 工作区: %s\n\n---\n\n%s", adhocDur, userCtx.WorkDir, resp2.ReplyText) saveQualityFile(t, scenarioADir, "adhoc_task_reply.md", header2) copyQualityWorkspace(t, userCtx.WorkDir, filepath.Join(scenarioADir, "workspace")) t.Logf("A-2 工作区已存档: %s", userCtx.WorkDir) // ========================================================================= // 场景 B:固定 workflow 标准任务(research 调研报告)治理运行 // ========================================================================= scenarioBDir := filepath.Join(reportRoot, "02-standard-research") _ = os.MkdirAll(scenarioBDir, 0o755) t.Log(">>> [B] 标准任务 research 真实执行(多步,预计 5-20 分钟)") t3 := time.Now() resB, err := svc.ExecuteTask(ctx, bizdev.TaskRunRequest{ TaskKey: "research", ProjectName: "掌上无线超声系统", CustomerName: "华西医院微创神经外科", DeviceType: "超声软组织手术吸引设备", IntendedUse: "脑组织空化乳化与同轴吸引", TargetMarket: "中国 NMPA 三类", Fields: map[string]string{"device_type": "超声软组织手术吸引设备", "target_market": "中国 NMPA 三类"}, AutoRepair: true, UserID: 1, ProjectID: 1, }) stdDur := time.Since(t3) if err != nil { t.Fatalf("B execute failed: %v (reason: %s)", err, func() string { if resB != nil { return resB.ErrorReason } return "" }()) } if !resB.Success { t.Fatalf("B task failed: %s", resB.ErrorReason) } t.Logf("B research 完成: 耗时=%v 工作区=%s 质量分=%.0f", stdDur, resB.WorkDir, qualityScore(resB)) saveQualityFile(t, scenarioBDir, "task_summary.md", fmt.Sprintf("# 场景B 标准任务 research 治理运行\n\n- 耗时: %v\n- 工作区: %s\n- 后端: %s\n- 质量门禁: %s\n", stdDur, resB.WorkDir, resB.BackendUsed, qualityVerdict(resB))) copyQualityWorkspace(t, resB.WorkDir, filepath.Join(scenarioBDir, "workspace")) // ========================================================================= // 汇总报告(供人工审阅与质量对比) // ========================================================================= firstAnswer, _ := os.ReadFile(filepath.Join(scenarioADir, "dialogue_first_answer.md")) adhocOut, _ := os.ReadFile(filepath.Join(scenarioADir, "workspace", "output.md")) var researchOut []byte if b, err := os.ReadFile(filepath.Join(scenarioBDir, "workspace", "output.md")); err == nil { researchOut = b } govA := coordinator.GovernanceSummaryForReport(userCtx.WorkDir) govB := coordinator.GovernanceSummaryForReport(resB.WorkDir) report := fmt.Sprintf(`# 长任务治理 × 路由层 真实质量评估报告 - **执行时间**: %s - **后端**: pi-agent (MiniMax-M3, 真实执行) - **设计规范**: .doc/archive/design/superpowers/specs/2026-09-09-longtask-governance-router-and-adhoc-design.md ## 场景 A:路由升级全链路(同题对比) | 指标 | A-1 对话助手即时初答(升级前用户所得) | A-2 特设小队交付(升级后用户所得) | |---|---|---| | 请求 | %s | (同一请求) | | 执行通道 | DIALOGUE 单轮 | SQUAD_ADHOC 多步(动态规划 4 阶段) | | 耗时 | %v | %v | | 产出形态 | 对话文本 | 受控文档 output.md + 质量门禁 + 归档锁定 | | 产出长度 | %d 字符 | %d 字符(不含表格与子任务中间产物) | | 治理审计 | — | %s | **人工审阅要点**: 1. dialogue_first_answer.md vs workspace/output.md:深度、结构化、多视角覆盖、引用可追溯性的差异; 2. workspace/trace.jsonl:每步执行轨迹与产物相似度;workspace/ledger.json:预算用量; 3. workspace/tasks/ 各子目录的 step_manifest.json:步骤级归档锁定指纹。 ## 场景 B:标准任务 research(固定 workflow + 治理运行) - 耗时 %v | 质量门禁 %s | 治理审计: %s - 交付物: %s(%d 字符) ## 治理功能清单(本次实装) - StepFSM 状态机收口(pending/running/verifying/repairing/completed/failed/aborted/skipped + 迁移守卫) - BudgetLedger 多级预算(步骤 12 / 修复轮 2 / agent 调用 16,软线 80%% 提醒、硬线 100%% 强停) - Trace 执行轨迹 + LoopDetector 重复识别(原地打转 / 收敛停滞 / 振荡;先提示后终止) - StepManifest 归档锁定(完成即冻结,SHA-256 指纹,显式重开带审计) - 质量修复闭环治理(修复预算预扣 + 跨轮停滞检测 + 反重复指令注入) - TaskRouter 路由层(确定性画像 + 升级建议复用挂起期望机制 + 每会话冷却) - TRIGGER_ADHOC_TASK 自由任务通道(对话 → 特设小队全链路) `, time.Now().Format("2006-01-02 15:04:05"), qualityComplexInput, a1Dur, adhocDur, len(firstAnswer), len(adhocOut), govA, stdDur, qualityVerdict(resB), govB, filepath.Join(scenarioBDir, "workspace", "output.md"), len(researchOut), ) saveQualityFile(t, reportRoot, "REPORT.md", report) t.Logf(">>> 质量评估完成,报告: %s", filepath.Join(reportRoot, "REPORT.md")) } // --- 小工具 --- func firstLine(s string) string { if idx := strings.Index(s, "\n"); idx > 0 { return s[:idx] } if len(s) > 80 { return s[:80] } return s } func qualityScore(res *bizdev.TaskRunResult) float64 { if res == nil || res.QualityReport == nil { return -1 } return res.QualityReport.Score } func qualityVerdict(res *bizdev.TaskRunResult) string { if res == nil || res.QualityReport == nil { return "无报告" } if res.QualityReport.Passed { return fmt.Sprintf("✅ 通过(评分 %.0f)", res.QualityReport.Score) } return fmt.Sprintf("⚠️ 未完全通过(评分 %.0f,违规: %s)", res.QualityReport.Score, strings.Join(res.QualityReport.Violations, "; ")) }