weichenandCursor
be9a6d4640
Move docs/superpowers to archive/design/superpowers with stubs; archive workflow rules to PROJECT SSOT; separate test-report assets. Update Go doc path references. Co-authored-by: Cursor <cursoragent@cursor.com>
294 lines
12 KiB
Go
294 lines
12 KiB
Go
package chat_test
|
||
|
||
// 长任务治理 × 路由层 真实后端质量评估 harness。
|
||
// 运行方式（内部测试目录为 cwd）：
|
||
//
|
||
// RUN_QUALITY_E2E=1 go test ./internal/chat/ -run TestLongTaskGovernance_QualityE2E -v -timeout 90m
|
||
//
|
||
// 产出存档至 .doc/test-reports/2026-09-09-longtask-governance-quality/ 供人工审阅对比：
|
||
// 01-router-upgrade/ 对话初答 vs 小队交付物（同题质量对比）
|
||
// 02-standard-research/ 固定 workflow 标准任务治理运行
|
||
// REPORT.md 指标汇总
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"chat-agent-system/internal/agent"
|
||
"chat-agent-system/internal/bizdev"
|
||
"chat-agent-system/internal/chat"
|
||
"chat-agent-system/internal/coordinator"
|
||
"chat-agent-system/internal/config"
|
||
"chat-agent-system/internal/intent"
|
||
)
|
||
|
||
const qualityComplexInput = "帮我做一份掌上超声与进口竞品（如 GE Vscan、Philips Lumify）的横向对标分析报告，覆盖法规注册路径、工程架构与商务定价三个视角，要详尽成体系，并给出 V0.5 阶段的切入建议"
|
||
|
||
func qualityReportDir(t *testing.T) string {
|
||
dir := os.Getenv("QUALITY_REPORT_DIR")
|
||
if dir == "" {
|
||
dir = "../../.doc/test-reports/2026-09-09-longtask-governance-quality"
|
||
}
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
t.Fatalf("mkdir report dir: %v", err)
|
||
}
|
||
return dir
|
||
}
|
||
|
||
func saveQualityFile(t *testing.T, dir, name, content string) {
|
||
t.Helper()
|
||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil {
|
||
t.Fatalf("write %s: %v", name, err)
|
||
}
|
||
}
|
||
|
||
func copyQualityWorkspace(t *testing.T, srcWorkDir, dstDir string) {
|
||
t.Helper()
|
||
if srcWorkDir == "" {
|
||
return
|
||
}
|
||
if err := os.MkdirAll(dstDir, 0o755); err != nil {
|
||
t.Fatalf("mkdir workspace archive: %v", err)
|
||
}
|
||
for _, name := range []string{
|
||
"output.md", "workflow.json", "blackboard.json", "ledger.json",
|
||
"trace.jsonl", "gov_audit.jsonl", "manifest.json", "workspace.json", "advisor_brief.json",
|
||
} {
|
||
if b, err := os.ReadFile(filepath.Join(srcWorkDir, name)); err == nil {
|
||
_ = os.WriteFile(filepath.Join(dstDir, name), b, 0o644)
|
||
}
|
||
}
|
||
// 各步骤 step_manifest 与 output
|
||
tasksDir := filepath.Join(srcWorkDir, "tasks")
|
||
if entries, err := os.ReadDir(tasksDir); err == nil {
|
||
for _, e := range entries {
|
||
if !e.IsDir() {
|
||
continue
|
||
}
|
||
subDst := filepath.Join(dstDir, "tasks", e.Name())
|
||
_ = os.MkdirAll(subDst, 0o755)
|
||
for _, f := range []string{"output.md", "step_manifest.json", "TASK.md"} {
|
||
if b, err := os.ReadFile(filepath.Join(tasksDir, e.Name(), f)); err == nil {
|
||
_ = os.WriteFile(filepath.Join(subDst, f), b, 0o644)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestLongTaskGovernance_QualityE2E(t *testing.T) {
|
||
if os.Getenv("RUN_QUALITY_E2E") == "" {
|
||
t.Skip("set RUN_QUALITY_E2E=1 to run the real-backend quality evaluation harness")
|
||
}
|
||
|
||
cfg, err := config.Load("../../.env")
|
||
if err != nil {
|
||
t.Fatalf("load config: %v", err)
|
||
}
|
||
// 关键：测试进程 cwd 为 internal/chat，而 DATA_DIR=./data/dev 为相对路径——
|
||
// 必须换算为仓库根绝对路径，否则工作区写入 internal/chat/data/... 与
|
||
// pi-agent runner 的容器挂载映射（仓库根 data/dev/workspaces ↔ /app/data/workspaces）
|
||
// 错位，导致容器内 context/template ENOENT 及治理文件落盘错位。
|
||
if !filepath.IsAbs(cfg.DataDir) {
|
||
repoRoot, _ := filepath.Abs(filepath.Join("..", ".."))
|
||
cfg.DataDir = filepath.Join(repoRoot, cfg.DataDir)
|
||
}
|
||
svc := bizdev.MustNewService(t, cfg)
|
||
dialogueBackend := agent.NewBackend("pi-agent", cfg)
|
||
engine := coordinator.NewCoordinatorEngineWithDialogue(svc, dialogueBackend, cfg.DataDir)
|
||
rec := intent.NewRecognizer()
|
||
reportRoot := qualityReportDir(t)
|
||
ctx := context.Background()
|
||
|
||
// =========================================================================
|
||
// 场景 A：路由升级全链路（同题对比 = 升级前单轮初答 vs 升级后小队交付物）
|
||
// =========================================================================
|
||
scenarioADir := filepath.Join(reportRoot, "01-router-upgrade")
|
||
_ = os.MkdirAll(scenarioADir, 0o755)
|
||
|
||
userCtx := &coordinator.UserProjectContext{
|
||
User: coordinator.UserInfo{ID: "1", Name: "张工", Role: "engineer", ProjectRole: "composer"},
|
||
Project: coordinator.ProjectInfo{
|
||
Name: "掌上无线超声系统",
|
||
CustomerName: "华西医院微创神经外科",
|
||
DeviceType: "超声软组织手术吸引设备",
|
||
IntendedUse: "脑组织空化乳化与同轴吸引",
|
||
TargetMarket: "中国 NMPA 三类",
|
||
CurrentStage: bizdev.StageV01_Triage,
|
||
},
|
||
BlackboardFacts: map[string]string{},
|
||
}
|
||
session := chat.NewSession(fmt.Sprintf("quality_e2e_%d", time.Now().Unix()), userCtx, rec, engine)
|
||
|
||
t.Log(">>> [A-1] 发送复杂自由任务请求，期望：即时初答 + 升级建议挂载")
|
||
t1 := time.Now()
|
||
resp1, err := session.SendMessage(ctx, qualityComplexInput)
|
||
if err != nil {
|
||
t.Fatalf("A-1 send failed: %v", err)
|
||
}
|
||
t.Logf("A-1 意图=%s 分支=%s 耗时=%v", resp1.IntentResult.TopIntent.Intent, resp1.Branch, time.Since(t1))
|
||
if resp1.PendingExpectation == nil || resp1.PendingExpectation.TargetAction != "launch_adhoc_task" {
|
||
t.Fatalf("A-1 expected upgrade expectation attached, got %+v", resp1.PendingExpectation)
|
||
}
|
||
|
||
header1 := fmt.Sprintf("# 场景A-1 对话助手即时初答（升级前的用户所得）\n\n- 请求: %s\n- 意图: %s | 分支: %s\n- 耗时: %v\n- 挂载升级期望: %s\n\n---\n\n",
|
||
qualityComplexInput, resp1.IntentResult.TopIntent.Intent, resp1.Branch, time.Since(t1), resp1.PendingExpectation.TargetAction)
|
||
saveQualityFile(t, scenarioADir, "dialogue_first_answer.md", header1+resp1.ReplyText)
|
||
a1Dur := time.Since(t1)
|
||
|
||
t.Log(">>> [A-2] 用户回复「好」→ 特设小队真实执行（多步，预计 5-20 分钟）")
|
||
t2 := time.Now()
|
||
resp2, err := session.SendMessage(ctx, "好")
|
||
if err != nil {
|
||
t.Fatalf("A-2 send failed: %v", err)
|
||
}
|
||
adhocDur := time.Since(t2)
|
||
t.Logf("A-2 意图=%s 耗时=%v 回复摘要: %s", resp2.IntentResult.TopIntent.Intent, adhocDur, firstLine(resp2.ReplyText))
|
||
|
||
if resp2.IntentResult.TopIntent.Intent != intent.IntentTriggerAdhocTask {
|
||
t.Fatalf("A-2 expected TRIGGER_ADHOC_TASK, got %s (reply: %s)", resp2.IntentResult.TopIntent.Intent, resp2.ReplyText)
|
||
}
|
||
if resp2.CoordinationResult == nil || resp2.CoordinationResult.Status != coordinator.StatusAccepted {
|
||
t.Fatalf("A-2 expected ACCEPTED, reply: %s", resp2.ReplyText)
|
||
}
|
||
if !strings.Contains(resp2.ReplyText, "已为您组建特设小队") {
|
||
t.Fatalf("A-2 unexpected reply (task may have failed): %s", resp2.ReplyText)
|
||
}
|
||
|
||
header2 := fmt.Sprintf("# 场景A-2 特设小队自由任务交付（升级后的用户所得）\n\n- 意图: TRIGGER_ADHOC_TASK | 耗时: %v\n- 工作区: %s\n\n---\n\n%s",
|
||
adhocDur, userCtx.WorkDir, resp2.ReplyText)
|
||
saveQualityFile(t, scenarioADir, "adhoc_task_reply.md", header2)
|
||
copyQualityWorkspace(t, userCtx.WorkDir, filepath.Join(scenarioADir, "workspace"))
|
||
t.Logf("A-2 工作区已存档: %s", userCtx.WorkDir)
|
||
|
||
// =========================================================================
|
||
// 场景 B：固定 workflow 标准任务（research 调研报告）治理运行
|
||
// =========================================================================
|
||
scenarioBDir := filepath.Join(reportRoot, "02-standard-research")
|
||
_ = os.MkdirAll(scenarioBDir, 0o755)
|
||
|
||
t.Log(">>> [B] 标准任务 research 真实执行（多步，预计 5-20 分钟）")
|
||
t3 := time.Now()
|
||
resB, err := svc.ExecuteTask(ctx, bizdev.TaskRunRequest{
|
||
TaskKey: "research",
|
||
ProjectName: "掌上无线超声系统",
|
||
CustomerName: "华西医院微创神经外科",
|
||
DeviceType: "超声软组织手术吸引设备",
|
||
IntendedUse: "脑组织空化乳化与同轴吸引",
|
||
TargetMarket: "中国 NMPA 三类",
|
||
Fields: map[string]string{"device_type": "超声软组织手术吸引设备", "target_market": "中国 NMPA 三类"},
|
||
AutoRepair: true,
|
||
UserID: 1,
|
||
ProjectID: 1,
|
||
})
|
||
stdDur := time.Since(t3)
|
||
if err != nil {
|
||
t.Fatalf("B execute failed: %v (reason: %s)", err, func() string {
|
||
if resB != nil {
|
||
return resB.ErrorReason
|
||
}
|
||
return ""
|
||
}())
|
||
}
|
||
if !resB.Success {
|
||
t.Fatalf("B task failed: %s", resB.ErrorReason)
|
||
}
|
||
t.Logf("B research 完成: 耗时=%v 工作区=%s 质量分=%.0f", stdDur, resB.WorkDir, qualityScore(resB))
|
||
|
||
saveQualityFile(t, scenarioBDir, "task_summary.md",
|
||
fmt.Sprintf("# 场景B 标准任务 research 治理运行\n\n- 耗时: %v\n- 工作区: %s\n- 后端: %s\n- 质量门禁: %s\n",
|
||
stdDur, resB.WorkDir, resB.BackendUsed, qualityVerdict(resB)))
|
||
copyQualityWorkspace(t, resB.WorkDir, filepath.Join(scenarioBDir, "workspace"))
|
||
|
||
// =========================================================================
|
||
// 汇总报告（供人工审阅与质量对比）
|
||
// =========================================================================
|
||
firstAnswer, _ := os.ReadFile(filepath.Join(scenarioADir, "dialogue_first_answer.md"))
|
||
adhocOut, _ := os.ReadFile(filepath.Join(scenarioADir, "workspace", "output.md"))
|
||
var researchOut []byte
|
||
if b, err := os.ReadFile(filepath.Join(scenarioBDir, "workspace", "output.md")); err == nil {
|
||
researchOut = b
|
||
}
|
||
govA := coordinator.GovernanceSummaryForReport(userCtx.WorkDir)
|
||
govB := coordinator.GovernanceSummaryForReport(resB.WorkDir)
|
||
|
||
report := fmt.Sprintf(`# 长任务治理 × 路由层 真实质量评估报告
|
||
|
||
- **执行时间**: %s
|
||
- **后端**: pi-agent (MiniMax-M3, 真实执行)
|
||
- **设计规范**: .doc/archive/design/superpowers/specs/2026-09-09-longtask-governance-router-and-adhoc-design.md
|
||
|
||
## 场景 A：路由升级全链路（同题对比）
|
||
|
||
| 指标 | A-1 对话助手即时初答（升级前用户所得） | A-2 特设小队交付（升级后用户所得） |
|
||
|---|---|---|
|
||
| 请求 | %s | （同一请求） |
|
||
| 执行通道 | DIALOGUE 单轮 | SQUAD_ADHOC 多步（动态规划 4 阶段） |
|
||
| 耗时 | %v | %v |
|
||
| 产出形态 | 对话文本 | 受控文档 output.md + 质量门禁 + 归档锁定 |
|
||
| 产出长度 | %d 字符 | %d 字符（不含表格与子任务中间产物） |
|
||
| 治理审计 | — | %s |
|
||
|
||
**人工审阅要点**：
|
||
1. dialogue_first_answer.md vs workspace/output.md：深度、结构化、多视角覆盖、引用可追溯性的差异；
|
||
2. workspace/trace.jsonl：每步执行轨迹与产物相似度；workspace/ledger.json：预算用量；
|
||
3. workspace/tasks/ 各子目录的 step_manifest.json：步骤级归档锁定指纹。
|
||
|
||
## 场景 B：标准任务 research（固定 workflow + 治理运行）
|
||
|
||
- 耗时 %v | 质量门禁 %s | 治理审计: %s
|
||
- 交付物: %s（%d 字符）
|
||
|
||
## 治理功能清单（本次实装）
|
||
|
||
- StepFSM 状态机收口（pending/running/verifying/repairing/completed/failed/aborted/skipped + 迁移守卫）
|
||
- BudgetLedger 多级预算（步骤 12 / 修复轮 2 / agent 调用 16，软线 80%% 提醒、硬线 100%% 强停）
|
||
- Trace 执行轨迹 + LoopDetector 重复识别（原地打转 / 收敛停滞 / 振荡；先提示后终止）
|
||
- StepManifest 归档锁定（完成即冻结，SHA-256 指纹，显式重开带审计）
|
||
- 质量修复闭环治理（修复预算预扣 + 跨轮停滞检测 + 反重复指令注入）
|
||
- TaskRouter 路由层（确定性画像 + 升级建议复用挂起期望机制 + 每会话冷却）
|
||
- TRIGGER_ADHOC_TASK 自由任务通道（对话 → 特设小队全链路）
|
||
`,
|
||
time.Now().Format("2006-01-02 15:04:05"),
|
||
qualityComplexInput, a1Dur, adhocDur, len(firstAnswer), len(adhocOut), govA,
|
||
stdDur, qualityVerdict(resB), govB, filepath.Join(scenarioBDir, "workspace", "output.md"), len(researchOut),
|
||
)
|
||
saveQualityFile(t, reportRoot, "REPORT.md", report)
|
||
t.Logf(">>> 质量评估完成，报告: %s", filepath.Join(reportRoot, "REPORT.md"))
|
||
}
|
||
|
||
// --- 小工具 ---
|
||
|
||
func firstLine(s string) string {
|
||
if idx := strings.Index(s, "\n"); idx > 0 {
|
||
return s[:idx]
|
||
}
|
||
if len(s) > 80 {
|
||
return s[:80]
|
||
}
|
||
return s
|
||
}
|
||
|
||
func qualityScore(res *bizdev.TaskRunResult) float64 {
|
||
if res == nil || res.QualityReport == nil {
|
||
return -1
|
||
}
|
||
return res.QualityReport.Score
|
||
}
|
||
|
||
func qualityVerdict(res *bizdev.TaskRunResult) string {
|
||
if res == nil || res.QualityReport == nil {
|
||
return "无报告"
|
||
}
|
||
if res.QualityReport.Passed {
|
||
return fmt.Sprintf("✅ 通过（评分 %.0f）", res.QualityReport.Score)
|
||
}
|
||
return fmt.Sprintf("⚠️ 未完全通过（评分 %.0f，违规: %s）", res.QualityReport.Score, strings.Join(res.QualityReport.Violations, "; "))
|
||
}
|
||
|