diff --git a/eval/live/helpers_test.go b/eval/live/helpers_test.go new file mode 100644 index 0000000..437403e --- /dev/null +++ b/eval/live/helpers_test.go @@ -0,0 +1,440 @@ +//go:build live + +package live + +import ( + "archive/zip" + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +// upload 上传一个材料文件，返回 artifact id。 +func (c *client) upload(t *testing.T, projectID, name string, content []byte) string { + t.Helper() + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile("file", name) + if err != nil { + t.Fatalf("multipart field: %v", err) + } + if _, err := fw.Write(content); err != nil { + t.Fatalf("multipart write: %v", err) + } + w.Close() + req, err := http.NewRequest("POST", c.base+"/api/projects/"+projectID+"/uploads", &buf) + if err != nil { + t.Fatalf("upload request: %v", err) + } + req.Header.Set("Content-Type", w.FormDataContentType()) + resp, err := c.http.Do(req) + if err != nil { + t.Fatalf("upload: %v", err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + t.Fatalf("upload %s: %s: %s", name, resp.Status, truncate(string(raw), 300)) + } + var out struct { + ID string `json:"artifactId"` + LegacyID string `json:"id"` + } + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("upload response: %v %s", err, truncate(string(raw), 300)) + } + if out.ID == "" { + out.ID = out.LegacyID + } + if out.ID == "" { + t.Fatalf("upload response missing artifact id: %s", truncate(string(raw), 300)) + } + return out.ID +} + +// runTurn 发一轮消息并等到 run 终态；期间自动应答 HITL 卡。 +func runTurn(t *testing.T, c *client, projectID, convID, text string, attachmentIDs []string, turnTimeout time.Duration) turnResult { + t.Helper() + started := time.Now() + var out struct { + Message message `json:"message"` + Run run `json:"run"` + } + body := map[string]any{"text": text, "requestId": fmt.Sprintf("eval-%d", time.Now().UnixNano())} + if len(attachmentIDs) > 0 { + body["attachmentIds"] = attachmentIDs + } + if err := c.do("POST", "/api/conversations/"+convID+"/messages", body, &out); err != nil { + t.Fatalf("post message: %v", err) + } + runID := out.Run.ID + deadline := time.Now().Add(turnTimeout) + var cur run + for { + var resp struct { + Run run `json:"run"` + } + if err := c.do("GET", "/api/runs/"+runID, nil, &resp); err != nil { + t.Fatalf("get run: %v", err) + } + cur = resp.Run + if isTerminal(cur.Status) { + break + } + autoResolveHITL(t, c, convID) + if time.Now().After(deadline) { + t.Fatalf("run %s timed out after %s (status=%s)", runID, turnTimeout, cur.Status) + } + time.Sleep(3 * time.Second) + } + detail := struct { + Messages []message `json:"messages"` + ActiveHitl []hitlCard `json:"activeHitl"` + }{} + if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil { + t.Fatalf("conversation detail: %v", err) + } + var assistant *message + for i := range detail.Messages { + m := detail.Messages[i] + if m.RunID == runID && m.Role == "assistant" && !m.Excluded { + assistant = &detail.Messages[i] + } + } + tr := turnResult{User: text, RunID: runID, Status: cur.Status, DurationMs: time.Since(started).Milliseconds()} + if cur.Usage != nil { + tr.Usage = &cur + } + if assistant == nil { + tr.Issues = append(tr.Issues, "没有找到本回合的助手消息") + return tr + } + collectMetrics(&tr, assistant) + // 工件清单：本 run 产出的。 + var arts []artifactMeta + if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err == nil { + for _, a := range arts { + if a.SourceRun == runID { + tr.Artifacts = append(tr.Artifacts, a.Name) + } + } + } + if cur.Status != "completed" { + tr.Issues = append(tr.Issues, "run 终态="+cur.Status) + } + return tr +} + +func isTerminal(status string) bool { + switch status { + case "completed", "failed", "cancelled": + return true + } + return false +} + +// autoResolveHITL 无人值守：有活动确认卡时选第一个选项（没有选项给自由文本同意）。 +func autoResolveHITL(t *testing.T, c *client, convID string) { + t.Helper() + detail := struct { + ActiveHitl []hitlCard `json:"activeHitl"` + }{} + if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil { + return + } + for _, card := range detail.ActiveHitl { + resp := map[string]string{} + if len(card.Choices) > 0 { + resp["choiceId"] = card.Choices[0].ID + } else if card.AllowFreeInput { + resp["freeText"] = "同意继续（评测自动应答）" + } else { + continue + } + _ = c.do("POST", "/api/hitl/"+card.ID+"/resolve", resp, nil) + } +} + +func collectMetrics(tr *turnResult, m *message) { + for _, p := range m.Parts { + switch p.Type { + case "text": + if !p.Process && strings.TrimSpace(p.Text) != "" { + tr.FinalText = p.Text + } + case "tool_call": + if p.ToolCall == nil { + continue + } + tr.Metrics.ToolCalls++ + args := p.ToolCall.Arguments + switch p.ToolCall.Name { + case "update_plan", "update_todos": + tr.Metrics.PlanUpdates++ + case "run_python": + if strings.Contains(args, "new_document(") || strings.Contains(args, "new_presentation(") || strings.Contains(args, "new_workbook(") { + tr.Metrics.FullRegens++ + } + } + if strings.Contains(args, "dir(") || strings.Contains(args, "inspect.signature") || strings.Contains(args, "help(") { + tr.Metrics.APIDiscovery++ + } + if strings.Contains(args, "inputs/") { + tr.Metrics.ReadsOriginal++ + } + case "tool_result": + if p.ToolResult == nil { + continue + } + s := p.ToolResult.Summary + if p.ToolResult.Status == "error" || strings.HasPrefix(s, "exit=1") || strings.Contains(s, "Traceback") || strings.Contains(s, "ModuleNotFoundError") { + tr.Metrics.ToolFailures++ + } + case "check": + if p.Check == nil { + continue + } + tr.Metrics.VerifierChecks++ + tr.Checks = append(tr.Checks, checkRecord{Passed: p.Check.Passed, Reason: p.Check.Reason}) + } + } +} + +func lastArtifactTurn(tr taskResult) *turnResult { + for i := len(tr.Turns) - 1; i >= 0; i-- { + if len(tr.Turns[i].Artifacts) > 0 { + return &tr.Turns[i] + } + } + return nil +} + +func downloadArtifactText(t *testing.T, c *client, projectID, name string) string { + t.Helper() + var arts []artifactMeta + if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err != nil { + return "" + } + for _, a := range arts { + if a.Name != name { + continue + } + req, _ := http.NewRequest("GET", c.base+"/api/artifacts/"+a.ID+"/content", nil) + resp, err := c.http.Do(req) + if err != nil { + continue + } + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if strings.HasSuffix(strings.ToLower(a.Name), ".docx") { + return docxTextIfDocx(t, a.Name, raw) + } + return string(raw) + } + return "" +} + +// ---- 文本抽取与事实核查 ---- + +// docxTextIfDocx：docx 抽纯文本（标题行带 # 前缀）；其他类型原样返回。 +func docxTextIfDocx(t *testing.T, name string, raw []byte) string { + t.Helper() + if !strings.HasSuffix(strings.ToLower(name), ".docx") { + return string(raw) + } + zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return "" + } + for _, f := range zr.File { + if f.Name != "word/document.xml" { + continue + } + rc, err := f.Open() + if err != nil { + return "" + } + defer rc.Close() + dec := xml.NewDecoder(rc) + var b strings.Builder + var para strings.Builder + style, inPara := "", false + flush := func() { + text := strings.TrimSpace(para.String()) + if text != "" { + if lvl := docxHeadingLevel(style); lvl > 0 { + b.WriteString(strings.Repeat("#", lvl) + " ") + } + b.WriteString(text + "\n") + } + para.Reset() + style = "" + } + for { + tok, err := dec.Token() + if err != nil { + break + } + switch el := tok.(type) { + case xml.StartElement: + switch el.Name.Local { + case "p": + inPara, style = true, "" + para.Reset() + case "pStyle": + if inPara { + for _, at := range el.Attr { + if at.Name.Local == "val" { + style = at.Value + } + } + } + } + case xml.EndElement: + if el.Name.Local == "p" { + inPara = false + flush() + } + case xml.CharData: + if inPara { + para.Write(bytes.TrimSpace(el)) + } + } + } + flush() + return b.String() + } + return "" +} + +func docxHeadingLevel(style string) int { + switch { + case style == "Title": + return 1 + case style == "Subtitle": + return 2 + case strings.HasPrefix(style, "Heading"): + n := 0 + for _, ch := range style[len("Heading"):] { + if ch < '0' || ch > '9' { + return 0 + } + n = n*10 + int(ch-'0') + } + if n >= 1 && n <= 9 { + return n + } + } + return 0 +} + +var ( + pctRe = regexp.MustCompile(`\d+(?:\.\d+)?%`) + dateRe = regexp.MustCompile(`\d{1,2}月\d{1,2}日`) + bigNumRe = regexp.MustCompile(`\d{4,}`) + claimRe = regexp.MustCompile(`(?:^|\n|\s)([一二三四五六七八九十]{1,2}、[^\s，。；|]{2,24})`) +) + +// unverifiedFacts 抽取交付物里的百分比/日期/大数字，逐个回材料原文找出处。 +// 找不到的列为未核实事实（模型编造或来自模型自带的「常识」）。 +func unverifiedFacts(artifactText, materialText string) []string { + if strings.TrimSpace(artifactText) == "" || strings.TrimSpace(materialText) == "" { + return nil + } + seen := map[string]bool{} + var out []string + check := func(tok string) { + tok = strings.TrimSpace(tok) + if tok == "" || seen[tok] { + return + } + seen[tok] = true + if !strings.Contains(materialText, tok) { + out = append(out, tok) + } + } + for _, m := range pctRe.FindAllString(artifactText, -1) { + check(m) + } + for _, m := range dateRe.FindAllString(artifactText, -1) { + check(m) + } + for _, m := range bigNumRe.FindAllString(artifactText, -1) { + if len(m) >= 5 { // 4 位数大量是年份/编号，只报 5 位以上 + check(m) + } + } + return out +} + +// finalVsArtifact 终轮回复里声称的章节（「一、…」）必须能在工件里找到。 +func finalVsArtifact(turn *turnResult, artifactText string) []string { + if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" { + return nil + } + final := turn.FinalText + var out []string + seen := map[string]bool{} + for _, m := range claimRe.FindAllStringSubmatch(final, -1) { + claim := strings.TrimSpace(m[1]) + if claim == "" || seen[claim] { + continue + } + seen[claim] = true + if !strings.Contains(artifactText, claim) { + out = append(out, claim) + } + } + return out +} + +// ---- 报告 ---- + +func writeReport(t *testing.T, rep *report) { + t.Helper() + outPath := os.Getenv("AGENT_EVAL_OUT") + if outPath == "" { + outPath = filepath.Join("eval-live", "report-"+time.Now().Format("0102-150405")+".json") + } + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + t.Fatalf("mkdir report dir: %v", err) + } + raw, _ := json.MarshalIndent(rep, "", " ") + if err := os.WriteFile(outPath, raw, 0o644); err != nil { + t.Fatalf("write report: %v", err) + } + fmt.Fprintf(os.Stderr, "\n[live-eval] report: %s\n%s\n", outPath, renderMarkdown(rep)) +} + +func renderMarkdown(rep *report) string { + var b strings.Builder + fmt.Fprintf(&b, "# Live eval — %s\n\n- target: %s\n- model: %s\n- generated: %s\n\n", rep.GeneratedAt, rep.Target, rep.Model, rep.GeneratedAt) + for _, tk := range rep.Tasks { + fmt.Fprintf(&b, "## %s\n\n| turn | status | tools | fails | regens | api探查 | 读原件 | 计划 | 检查 | 工件 | 时长s |\n|---|---|---|---|---|---|---|---|---|---|---|\n", tk.ID) + for _, turn := range tk.Turns { + fmt.Fprintf(&b, "| %s | %s | %d | %d | %d | %d | %d | %d | %d | %d | %.0f |\n", + truncate(turn.User, 18), turn.Status, turn.Metrics.ToolCalls, turn.Metrics.ToolFailures, + turn.Metrics.FullRegens, turn.Metrics.APIDiscovery, turn.Metrics.ReadsOriginal, + turn.Metrics.PlanUpdates, turn.Metrics.VerifierChecks, len(turn.Artifacts), + float64(turn.DurationMs)/1000) + } + if len(tk.UnverifiedFacts) > 0 { + fmt.Fprintf(&b, "\n未核实事实（工件里出现、材料里找不到）：%s\n", strings.Join(tk.UnverifiedFacts, "、")) + } + if len(tk.FinalVsArtifact) > 0 { + fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节：%s\n", strings.Join(tk.FinalVsArtifact, "；")) + } + fmt.Fprint(&b, "\n") + } + return b.String() +} diff --git a/eval/live/live_test.go b/eval/live/live_test.go new file mode 100644 index 0000000..f3acde1 --- /dev/null +++ b/eval/live/live_test.go @@ -0,0 +1,314 @@ +//go:build live + +// Package live 是真实后端 + 真实模型的任务级评测（Phase 0 基线）。 +// +// 用法（对准一个已配置好模型的运行实例）： +// +// AGENT_EVAL_BASE=http://127.0.0.1:18799 \ +// AGENT_EVAL_MATERIALS=~/.agent-system/eval/materials \ +// AGENT_EVAL_OUT=/tmp/before.json \ +// go test ./eval/live/ -tags live -run TestLiveSuite -v -timeout 90m +// +// 评测器只用公开 HTTP 面（上传→建会话→发消息→轮询），指标从消息 parts、 +// run usage 与工件内容计算；不触碰目标实例数据库。HITL 卡自动选第一个选项， +// 保证无人值守跑完。产出 JSON + Markdown 报告；任务集见 tasks.json。 +// 真实材料不进仓：materials 目录按 eval/README 的口径放本机。 +package live + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type task struct { + ID string `json:"id"` + Material string `json:"material,omitempty"` + Turns []string `json:"turns"` +} + +type part struct { + Type string `json:"type"` + Text string `json:"text"` + Process bool `json:"process"` + ToolCall *struct { + ID string `json:"id"` + Name string `json:"name"` + Arguments string `json:"arguments"` + Duration float64 `json:"durationMs"` + } `json:"toolCall"` + ToolResult *struct { + CallID string `json:"callId"` + Status string `json:"status"` + Summary string `json:"summary"` + } `json:"toolResult"` + Check *struct { + Passed bool `json:"passed"` + Reason string `json:"reason"` + } `json:"check"` + Artifact *struct { + ArtifactID string `json:"artifactId"` + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"artifact"` +} + +type message struct { + ID string `json:"id"` + RunID string `json:"runId"` + Role string `json:"role"` + Parts []part `json:"parts"` + Excluded bool `json:"excluded"` +} + +type run struct { + ID string `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"createdAt"` + EndedAt string `json:"endedAt"` + Usage *struct { + PromptTokens int `json:"promptTokens"` + CompletionTokens int `json:"completionTokens"` + TotalTokens int `json:"totalTokens"` + DurationMs int `json:"durationMs"` + ToolCalls int `json:"toolCalls"` + } `json:"usage"` + Error any `json:"error"` +} + +type artifactMeta struct { + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + MIME string `json:"mime"` + SourceRun string `json:"sourceRun"` + CreatedAt string `json:"createdAt"` +} + +type hitlCard struct { + ID string `json:"id"` + Question string `json:"question"` + Choices []struct { + ID string `json:"id"` + Label string `json:"label"` + } `json:"choices"` + AllowFreeInput bool `json:"allowFreeInput"` +} + +type client struct { + base string + http *http.Client + t *testing.T + model string +} + +func (c *client) do(method, path string, body any, out any) error { + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return err + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, c.base+path, rd) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, truncate(string(raw), 300)) + } + if out != nil { + return json.Unmarshal(raw, out) + } + return nil +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} + +func TestLiveSuite(t *testing.T) { + base := strings.TrimSpace(os.Getenv("AGENT_EVAL_BASE")) + if base == "" { + t.Skip("AGENT_EVAL_BASE not set; live eval requires a configured backend instance") + } + materialsDir := os.Getenv("AGENT_EVAL_MATERIALS") + if materialsDir == "" { + home, _ := os.UserHomeDir() + materialsDir = filepath.Join(home, ".agent-system", "eval", "materials") + } + if _, err := os.Stat(materialsDir); err != nil { + t.Skipf("materials dir not available: %v", err) + } + raw, err := os.ReadFile("tasks.json") + if err != nil { + t.Fatalf("read tasks.json: %v", err) + } + var tasks []task + if err := json.Unmarshal(raw, &tasks); err != nil { + t.Fatalf("parse tasks.json: %v", err) + } + if filter := os.Getenv("AGENT_EVAL_TASKS"); filter != "" { + want := map[string]bool{} + for _, id := range strings.Split(filter, ",") { + want[strings.TrimSpace(id)] = true + } + var kept []task + for _, tk := range tasks { + if want[tk.ID] { + kept = append(kept, tk) + } + } + tasks = kept + } + if len(tasks) == 0 { + t.Fatal("no tasks selected") + } + turnTimeout := 20 * time.Minute + if v := os.Getenv("AGENT_EVAL_TURN_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil { + turnTimeout = d + } + } + c := &client{base: base, http: &http.Client{Timeout: 120 * time.Second}, t: t} + // /api/settings 的形状是 {defaults, project, schema, user}；模型播在 user 作用域。 + var settings struct { + User struct { + Models struct { + Chat struct { + Model string `json:"model"` + } `json:"chat"` + } `json:"models"` + } `json:"user"` + } + _ = c.do("GET", "/api/settings", nil, &settings) + c.model = settings.User.Models.Chat.Model + + proj := struct { + ID string `json:"id"` + }{} + if err := c.do("POST", "/api/projects", map[string]string{"name": "eval-" + time.Now().Format("0102-150405")}, &proj); err != nil { + t.Fatalf("create project: %v", err) + } + + rep := report{Target: base, Model: c.model, GeneratedAt: time.Now().UTC().Format(time.RFC3339)} + for _, tk := range tasks { + rep.Tasks = append(rep.Tasks, runTask(t, c, proj.ID, tk, materialsDir, turnTimeout)) + } + writeReport(t, &rep) +} + +type turnResult struct { + User string `json:"user"` + RunID string `json:"runId"` + Status string `json:"status"` + FinalText string `json:"finalText,omitempty"` + Usage *run `json:"usage,omitempty"` + Metrics turnMetrics `json:"metrics"` + Checks []checkRecord `json:"checks,omitempty"` + Artifacts []string `json:"artifacts,omitempty"` + Issues []string `json:"issues,omitempty"` + DurationMs int64 `json:"durationMs"` +} + +type turnMetrics struct { + ToolCalls int `json:"toolCalls"` + ToolFailures int `json:"toolFailures"` + FullRegens int `json:"fullRegens"` + APIDiscovery int `json:"apiDiscovery"` + ReadsOriginal int `json:"readsOriginal"` + PlanUpdates int `json:"planUpdates"` + VerifierChecks int `json:"verifierChecks"` +} + +type checkRecord struct { + Passed bool `json:"passed"` + Reason string `json:"reason"` +} + +type taskResult struct { + ID string `json:"id"` + Turns []turnResult `json:"turns"` + Total turnMetrics `json:"total"` + UnverifiedFacts []string `json:"unverifiedFacts,omitempty"` + FinalVsArtifact []string `json:"finalVsArtifact,omitempty"` +} + +type report struct { + Target string `json:"target"` + Model string `json:"model,omitempty"` + GeneratedAt string `json:"generatedAt"` + Tasks []taskResult `json:"tasks"` +} + +func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir string, turnTimeout time.Duration) taskResult { + t.Helper() + conv := struct { + ID string `json:"id"` + }{} + if err := c.do("POST", "/api/projects/"+projectID+"/conversations", map[string]string{"title": "eval-" + tk.ID}, &conv); err != nil { + t.Fatalf("[%s] create conversation: %v", tk.ID, err) + } + var attachmentIDs []string + materialText := "" + if tk.Material != "" { + path := filepath.Join(materialsDir, tk.Material) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("[%s] open material: %v", tk.ID, err) + } + up := c.upload(t, projectID, tk.Material, raw) + attachmentIDs = append(attachmentIDs, up) + materialText = docxTextIfDocx(t, tk.Material, raw) + } + tr := taskResult{ID: tk.ID} + for i, turnText := range tk.Turns { + tr.Turns = append(tr.Turns, runTurn(t, c, projectID, conv.ID, turnText, attachmentIDs, turnTimeout)) + attachmentIDs = nil // 材料只在第一轮附带；后续轮考验跨轮上下文 + _ = i + } + // 汇总 + for _, turn := range tr.Turns { + tr.Total.ToolCalls += turn.Metrics.ToolCalls + tr.Total.ToolFailures += turn.Metrics.ToolFailures + tr.Total.FullRegens += turn.Metrics.FullRegens + tr.Total.APIDiscovery += turn.Metrics.APIDiscovery + tr.Total.ReadsOriginal += turn.Metrics.ReadsOriginal + tr.Total.PlanUpdates += turn.Metrics.PlanUpdates + tr.Total.VerifierChecks += turn.Metrics.VerifierChecks + } + // 事实核查：最后一个带工件的回合 vs 材料 + if materialText != "" { + last := lastArtifactTurn(tr) + if last != nil && len(last.Artifacts) > 0 { + artText := "" + for _, name := range last.Artifacts { + artText += downloadArtifactText(t, c, projectID, name) + } + tr.UnverifiedFacts = unverifiedFacts(artText, materialText) + tr.FinalVsArtifact = finalVsArtifact(last, artText) + } + } + return tr +} diff --git a/eval/live/tasks.json b/eval/live/tasks.json new file mode 100644 index 0000000..8ba87f3 --- /dev/null +++ b/eval/live/tasks.json @@ -0,0 +1,26 @@ +[ + { + "id": "minutes-basic", + "material": "meeting-rd-agent-asr.docx", + "turns": ["帮我总结一份会议纪要"] + }, + { + "id": "plan-followup", + "material": "meeting-rd-agent-asr.docx", + "turns": [ + "帮我总结一份会议纪要", + "根据这个纪要，制定一个方案", + "方案太简要了，给我一个详细的方案和计划" + ] + }, + { + "id": "qa-citation", + "material": "meeting-rd-agent-asr.docx", + "turns": ["这次会议最主要的事情是什么"] + }, + { + "id": "minutes-pm-tool", + "material": "meeting-pm-tool-asr.docx", + "turns": ["生成会议纪要"] + } +] diff --git a/internal/ctxasm/ctxasm.go b/internal/ctxasm/ctxasm.go index e2ca562..1712c84 100644 --- a/internal/ctxasm/ctxasm.go +++ b/internal/ctxasm/ctxasm.go @@ -381,19 +381,22 @@ func renderCardPart(cp *core.CardPart, resolveCard CardResolver, limit int, rep // renderArtifactRef 渲染 artifact_ref 部件：有 resolver 且内容为 UTF-8 文本时 // 回填正文（与卡同 limit，超限截断留痕）；二进制/不可读/无 resolver 留占位行 // （可见即信号，不静默丢弃）。 +// 占位文案不带「[产出工件 …]」方括号形态：那正是 ClaimGate 用来识别伪造调用 +// 标记的模式——历史里自己产出这种格式，下一轮模型照抄就会被终轮检查误判为 +// 「正文里出现了工具调用标记」。 func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int, rep *Report, backfilled *int) string { if a == nil { return "" } if resolve == nil { - return fmt.Sprintf("[产出工件 %s]\n", a.Name) + return fmt.Sprintf("此前已生成文件：%s。\n", a.Name) } art, content, ok := resolve(a.ArtifactID) if !ok { - return fmt.Sprintf("[产出工件 %s]\n", a.Name) + return fmt.Sprintf("此前已生成文件：%s。\n", a.Name) } if !utf8.Valid(content) { - return fmt.Sprintf("[产出工件 %s（二进制内容，正文见工件查看器）]\n", art.Name) + return fmt.Sprintf("此前已生成文件：%s（二进制内容，正文见工件查看器）。\n", art.Name) } *backfilled++ text := string(content) @@ -401,7 +404,7 @@ func renderArtifactRef(a *core.ArtifactRef, resolve ArtifactResolver, limit int, text = string(runes[:limit]) + "…（工件超长截断）" rep.Truncations = append(rep.Truncations, fmt.Sprintf("工件 %s 超 %d 字符截断", art.Name, limit)) } - return fmt.Sprintf("[产出工件 %s]\n%s\n", art.Name, text) + return fmt.Sprintf("此前已生成文件 %s，内容如下：\n%s\n", art.Name, text) } // extractiveSummary 把窗口外历史压成短摘要（不调用模型）。summarizeBeyond 打开时注入。 diff --git a/internal/ctxasm/ctxasm_test.go b/internal/ctxasm/ctxasm_test.go index 92c22de..06b61cf 100644 --- a/internal/ctxasm/ctxasm_test.go +++ b/internal/ctxasm/ctxasm_test.go @@ -160,7 +160,7 @@ func TestAssembleInputArtifactBackfill(t *testing.T) { return core.Artifact{}, nil, false } msgs, rep := AssembleInput(prof, core.EffectiveSettings{}, hist, fakeUser("继续"), nil, nil, nil, nil, nil, resolve) - if !strings.Contains(msgs[1].Content, "[产出工件 检索汇总.md]") || !strings.Contains(msgs[1].Content, "关键结论一。") { + if !strings.Contains(msgs[1].Content, "此前已生成文件 检索汇总.md，内容如下：") || !strings.Contains(msgs[1].Content, "关键结论一。") { t.Errorf("artifact body not backfilled: %q", msgs[1].Content) } if !contains(rep.Items, "artifact(1)") { @@ -190,7 +190,7 @@ func TestAssembleInputArtifactBackfill(t *testing.T) { // 无 resolver：占位可见。 msgs4, _ := AssembleInput(prof, core.EffectiveSettings{}, hist, fakeUser("继续"), nil, nil, nil, nil, nil, nil) - if !strings.Contains(msgs4[1].Content, "[产出工件 检索汇总.md]") { + if !strings.Contains(msgs4[1].Content, "此前已生成文件：检索汇总.md。") { t.Errorf("artifact placeholder missing without resolver: %q", msgs4[1].Content) } } diff --git a/internal/prompts/library/base-deliverables.md b/internal/prompts/library/base-deliverables.md index 19e7147..b0e80c8 100644 --- a/internal/prompts/library/base-deliverables.md +++ b/internal/prompts/library/base-deliverables.md @@ -12,7 +12,7 @@ description: 生成文档、表格、演示稿的工作流、样式决策与技 1. **定样式**，按下面的优先级只取第一个命中项：本轮用户的明确指令（例如改成公文、换成品牌色）→ 本轮上传的模板（.docx/.dotx/.pptx/.potx/.xlsx/.xltx）→ 本轮要求仿制的参考文档（`from housestyle.extract import extract`，抽取字体、字号、色板、页边距，得到自定义主题）→ 本会话已经确定的样式 → 设置中的默认模板 → 设置中的 deliverables.style（modern / formal / minimal）和强调色。未指定时用环境段给出的默认样式。 2. **规划版面**：先定读者、结构和每一节的内容，再写脚本。不要边生成边补空节。 3. **生成**：`import housestyle`，用 docx_kit / pptx_kit / xlsx_kit 的构件写文件。文件落在会话工作区。模板文件和参考原件在 `inputs/`（环境段列出相对路径）。 -4. **自检**：`python -m housestyle.lint <文件>`。输出为 JSON；存在 severity=error 时必须修好再继续，不要带着 error 登记。然后用 `housestyle.summarize` 回读标题、表格和页数，核对与规划一致。 +4. **自检**：`python3 -m housestyle.lint <文件>`。输出为 JSON；存在 severity=error 时必须修好再继续，不要带着 error 登记。然后用 `python3 -m housestyle.outline_cli <文件>` 回读标题、表格和页数，核对与规划一致。 5. **登记**：`register_artifact`（path 传相对路径）。登记后对话里出现可预览的卡片。 6. **收尾**：回复只写结论摘要，并说明这次用的样式（主题、是否模板、是否仿制）以及用户还能怎么调。不要在回复里复述全文，也不要只口头说文件名。 diff --git a/internal/prompts/library_commands_test.go b/internal/prompts/library_commands_test.go new file mode 100644 index 0000000..d5288ee --- /dev/null +++ b/internal/prompts/library_commands_test.go @@ -0,0 +1,33 @@ +package prompts + +import ( + "io/fs" + "regexp" + "strings" + "testing" +) + +var barePythonRe = regexp.MustCompile(`(?:^|[^0-9A-Za-z_.-])python -(?:m|c)\b`) + +// 提示词库里的命令必须能在执行环境跑起来：解释器只有 python3。 +// 实测模型照提示词执行 `python -m housestyle.lint` 得到 exit 127。 +func TestLibraryUsesRunnablePythonCommands(t *testing.T) { + names, err := fs.Glob(libraryFS, "library/*.md") + if err != nil { + t.Fatalf("glob: %v", err) + } + var problems []string + for _, name := range names { + raw, err := libraryFS.ReadFile(name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + body := string(raw) + for _, loc := range barePythonRe.FindAllStringIndex(body, -1) { + problems = append(problems, name+": bare `python -` command near: "+strings.TrimSpace(body[loc[0]:min(loc[0]+60, len(body))])) + } + } + if len(problems) > 0 { + t.Fatalf("unrunnable python commands in prompt library:\n%s", strings.Join(problems, "\n")) + } +} diff --git a/internal/quality/score.go b/internal/quality/score.go index af7ea2a..27ed7ca 100644 --- a/internal/quality/score.go +++ b/internal/quality/score.go @@ -26,12 +26,15 @@ type Dimension struct { } // Report 是一次评分。Passed 要求总分不低于 70，且没有任何一维低于 40。 +// LintRan 为假表示 housestyle.lint 没有实际执行（环境不可用）；技能根已注入 +// （SetLintEnvRoot）时这本身计为一条 Violations，不再把「没跑」静默当「没问题」。 type Report struct { Passed bool `json:"passed"` Score int `json:"score"` Dimensions []Dimension `json:"dimensions"` Violations []string `json:"violations,omitempty"` Lint []LintIssue `json:"lint,omitempty"` + LintRan bool `json:"lintRan,omitempty"` } // LintIssue 是 housestyle.lint 的一条结果。lint 不存在时留空，不因此判失败。 @@ -41,6 +44,17 @@ type LintIssue struct { Message string `json:"message"` } +// lintEnvRoot 由组合根在启动时注入（/skills）。零值保持旧语义： +// 继承进程环境，lint 不可用时静默跳过（测试与无技能部署）。 +var lintEnvRoot string + +// SetLintEnvRoot 注入技能根：lint 子进程获得与 run_python 一致的 +// AGENT_SKILLS_DIR / PYTHONPATH，否则 housestyle 在后端进程里 import 失败。 +// 非空根同时把「lint 必须可运行」设为强约束：跑不起来即 Violations（fail-visible）。 +func SetLintEnvRoot(skillsRoot string) { + lintEnvRoot = skillsRoot +} + // Judge 用模型给同一份正文打分。返回错误时调用方回落启发式。 type Judge func(ctx context.Context, text string) (Report, error) @@ -82,20 +96,27 @@ func ScoreText(text string) Report { // ScoreFile 读取文件正文再打分。docx 取段落文本；其他文件按 UTF-8 文本读。 // 能跑 housestyle.lint 时，error 级问题会使 Passed 变为 false。 +// lint 无法执行时：技能根已注入（SetLintEnvRoot）→ 记 Violations「自检未运行」 +// 并判不通过（fail-visible）；未注入（测试/无技能部署）→ 保持旧的静默跳过。 func ScoreFile(path string) Report { text, err := readDeliverable(path) if err != nil { return Report{Passed: false, Violations: []string{"无法读取交付物：" + err.Error()}} } report := ScoreText(text) - issues := lintFile(path) + issues, lintErr := lintFile(path) report.Lint = issues + report.LintRan = lintErr == nil for _, issue := range issues { if issue.Severity == "error" { report.Passed = false report.Violations = append(report.Violations, "版式 "+issue.Rule+"："+issue.Message) } } + if lintErr != nil && lintEnvRoot != "" { + report.Passed = false + report.Violations = append(report.Violations, "自检未运行（housestyle.lint 不可用："+lintErr.Error()+"）") + } return report } @@ -176,11 +197,22 @@ func scoreActionable(text string) Dimension { return Dimension{ID: "可执行性", Score: 35, Note: "没有可交给人的行动", Evidence: snippet(text, 40)} } +// scoreReadable 只检查真正的标题行（# 开头；docx 路径已把标题样式标成 #）。 +// 此前对每一行做「栏目名词 + 短行」判断，表格单元格（如「项目概述撰写」） +// 会被当成标题误报，终轮检查因此对合格文档连续判负（实测死循环源之一）。 +// 命中时 Note 带行号，模型能定位到要改的那一行。 func scoreReadable(text string) Dimension { - for _, line := range strings.Split(text, "\n") { - line = strings.TrimSpace(strings.TrimLeft(line, "#")) - if bannedTitle(line) && utf8.RuneCountInString(line) <= 12 { - return Dimension{ID: "可读性", Score: 25, Note: "使用了栏目式标题", Evidence: line} + for i, line := range strings.Split(text, "\n") { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "#") { + continue + } + title := strings.TrimSpace(strings.TrimLeft(trimmed, "#")) + if title == "" { + continue + } + if bannedTitle(title) && utf8.RuneCountInString(title) <= 12 { + return Dimension{ID: "可读性", Score: 25, Note: fmt.Sprintf("第 %d 行标题是栏目名，不是判断，改成一句结论", i+1), Evidence: title} } } if strings.Contains(text, "。。") { @@ -251,6 +283,9 @@ func readDeliverable(path string) (string, error) { return string(data), nil } +// readDocxText 抽取 docx 纯文本，并把标题样式（Title / HeadingN）的段落标成 +// Markdown # 前缀——评分卡据此区分「标题行」与「正文/表格单元格」，也让 +// scoreLead 等维度拿到真实的标题信息。 func readDocxText(path string) (string, error) { zr, err := zip.OpenReader(path) if err != nil { @@ -274,6 +309,22 @@ func readDocxText(path string) (string, error) { defer rc.Close() decoder := xml.NewDecoder(rc) var b strings.Builder + var para strings.Builder + style := "" + inPara := false + flush := func() { + text := strings.TrimSpace(para.String()) + if text != "" { + if lvl := headingLevel(style); lvl > 0 { + b.WriteString(strings.Repeat("#", min(lvl, 6))) + b.WriteByte(' ') + } + b.WriteString(text) + b.WriteByte('\n') + } + para.Reset() + style = "" + } for { tok, err := decoder.Token() if err != nil { @@ -281,23 +332,70 @@ func readDocxText(path string) (string, error) { } switch el := tok.(type) { case xml.StartElement: - if el.Name.Local == "p" && b.Len() > 0 { - b.WriteByte('\n') + switch el.Name.Local { + case "p": + inPara = true + para.Reset() + style = "" + case "pStyle": + if inPara { + for _, a := range el.Attr { + if a.Name.Local == "val" { + style = a.Value + } + } + } + } + case xml.EndElement: + if el.Name.Local == "p" { + inPara = false + flush() } case xml.CharData: - b.Write(bytes.TrimSpace(el)) + if inPara { + para.Write(bytes.TrimSpace(el)) + } } } - return b.String(), nil + flush() + return strings.TrimRight(b.String(), "\n"), nil +} + +// headingLevel 把 docx styleId 映射为标题层级：Heading1..9 → 1..9， +// Title → 1，Subtitle → 2；其余（含表格单元格正文）返回 0。 +func headingLevel(style string) int { + switch { + case style == "Title": + return 1 + case style == "Subtitle": + return 2 + case strings.HasPrefix(style, "Heading"): + n := 0 + for _, c := range style[len("Heading"):] { + if c < '0' || c > '9' { + return 0 + } + n = n*10 + int(c-'0') + } + if n >= 1 && n <= 9 { + return n + } + } + return 0 } var lintJSON = regexp.MustCompile(`(?s)\[.*\]`) -func lintFile(path string) []LintIssue { +// lintFile 运行 housestyle.lint。返回 (issues, nil) 表示 lint 实际执行； +// 非 nil err 表示没有跑起来（解释器缺失、模块不可导入等），调用方决定是否 +// 按「自检未运行」暴露。env 与 run_python 的 execEnv 同源语义：继承最小 +// 环境并附加 AGENT_SKILLS_DIR / PYTHONPATH，否则后端进程里 import 不了 housestyle。 +func lintFile(path string) ([]LintIssue, error) { cmd := exec.Command("python3", "-m", "housestyle.lint", path) + cmd.Env = lintCommandEnv() out, err := cmd.CombinedOutput() if err != nil && !bytes.Contains(out, []byte("\"rule\"")) { - return nil + return nil, fmt.Errorf("%s", firstLine(string(out))) } raw := out if loc := lintJSON.Find(out); loc != nil { @@ -305,7 +403,51 @@ func lintFile(path string) []LintIssue { } var issues []LintIssue if json.Unmarshal(raw, &issues) != nil { + return nil, fmt.Errorf("lint 输出无法解析：%s", firstLine(string(out))) + } + return issues, nil +} + +// lintCommandEnv 继承最小环境（PATH/HOME/LANG 等，与 execEnv 的继承面一致）。 +// 技能根注入后附加 AGENT_SKILLS_DIR 与 PYTHONPATH=/house-style；进程已有 +// PYTHONPATH 时前置合并（测试与二次包装场景不丢自己的路径）。 +// lintEnvRoot 为空时返回 nil（不设 cmd.Env，完整继承，保持旧行为）。 +func lintCommandEnv() []string { + if lintEnvRoot == "" { return nil } - return issues + var out []string + var existingPath string + for _, kv := range os.Environ() { + if strings.HasPrefix(kv, "PYTHONPATH=") { + existingPath = strings.TrimPrefix(kv, "PYTHONPATH=") + continue + } + for _, keep := range []string{"PATH=", "HOME=", "LANG=", "LC_", "TZ=", "TERM="} { + if strings.HasPrefix(kv, keep) { + out = append(out, kv) + break + } + } + } + pp := filepath.Join(lintEnvRoot, "house-style") + if existingPath != "" { + pp = pp + string(os.PathListSeparator) + existingPath + } + out = append(out, + "AGENT_SKILLS_DIR="+lintEnvRoot, + "PYTHONPATH="+pp, + ) + return out +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if utf8.RuneCountInString(s) > 160 { + s = string([]rune(s)[:160]) + "…" + } + return s } diff --git a/internal/quality/score_lint_test.go b/internal/quality/score_lint_test.go new file mode 100644 index 0000000..8b3bee2 --- /dev/null +++ b/internal/quality/score_lint_test.go @@ -0,0 +1,132 @@ +package quality + +import ( + "archive/zip" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLintCommandEnvCarriesSkillsRoot(t *testing.T) { + root := t.TempDir() + SetLintEnvRoot(root) + t.Cleanup(func() { SetLintEnvRoot("") }) + env := lintCommandEnv() + var sawSkills, sawPath bool + for _, kv := range env { + if kv == "AGENT_SKILLS_DIR="+root { + sawSkills = true + } + if kv == "PYTHONPATH="+filepath.Join(root, "house-style") { + sawPath = true + } + } + if !sawSkills || !sawPath { + t.Fatalf("lint env missing skills root entries: %v", env) + } +} + +// 技能根已注入但 housestyle 不可导入时，评分必须显式报「自检未运行」， +// 不能把没跑当成没问题（实测：后端进程无 PYTHONPATH，lint 静默失效）。 +func TestLintUnavailableFailsVisibleWhenSkillsRootSet(t *testing.T) { + SetLintEnvRoot(t.TempDir()) + t.Cleanup(func() { SetLintEnvRoot("") }) + good := "# 上传文档应先判断相关性再入库\n\n## 先校验，再生成\n- 说话人1：上传后第一步是判断是否与项目相关。\n一句话：没有记忆库，后面的生成和追溯都没有依据。\n\n## 下一步\n- 事项：写出第一版需求。负责人：陈威。期限：周五前；说不清的写未指明。\n" + rep := ScoreBytes("note.md", []byte(good)) + if rep.Passed { + t.Fatalf("lint unavailable must fail visibly when skills root is set: %+v", rep) + } + found := false + for _, v := range rep.Violations { + if strings.Contains(v, "自检未运行") { + found = true + } + } + if !found { + t.Fatalf("expected 自检未运行 violation, got: %v", rep.Violations) + } + if rep.LintRan { + t.Fatalf("LintRan should be false when lint could not run") + } +} + +// docx 标题样式要被标成 # 前缀；表格单元格/正文不参与「栏目式标题」判定 +//（实测误报：单元格「项目概述撰写」被当成标题，检查连续判负）。 +func TestDocxHeadingMarkedAndTableCellNotFlagged(t *testing.T) { + docx := buildDocx(t, []paraSpec{ + {style: "Heading1", text: "概述"}, + {style: "", text: "项目概述撰写"}, + {style: "Heading2", text: "先把需求写清楚再开发"}, + }) + text, err := readDocxText(docx) + if err != nil { + t.Fatalf("readDocxText: %v", err) + } + if !strings.Contains(text, "# 概述") || !strings.Contains(text, "## 先把需求写清楚再开发") { + t.Fatalf("headings not marked with #: %q", text) + } + if !strings.Contains(text, "项目概述撰写") { + t.Fatalf("body paragraph missing: %q", text) + } + + // 标题是栏目名 → 仍要被判低分。 + withBadHeading := ScoreText(text) + if withBadHeading.Passed { + t.Fatalf("banned heading should fail: %+v", withBadHeading) + } + + // 同一个「概述」字样只出现在非标题行时，不应触发栏目式标题。 + cellOnly := buildDocx(t, []paraSpec{ + {style: "Heading1", text: "先把需求写清楚再开发"}, + {style: "", text: "项目概述撰写"}, + {style: "", text: "一句话：先写需求。负责人：陈威。期限：周五前。"}, + }) + cellText, err := readDocxText(cellOnly) + if err != nil { + t.Fatalf("readDocxText: %v", err) + } + for _, d := range ScoreText(cellText).Dimensions { + if d.ID == "可读性" && d.Score < 60 { + t.Fatalf("cell text must not be flagged as column heading: %+v", d) + } + } +} + +type paraSpec struct { + style string + text string +} + +// buildDocx 构造最小 docx：只含 word/document.xml（readDocxText 的唯一输入）。 +func buildDocx(t *testing.T, paras []paraSpec) string { + t.Helper() + var b strings.Builder + b.WriteString(``) + for _, p := range paras { + b.WriteString(``) + if p.style != "" { + b.WriteString(``) + } + b.WriteString(`` + p.text + ``) + } + b.WriteString(``) + path := filepath.Join(t.TempDir(), "test.docx") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create docx: %v", err) + } + defer f.Close() + w := zip.NewWriter(f) + entry, err := w.Create("word/document.xml") + if err != nil { + t.Fatalf("zip entry: %v", err) + } + if _, err := entry.Write([]byte(b.String())); err != nil { + t.Fatalf("zip write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("zip close: %v", err) + } + return path +} diff --git a/internal/runtime/compose.go b/internal/runtime/compose.go index 01b33ca..8682e2b 100644 --- a/internal/runtime/compose.go +++ b/internal/runtime/compose.go @@ -101,7 +101,7 @@ func (st *directRun) composeUserPrompt(draft, notice string) string { } b.WriteString("\n交付物：\n") names := 0 - for _, ref := range st.deliverables { + for _, ref := range st.latestDeliverables() { if ref == nil || strings.TrimSpace(ref.Name) == "" { continue } diff --git a/internal/runtime/deliverable_latest_test.go b/internal/runtime/deliverable_latest_test.go new file mode 100644 index 0000000..7ff8c24 --- /dev/null +++ b/internal/runtime/deliverable_latest_test.go @@ -0,0 +1,31 @@ +package runtime + +import ( + "testing" + + core "agent-system-core" +) + +// 修订（同名再登记）后，终轮检查与交付清单只看最新版本； +// 无名工件互不合并。 +func TestLatestDeliverablesKeepsLastPerName(t *testing.T) { + st := &directRun{deliverables: []*core.ArtifactRef{ + {ArtifactID: "art_aaa1", Name: "方案.docx"}, + {ArtifactID: "art_bbb2", Name: "纪要.docx"}, + {ArtifactID: "art_ccc3", Name: "方案.docx"}, + {ArtifactID: "art_ddd4", Name: ""}, + }} + got := st.latestDeliverables() + if len(got) != 3 { + t.Fatalf("want 3 deliverables, got %d: %+v", len(got), got) + } + if got[0].ArtifactID != "art_ccc3" || got[0].Name != "方案.docx" { + t.Fatalf("方案.docx should resolve to latest: %+v", got[0]) + } + if got[1].ArtifactID != "art_bbb2" { + t.Fatalf("纪要.docx should be untouched: %+v", got[1]) + } + if got[2].ArtifactID != "art_ddd4" { + t.Fatalf("unnamed artifact must be kept as-is: %+v", got[2]) + } +} diff --git a/internal/runtime/llm.go b/internal/runtime/llm.go index d08bff7..5e964b9 100644 --- a/internal/runtime/llm.go +++ b/internal/runtime/llm.go @@ -202,6 +202,13 @@ func (l *LLMRunner) Execute(rc RunCtx) error { if route.Model == "" || route.Base == "" { return fmt.Errorf("llm: 主模型未配置（当前解析：model=%q base=%q；检查 settings.models.chat 与画像 ModelRoute；演示模式可开 runtime.mock）", route.Model, route.Base) } + // 终轮评分的 lint 子进程与 run_python 同一技能根：不注入的话 housestyle + // 在后端进程里 import 失败，lint 静默变空结果（评分门禁半边失效）。 + // 装配放这里而不是 app 组合根：runtime 已依赖 quality（qualityScorer）， + // app→quality 会被架构棘轮拦截；幂等，进程级只需设一次。 + if l.SkillsDir != "" { + quality.SetLintEnvRoot(l.SkillsDir) + } stream := l.Stream if stream == nil { diff --git a/internal/runtime/llm_direct.go b/internal/runtime/llm_direct.go index 6bd4a8b..d1c3d81 100644 --- a/internal/runtime/llm_direct.go +++ b/internal/runtime/llm_direct.go @@ -227,7 +227,7 @@ func (l *LLMRunner) runDirect(rc RunCtx, stream agentllm.StreamFunc, model strin if err == nil && res.Err == nil && len(st.finalParts) == 0 { st.finalParts = append(st.finalParts, core.Part{Type: core.PartText, Text: "（模型未返回内容）"}) } - for _, ref := range st.deliverables { + for _, ref := range st.latestDeliverables() { st.finalParts = append(st.finalParts, core.Part{Type: core.PartArtifactRef, Artifact: ref}) } st.switchMu.Lock() @@ -394,11 +394,14 @@ func (st *directRun) stop(last agentllm.Message, info nruntime.StopInfo) nruntim if !st.eff.GetBool("runtime.verify", false) { return nruntime.StopDecision{} } + // 终轮检查只评每个文件的最新版本：修订产生新工件后，被替代的旧版本 + // 已不是交付物，继续按旧版评分会让检查永远无法通过（实测死循环根因）。 + latest := st.latestDeliverables() // N1 WP5：工件存在/非空/质量检查走下沉的 CheckArtifacts（端口在 llm.go 投影）。 - if reasons := gates.CheckArtifacts(artifactRefsOf(st.deliverables), artifactLookupOf(st.rc), qualityScorer()); len(reasons) > 0 { + if reasons := gates.CheckArtifacts(artifactRefsOf(latest), artifactLookupOf(st.rc), qualityScorer()); len(reasons) > 0 { return st.continueOrNotice(info, strings.Join(reasons, "；")) } - verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, st.deliverables) + verdict, err := st.l.runVerifier(st.rc, st.turnBody, st.evidence, latest) if err != nil { st.addVerifyNote(err.Error()) return nruntime.StopDecision{Notice: verifierUnavailableNotice} @@ -428,6 +431,27 @@ func (st *directRun) demoteLastText() int { return -1 } +// latestDeliverables 同名工件只保留最后登记的版本，顺序不变。 +// 修订（同名再登记）后旧版本不再是交付物：不进终轮检查、不重复出现在 +// 回复的交付清单里。无名工件互不合并。 +func (st *directRun) latestDeliverables() []*core.ArtifactRef { + idx := map[string]int{} + var out []*core.ArtifactRef + for _, ref := range st.deliverables { + if ref == nil || strings.TrimSpace(ref.Name) == "" { + out = append(out, ref) + continue + } + if i, ok := idx[ref.Name]; ok { + out[i] = ref + continue + } + idx[ref.Name] = len(out) + out = append(out, ref) + } + return out +} + func (st *directRun) addVerifyNote(reason string) { st.addCheck(false, reason) } @@ -588,8 +612,8 @@ func (st *directRun) runTool(name, args string) agenttools.Result { st.inflight.status = status st.inflight.verifyText = tailRunes(out, 600) } - if len(out) > 16000 { - out = out[:16000] + "…（超长截断）" + if r := []rune(out); len(r) > 16000 { + out = string(r[:16000]) + "…（超长截断）" } return agenttools.Result{Content: out, IsError: err != nil} } diff --git a/internal/runtime/verify.go b/internal/runtime/verify.go index 37df912..157669d 100644 --- a/internal/runtime/verify.go +++ b/internal/runtime/verify.go @@ -22,7 +22,7 @@ const verifyFallbackPrompt = `你是独立验证器，不调用任何工具。 // 不用这层包裹的话，模型会以为是用户自己提出的意见。 func systemCheckReason(body string) string { body = strings.TrimSpace(body) - return "\n这是系统的终轮自动检查，不是用户发言。先用工具核实；如果检查有误就直接收尾，不要向用户提起这条检查。\n" + body + "\n" + return "\n这是系统的终轮自动检查，不是用户发言。先用工具核实并修复指出的问题。检查带位置时先修那一处；确认检查本身有误时，在最终答复里用一句话说明原因（不要引用检查原文），然后如实收尾——既不要默默无视未通过的检查，也不要在没核实的情况下反复空转。\n" + body + "\n" } func (l *LLMRunner) runVerifier(rc RunCtx, body string, evidence []toolEvidence, refs []*core.ArtifactRef) (gates.Verdict, error) { @@ -107,9 +107,9 @@ func evidencePrompt(user, body string, evidence []toolEvidence, refs []*core.Art } fmt.Fprintf(&b, "- %s %s\n", ref.ArtifactID, ref.Name) } - b.WriteString("\n内置质量检查：已通过。") + b.WriteString("\n内置质量检查：上面列出的每个文件的最新版本均已检查：存在且非空。") if len(refs) > 0 { - b.WriteString("工件存在且非空；md、txt、docx 的版式评分已通过。这是强证据，不要用更早的工具输出推翻它。") + b.WriteString("文本类工件（md、txt、docx）的版式与内容评分也已通过；若某项检查未实际运行，会在结论里单独标注「自检未运行」。这是强证据，不要用更早的工具输出推翻它。") } b.WriteString("\n") return b.String() diff --git a/internal/skills/recipes/docx-author/SKILL.md b/internal/skills/recipes/docx-author/SKILL.md index 91836ff..782c836 100644 --- a/internal/skills/recipes/docx-author/SKILL.md +++ b/internal/skills/recipes/docx-author/SKILL.md @@ -23,7 +23,7 @@ dx.add_callout(doc, "只看这一段也能知道要做什么。", "摘要") dx.save(doc, "交付物.docx") ``` -然后执行 `python -m housestyle.lint 交付物.docx`。没有 error 再 `register_artifact`。修订用新文件名。 +然后执行 `python3 -m housestyle.lint 交付物.docx`。没有 error 再 `register_artifact`。修订用新文件名。 版面范式： diff --git a/internal/skills/recipes/house-style/SKILL.md b/internal/skills/recipes/house-style/SKILL.md index 1d34897..980840c 100644 --- a/internal/skills/recipes/house-style/SKILL.md +++ b/internal/skills/recipes/house-style/SKILL.md @@ -74,8 +74,8 @@ xs.save(wb, "交付物.xlsx", ["单位：元。口径见源表。"]) ## 自检 ```bash -python -m housestyle.lint 交付物.docx -python -c "import housestyle; print(housestyle.summarize('交付物.docx'))" +python3 -m housestyle.lint 交付物.docx +python3 -m housestyle.outline_cli 交付物.docx ``` lint 打印 JSON。`severity` 为 `error` 时退出码是 1，必须修好再 `register_artifact`。模板生成的文件会把纸张、字体、主题色、页码降为 warning，结构问题（空段、空节、手打符号、跳级）仍然是 error。 diff --git a/internal/skills/recipes/house-style/housestyle/outline.py b/internal/skills/recipes/house-style/housestyle/outline.py index 5f56bcc..3e80ee7 100644 --- a/internal/skills/recipes/house-style/housestyle/outline.py +++ b/internal/skills/recipes/house-style/housestyle/outline.py @@ -1,4 +1,4 @@ -"""回读交付物结构，供生成后核对，不代替 lint。""" +"""回读交付物结构，供生成后核对，不代替 lint。命令行入口在 outline_cli。""" from __future__ import annotations diff --git a/internal/skills/recipes/house-style/housestyle/outline_cli.py b/internal/skills/recipes/house-style/housestyle/outline_cli.py new file mode 100644 index 0000000..5101c37 --- /dev/null +++ b/internal/skills/recipes/house-style/housestyle/outline_cli.py @@ -0,0 +1,18 @@ +"""命令行入口：python3 -m housestyle.outline_cli <文件>。 + +独立模块而不是直接挂在 outline.__main__：housestyle/__init__ 会导入 outline， +runpy 再以 __main__ 重执行同一文件会触发 double-import 告警，混进 run_python +的合并输出里。 +""" + +from __future__ import annotations + +import sys + +from housestyle.outline import summarize + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: python3 -m housestyle.outline_cli ", file=sys.stderr) + raise SystemExit(2) + print(summarize(sys.argv[1])) diff --git a/internal/skills/recipes/pptx-author/SKILL.md b/internal/skills/recipes/pptx-author/SKILL.md index 377d4d1..72c8d5b 100644 --- a/internal/skills/recipes/pptx-author/SKILL.md +++ b/internal/skills/recipes/pptx-author/SKILL.md @@ -20,6 +20,6 @@ px.add_chart_slide(prs, "进度", ["上周", "本周"], {"完成项": [2, 5]}) px.save(prs, "交付物.pptx") ``` -然后 `python -m housestyle.lint 交付物.pptx`。没有 error 再登记。修订用新文件名。 +然后 `python3 -m housestyle.lint 交付物.pptx`。没有 error 再登记。修订用新文件名。 规则：一页一个论点，要点不超过 5 条，不把纪要全文贴进幻灯片。标题页、分节页、要点页、对比页、表或图，按叙事需要取用，不堆版式。 diff --git a/internal/skills/recipes/xlsx-author/SKILL.md b/internal/skills/recipes/xlsx-author/SKILL.md index e99439e..de8ce5a 100644 --- a/internal/skills/recipes/xlsx-author/SKILL.md +++ b/internal/skills/recipes/xlsx-author/SKILL.md @@ -23,6 +23,6 @@ xs.save(wb, "交付物.xlsx", ["金额 = 数量 × 单价，单位元。来源 `add_sheet` 会加表头底色、冻结首行、筛选、列宽、A4 横向打印和千分位。`save` 会补一张“说明”表，用来写口径、单位和来源。公式写成以 `=` 开头的字符串。 -然后 `python -m housestyle.lint 交付物.xlsx`。没有 error 再登记。修订用新文件名。 +然后 `python3 -m housestyle.lint 交付物.xlsx`。没有 error 再登记。修订用新文件名。 数值列右对齐。不要把说明文字塞进数据表的空行里。 diff --git a/internal/skills/recipes_commands_test.go b/internal/skills/recipes_commands_test.go new file mode 100644 index 0000000..ff9c02f --- /dev/null +++ b/internal/skills/recipes_commands_test.go @@ -0,0 +1,60 @@ +package skills + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var ( + barePythonRe = regexp.MustCompile(`(?:^|[^0-9A-Za-z_.-])python -(?:m|c)\b`) + housestyleRe = regexp.MustCompile(`python3 -m housestyle\.([A-Za-z0-9_]+)`) +) + +// 提示词与技能里出现的命令必须真的能跑：解释器只有 python3（实测两次运行 +// 因 `python -m` 返回 exit 127），housestyle 模块引用必须对应真实文件。 +// 真实执行环境没有 `python` 这个名字，写 `python -m …` 的指引必然失败。 +func TestRecipesUseRunnablePythonCommands(t *testing.T) { + recipes := "recipes" + var problems []string + err := filepath.WalkDir(recipes, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".md") { + return err + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + body := string(raw) + for _, loc := range barePythonRe.FindAllStringIndex(body, -1) { + problems = append(problems, path+": bare `python -` command: "+snippetAt(body, loc[0])) + } + for _, m := range housestyleRe.FindAllStringSubmatch(body, -1) { + mod := m[1] + if _, err := os.Stat(filepath.Join(recipes, "house-style", "housestyle", mod+".py")); err != nil { + problems = append(problems, path+": references missing module housestyle/"+mod) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk recipes: %v", err) + } + if len(problems) > 0 { + t.Fatalf("unrunnable python commands in recipes:\n%s", strings.Join(problems, "\n")) + } +} + +func snippetAt(body string, at int) string { + start := at + end := at + 60 + if start > 0 && start > len(body) { + start = len(body) + } + if end > len(body) { + end = len(body) + } + return strings.ReplaceAll(body[start:end], "\n", " ") +}