From 613fb27ba06761f93c7320c8feaeac99725b3155 Mon Sep 17 00:00:00 2001 From: weichen Date: Fri, 9 Oct 2026 12:49:38 +0000 Subject: [PATCH] Add the live-model task eval harness (phase 0 baseline) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: No task-level measurement of real agent behavior existed, so the failure chain documented in the diagnosis was invisible to CI. The harness drives a configured backend over its public HTTP surface (upload, message, poll, HITL auto-resolve), computes per-turn metrics (tool failures, full regenerations, API-discovery calls, original-material reads, plan usage, gate checks) and outcome checks (unverified facts vs material text, final claims vs artifact headings), and writes a JSON + Markdown report. Evidence: SELF - built behind the live build tag (CI skips); vet passes with -tags live; drove the qa instance (remote main 1ebd46e) through the four golden tasks end to end; report referenced on the phase01 task card. Impact: New eval/live package (tasks.json + runner). Materials stay outside the repo per eval/README; runs are opt-in via AGENT_EVAL_BASE. Refs: Dev task feature/agent-quality-phase01; documents proposal 2026-10-09-agent-quality-improvement-plan.md §4 P0. --- eval/live/helpers_test.go | 440 ++++++++++++++++++++++++++++++++++++++ eval/live/live_test.go | 311 +++++++++++++++++++++++++++ eval/live/tasks.json | 26 +++ 3 files changed, 777 insertions(+) create mode 100644 eval/live/helpers_test.go create mode 100644 eval/live/live_test.go create mode 100644 eval/live/tasks.json diff --git a/eval/live/helpers_test.go b/eval/live/helpers_test.go new file mode 100644 index 0000000..437403e --- /dev/null +++ b/eval/live/helpers_test.go @@ -0,0 +1,440 @@ +//go:build live + +package live + +import ( + "archive/zip" + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +// upload 上传一个材料文件，返回 artifact id。 +func (c *client) upload(t *testing.T, projectID, name string, content []byte) string { + t.Helper() + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile("file", name) + if err != nil { + t.Fatalf("multipart field: %v", err) + } + if _, err := fw.Write(content); err != nil { + t.Fatalf("multipart write: %v", err) + } + w.Close() + req, err := http.NewRequest("POST", c.base+"/api/projects/"+projectID+"/uploads", &buf) + if err != nil { + t.Fatalf("upload request: %v", err) + } + req.Header.Set("Content-Type", w.FormDataContentType()) + resp, err := c.http.Do(req) + if err != nil { + t.Fatalf("upload: %v", err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + t.Fatalf("upload %s: %s: %s", name, resp.Status, truncate(string(raw), 300)) + } + var out struct { + ID string `json:"artifactId"` + LegacyID string `json:"id"` + } + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("upload response: %v %s", err, truncate(string(raw), 300)) + } + if out.ID == "" { + out.ID = out.LegacyID + } + if out.ID == "" { + t.Fatalf("upload response missing artifact id: %s", truncate(string(raw), 300)) + } + return out.ID +} + +// runTurn 发一轮消息并等到 run 终态；期间自动应答 HITL 卡。 +func runTurn(t *testing.T, c *client, projectID, convID, text string, attachmentIDs []string, turnTimeout time.Duration) turnResult { + t.Helper() + started := time.Now() + var out struct { + Message message `json:"message"` + Run run `json:"run"` + } + body := map[string]any{"text": text, "requestId": fmt.Sprintf("eval-%d", time.Now().UnixNano())} + if len(attachmentIDs) > 0 { + body["attachmentIds"] = attachmentIDs + } + if err := c.do("POST", "/api/conversations/"+convID+"/messages", body, &out); err != nil { + t.Fatalf("post message: %v", err) + } + runID := out.Run.ID + deadline := time.Now().Add(turnTimeout) + var cur run + for { + var resp struct { + Run run `json:"run"` + } + if err := c.do("GET", "/api/runs/"+runID, nil, &resp); err != nil { + t.Fatalf("get run: %v", err) + } + cur = resp.Run + if isTerminal(cur.Status) { + break + } + autoResolveHITL(t, c, convID) + if time.Now().After(deadline) { + t.Fatalf("run %s timed out after %s (status=%s)", runID, turnTimeout, cur.Status) + } + time.Sleep(3 * time.Second) + } + detail := struct { + Messages []message `json:"messages"` + ActiveHitl []hitlCard `json:"activeHitl"` + }{} + if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil { + t.Fatalf("conversation detail: %v", err) + } + var assistant *message + for i := range detail.Messages { + m := detail.Messages[i] + if m.RunID == runID && m.Role == "assistant" && !m.Excluded { + assistant = &detail.Messages[i] + } + } + tr := turnResult{User: text, RunID: runID, Status: cur.Status, DurationMs: time.Since(started).Milliseconds()} + if cur.Usage != nil { + tr.Usage = &cur + } + if assistant == nil { + tr.Issues = append(tr.Issues, "没有找到本回合的助手消息") + return tr + } + collectMetrics(&tr, assistant) + // 工件清单：本 run 产出的。 + var arts []artifactMeta + if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err == nil { + for _, a := range arts { + if a.SourceRun == runID { + tr.Artifacts = append(tr.Artifacts, a.Name) + } + } + } + if cur.Status != "completed" { + tr.Issues = append(tr.Issues, "run 终态="+cur.Status) + } + return tr +} + +func isTerminal(status string) bool { + switch status { + case "completed", "failed", "cancelled": + return true + } + return false +} + +// autoResolveHITL 无人值守：有活动确认卡时选第一个选项（没有选项给自由文本同意）。 +func autoResolveHITL(t *testing.T, c *client, convID string) { + t.Helper() + detail := struct { + ActiveHitl []hitlCard `json:"activeHitl"` + }{} + if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil { + return + } + for _, card := range detail.ActiveHitl { + resp := map[string]string{} + if len(card.Choices) > 0 { + resp["choiceId"] = card.Choices[0].ID + } else if card.AllowFreeInput { + resp["freeText"] = "同意继续（评测自动应答）" + } else { + continue + } + _ = c.do("POST", "/api/hitl/"+card.ID+"/resolve", resp, nil) + } +} + +func collectMetrics(tr *turnResult, m *message) { + for _, p := range m.Parts { + switch p.Type { + case "text": + if !p.Process && strings.TrimSpace(p.Text) != "" { + tr.FinalText = p.Text + } + case "tool_call": + if p.ToolCall == nil { + continue + } + tr.Metrics.ToolCalls++ + args := p.ToolCall.Arguments + switch p.ToolCall.Name { + case "update_plan", "update_todos": + tr.Metrics.PlanUpdates++ + case "run_python": + if strings.Contains(args, "new_document(") || strings.Contains(args, "new_presentation(") || strings.Contains(args, "new_workbook(") { + tr.Metrics.FullRegens++ + } + } + if strings.Contains(args, "dir(") || strings.Contains(args, "inspect.signature") || strings.Contains(args, "help(") { + tr.Metrics.APIDiscovery++ + } + if strings.Contains(args, "inputs/") { + tr.Metrics.ReadsOriginal++ + } + case "tool_result": + if p.ToolResult == nil { + continue + } + s := p.ToolResult.Summary + if p.ToolResult.Status == "error" || strings.HasPrefix(s, "exit=1") || strings.Contains(s, "Traceback") || strings.Contains(s, "ModuleNotFoundError") { + tr.Metrics.ToolFailures++ + } + case "check": + if p.Check == nil { + continue + } + tr.Metrics.VerifierChecks++ + tr.Checks = append(tr.Checks, checkRecord{Passed: p.Check.Passed, Reason: p.Check.Reason}) + } + } +} + +func lastArtifactTurn(tr taskResult) *turnResult { + for i := len(tr.Turns) - 1; i >= 0; i-- { + if len(tr.Turns[i].Artifacts) > 0 { + return &tr.Turns[i] + } + } + return nil +} + +func downloadArtifactText(t *testing.T, c *client, projectID, name string) string { + t.Helper() + var arts []artifactMeta + if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err != nil { + return "" + } + for _, a := range arts { + if a.Name != name { + continue + } + req, _ := http.NewRequest("GET", c.base+"/api/artifacts/"+a.ID+"/content", nil) + resp, err := c.http.Do(req) + if err != nil { + continue + } + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if strings.HasSuffix(strings.ToLower(a.Name), ".docx") { + return docxTextIfDocx(t, a.Name, raw) + } + return string(raw) + } + return "" +} + +// ---- 文本抽取与事实核查 ---- + +// docxTextIfDocx：docx 抽纯文本（标题行带 # 前缀）；其他类型原样返回。 +func docxTextIfDocx(t *testing.T, name string, raw []byte) string { + t.Helper() + if !strings.HasSuffix(strings.ToLower(name), ".docx") { + return string(raw) + } + zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + return "" + } + for _, f := range zr.File { + if f.Name != "word/document.xml" { + continue + } + rc, err := f.Open() + if err != nil { + return "" + } + defer rc.Close() + dec := xml.NewDecoder(rc) + var b strings.Builder + var para strings.Builder + style, inPara := "", false + flush := func() { + text := strings.TrimSpace(para.String()) + if text != "" { + if lvl := docxHeadingLevel(style); lvl > 0 { + b.WriteString(strings.Repeat("#", lvl) + " ") + } + b.WriteString(text + "\n") + } + para.Reset() + style = "" + } + for { + tok, err := dec.Token() + if err != nil { + break + } + switch el := tok.(type) { + case xml.StartElement: + switch el.Name.Local { + case "p": + inPara, style = true, "" + para.Reset() + case "pStyle": + if inPara { + for _, at := range el.Attr { + if at.Name.Local == "val" { + style = at.Value + } + } + } + } + case xml.EndElement: + if el.Name.Local == "p" { + inPara = false + flush() + } + case xml.CharData: + if inPara { + para.Write(bytes.TrimSpace(el)) + } + } + } + flush() + return b.String() + } + return "" +} + +func docxHeadingLevel(style string) int { + switch { + case style == "Title": + return 1 + case style == "Subtitle": + return 2 + case strings.HasPrefix(style, "Heading"): + n := 0 + for _, ch := range style[len("Heading"):] { + if ch < '0' || ch > '9' { + return 0 + } + n = n*10 + int(ch-'0') + } + if n >= 1 && n <= 9 { + return n + } + } + return 0 +} + +var ( + pctRe = regexp.MustCompile(`\d+(?:\.\d+)?%`) + dateRe = regexp.MustCompile(`\d{1,2}月\d{1,2}日`) + bigNumRe = regexp.MustCompile(`\d{4,}`) + claimRe = regexp.MustCompile(`(?:^|\n|\s)([一二三四五六七八九十]{1,2}、[^\s，。；|]{2,24})`) +) + +// unverifiedFacts 抽取交付物里的百分比/日期/大数字，逐个回材料原文找出处。 +// 找不到的列为未核实事实（模型编造或来自模型自带的「常识」）。 +func unverifiedFacts(artifactText, materialText string) []string { + if strings.TrimSpace(artifactText) == "" || strings.TrimSpace(materialText) == "" { + return nil + } + seen := map[string]bool{} + var out []string + check := func(tok string) { + tok = strings.TrimSpace(tok) + if tok == "" || seen[tok] { + return + } + seen[tok] = true + if !strings.Contains(materialText, tok) { + out = append(out, tok) + } + } + for _, m := range pctRe.FindAllString(artifactText, -1) { + check(m) + } + for _, m := range dateRe.FindAllString(artifactText, -1) { + check(m) + } + for _, m := range bigNumRe.FindAllString(artifactText, -1) { + if len(m) >= 5 { // 4 位数大量是年份/编号，只报 5 位以上 + check(m) + } + } + return out +} + +// finalVsArtifact 终轮回复里声称的章节（「一、…」）必须能在工件里找到。 +func finalVsArtifact(turn *turnResult, artifactText string) []string { + if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" { + return nil + } + final := turn.FinalText + var out []string + seen := map[string]bool{} + for _, m := range claimRe.FindAllStringSubmatch(final, -1) { + claim := strings.TrimSpace(m[1]) + if claim == "" || seen[claim] { + continue + } + seen[claim] = true + if !strings.Contains(artifactText, claim) { + out = append(out, claim) + } + } + return out +} + +// ---- 报告 ---- + +func writeReport(t *testing.T, rep *report) { + t.Helper() + outPath := os.Getenv("AGENT_EVAL_OUT") + if outPath == "" { + outPath = filepath.Join("eval-live", "report-"+time.Now().Format("0102-150405")+".json") + } + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + t.Fatalf("mkdir report dir: %v", err) + } + raw, _ := json.MarshalIndent(rep, "", " ") + if err := os.WriteFile(outPath, raw, 0o644); err != nil { + t.Fatalf("write report: %v", err) + } + fmt.Fprintf(os.Stderr, "\n[live-eval] report: %s\n%s\n", outPath, renderMarkdown(rep)) +} + +func renderMarkdown(rep *report) string { + var b strings.Builder + fmt.Fprintf(&b, "# Live eval — %s\n\n- target: %s\n- model: %s\n- generated: %s\n\n", rep.GeneratedAt, rep.Target, rep.Model, rep.GeneratedAt) + for _, tk := range rep.Tasks { + fmt.Fprintf(&b, "## %s\n\n| turn | status | tools | fails | regens | api探查 | 读原件 | 计划 | 检查 | 工件 | 时长s |\n|---|---|---|---|---|---|---|---|---|---|---|\n", tk.ID) + for _, turn := range tk.Turns { + fmt.Fprintf(&b, "| %s | %s | %d | %d | %d | %d | %d | %d | %d | %d | %.0f |\n", + truncate(turn.User, 18), turn.Status, turn.Metrics.ToolCalls, turn.Metrics.ToolFailures, + turn.Metrics.FullRegens, turn.Metrics.APIDiscovery, turn.Metrics.ReadsOriginal, + turn.Metrics.PlanUpdates, turn.Metrics.VerifierChecks, len(turn.Artifacts), + float64(turn.DurationMs)/1000) + } + if len(tk.UnverifiedFacts) > 0 { + fmt.Fprintf(&b, "\n未核实事实（工件里出现、材料里找不到）：%s\n", strings.Join(tk.UnverifiedFacts, "、")) + } + if len(tk.FinalVsArtifact) > 0 { + fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节：%s\n", strings.Join(tk.FinalVsArtifact, "；")) + } + fmt.Fprint(&b, "\n") + } + return b.String() +} diff --git a/eval/live/live_test.go b/eval/live/live_test.go new file mode 100644 index 0000000..4928450 --- /dev/null +++ b/eval/live/live_test.go @@ -0,0 +1,311 @@ +//go:build live + +// Package live 是真实后端 + 真实模型的任务级评测（Phase 0 基线）。 +// +// 用法（对准一个已配置好模型的运行实例）： +// +// AGENT_EVAL_BASE=http://127.0.0.1:18799 \ +// AGENT_EVAL_MATERIALS=~/.agent-system/eval/materials \ +// AGENT_EVAL_OUT=/tmp/before.json \ +// go test ./eval/live/ -tags live -run TestLiveSuite -v -timeout 90m +// +// 评测器只用公开 HTTP 面（上传→建会话→发消息→轮询），指标从消息 parts、 +// run usage 与工件内容计算；不触碰目标实例数据库。HITL 卡自动选第一个选项， +// 保证无人值守跑完。产出 JSON + Markdown 报告；任务集见 tasks.json。 +// 真实材料不进仓：materials 目录按 eval/README 的口径放本机。 +package live + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +type task struct { + ID string `json:"id"` + Material string `json:"material,omitempty"` + Turns []string `json:"turns"` +} + +type part struct { + Type string `json:"type"` + Text string `json:"text"` + Process bool `json:"process"` + ToolCall *struct { + ID string `json:"id"` + Name string `json:"name"` + Arguments string `json:"arguments"` + Duration float64 `json:"durationMs"` + } `json:"toolCall"` + ToolResult *struct { + CallID string `json:"callId"` + Status string `json:"status"` + Summary string `json:"summary"` + } `json:"toolResult"` + Check *struct { + Passed bool `json:"passed"` + Reason string `json:"reason"` + } `json:"check"` + Artifact *struct { + ArtifactID string `json:"artifactId"` + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"artifact"` +} + +type message struct { + ID string `json:"id"` + RunID string `json:"runId"` + Role string `json:"role"` + Parts []part `json:"parts"` + Excluded bool `json:"excluded"` +} + +type run struct { + ID string `json:"id"` + Status string `json:"status"` + CreatedAt string `json:"createdAt"` + EndedAt string `json:"endedAt"` + Usage *struct { + PromptTokens int `json:"promptTokens"` + CompletionTokens int `json:"completionTokens"` + TotalTokens int `json:"totalTokens"` + DurationMs int `json:"durationMs"` + ToolCalls int `json:"toolCalls"` + } `json:"usage"` + Error any `json:"error"` +} + +type artifactMeta struct { + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + MIME string `json:"mime"` + SourceRun string `json:"sourceRun"` + CreatedAt string `json:"createdAt"` +} + +type hitlCard struct { + ID string `json:"id"` + Question string `json:"question"` + Choices []struct { + ID string `json:"id"` + Label string `json:"label"` + } `json:"choices"` + AllowFreeInput bool `json:"allowFreeInput"` +} + +type client struct { + base string + http *http.Client + t *testing.T + model string +} + +func (c *client) do(method, path string, body any, out any) error { + var rd io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return err + } + rd = bytes.NewReader(b) + } + req, err := http.NewRequest(method, c.base+path, rd) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.http.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, truncate(string(raw), 300)) + } + if out != nil { + return json.Unmarshal(raw, out) + } + return nil +} + +func truncate(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "…" +} + +func TestLiveSuite(t *testing.T) { + base := strings.TrimSpace(os.Getenv("AGENT_EVAL_BASE")) + if base == "" { + t.Skip("AGENT_EVAL_BASE not set; live eval requires a configured backend instance") + } + materialsDir := os.Getenv("AGENT_EVAL_MATERIALS") + if materialsDir == "" { + home, _ := os.UserHomeDir() + materialsDir = filepath.Join(home, ".agent-system", "eval", "materials") + } + if _, err := os.Stat(materialsDir); err != nil { + t.Skipf("materials dir not available: %v", err) + } + raw, err := os.ReadFile("tasks.json") + if err != nil { + t.Fatalf("read tasks.json: %v", err) + } + var tasks []task + if err := json.Unmarshal(raw, &tasks); err != nil { + t.Fatalf("parse tasks.json: %v", err) + } + if filter := os.Getenv("AGENT_EVAL_TASKS"); filter != "" { + want := map[string]bool{} + for _, id := range strings.Split(filter, ",") { + want[strings.TrimSpace(id)] = true + } + var kept []task + for _, tk := range tasks { + if want[tk.ID] { + kept = append(kept, tk) + } + } + tasks = kept + } + if len(tasks) == 0 { + t.Fatal("no tasks selected") + } + turnTimeout := 20 * time.Minute + if v := os.Getenv("AGENT_EVAL_TURN_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil { + turnTimeout = d + } + } + c := &client{base: base, http: &http.Client{Timeout: 120 * time.Second}, t: t} + var settings struct { + Models struct { + Chat struct { + Model string `json:"model"` + } `json:"chat"` + } `json:"models"` + } + _ = c.do("GET", "/api/settings", nil, &settings) + c.model = settings.Models.Chat.Model + + proj := struct { + ID string `json:"id"` + }{} + if err := c.do("POST", "/api/projects", map[string]string{"name": "eval-" + time.Now().Format("0102-150405")}, &proj); err != nil { + t.Fatalf("create project: %v", err) + } + + rep := report{Target: base, Model: c.model, GeneratedAt: time.Now().UTC().Format(time.RFC3339)} + for _, tk := range tasks { + rep.Tasks = append(rep.Tasks, runTask(t, c, proj.ID, tk, materialsDir, turnTimeout)) + } + writeReport(t, &rep) +} + +type turnResult struct { + User string `json:"user"` + RunID string `json:"runId"` + Status string `json:"status"` + FinalText string `json:"finalText,omitempty"` + Usage *run `json:"usage,omitempty"` + Metrics turnMetrics `json:"metrics"` + Checks []checkRecord `json:"checks,omitempty"` + Artifacts []string `json:"artifacts,omitempty"` + Issues []string `json:"issues,omitempty"` + DurationMs int64 `json:"durationMs"` +} + +type turnMetrics struct { + ToolCalls int `json:"toolCalls"` + ToolFailures int `json:"toolFailures"` + FullRegens int `json:"fullRegens"` + APIDiscovery int `json:"apiDiscovery"` + ReadsOriginal int `json:"readsOriginal"` + PlanUpdates int `json:"planUpdates"` + VerifierChecks int `json:"verifierChecks"` +} + +type checkRecord struct { + Passed bool `json:"passed"` + Reason string `json:"reason"` +} + +type taskResult struct { + ID string `json:"id"` + Turns []turnResult `json:"turns"` + Total turnMetrics `json:"total"` + UnverifiedFacts []string `json:"unverifiedFacts,omitempty"` + FinalVsArtifact []string `json:"finalVsArtifact,omitempty"` +} + +type report struct { + Target string `json:"target"` + Model string `json:"model,omitempty"` + GeneratedAt string `json:"generatedAt"` + Tasks []taskResult `json:"tasks"` +} + +func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir string, turnTimeout time.Duration) taskResult { + t.Helper() + conv := struct { + ID string `json:"id"` + }{} + if err := c.do("POST", "/api/projects/"+projectID+"/conversations", map[string]string{"title": "eval-" + tk.ID}, &conv); err != nil { + t.Fatalf("[%s] create conversation: %v", tk.ID, err) + } + var attachmentIDs []string + materialText := "" + if tk.Material != "" { + path := filepath.Join(materialsDir, tk.Material) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("[%s] open material: %v", tk.ID, err) + } + up := c.upload(t, projectID, tk.Material, raw) + attachmentIDs = append(attachmentIDs, up) + materialText = docxTextIfDocx(t, tk.Material, raw) + } + tr := taskResult{ID: tk.ID} + for i, turnText := range tk.Turns { + tr.Turns = append(tr.Turns, runTurn(t, c, projectID, conv.ID, turnText, attachmentIDs, turnTimeout)) + attachmentIDs = nil // 材料只在第一轮附带；后续轮考验跨轮上下文 + _ = i + } + // 汇总 + for _, turn := range tr.Turns { + tr.Total.ToolCalls += turn.Metrics.ToolCalls + tr.Total.ToolFailures += turn.Metrics.ToolFailures + tr.Total.FullRegens += turn.Metrics.FullRegens + tr.Total.APIDiscovery += turn.Metrics.APIDiscovery + tr.Total.ReadsOriginal += turn.Metrics.ReadsOriginal + tr.Total.PlanUpdates += turn.Metrics.PlanUpdates + tr.Total.VerifierChecks += turn.Metrics.VerifierChecks + } + // 事实核查：最后一个带工件的回合 vs 材料 + if materialText != "" { + last := lastArtifactTurn(tr) + if last != nil && len(last.Artifacts) > 0 { + artText := "" + for _, name := range last.Artifacts { + artText += downloadArtifactText(t, c, projectID, name) + } + tr.UnverifiedFacts = unverifiedFacts(artText, materialText) + tr.FinalVsArtifact = finalVsArtifact(last, artText) + } + } + return tr +} diff --git a/eval/live/tasks.json b/eval/live/tasks.json new file mode 100644 index 0000000..8ba87f3 --- /dev/null +++ b/eval/live/tasks.json @@ -0,0 +1,26 @@ +[ + { + "id": "minutes-basic", + "material": "meeting-rd-agent-asr.docx", + "turns": ["帮我总结一份会议纪要"] + }, + { + "id": "plan-followup", + "material": "meeting-rd-agent-asr.docx", + "turns": [ + "帮我总结一份会议纪要", + "根据这个纪要，制定一个方案", + "方案太简要了，给我一个详细的方案和计划" + ] + }, + { + "id": "qa-citation", + "material": "meeting-rd-agent-asr.docx", + "turns": ["这次会议最主要的事情是什么"] + }, + { + "id": "minutes-pm-tool", + "material": "meeting-pm-tool-asr.docx", + "turns": ["生成会议纪要"] + } +]