Add the live-model task eval harness (phase 0 baseline)
Why: No task-level measurement of real agent behavior existed, so the failure chain documented in the diagnosis was invisible to CI. The harness drives a configured backend over its public HTTP surface (upload, message, poll, HITL auto-resolve), computes per-turn metrics (tool failures, full regenerations, API-discovery calls, original-material reads, plan usage, gate checks) and outcome checks (unverified facts vs material text, final claims vs artifact headings), and writes a JSON + Markdown report. Evidence: SELF - built behind the live build tag (CI skips); vet passes with -tags live; drove the qa instance (remote main 1ebd46e) through the four golden tasks end to end; report referenced on the phase01 task card. Impact: New eval/live package (tasks.json + runner). Materials stay outside the repo per eval/README; runs are opt-in via AGENT_EVAL_BASE. Refs: Dev task feature/agent-quality-phase01; documents proposal 2026-10-09-agent-quality-improvement-plan.md §4 P0.
This commit is contained in:
@@ -0,0 +1,440 @@
|
||||
//go:build live
|
||||
|
||||
package live
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// upload 上传一个材料文件，返回 artifact id。
|
||||
func (c *client) upload(t *testing.T, projectID, name string, content []byte) string {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
fw, err := w.CreateFormFile("file", name)
|
||||
if err != nil {
|
||||
t.Fatalf("multipart field: %v", err)
|
||||
}
|
||||
if _, err := fw.Write(content); err != nil {
|
||||
t.Fatalf("multipart write: %v", err)
|
||||
}
|
||||
w.Close()
|
||||
req, err := http.NewRequest("POST", c.base+"/api/projects/"+projectID+"/uploads", &buf)
|
||||
if err != nil {
|
||||
t.Fatalf("upload request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 300 {
|
||||
t.Fatalf("upload %s: %s: %s", name, resp.Status, truncate(string(raw), 300))
|
||||
}
|
||||
var out struct {
|
||||
ID string `json:"artifactId"`
|
||||
LegacyID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatalf("upload response: %v %s", err, truncate(string(raw), 300))
|
||||
}
|
||||
if out.ID == "" {
|
||||
out.ID = out.LegacyID
|
||||
}
|
||||
if out.ID == "" {
|
||||
t.Fatalf("upload response missing artifact id: %s", truncate(string(raw), 300))
|
||||
}
|
||||
return out.ID
|
||||
}
|
||||
|
||||
// runTurn 发一轮消息并等到 run 终态；期间自动应答 HITL 卡。
|
||||
func runTurn(t *testing.T, c *client, projectID, convID, text string, attachmentIDs []string, turnTimeout time.Duration) turnResult {
|
||||
t.Helper()
|
||||
started := time.Now()
|
||||
var out struct {
|
||||
Message message `json:"message"`
|
||||
Run run `json:"run"`
|
||||
}
|
||||
body := map[string]any{"text": text, "requestId": fmt.Sprintf("eval-%d", time.Now().UnixNano())}
|
||||
if len(attachmentIDs) > 0 {
|
||||
body["attachmentIds"] = attachmentIDs
|
||||
}
|
||||
if err := c.do("POST", "/api/conversations/"+convID+"/messages", body, &out); err != nil {
|
||||
t.Fatalf("post message: %v", err)
|
||||
}
|
||||
runID := out.Run.ID
|
||||
deadline := time.Now().Add(turnTimeout)
|
||||
var cur run
|
||||
for {
|
||||
var resp struct {
|
||||
Run run `json:"run"`
|
||||
}
|
||||
if err := c.do("GET", "/api/runs/"+runID, nil, &resp); err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
cur = resp.Run
|
||||
if isTerminal(cur.Status) {
|
||||
break
|
||||
}
|
||||
autoResolveHITL(t, c, convID)
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("run %s timed out after %s (status=%s)", runID, turnTimeout, cur.Status)
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
detail := struct {
|
||||
Messages []message `json:"messages"`
|
||||
ActiveHitl []hitlCard `json:"activeHitl"`
|
||||
}{}
|
||||
if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
|
||||
t.Fatalf("conversation detail: %v", err)
|
||||
}
|
||||
var assistant *message
|
||||
for i := range detail.Messages {
|
||||
m := detail.Messages[i]
|
||||
if m.RunID == runID && m.Role == "assistant" && !m.Excluded {
|
||||
assistant = &detail.Messages[i]
|
||||
}
|
||||
}
|
||||
tr := turnResult{User: text, RunID: runID, Status: cur.Status, DurationMs: time.Since(started).Milliseconds()}
|
||||
if cur.Usage != nil {
|
||||
tr.Usage = &cur
|
||||
}
|
||||
if assistant == nil {
|
||||
tr.Issues = append(tr.Issues, "没有找到本回合的助手消息")
|
||||
return tr
|
||||
}
|
||||
collectMetrics(&tr, assistant)
|
||||
// 工件清单：本 run 产出的。
|
||||
var arts []artifactMeta
|
||||
if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err == nil {
|
||||
for _, a := range arts {
|
||||
if a.SourceRun == runID {
|
||||
tr.Artifacts = append(tr.Artifacts, a.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if cur.Status != "completed" {
|
||||
tr.Issues = append(tr.Issues, "run 终态="+cur.Status)
|
||||
}
|
||||
return tr
|
||||
}
|
||||
|
||||
func isTerminal(status string) bool {
|
||||
switch status {
|
||||
case "completed", "failed", "cancelled":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// autoResolveHITL 无人值守：有活动确认卡时选第一个选项（没有选项给自由文本同意）。
|
||||
func autoResolveHITL(t *testing.T, c *client, convID string) {
|
||||
t.Helper()
|
||||
detail := struct {
|
||||
ActiveHitl []hitlCard `json:"activeHitl"`
|
||||
}{}
|
||||
if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
|
||||
return
|
||||
}
|
||||
for _, card := range detail.ActiveHitl {
|
||||
resp := map[string]string{}
|
||||
if len(card.Choices) > 0 {
|
||||
resp["choiceId"] = card.Choices[0].ID
|
||||
} else if card.AllowFreeInput {
|
||||
resp["freeText"] = "同意继续（评测自动应答）"
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
_ = c.do("POST", "/api/hitl/"+card.ID+"/resolve", resp, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func collectMetrics(tr *turnResult, m *message) {
|
||||
for _, p := range m.Parts {
|
||||
switch p.Type {
|
||||
case "text":
|
||||
if !p.Process && strings.TrimSpace(p.Text) != "" {
|
||||
tr.FinalText = p.Text
|
||||
}
|
||||
case "tool_call":
|
||||
if p.ToolCall == nil {
|
||||
continue
|
||||
}
|
||||
tr.Metrics.ToolCalls++
|
||||
args := p.ToolCall.Arguments
|
||||
switch p.ToolCall.Name {
|
||||
case "update_plan", "update_todos":
|
||||
tr.Metrics.PlanUpdates++
|
||||
case "run_python":
|
||||
if strings.Contains(args, "new_document(") || strings.Contains(args, "new_presentation(") || strings.Contains(args, "new_workbook(") {
|
||||
tr.Metrics.FullRegens++
|
||||
}
|
||||
}
|
||||
if strings.Contains(args, "dir(") || strings.Contains(args, "inspect.signature") || strings.Contains(args, "help(") {
|
||||
tr.Metrics.APIDiscovery++
|
||||
}
|
||||
if strings.Contains(args, "inputs/") {
|
||||
tr.Metrics.ReadsOriginal++
|
||||
}
|
||||
case "tool_result":
|
||||
if p.ToolResult == nil {
|
||||
continue
|
||||
}
|
||||
s := p.ToolResult.Summary
|
||||
if p.ToolResult.Status == "error" || strings.HasPrefix(s, "exit=1") || strings.Contains(s, "Traceback") || strings.Contains(s, "ModuleNotFoundError") {
|
||||
tr.Metrics.ToolFailures++
|
||||
}
|
||||
case "check":
|
||||
if p.Check == nil {
|
||||
continue
|
||||
}
|
||||
tr.Metrics.VerifierChecks++
|
||||
tr.Checks = append(tr.Checks, checkRecord{Passed: p.Check.Passed, Reason: p.Check.Reason})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lastArtifactTurn(tr taskResult) *turnResult {
|
||||
for i := len(tr.Turns) - 1; i >= 0; i-- {
|
||||
if len(tr.Turns[i].Artifacts) > 0 {
|
||||
return &tr.Turns[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadArtifactText(t *testing.T, c *client, projectID, name string) string {
|
||||
t.Helper()
|
||||
var arts []artifactMeta
|
||||
if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, a := range arts {
|
||||
if a.Name != name {
|
||||
continue
|
||||
}
|
||||
req, _ := http.NewRequest("GET", c.base+"/api/artifacts/"+a.ID+"/content", nil)
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if strings.HasSuffix(strings.ToLower(a.Name), ".docx") {
|
||||
return docxTextIfDocx(t, a.Name, raw)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ---- 文本抽取与事实核查 ----
|
||||
|
||||
// docxTextIfDocx：docx 抽纯文本（标题行带 # 前缀）；其他类型原样返回。
|
||||
func docxTextIfDocx(t *testing.T, name string, raw []byte) string {
|
||||
t.Helper()
|
||||
if !strings.HasSuffix(strings.ToLower(name), ".docx") {
|
||||
return string(raw)
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
if f.Name != "word/document.xml" {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer rc.Close()
|
||||
dec := xml.NewDecoder(rc)
|
||||
var b strings.Builder
|
||||
var para strings.Builder
|
||||
style, inPara := "", false
|
||||
flush := func() {
|
||||
text := strings.TrimSpace(para.String())
|
||||
if text != "" {
|
||||
if lvl := docxHeadingLevel(style); lvl > 0 {
|
||||
b.WriteString(strings.Repeat("#", lvl) + " ")
|
||||
}
|
||||
b.WriteString(text + "\n")
|
||||
}
|
||||
para.Reset()
|
||||
style = ""
|
||||
}
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch el := tok.(type) {
|
||||
case xml.StartElement:
|
||||
switch el.Name.Local {
|
||||
case "p":
|
||||
inPara, style = true, ""
|
||||
para.Reset()
|
||||
case "pStyle":
|
||||
if inPara {
|
||||
for _, at := range el.Attr {
|
||||
if at.Name.Local == "val" {
|
||||
style = at.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
if el.Name.Local == "p" {
|
||||
inPara = false
|
||||
flush()
|
||||
}
|
||||
case xml.CharData:
|
||||
if inPara {
|
||||
para.Write(bytes.TrimSpace(el))
|
||||
}
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return b.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func docxHeadingLevel(style string) int {
|
||||
switch {
|
||||
case style == "Title":
|
||||
return 1
|
||||
case style == "Subtitle":
|
||||
return 2
|
||||
case strings.HasPrefix(style, "Heading"):
|
||||
n := 0
|
||||
for _, ch := range style[len("Heading"):] {
|
||||
if ch < '0' || ch > '9' {
|
||||
return 0
|
||||
}
|
||||
n = n*10 + int(ch-'0')
|
||||
}
|
||||
if n >= 1 && n <= 9 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var (
|
||||
pctRe = regexp.MustCompile(`\d+(?:\.\d+)?%`)
|
||||
dateRe = regexp.MustCompile(`\d{1,2}月\d{1,2}日`)
|
||||
bigNumRe = regexp.MustCompile(`\d{4,}`)
|
||||
claimRe = regexp.MustCompile(`(?:^|\n|\s)([一二三四五六七八九十]{1,2}、[^\s，。；|]{2,24})`)
|
||||
)
|
||||
|
||||
// unverifiedFacts 抽取交付物里的百分比/日期/大数字，逐个回材料原文找出处。
|
||||
// 找不到的列为未核实事实（模型编造或来自模型自带的「常识」）。
|
||||
func unverifiedFacts(artifactText, materialText string) []string {
|
||||
if strings.TrimSpace(artifactText) == "" || strings.TrimSpace(materialText) == "" {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
check := func(tok string) {
|
||||
tok = strings.TrimSpace(tok)
|
||||
if tok == "" || seen[tok] {
|
||||
return
|
||||
}
|
||||
seen[tok] = true
|
||||
if !strings.Contains(materialText, tok) {
|
||||
out = append(out, tok)
|
||||
}
|
||||
}
|
||||
for _, m := range pctRe.FindAllString(artifactText, -1) {
|
||||
check(m)
|
||||
}
|
||||
for _, m := range dateRe.FindAllString(artifactText, -1) {
|
||||
check(m)
|
||||
}
|
||||
for _, m := range bigNumRe.FindAllString(artifactText, -1) {
|
||||
if len(m) >= 5 { // 4 位数大量是年份/编号，只报 5 位以上
|
||||
check(m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// finalVsArtifact 终轮回复里声称的章节（「一、…」）必须能在工件里找到。
|
||||
func finalVsArtifact(turn *turnResult, artifactText string) []string {
|
||||
if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" {
|
||||
return nil
|
||||
}
|
||||
final := turn.FinalText
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, m := range claimRe.FindAllStringSubmatch(final, -1) {
|
||||
claim := strings.TrimSpace(m[1])
|
||||
if claim == "" || seen[claim] {
|
||||
continue
|
||||
}
|
||||
seen[claim] = true
|
||||
if !strings.Contains(artifactText, claim) {
|
||||
out = append(out, claim)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- 报告 ----
|
||||
|
||||
func writeReport(t *testing.T, rep *report) {
|
||||
t.Helper()
|
||||
outPath := os.Getenv("AGENT_EVAL_OUT")
|
||||
if outPath == "" {
|
||||
outPath = filepath.Join("eval-live", "report-"+time.Now().Format("0102-150405")+".json")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||||
t.Fatalf("mkdir report dir: %v", err)
|
||||
}
|
||||
raw, _ := json.MarshalIndent(rep, "", " ")
|
||||
if err := os.WriteFile(outPath, raw, 0o644); err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "\n[live-eval] report: %s\n%s\n", outPath, renderMarkdown(rep))
|
||||
}
|
||||
|
||||
func renderMarkdown(rep *report) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "# Live eval — %s\n\n- target: %s\n- model: %s\n- generated: %s\n\n", rep.GeneratedAt, rep.Target, rep.Model, rep.GeneratedAt)
|
||||
for _, tk := range rep.Tasks {
|
||||
fmt.Fprintf(&b, "## %s\n\n| turn | status | tools | fails | regens | api探查 | 读原件 | 计划 | 检查 | 工件 | 时长s |\n|---|---|---|---|---|---|---|---|---|---|---|\n", tk.ID)
|
||||
for _, turn := range tk.Turns {
|
||||
fmt.Fprintf(&b, "| %s | %s | %d | %d | %d | %d | %d | %d | %d | %d | %.0f |\n",
|
||||
truncate(turn.User, 18), turn.Status, turn.Metrics.ToolCalls, turn.Metrics.ToolFailures,
|
||||
turn.Metrics.FullRegens, turn.Metrics.APIDiscovery, turn.Metrics.ReadsOriginal,
|
||||
turn.Metrics.PlanUpdates, turn.Metrics.VerifierChecks, len(turn.Artifacts),
|
||||
float64(turn.DurationMs)/1000)
|
||||
}
|
||||
if len(tk.UnverifiedFacts) > 0 {
|
||||
fmt.Fprintf(&b, "\n未核实事实（工件里出现、材料里找不到）：%s\n", strings.Join(tk.UnverifiedFacts, "、"))
|
||||
}
|
||||
if len(tk.FinalVsArtifact) > 0 {
|
||||
fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节：%s\n", strings.Join(tk.FinalVsArtifact, "；"))
|
||||
}
|
||||
fmt.Fprint(&b, "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
//go:build live
|
||||
|
||||
// Package live 是真实后端 + 真实模型的任务级评测（Phase 0 基线）。
|
||||
//
|
||||
// 用法（对准一个已配置好模型的运行实例）：
|
||||
//
|
||||
// AGENT_EVAL_BASE=http://127.0.0.1:18799 \
|
||||
// AGENT_EVAL_MATERIALS=~/.agent-system/eval/materials \
|
||||
// AGENT_EVAL_OUT=/tmp/before.json \
|
||||
// go test ./eval/live/ -tags live -run TestLiveSuite -v -timeout 90m
|
||||
//
|
||||
// 评测器只用公开 HTTP 面（上传→建会话→发消息→轮询），指标从消息 parts、
|
||||
// run usage 与工件内容计算；不触碰目标实例数据库。HITL 卡自动选第一个选项，
|
||||
// 保证无人值守跑完。产出 JSON + Markdown 报告；任务集见 tasks.json。
|
||||
// 真实材料不进仓：materials 目录按 eval/README 的口径放本机。
|
||||
package live
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type task struct {
|
||||
ID string `json:"id"`
|
||||
Material string `json:"material,omitempty"`
|
||||
Turns []string `json:"turns"`
|
||||
}
|
||||
|
||||
type part struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
Process bool `json:"process"`
|
||||
ToolCall *struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
Duration float64 `json:"durationMs"`
|
||||
} `json:"toolCall"`
|
||||
ToolResult *struct {
|
||||
CallID string `json:"callId"`
|
||||
Status string `json:"status"`
|
||||
Summary string `json:"summary"`
|
||||
} `json:"toolResult"`
|
||||
Check *struct {
|
||||
Passed bool `json:"passed"`
|
||||
Reason string `json:"reason"`
|
||||
} `json:"check"`
|
||||
Artifact *struct {
|
||||
ArtifactID string `json:"artifactId"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
} `json:"artifact"`
|
||||
}
|
||||
|
||||
type message struct {
|
||||
ID string `json:"id"`
|
||||
RunID string `json:"runId"`
|
||||
Role string `json:"role"`
|
||||
Parts []part `json:"parts"`
|
||||
Excluded bool `json:"excluded"`
|
||||
}
|
||||
|
||||
type run struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
EndedAt string `json:"endedAt"`
|
||||
Usage *struct {
|
||||
PromptTokens int `json:"promptTokens"`
|
||||
CompletionTokens int `json:"completionTokens"`
|
||||
TotalTokens int `json:"totalTokens"`
|
||||
DurationMs int `json:"durationMs"`
|
||||
ToolCalls int `json:"toolCalls"`
|
||||
} `json:"usage"`
|
||||
Error any `json:"error"`
|
||||
}
|
||||
|
||||
type artifactMeta struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
MIME string `json:"mime"`
|
||||
SourceRun string `json:"sourceRun"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
}
|
||||
|
||||
type hitlCard struct {
|
||||
ID string `json:"id"`
|
||||
Question string `json:"question"`
|
||||
Choices []struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
} `json:"choices"`
|
||||
AllowFreeInput bool `json:"allowFreeInput"`
|
||||
}
|
||||
|
||||
type client struct {
|
||||
base string
|
||||
http *http.Client
|
||||
t *testing.T
|
||||
model string
|
||||
}
|
||||
|
||||
func (c *client) do(method, path string, body any, out any) error {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequest(method, c.base+path, rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, truncate(string(raw), 300))
|
||||
}
|
||||
if out != nil {
|
||||
return json.Unmarshal(raw, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
||||
func TestLiveSuite(t *testing.T) {
|
||||
base := strings.TrimSpace(os.Getenv("AGENT_EVAL_BASE"))
|
||||
if base == "" {
|
||||
t.Skip("AGENT_EVAL_BASE not set; live eval requires a configured backend instance")
|
||||
}
|
||||
materialsDir := os.Getenv("AGENT_EVAL_MATERIALS")
|
||||
if materialsDir == "" {
|
||||
home, _ := os.UserHomeDir()
|
||||
materialsDir = filepath.Join(home, ".agent-system", "eval", "materials")
|
||||
}
|
||||
if _, err := os.Stat(materialsDir); err != nil {
|
||||
t.Skipf("materials dir not available: %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile("tasks.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read tasks.json: %v", err)
|
||||
}
|
||||
var tasks []task
|
||||
if err := json.Unmarshal(raw, &tasks); err != nil {
|
||||
t.Fatalf("parse tasks.json: %v", err)
|
||||
}
|
||||
if filter := os.Getenv("AGENT_EVAL_TASKS"); filter != "" {
|
||||
want := map[string]bool{}
|
||||
for _, id := range strings.Split(filter, ",") {
|
||||
want[strings.TrimSpace(id)] = true
|
||||
}
|
||||
var kept []task
|
||||
for _, tk := range tasks {
|
||||
if want[tk.ID] {
|
||||
kept = append(kept, tk)
|
||||
}
|
||||
}
|
||||
tasks = kept
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("no tasks selected")
|
||||
}
|
||||
turnTimeout := 20 * time.Minute
|
||||
if v := os.Getenv("AGENT_EVAL_TURN_TIMEOUT"); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
turnTimeout = d
|
||||
}
|
||||
}
|
||||
c := &client{base: base, http: &http.Client{Timeout: 120 * time.Second}, t: t}
|
||||
var settings struct {
|
||||
Models struct {
|
||||
Chat struct {
|
||||
Model string `json:"model"`
|
||||
} `json:"chat"`
|
||||
} `json:"models"`
|
||||
}
|
||||
_ = c.do("GET", "/api/settings", nil, &settings)
|
||||
c.model = settings.Models.Chat.Model
|
||||
|
||||
proj := struct {
|
||||
ID string `json:"id"`
|
||||
}{}
|
||||
if err := c.do("POST", "/api/projects", map[string]string{"name": "eval-" + time.Now().Format("0102-150405")}, &proj); err != nil {
|
||||
t.Fatalf("create project: %v", err)
|
||||
}
|
||||
|
||||
rep := report{Target: base, Model: c.model, GeneratedAt: time.Now().UTC().Format(time.RFC3339)}
|
||||
for _, tk := range tasks {
|
||||
rep.Tasks = append(rep.Tasks, runTask(t, c, proj.ID, tk, materialsDir, turnTimeout))
|
||||
}
|
||||
writeReport(t, &rep)
|
||||
}
|
||||
|
||||
type turnResult struct {
|
||||
User string `json:"user"`
|
||||
RunID string `json:"runId"`
|
||||
Status string `json:"status"`
|
||||
FinalText string `json:"finalText,omitempty"`
|
||||
Usage *run `json:"usage,omitempty"`
|
||||
Metrics turnMetrics `json:"metrics"`
|
||||
Checks []checkRecord `json:"checks,omitempty"`
|
||||
Artifacts []string `json:"artifacts,omitempty"`
|
||||
Issues []string `json:"issues,omitempty"`
|
||||
DurationMs int64 `json:"durationMs"`
|
||||
}
|
||||
|
||||
type turnMetrics struct {
|
||||
ToolCalls int `json:"toolCalls"`
|
||||
ToolFailures int `json:"toolFailures"`
|
||||
FullRegens int `json:"fullRegens"`
|
||||
APIDiscovery int `json:"apiDiscovery"`
|
||||
ReadsOriginal int `json:"readsOriginal"`
|
||||
PlanUpdates int `json:"planUpdates"`
|
||||
VerifierChecks int `json:"verifierChecks"`
|
||||
}
|
||||
|
||||
type checkRecord struct {
|
||||
Passed bool `json:"passed"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type taskResult struct {
|
||||
ID string `json:"id"`
|
||||
Turns []turnResult `json:"turns"`
|
||||
Total turnMetrics `json:"total"`
|
||||
UnverifiedFacts []string `json:"unverifiedFacts,omitempty"`
|
||||
FinalVsArtifact []string `json:"finalVsArtifact,omitempty"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
Target string `json:"target"`
|
||||
Model string `json:"model,omitempty"`
|
||||
GeneratedAt string `json:"generatedAt"`
|
||||
Tasks []taskResult `json:"tasks"`
|
||||
}
|
||||
|
||||
func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir string, turnTimeout time.Duration) taskResult {
|
||||
t.Helper()
|
||||
conv := struct {
|
||||
ID string `json:"id"`
|
||||
}{}
|
||||
if err := c.do("POST", "/api/projects/"+projectID+"/conversations", map[string]string{"title": "eval-" + tk.ID}, &conv); err != nil {
|
||||
t.Fatalf("[%s] create conversation: %v", tk.ID, err)
|
||||
}
|
||||
var attachmentIDs []string
|
||||
materialText := ""
|
||||
if tk.Material != "" {
|
||||
path := filepath.Join(materialsDir, tk.Material)
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("[%s] open material: %v", tk.ID, err)
|
||||
}
|
||||
up := c.upload(t, projectID, tk.Material, raw)
|
||||
attachmentIDs = append(attachmentIDs, up)
|
||||
materialText = docxTextIfDocx(t, tk.Material, raw)
|
||||
}
|
||||
tr := taskResult{ID: tk.ID}
|
||||
for i, turnText := range tk.Turns {
|
||||
tr.Turns = append(tr.Turns, runTurn(t, c, projectID, conv.ID, turnText, attachmentIDs, turnTimeout))
|
||||
attachmentIDs = nil // 材料只在第一轮附带；后续轮考验跨轮上下文
|
||||
_ = i
|
||||
}
|
||||
// 汇总
|
||||
for _, turn := range tr.Turns {
|
||||
tr.Total.ToolCalls += turn.Metrics.ToolCalls
|
||||
tr.Total.ToolFailures += turn.Metrics.ToolFailures
|
||||
tr.Total.FullRegens += turn.Metrics.FullRegens
|
||||
tr.Total.APIDiscovery += turn.Metrics.APIDiscovery
|
||||
tr.Total.ReadsOriginal += turn.Metrics.ReadsOriginal
|
||||
tr.Total.PlanUpdates += turn.Metrics.PlanUpdates
|
||||
tr.Total.VerifierChecks += turn.Metrics.VerifierChecks
|
||||
}
|
||||
// 事实核查：最后一个带工件的回合 vs 材料
|
||||
if materialText != "" {
|
||||
last := lastArtifactTurn(tr)
|
||||
if last != nil && len(last.Artifacts) > 0 {
|
||||
artText := ""
|
||||
for _, name := range last.Artifacts {
|
||||
artText += downloadArtifactText(t, c, projectID, name)
|
||||
}
|
||||
tr.UnverifiedFacts = unverifiedFacts(artText, materialText)
|
||||
tr.FinalVsArtifact = finalVsArtifact(last, artText)
|
||||
}
|
||||
}
|
||||
return tr
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"id": "minutes-basic",
|
||||
"material": "meeting-rd-agent-asr.docx",
|
||||
"turns": ["帮我总结一份会议纪要"]
|
||||
},
|
||||
{
|
||||
"id": "plan-followup",
|
||||
"material": "meeting-rd-agent-asr.docx",
|
||||
"turns": [
|
||||
"帮我总结一份会议纪要",
|
||||
"根据这个纪要，制定一个方案",
|
||||
"方案太简要了，给我一个详细的方案和计划"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "qa-citation",
|
||||
"material": "meeting-rd-agent-asr.docx",
|
||||
"turns": ["这次会议最主要的事情是什么"]
|
||||
},
|
||||
{
|
||||
"id": "minutes-pm-tool",
|
||||
"material": "meeting-pm-tool-asr.docx",
|
||||
"turns": ["生成会议纪要"]
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user