weichen
b68833f320
ci / vet-and-test (pull_request) Successful in 47s
Why: The live report's model field came back empty because the settings
endpoint nests models under the user scope ({defaults, project, schema, user}),
not at the top level; the before-baseline report had to be annotated by hand.
Evidence: SELF - vet with -tags live passes; the scope shape verified against
GET /api/settings on the qa instance.
Impact: Future live reports carry the served model without manual annotation.
315 lines
8.9 KiB
Go
315 lines
8.9 KiB
Go
//go:build live
|
|
|
|
// Package live 是真实后端 + 真实模型的任务级评测（Phase 0 基线）。
|
|
//
|
|
// 用法（对准一个已配置好模型的运行实例）：
|
|
//
|
|
// AGENT_EVAL_BASE=http://127.0.0.1:18799 \
|
|
// AGENT_EVAL_MATERIALS=~/.agent-system/eval/materials \
|
|
// AGENT_EVAL_OUT=/tmp/before.json \
|
|
// go test ./eval/live/ -tags live -run TestLiveSuite -v -timeout 90m
|
|
//
|
|
// 评测器只用公开 HTTP 面（上传→建会话→发消息→轮询），指标从消息 parts、
|
|
// run usage 与工件内容计算；不触碰目标实例数据库。HITL 卡自动选第一个选项，
|
|
// 保证无人值守跑完。产出 JSON + Markdown 报告；任务集见 tasks.json。
|
|
// 真实材料不进仓：materials 目录按 eval/README 的口径放本机。
|
|
package live
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type task struct {
|
|
ID string `json:"id"`
|
|
Material string `json:"material,omitempty"`
|
|
Turns []string `json:"turns"`
|
|
}
|
|
|
|
type part struct {
|
|
Type string `json:"type"`
|
|
Text string `json:"text"`
|
|
Process bool `json:"process"`
|
|
ToolCall *struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Arguments string `json:"arguments"`
|
|
Duration float64 `json:"durationMs"`
|
|
} `json:"toolCall"`
|
|
ToolResult *struct {
|
|
CallID string `json:"callId"`
|
|
Status string `json:"status"`
|
|
Summary string `json:"summary"`
|
|
} `json:"toolResult"`
|
|
Check *struct {
|
|
Passed bool `json:"passed"`
|
|
Reason string `json:"reason"`
|
|
} `json:"check"`
|
|
Artifact *struct {
|
|
ArtifactID string `json:"artifactId"`
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
} `json:"artifact"`
|
|
}
|
|
|
|
type message struct {
|
|
ID string `json:"id"`
|
|
RunID string `json:"runId"`
|
|
Role string `json:"role"`
|
|
Parts []part `json:"parts"`
|
|
Excluded bool `json:"excluded"`
|
|
}
|
|
|
|
type run struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
CreatedAt string `json:"createdAt"`
|
|
EndedAt string `json:"endedAt"`
|
|
Usage *struct {
|
|
PromptTokens int `json:"promptTokens"`
|
|
CompletionTokens int `json:"completionTokens"`
|
|
TotalTokens int `json:"totalTokens"`
|
|
DurationMs int `json:"durationMs"`
|
|
ToolCalls int `json:"toolCalls"`
|
|
} `json:"usage"`
|
|
Error any `json:"error"`
|
|
}
|
|
|
|
type artifactMeta struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
MIME string `json:"mime"`
|
|
SourceRun string `json:"sourceRun"`
|
|
CreatedAt string `json:"createdAt"`
|
|
}
|
|
|
|
type hitlCard struct {
|
|
ID string `json:"id"`
|
|
Question string `json:"question"`
|
|
Choices []struct {
|
|
ID string `json:"id"`
|
|
Label string `json:"label"`
|
|
} `json:"choices"`
|
|
AllowFreeInput bool `json:"allowFreeInput"`
|
|
}
|
|
|
|
type client struct {
|
|
base string
|
|
http *http.Client
|
|
t *testing.T
|
|
model string
|
|
}
|
|
|
|
func (c *client) do(method, path string, body any, out any) error {
|
|
var rd io.Reader
|
|
if body != nil {
|
|
b, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rd = bytes.NewReader(b)
|
|
}
|
|
req, err := http.NewRequest(method, c.base+path, rd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
raw, _ := io.ReadAll(resp.Body)
|
|
if resp.StatusCode >= 300 {
|
|
return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, truncate(string(raw), 300))
|
|
}
|
|
if out != nil {
|
|
return json.Unmarshal(raw, out)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
r := []rune(s)
|
|
if len(r) <= n {
|
|
return s
|
|
}
|
|
return string(r[:n]) + "…"
|
|
}
|
|
|
|
func TestLiveSuite(t *testing.T) {
|
|
base := strings.TrimSpace(os.Getenv("AGENT_EVAL_BASE"))
|
|
if base == "" {
|
|
t.Skip("AGENT_EVAL_BASE not set; live eval requires a configured backend instance")
|
|
}
|
|
materialsDir := os.Getenv("AGENT_EVAL_MATERIALS")
|
|
if materialsDir == "" {
|
|
home, _ := os.UserHomeDir()
|
|
materialsDir = filepath.Join(home, ".agent-system", "eval", "materials")
|
|
}
|
|
if _, err := os.Stat(materialsDir); err != nil {
|
|
t.Skipf("materials dir not available: %v", err)
|
|
}
|
|
raw, err := os.ReadFile("tasks.json")
|
|
if err != nil {
|
|
t.Fatalf("read tasks.json: %v", err)
|
|
}
|
|
var tasks []task
|
|
if err := json.Unmarshal(raw, &tasks); err != nil {
|
|
t.Fatalf("parse tasks.json: %v", err)
|
|
}
|
|
if filter := os.Getenv("AGENT_EVAL_TASKS"); filter != "" {
|
|
want := map[string]bool{}
|
|
for _, id := range strings.Split(filter, ",") {
|
|
want[strings.TrimSpace(id)] = true
|
|
}
|
|
var kept []task
|
|
for _, tk := range tasks {
|
|
if want[tk.ID] {
|
|
kept = append(kept, tk)
|
|
}
|
|
}
|
|
tasks = kept
|
|
}
|
|
if len(tasks) == 0 {
|
|
t.Fatal("no tasks selected")
|
|
}
|
|
turnTimeout := 20 * time.Minute
|
|
if v := os.Getenv("AGENT_EVAL_TURN_TIMEOUT"); v != "" {
|
|
if d, err := time.ParseDuration(v); err == nil {
|
|
turnTimeout = d
|
|
}
|
|
}
|
|
c := &client{base: base, http: &http.Client{Timeout: 120 * time.Second}, t: t}
|
|
// /api/settings 的形状是 {defaults, project, schema, user}；模型播在 user 作用域。
|
|
var settings struct {
|
|
User struct {
|
|
Models struct {
|
|
Chat struct {
|
|
Model string `json:"model"`
|
|
} `json:"chat"`
|
|
} `json:"models"`
|
|
} `json:"user"`
|
|
}
|
|
_ = c.do("GET", "/api/settings", nil, &settings)
|
|
c.model = settings.User.Models.Chat.Model
|
|
|
|
proj := struct {
|
|
ID string `json:"id"`
|
|
}{}
|
|
if err := c.do("POST", "/api/projects", map[string]string{"name": "eval-" + time.Now().Format("0102-150405")}, &proj); err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
rep := report{Target: base, Model: c.model, GeneratedAt: time.Now().UTC().Format(time.RFC3339)}
|
|
for _, tk := range tasks {
|
|
rep.Tasks = append(rep.Tasks, runTask(t, c, proj.ID, tk, materialsDir, turnTimeout))
|
|
}
|
|
writeReport(t, &rep)
|
|
}
|
|
|
|
type turnResult struct {
|
|
User string `json:"user"`
|
|
RunID string `json:"runId"`
|
|
Status string `json:"status"`
|
|
FinalText string `json:"finalText,omitempty"`
|
|
Usage *run `json:"usage,omitempty"`
|
|
Metrics turnMetrics `json:"metrics"`
|
|
Checks []checkRecord `json:"checks,omitempty"`
|
|
Artifacts []string `json:"artifacts,omitempty"`
|
|
Issues []string `json:"issues,omitempty"`
|
|
DurationMs int64 `json:"durationMs"`
|
|
}
|
|
|
|
type turnMetrics struct {
|
|
ToolCalls int `json:"toolCalls"`
|
|
ToolFailures int `json:"toolFailures"`
|
|
FullRegens int `json:"fullRegens"`
|
|
APIDiscovery int `json:"apiDiscovery"`
|
|
ReadsOriginal int `json:"readsOriginal"`
|
|
PlanUpdates int `json:"planUpdates"`
|
|
VerifierChecks int `json:"verifierChecks"`
|
|
}
|
|
|
|
type checkRecord struct {
|
|
Passed bool `json:"passed"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
type taskResult struct {
|
|
ID string `json:"id"`
|
|
Turns []turnResult `json:"turns"`
|
|
Total turnMetrics `json:"total"`
|
|
UnverifiedFacts []string `json:"unverifiedFacts,omitempty"`
|
|
FinalVsArtifact []string `json:"finalVsArtifact,omitempty"`
|
|
}
|
|
|
|
type report struct {
|
|
Target string `json:"target"`
|
|
Model string `json:"model,omitempty"`
|
|
GeneratedAt string `json:"generatedAt"`
|
|
Tasks []taskResult `json:"tasks"`
|
|
}
|
|
|
|
func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir string, turnTimeout time.Duration) taskResult {
|
|
t.Helper()
|
|
conv := struct {
|
|
ID string `json:"id"`
|
|
}{}
|
|
if err := c.do("POST", "/api/projects/"+projectID+"/conversations", map[string]string{"title": "eval-" + tk.ID}, &conv); err != nil {
|
|
t.Fatalf("[%s] create conversation: %v", tk.ID, err)
|
|
}
|
|
var attachmentIDs []string
|
|
materialText := ""
|
|
if tk.Material != "" {
|
|
path := filepath.Join(materialsDir, tk.Material)
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatalf("[%s] open material: %v", tk.ID, err)
|
|
}
|
|
up := c.upload(t, projectID, tk.Material, raw)
|
|
attachmentIDs = append(attachmentIDs, up)
|
|
materialText = docxTextIfDocx(t, tk.Material, raw)
|
|
}
|
|
tr := taskResult{ID: tk.ID}
|
|
for i, turnText := range tk.Turns {
|
|
tr.Turns = append(tr.Turns, runTurn(t, c, projectID, conv.ID, turnText, attachmentIDs, turnTimeout))
|
|
attachmentIDs = nil // 材料只在第一轮附带；后续轮考验跨轮上下文
|
|
_ = i
|
|
}
|
|
// 汇总
|
|
for _, turn := range tr.Turns {
|
|
tr.Total.ToolCalls += turn.Metrics.ToolCalls
|
|
tr.Total.ToolFailures += turn.Metrics.ToolFailures
|
|
tr.Total.FullRegens += turn.Metrics.FullRegens
|
|
tr.Total.APIDiscovery += turn.Metrics.APIDiscovery
|
|
tr.Total.ReadsOriginal += turn.Metrics.ReadsOriginal
|
|
tr.Total.PlanUpdates += turn.Metrics.PlanUpdates
|
|
tr.Total.VerifierChecks += turn.Metrics.VerifierChecks
|
|
}
|
|
// 事实核查：最后一个带工件的回合 vs 材料
|
|
if materialText != "" {
|
|
last := lastArtifactTurn(tr)
|
|
if last != nil && len(last.Artifacts) > 0 {
|
|
artText := ""
|
|
for _, name := range last.Artifacts {
|
|
artText += downloadArtifactText(t, c, projectID, name)
|
|
}
|
|
tr.UnverifiedFacts = unverifiedFacts(artText, materialText)
|
|
tr.FinalVsArtifact = finalVsArtifact(last, artText)
|
|
}
|
|
}
|
|
return tr
|
|
}
|