Agent-System-Backend/eval/live/live_test.go

//go:build live

// Package live 是真实后端 + 真实模型的任务级评测（Phase 0 基线）。
//
// 用法（对准一个已配置好模型的运行实例）：
//
//	AGENT_EVAL_BASE=http://127.0.0.1:18799 \
//	AGENT_EVAL_MATERIALS=~/.agent-system/eval/materials \
//	AGENT_EVAL_OUT=/tmp/before.json \
//	go test ./eval/live/ -tags live -run TestLiveSuite -v -timeout 90m
//
// 评测器只用公开 HTTP 面（上传→建会话→发消息→轮询），指标从消息 parts、
// run usage 与工件内容计算；不触碰目标实例数据库。HITL 卡自动选第一个选项，
// 保证无人值守跑完。产出 JSON + Markdown 报告；任务集见 tasks.json。
// 真实材料不进仓：materials 目录按 eval/README 的口径放本机。
package live

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"
)

type task struct {
	ID       string   `json:"id"`
	Material string   `json:"material,omitempty"`
	Turns    []string `json:"turns"`
}

type part struct {
	Type     string `json:"type"`
	Text     string `json:"text"`
	Process  bool   `json:"process"`
	ToolCall *struct {
		ID        string  `json:"id"`
		Name      string  `json:"name"`
		Arguments string  `json:"arguments"`
		Duration  float64 `json:"durationMs"`
	} `json:"toolCall"`
	ToolResult *struct {
		CallID  string `json:"callId"`
		Status  string `json:"status"`
		Summary string `json:"summary"`
	} `json:"toolResult"`
	Check *struct {
		Passed bool   `json:"passed"`
		Reason string `json:"reason"`
	} `json:"check"`
	Artifact *struct {
		ArtifactID string `json:"artifactId"`
		Name       string `json:"name"`
		Kind       string `json:"kind"`
	} `json:"artifact"`
}

type message struct {
	ID       string `json:"id"`
	RunID    string `json:"runId"`
	Role     string `json:"role"`
	Parts    []part `json:"parts"`
	Excluded bool   `json:"excluded"`
}

type run struct {
	ID        string `json:"id"`
	Status    string `json:"status"`
	CreatedAt string `json:"createdAt"`
	EndedAt   string `json:"endedAt"`
	Usage     *struct {
		PromptTokens     int `json:"promptTokens"`
		CompletionTokens int `json:"completionTokens"`
		TotalTokens      int `json:"totalTokens"`
		DurationMs       int `json:"durationMs"`
		ToolCalls        int `json:"toolCalls"`
	} `json:"usage"`
	Error any `json:"error"`
}

type artifactMeta struct {
	ID        string `json:"id"`
	Name      string `json:"name"`
	Kind      string `json:"kind"`
	MIME      string `json:"mime"`
	SourceRun string `json:"sourceRun"`
	CreatedAt string `json:"createdAt"`
}

type hitlCard struct {
	ID       string `json:"id"`
	Question string `json:"question"`
	Choices  []struct {
		ID    string `json:"id"`
		Label string `json:"label"`
	} `json:"choices"`
	AllowFreeInput bool `json:"allowFreeInput"`
}

type client struct {
	base  string
	http  *http.Client
	t     *testing.T
	model string
}

func (c *client) do(method, path string, body any, out any) error {
	var rd io.Reader
	if body != nil {
		b, err := json.Marshal(body)
		if err != nil {
			return err
		}
		rd = bytes.NewReader(b)
	}
	req, err := http.NewRequest(method, c.base+path, rd)
	if err != nil {
		return err
	}
	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	resp, err := c.http.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	raw, _ := io.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, truncate(string(raw), 300))
	}
	if out != nil {
		return json.Unmarshal(raw, out)
	}
	return nil
}

func truncate(s string, n int) string {
	r := []rune(s)
	if len(r) <= n {
		return s
	}
	return string(r[:n]) + "…"
}

func TestLiveSuite(t *testing.T) {
	base := strings.TrimSpace(os.Getenv("AGENT_EVAL_BASE"))
	if base == "" {
		t.Skip("AGENT_EVAL_BASE not set; live eval requires a configured backend instance")
	}
	materialsDir := os.Getenv("AGENT_EVAL_MATERIALS")
	if materialsDir == "" {
		home, _ := os.UserHomeDir()
		materialsDir = filepath.Join(home, ".agent-system", "eval", "materials")
	}
	if _, err := os.Stat(materialsDir); err != nil {
		t.Skipf("materials dir not available: %v", err)
	}
	raw, err := os.ReadFile("tasks.json")
	if err != nil {
		t.Fatalf("read tasks.json: %v", err)
	}
	var tasks []task
	if err := json.Unmarshal(raw, &tasks); err != nil {
		t.Fatalf("parse tasks.json: %v", err)
	}
	if filter := os.Getenv("AGENT_EVAL_TASKS"); filter != "" {
		want := map[string]bool{}
		for _, id := range strings.Split(filter, ",") {
			want[strings.TrimSpace(id)] = true
		}
		var kept []task
		for _, tk := range tasks {
			if want[tk.ID] {
				kept = append(kept, tk)
			}
		}
		tasks = kept
	}
	if len(tasks) == 0 {
		t.Fatal("no tasks selected")
	}
	turnTimeout := 20 * time.Minute
	if v := os.Getenv("AGENT_EVAL_TURN_TIMEOUT"); v != "" {
		if d, err := time.ParseDuration(v); err == nil {
			turnTimeout = d
		}
	}
	c := &client{base: base, http: &http.Client{Timeout: 120 * time.Second}, t: t}
	// /api/settings 的形状是 {defaults, project, schema, user}；模型播在 user 作用域。
	var settings struct {
		User struct {
			Models struct {
				Chat struct {
					Model string `json:"model"`
				} `json:"chat"`
			} `json:"models"`
		} `json:"user"`
	}
	_ = c.do("GET", "/api/settings", nil, &settings)
	c.model = settings.User.Models.Chat.Model

	proj := struct {
		ID string `json:"id"`
	}{}
	if err := c.do("POST", "/api/projects", map[string]string{"name": "eval-" + time.Now().Format("0102-150405")}, &proj); err != nil {
		t.Fatalf("create project: %v", err)
	}

	rep := report{Target: base, Model: c.model, GeneratedAt: time.Now().UTC().Format(time.RFC3339)}
	for _, tk := range tasks {
		rep.Tasks = append(rep.Tasks, runTask(t, c, proj.ID, tk, materialsDir, turnTimeout))
	}
	writeReport(t, &rep)
}

type turnResult struct {
	User       string        `json:"user"`
	RunID      string        `json:"runId"`
	Status     string        `json:"status"`
	FinalText  string        `json:"finalText,omitempty"`
	Usage      *run          `json:"usage,omitempty"`
	Metrics    turnMetrics   `json:"metrics"`
	Checks     []checkRecord `json:"checks,omitempty"`
	Artifacts  []string      `json:"artifacts,omitempty"`
	Issues     []string      `json:"issues,omitempty"`
	DurationMs int64         `json:"durationMs"`
}

type turnMetrics struct {
	ToolCalls      int `json:"toolCalls"`
	ToolFailures   int `json:"toolFailures"`
	FullRegens     int `json:"fullRegens"`
	APIDiscovery   int `json:"apiDiscovery"`
	ReadsOriginal  int `json:"readsOriginal"`
	PlanUpdates    int `json:"planUpdates"`
	VerifierChecks int `json:"verifierChecks"`
}

type checkRecord struct {
	Passed bool   `json:"passed"`
	Reason string `json:"reason"`
}

type taskResult struct {
	ID              string       `json:"id"`
	Turns           []turnResult `json:"turns"`
	Total           turnMetrics  `json:"total"`
	UnverifiedFacts []string     `json:"unverifiedFacts,omitempty"`
	// UnverifiedUnmarkedFacts：无出处且未按「目标/预计/假设」标注的子集
	// （E3 口径：严格口径 unverifiedFacts 保留原义以维持历史可比性）。
	UnverifiedUnmarkedFacts []string `json:"unverifiedUnmarkedFacts,omitempty"`
	FinalVsArtifact []string     `json:"finalVsArtifact,omitempty"`
}

type report struct {
	Target      string       `json:"target"`
	Model       string       `json:"model,omitempty"`
	GeneratedAt string       `json:"generatedAt"`
	Tasks       []taskResult `json:"tasks"`
}

func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir string, turnTimeout time.Duration) taskResult {
	t.Helper()
	conv := struct {
		ID string `json:"id"`
	}{}
	if err := c.do("POST", "/api/projects/"+projectID+"/conversations", map[string]string{"title": "eval-" + tk.ID}, &conv); err != nil {
		t.Fatalf("[%s] create conversation: %v", tk.ID, err)
	}
	var attachmentIDs []string
	materialText := ""
	if tk.Material != "" {
		path := filepath.Join(materialsDir, tk.Material)
		raw, err := os.ReadFile(path)
		if err != nil {
			t.Fatalf("[%s] open material: %v", tk.ID, err)
		}
		up := c.upload(t, projectID, tk.Material, raw)
		attachmentIDs = append(attachmentIDs, up)
		materialText = docxTextIfDocx(t, tk.Material, raw)
	}
	tr := taskResult{ID: tk.ID}
	for i, turnText := range tk.Turns {
		tr.Turns = append(tr.Turns, runTurn(t, c, projectID, conv.ID, turnText, attachmentIDs, turnTimeout))
		attachmentIDs = nil // 材料只在第一轮附带；后续轮考验跨轮上下文
		_ = i
	}
	// 汇总
	for _, turn := range tr.Turns {
		tr.Total.ToolCalls += turn.Metrics.ToolCalls
		tr.Total.ToolFailures += turn.Metrics.ToolFailures
		tr.Total.FullRegens += turn.Metrics.FullRegens
		tr.Total.APIDiscovery += turn.Metrics.APIDiscovery
		tr.Total.ReadsOriginal += turn.Metrics.ReadsOriginal
		tr.Total.PlanUpdates += turn.Metrics.PlanUpdates
		tr.Total.VerifierChecks += turn.Metrics.VerifierChecks
	}
	// 事实核查：最后一个带工件的回合 vs 材料
	if materialText != "" {
		last := lastArtifactTurn(tr)
		if last != nil && len(last.Artifacts) > 0 {
			artText := ""
			for _, name := range last.Artifacts {
				artText += downloadArtifactText(t, c, projectID, name)
			}
			tr.UnverifiedFacts = unverifiedFacts(artText, materialText)
			tr.UnverifiedUnmarkedFacts = unmarkedFacts(tr.UnverifiedFacts, artText)
			tr.FinalVsArtifact = finalVsArtifact(last, artText)
		}
	}
	return tr
}