Agent-System-Backend/eval/live/helpers_test.go

//go:build live

package live

import (
	"archive/zip"
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
	"time"

	"agent-system-backend/internal/runtime"
)

// upload 上传一个材料文件，返回 artifact id。
func (c *client) upload(t *testing.T, projectID, name string, content []byte) string {
	t.Helper()
	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)
	fw, err := w.CreateFormFile("file", name)
	if err != nil {
		t.Fatalf("multipart field: %v", err)
	}
	if _, err := fw.Write(content); err != nil {
		t.Fatalf("multipart write: %v", err)
	}
	w.Close()
	req, err := http.NewRequest("POST", c.base+"/api/projects/"+projectID+"/uploads", &buf)
	if err != nil {
		t.Fatalf("upload request: %v", err)
	}
	req.Header.Set("Content-Type", w.FormDataContentType())
	resp, err := c.http.Do(req)
	if err != nil {
		t.Fatalf("upload: %v", err)
	}
	defer resp.Body.Close()
	raw, _ := io.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		t.Fatalf("upload %s: %s: %s", name, resp.Status, truncate(string(raw), 300))
	}
	var out struct {
		ID       string `json:"artifactId"`
		LegacyID string `json:"id"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		t.Fatalf("upload response: %v %s", err, truncate(string(raw), 300))
	}
	if out.ID == "" {
		out.ID = out.LegacyID
	}
	if out.ID == "" {
		t.Fatalf("upload response missing artifact id: %s", truncate(string(raw), 300))
	}
	return out.ID
}

// runTurn 发一轮消息并等到 run 终态；期间自动应答 HITL 卡。
func runTurn(t *testing.T, c *client, projectID, convID, text string, attachmentIDs []string, turnTimeout time.Duration) turnResult {
	t.Helper()
	started := time.Now()
	var out struct {
		Message message `json:"message"`
		Run     run     `json:"run"`
	}
	body := map[string]any{"text": text, "requestId": fmt.Sprintf("eval-%d", time.Now().UnixNano())}
	if len(attachmentIDs) > 0 {
		body["attachmentIds"] = attachmentIDs
	}
	if err := c.do("POST", "/api/conversations/"+convID+"/messages", body, &out); err != nil {
		t.Fatalf("post message: %v", err)
	}
	runID := out.Run.ID
	deadline := time.Now().Add(turnTimeout)
	var cur run
	for {
		var resp struct {
			Run run `json:"run"`
		}
		if err := c.do("GET", "/api/runs/"+runID, nil, &resp); err != nil {
			t.Fatalf("get run: %v", err)
		}
		cur = resp.Run
		if isTerminal(cur.Status) {
			break
		}
		autoResolveHITL(t, c, convID)
		if time.Now().After(deadline) {
			t.Fatalf("run %s timed out after %s (status=%s)", runID, turnTimeout, cur.Status)
		}
		time.Sleep(3 * time.Second)
	}
	detail := struct {
		Messages   []message  `json:"messages"`
		ActiveHitl []hitlCard `json:"activeHitl"`
	}{}
	if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
		t.Fatalf("conversation detail: %v", err)
	}
	var assistant *message
	for i := range detail.Messages {
		m := detail.Messages[i]
		if m.RunID == runID && m.Role == "assistant" && !m.Excluded {
			assistant = &detail.Messages[i]
		}
	}
	tr := turnResult{User: text, RunID: runID, Status: cur.Status, DurationMs: time.Since(started).Milliseconds()}
	if cur.Usage != nil {
		tr.Usage = &cur
	}
	if assistant == nil {
		tr.Issues = append(tr.Issues, "没有找到本回合的助手消息")
		return tr
	}
	collectMetrics(&tr, assistant)
	// 工件清单：本 run 产出的。
	var arts []artifactMeta
	if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err == nil {
		for _, a := range arts {
			if a.SourceRun == runID {
				tr.Artifacts = append(tr.Artifacts, a.Name)
			}
		}
	}
	if cur.Status != "completed" {
		tr.Issues = append(tr.Issues, "run 终态="+cur.Status)
	}
	return tr
}

func isTerminal(status string) bool {
	switch status {
	case "completed", "failed", "cancelled":
		return true
	}
	return false
}

// autoResolveHITL 无人值守：有活动确认卡时选第一个选项（没有选项给自由文本同意）。
func autoResolveHITL(t *testing.T, c *client, convID string) {
	t.Helper()
	detail := struct {
		ActiveHitl []hitlCard `json:"activeHitl"`
	}{}
	if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
		return
	}
	for _, card := range detail.ActiveHitl {
		resp := map[string]string{}
		if len(card.Choices) > 0 {
			resp["choiceId"] = card.Choices[0].ID
		} else if card.AllowFreeInput {
			resp["freeText"] = "同意继续（评测自动应答）"
		} else {
			continue
		}
		_ = c.do("POST", "/api/hitl/"+card.ID+"/resolve", resp, nil)
	}
}

func collectMetrics(tr *turnResult, m *message) {
	for _, p := range m.Parts {
		switch p.Type {
		case "text":
			if !p.Process && strings.TrimSpace(p.Text) != "" {
				tr.FinalText = p.Text
			}
		case "tool_call":
			if p.ToolCall == nil {
				continue
			}
			tr.Metrics.ToolCalls++
			args := p.ToolCall.Arguments
			switch p.ToolCall.Name {
			case "update_plan", "update_todos":
				tr.Metrics.PlanUpdates++
			case "run_python":
				if strings.Contains(args, "new_document(") || strings.Contains(args, "new_presentation(") || strings.Contains(args, "new_workbook(") {
					tr.Metrics.FullRegens++
				}
			}
			if strings.Contains(args, "dir(") || strings.Contains(args, "inspect.signature") || strings.Contains(args, "help(") {
				tr.Metrics.APIDiscovery++
			}
			if strings.Contains(args, "inputs/") {
				tr.Metrics.ReadsOriginal++
			}
		case "tool_result":
			if p.ToolResult == nil {
				continue
			}
			s := p.ToolResult.Summary
			if p.ToolResult.Status == "error" || strings.HasPrefix(s, "exit=1") || strings.Contains(s, "Traceback") || strings.Contains(s, "ModuleNotFoundError") {
				tr.Metrics.ToolFailures++
			}
		case "check":
			if p.Check == nil {
				continue
			}
			tr.Metrics.VerifierChecks++
			tr.Checks = append(tr.Checks, checkRecord{Passed: p.Check.Passed, Reason: p.Check.Reason})
		}
	}
}

func lastArtifactTurn(tr taskResult) *turnResult {
	for i := len(tr.Turns) - 1; i >= 0; i-- {
		if len(tr.Turns[i].Artifacts) > 0 {
			return &tr.Turns[i]
		}
	}
	return nil
}

func downloadArtifactText(t *testing.T, c *client, projectID, name string) string {
	t.Helper()
	var arts []artifactMeta
	if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err != nil {
		return ""
	}
	for _, a := range arts {
		if a.Name != name {
			continue
		}
		req, _ := http.NewRequest("GET", c.base+"/api/artifacts/"+a.ID+"/content", nil)
		resp, err := c.http.Do(req)
		if err != nil {
			continue
		}
		raw, _ := io.ReadAll(resp.Body)
		resp.Body.Close()
		if strings.HasSuffix(strings.ToLower(a.Name), ".docx") {
			return docxTextIfDocx(t, a.Name, raw)
		}
		return string(raw)
	}
	return ""
}

// ---- 文本抽取与事实核查 ----

// docxTextIfDocx：docx 抽纯文本（标题行带 # 前缀）；其他类型原样返回。
func docxTextIfDocx(t *testing.T, name string, raw []byte) string {
	t.Helper()
	if !strings.HasSuffix(strings.ToLower(name), ".docx") {
		return string(raw)
	}
	zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
	if err != nil {
		return ""
	}
	for _, f := range zr.File {
		if f.Name != "word/document.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			return ""
		}
		defer rc.Close()
		dec := xml.NewDecoder(rc)
		var b strings.Builder
		var para strings.Builder
		style, inPara := "", false
		flush := func() {
			text := strings.TrimSpace(para.String())
			if text != "" {
				if lvl := docxHeadingLevel(style); lvl > 0 {
					b.WriteString(strings.Repeat("#", lvl) + " ")
				}
				b.WriteString(text + "\n")
			}
			para.Reset()
			style = ""
		}
		for {
			tok, err := dec.Token()
			if err != nil {
				break
			}
			switch el := tok.(type) {
			case xml.StartElement:
				switch el.Name.Local {
				case "p":
					inPara, style = true, ""
					para.Reset()
				case "pStyle":
					if inPara {
						for _, at := range el.Attr {
							if at.Name.Local == "val" {
								style = at.Value
							}
						}
					}
				}
			case xml.EndElement:
				if el.Name.Local == "p" {
					inPara = false
					flush()
				}
			case xml.CharData:
				if inPara {
					para.Write(bytes.TrimSpace(el))
				}
			}
		}
		flush()
		return b.String()
	}
	return ""
}

func docxHeadingLevel(style string) int {
	switch {
	case style == "Title":
		return 1
	case style == "Subtitle":
		return 2
	case strings.HasPrefix(style, "Heading"):
		n := 0
		for _, ch := range style[len("Heading"):] {
			if ch < '0' || ch > '9' {
				return 0
			}
			n = n*10 + int(ch-'0')
		}
		if n >= 1 && n <= 9 {
			return n
		}
	}
	return 0
}

var (
	pctRe    = regexp.MustCompile(`\d+(?:\.\d+)?%`)
	dateRe   = regexp.MustCompile(`\d{1,2}月\d{1,2}日`)
	bigNumRe = regexp.MustCompile(`\d{4,}`)
	claimRe  = regexp.MustCompile(`(?:^|\n|\s)([一二三四五六七八九十]{1,2}、[^\s，。；|]{2,24})`)
)

// unverifiedFacts 抽取交付物里的百分比/日期/大数字，逐个回材料原文找出处。
// 找不到的列为未核实事实（模型编造或来自模型自带的「常识」）。
func unverifiedFacts(artifactText, materialText string) []string {
	if strings.TrimSpace(artifactText) == "" || strings.TrimSpace(materialText) == "" {
		return nil
	}
	seen := map[string]bool{}
	var out []string
	check := func(tok string) {
		tok = strings.TrimSpace(tok)
		if tok == "" || seen[tok] {
			return
		}
		seen[tok] = true
		if !strings.Contains(materialText, tok) {
			out = append(out, tok)
		}
	}
	for _, m := range pctRe.FindAllString(artifactText, -1) {
		check(m)
	}
	for _, m := range dateRe.FindAllString(artifactText, -1) {
		check(m)
	}
	for _, m := range bigNumRe.FindAllString(artifactText, -1) {
		if len(m) >= 5 { // 4 位数大量是年份/编号，只报 5 位以上
			check(m)
		}
	}
	return out
}

// unmarkedFacts 从未核实清单里剔除已按「目标/预计/假设」标注的（与后端
// runtime.MarkedAsEstimate 同一口径，标注词邻域 ±32 字符）。
func unmarkedFacts(tokens []string, artifactText string) []string {
	var out []string
	for _, tok := range tokens {
		if !runtime.MarkedAsEstimate(artifactText, tok) {
			out = append(out, tok)
		}
	}
	return out
}

// finalVsArtifact 终轮回复里声称的章节（「一、…」）必须能在工件里找到。
func finalVsArtifact(turn *turnResult, artifactText string) []string {
	if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" {
		return nil
	}
	final := turn.FinalText
	var out []string
	seen := map[string]bool{}
	for _, m := range claimRe.FindAllStringSubmatch(final, -1) {
		claim := strings.TrimSpace(m[1])
		if claim == "" || seen[claim] {
			continue
		}
		seen[claim] = true
		if !strings.Contains(artifactText, claim) {
			out = append(out, claim)
		}
	}
	return out
}

// ---- 报告 ----

func writeReport(t *testing.T, rep *report) {
	t.Helper()
	outPath := os.Getenv("AGENT_EVAL_OUT")
	if outPath == "" {
		outPath = filepath.Join("eval-live", "report-"+time.Now().Format("0102-150405")+".json")
	}
	if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
		t.Fatalf("mkdir report dir: %v", err)
	}
	raw, _ := json.MarshalIndent(rep, "", "  ")
	if err := os.WriteFile(outPath, raw, 0o644); err != nil {
		t.Fatalf("write report: %v", err)
	}
	fmt.Fprintf(os.Stderr, "\n[live-eval] report: %s\n%s\n", outPath, renderMarkdown(rep))
}

func renderMarkdown(rep *report) string {
	var b strings.Builder
	fmt.Fprintf(&b, "# Live eval — %s\n\n- target: %s\n- model: %s\n- generated: %s\n\n", rep.GeneratedAt, rep.Target, rep.Model, rep.GeneratedAt)
	for _, tk := range rep.Tasks {
		fmt.Fprintf(&b, "## %s\n\n| turn | status | tools | fails | regens | api探查 | 读原件 | 计划 | 检查 | 工件 | 时长s |\n|---|---|---|---|---|---|---|---|---|---|---|\n", tk.ID)
		for _, turn := range tk.Turns {
			fmt.Fprintf(&b, "| %s | %s | %d | %d | %d | %d | %d | %d | %d | %d | %.0f |\n",
				truncate(turn.User, 18), turn.Status, turn.Metrics.ToolCalls, turn.Metrics.ToolFailures,
				turn.Metrics.FullRegens, turn.Metrics.APIDiscovery, turn.Metrics.ReadsOriginal,
				turn.Metrics.PlanUpdates, turn.Metrics.VerifierChecks, len(turn.Artifacts),
				float64(turn.DurationMs)/1000)
		}
		if len(tk.UnverifiedFacts) > 0 {
			fmt.Fprintf(&b, "\n未核实事实（工件里出现、材料里找不到）：%s\n", strings.Join(tk.UnverifiedFacts, "、"))
		if len(tk.UnverifiedUnmarkedFacts) > 0 {
			fmt.Fprintf(&b, "其中未标注（无出处且无目标/预计/假设标注，E3 闸门口径）：%s\n", strings.Join(tk.UnverifiedUnmarkedFacts, "、"))
		}
		}
		if len(tk.FinalVsArtifact) > 0 {
			fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节：%s\n", strings.Join(tk.FinalVsArtifact, "；"))
		}
		fmt.Fprint(&b, "\n")
	}
	return b.String()
}