weichen
f1302ed5b3
Why: Allow consumers to download an immutable module revision without nested source checkouts or local path replacements. Evidence: SELF package tests and the prior canonical-path rejection are recorded in the Dev dependency-workspace task card; remote consumption is replayed against the committed candidate. Impact: Change the module identity and its Go imports while preserving API behavior. Existing historical submodule revisions remain buildable. Refs: Versioned dependencies U1/U3; feature/dependency-workspace
458 lines
12 KiB
Go
458 lines
12 KiB
Go
//go:build live
|
||
|
||
package live
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"encoding/json"
|
||
"encoding/xml"
|
||
"fmt"
|
||
"io"
|
||
"mime/multipart"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"git-ssh.u1680832.nyat.app/weichen/Agent-System-Backend.git/internal/runtime"
|
||
)
|
||
|
||
// upload 上传一个材料文件，返回 artifact id。
|
||
func (c *client) upload(t *testing.T, projectID, name string, content []byte) string {
|
||
t.Helper()
|
||
var buf bytes.Buffer
|
||
w := multipart.NewWriter(&buf)
|
||
fw, err := w.CreateFormFile("file", name)
|
||
if err != nil {
|
||
t.Fatalf("multipart field: %v", err)
|
||
}
|
||
if _, err := fw.Write(content); err != nil {
|
||
t.Fatalf("multipart write: %v", err)
|
||
}
|
||
w.Close()
|
||
req, err := http.NewRequest("POST", c.base+"/api/projects/"+projectID+"/uploads", &buf)
|
||
if err != nil {
|
||
t.Fatalf("upload request: %v", err)
|
||
}
|
||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||
resp, err := c.http.Do(req)
|
||
if err != nil {
|
||
t.Fatalf("upload: %v", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
if resp.StatusCode >= 300 {
|
||
t.Fatalf("upload %s: %s: %s", name, resp.Status, truncate(string(raw), 300))
|
||
}
|
||
var out struct {
|
||
ID string `json:"artifactId"`
|
||
LegacyID string `json:"id"`
|
||
}
|
||
if err := json.Unmarshal(raw, &out); err != nil {
|
||
t.Fatalf("upload response: %v %s", err, truncate(string(raw), 300))
|
||
}
|
||
if out.ID == "" {
|
||
out.ID = out.LegacyID
|
||
}
|
||
if out.ID == "" {
|
||
t.Fatalf("upload response missing artifact id: %s", truncate(string(raw), 300))
|
||
}
|
||
return out.ID
|
||
}
|
||
|
||
// runTurn 发一轮消息并等到 run 终态；期间自动应答 HITL 卡。
|
||
func runTurn(t *testing.T, c *client, projectID, convID, text string, attachmentIDs []string, turnTimeout time.Duration) turnResult {
|
||
t.Helper()
|
||
started := time.Now()
|
||
var out struct {
|
||
Message message `json:"message"`
|
||
Run run `json:"run"`
|
||
}
|
||
body := map[string]any{"text": text, "requestId": fmt.Sprintf("eval-%d", time.Now().UnixNano())}
|
||
if len(attachmentIDs) > 0 {
|
||
body["attachmentIds"] = attachmentIDs
|
||
}
|
||
if err := c.do("POST", "/api/conversations/"+convID+"/messages", body, &out); err != nil {
|
||
t.Fatalf("post message: %v", err)
|
||
}
|
||
runID := out.Run.ID
|
||
deadline := time.Now().Add(turnTimeout)
|
||
var cur run
|
||
for {
|
||
var resp struct {
|
||
Run run `json:"run"`
|
||
}
|
||
if err := c.do("GET", "/api/runs/"+runID, nil, &resp); err != nil {
|
||
t.Fatalf("get run: %v", err)
|
||
}
|
||
cur = resp.Run
|
||
if isTerminal(cur.Status) {
|
||
break
|
||
}
|
||
autoResolveHITL(t, c, convID)
|
||
if time.Now().After(deadline) {
|
||
t.Fatalf("run %s timed out after %s (status=%s)", runID, turnTimeout, cur.Status)
|
||
}
|
||
time.Sleep(3 * time.Second)
|
||
}
|
||
detail := struct {
|
||
Messages []message `json:"messages"`
|
||
ActiveHitl []hitlCard `json:"activeHitl"`
|
||
}{}
|
||
if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
|
||
t.Fatalf("conversation detail: %v", err)
|
||
}
|
||
var assistant *message
|
||
for i := range detail.Messages {
|
||
m := detail.Messages[i]
|
||
if m.RunID == runID && m.Role == "assistant" && !m.Excluded {
|
||
assistant = &detail.Messages[i]
|
||
}
|
||
}
|
||
tr := turnResult{User: text, RunID: runID, Status: cur.Status, DurationMs: time.Since(started).Milliseconds()}
|
||
if cur.Usage != nil {
|
||
tr.Usage = &cur
|
||
}
|
||
if assistant == nil {
|
||
tr.Issues = append(tr.Issues, "没有找到本回合的助手消息")
|
||
return tr
|
||
}
|
||
collectMetrics(&tr, assistant)
|
||
// 工件清单：本 run 产出的。
|
||
var arts []artifactMeta
|
||
if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err == nil {
|
||
for _, a := range arts {
|
||
if a.SourceRun == runID {
|
||
tr.Artifacts = append(tr.Artifacts, a.Name)
|
||
}
|
||
}
|
||
}
|
||
if cur.Status != "completed" {
|
||
tr.Issues = append(tr.Issues, "run 终态="+cur.Status)
|
||
}
|
||
return tr
|
||
}
|
||
|
||
func isTerminal(status string) bool {
|
||
switch status {
|
||
case "completed", "failed", "cancelled":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// autoResolveHITL 无人值守：有活动确认卡时选第一个选项（没有选项给自由文本同意）。
|
||
func autoResolveHITL(t *testing.T, c *client, convID string) {
|
||
t.Helper()
|
||
detail := struct {
|
||
ActiveHitl []hitlCard `json:"activeHitl"`
|
||
}{}
|
||
if err := c.do("GET", "/api/conversations/"+convID, nil, &detail); err != nil {
|
||
return
|
||
}
|
||
for _, card := range detail.ActiveHitl {
|
||
resp := map[string]string{}
|
||
if len(card.Choices) > 0 {
|
||
resp["choiceId"] = card.Choices[0].ID
|
||
} else if card.AllowFreeInput {
|
||
resp["freeText"] = "同意继续（评测自动应答）"
|
||
} else {
|
||
continue
|
||
}
|
||
_ = c.do("POST", "/api/hitl/"+card.ID+"/resolve", resp, nil)
|
||
}
|
||
}
|
||
|
||
func collectMetrics(tr *turnResult, m *message) {
|
||
for _, p := range m.Parts {
|
||
switch p.Type {
|
||
case "text":
|
||
if !p.Process && strings.TrimSpace(p.Text) != "" {
|
||
tr.FinalText = p.Text
|
||
}
|
||
case "tool_call":
|
||
if p.ToolCall == nil {
|
||
continue
|
||
}
|
||
tr.Metrics.ToolCalls++
|
||
args := p.ToolCall.Arguments
|
||
switch p.ToolCall.Name {
|
||
case "update_plan", "update_todos":
|
||
tr.Metrics.PlanUpdates++
|
||
case "run_python":
|
||
if strings.Contains(args, "new_document(") || strings.Contains(args, "new_presentation(") || strings.Contains(args, "new_workbook(") {
|
||
tr.Metrics.FullRegens++
|
||
}
|
||
}
|
||
if strings.Contains(args, "dir(") || strings.Contains(args, "inspect.signature") || strings.Contains(args, "help(") {
|
||
tr.Metrics.APIDiscovery++
|
||
}
|
||
if strings.Contains(args, "inputs/") {
|
||
tr.Metrics.ReadsOriginal++
|
||
}
|
||
case "tool_result":
|
||
if p.ToolResult == nil {
|
||
continue
|
||
}
|
||
s := p.ToolResult.Summary
|
||
if p.ToolResult.Status == "error" || strings.HasPrefix(s, "exit=1") || strings.Contains(s, "Traceback") || strings.Contains(s, "ModuleNotFoundError") {
|
||
tr.Metrics.ToolFailures++
|
||
}
|
||
case "check":
|
||
if p.Check == nil {
|
||
continue
|
||
}
|
||
tr.Metrics.VerifierChecks++
|
||
tr.Checks = append(tr.Checks, checkRecord{Passed: p.Check.Passed, Reason: p.Check.Reason})
|
||
}
|
||
}
|
||
}
|
||
|
||
func lastArtifactTurn(tr taskResult) *turnResult {
|
||
for i := len(tr.Turns) - 1; i >= 0; i-- {
|
||
if len(tr.Turns[i].Artifacts) > 0 {
|
||
return &tr.Turns[i]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func downloadArtifactText(t *testing.T, c *client, projectID, name string) string {
|
||
t.Helper()
|
||
var arts []artifactMeta
|
||
if err := c.do("GET", "/api/projects/"+projectID+"/artifacts", nil, &arts); err != nil {
|
||
return ""
|
||
}
|
||
for _, a := range arts {
|
||
if a.Name != name {
|
||
continue
|
||
}
|
||
req, _ := http.NewRequest("GET", c.base+"/api/artifacts/"+a.ID+"/content", nil)
|
||
resp, err := c.http.Do(req)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
resp.Body.Close()
|
||
if strings.HasSuffix(strings.ToLower(a.Name), ".docx") {
|
||
return docxTextIfDocx(t, a.Name, raw)
|
||
}
|
||
return string(raw)
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// ---- 文本抽取与事实核查 ----
|
||
|
||
// docxTextIfDocx：docx 抽纯文本（标题行带 # 前缀）；其他类型原样返回。
|
||
func docxTextIfDocx(t *testing.T, name string, raw []byte) string {
|
||
t.Helper()
|
||
if !strings.HasSuffix(strings.ToLower(name), ".docx") {
|
||
return string(raw)
|
||
}
|
||
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, f := range zr.File {
|
||
if f.Name != "word/document.xml" {
|
||
continue
|
||
}
|
||
rc, err := f.Open()
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
defer rc.Close()
|
||
dec := xml.NewDecoder(rc)
|
||
var b strings.Builder
|
||
var para strings.Builder
|
||
style, inPara := "", false
|
||
flush := func() {
|
||
text := strings.TrimSpace(para.String())
|
||
if text != "" {
|
||
if lvl := docxHeadingLevel(style); lvl > 0 {
|
||
b.WriteString(strings.Repeat("#", lvl) + " ")
|
||
}
|
||
b.WriteString(text + "\n")
|
||
}
|
||
para.Reset()
|
||
style = ""
|
||
}
|
||
for {
|
||
tok, err := dec.Token()
|
||
if err != nil {
|
||
break
|
||
}
|
||
switch el := tok.(type) {
|
||
case xml.StartElement:
|
||
switch el.Name.Local {
|
||
case "p":
|
||
inPara, style = true, ""
|
||
para.Reset()
|
||
case "pStyle":
|
||
if inPara {
|
||
for _, at := range el.Attr {
|
||
if at.Name.Local == "val" {
|
||
style = at.Value
|
||
}
|
||
}
|
||
}
|
||
}
|
||
case xml.EndElement:
|
||
if el.Name.Local == "p" {
|
||
inPara = false
|
||
flush()
|
||
}
|
||
case xml.CharData:
|
||
if inPara {
|
||
para.Write(bytes.TrimSpace(el))
|
||
}
|
||
}
|
||
}
|
||
flush()
|
||
return b.String()
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func docxHeadingLevel(style string) int {
|
||
switch {
|
||
case style == "Title":
|
||
return 1
|
||
case style == "Subtitle":
|
||
return 2
|
||
case strings.HasPrefix(style, "Heading"):
|
||
n := 0
|
||
for _, ch := range style[len("Heading"):] {
|
||
if ch < '0' || ch > '9' {
|
||
return 0
|
||
}
|
||
n = n*10 + int(ch-'0')
|
||
}
|
||
if n >= 1 && n <= 9 {
|
||
return n
|
||
}
|
||
}
|
||
return 0
|
||
}
|
||
|
||
var (
|
||
pctRe = regexp.MustCompile(`\d+(?:\.\d+)?%`)
|
||
dateRe = regexp.MustCompile(`\d{1,2}月\d{1,2}日`)
|
||
bigNumRe = regexp.MustCompile(`\d{4,}`)
|
||
claimRe = regexp.MustCompile(`(?:^|\n|\s)([一二三四五六七八九十]{1,2}、[^\s，。；|]{2,24})`)
|
||
)
|
||
|
||
// unverifiedFacts 抽取交付物里的百分比/日期/大数字，逐个回材料原文找出处。
|
||
// 找不到的列为未核实事实（模型编造或来自模型自带的「常识」）。
|
||
func unverifiedFacts(artifactText, materialText string) []string {
|
||
if strings.TrimSpace(artifactText) == "" || strings.TrimSpace(materialText) == "" {
|
||
return nil
|
||
}
|
||
seen := map[string]bool{}
|
||
var out []string
|
||
check := func(tok string) {
|
||
tok = strings.TrimSpace(tok)
|
||
if tok == "" || seen[tok] {
|
||
return
|
||
}
|
||
seen[tok] = true
|
||
if !strings.Contains(materialText, tok) {
|
||
out = append(out, tok)
|
||
}
|
||
}
|
||
for _, m := range pctRe.FindAllString(artifactText, -1) {
|
||
check(m)
|
||
}
|
||
for _, m := range dateRe.FindAllString(artifactText, -1) {
|
||
check(m)
|
||
}
|
||
for _, m := range bigNumRe.FindAllString(artifactText, -1) {
|
||
if len(m) >= 5 { // 4 位数大量是年份/编号，只报 5 位以上
|
||
check(m)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// unmarkedFacts 从未核实清单里剔除已按「目标/预计/假设」标注的（与后端
|
||
// runtime.MarkedAsEstimate 同一口径，标注词邻域 ±32 字符）。
|
||
func unmarkedFacts(tokens []string, artifactText string) []string {
|
||
var out []string
|
||
for _, tok := range tokens {
|
||
if !runtime.MarkedAsEstimate(artifactText, tok) {
|
||
out = append(out, tok)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// finalVsArtifact 终轮回复里声称的章节（「一、…」）必须能在工件里找到。
|
||
func finalVsArtifact(turn *turnResult, artifactText string) []string {
|
||
if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" {
|
||
return nil
|
||
}
|
||
final := turn.FinalText
|
||
var out []string
|
||
seen := map[string]bool{}
|
||
for _, m := range claimRe.FindAllStringSubmatch(final, -1) {
|
||
claim := strings.TrimSpace(m[1])
|
||
if claim == "" || seen[claim] {
|
||
continue
|
||
}
|
||
seen[claim] = true
|
||
if !strings.Contains(artifactText, claim) {
|
||
out = append(out, claim)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ---- 报告 ----
|
||
|
||
func writeReport(t *testing.T, rep *report) {
|
||
t.Helper()
|
||
outPath := os.Getenv("AGENT_EVAL_OUT")
|
||
if outPath == "" {
|
||
outPath = filepath.Join("eval-live", "report-"+time.Now().Format("0102-150405")+".json")
|
||
}
|
||
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
|
||
t.Fatalf("mkdir report dir: %v", err)
|
||
}
|
||
raw, _ := json.MarshalIndent(rep, "", " ")
|
||
if err := os.WriteFile(outPath, raw, 0o644); err != nil {
|
||
t.Fatalf("write report: %v", err)
|
||
}
|
||
fmt.Fprintf(os.Stderr, "\n[live-eval] report: %s\n%s\n", outPath, renderMarkdown(rep))
|
||
}
|
||
|
||
func renderMarkdown(rep *report) string {
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, "# Live eval — %s\n\n- target: %s\n- model: %s\n- generated: %s\n\n", rep.GeneratedAt, rep.Target, rep.Model, rep.GeneratedAt)
|
||
for _, tk := range rep.Tasks {
|
||
fmt.Fprintf(&b, "## %s\n\n| turn | status | tools | fails | regens | api探查 | 读原件 | 计划 | 检查 | 工件 | 时长s |\n|---|---|---|---|---|---|---|---|---|---|---|\n", tk.ID)
|
||
for _, turn := range tk.Turns {
|
||
fmt.Fprintf(&b, "| %s | %s | %d | %d | %d | %d | %d | %d | %d | %d | %.0f |\n",
|
||
truncate(turn.User, 18), turn.Status, turn.Metrics.ToolCalls, turn.Metrics.ToolFailures,
|
||
turn.Metrics.FullRegens, turn.Metrics.APIDiscovery, turn.Metrics.ReadsOriginal,
|
||
turn.Metrics.PlanUpdates, turn.Metrics.VerifierChecks, len(turn.Artifacts),
|
||
float64(turn.DurationMs)/1000)
|
||
}
|
||
if len(tk.UnverifiedFacts) > 0 {
|
||
fmt.Fprintf(&b, "\n未核实事实（工件里出现、材料里找不到）：%s\n", strings.Join(tk.UnverifiedFacts, "、"))
|
||
if len(tk.UnverifiedUnmarkedFacts) > 0 {
|
||
fmt.Fprintf(&b, "其中未标注（无出处且无目标/预计/假设标注，E3 闸门口径）：%s\n", strings.Join(tk.UnverifiedUnmarkedFacts, "、"))
|
||
}
|
||
}
|
||
if len(tk.FinalVsArtifact) > 0 {
|
||
fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节：%s\n", strings.Join(tk.FinalVsArtifact, "；"))
|
||
}
|
||
fmt.Fprint(&b, "\n")
|
||
}
|
||
return b.String()
|
||
}
|