Files
Agent-System-Backend/internal/quality/score.go
T
weichenandCursor a3ae1b986d Check final replies before the direct run stops
Why: A direct run could stop on a missing file, a failed lint, or an unverified claim. The final turn now checks registered artifacts and asks a tool-less verifier, and nudges stale plans and todos.
Evidence: go test ./... passed. New runtime tests cover missing and empty artifacts, docx lint errors, verifier reject-then-pass, unparseable and failed verifier calls, nil verifier tools, todo reminders, and the false-deliverable fixture.
Impact: Direct runs with runtime.verify (default on for new runs) spend one extra lightweight call on every final turn. Setting it off keeps the previous stop behavior. Squad and SubLLMRunner are unchanged. The agent-native pin stays a34689c.
Refs: feature/agent-harness-p2
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-01 13:54:15 +00:00

312 lines
8.9 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package quality 给交付物打分。直通路径的 quality_review 和小队 Checker 用同一套规则。
// 默认是可复现的启发式评分;调用方可以注入 Judge,用轻量模型覆盖启发式结果。
package quality
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"encoding/xml"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"unicode/utf8"
)
// Dimension 是评分卡上的一维。Score 为 0–100。Evidence 必须引用被评文本里的原句。
type Dimension struct {
ID string `json:"id"`
Score int `json:"score"`
Note string `json:"note"`
Evidence string `json:"evidence"`
}
// Report 是一次评分。Passed 要求总分不低于 70,且没有任何一维低于 40。
type Report struct {
Passed bool `json:"passed"`
Score int `json:"score"`
Dimensions []Dimension `json:"dimensions"`
Violations []string `json:"violations,omitempty"`
Lint []LintIssue `json:"lint,omitempty"`
}
// LintIssue 是 housestyle.lint 的一条结果。lint 不存在时留空,不因此判失败。
type LintIssue struct {
Severity string `json:"severity"`
Rule string `json:"rule"`
Message string `json:"message"`
}
// Judge 用模型给同一份正文打分。返回错误时调用方回落启发式。
type Judge func(ctx context.Context, text string) (Report, error)
// ScoreText 按六个维度给纯文本打分。
func ScoreText(text string) Report {
text = strings.TrimSpace(text)
dims := []Dimension{
scoreLead(text),
scoreStructure(text),
scoreComplete(text),
scoreEvidence(text),
scoreActionable(text),
scoreReadable(text),
}
total := 0
low := false
var violations []string
for _, d := range dims {
total += d.Score
if d.Score < 40 {
low = true
violations = append(violations, d.ID+":"+d.Note)
}
}
score := 0
if len(dims) > 0 {
score = total / len(dims)
}
if score < 70 {
violations = append(violations, fmt.Sprintf("总分 %d,低于 70", score))
}
return Report{
Passed: score >= 70 && !low,
Score: score,
Dimensions: dims,
Violations: violations,
}
}
// ScoreFile 读取文件正文再打分。docx 取段落文本;其他文件按 UTF-8 文本读。
// 能跑 housestyle.lint 时,error 级问题会使 Passed 变为 false。
func ScoreFile(path string) Report {
text, err := readDeliverable(path)
if err != nil {
return Report{Passed: false, Violations: []string{"无法读取交付物:" + err.Error()}}
}
report := ScoreText(text)
issues := lintFile(path)
report.Lint = issues
for _, issue := range issues {
if issue.Severity == "error" {
report.Passed = false
report.Violations = append(report.Violations, "版式 "+issue.Rule+":"+issue.Message)
}
}
return report
}
// ScoreBytes 把内容落到带扩展名的临时文件,再走 ScoreFile。
// docx 与 housestyle.lint 因此和按路径评分用同一条路径。
func ScoreBytes(name string, content []byte) Report {
ext := filepath.Ext(name)
if ext == "" {
ext = ".txt"
}
f, err := os.CreateTemp("", "score-*"+ext)
if err != nil {
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
}
path := f.Name()
defer os.Remove(path)
if _, err := f.Write(content); err != nil {
f.Close()
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
}
if err := f.Close(); err != nil {
return Report{Passed: false, Violations: []string{"无法写入临时交付物:" + err.Error()}}
}
return ScoreFile(path)
}
func scoreLead(text string) Dimension {
title := firstHeading(text)
if title == "" {
return Dimension{ID: "结论先行", Score: 30, Note: "开头没有标题或判断", Evidence: snippet(text, 40)}
}
if bannedTitle(title) {
return Dimension{ID: "结论先行", Score: 20, Note: "标题是栏目名,不是判断", Evidence: title}
}
if containsAny(title, "应", "先", "不要", "建议", "必须", "改为", "停止") || utf8.RuneCountInString(title) >= 8 {
return Dimension{ID: "结论先行", Score: 90, Note: "标题本身是判断", Evidence: title}
}
return Dimension{ID: "结论先行", Score: 55, Note: "有标题,但还不是一句判断", Evidence: title}
}
func scoreStructure(text string) Dimension {
if strings.Contains(text, "一句话") {
return Dimension{ID: "结构", Score: 90, Note: "要点组有收束句", Evidence: evidenceLine(text, "一句话")}
}
if strings.Count(text, "#") >= 2 || strings.Count(text, "\n") >= 4 {
return Dimension{ID: "结构", Score: 60, Note: "有分层,但没有一句话收束", Evidence: snippet(text, 40)}
}
return Dimension{ID: "结构", Score: 30, Note: "几乎没有结构", Evidence: snippet(text, 40)}
}
func scoreComplete(text string) Dimension {
n := utf8.RuneCountInString(text)
switch {
case n >= 180:
return Dimension{ID: "完整性", Score: 85, Note: "篇幅足以覆盖一个决定", Evidence: fmt.Sprintf("%d 字", n)}
case n >= 80:
return Dimension{ID: "完整性", Score: 60, Note: "偏短,可能漏了材料里的决定", Evidence: fmt.Sprintf("%d 字", n)}
default:
return Dimension{ID: "完整性", Score: 25, Note: "内容过短", Evidence: fmt.Sprintf("%d 字", n)}
}
}
func scoreEvidence(text string) Dimension {
for _, word := range []string{"说话人", "依据", "出处", "原文", "文件"} {
if line := evidenceLine(text, word); line != "" {
return Dimension{ID: "证据", Score: 85, Note: "判断能回到材料", Evidence: line}
}
}
return Dimension{ID: "证据", Score: 30, Note: "没有出处", Evidence: snippet(text, 40)}
}
func scoreActionable(text string) Dimension {
for _, word := range []string{"未指明", "负责人", "期限", "之前完成", "周五"} {
if line := evidenceLine(text, word); line != "" {
return Dimension{ID: "可执行性", Score: 85, Note: "行动写了人或时间", Evidence: line}
}
}
return Dimension{ID: "可执行性", Score: 35, Note: "没有可交给人的行动", Evidence: snippet(text, 40)}
}
func scoreReadable(text string) Dimension {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(strings.TrimLeft(line, "#"))
if bannedTitle(line) && utf8.RuneCountInString(line) <= 12 {
return Dimension{ID: "可读性", Score: 25, Note: "使用了栏目式标题", Evidence: line}
}
}
if strings.Contains(text, "。。") {
return Dimension{ID: "可读性", Score: 40, Note: "有连续句号", Evidence: "。。"}
}
return Dimension{ID: "可读性", Score: 85, Note: "标题可扫读", Evidence: firstHeading(text)}
}
func firstHeading(text string) string {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(strings.TrimLeft(line, "#"))
line = strings.TrimSpace(line)
if line != "" {
return line
}
}
return ""
}
func bannedTitle(title string) bool {
for _, word := range []string{"讨论过程", "概述", "补充说明", "其他"} {
if strings.Contains(title, word) {
return true
}
}
return false
}
func containsAny(s string, words ...string) bool {
for _, word := range words {
if strings.Contains(s, word) {
return true
}
}
return false
}
func evidenceLine(text, word string) string {
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(line)
if strings.Contains(line, word) {
return snippet(line, 80)
}
}
return ""
}
func snippet(text string, n int) string {
text = strings.TrimSpace(strings.ReplaceAll(text, "\n", " "))
if utf8.RuneCountInString(text) <= n {
return text
}
runes := []rune(text)
return string(runes[:n]) + "…"
}
func readDeliverable(path string) (string, error) {
if strings.EqualFold(filepath.Ext(path), ".docx") {
return readDocxText(path)
}
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
if !utf8.Valid(data) {
return "", fmt.Errorf("不是文本文件")
}
return string(data), nil
}
func readDocxText(path string) (string, error) {
zr, err := zip.OpenReader(path)
if err != nil {
return "", err
}
defer zr.Close()
var doc *zip.File
for _, f := range zr.File {
if f.Name == "word/document.xml" {
doc = f
break
}
}
if doc == nil {
return "", fmt.Errorf("docx 没有 document.xml")
}
rc, err := doc.Open()
if err != nil {
return "", err
}
defer rc.Close()
decoder := xml.NewDecoder(rc)
var b strings.Builder
for {
tok, err := decoder.Token()
if err != nil {
break
}
switch el := tok.(type) {
case xml.StartElement:
if el.Name.Local == "p" && b.Len() > 0 {
b.WriteByte('\n')
}
case xml.CharData:
b.Write(bytes.TrimSpace(el))
}
}
return b.String(), nil
}
var lintJSON = regexp.MustCompile(`(?s)\[.*\]`)
func lintFile(path string) []LintIssue {
cmd := exec.Command("python3", "-m", "housestyle.lint", path)
out, err := cmd.CombinedOutput()
if err != nil && !bytes.Contains(out, []byte("\"rule\"")) {
return nil
}
raw := out
if loc := lintJSON.Find(out); loc != nil {
raw = loc
}
var issues []LintIssue
if json.Unmarshal(raw, &issues) != nil {
return nil
}
return issues
}