Agent-Intent/intent/corpus.go

package intent

import (
	"context"
	"fmt"
)

// CorpusCase 是评测用例：v1 单轮（input/expect_*）与 v2 多轮（turns）共用。
// v1 字段在 Turns 为空时生效，并自动等价于一个单 turn 用例（不含命令断言）。
type CorpusCase struct {
	Name               string            `json:"name"`
	Input              string            `json:"input,omitempty"`
	ExpectBranch       string            `json:"expect_branch,omitempty"`
	ExpectTop          string            `json:"expect_top,omitempty"`
	ExpectSlots        map[string]string `json:"expect_slots,omitempty"`
	ExpectMissingSlots []string          `json:"expect_missing_slots,omitempty"`
	Turns              []CorpusTurn      `json:"turns,omitempty"`
}

// CorpusTurn 是多轮用例中的一轮：输入与逐轮期望（分支/top/命令列表）。
type CorpusTurn struct {
	Input          string          `json:"input"`
	ExpectBranch   string          `json:"expect_branch,omitempty"`
	ExpectTop      string          `json:"expect_top,omitempty"`
	ExpectCommands []CommandExpect `json:"expect_commands,omitempty"`
}

// CommandExpect 是对一条命令的期望；只有写出的字段参与断言。
type CommandExpect struct {
	Kind       string            `json:"kind,omitempty"`
	ActionKind string            `json:"action_kind,omitempty"`
	ActionID   string            `json:"action_id,omitempty"`
	Slots      map[string]string `json:"slots,omitempty"`
	Name       string            `json:"name,omitempty"`
	Value      string            `json:"value,omitempty"`
	Missing    []string          `json:"missing,omitempty"`
	Question   string            `json:"question,omitempty"`
	Candidates []string          `json:"candidates,omitempty"`
}

// TurnResult 一轮的实际结果与差异。
type TurnResult struct {
	Index      int
	Input      string
	Branch     string
	TopID      string
	Commands   []Command
	Mismatches []string
}

// CaseResult 一个用例的全部轮次结果。Passed = 所有轮次无差异。
type CaseResult struct {
	Name  string
	Turns []TurnResult
}

// Passed 报告用例是否全部轮次通过。
func (c CaseResult) Passed() bool {
	for _, t := range c.Turns {
		if len(t.Mismatches) > 0 {
			return false
		}
	}
	return true
}

// chainedExpectation 模拟宿主的挂起预期链：上一轮 Top 有必填缺失或非法
// 槽位时，下一轮挂起该意图（与产品「带槽启动后追问」的行为对应）。
type chainedExpectation struct {
	next *PendingExpectation
}

func (c *chainedExpectation) Pending(ctx context.Context) *PendingExpectation {
	return c.next
}

// RunCorpusCase 执行一个用例的全部轮次（v1 用例自动包装为单轮）。
// 识别器按轮次构建在链式预期上，保证 SetSlot+Start 这类组合可在评测中到达。
func RunCorpusCase(ctx context.Context, snap *Snapshot, tc CorpusCase) CaseResult {
	turns := tc.Turns
	if len(turns) == 0 {
		turns = []CorpusTurn{{Input: tc.Input, ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
	}
	exp := &chainedExpectation{}
	rec := NewRecognizer(Options{}, exp, nil)
	out := CaseResult{Name: tc.Name}
	for i, turn := range turns {
		tr := TurnResult{Index: i, Input: turn.Input}
		res, err := rec.Recognize(ctx, turn.Input, snap)
		if err != nil {
			tr.Mismatches = append(tr.Mismatches, "识别错误："+err.Error())
			out.Turns = append(out.Turns, tr)
			break
		}
		tr.Branch = string(res.Branch)
		if res.Top != nil {
			tr.TopID = res.Top.ID
		}
		tr.Commands = res.Commands
		tr.Mismatches = matchTurn(turn, res)
		out.Turns = append(out.Turns, tr)
		exp.next = nil
		if res.Top != nil && (len(res.Top.MissingRequiredSlots) > 0 || len(res.Top.InvalidSlots) > 0) {
			exp.next = &PendingExpectation{IntentIDs: []string{res.Top.ID}, Reason: "eval chained"}
		}
	}
	return out
}

func matchTurn(turn CorpusTurn, res Result) []string {
	var mm []string
	if turn.ExpectBranch != "" && string(res.Branch) != turn.ExpectBranch {
		mm = append(mm, fmt.Sprintf("branch=%s(期望 %s)", res.Branch, turn.ExpectBranch))
	}
	if turn.ExpectTop != "" {
		got := ""
		if res.Top != nil {
			got = res.Top.ID
		}
		if got != turn.ExpectTop {
			mm = append(mm, fmt.Sprintf("top=%s(期望 %s)", got, turn.ExpectTop))
		}
	}
	if turn.ExpectCommands != nil {
		mm = append(mm, matchCommands(turn.ExpectCommands, res.Commands)...)
	}
	return mm
}

func matchCommands(want []CommandExpect, got []Command) []string {
	if len(want) != len(got) {
		return []string{fmt.Sprintf("commands 数量=%d(期望 %d)", len(got), len(want))}
	}
	var mm []string
	for i, w := range want {
		g := got[i]
		if w.Kind != "" && string(g.Kind) != w.Kind {
			mm = append(mm, fmt.Sprintf("cmd[%d].kind=%s(期望 %s)", i, g.Kind, w.Kind))
			continue
		}
		if w.ActionKind != "" && g.Action.Kind != w.ActionKind {
			mm = append(mm, fmt.Sprintf("cmd[%d].action.kind=%s(期望 %s)", i, g.Action.Kind, w.ActionKind))
		}
		if w.ActionID != "" && g.Action.ID != w.ActionID {
			mm = append(mm, fmt.Sprintf("cmd[%d].action.id=%s(期望 %s)", i, g.Action.ID, w.ActionID))
		}
		for k, v := range w.Slots {
			if g.Slots[k] != v {
				mm = append(mm, fmt.Sprintf("cmd[%d].slots[%s]=%q(期望 %q)", i, k, g.Slots[k], v))
			}
		}
		if w.Name != "" && g.Name != w.Name {
			mm = append(mm, fmt.Sprintf("cmd[%d].name=%s(期望 %s)", i, g.Name, w.Name))
		}
		if w.Value != "" && g.Value != w.Value {
			mm = append(mm, fmt.Sprintf("cmd[%d].value=%s(期望 %s)", i, g.Value, w.Value))
		}
		for _, m := range w.Missing {
			if !containsString(g.Missing, m) {
				mm = append(mm, fmt.Sprintf("cmd[%d].missing 缺 %s", i, m))
			}
		}
		if w.Question != "" && g.Question != w.Question {
			mm = append(mm, fmt.Sprintf("cmd[%d].question=%q(期望 %q)", i, g.Question, w.Question))
		}
		for _, c := range w.Candidates {
			if !containsString(g.Candidates, c) {
				mm = append(mm, fmt.Sprintf("cmd[%d].candidates 缺 %s", i, c))
			}
		}
	}
	return mm
}

func containsString(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}

// CorpusMetrics 评测指标（I2）：按意图标签（matched/top）计 TP/FP/FN。
// 期望非 matched（fallback/ambiguous）的轮次作负例、不进指标。
type CorpusMetrics struct {
	TP, FP, FN int
}

// Precision 精确率；分母为 0 时返回 (0, false)（未定义，由调用方决定展示）。
func (m CorpusMetrics) Precision() (float64, bool) {
	if m.TP+m.FP == 0 {
		return 0, false
	}
	return float64(m.TP) / float64(m.TP+m.FP), true
}

// Recall 召回率；分母为 0 时返回 (0, false)。
func (m CorpusMetrics) Recall() (float64, bool) {
	if m.TP+m.FN == 0 {
		return 0, false
	}
	return float64(m.TP) / float64(m.TP+m.FN), true
}

// MetricsOf 逐轮累计指标（I2）：
//   - TP：期望 matched/top=X 且实际 matched/top=X；
//   - FP：实际 matched/top=Y 但期望的标签不是 matched/top=Y
//     （期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同）；
//   - FN：期望 matched/top=X 但实际没拿到它（branch 不对或 top 不同）。
// 只有写了 expect_top 的轮次进入指标（意图识别通行口径）。
func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
	var m CorpusMetrics
	byName := map[string]CorpusCase{}
	for _, tc := range cases {
		byName[tc.Name] = tc
	}
	for _, res := range results {
		tc, ok := byName[res.Name]
		if !ok {
			continue
		}
		turns := tc.Turns
		if len(turns) == 0 {
			turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
		}
		for i, turn := range turns {
			if turn.ExpectTop == "" || i >= len(res.Turns) {
				continue
			}
			got := res.Turns[i]
			wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit"
			gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit"
			if gotMatched && got.TopID == turn.ExpectTop && wantMatched {
				m.TP++
				continue
			}
			if gotMatched && got.TopID != "" {
				// 错配同时是假阳（拿了错的）与假阴（期望的没拿到）。
				m.FP++
				if wantMatched {
					m.FN++
				}
				continue
			}
			if wantMatched {
				m.FN++
			}
		}
	}
	return m
}