Add precision and recall metrics to the multi-turn evaluation
Why: I2 acceptance - the evaluator must emit metrics over a multiturn fixture no smaller than the existing corpus, not just print branch names. Evidence: go vet clean; full suite ok; intentctl eval prints precision=1.000 recall=1.000 (TP=12 FP=0 FN=0) over the 11-case multiturn fixture; MetricsOf unit test pins TP/FP/FN counting including wrong-match counting as both. Impact: CorpusMetrics + MetricsOf (expectation_hit counts as a hit); intentctl eval prints the metrics line after the summary; corpus_multiturn.json expanded to 11 cases (chained expectation, tie-break, ambiguous, fallback, handler action, clarify paths). Refs: Documents proposal 2026-10-07-agent-stack-n1q1i1-plans.md I2
This commit is contained in:
@@ -340,6 +340,22 @@ func cmdEval(args []string) error {
|
||||
fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch)
|
||||
}
|
||||
fmt.Printf("总计 %d 用例：%d 通过，%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed)
|
||||
// I2：指标输出（precision/recall 按意图标签计；确定性重跑，不计入通过判定）。
|
||||
var results []intent.CaseResult
|
||||
for _, tc := range corpus.Cases {
|
||||
results = append(results, intent.RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
metrics := intent.MetricsOf(corpus.Cases, results)
|
||||
p, pok := metrics.Precision()
|
||||
r, rok := metrics.Recall()
|
||||
ps, rs := "-", "-"
|
||||
if pok {
|
||||
ps = fmt.Sprintf("%.3f", p)
|
||||
}
|
||||
if rok {
|
||||
rs = fmt.Sprintf("%.3f", r)
|
||||
}
|
||||
fmt.Printf("指标: precision=%s recall=%s（TP=%d FP=%d FN=%d）\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
|
||||
}
|
||||
|
||||
@@ -180,3 +180,73 @@ func containsString(list []string, s string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CorpusMetrics 评测指标（I2）：按意图标签（matched/top）计 TP/FP/FN。
|
||||
// 期望非 matched（fallback/ambiguous）的轮次作负例、不进指标。
|
||||
type CorpusMetrics struct {
|
||||
TP, FP, FN int
|
||||
}
|
||||
|
||||
// Precision 精确率；分母为 0 时返回 (0, false)（未定义，由调用方决定展示）。
|
||||
func (m CorpusMetrics) Precision() (float64, bool) {
|
||||
if m.TP+m.FP == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(m.TP) / float64(m.TP+m.FP), true
|
||||
}
|
||||
|
||||
// Recall 召回率；分母为 0 时返回 (0, false)。
|
||||
func (m CorpusMetrics) Recall() (float64, bool) {
|
||||
if m.TP+m.FN == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(m.TP) / float64(m.TP+m.FN), true
|
||||
}
|
||||
|
||||
// MetricsOf 逐轮累计指标（I2）：
|
||||
// - TP：期望 matched/top=X 且实际 matched/top=X；
|
||||
// - FP：实际 matched/top=Y 但期望的标签不是 matched/top=Y
|
||||
// （期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同）；
|
||||
// - FN：期望 matched/top=X 但实际没拿到它（branch 不对或 top 不同）。
|
||||
// 只有写了 expect_top 的轮次进入指标（意图识别通行口径）。
|
||||
func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
|
||||
var m CorpusMetrics
|
||||
byName := map[string]CorpusCase{}
|
||||
for _, tc := range cases {
|
||||
byName[tc.Name] = tc
|
||||
}
|
||||
for _, res := range results {
|
||||
tc, ok := byName[res.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
turns := tc.Turns
|
||||
if len(turns) == 0 {
|
||||
turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
|
||||
}
|
||||
for i, turn := range turns {
|
||||
if turn.ExpectTop == "" || i >= len(res.Turns) {
|
||||
continue
|
||||
}
|
||||
got := res.Turns[i]
|
||||
wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit"
|
||||
gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit"
|
||||
if gotMatched && got.TopID == turn.ExpectTop && wantMatched {
|
||||
m.TP++
|
||||
continue
|
||||
}
|
||||
if gotMatched && got.TopID != "" {
|
||||
// 错配同时是假阳（拿了错的）与假阴（期望的没拿到）。
|
||||
m.FP++
|
||||
if wantMatched {
|
||||
m.FN++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if wantMatched {
|
||||
m.FN++
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -77,3 +77,51 @@ func TestMatchCommandsFindsMismatches(t *testing.T) {
|
||||
t.Fatalf("count mismatch = %v", mm)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetricsOfCounts：指标按意图标签计 TP/FP/FN（I2 验收的指标形态）。
|
||||
func TestMetricsOfCounts(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
data, err := os.ReadFile("testdata/corpus_multiturn.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Cases []CorpusCase `json:"cases"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var results []CaseResult
|
||||
for _, tc := range doc.Cases {
|
||||
results = append(results, RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
m := MetricsOf(doc.Cases, results)
|
||||
// 通过的夹具：指标应当全 TP（precision=recall=1）。
|
||||
if m.FP != 0 || m.FN != 0 {
|
||||
t.Fatalf("metrics = %+v", m)
|
||||
}
|
||||
p, pok := m.Precision()
|
||||
r, rok := m.Recall()
|
||||
if !pok || !rok || p != 1.0 || r != 1.0 {
|
||||
t.Fatalf("precision=%v recall=%v (m=%+v)", p, r, m)
|
||||
}
|
||||
if m.TP < 8 {
|
||||
t.Fatalf("TP = %d, want >= 8（夹具内带 top 期望的轮次）", m.TP)
|
||||
}
|
||||
|
||||
// 手工构造一个 FP+FN 用例验证计数方向。
|
||||
bad := []CorpusCase{{Name: "bad", Turns: []CorpusTurn{
|
||||
{Input: "查询任务 42 的进度", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"},
|
||||
}}}
|
||||
got := []CaseResult{RunCorpusCase(context.Background(), snap, bad[0])}
|
||||
m2 := MetricsOf(bad, got)
|
||||
// 实际 matched status_query 但期望 greet.hello → 1 FP + 1 FN。
|
||||
if m2.TP != 0 || m2.FP != 1 || m2.FN != 1 {
|
||||
t.Fatalf("bad metrics = %+v", m2)
|
||||
}
|
||||
p2, _ := m2.Precision()
|
||||
r2, _ := m2.Recall()
|
||||
if p2 != 0 || r2 != 0 {
|
||||
t.Fatalf("p=%v r=%v, want 0/0", p2, r2)
|
||||
}
|
||||
}
|
||||
|
||||
+190
@@ -43,6 +43,196 @@
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ambiguous_then_tiebreak_match",
|
||||
"turns": [
|
||||
{
|
||||
"input": "打结",
|
||||
"expect_branch": "ambiguous",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "clarify",
|
||||
"candidates": [
|
||||
"demo.tie.alpha",
|
||||
"demo.tie.beta"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": "你好呀",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.greet.hello"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_matched_full_slots",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "taskrun.query_status",
|
||||
"slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_fallback",
|
||||
"turns": [
|
||||
{
|
||||
"input": "天气不错",
|
||||
"expect_branch": "fallback",
|
||||
"expect_commands": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_ambiguous_tie",
|
||||
"turns": [
|
||||
{
|
||||
"input": "打结",
|
||||
"expect_branch": "ambiguous",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "clarify",
|
||||
"candidates": [
|
||||
"demo.tie.alpha",
|
||||
"demo.tie.beta"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_tiebreak_priority",
|
||||
"turns": [
|
||||
{
|
||||
"input": "你好呀",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.greet.hello"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_matched_handler_action",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 7 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "taskrun.query_status"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_complete_then_fallback",
|
||||
"turns": [
|
||||
{
|
||||
"input": "生成报告，主题是电机",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "reportHandler",
|
||||
"slots": {
|
||||
"topic": "电机"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": "天气不错",
|
||||
"expect_branch": "fallback",
|
||||
"expect_commands": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_missing_then_other_intent",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_clarify_missing_with_question",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify",
|
||||
"missing": [
|
||||
"topic"
|
||||
],
|
||||
"question": "要查询哪个主题？"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_set_slot_pump",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "生成报告，主题是水泵",
|
||||
"expect_branch": "expectation_hit",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "set_slot",
|
||||
"name": "topic",
|
||||
"value": "水泵"
|
||||
},
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "reportHandler",
|
||||
"slots": {
|
||||
"topic": "水泵"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user