From db79b535ec710179a77ccfb32215de4435151a0f Mon Sep 17 00:00:00 2001 From: weichen Date: Wed, 7 Oct 2026 11:02:28 +0000 Subject: [PATCH] Add precision and recall metrics to the multi-turn evaluation Why: I2 acceptance - the evaluator must emit metrics over a multiturn fixture no smaller than the existing corpus, not just print branch names. Evidence: go vet clean; full suite ok; intentctl eval prints precision=1.000 recall=1.000 (TP=12 FP=0 FN=0) over the 11-case multiturn fixture; MetricsOf unit test pins TP/FP/FN counting including wrong-match counting as both. Impact: CorpusMetrics + MetricsOf (expectation_hit counts as a hit); intentctl eval prints the metrics line after the summary; corpus_multiturn.json expanded to 11 cases (chained expectation, tie-break, ambiguous, fallback, handler action, clarify paths). Refs: Documents proposal 2026-10-07-agent-stack-n1q1i1-plans.md I2 --- cmd/intentctl/main.go | 16 +++ intent/corpus.go | 70 ++++++++++ intent/corpus_test.go | 48 +++++++ intent/testdata/corpus_multiturn.json | 190 ++++++++++++++++++++++++++ 4 files changed, 324 insertions(+) diff --git a/cmd/intentctl/main.go b/cmd/intentctl/main.go index 7b8b674..eb5f95f 100644 --- a/cmd/intentctl/main.go +++ b/cmd/intentctl/main.go @@ -340,6 +340,22 @@ func cmdEval(args []string) error { fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch) } fmt.Printf("总计 %d 用例：%d 通过，%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed) + // I2：指标输出（precision/recall 按意图标签计；确定性重跑，不计入通过判定）。 + var results []intent.CaseResult + for _, tc := range corpus.Cases { + results = append(results, intent.RunCorpusCase(context.Background(), snap, tc)) + } + metrics := intent.MetricsOf(corpus.Cases, results) + p, pok := metrics.Precision() + r, rok := metrics.Recall() + ps, rs := "-", "-" + if pok { + ps = fmt.Sprintf("%.3f", p) + } + if rok { + rs = fmt.Sprintf("%.3f", r) + } + fmt.Printf("指标: precision=%s recall=%s（TP=%d FP=%d FN=%d）\n", ps, rs, metrics.TP, metrics.FP, metrics.FN) if failed > 0 { return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases)) } diff --git a/intent/corpus.go b/intent/corpus.go index f820426..f3993c4 100644 --- a/intent/corpus.go +++ b/intent/corpus.go @@ -180,3 +180,73 @@ func containsString(list []string, s string) bool { } return false } + +// CorpusMetrics 评测指标（I2）：按意图标签（matched/top）计 TP/FP/FN。 +// 期望非 matched（fallback/ambiguous）的轮次作负例、不进指标。 +type CorpusMetrics struct { + TP, FP, FN int +} + +// Precision 精确率；分母为 0 时返回 (0, false)（未定义，由调用方决定展示）。 +func (m CorpusMetrics) Precision() (float64, bool) { + if m.TP+m.FP == 0 { + return 0, false + } + return float64(m.TP) / float64(m.TP+m.FP), true +} + +// Recall 召回率；分母为 0 时返回 (0, false)。 +func (m CorpusMetrics) Recall() (float64, bool) { + if m.TP+m.FN == 0 { + return 0, false + } + return float64(m.TP) / float64(m.TP+m.FN), true +} + +// MetricsOf 逐轮累计指标（I2）： +// - TP：期望 matched/top=X 且实际 matched/top=X； +// - FP：实际 matched/top=Y 但期望的标签不是 matched/top=Y +// （期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同）； +// - FN：期望 matched/top=X 但实际没拿到它（branch 不对或 top 不同）。 +// 只有写了 expect_top 的轮次进入指标（意图识别通行口径）。 +func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics { + var m CorpusMetrics + byName := map[string]CorpusCase{} + for _, tc := range cases { + byName[tc.Name] = tc + } + for _, res := range results { + tc, ok := byName[res.Name] + if !ok { + continue + } + turns := tc.Turns + if len(turns) == 0 { + turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}} + } + for i, turn := range turns { + if turn.ExpectTop == "" || i >= len(res.Turns) { + continue + } + got := res.Turns[i] + wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit" + gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit" + if gotMatched && got.TopID == turn.ExpectTop && wantMatched { + m.TP++ + continue + } + if gotMatched && got.TopID != "" { + // 错配同时是假阳（拿了错的）与假阴（期望的没拿到）。 + m.FP++ + if wantMatched { + m.FN++ + } + continue + } + if wantMatched { + m.FN++ + } + } + } + return m +} diff --git a/intent/corpus_test.go b/intent/corpus_test.go index 352c227..1a06006 100644 --- a/intent/corpus_test.go +++ b/intent/corpus_test.go @@ -77,3 +77,51 @@ func TestMatchCommandsFindsMismatches(t *testing.T) { t.Fatalf("count mismatch = %v", mm) } } + +// TestMetricsOfCounts：指标按意图标签计 TP/FP/FN（I2 验收的指标形态）。 +func TestMetricsOfCounts(t *testing.T) { + _, snap := buildDemoRegistry(t) + data, err := os.ReadFile("testdata/corpus_multiturn.json") + if err != nil { + t.Fatal(err) + } + var doc struct { + Cases []CorpusCase `json:"cases"` + } + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + var results []CaseResult + for _, tc := range doc.Cases { + results = append(results, RunCorpusCase(context.Background(), snap, tc)) + } + m := MetricsOf(doc.Cases, results) + // 通过的夹具：指标应当全 TP（precision=recall=1）。 + if m.FP != 0 || m.FN != 0 { + t.Fatalf("metrics = %+v", m) + } + p, pok := m.Precision() + r, rok := m.Recall() + if !pok || !rok || p != 1.0 || r != 1.0 { + t.Fatalf("precision=%v recall=%v (m=%+v)", p, r, m) + } + if m.TP < 8 { + t.Fatalf("TP = %d, want >= 8（夹具内带 top 期望的轮次）", m.TP) + } + + // 手工构造一个 FP+FN 用例验证计数方向。 + bad := []CorpusCase{{Name: "bad", Turns: []CorpusTurn{ + {Input: "查询任务 42 的进度", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"}, + }}} + got := []CaseResult{RunCorpusCase(context.Background(), snap, bad[0])} + m2 := MetricsOf(bad, got) + // 实际 matched status_query 但期望 greet.hello → 1 FP + 1 FN。 + if m2.TP != 0 || m2.FP != 1 || m2.FN != 1 { + t.Fatalf("bad metrics = %+v", m2) + } + p2, _ := m2.Precision() + r2, _ := m2.Recall() + if p2 != 0 || r2 != 0 { + t.Fatalf("p=%v r=%v, want 0/0", p2, r2) + } +} diff --git a/intent/testdata/corpus_multiturn.json b/intent/testdata/corpus_multiturn.json index 8824feb..bb2930d 100644 --- a/intent/testdata/corpus_multiturn.json +++ b/intent/testdata/corpus_multiturn.json @@ -43,6 +43,196 @@ ] } ] + }, + { + "name": "ambiguous_then_tiebreak_match", + "turns": [ + { + "input": "打结", + "expect_branch": "ambiguous", + "expect_commands": [ + { + "kind": "clarify", + "candidates": [ + "demo.tie.alpha", + "demo.tie.beta" + ] + } + ] + }, + { + "input": "你好呀", + "expect_branch": "matched", + "expect_top": "demo.greet.hello" + } + ] + }, + { + "name": "single_matched_full_slots", + "turns": [ + { + "input": "查询任务 42 的进度", + "expect_branch": "matched", + "expect_top": "demo.run.status_query", + "expect_commands": [ + { + "kind": "start", + "action_kind": "handler", + "action_id": "taskrun.query_status", + "slots": { + "run_id": "42" + } + } + ] + } + ] + }, + { + "name": "single_fallback", + "turns": [ + { + "input": "天气不错", + "expect_branch": "fallback", + "expect_commands": [] + } + ] + }, + { + "name": "single_ambiguous_tie", + "turns": [ + { + "input": "打结", + "expect_branch": "ambiguous", + "expect_commands": [ + { + "kind": "clarify", + "candidates": [ + "demo.tie.alpha", + "demo.tie.beta" + ] + } + ] + } + ] + }, + { + "name": "single_tiebreak_priority", + "turns": [ + { + "input": "你好呀", + "expect_branch": "matched", + "expect_top": "demo.greet.hello" + } + ] + }, + { + "name": "single_matched_handler_action", + "turns": [ + { + "input": "查询任务 7 的进度", + "expect_branch": "matched", + "expect_top": "demo.run.status_query", + "expect_commands": [ + { + "kind": "start", + "action_kind": "handler", + "action_id": "taskrun.query_status" + } + ] + } + ] + }, + { + "name": "two_turn_complete_then_fallback", + "turns": [ + { + "input": "生成报告，主题是电机", + "expect_branch": "matched", + "expect_top": "report.generate", + "expect_commands": [ + { + "kind": "start", + "action_kind": "handler", + "action_id": "reportHandler", + "slots": { + "topic": "电机" + } + } + ] + }, + { + "input": "天气不错", + "expect_branch": "fallback", + "expect_commands": [] + } + ] + }, + { + "name": "two_turn_missing_then_other_intent", + "turns": [ + { + "input": "帮我生成报告", + "expect_branch": "matched", + "expect_top": "report.generate" + }, + { + "input": "查询任务 42 的进度", + "expect_branch": "matched", + "expect_top": "demo.run.status_query" + } + ] + }, + { + "name": "single_clarify_missing_with_question", + "turns": [ + { + "input": "帮我生成报告", + "expect_branch": "matched", + "expect_top": "report.generate", + "expect_commands": [ + { + "kind": "start" + }, + { + "kind": "clarify", + "missing": [ + "topic" + ], + "question": "要查询哪个主题？" + } + ] + } + ] + }, + { + "name": "two_turn_set_slot_pump", + "turns": [ + { + "input": "帮我生成报告", + "expect_branch": "matched", + "expect_top": "report.generate" + }, + { + "input": "生成报告，主题是水泵", + "expect_branch": "expectation_hit", + "expect_top": "report.generate", + "expect_commands": [ + { + "kind": "set_slot", + "name": "topic", + "value": "水泵" + }, + { + "kind": "start", + "action_kind": "handler", + "action_id": "reportHandler", + "slots": { + "topic": "水泵" + } + } + ] + } + ] } ] } \ No newline at end of file -- 2.54.0