Add precision and recall metrics to the multi-turn evaluation
Why: I2 acceptance - the evaluator must emit metrics over a multiturn fixture no smaller than the existing corpus, not just print branch names. Evidence: go vet clean; full suite ok; intentctl eval prints precision=1.000 recall=1.000 (TP=12 FP=0 FN=0) over the 11-case multiturn fixture; MetricsOf unit test pins TP/FP/FN counting including wrong-match counting as both. Impact: CorpusMetrics + MetricsOf (expectation_hit counts as a hit); intentctl eval prints the metrics line after the summary; corpus_multiturn.json expanded to 11 cases (chained expectation, tie-break, ambiguous, fallback, handler action, clarify paths). Refs: Documents proposal 2026-10-07-agent-stack-n1q1i1-plans.md I2
This commit is contained in:
@@ -340,6 +340,22 @@ func cmdEval(args []string) error {
|
||||
fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch)
|
||||
}
|
||||
fmt.Printf("总计 %d 用例:%d 通过,%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed)
|
||||
// I2:指标输出(precision/recall 按意图标签计;确定性重跑,不计入通过判定)。
|
||||
var results []intent.CaseResult
|
||||
for _, tc := range corpus.Cases {
|
||||
results = append(results, intent.RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
metrics := intent.MetricsOf(corpus.Cases, results)
|
||||
p, pok := metrics.Precision()
|
||||
r, rok := metrics.Recall()
|
||||
ps, rs := "-", "-"
|
||||
if pok {
|
||||
ps = fmt.Sprintf("%.3f", p)
|
||||
}
|
||||
if rok {
|
||||
rs = fmt.Sprintf("%.3f", r)
|
||||
}
|
||||
fmt.Printf("指标: precision=%s recall=%s(TP=%d FP=%d FN=%d)\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user