weichen
6aa4925301
Why: A3 - the recall decision (retrieval? model tuning?) needs a repeatable offline baseline; the eval mechanism existed but had no corpus and no metrics. Evidence: five-category corpus (30+ cases) derived from fixture specs; intentctl eval prints per-category metrics and writes metrics.json; structural coverage test (verify-a3.json, exit=0). Impact: evaluation corpus/metrics/baseline report + eval metrics output; intentctl contract backward compatible. Refs: feature/catalog-recall-eval-a3
91 lines
2.8 KiB
Go
91 lines
2.8 KiB
Go
package intent
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
// corpusCase 与 intentctl eval 共用同一份评测集格式（testdata/corpus.json）。
|
|
type corpusCase struct {
|
|
Name string `json:"name"`
|
|
Input string `json:"input"`
|
|
ExpectBranch string `json:"expect_branch"`
|
|
ExpectTop string `json:"expect_top"`
|
|
ExpectSlots map[string]string `json:"expect_slots"`
|
|
ExpectMissingSlots []string `json:"expect_missing_slots"`
|
|
}
|
|
|
|
type corpusDoc struct {
|
|
Cases []corpusCase `json:"cases"`
|
|
}
|
|
|
|
func runCorpus(t *testing.T, rec Recognizer, snap *Snapshot, cases []corpusCase) {
|
|
t.Helper()
|
|
for _, tc := range cases {
|
|
t.Run(tc.Name, func(t *testing.T) {
|
|
res, err := rec.Recognize(context.Background(), tc.Input, snap)
|
|
if err != nil {
|
|
t.Fatalf("recognize: %v", err)
|
|
}
|
|
if string(res.Branch) != tc.ExpectBranch {
|
|
t.Fatalf("branch = %s, want %s (top=%+v)", res.Branch, tc.ExpectBranch, res.Top)
|
|
}
|
|
if tc.ExpectTop == "" {
|
|
if res.Top != nil {
|
|
t.Fatalf("expected no top, got %+v", res.Top)
|
|
}
|
|
} else if res.Top == nil || res.Top.ID != tc.ExpectTop {
|
|
t.Fatalf("top = %+v, want %s", res.Top, tc.ExpectTop)
|
|
}
|
|
for k, v := range tc.ExpectSlots {
|
|
if res.Top == nil || res.Top.Slots[k] != v {
|
|
t.Fatalf("slot %s = %v, want %v (top=%+v)", k, res.Top.Slots, v, res.Top)
|
|
}
|
|
}
|
|
if len(tc.ExpectMissingSlots) > 0 {
|
|
if res.Top == nil || len(res.Top.MissingRequiredSlots) != len(tc.ExpectMissingSlots) {
|
|
t.Fatalf("missing slots = %+v, want %v", res.Top, tc.ExpectMissingSlots)
|
|
}
|
|
for i, k := range tc.ExpectMissingSlots {
|
|
if res.Top.MissingRequiredSlots[i] != k {
|
|
t.Fatalf("missing slots = %+v, want %v", res.Top.MissingRequiredSlots, tc.ExpectMissingSlots)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestEvalCorpus 是模块自带的评测集回归：守护内容 Go 内置 + 业务意图 JSON 配置，
|
|
// 覆盖正常路径、守卫拦截、否定写硬负例、必填槽位缺失与大小写折叠。
|
|
func TestEvalCorpus(t *testing.T) {
|
|
data, err := os.ReadFile("testdata/corpus.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var doc corpusDoc
|
|
if err := json.Unmarshal(data, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(doc.Cases) == 0 {
|
|
t.Fatal("empty corpus")
|
|
}
|
|
_, snap := buildDemoRegistry(t)
|
|
rec := NewRecognizer(Options{}, nil, nil)
|
|
runCorpus(t, rec, snap, doc.Cases)
|
|
|
|
var shared struct {
|
|
Cases []CorpusCase `json:"cases"`
|
|
}
|
|
if err := json.Unmarshal(data, &shared); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var results []CaseResult
|
|
for _, tc := range shared.Cases {
|
|
results = append(results, RunCorpusCase(context.Background(), snap, tc))
|
|
}
|
|
t.Log("\n" + SummarizeCorpus(CorpusDigest(data), shared.Cases, results).Table())
|
|
}
|