Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ab55b54e9b | ||
|
|
db79b535ec | ||
|
|
fcf9ec4b2a | ||
|
|
89a43abf87 | ||
|
|
f57645c645 |
@@ -340,6 +340,22 @@ func cmdEval(args []string) error {
|
||||
fmt.Printf("PASS %-28s %s\n", tc.Name, res.Branch)
|
||||
}
|
||||
fmt.Printf("总计 %d 用例:%d 通过,%d 失败\n", len(corpus.Cases), len(corpus.Cases)-failed, failed)
|
||||
// I2:指标输出(precision/recall 按意图标签计;确定性重跑,不计入通过判定)。
|
||||
var results []intent.CaseResult
|
||||
for _, tc := range corpus.Cases {
|
||||
results = append(results, intent.RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
metrics := intent.MetricsOf(corpus.Cases, results)
|
||||
p, pok := metrics.Precision()
|
||||
r, rok := metrics.Recall()
|
||||
ps, rs := "-", "-"
|
||||
if pok {
|
||||
ps = fmt.Sprintf("%.3f", p)
|
||||
}
|
||||
if rok {
|
||||
rs = fmt.Sprintf("%.3f", r)
|
||||
}
|
||||
fmt.Printf("指标: precision=%s recall=%s(TP=%d FP=%d FN=%d)\n", ps, rs, metrics.TP, metrics.FP, metrics.FN)
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d/%d 用例失败", failed, len(corpus.Cases))
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
@@ -119,3 +121,25 @@ func specSlotFormat(spec IntentSpec, key string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CatalogTopK 取目录的前 k 条:按 Priority 降序、ID 升序(确定性排序;
|
||||
// 排序口径是 priority,不是语义相关度——相关度排序由宿主经 examples 自行计算)。
|
||||
// k <= 0 时等同 Catalog() 全量。
|
||||
func (s *Snapshot) CatalogTopK(k int) []CatalogSpec {
|
||||
all := s.Catalog()
|
||||
if k <= 0 || len(all) <= k {
|
||||
return all
|
||||
}
|
||||
// Priority 不在目录视图里;用 spec 表内的 priority 排序。
|
||||
prio := map[string]int{}
|
||||
for _, cs := range s.specs {
|
||||
prio[cs.spec.ID] = cs.spec.Priority
|
||||
}
|
||||
sort.SliceStable(all, func(i, j int) bool {
|
||||
if prio[all[i].ID] != prio[all[j].ID] {
|
||||
return prio[all[i].ID] > prio[all[j].ID]
|
||||
}
|
||||
return all[i].ID < all[j].ID
|
||||
})
|
||||
return all[:k]
|
||||
}
|
||||
|
||||
@@ -180,3 +180,73 @@ func containsString(list []string, s string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CorpusMetrics 评测指标(I2):按意图标签(matched/top)计 TP/FP/FN。
|
||||
// 期望非 matched(fallback/ambiguous)的轮次作负例、不进指标。
|
||||
type CorpusMetrics struct {
|
||||
TP, FP, FN int
|
||||
}
|
||||
|
||||
// Precision 精确率;分母为 0 时返回 (0, false)(未定义,由调用方决定展示)。
|
||||
func (m CorpusMetrics) Precision() (float64, bool) {
|
||||
if m.TP+m.FP == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(m.TP) / float64(m.TP+m.FP), true
|
||||
}
|
||||
|
||||
// Recall 召回率;分母为 0 时返回 (0, false)。
|
||||
func (m CorpusMetrics) Recall() (float64, bool) {
|
||||
if m.TP+m.FN == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return float64(m.TP) / float64(m.TP+m.FN), true
|
||||
}
|
||||
|
||||
// MetricsOf 逐轮累计指标(I2):
|
||||
// - TP:期望 matched/top=X 且实际 matched/top=X;
|
||||
// - FP:实际 matched/top=Y 但期望的标签不是 matched/top=Y
|
||||
// (期望别的 top、期望 fallback/ambiguous、或期望 matched 但 top 不同);
|
||||
// - FN:期望 matched/top=X 但实际没拿到它(branch 不对或 top 不同)。
|
||||
// 只有写了 expect_top 的轮次进入指标(意图识别通行口径)。
|
||||
func MetricsOf(cases []CorpusCase, results []CaseResult) CorpusMetrics {
|
||||
var m CorpusMetrics
|
||||
byName := map[string]CorpusCase{}
|
||||
for _, tc := range cases {
|
||||
byName[tc.Name] = tc
|
||||
}
|
||||
for _, res := range results {
|
||||
tc, ok := byName[res.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
turns := tc.Turns
|
||||
if len(turns) == 0 {
|
||||
turns = []CorpusTurn{{ExpectBranch: tc.ExpectBranch, ExpectTop: tc.ExpectTop}}
|
||||
}
|
||||
for i, turn := range turns {
|
||||
if turn.ExpectTop == "" || i >= len(res.Turns) {
|
||||
continue
|
||||
}
|
||||
got := res.Turns[i]
|
||||
wantMatched := turn.ExpectBranch == "" || turn.ExpectBranch == "matched" || turn.ExpectBranch == "expectation_hit"
|
||||
gotMatched := got.Branch == "matched" || got.Branch == "expectation_hit"
|
||||
if gotMatched && got.TopID == turn.ExpectTop && wantMatched {
|
||||
m.TP++
|
||||
continue
|
||||
}
|
||||
if gotMatched && got.TopID != "" {
|
||||
// 错配同时是假阳(拿了错的)与假阴(期望的没拿到)。
|
||||
m.FP++
|
||||
if wantMatched {
|
||||
m.FN++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if wantMatched {
|
||||
m.FN++
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -77,3 +77,51 @@ func TestMatchCommandsFindsMismatches(t *testing.T) {
|
||||
t.Fatalf("count mismatch = %v", mm)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMetricsOfCounts:指标按意图标签计 TP/FP/FN(I2 验收的指标形态)。
|
||||
func TestMetricsOfCounts(t *testing.T) {
|
||||
_, snap := buildDemoRegistry(t)
|
||||
data, err := os.ReadFile("testdata/corpus_multiturn.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Cases []CorpusCase `json:"cases"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var results []CaseResult
|
||||
for _, tc := range doc.Cases {
|
||||
results = append(results, RunCorpusCase(context.Background(), snap, tc))
|
||||
}
|
||||
m := MetricsOf(doc.Cases, results)
|
||||
// 通过的夹具:指标应当全 TP(precision=recall=1)。
|
||||
if m.FP != 0 || m.FN != 0 {
|
||||
t.Fatalf("metrics = %+v", m)
|
||||
}
|
||||
p, pok := m.Precision()
|
||||
r, rok := m.Recall()
|
||||
if !pok || !rok || p != 1.0 || r != 1.0 {
|
||||
t.Fatalf("precision=%v recall=%v (m=%+v)", p, r, m)
|
||||
}
|
||||
if m.TP < 8 {
|
||||
t.Fatalf("TP = %d, want >= 8(夹具内带 top 期望的轮次)", m.TP)
|
||||
}
|
||||
|
||||
// 手工构造一个 FP+FN 用例验证计数方向。
|
||||
bad := []CorpusCase{{Name: "bad", Turns: []CorpusTurn{
|
||||
{Input: "查询任务 42 的进度", ExpectBranch: "matched", ExpectTop: "demo.greet.hello"},
|
||||
}}}
|
||||
got := []CaseResult{RunCorpusCase(context.Background(), snap, bad[0])}
|
||||
m2 := MetricsOf(bad, got)
|
||||
// 实际 matched status_query 但期望 greet.hello → 1 FP + 1 FN。
|
||||
if m2.TP != 0 || m2.FP != 1 || m2.FN != 1 {
|
||||
t.Fatalf("bad metrics = %+v", m2)
|
||||
}
|
||||
p2, _ := m2.Precision()
|
||||
r2, _ := m2.Recall()
|
||||
if p2 != 0 || r2 != 0 {
|
||||
t.Fatalf("p=%v r=%v, want 0/0", p2, r2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package intent
|
||||
|
||||
import "context"
|
||||
|
||||
// CommandGenerator 模型命令生成端口(I2):宿主用模型实现
|
||||
// (借鉴 Rasa 的 LLM command generators:只把相关流程交给生成器)。
|
||||
// 只在规则与语义都未命中(fallback)时被调用;实现自担延迟与成本。
|
||||
// 返回错误时识别保持 fallback,不会使 Recognize 失败。
|
||||
//
|
||||
// 库侧校验与 deriveCommands 同款:start 命令的 ActionRef 必须可解析——
|
||||
// auto/command/workflow/blueprint 的目标必须在 catalog 的 spec ID 集合内,
|
||||
// handler 的目标必须在 catalog 的 Handler 集合内;槽位键必须在目标 spec 的
|
||||
// 槽位键集合内(可解析时)。生成器幻觉一律 fail-closed 回 fallback。
|
||||
type CommandGenerator interface {
|
||||
// Generate 基于输入与能力目录产出命令列表(catalog 由调用方按 top-k 给)。
|
||||
Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error)
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package intent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeScorer 按 examples 里是否包含特定子串给分(测试用确定性打分器)。
|
||||
type fakeScorer struct {
|
||||
scores map[string]float64 // key: examples[0]
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeScorer) Score(ctx context.Context, input string, examples []string) (float64, error) {
|
||||
if f.err != nil {
|
||||
return 0, f.err
|
||||
}
|
||||
if len(examples) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if s, ok := f.scores[examples[0]]; ok {
|
||||
return s, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func ambiguousFixture(t *testing.T, exA, exB []string) *Snapshot {
|
||||
t.Helper()
|
||||
b := NewRegistryBuilder(Options{})
|
||||
if err := b.Register(IntentSpec{ID: "demo.alpha", Domain: "demo", Version: 1, Priority: 5,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"模糊词"}}, Examples: exA}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.Register(IntentSpec{ID: "demo.beta", Domain: "demo", Version: 1, Priority: 5,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"模糊词"}}, Examples: exB}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return reg.Snapshot()
|
||||
}
|
||||
|
||||
// TestSemanticRerankResolvesAmbiguous:语义过阈且分差够 → matched(I2 分层)。
|
||||
func TestSemanticRerankResolvesAmbiguous(t *testing.T) {
|
||||
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"闲聊问候"})
|
||||
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.95, "闲聊问候": 0.4}}}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "模糊词", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "demo.alpha" {
|
||||
t.Fatalf("semantic rerank: %+v", res)
|
||||
}
|
||||
if !strings.Contains(res.Top.Reason, "refined_by=semantic") {
|
||||
t.Fatalf("reason = %q", res.Top.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSemanticBelowThresholdKeepsAmbiguous:全部不达阈 → 保持 ambiguous。
|
||||
func TestSemanticBelowThresholdKeepsAmbiguous(t *testing.T) {
|
||||
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"闲聊问候"})
|
||||
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.5}}}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "模糊词", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchAmbiguous {
|
||||
t.Fatalf("below threshold: branch = %s", res.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSemanticTieKeepsAmbiguous:两个过阈但分差不足 → 保持 ambiguous。
|
||||
func TestSemanticTieKeepsAmbiguous(t *testing.T) {
|
||||
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"查运行状态"})
|
||||
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.9, "查运行状态": 0.89}}}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "模糊词", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchAmbiguous {
|
||||
t.Fatalf("tie: branch = %s", res.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSemanticErrorKeepsAmbiguousAndCallsL2Next:打分错误 → ambiguous,且 L2 接管。
|
||||
func TestSemanticErrorKeepsAmbiguousAndCallsL2Next(t *testing.T) {
|
||||
snap := ambiguousFixture(t, []string{"查运行进度"}, nil)
|
||||
called := false
|
||||
cls := rewriteClassifier{fn: func(c []Candidate) []Candidate {
|
||||
called = true
|
||||
out := append([]Candidate(nil), c...)
|
||||
out[0].Confidence = 0.99
|
||||
out[1].Confidence = 0.1
|
||||
return out
|
||||
}}
|
||||
rec := NewRecognizer(Options{Semantic: fakeScorer{err: errors.New("embed down")}}, nil, cls)
|
||||
res, err := rec.Recognize(context.Background(), "模糊词", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called {
|
||||
t.Fatal("L2 should be consulted after semantic failure")
|
||||
}
|
||||
if res.Branch != BranchMatched || res.Top.ID != "demo.alpha" {
|
||||
t.Fatalf("L2 fallback after semantic error: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratorValidStartBecomesMatched:合法 start 命令 → matched + 合成 Top(I2)。
|
||||
func TestGeneratorValidStartBecomesMatched(t *testing.T) {
|
||||
b := NewRegistryBuilder(Options{})
|
||||
if err := b.Register(IntentSpec{ID: "report.generate", Domain: "report", Version: 1,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
|
||||
Slots: []SlotSpec{{Key: "topic", Pattern: `主题是(.+)`}},
|
||||
Handler: "reportHandler"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gen := stubGenerator{cmds: []Command{
|
||||
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "report.generate"},
|
||||
Slots: map[string]string{"topic": "电机"}},
|
||||
}}
|
||||
rec := NewRecognizer(Options{Generator: gen}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", reg.Snapshot())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "report.generate" {
|
||||
t.Fatalf("generated: %+v", res)
|
||||
}
|
||||
if len(res.Commands) != 1 || res.Commands[0].Kind != CommandStart || res.Commands[0].Slots["topic"] != "电机" {
|
||||
t.Fatalf("commands = %+v", res.Commands)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratorInvalidOutputFallsBack:幻觉动作/槽位 → 保持 fallback(fail-closed)。
|
||||
func TestGeneratorInvalidOutputFallsBack(t *testing.T) {
|
||||
b := NewRegistryBuilder(Options{})
|
||||
if err := b.Register(IntentSpec{ID: "report.generate", Domain: "report", Version: 1,
|
||||
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
|
||||
Slots: []SlotSpec{{Key: "topic", Pattern: `(.+)`}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snap := reg.Snapshot()
|
||||
cases := map[string][]Command{
|
||||
"unknown spec id": {
|
||||
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "ghost.flow"}},
|
||||
},
|
||||
"unknown kind": {
|
||||
{Kind: CommandStart, Action: ActionRef{Kind: "bogus", ID: "report.generate"}},
|
||||
},
|
||||
"unknown slot key": {
|
||||
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "report.generate"}, Slots: map[string]string{"bogus": "v"}},
|
||||
},
|
||||
"unknown cmd kind": {
|
||||
{Kind: "teleport"},
|
||||
},
|
||||
"empty": {},
|
||||
}
|
||||
for name, cmds := range cases {
|
||||
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: cmds}}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", snap)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchFallback {
|
||||
t.Fatalf("%s: branch = %s, want fallback", name, res.Branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratorErrorKeepsFallback:生成器错误 → 保持 fallback。
|
||||
func TestGeneratorErrorKeepsFallback(t *testing.T) {
|
||||
reg, err := NewRegistryBuilder(Options{}).Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := NewRecognizer(Options{Generator: stubGenerator{err: errors.New("model down")}}, nil, nil)
|
||||
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", reg.Snapshot())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Branch != BranchFallback {
|
||||
t.Fatalf("branch = %s", res.Branch)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCatalogTopK:按 priority 降序取 k(确定性口径)。
|
||||
func TestCatalogTopK(t *testing.T) {
|
||||
b := NewRegistryBuilder(Options{})
|
||||
if err := b.Register(IntentSpec{ID: "demo.low", Domain: "demo", Version: 1, Priority: 1, Matcher: MatchSpec{IncludesAny: []string{"a"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.Register(IntentSpec{ID: "demo.high", Domain: "demo", Version: 1, Priority: 9, Matcher: MatchSpec{IncludesAny: []string{"b"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.Register(IntentSpec{ID: "demo.mid", Domain: "demo", Version: 1, Priority: 5, Matcher: MatchSpec{IncludesAny: []string{"c"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, err := b.Build()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
top := reg.Snapshot().CatalogTopK(2)
|
||||
if len(top) != 2 || top[0].ID != "demo.high" || top[1].ID != "demo.mid" {
|
||||
t.Fatalf("topk = %+v", top)
|
||||
}
|
||||
if all := reg.Snapshot().CatalogTopK(0); len(all) != 3 {
|
||||
t.Fatalf("k=0 should be full catalog: %d", len(all))
|
||||
}
|
||||
}
|
||||
|
||||
type stubGenerator struct {
|
||||
cmds []Command
|
||||
err error
|
||||
}
|
||||
|
||||
func (s stubGenerator) Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error) {
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
}
|
||||
return s.cmds, nil
|
||||
}
|
||||
+170
-4
@@ -19,17 +19,22 @@ type Recognizer interface {
|
||||
|
||||
// FunnelRecognizer 四级漏斗实现:guard → expectation → rules → adjudicate,
|
||||
// 可选注入挂起预期提供者与 Ambiguous 分支的 L2 分类器。
|
||||
// I2 的分层:歧义先经语义重排(确定性宿主打分)再到 L2 分类器(模型)。
|
||||
type FunnelRecognizer struct {
|
||||
log *logsys.Logger
|
||||
exp ExpectationProvider
|
||||
cls Classifier
|
||||
sem SemanticScorer
|
||||
thr float64
|
||||
gen CommandGenerator
|
||||
}
|
||||
|
||||
// NewRecognizer 构建漏斗识别器。opts 仅使用 Log 字段(阈值类参数在快照内);
|
||||
// exp / cls 可为 nil(对应阶段不启用)。
|
||||
// exp / cls 可为 nil(对应阶段不启用)。语义与生成端口经 Options 注入(I2)。
|
||||
func NewRecognizer(opts Options, exp ExpectationProvider, cls Classifier) *FunnelRecognizer {
|
||||
o := opts.normalized()
|
||||
return &FunnelRecognizer{log: o.Log, exp: exp, cls: cls}
|
||||
return &FunnelRecognizer{log: o.Log, exp: exp, cls: cls,
|
||||
sem: o.Semantic, thr: o.SemanticThreshold, gen: o.Generator}
|
||||
}
|
||||
|
||||
// Recognize 识别一次输入。确定性:同输入 + 同快照 → 同结果(日志中的耗时除外)。
|
||||
@@ -46,7 +51,10 @@ func (r *FunnelRecognizer) Recognize(ctx context.Context, input string, snap *Sn
|
||||
}
|
||||
start := time.Now()
|
||||
res := r.recognize(ctx, input, snap)
|
||||
res.Commands = deriveCommands(res, snap)
|
||||
// 生成端口已经给出经过校验的命令列表时不再用规则展开覆盖(I2)。
|
||||
if res.Commands == nil {
|
||||
res.Commands = deriveCommands(res, snap)
|
||||
}
|
||||
topID, conf := "", 0.0
|
||||
if res.Top != nil {
|
||||
topID, conf = res.Top.ID, res.Top.Confidence
|
||||
@@ -89,6 +97,12 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
|
||||
hits := snap.evalRules(matchInput, folded)
|
||||
switch {
|
||||
case len(hits) == 0:
|
||||
// I2:规则与语义都未命中时,可选经模型命令生成端口兜底(校验同款)。
|
||||
if r.gen != nil {
|
||||
if res, ok := r.generateCommands(ctx, input, snap); ok {
|
||||
return res
|
||||
}
|
||||
}
|
||||
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
|
||||
case len(hits) == 1 || hits[0].score-hits[1].score >= snap.margin || tieBreakByPriority(hits):
|
||||
cands := candidatesOf(trimHits(hits, snap.topN), input, true)
|
||||
@@ -96,8 +110,15 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
|
||||
return Result{Input: input, Branch: BranchMatched, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
|
||||
}
|
||||
|
||||
// Ambiguous:可选 L2 分类器重排;失败或结果不合法都保持 Ambiguous。
|
||||
// Ambiguous:先语义重排(I2 确定性宿主打分),失败或不达阈才到 L2 分类器。
|
||||
cands := candidatesOf(trimHits(hits, snap.topN), input, false)
|
||||
if r.sem != nil {
|
||||
if top, ok := r.semanticRerank(ctx, input, cands, snap); ok {
|
||||
snap.fillTopSlots(&top, input)
|
||||
top.Reason = top.Reason + "; refined_by=semantic"
|
||||
return Result{Input: input, Branch: BranchMatched, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
|
||||
}
|
||||
}
|
||||
if r.cls != nil {
|
||||
refined, err := r.cls.Refine(ctx, input, cands)
|
||||
if err != nil {
|
||||
@@ -263,3 +284,148 @@ func sortScored(ss []ScoredSpec) {
|
||||
func sortRejected(rs []RejectedSpec) {
|
||||
sort.SliceStable(rs, func(i, j int) bool { return rs[i].ID < rs[j].ID })
|
||||
}
|
||||
|
||||
// semanticRerank 用宿主打分器对歧义候选按 examples 重排(I2)。
|
||||
// 规则:候选必须来自当前候选集(构造上不可能引入未知 ID);
|
||||
// 只取相似度 ≥ 阈值的候选——唯一过阈或前两名分差 ≥ margin 才判 matched,
|
||||
// 否则保持 ambiguous(与规则裁决同款语义)。打分错误一律记日志并保持 ambiguous。
|
||||
func (r *FunnelRecognizer) semanticRerank(ctx context.Context, input string, cands []Candidate, snap *Snapshot) (Candidate, bool) {
|
||||
type scored struct {
|
||||
cand Candidate
|
||||
score float64
|
||||
}
|
||||
var above []scored
|
||||
for _, c := range cands {
|
||||
spec, ok := snap.SpecByID(c.ID)
|
||||
if !ok || len(spec.Examples) == 0 {
|
||||
continue
|
||||
}
|
||||
s, err := r.sem.Score(ctx, input, spec.Examples)
|
||||
if err != nil {
|
||||
r.log.Module("intent").WithContext(ctx).Warn("intent semantic scorer failed; keep ambiguous",
|
||||
"error", err.Error(), "id", c.ID)
|
||||
continue
|
||||
}
|
||||
if s >= r.thr {
|
||||
above = append(above, scored{cand: c, score: s})
|
||||
}
|
||||
}
|
||||
if len(above) == 0 {
|
||||
return Candidate{}, false
|
||||
}
|
||||
for i := 0; i+1 < len(above); i++ {
|
||||
for j := i + 1; j < len(above); j++ {
|
||||
if above[j].score > above[i].score {
|
||||
above[i], above[j] = above[j], above[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(above) == 1 || above[0].score-above[1].score >= snap.margin {
|
||||
return above[0].cand, true
|
||||
}
|
||||
return Candidate{}, false
|
||||
}
|
||||
|
||||
// generateCommands 经模型命令生成端口兜底(I2)。校验:
|
||||
// start 的 ActionRef 可解析(spec ID 集或 Handler 集内)、命令种类受限、
|
||||
// 槽位键在目标 spec 的键集内(可解析时);全部通过则把首个 start 的目标
|
||||
// spec 合成 Top(refined_by=generator 标注),任一项不过则保持 fallback。
|
||||
func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, snap *Snapshot) (Result, bool) {
|
||||
catalog := snap.CatalogTopK(DefaultGeneratorCatalogK)
|
||||
cmds, err := r.gen.Generate(ctx, input, catalog)
|
||||
if err != nil {
|
||||
r.log.Module("intent").WithContext(ctx).Warn("intent command generator failed; keep fallback", "error", err.Error())
|
||||
return Result{}, false
|
||||
}
|
||||
if err := validateGenerated(cmds, catalog); err != nil {
|
||||
r.log.Module("intent").WithContext(ctx).Warn("intent generator output rejected; keep fallback", "error", err.Error())
|
||||
return Result{}, false
|
||||
}
|
||||
res := Result{Input: input, Branch: BranchFallback, Commands: cmds, SnapshotRevision: snap.rev}
|
||||
for _, c := range cmds {
|
||||
if c.Kind != CommandStart {
|
||||
continue
|
||||
}
|
||||
if spec, ok := generatedSpecOf(c.Action, catalog); ok {
|
||||
top := Candidate{
|
||||
ID: spec.ID, Domain: spec.Domain, Version: spec.Version,
|
||||
Confidence: 0, Reason: "generated_by=command_generator",
|
||||
Slots: c.Slots,
|
||||
}
|
||||
res.Top = &top
|
||||
res.Branch = BranchMatched
|
||||
res.Candidates = []Candidate{top}
|
||||
}
|
||||
break
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
// DefaultGeneratorCatalogK 生成器目录默认 top-k。
|
||||
const DefaultGeneratorCatalogK = 8
|
||||
|
||||
// validateGenerated 校验生成命令(I2;与 spec 级校验同款 fail-closed)。
|
||||
func validateGenerated(cmds []Command, catalog []CatalogSpec) error {
|
||||
if len(cmds) == 0 {
|
||||
return fmt.Errorf("empty command list")
|
||||
}
|
||||
specIDs := map[string]bool{}
|
||||
handlers := map[string]bool{}
|
||||
slotKeys := map[string]map[string]bool{}
|
||||
for _, cs := range catalog {
|
||||
specIDs[cs.ID] = true
|
||||
if cs.Handler != "" {
|
||||
handlers[cs.Handler] = true
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, sl := range cs.Slots {
|
||||
keys[sl.Key] = true
|
||||
}
|
||||
slotKeys[cs.ID] = keys
|
||||
}
|
||||
for i, c := range cmds {
|
||||
switch c.Kind {
|
||||
case CommandStart:
|
||||
ref := c.Action
|
||||
switch ref.Kind {
|
||||
case ActionAuto, ActionCommand, ActionWorkflow, ActionBlueprint:
|
||||
if !specIDs[ref.ID] {
|
||||
return fmt.Errorf("cmd[%d] start: action.id %q 不在目录内", i, ref.ID)
|
||||
}
|
||||
case ActionHandler:
|
||||
if !handlers[ref.ID] {
|
||||
return fmt.Errorf("cmd[%d] start: handler %q 不在目录内", i, ref.ID)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("cmd[%d] start: 未知 action.kind %q", i, ref.Kind)
|
||||
}
|
||||
if keys, ok := slotKeys[ref.ID]; ok {
|
||||
for k := range c.Slots {
|
||||
if !keys[k] {
|
||||
return fmt.Errorf("cmd[%d] start: 槽位 %q 不在 %s 的槽位键集内", i, k, ref.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
case CommandClarify, CommandCancel, CommandSetSlot, CommandChangeSlot, CommandChitchat, CommandHandoff:
|
||||
default:
|
||||
return fmt.Errorf("cmd[%d]: 未知命令种类 %q", i, c.Kind)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// generatedSpecOf 把 start 的 ActionRef 解析回目录 spec(handler 键按 Handler 反查)。
|
||||
func generatedSpecOf(ref ActionRef, catalog []CatalogSpec) (CatalogSpec, bool) {
|
||||
for _, cs := range catalog {
|
||||
if ref.Kind == ActionHandler {
|
||||
if cs.Handler == ref.ID {
|
||||
return cs, true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if cs.ID == ref.ID {
|
||||
return cs, true
|
||||
}
|
||||
}
|
||||
return CatalogSpec{}, false
|
||||
}
|
||||
|
||||
@@ -35,6 +35,12 @@ type Options struct {
|
||||
MaxSlotsPerSpec int
|
||||
// Log 可选日志器;nil 时使用 Nop。识别日志不记录原始输入。
|
||||
Log *logsys.Logger
|
||||
// Semantic 语义相似度端口(I2):nil 时语义层不启用。
|
||||
Semantic SemanticScorer
|
||||
// SemanticThreshold 语义重排的过阈值(默认 0.8)。
|
||||
SemanticThreshold float64
|
||||
// Generator 模型命令生成端口(I2):nil 时 fallback 不生成。
|
||||
Generator CommandGenerator
|
||||
// SlotValidators 作用域槽位校验器。构建与校验时先查此表,查不到再回落
|
||||
// 已弃用的全局表(RegisterSlotValidator)。不同 Registry 的同名校验器互不影响。
|
||||
SlotValidators map[string]SlotValidator
|
||||
@@ -61,6 +67,9 @@ func (o Options) normalized() Options {
|
||||
if n.AmbiguityMargin <= 0 {
|
||||
n.AmbiguityMargin = DefaultAmbiguityMargin
|
||||
}
|
||||
if n.SemanticThreshold <= 0 {
|
||||
n.SemanticThreshold = DefaultSemanticThreshold
|
||||
}
|
||||
if n.MaxInputLen <= 0 {
|
||||
n.MaxInputLen = DefaultMaxInputLen
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package intent
|
||||
|
||||
import "context"
|
||||
|
||||
// SemanticScorer 语义相似度端口(I2 分层与评测):宿主注入向量模型,
|
||||
// 核心不做 I/O——打分由宿主计算,库只消费分数。
|
||||
// 只在规则裁决为 Ambiguous 时被调用(规则→语义→模型分层的中位);
|
||||
// 实现应幂等且无副作用(同一输入与示例集应得到同一分数)。
|
||||
type SemanticScorer interface {
|
||||
// Score 返回 input 与 examples 的最高相似度,取值 [0,1]。
|
||||
// 实现返回错误时识别保持 Ambiguous,不会使 Recognize 失败。
|
||||
Score(ctx context.Context, input string, examples []string) (float64, error)
|
||||
}
|
||||
@@ -18,6 +18,8 @@ const (
|
||||
DefaultBase = 0.7
|
||||
DefaultPatternBonus = 0.2
|
||||
DefaultThreshold = 0.6
|
||||
// DefaultSemanticThreshold 语义重排默认过阈值。
|
||||
DefaultSemanticThreshold = 0.8
|
||||
)
|
||||
|
||||
// MatchSpec 声明式匹配规则。
|
||||
@@ -87,6 +89,9 @@ type IntentSpec struct {
|
||||
// 声明后优先于 Handler 与 Metadata["blueprint"] 被 ActionRef 采用。
|
||||
// kind 必须是 command/workflow/blueprint/handler 之一。
|
||||
Action *ActionRef `json:"action,omitempty"`
|
||||
// Examples 示例句(I2 语义端口):歧义时语义打分器按它们路由
|
||||
// (semantic-router 借鉴;向量模型由宿主注入,库不做 I/O)。
|
||||
Examples []string `json:"examples,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
+190
@@ -43,6 +43,196 @@
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "ambiguous_then_tiebreak_match",
|
||||
"turns": [
|
||||
{
|
||||
"input": "打结",
|
||||
"expect_branch": "ambiguous",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "clarify",
|
||||
"candidates": [
|
||||
"demo.tie.alpha",
|
||||
"demo.tie.beta"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": "你好呀",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.greet.hello"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_matched_full_slots",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "taskrun.query_status",
|
||||
"slots": {
|
||||
"run_id": "42"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_fallback",
|
||||
"turns": [
|
||||
{
|
||||
"input": "天气不错",
|
||||
"expect_branch": "fallback",
|
||||
"expect_commands": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_ambiguous_tie",
|
||||
"turns": [
|
||||
{
|
||||
"input": "打结",
|
||||
"expect_branch": "ambiguous",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "clarify",
|
||||
"candidates": [
|
||||
"demo.tie.alpha",
|
||||
"demo.tie.beta"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_tiebreak_priority",
|
||||
"turns": [
|
||||
{
|
||||
"input": "你好呀",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.greet.hello"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_matched_handler_action",
|
||||
"turns": [
|
||||
{
|
||||
"input": "查询任务 7 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "taskrun.query_status"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_complete_then_fallback",
|
||||
"turns": [
|
||||
{
|
||||
"input": "生成报告,主题是电机",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "reportHandler",
|
||||
"slots": {
|
||||
"topic": "电机"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"input": "天气不错",
|
||||
"expect_branch": "fallback",
|
||||
"expect_commands": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_missing_then_other_intent",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "查询任务 42 的进度",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "demo.run.status_query"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "single_clarify_missing_with_question",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "start"
|
||||
},
|
||||
{
|
||||
"kind": "clarify",
|
||||
"missing": [
|
||||
"topic"
|
||||
],
|
||||
"question": "要查询哪个主题?"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "two_turn_set_slot_pump",
|
||||
"turns": [
|
||||
{
|
||||
"input": "帮我生成报告",
|
||||
"expect_branch": "matched",
|
||||
"expect_top": "report.generate"
|
||||
},
|
||||
{
|
||||
"input": "生成报告,主题是水泵",
|
||||
"expect_branch": "expectation_hit",
|
||||
"expect_top": "report.generate",
|
||||
"expect_commands": [
|
||||
{
|
||||
"kind": "set_slot",
|
||||
"name": "topic",
|
||||
"value": "水泵"
|
||||
},
|
||||
{
|
||||
"kind": "start",
|
||||
"action_kind": "handler",
|
||||
"action_id": "reportHandler",
|
||||
"slots": {
|
||||
"topic": "水泵"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user