Add the semantic rerank port, examples, and the command generator port (I2) #9

Merged
weichen merged 1 commits from feature/agent-stack-i2 into main 2026-10-07 10:57:50 +00:00
7 changed files with 472 additions and 4 deletions
+24
View File
@@ -1,6 +1,8 @@
package intent
import (
"sort"
"fmt"
"strings"
)
@@ -119,3 +121,25 @@ func specSlotFormat(spec IntentSpec, key string) string {
}
return ""
}
// CatalogTopK 取目录的前 k 条:按 Priority 降序、ID 升序(确定性排序;
// 排序口径是 priority,不是语义相关度——相关度排序由宿主经 examples 自行计算)。
// k <= 0 时等同 Catalog() 全量。
func (s *Snapshot) CatalogTopK(k int) []CatalogSpec {
all := s.Catalog()
if k <= 0 || len(all) <= k {
return all
}
// Priority 不在目录视图里;用 spec 表内的 priority 排序。
prio := map[string]int{}
for _, cs := range s.specs {
prio[cs.spec.ID] = cs.spec.Priority
}
sort.SliceStable(all, func(i, j int) bool {
if prio[all[i].ID] != prio[all[j].ID] {
return prio[all[i].ID] > prio[all[j].ID]
}
return all[i].ID < all[j].ID
})
return all[:k]
}
+17
View File
@@ -0,0 +1,17 @@
package intent
import "context"
// CommandGenerator 模型命令生成端口(I2):宿主用模型实现
// (借鉴 Rasa 的 LLM command generators:只把相关流程交给生成器)。
// 只在规则与语义都未命中(fallback)时被调用;实现自担延迟与成本。
// 返回错误时识别保持 fallback,不会使 Recognize 失败。
//
// 库侧校验与 deriveCommands 同款:start 命令的 ActionRef 必须可解析——
// auto/command/workflow/blueprint 的目标必须在 catalog 的 spec ID 集合内,
// handler 的目标必须在 catalog 的 Handler 集合内;槽位键必须在目标 spec 的
// 槽位键集合内(可解析时)。生成器幻觉一律 fail-closed 回 fallback。
type CommandGenerator interface {
// Generate 基于输入与能力目录产出命令列表(catalog 由调用方按 top-k 给)。
Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error)
}
+234
View File
@@ -0,0 +1,234 @@
package intent
import (
"context"
"errors"
"strings"
"testing"
)
// fakeScorer 按 examples 里是否包含特定子串给分(测试用确定性打分器)。
type fakeScorer struct {
scores map[string]float64 // key: examples[0]
err error
}
func (f fakeScorer) Score(ctx context.Context, input string, examples []string) (float64, error) {
if f.err != nil {
return 0, f.err
}
if len(examples) == 0 {
return 0, nil
}
if s, ok := f.scores[examples[0]]; ok {
return s, nil
}
return 0, nil
}
func ambiguousFixture(t *testing.T, exA, exB []string) *Snapshot {
t.Helper()
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{ID: "demo.alpha", Domain: "demo", Version: 1, Priority: 5,
Matcher: MatchSpec{IncludesAny: []string{"模糊词"}}, Examples: exA}); err != nil {
t.Fatal(err)
}
if err := b.Register(IntentSpec{ID: "demo.beta", Domain: "demo", Version: 1, Priority: 5,
Matcher: MatchSpec{IncludesAny: []string{"模糊词"}}, Examples: exB}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
return reg.Snapshot()
}
// TestSemanticRerankResolvesAmbiguous:语义过阈且分差够 → matched(I2 分层)。
func TestSemanticRerankResolvesAmbiguous(t *testing.T) {
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"闲聊问候"})
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.95, "闲聊问候": 0.4}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "demo.alpha" {
t.Fatalf("semantic rerank: %+v", res)
}
if !strings.Contains(res.Top.Reason, "refined_by=semantic") {
t.Fatalf("reason = %q", res.Top.Reason)
}
}
// TestSemanticBelowThresholdKeepsAmbiguous:全部不达阈 → 保持 ambiguous。
func TestSemanticBelowThresholdKeepsAmbiguous(t *testing.T) {
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"闲聊问候"})
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.5}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous {
t.Fatalf("below threshold: branch = %s", res.Branch)
}
}
// TestSemanticTieKeepsAmbiguous:两个过阈但分差不足 → 保持 ambiguous。
func TestSemanticTieKeepsAmbiguous(t *testing.T) {
snap := ambiguousFixture(t, []string{"查运行进度"}, []string{"查运行状态"})
rec := NewRecognizer(Options{Semantic: fakeScorer{scores: map[string]float64{"查运行进度": 0.9, "查运行状态": 0.89}}}, nil, nil)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchAmbiguous {
t.Fatalf("tie: branch = %s", res.Branch)
}
}
// TestSemanticErrorKeepsAmbiguousAndCallsL2Next:打分错误 → ambiguous,且 L2 接管。
func TestSemanticErrorKeepsAmbiguousAndCallsL2Next(t *testing.T) {
snap := ambiguousFixture(t, []string{"查运行进度"}, nil)
called := false
cls := rewriteClassifier{fn: func(c []Candidate) []Candidate {
called = true
out := append([]Candidate(nil), c...)
out[0].Confidence = 0.99
out[1].Confidence = 0.1
return out
}}
rec := NewRecognizer(Options{Semantic: fakeScorer{err: errors.New("embed down")}}, nil, cls)
res, err := rec.Recognize(context.Background(), "模糊词", snap)
if err != nil {
t.Fatal(err)
}
if !called {
t.Fatal("L2 should be consulted after semantic failure")
}
if res.Branch != BranchMatched || res.Top.ID != "demo.alpha" {
t.Fatalf("L2 fallback after semantic error: %+v", res)
}
}
// TestGeneratorValidStartBecomesMatched:合法 start 命令 → matched + 合成 Top(I2)。
func TestGeneratorValidStartBecomesMatched(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{ID: "report.generate", Domain: "report", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
Slots: []SlotSpec{{Key: "topic", Pattern: `主题是(.+)`}},
Handler: "reportHandler"}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
gen := stubGenerator{cmds: []Command{
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "report.generate"},
Slots: map[string]string{"topic": "电机"}},
}}
rec := NewRecognizer(Options{Generator: gen}, nil, nil)
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", reg.Snapshot())
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchMatched || res.Top == nil || res.Top.ID != "report.generate" {
t.Fatalf("generated: %+v", res)
}
if len(res.Commands) != 1 || res.Commands[0].Kind != CommandStart || res.Commands[0].Slots["topic"] != "电机" {
t.Fatalf("commands = %+v", res.Commands)
}
}
// TestGeneratorInvalidOutputFallsBack:幻觉动作/槽位 → 保持 fallback(fail-closed)。
func TestGeneratorInvalidOutputFallsBack(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{ID: "report.generate", Domain: "report", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"生成报告"}},
Slots: []SlotSpec{{Key: "topic", Pattern: `(.+)`}}}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
cases := map[string][]Command{
"unknown spec id": {
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "ghost.flow"}},
},
"unknown kind": {
{Kind: CommandStart, Action: ActionRef{Kind: "bogus", ID: "report.generate"}},
},
"unknown slot key": {
{Kind: CommandStart, Action: ActionRef{Kind: ActionWorkflow, ID: "report.generate"}, Slots: map[string]string{"bogus": "v"}},
},
"unknown cmd kind": {
{Kind: "teleport"},
},
"empty": {},
}
for name, cmds := range cases {
rec := NewRecognizer(Options{Generator: stubGenerator{cmds: cmds}}, nil, nil)
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", snap)
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchFallback {
t.Fatalf("%s: branch = %s, want fallback", name, res.Branch)
}
}
}
// TestGeneratorErrorKeepsFallback:生成器错误 → 保持 fallback。
func TestGeneratorErrorKeepsFallback(t *testing.T) {
reg, err := NewRegistryBuilder(Options{}).Build()
if err != nil {
t.Fatal(err)
}
rec := NewRecognizer(Options{Generator: stubGenerator{err: errors.New("model down")}}, nil, nil)
res, err := rec.Recognize(context.Background(), "完全不相关的一句话", reg.Snapshot())
if err != nil {
t.Fatal(err)
}
if res.Branch != BranchFallback {
t.Fatalf("branch = %s", res.Branch)
}
}
// TestCatalogTopK:按 priority 降序取 k(确定性口径)。
func TestCatalogTopK(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{ID: "demo.low", Domain: "demo", Version: 1, Priority: 1, Matcher: MatchSpec{IncludesAny: []string{"a"}}}); err != nil {
t.Fatal(err)
}
if err := b.Register(IntentSpec{ID: "demo.high", Domain: "demo", Version: 1, Priority: 9, Matcher: MatchSpec{IncludesAny: []string{"b"}}}); err != nil {
t.Fatal(err)
}
if err := b.Register(IntentSpec{ID: "demo.mid", Domain: "demo", Version: 1, Priority: 5, Matcher: MatchSpec{IncludesAny: []string{"c"}}}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
top := reg.Snapshot().CatalogTopK(2)
if len(top) != 2 || top[0].ID != "demo.high" || top[1].ID != "demo.mid" {
t.Fatalf("topk = %+v", top)
}
if all := reg.Snapshot().CatalogTopK(0); len(all) != 3 {
t.Fatalf("k=0 should be full catalog: %d", len(all))
}
}
type stubGenerator struct {
cmds []Command
err error
}
func (s stubGenerator) Generate(ctx context.Context, input string, catalog []CatalogSpec) ([]Command, error) {
if s.err != nil {
return nil, s.err
}
return s.cmds, nil
}
+170 -4
View File
@@ -19,17 +19,22 @@ type Recognizer interface {
// FunnelRecognizer 四级漏斗实现:guard → expectation → rules → adjudicate,
// 可选注入挂起预期提供者与 Ambiguous 分支的 L2 分类器。
// I2 的分层:歧义先经语义重排(确定性宿主打分)再到 L2 分类器(模型)。
type FunnelRecognizer struct {
log *logsys.Logger
exp ExpectationProvider
cls Classifier
sem SemanticScorer
thr float64
gen CommandGenerator
}
// NewRecognizer 构建漏斗识别器。opts 仅使用 Log 字段(阈值类参数在快照内);
// exp / cls 可为 nil(对应阶段不启用)。
// exp / cls 可为 nil(对应阶段不启用)。语义与生成端口经 Options 注入(I2)。
func NewRecognizer(opts Options, exp ExpectationProvider, cls Classifier) *FunnelRecognizer {
o := opts.normalized()
return &FunnelRecognizer{log: o.Log, exp: exp, cls: cls}
return &FunnelRecognizer{log: o.Log, exp: exp, cls: cls,
sem: o.Semantic, thr: o.SemanticThreshold, gen: o.Generator}
}
// Recognize 识别一次输入。确定性:同输入 + 同快照 → 同结果(日志中的耗时除外)。
@@ -46,7 +51,10 @@ func (r *FunnelRecognizer) Recognize(ctx context.Context, input string, snap *Sn
}
start := time.Now()
res := r.recognize(ctx, input, snap)
res.Commands = deriveCommands(res, snap)
// 生成端口已经给出经过校验的命令列表时不再用规则展开覆盖(I2)。
if res.Commands == nil {
res.Commands = deriveCommands(res, snap)
}
topID, conf := "", 0.0
if res.Top != nil {
topID, conf = res.Top.ID, res.Top.Confidence
@@ -89,6 +97,12 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
hits := snap.evalRules(matchInput, folded)
switch {
case len(hits) == 0:
// I2:规则与语义都未命中时,可选经模型命令生成端口兜底(校验同款)。
if r.gen != nil {
if res, ok := r.generateCommands(ctx, input, snap); ok {
return res
}
}
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
case len(hits) == 1 || hits[0].score-hits[1].score >= snap.margin || tieBreakByPriority(hits):
cands := candidatesOf(trimHits(hits, snap.topN), input, true)
@@ -96,8 +110,15 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
return Result{Input: input, Branch: BranchMatched, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
}
// Ambiguous:可选 L2 分类器重排;失败或结果不合法都保持 Ambiguous。
// Ambiguous:先语义重排(I2 确定性宿主打分),失败或不达阈才到 L2 分类器。
cands := candidatesOf(trimHits(hits, snap.topN), input, false)
if r.sem != nil {
if top, ok := r.semanticRerank(ctx, input, cands, snap); ok {
snap.fillTopSlots(&top, input)
top.Reason = top.Reason + "; refined_by=semantic"
return Result{Input: input, Branch: BranchMatched, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
}
}
if r.cls != nil {
refined, err := r.cls.Refine(ctx, input, cands)
if err != nil {
@@ -263,3 +284,148 @@ func sortScored(ss []ScoredSpec) {
func sortRejected(rs []RejectedSpec) {
sort.SliceStable(rs, func(i, j int) bool { return rs[i].ID < rs[j].ID })
}
// semanticRerank 用宿主打分器对歧义候选按 examples 重排(I2)。
// 规则:候选必须来自当前候选集(构造上不可能引入未知 ID);
// 只取相似度 ≥ 阈值的候选——唯一过阈或前两名分差 ≥ margin 才判 matched,
// 否则保持 ambiguous(与规则裁决同款语义)。打分错误一律记日志并保持 ambiguous。
func (r *FunnelRecognizer) semanticRerank(ctx context.Context, input string, cands []Candidate, snap *Snapshot) (Candidate, bool) {
type scored struct {
cand Candidate
score float64
}
var above []scored
for _, c := range cands {
spec, ok := snap.SpecByID(c.ID)
if !ok || len(spec.Examples) == 0 {
continue
}
s, err := r.sem.Score(ctx, input, spec.Examples)
if err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent semantic scorer failed; keep ambiguous",
"error", err.Error(), "id", c.ID)
continue
}
if s >= r.thr {
above = append(above, scored{cand: c, score: s})
}
}
if len(above) == 0 {
return Candidate{}, false
}
for i := 0; i+1 < len(above); i++ {
for j := i + 1; j < len(above); j++ {
if above[j].score > above[i].score {
above[i], above[j] = above[j], above[i]
}
}
}
if len(above) == 1 || above[0].score-above[1].score >= snap.margin {
return above[0].cand, true
}
return Candidate{}, false
}
// generateCommands 经模型命令生成端口兜底(I2)。校验:
// start 的 ActionRef 可解析(spec ID 集或 Handler 集内)、命令种类受限、
// 槽位键在目标 spec 的键集内(可解析时);全部通过则把首个 start 的目标
// spec 合成 Top(refined_by=generator 标注),任一项不过则保持 fallback。
func (r *FunnelRecognizer) generateCommands(ctx context.Context, input string, snap *Snapshot) (Result, bool) {
catalog := snap.CatalogTopK(DefaultGeneratorCatalogK)
cmds, err := r.gen.Generate(ctx, input, catalog)
if err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent command generator failed; keep fallback", "error", err.Error())
return Result{}, false
}
if err := validateGenerated(cmds, catalog); err != nil {
r.log.Module("intent").WithContext(ctx).Warn("intent generator output rejected; keep fallback", "error", err.Error())
return Result{}, false
}
res := Result{Input: input, Branch: BranchFallback, Commands: cmds, SnapshotRevision: snap.rev}
for _, c := range cmds {
if c.Kind != CommandStart {
continue
}
if spec, ok := generatedSpecOf(c.Action, catalog); ok {
top := Candidate{
ID: spec.ID, Domain: spec.Domain, Version: spec.Version,
Confidence: 0, Reason: "generated_by=command_generator",
Slots: c.Slots,
}
res.Top = &top
res.Branch = BranchMatched
res.Candidates = []Candidate{top}
}
break
}
return res, true
}
// DefaultGeneratorCatalogK 生成器目录默认 top-k。
const DefaultGeneratorCatalogK = 8
// validateGenerated 校验生成命令(I2;与 spec 级校验同款 fail-closed)。
func validateGenerated(cmds []Command, catalog []CatalogSpec) error {
if len(cmds) == 0 {
return fmt.Errorf("empty command list")
}
specIDs := map[string]bool{}
handlers := map[string]bool{}
slotKeys := map[string]map[string]bool{}
for _, cs := range catalog {
specIDs[cs.ID] = true
if cs.Handler != "" {
handlers[cs.Handler] = true
}
keys := map[string]bool{}
for _, sl := range cs.Slots {
keys[sl.Key] = true
}
slotKeys[cs.ID] = keys
}
for i, c := range cmds {
switch c.Kind {
case CommandStart:
ref := c.Action
switch ref.Kind {
case ActionAuto, ActionCommand, ActionWorkflow, ActionBlueprint:
if !specIDs[ref.ID] {
return fmt.Errorf("cmd[%d] start: action.id %q 不在目录内", i, ref.ID)
}
case ActionHandler:
if !handlers[ref.ID] {
return fmt.Errorf("cmd[%d] start: handler %q 不在目录内", i, ref.ID)
}
default:
return fmt.Errorf("cmd[%d] start: 未知 action.kind %q", i, ref.Kind)
}
if keys, ok := slotKeys[ref.ID]; ok {
for k := range c.Slots {
if !keys[k] {
return fmt.Errorf("cmd[%d] start: 槽位 %q 不在 %s 的槽位键集内", i, k, ref.ID)
}
}
}
case CommandClarify, CommandCancel, CommandSetSlot, CommandChangeSlot, CommandChitchat, CommandHandoff:
default:
return fmt.Errorf("cmd[%d]: 未知命令种类 %q", i, c.Kind)
}
}
return nil
}
// generatedSpecOf 把 start 的 ActionRef 解析回目录 spec(handler 键按 Handler 反查)。
func generatedSpecOf(ref ActionRef, catalog []CatalogSpec) (CatalogSpec, bool) {
for _, cs := range catalog {
if ref.Kind == ActionHandler {
if cs.Handler == ref.ID {
return cs, true
}
continue
}
if cs.ID == ref.ID {
return cs, true
}
}
return CatalogSpec{}, false
}
+9
View File
@@ -35,6 +35,12 @@ type Options struct {
MaxSlotsPerSpec int
// Log 可选日志器;nil 时使用 Nop。识别日志不记录原始输入。
Log *logsys.Logger
// Semantic 语义相似度端口(I2):nil 时语义层不启用。
Semantic SemanticScorer
// SemanticThreshold 语义重排的过阈值(默认 0.8)。
SemanticThreshold float64
// Generator 模型命令生成端口(I2):nil 时 fallback 不生成。
Generator CommandGenerator
// SlotValidators 作用域槽位校验器。构建与校验时先查此表,查不到再回落
// 已弃用的全局表(RegisterSlotValidator)。不同 Registry 的同名校验器互不影响。
SlotValidators map[string]SlotValidator
@@ -61,6 +67,9 @@ func (o Options) normalized() Options {
if n.AmbiguityMargin <= 0 {
n.AmbiguityMargin = DefaultAmbiguityMargin
}
if n.SemanticThreshold <= 0 {
n.SemanticThreshold = DefaultSemanticThreshold
}
if n.MaxInputLen <= 0 {
n.MaxInputLen = DefaultMaxInputLen
}
+13
View File
@@ -0,0 +1,13 @@
package intent
import "context"
// SemanticScorer 语义相似度端口(I2 分层与评测):宿主注入向量模型,
// 核心不做 I/O——打分由宿主计算,库只消费分数。
// 只在规则裁决为 Ambiguous 时被调用(规则→语义→模型分层的中位);
// 实现应幂等且无副作用(同一输入与示例集应得到同一分数)。
type SemanticScorer interface {
// Score 返回 input 与 examples 的最高相似度,取值 [0,1]。
// 实现返回错误时识别保持 Ambiguous,不会使 Recognize 失败。
Score(ctx context.Context, input string, examples []string) (float64, error)
}
+5
View File
@@ -18,6 +18,8 @@ const (
DefaultBase = 0.7
DefaultPatternBonus = 0.2
DefaultThreshold = 0.6
// DefaultSemanticThreshold 语义重排默认过阈值。
DefaultSemanticThreshold = 0.8
)
// MatchSpec 声明式匹配规则。
@@ -87,6 +89,9 @@ type IntentSpec struct {
// 声明后优先于 Handler 与 Metadata["blueprint"] 被 ActionRef 采用。
// kind 必须是 command/workflow/blueprint/handler 之一。
Action *ActionRef `json:"action,omitempty"`
// Examples 示例句(I2 语义端口):歧义时语义打分器按它们路由
// (semantic-router 借鉴;向量模型由宿主注入,库不做 I/O)。
Examples []string `json:"examples,omitempty"`
}
var (