From 05e74d8e320647fe41c4ca9cd20e31065323d6af Mon Sep 17 00:00:00 2001 From: weichen Date: Sat, 3 Oct 2026 12:56:55 +0000 Subject: [PATCH] Fold NFKC input and stop catalog metadata leaks Why: fullwidth text missed case-folded terms, SpecByID shared maps, the catalog prompt printed handlers and private metadata, overlapping slots kept both values, and negation was easy to mistake for semantics. Evidence: GOWORK=off go vet ./... && go test -count=1 -race ./... passed, including TestR2NFKCFoldsFullwidthButNotTraditional, TestR2SpecByIDDeepCopiesMetadata, TestR2CatalogHidesPrivateMetadataAndHandlerInMarkdown, TestR2EvalRulesDocumentsMaxSpecs, TestR2OverlappingSlotsAreAmbiguous, and TestR2NegationIsOnlyExcludesAny. Impact: NFKC is opt-in and does not convert traditional Chinese; catalog markdown omits handlers and non-allowlisted metadata; overlapping slot spans are invalid. Refs: I4, I7, I8, I10, I11, I12 Co-authored-by: Cursor --- go.mod | 2 + go.sum | 2 + intent/catalog.go | 4 +- intent/fold.go | 80 ++++++++++++++++ intent/r2_regression_test.go | 173 +++++++++++++++++++++++++++++++++++ intent/recognizer.go | 15 +-- intent/registry.go | 24 +++-- intent/slots.go | 67 ++++++++++++-- 8 files changed, 339 insertions(+), 28 deletions(-) create mode 100644 go.sum create mode 100644 intent/fold.go create mode 100644 intent/r2_regression_test.go diff --git a/go.mod b/go.mod index ac7fad7..13fa331 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,8 @@ require ( logging-system v0.0.0 ) +require golang.org/x/text v0.29.0 // indirect + replace error-code-system => ./deps/error-code-system replace logging-system => ./deps/logging-system diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3adec61 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= diff --git a/intent/catalog.go b/intent/catalog.go index cbd0219..ce85d3b 100644 --- a/intent/catalog.go +++ b/intent/catalog.go @@ -45,7 +45,7 @@ func (s *Snapshot) Catalog() []CatalogSpec { Risk: cs.spec.Risk, RequiresConfirm: cs.spec.RequiresConfirm, Handler: cs.spec.Handler, - Metadata: cs.spec.Metadata, + Metadata: catalogMetadata(cs.spec.Metadata), } if view.Risk == "" { view.Risk = RiskLow @@ -83,7 +83,7 @@ func (s *Snapshot) CatalogMarkdown() string { if sp.RequiresConfirm { confirm = "是" } - fmt.Fprintf(&b, "- 风险：%s；执行前需人工确认：%s；handler：%s\n", sp.Risk, confirm, sp.Handler) + fmt.Fprintf(&b, "- 风险：%s；执行前需人工确认：%s\n", sp.Risk, confirm) for _, sl := range sp.Slots { req := "选填" if sl.Required { diff --git a/intent/fold.go b/intent/fold.go new file mode 100644 index 0000000..c75d2fb --- /dev/null +++ b/intent/fold.go @@ -0,0 +1,80 @@ +package intent + +import ( + "strings" + + "golang.org/x/text/unicode/norm" +) + +// foldMatchText 做识别用折叠。nfkc 只做兼容等价（全角 ASCII、组合字符）， +// 不做繁简转换。caseSensitive 为真时保留大小写。 +func foldMatchText(s string, nfkc, caseSensitive bool) string { + if nfkc { + s = norm.NFKC.String(s) + } + if !caseSensitive { + s = strings.ToLower(s) + } + return s +} + +// foldInput 返回用于匹配的文本：第一段保留大小写（可选 NFKC），第二段是其小写。 +func (s *Snapshot) foldInput(input string) (matchInput, folded string) { + if s == nil { + return input, strings.ToLower(input) + } + matchInput = foldMatchText(input, s.nfkc, true) + return matchInput, strings.ToLower(matchInput) +} + +func cloneStrings(in []string) []string { + if in == nil { + return nil + } + return append([]string(nil), in...) +} + +func cloneIntentSpec(in IntentSpec) IntentSpec { + out := in + out.Matcher.IncludesAny = cloneStrings(in.Matcher.IncludesAny) + out.Matcher.IncludesAll = cloneStrings(in.Matcher.IncludesAll) + out.Matcher.ExcludesAny = cloneStrings(in.Matcher.ExcludesAny) + out.Matcher.Patterns = cloneStrings(in.Matcher.Patterns) + if in.Slots != nil { + out.Slots = append([]SlotSpec(nil), in.Slots...) + for i := range out.Slots { + out.Slots[i].Enum = cloneStrings(in.Slots[i].Enum) + out.Slots[i].Normalize = cloneStrings(in.Slots[i].Normalize) + } + } + if in.Metadata != nil { + out.Metadata = make(map[string]string, len(in.Metadata)) + for k, v := range in.Metadata { + out.Metadata[k] = v + } + } + return out +} + +// catalogMetadataAllow 是目录与 Markdown 允许带出的 metadata 键。 +// handler 仍留在 CatalogSpec 供执行器按名分派，但不写入 Markdown。 +var catalogMetadataAllow = map[string]bool{ + "blueprint": true, + "guidance": true, +} + +func catalogMetadata(in map[string]string) map[string]string { + if len(in) == 0 { + return nil + } + out := map[string]string{} + for k, v := range in { + if catalogMetadataAllow[k] { + out[k] = v + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/intent/r2_regression_test.go b/intent/r2_regression_test.go new file mode 100644 index 0000000..cae0a17 --- /dev/null +++ b/intent/r2_regression_test.go @@ -0,0 +1,173 @@ +package intent + +import ( + "context" + "strings" + "testing" +) + +func TestR2NFKCFoldsFullwidthButNotTraditional(t *testing.T) { + spec := IntentSpec{ + ID: "demo.greet.hello", Domain: "demo", Version: 1, + Matcher: MatchSpec{IncludesAny: []string{"hello"}}, + } + b := NewRegistryBuilder(Options{NFKC: true}) + if err := b.Register(spec); err != nil { + t.Fatal(err) + } + reg, err := b.Build() + if err != nil { + t.Fatal(err) + } + rec := NewRecognizer(Options{NFKC: true}, nil, nil) + res, err := rec.Recognize(context.Background(), "Ｈｅｌｌｏ", reg.Snapshot()) + if err != nil || res.Top == nil || res.Top.ID != spec.ID { + t.Fatalf("fullwidth hello: %+v %v", res, err) + } + trad, err := rec.Recognize(context.Background(), "寫入", reg.Snapshot()) + if err != nil { + t.Fatal(err) + } + if trad.Branch != BranchFallback { + t.Fatalf("NFKC must not convert traditional Chinese, got %+v", trad) + } +} + +func TestR2SpecByIDDeepCopiesMetadata(t *testing.T) { + b := NewRegistryBuilder(Options{}) + if err := b.Register(IntentSpec{ + ID: "demo.meta.copy", Domain: "demo", Version: 1, + Matcher: MatchSpec{IncludesAny: []string{"复制"}}, + Metadata: map[string]string{"guidance": "keep"}, + }); err != nil { + t.Fatal(err) + } + reg, err := b.Build() + if err != nil { + t.Fatal(err) + } + got, ok := reg.Snapshot().SpecByID("demo.meta.copy") + if !ok { + t.Fatal("missing spec") + } + got.Metadata["guidance"] = "mutated" + again, _ := reg.Snapshot().SpecByID("demo.meta.copy") + if again.Metadata["guidance"] != "keep" { + t.Fatalf("snapshot metadata changed: %q", again.Metadata["guidance"]) + } +} + +func TestR2CatalogHidesPrivateMetadataAndHandlerInMarkdown(t *testing.T) { + b := NewRegistryBuilder(Options{}) + if err := b.Register(IntentSpec{ + ID: "demo.catalog.public", Domain: "demo", Version: 1, + Handler: "secret.handler", + Matcher: MatchSpec{IncludesAny: []string{"目录"}}, + Metadata: map[string]string{ + "blueprint": "bp", + "token": "do-not-leak", + }, + }); err != nil { + t.Fatal(err) + } + reg, err := b.Build() + if err != nil { + t.Fatal(err) + } + snap := reg.Snapshot() + cat := snap.Catalog() + if len(cat) != 1 { + t.Fatalf("catalog: %+v", cat) + } + if cat[0].Metadata["blueprint"] != "bp" || cat[0].Metadata["token"] != "" { + t.Fatalf("metadata allowlist: %+v", cat[0].Metadata) + } + md := snap.CatalogMarkdown() + if strings.Contains(md, "secret.handler") || strings.Contains(md, "do-not-leak") || strings.Contains(md, "handler：") { + t.Fatalf("markdown leaked private fields:\n%s", md) + } +} + +func TestR2EvalRulesDocumentsMaxSpecs(t *testing.T) { + if DefaultMaxSpecs != 2048 { + t.Fatalf("linear scan cap changed: %d", DefaultMaxSpecs) + } + b := NewRegistryBuilder(Options{MaxSpecs: 1}) + if err := b.Register( + IntentSpec{ID: "demo.cap.a", Domain: "demo", Version: 1, Matcher: MatchSpec{IncludesAny: []string{"a"}}}, + IntentSpec{ID: "demo.cap.b", Domain: "demo", Version: 1, Matcher: MatchSpec{IncludesAny: []string{"b"}}}, + ); err != nil { + t.Fatal(err) + } + if _, err := b.Build(); err == nil { + t.Fatal("build must reject specs above MaxSpecs") + } +} + +func TestR2OverlappingSlotsAreAmbiguous(t *testing.T) { + b := NewRegistryBuilder(Options{}) + if err := b.Register(IntentSpec{ + ID: "demo.slot.overlap", Domain: "demo", Version: 1, + Matcher: MatchSpec{IncludesAny: []string{"abcd"}}, + Slots: []SlotSpec{ + {Key: "left", Pattern: "(abc)"}, + {Key: "right", Pattern: "(bcd)"}, + }, + }); err != nil { + t.Fatal(err) + } + reg, err := b.Build() + if err != nil { + t.Fatal(err) + } + res, err := NewRecognizer(Options{}, nil, nil).Recognize(context.Background(), "abcd", reg.Snapshot()) + if err != nil || res.Top == nil { + t.Fatalf("recognize: %+v %v", res, err) + } + if len(res.Top.Slots) != 0 { + t.Fatalf("overlapping slots must not be kept: %+v", res.Top.Slots) + } + if res.Top.InvalidSlots["left"] != "span overlap" || res.Top.InvalidSlots["right"] != "span overlap" { + t.Fatalf("invalid slots: %+v", res.Top.InvalidSlots) + } +} + +func TestR2NegationIsOnlyExcludesAny(t *testing.T) { + plain := IntentSpec{ + ID: "demo.write.file", Domain: "demo", Version: 1, + Matcher: MatchSpec{IncludesAny: []string{"写入"}}, + } + excluded := plain + excluded.ID = "demo.write.excluded" + excluded.Matcher.ExcludesAny = []string{"不要"} + rec := NewRecognizer(Options{}, nil, nil) + + plainBuilder := NewRegistryBuilder(Options{}) + if err := plainBuilder.Register(plain); err != nil { + t.Fatal(err) + } + built, err := plainBuilder.Build() + if err != nil { + t.Fatal(err) + } + hit, err := rec.Recognize(context.Background(), "不要写入文件", built.Snapshot()) + if err != nil || hit.Top == nil || hit.Top.ID != plain.ID { + t.Fatalf("without excludes_any the negated phrase still matches: %+v %v", hit, err) + } + + exBuilder := NewRegistryBuilder(Options{}) + if err := exBuilder.Register(excluded); err != nil { + t.Fatal(err) + } + builtEx, err := exBuilder.Build() + if err != nil { + t.Fatal(err) + } + miss, err := rec.Recognize(context.Background(), "不要写入文件", builtEx.Snapshot()) + if err != nil { + t.Fatal(err) + } + if miss.Branch != BranchFallback { + t.Fatalf("excludes_any must reject, got %+v", miss) + } +} diff --git a/intent/recognizer.go b/intent/recognizer.go index 0ef23ae..f101148 100644 --- a/intent/recognizer.go +++ b/intent/recognizer.go @@ -63,10 +63,11 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn if strings.TrimSpace(input) == "" { return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev} } - folded := strings.ToLower(input) + // 否定只认 excludes_any 子串，不做语义否定；「不要写入」这类说法交给命令层 Cancel。 + matchInput, folded := snap.foldInput(input) // Stage 0：守卫命中即短路，可表达"识别即拒绝"。 - if guardHits := snap.evalGuard(input, folded); len(guardHits) > 0 { + if guardHits := snap.evalGuard(matchInput, folded); len(guardHits) > 0 { cands := candidatesOf(trimHits(guardHits, snap.topN), input, true) top := cands[0] return Result{Input: input, Branch: BranchGuardHit, Top: &top, Candidates: cands, SnapshotRevision: snap.rev} @@ -75,7 +76,7 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn // Stage 1：会话挂起预期优先于一般规则（宿主可选注入）。 if r.exp != nil { if pending := r.exp.Pending(ctx); pending != nil && len(pending.IntentIDs) > 0 { - if hits := snap.evalIDs(pending.IntentIDs, input, folded); len(hits) > 0 { + if hits := snap.evalIDs(pending.IntentIDs, matchInput, folded); len(hits) > 0 { cands := candidatesOf(trimHits(hits, snap.topN), input, true) top := cands[0] return Result{Input: input, Branch: BranchExpectationHit, Top: &top, Candidates: cands, SnapshotRevision: snap.rev} @@ -84,7 +85,7 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn } // Stage 2 + 3：规则计分与置信度裁决。 - hits := snap.evalRules(input, folded) + hits := snap.evalRules(matchInput, folded) switch { case len(hits) == 0: return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev} @@ -196,11 +197,11 @@ func (s *Snapshot) Explain(input string) Trace { trace.Branch = BranchFallback return trace } - folded := strings.ToLower(input) + matchInput, folded := s.foldInput(input) for _, idx := range s.guardIdx { cs := s.specs[idx] - o := cs.evaluate(input, folded) + o := cs.evaluate(matchInput, folded) if o.passed { trace.Guard = append(trace.Guard, ScoredSpec{ID: cs.spec.ID, Score: o.score, HitTerms: o.hitTerms, HitPatterns: o.hitPat}) } else { @@ -220,7 +221,7 @@ func (s *Snapshot) Explain(input string) Trace { if cs.guard { continue } - o := cs.evaluate(input, folded) + o := cs.evaluate(matchInput, folded) if o.passed { hits = append(hits, specHit{cs: cs, score: o.score, terms: o.hitTerms, pats: o.hitPat}) trace.Rules = append(trace.Rules, ScoredSpec{ID: cs.spec.ID, Score: o.score, HitTerms: o.hitTerms, HitPatterns: o.hitPat}) diff --git a/intent/registry.go b/intent/registry.go index 02fe307..aa0d770 100644 --- a/intent/registry.go +++ b/intent/registry.go @@ -38,6 +38,9 @@ type Options struct { // SlotValidators 作用域槽位校验器。构建与校验时先查此表，查不到再回落 // 已弃用的全局表（RegisterSlotValidator）。不同 Registry 的同名校验器互不影响。 SlotValidators map[string]SlotValidator + // NFKC 为真时，词条与输入先做兼容等价归一（全角 ASCII、组合字符），再按 + // case_sensitive 决定是否小写。NFKC 不做繁简转换。 + NFKC bool } // Options 字段默认值。 @@ -173,16 +176,12 @@ func (cs *compiledSpec) evaluate(input, folded string) matchOutcome { return matchOutcome{score: math.Round(score*10000) / 10000, passed: score >= cs.threshold, hitTerms: hitTerms, hitPat: hitPat} } -func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotValidator) (*compiledSpec, error) { +func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotValidator, nfkc bool) (*compiledSpec, error) { m := spec.Matcher fold := func(terms []string) []string { out := make([]string, len(terms)) for i, t := range terms { - if m.CaseSensitive { - out[i] = t - } else { - out[i] = strings.ToLower(t) - } + out[i] = foldMatchText(t, nfkc, m.CaseSensitive) } return out } @@ -239,6 +238,7 @@ type Snapshot struct { disabled []string // 已禁用 id（overlay disable 与 spec.disabled 两个入口） sources map[string]string total int // 含禁用 + nfkc bool } // Revision 返回快照版本号，随每次成功 Reload 递增。 @@ -267,10 +267,11 @@ func (s *Snapshot) Source(id string) (string, bool) { return src, ok } -// SpecByID 返回指定 id 的启用 spec 定义副本。不存在或已禁用时第二个返回值为 false。 +// SpecByID 返回指定 id 的启用 spec 定义副本。切片与 Metadata 深拷贝， +// 调用方改返回值不会改到快照。不存在或已禁用时第二个返回值为 false。 func (s *Snapshot) SpecByID(id string) (IntentSpec, bool) { if idx, ok := s.byID[id]; ok { - return s.specs[idx].spec, true + return cloneIntentSpec(s.specs[idx].spec), true } return IntentSpec{}, false } @@ -367,7 +368,9 @@ func (s *Snapshot) evalIDs(ids []string, input, folded string) []specHit { return hits } -// evalRules 评估全部非守卫 spec，返回过阈命中（已排序）。 +// evalRules 对全部非守卫 spec 做线性扫描，返回过阈命中（已排序）。 +// 规模上限是 Options.MaxSpecs（默认 DefaultMaxSpecs = 2048）。倒排索引留到 +// 超过该上限仍不够用时再做；构建期已经拒绝超过上限的快照。 func (s *Snapshot) evalRules(input, folded string) []specHit { var hits []specHit for _, cs := range s.specs { @@ -632,6 +635,7 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry, disabled: disabledList, sources: sources, total: total, + nfkc: opts.NFKC, } nextIndex := 0 for _, spec := range ordered { @@ -639,7 +643,7 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry, snap.disabled = append(snap.disabled, spec.ID) continue } - cs, err := compileSpec(spec, guards[spec.Domain], nextIndex, opts.SlotValidators) + cs, err := compileSpec(spec, guards[spec.Domain], nextIndex, opts.SlotValidators, opts.NFKC) if err != nil { return nil, err } diff --git a/intent/slots.go b/intent/slots.go index 95959b4..fc3f186 100644 --- a/intent/slots.go +++ b/intent/slots.go @@ -87,33 +87,82 @@ func (s *compiledSlot) validate(v string) (string, bool) { return "", true } +// slotCapture 一次槽位抽取的捕获区间。多槽位各自第一次匹配；区间重叠时 +// 两边都标 span overlap，不写入 slots。库内不做重叠消解。 +type slotCapture struct { + key string + start int + end int + value string + ok bool + reason string + required bool + matched bool +} + // extractSlots 抽取并归一化槽位。 -// missing：必填槽位未捕获（模式没匹配上）；invalid：捕获到但校验不合法（key→原因）。 +// missing：必填槽位未捕获（模式没匹配上）；invalid：捕获到但校验不合法，或与另一槽位区间重叠（key→原因）。 // 必填且非法只进 invalid（宿主可区分"没给"与"给错了"来决定澄清话术）。 func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, missing []string, invalid map[string]string) { if len(cs.slots) == 0 { return nil, nil, nil } + caps := make([]slotCapture, 0, len(cs.slots)) for _, s := range cs.slots { - m := s.re.FindStringSubmatch(input) - if m == nil || s.group >= len(m) || m[s.group] == "" { - if s.required { - missing = append(missing, s.key) + loc := s.re.FindStringSubmatchIndex(input) + g := s.group + if loc == nil || 2*g+1 >= len(loc) || loc[2*g] < 0 || loc[2*g] == loc[2*g+1] { + caps = append(caps, slotCapture{key: s.key, required: s.required}) + continue + } + raw := input[loc[2*g]:loc[2*g+1]] + v := normalizeSlotValue(raw, s.normalize) + reason, ok := s.validate(v) + caps = append(caps, slotCapture{ + key: s.key, start: loc[2*g], end: loc[2*g+1], value: v, + ok: ok, reason: reason, required: s.required, matched: true, + }) + } + overlap := map[int]bool{} + for i := range caps { + if !caps[i].matched { + continue + } + for j := i + 1; j < len(caps); j++ { + if !caps[j].matched { + continue + } + if caps[i].start < caps[j].end && caps[j].start < caps[i].end { + overlap[i] = true + overlap[j] = true + } + } + } + for i, c := range caps { + if !c.matched { + if c.required { + missing = append(missing, c.key) } continue } - v := normalizeSlotValue(m[s.group], s.normalize) - if reason, ok := s.validate(v); !ok { + if overlap[i] { if invalid == nil { invalid = map[string]string{} } - invalid[s.key] = reason + invalid[c.key] = "span overlap" + continue + } + if !c.ok { + if invalid == nil { + invalid = map[string]string{} + } + invalid[c.key] = c.reason continue } if slots == nil { slots = map[string]string{} } - slots[s.key] = v + slots[c.key] = c.value } return slots, missing, invalid }