weichen/Agent-Intent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fold NFKC input and stop catalog metadata leaks #4

Merged
weichen merged 1 commits from feature/agent-stack-r2 into main 2026-10-03 13:22:54 +00:00
Conversation 0 Commits 1 Files Changed 8
+339 -28
8 changed files with 339 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2
View File
@@ -7,6 +7,8 @@ require (
logging-system v0.0.0
)
require golang.org/x/text v0.29.0 // indirect
replace error-code-system => ./deps/error-code-system
replace logging-system => ./deps/logging-system
go.sum
+2
View File
@@ -0,0 +1,2 @@
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
intent/catalog.go
+2 -2
View File
@@ -45,7 +45,7 @@ func (s *Snapshot) Catalog() []CatalogSpec {
Risk: cs.spec.Risk,
RequiresConfirm: cs.spec.RequiresConfirm,
Handler: cs.spec.Handler,
Metadata: cs.spec.Metadata,
Metadata: catalogMetadata(cs.spec.Metadata),
}
if view.Risk == "" {
view.Risk = RiskLow
@@ -83,7 +83,7 @@ func (s *Snapshot) CatalogMarkdown() string {
if sp.RequiresConfirm {
confirm = "是"
}
fmt.Fprintf(&b, "- 风险：%s；执行前需人工确认：%s；handler：%s\n", sp.Risk, confirm, sp.Handler)
fmt.Fprintf(&b, "- 风险：%s；执行前需人工确认：%s\n", sp.Risk, confirm)
for _, sl := range sp.Slots {
req := "选填"
if sl.Required {
intent/fold.go
+80
View File
@@ -0,0 +1,80 @@
package intent
import (
"strings"
"golang.org/x/text/unicode/norm"
)
// foldMatchText 做识别用折叠。nfkc 只做兼容等价（全角 ASCII、组合字符），
// 不做繁简转换。caseSensitive 为真时保留大小写。
func foldMatchText(s string, nfkc, caseSensitive bool) string {
if nfkc {
s = norm.NFKC.String(s)
}
if !caseSensitive {
s = strings.ToLower(s)
}
return s
}
// foldInput 返回用于匹配的文本：第一段保留大小写（可选 NFKC），第二段是其小写。
func (s *Snapshot) foldInput(input string) (matchInput, folded string) {
if s == nil {
return input, strings.ToLower(input)
}
matchInput = foldMatchText(input, s.nfkc, true)
return matchInput, strings.ToLower(matchInput)
}
func cloneStrings(in []string) []string {
if in == nil {
return nil
}
return append([]string(nil), in...)
}
func cloneIntentSpec(in IntentSpec) IntentSpec {
out := in
out.Matcher.IncludesAny = cloneStrings(in.Matcher.IncludesAny)
out.Matcher.IncludesAll = cloneStrings(in.Matcher.IncludesAll)
out.Matcher.ExcludesAny = cloneStrings(in.Matcher.ExcludesAny)
out.Matcher.Patterns = cloneStrings(in.Matcher.Patterns)
if in.Slots != nil {
out.Slots = append([]SlotSpec(nil), in.Slots...)
for i := range out.Slots {
out.Slots[i].Enum = cloneStrings(in.Slots[i].Enum)
out.Slots[i].Normalize = cloneStrings(in.Slots[i].Normalize)
}
}
if in.Metadata != nil {
out.Metadata = make(map[string]string, len(in.Metadata))
for k, v := range in.Metadata {
out.Metadata[k] = v
}
}
return out
}
// catalogMetadataAllow 是目录与 Markdown 允许带出的 metadata 键。
// handler 仍留在 CatalogSpec 供执行器按名分派，但不写入 Markdown。
var catalogMetadataAllow = map[string]bool{
"blueprint": true,
"guidance": true,
}
func catalogMetadata(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := map[string]string{}
for k, v := range in {
if catalogMetadataAllow[k] {
out[k] = v
}
}
if len(out) == 0 {
return nil
}
return out
}
intent/r2_regression_test.go
+173
View File
@@ -0,0 +1,173 @@
package intent
import (
"context"
"strings"
"testing"
)
func TestR2NFKCFoldsFullwidthButNotTraditional(t *testing.T) {
spec := IntentSpec{
ID: "demo.greet.hello", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"hello"}},
}
b := NewRegistryBuilder(Options{NFKC: true})
if err := b.Register(spec); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
rec := NewRecognizer(Options{NFKC: true}, nil, nil)
res, err := rec.Recognize(context.Background(), "Ｈｅｌｌｏ", reg.Snapshot())
if err != nil || res.Top == nil || res.Top.ID != spec.ID {
t.Fatalf("fullwidth hello: %+v %v", res, err)
}
trad, err := rec.Recognize(context.Background(), "寫入", reg.Snapshot())
if err != nil {
t.Fatal(err)
}
if trad.Branch != BranchFallback {
t.Fatalf("NFKC must not convert traditional Chinese, got %+v", trad)
}
}
func TestR2SpecByIDDeepCopiesMetadata(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{
ID: "demo.meta.copy", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"复制"}},
Metadata: map[string]string{"guidance": "keep"},
}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
got, ok := reg.Snapshot().SpecByID("demo.meta.copy")
if !ok {
t.Fatal("missing spec")
}
got.Metadata["guidance"] = "mutated"
again, _ := reg.Snapshot().SpecByID("demo.meta.copy")
if again.Metadata["guidance"] != "keep" {
t.Fatalf("snapshot metadata changed: %q", again.Metadata["guidance"])
}
}
func TestR2CatalogHidesPrivateMetadataAndHandlerInMarkdown(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{
ID: "demo.catalog.public", Domain: "demo", Version: 1,
Handler: "secret.handler",
Matcher: MatchSpec{IncludesAny: []string{"目录"}},
Metadata: map[string]string{
"blueprint": "bp",
"token": "do-not-leak",
},
}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
snap := reg.Snapshot()
cat := snap.Catalog()
if len(cat) != 1 {
t.Fatalf("catalog: %+v", cat)
}
if cat[0].Metadata["blueprint"] != "bp" || cat[0].Metadata["token"] != "" {
t.Fatalf("metadata allowlist: %+v", cat[0].Metadata)
}
md := snap.CatalogMarkdown()
if strings.Contains(md, "secret.handler") || strings.Contains(md, "do-not-leak") || strings.Contains(md, "handler：") {
t.Fatalf("markdown leaked private fields:\n%s", md)
}
}
func TestR2EvalRulesDocumentsMaxSpecs(t *testing.T) {
if DefaultMaxSpecs != 2048 {
t.Fatalf("linear scan cap changed: %d", DefaultMaxSpecs)
}
b := NewRegistryBuilder(Options{MaxSpecs: 1})
if err := b.Register(
IntentSpec{ID: "demo.cap.a", Domain: "demo", Version: 1, Matcher: MatchSpec{IncludesAny: []string{"a"}}},
IntentSpec{ID: "demo.cap.b", Domain: "demo", Version: 1, Matcher: MatchSpec{IncludesAny: []string{"b"}}},
); err != nil {
t.Fatal(err)
}
if _, err := b.Build(); err == nil {
t.Fatal("build must reject specs above MaxSpecs")
}
}
func TestR2OverlappingSlotsAreAmbiguous(t *testing.T) {
b := NewRegistryBuilder(Options{})
if err := b.Register(IntentSpec{
ID: "demo.slot.overlap", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"abcd"}},
Slots: []SlotSpec{
{Key: "left", Pattern: "(abc)"},
{Key: "right", Pattern: "(bcd)"},
},
}); err != nil {
t.Fatal(err)
}
reg, err := b.Build()
if err != nil {
t.Fatal(err)
}
res, err := NewRecognizer(Options{}, nil, nil).Recognize(context.Background(), "abcd", reg.Snapshot())
if err != nil || res.Top == nil {
t.Fatalf("recognize: %+v %v", res, err)
}
if len(res.Top.Slots) != 0 {
t.Fatalf("overlapping slots must not be kept: %+v", res.Top.Slots)
}
if res.Top.InvalidSlots["left"] != "span overlap" || res.Top.InvalidSlots["right"] != "span overlap" {
t.Fatalf("invalid slots: %+v", res.Top.InvalidSlots)
}
}
func TestR2NegationIsOnlyExcludesAny(t *testing.T) {
plain := IntentSpec{
ID: "demo.write.file", Domain: "demo", Version: 1,
Matcher: MatchSpec{IncludesAny: []string{"写入"}},
}
excluded := plain
excluded.ID = "demo.write.excluded"
excluded.Matcher.ExcludesAny = []string{"不要"}
rec := NewRecognizer(Options{}, nil, nil)
plainBuilder := NewRegistryBuilder(Options{})
if err := plainBuilder.Register(plain); err != nil {
t.Fatal(err)
}
built, err := plainBuilder.Build()
if err != nil {
t.Fatal(err)
}
hit, err := rec.Recognize(context.Background(), "不要写入文件", built.Snapshot())
if err != nil || hit.Top == nil || hit.Top.ID != plain.ID {
t.Fatalf("without excludes_any the negated phrase still matches: %+v %v", hit, err)
}
exBuilder := NewRegistryBuilder(Options{})
if err := exBuilder.Register(excluded); err != nil {
t.Fatal(err)
}
builtEx, err := exBuilder.Build()
if err != nil {
t.Fatal(err)
}
miss, err := rec.Recognize(context.Background(), "不要写入文件", builtEx.Snapshot())
if err != nil {
t.Fatal(err)
}
if miss.Branch != BranchFallback {
t.Fatalf("excludes_any must reject, got %+v", miss)
}
}
intent/recognizer.go
+8 -7
View File
@@ -63,10 +63,11 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
if strings.TrimSpace(input) == "" {
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
}
folded := strings.ToLower(input)
// 否定只认 excludes_any 子串，不做语义否定；「不要写入」这类说法交给命令层 Cancel。
matchInput, folded := snap.foldInput(input)
// Stage 0：守卫命中即短路，可表达"识别即拒绝"。
if guardHits := snap.evalGuard(input, folded); len(guardHits) > 0 {
if guardHits := snap.evalGuard(matchInput, folded); len(guardHits) > 0 {
cands := candidatesOf(trimHits(guardHits, snap.topN), input, true)
top := cands[0]
return Result{Input: input, Branch: BranchGuardHit, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
@@ -75,7 +76,7 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
// Stage 1：会话挂起预期优先于一般规则（宿主可选注入）。
if r.exp != nil {
if pending := r.exp.Pending(ctx); pending != nil && len(pending.IntentIDs) > 0 {
if hits := snap.evalIDs(pending.IntentIDs, input, folded); len(hits) > 0 {
if hits := snap.evalIDs(pending.IntentIDs, matchInput, folded); len(hits) > 0 {
cands := candidatesOf(trimHits(hits, snap.topN), input, true)
top := cands[0]
return Result{Input: input, Branch: BranchExpectationHit, Top: &top, Candidates: cands, SnapshotRevision: snap.rev}
@@ -84,7 +85,7 @@ func (r *FunnelRecognizer) recognize(ctx context.Context, input string, snap *Sn
}
// Stage 2 + 3：规则计分与置信度裁决。
hits := snap.evalRules(input, folded)
hits := snap.evalRules(matchInput, folded)
switch {
case len(hits) == 0:
return Result{Input: input, Branch: BranchFallback, SnapshotRevision: snap.rev}
@@ -196,11 +197,11 @@ func (s *Snapshot) Explain(input string) Trace {
trace.Branch = BranchFallback
return trace
}
folded := strings.ToLower(input)
matchInput, folded := s.foldInput(input)
for _, idx := range s.guardIdx {
cs := s.specs[idx]
o := cs.evaluate(input, folded)
o := cs.evaluate(matchInput, folded)
if o.passed {
trace.Guard = append(trace.Guard, ScoredSpec{ID: cs.spec.ID, Score: o.score, HitTerms: o.hitTerms, HitPatterns: o.hitPat})
} else {
@@ -220,7 +221,7 @@ func (s *Snapshot) Explain(input string) Trace {
if cs.guard {
continue
}
o := cs.evaluate(input, folded)
o := cs.evaluate(matchInput, folded)
if o.passed {
hits = append(hits, specHit{cs: cs, score: o.score, terms: o.hitTerms, pats: o.hitPat})
trace.Rules = append(trace.Rules, ScoredSpec{ID: cs.spec.ID, Score: o.score, HitTerms: o.hitTerms, HitPatterns: o.hitPat})
intent/registry.go
+14 -10
View File
@@ -38,6 +38,9 @@ type Options struct {
// SlotValidators 作用域槽位校验器。构建与校验时先查此表，查不到再回落
// 已弃用的全局表（RegisterSlotValidator）。不同 Registry 的同名校验器互不影响。
SlotValidators map[string]SlotValidator
// NFKC 为真时，词条与输入先做兼容等价归一（全角 ASCII、组合字符），再按
// case_sensitive 决定是否小写。NFKC 不做繁简转换。
NFKC bool
}
// Options 字段默认值。
@@ -173,16 +176,12 @@ func (cs *compiledSpec) evaluate(input, folded string) matchOutcome {
return matchOutcome{score: math.Round(score*10000) / 10000, passed: score >= cs.threshold, hitTerms: hitTerms, hitPat: hitPat}
}
func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotValidator) (*compiledSpec, error) {
func compileSpec(spec IntentSpec, guard bool, index int, scoped map[string]SlotValidator, nfkc bool) (*compiledSpec, error) {
m := spec.Matcher
fold := func(terms []string) []string {
out := make([]string, len(terms))
for i, t := range terms {
if m.CaseSensitive {
out[i] = t
} else {
out[i] = strings.ToLower(t)
}
out[i] = foldMatchText(t, nfkc, m.CaseSensitive)
}
return out
}
@@ -239,6 +238,7 @@ type Snapshot struct {
disabled []string // 已禁用 id（overlay disable 与 spec.disabled 两个入口）
sources map[string]string
total int // 含禁用
nfkc bool
}
// Revision 返回快照版本号，随每次成功 Reload 递增。
@@ -267,10 +267,11 @@ func (s *Snapshot) Source(id string) (string, bool) {
return src, ok
}
// SpecByID 返回指定 id 的启用 spec 定义副本。不存在或已禁用时第二个返回值为 false。
// SpecByID 返回指定 id 的启用 spec 定义副本。切片与 Metadata 深拷贝，
// 调用方改返回值不会改到快照。不存在或已禁用时第二个返回值为 false。
func (s *Snapshot) SpecByID(id string) (IntentSpec, bool) {
if idx, ok := s.byID[id]; ok {
return s.specs[idx].spec, true
return cloneIntentSpec(s.specs[idx].spec), true
}
return IntentSpec{}, false
}
@@ -367,7 +368,9 @@ func (s *Snapshot) evalIDs(ids []string, input, folded string) []specHit {
return hits
}
// evalRules 评估全部非守卫 spec，返回过阈命中（已排序）。
// evalRules 对全部非守卫 spec 做线性扫描，返回过阈命中（已排序）。
// 规模上限是 Options.MaxSpecs（默认 DefaultMaxSpecs = 2048）。倒排索引留到
// 超过该上限仍不够用时再做；构建期已经拒绝超过上限的快照。
func (s *Snapshot) evalRules(input, folded string) []specHit {
var hits []specHit
for _, cs := range s.specs {
@@ -632,6 +635,7 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry,
disabled: disabledList,
sources: sources,
total: total,
nfkc: opts.NFKC,
}
nextIndex := 0
for _, spec := range ordered {
@@ -639,7 +643,7 @@ func buildSnapshot(opts Options, baseline []IntentSpec, entries []overlayEntry,
snap.disabled = append(snap.disabled, spec.ID)
continue
}
cs, err := compileSpec(spec, guards[spec.Domain], nextIndex, opts.SlotValidators)
cs, err := compileSpec(spec, guards[spec.Domain], nextIndex, opts.SlotValidators, opts.NFKC)
if err != nil {
return nil, err
}
intent/slots.go
+58 -9
View File
@@ -87,33 +87,82 @@ func (s *compiledSlot) validate(v string) (string, bool) {
return "", true
}
// slotCapture 一次槽位抽取的捕获区间。多槽位各自第一次匹配；区间重叠时
// 两边都标 span overlap，不写入 slots。库内不做重叠消解。
type slotCapture struct {
key string
start int
end int
value string
ok bool
reason string
required bool
matched bool
}
// extractSlots 抽取并归一化槽位。
// missing：必填槽位未捕获（模式没匹配上）；invalid：捕获到但校验不合法（key→原因）。
// missing：必填槽位未捕获（模式没匹配上）；invalid：捕获到但校验不合法，或与另一槽位区间重叠（key→原因）。
// 必填且非法只进 invalid（宿主可区分"没给"与"给错了"来决定澄清话术）。
func (cs *compiledSpec) extractSlots(input string) (slots map[string]string, missing []string, invalid map[string]string) {
if len(cs.slots) == 0 {
return nil, nil, nil
}
caps := make([]slotCapture, 0, len(cs.slots))
for _, s := range cs.slots {
m := s.re.FindStringSubmatch(input)
if m == nil || s.group >= len(m) || m[s.group] == "" {
if s.required {
missing = append(missing, s.key)
loc := s.re.FindStringSubmatchIndex(input)
g := s.group
if loc == nil || 2*g+1 >= len(loc) || loc[2*g] < 0 || loc[2*g] == loc[2*g+1] {
caps = append(caps, slotCapture{key: s.key, required: s.required})
continue
}
raw := input[loc[2*g]:loc[2*g+1]]
v := normalizeSlotValue(raw, s.normalize)
reason, ok := s.validate(v)
caps = append(caps, slotCapture{
key: s.key, start: loc[2*g], end: loc[2*g+1], value: v,
ok: ok, reason: reason, required: s.required, matched: true,
})
}
overlap := map[int]bool{}
for i := range caps {
if !caps[i].matched {
continue
}
for j := i + 1; j < len(caps); j++ {
if !caps[j].matched {
continue
}
if caps[i].start < caps[j].end && caps[j].start < caps[i].end {
overlap[i] = true
overlap[j] = true
}
}
}
for i, c := range caps {
if !c.matched {
if c.required {
missing = append(missing, c.key)
}
continue
}
v := normalizeSlotValue(m[s.group], s.normalize)
if reason, ok := s.validate(v); !ok {
if overlap[i] {
if invalid == nil {
invalid = map[string]string{}
}
invalid[s.key] = reason
invalid[c.key] = "span overlap"
continue
}
if !c.ok {
if invalid == nil {
invalid = map[string]string{}
}
invalid[c.key] = c.reason
continue
}
if slots == nil {
slots = map[string]string{}
}
slots[s.key] = v
slots[c.key] = c.value
}
return slots, missing, invalid
}