From edf2f759ae0d6b70b21bb03cd852e5f2fa15ac64 Mon Sep 17 00:00:00 2001 From: WeiCHEN Date: Tue, 8 Sep 2026 13:34:44 +0800 Subject: [PATCH] =?UTF-8?q?feat(ctxmgr):=20=E5=A2=9E=E5=8A=A0=E7=A6=BB?= =?UTF-8?q?=E7=BA=BF=E7=A1=AE=E5=AE=9A=E6=80=A7=E9=AB=98=E7=BB=B4=E5=B5=8C?= =?UTF-8?q?=E5=85=A5=E7=94=9F=E6=88=90=E5=99=A8=E4=B8=8E=E5=90=91=E9=87=8F?= =?UTF-8?q?=E5=AD=97=E9=9D=A2=E9=87=8F=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- internal/ctxmgr/embed.go | 113 ++++++++++++++++++++++++++++++++++ internal/ctxmgr/embed_test.go | 66 ++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 internal/ctxmgr/embed.go create mode 100644 internal/ctxmgr/embed_test.go diff --git a/internal/ctxmgr/embed.go b/internal/ctxmgr/embed.go new file mode 100644 index 00000000..8b87d812 --- /dev/null +++ b/internal/ctxmgr/embed.go @@ -0,0 +1,113 @@ +package ctxmgr + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "math" + "strings" +) + +// VectorEmbedder 统一向量化接口 +type VectorEmbedder interface { + Embed(ctx context.Context, texts []string) ([][]float32, error) +} + +// DeterministicEmbedder 确定性特征高维投影嵌入器（无外部网络依赖，测试与离线兜底） +type DeterministicEmbedder struct { + Dim int +} + +// NewDeterministicEmbedder 创建确定性嵌入生成器 +func NewDeterministicEmbedder(dim int) *DeterministicEmbedder { + if dim <= 0 { + dim = 1024 + } + return &DeterministicEmbedder{Dim: dim} +} + +// Embed 为文本切片生成归一化的特征向量 +func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) { + if len(texts) == 0 { + return nil, nil + } + + results := make([][]float32, len(texts)) + for idx, text := range texts { + vec := make([]float32, e.Dim) + cleaned := strings.TrimSpace(strings.ToLower(text)) + if cleaned == "" { + vec[0] = 1.0 + results[idx] = vec + continue + } + + // 分词提取（简单按空格或中文按单字/双字切分） + tokens := tokenizeForEmbedding(cleaned) + for _, tok := range tokens { + h := sha256.Sum256([]byte(tok)) + for i := 0; i < e.Dim; i++ { + offset := (i * 4) % (len(h) - 4) + val := binary.BigEndian.Uint32(h[offset : offset+4]) + // 映射到 [-1.0, 1.0] + fVal := float32((float64(val)/float64(math.MaxUint32))*2.0 - 1.0) + vec[i] += fVal + } + } + + // L2 归一化 (Unit Normalization) + var sumSq float64 + for _, v := range vec { + sumSq += float64(v * v) + } + norm := float32(math.Sqrt(sumSq)) + if norm > 1e-9 { + for i := range vec { + vec[i] /= norm + } + } else { + vec[0] = 1.0 + } + + results[idx] = vec + } + + return results, nil +} + +// tokenizeForEmbedding 字符级与词级特征切分 +func tokenizeForEmbedding(s string) []string { + var tokens []string + words := strings.Fields(s) + if len(words) > 1 { + tokens = append(tokens, words...) + } + + runes := []rune(s) + // 提取 2-gram 和单字符 + for i := 0; i < len(runes); i++ { + tokens = append(tokens, string(runes[i])) + if i+1 < len(runes) { + tokens = append(tokens, string(runes[i:i+2])) + } + } + return tokens +} + +// VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...] +func VectorLiteral(vec []float32) string { + if len(vec) == 0 { + return "" + } + var sb strings.Builder + sb.WriteByte('[') + for i, f := range vec { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(fmt.Sprintf("%g", f)) + } + sb.WriteByte(']') + return sb.String() +} diff --git a/internal/ctxmgr/embed_test.go b/internal/ctxmgr/embed_test.go new file mode 100644 index 00000000..db690624 --- /dev/null +++ b/internal/ctxmgr/embed_test.go @@ -0,0 +1,66 @@ +package ctxmgr_test + +import ( + "context" + "math" + "testing" + + "chat-agent-system/internal/ctxmgr" +) + +func TestDeterministicEmbedder_EmbeddingProperties(t *testing.T) { + emb := ctxmgr.NewDeterministicEmbedder(1024) + ctx := context.Background() + + texts := []string{ + "超声软组织切割止血手术设备", + "微创外科硬管内窥镜摄像系统", + "超声软组织切割止血手术设备", // 重复文本验证确定性 + } + + vecs, err := emb.Embed(ctx, texts) + if err != nil { + t.Fatalf("embed failed: %v", err) + } + if len(vecs) != 3 { + t.Fatalf("expected 3 vectors, got %d", len(vecs)) + } + + // 1. 验证维度 + for i, v := range vecs { + if len(v) != 1024 { + t.Fatalf("vector %d length expected 1024, got %d", i, len(v)) + } + // 验证单位化模长 (L2 norm ≈ 1.0) + var sumSq float64 + for _, val := range v { + sumSq += float64(val * val) + } + norm := math.Sqrt(sumSq) + if math.Abs(norm-1.0) > 1e-4 { + t.Fatalf("vector %d not normalized, norm=%f", i, norm) + } + } + + // 2. 验证确定性：相同文本向量完全一致 + for i := 0; i < 1024; i++ { + if vecs[0][i] != vecs[2][i] { + t.Fatalf("deterministic mismatch at index %d: %v vs %v", i, vecs[0][i], vecs[2][i]) + } + } + + // 3. 验证语义区别：不同文本余弦相似度应小于 0.95 + var dotProduct float32 + for i := 0; i < 1024; i++ { + dotProduct += vecs[0][i] * vecs[1][i] + } + if dotProduct > 0.95 { + t.Fatalf("distinct texts should not have near-identical embeddings: dotProduct=%f", dotProduct) + } + + // 4. 验证 VectorLiteral + lit := ctxmgr.VectorLiteral(vecs[0][:3]) + if lit == "" || lit[0] != '[' || lit[len(lit)-1] != ']' { + t.Fatalf("invalid vector literal: %s", lit) + } +}