package ctxmgr import ( "context" "crypto/sha256" "encoding/binary" "fmt" "math" "strings" ) // VectorEmbedder 统一向量化接口 type VectorEmbedder interface { Embed(ctx context.Context, texts []string) ([][]float32, error) } // DeterministicEmbedder 确定性特征高维投影嵌入器(无外部网络依赖,测试与离线兜底) type DeterministicEmbedder struct { Dim int } // NewDeterministicEmbedder 创建确定性嵌入生成器 func NewDeterministicEmbedder(dim int) *DeterministicEmbedder { if dim <= 0 { dim = 1024 } return &DeterministicEmbedder{Dim: dim} } // Embed 为文本切片生成归一化的特征向量 func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) { if len(texts) == 0 { return nil, nil } results := make([][]float32, len(texts)) for idx, text := range texts { vec := make([]float32, e.Dim) cleaned := strings.TrimSpace(strings.ToLower(text)) if cleaned == "" { vec[0] = 1.0 results[idx] = vec continue } // 分词提取(简单按空格或中文按单字/双字切分) tokens := tokenizeForEmbedding(cleaned) for _, tok := range tokens { h := sha256.Sum256([]byte(tok)) // 将每个 token 投射到特征索引上 dimIdx := int(binary.BigEndian.Uint32(h[0:4])) % e.Dim signVal := float32(1.0) if h[4]&1 == 1 { signVal = -1.0 } vec[dimIdx] += signVal // 次级哈希散列 dimIdx2 := int(binary.BigEndian.Uint32(h[5:9])) % e.Dim vec[dimIdx2] += signVal * 0.5 } // L2 归一化 (Unit Normalization) var sumSq float64 for _, v := range vec { sumSq += float64(v * v) } norm := float32(math.Sqrt(sumSq)) if norm > 1e-9 { for i := range vec { vec[i] /= norm } } else { vec[0] = 1.0 } results[idx] = vec } return results, nil } // tokenizeForEmbedding 字符级与词级特征切分 func tokenizeForEmbedding(s string) []string { var tokens []string words := strings.Fields(s) if len(words) > 1 { tokens = append(tokens, words...) } runes := []rune(s) // 提取 1-gram 和 2-gram 特征 for i := 0; i < len(runes); i++ { tokens = append(tokens, string(runes[i])) if i+1 < len(runes) { tokens = append(tokens, string(runes[i:i+2])) } } return tokens } // VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...] func VectorLiteral(vec []float32) string { if len(vec) == 0 { return "" } var sb strings.Builder sb.WriteByte('[') for i, f := range vec { if i > 0 { sb.WriteByte(',') } sb.WriteString(fmt.Sprintf("%g", f)) } sb.WriteByte(']') return sb.String() }