118 lines
2.6 KiB
Go
118 lines
2.6 KiB
Go
package ctxmgr
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
)
|
|
|
|
// VectorEmbedder 统一向量化接口
|
|
type VectorEmbedder interface {
|
|
Embed(ctx context.Context, texts []string) ([][]float32, error)
|
|
}
|
|
|
|
// DeterministicEmbedder 确定性特征高维投影嵌入器（无外部网络依赖，测试与离线兜底）
|
|
type DeterministicEmbedder struct {
|
|
Dim int
|
|
}
|
|
|
|
// NewDeterministicEmbedder 创建确定性嵌入生成器
|
|
func NewDeterministicEmbedder(dim int) *DeterministicEmbedder {
|
|
if dim <= 0 {
|
|
dim = 1024
|
|
}
|
|
return &DeterministicEmbedder{Dim: dim}
|
|
}
|
|
|
|
// Embed 为文本切片生成归一化的特征向量
|
|
func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
|
if len(texts) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
results := make([][]float32, len(texts))
|
|
for idx, text := range texts {
|
|
vec := make([]float32, e.Dim)
|
|
cleaned := strings.TrimSpace(strings.ToLower(text))
|
|
if cleaned == "" {
|
|
vec[0] = 1.0
|
|
results[idx] = vec
|
|
continue
|
|
}
|
|
|
|
// 分词提取（简单按空格或中文按单字/双字切分）
|
|
tokens := tokenizeForEmbedding(cleaned)
|
|
for _, tok := range tokens {
|
|
h := sha256.Sum256([]byte(tok))
|
|
// 将每个 token 投射到特征索引上
|
|
dimIdx := int(binary.BigEndian.Uint32(h[0:4])) % e.Dim
|
|
signVal := float32(1.0)
|
|
if h[4]&1 == 1 {
|
|
signVal = -1.0
|
|
}
|
|
vec[dimIdx] += signVal
|
|
|
|
// 次级哈希散列
|
|
dimIdx2 := int(binary.BigEndian.Uint32(h[5:9])) % e.Dim
|
|
vec[dimIdx2] += signVal * 0.5
|
|
}
|
|
|
|
// L2 归一化 (Unit Normalization)
|
|
var sumSq float64
|
|
for _, v := range vec {
|
|
sumSq += float64(v * v)
|
|
}
|
|
norm := float32(math.Sqrt(sumSq))
|
|
if norm > 1e-9 {
|
|
for i := range vec {
|
|
vec[i] /= norm
|
|
}
|
|
} else {
|
|
vec[0] = 1.0
|
|
}
|
|
|
|
results[idx] = vec
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
// tokenizeForEmbedding 字符级与词级特征切分
|
|
func tokenizeForEmbedding(s string) []string {
|
|
var tokens []string
|
|
words := strings.Fields(s)
|
|
if len(words) > 1 {
|
|
tokens = append(tokens, words...)
|
|
}
|
|
|
|
runes := []rune(s)
|
|
// 提取 1-gram 和 2-gram 特征
|
|
for i := 0; i < len(runes); i++ {
|
|
tokens = append(tokens, string(runes[i]))
|
|
if i+1 < len(runes) {
|
|
tokens = append(tokens, string(runes[i:i+2]))
|
|
}
|
|
}
|
|
return tokens
|
|
}
|
|
|
|
// VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...]
|
|
func VectorLiteral(vec []float32) string {
|
|
if len(vec) == 0 {
|
|
return ""
|
|
}
|
|
var sb strings.Builder
|
|
sb.WriteByte('[')
|
|
for i, f := range vec {
|
|
if i > 0 {
|
|
sb.WriteByte(',')
|
|
}
|
|
sb.WriteString(fmt.Sprintf("%g", f))
|
|
}
|
|
sb.WriteByte(']')
|
|
return sb.String()
|
|
}
|