weichen
5d5cef9fcd
ci / vet-and-test (pull_request) Successful in 56s
Why: plan U6 vector slice (KTD11 correctness step) - same-dimension vectors from different models must never mix, and PG tiers should run filtered exact top-k in the database instead of scanning every row in-process. Evidence: full suite green on SQLite and real PostgreSQL 18.6 with pgvector (35 packages each tier) - pgvector_test covers model-identity and dimension isolation, update/delete atomicity (BLOB and vec_pg swap together), shared top-5 with score monotonicity, Recall@10 = exact-scan parity, deny-all predicate leakage; storage-migrate tests green after excluding the derived vec_pg column. Impact: v25 migration adds model/content_hash columns and (PG, best-effort DO block) the pgvector extension plus vec_pg column - absence degrades to in-process scan; UpsertKnowledgeVectorExt dual-writes BLOB (cross-tier authority) and vec_pg; SearchKnowledgeVectorsModel/SharedModel filter by model identity and use SQL exact top-k (ORDER BY vec_pg <=> q) with automatic degradation; embedders expose Model() and the rag pipeline threads it through indexing and both retrieve paths; storage-migrate excludes vec_pg as a derived column. Refs: proposals/2026-10-11-collaboration-storage-and-distribution.md U6 slice 3, KTD11; R9.
75 lines
1.8 KiB
Go
75 lines
1.8 KiB
Go
package rag
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"hash/fnv"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
const hashDim = 32
|
||
|
||
// HashEmbedder 是零依赖的本地嵌入档。语义弱于真实模型，只保证同文本稳定、近文本可比较。
|
||
// 由 settings models.embedding.provider=hash 显式打开；未设置时仍按 D23 退化为全文。
|
||
type HashEmbedder struct{}
|
||
|
||
// Model 本地哈希嵌入的身份（U6-3：同源写入/检索，禁止与真实模型混查）。
|
||
func (HashEmbedder) Model() string { return "hash:" + fmt.Sprint(hashDim) }
|
||
|
||
func (HashEmbedder) Embed(_ context.Context, texts []string) ([][]float32, error) {
|
||
out := make([][]float32, len(texts))
|
||
for i, text := range texts {
|
||
out[i] = hashVec(text)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func hashVec(text string) []float32 {
|
||
v := make([]float32, hashDim)
|
||
for _, r := range text {
|
||
h := fnv.New32a()
|
||
var buf [4]byte
|
||
n := utf8.EncodeRune(buf[:], r)
|
||
_, _ = h.Write(buf[:n])
|
||
idx := int(h.Sum32() % hashDim)
|
||
v[idx] += 1
|
||
}
|
||
var norm float32
|
||
for _, x := range v {
|
||
norm += x * x
|
||
}
|
||
if norm == 0 {
|
||
return v
|
||
}
|
||
inv := float32(1)
|
||
// 粗略归一，避免余弦被长度主导。
|
||
for norm > 1 {
|
||
norm = norm / 4
|
||
inv = inv / 2
|
||
}
|
||
for i := range v {
|
||
v[i] *= inv
|
||
}
|
||
return v
|
||
}
|
||
|
||
// SwitchingEmbedder 在 provider=hash 时走本地档，否则走 HTTP。
|
||
type SwitchingEmbedder struct {
|
||
HTTP *HTTPEmbedder
|
||
Provider func() string
|
||
}
|
||
|
||
func (e SwitchingEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
||
if e.Provider != nil && e.Provider() == "hash" {
|
||
return HashEmbedder{}.Embed(ctx, texts)
|
||
}
|
||
if e.HTTP == nil {
|
||
return nil, fmt.Errorf("rag: embedding 模型未配置（settings.models.embedding）")
|
||
}
|
||
return e.HTTP.Embed(ctx, texts)
|
||
}
|
||
|
||
func (e SwitchingEmbedder) Offline() bool {
|
||
return e.Provider != nil && e.Provider() == "hash"
|
||
}
|