weichen
63f1fa5f87
- internal/controlcenter/embed: OpenAI-compatible /embeddings client
(batch<=16, 30s timeout, 5xx/network retried once, 4xx not), Embedder
interface, deterministic offline Stub, pgvector literal formatting
- internal/controlcenter/docs: .doc/**/*.md sync into cc_documents with
sha256 diffing and vanished-file removal; YAML front matter -> jsonb,
title fallback, taxonomy from first .doc dir; heading-based chunking
with paragraph bisection over ~1800 est. tokens; chunk_hash idempotent
inserts (unchanged chunks keep embeddings); embedding failures degrade
to NULL vectors without failing the sync; IndexArtifact for
cc_stage_artifacts; StartPeriodic ticker (CC_DOC_SYNC_SECONDS)
- server: GET /api/v1/docs (taxonomy/q filters), /api/v1/docs/{id},
/api/v1/docs/search (semantic|keyword|hybrid, ts_rank+ILIKE keyword
fallback, 0.6/0.4 max-normalized fusion, degraded flag when embedding
unavailable); async artifact indexing hooked into agent artifacts and
external-submit handlers
- archtest module-boundaries allowlist updated for new packages
113 lines
3.8 KiB
Go
113 lines
3.8 KiB
Go
package docs
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestChunkMarkdownHeadings(t *testing.T) {
|
|
md := "# Doc Title\n\nintro text\n\n## Section A\n\nalpha body\n\n### Sub A1\n\nsub body\n\n## Section B\n\nbeta body\n"
|
|
chunks := chunkMarkdown(md)
|
|
require.Len(t, chunks, 4)
|
|
assert.Equal(t, "# Doc Title", chunks[0].Heading)
|
|
assert.Contains(t, chunks[0].Content, "intro text")
|
|
assert.Equal(t, "## Section A", chunks[1].Heading)
|
|
assert.Contains(t, chunks[1].Content, "alpha body")
|
|
assert.NotContains(t, chunks[1].Content, "sub body")
|
|
assert.Equal(t, "### Sub A1", chunks[2].Heading)
|
|
assert.Equal(t, "## Section B", chunks[3].Heading)
|
|
for i, c := range chunks {
|
|
assert.Equal(t, i, c.Seq)
|
|
}
|
|
}
|
|
|
|
func TestChunkMarkdownPreambleOnly(t *testing.T) {
|
|
chunks := chunkMarkdown("just some text\nwithout headings\n")
|
|
require.Len(t, chunks, 1)
|
|
assert.Equal(t, "", chunks[0].Heading)
|
|
assert.Contains(t, chunks[0].Content, "just some text")
|
|
}
|
|
|
|
func TestChunkMarkdownEmpty(t *testing.T) {
|
|
assert.Empty(t, chunkMarkdown(""))
|
|
assert.Empty(t, chunkMarkdown("\n\n \n"))
|
|
}
|
|
|
|
func TestChunkMarkdownLongSectionBisected(t *testing.T) {
|
|
// Build one section with many paragraphs, well over maxChunkTokens.
|
|
var sb strings.Builder
|
|
sb.WriteString("## Big\n\n")
|
|
for i := 0; i < 60; i++ {
|
|
fmt.Fprintf(&sb, "paragraph %d %s\n\n", i, strings.Repeat("word ", 100))
|
|
}
|
|
chunks := chunkMarkdown(sb.String())
|
|
require.Greater(t, len(chunks), 1, "oversized section must be split")
|
|
for _, c := range chunks {
|
|
assert.Equal(t, "## Big", c.Heading)
|
|
assert.LessOrEqual(t, estimateTokens(c.Content), maxChunkTokens)
|
|
}
|
|
// all paragraphs survive the split
|
|
joined := strings.Join(func() []string {
|
|
var out []string
|
|
for _, c := range chunks {
|
|
out = append(out, c.Content)
|
|
}
|
|
return out
|
|
}(), "\n")
|
|
for i := 0; i < 60; i++ {
|
|
assert.Contains(t, joined, fmt.Sprintf("paragraph %d", i))
|
|
}
|
|
}
|
|
|
|
func TestChunkMarkdownSingleHugeParagraph(t *testing.T) {
|
|
md := "## Blob\n\n" + strings.Repeat("汉字", 3000) + "\n"
|
|
chunks := chunkMarkdown(md)
|
|
require.Greater(t, len(chunks), 1)
|
|
for _, c := range chunks {
|
|
assert.LessOrEqual(t, estimateTokens(c.Content), maxChunkTokens)
|
|
}
|
|
}
|
|
|
|
func TestChunkHashStable(t *testing.T) {
|
|
h1 := chunkHash("doc", ".doc/a.md", 0, "content")
|
|
h2 := chunkHash("doc", ".doc/a.md", 0, "content")
|
|
assert.Equal(t, h1, h2)
|
|
assert.NotEqual(t, h1, chunkHash("doc", ".doc/a.md", 1, "content"), "seq matters")
|
|
assert.NotEqual(t, h1, chunkHash("artifact", ".doc/a.md", 0, "content"), "owner kind matters")
|
|
}
|
|
|
|
func TestSplitFrontMatter(t *testing.T) {
|
|
fm, body := splitFrontMatter("---\ntitle: My Doc\nstatus: done\n---\n# Heading\nbody\n")
|
|
assert.Equal(t, "My Doc", fm["title"])
|
|
assert.Equal(t, "done", fm["status"])
|
|
assert.True(t, strings.HasPrefix(body, "# Heading"))
|
|
|
|
fm, body = splitFrontMatter("# No front matter\n")
|
|
assert.Empty(t, fm)
|
|
assert.Contains(t, body, "# No front matter")
|
|
}
|
|
|
|
func TestDocTitleFallback(t *testing.T) {
|
|
assert.Equal(t, "FM Title", docTitle(map[string]any{"title": "FM Title"}, "# H1\n", "x.md"))
|
|
assert.Equal(t, "H1 Title", docTitle(map[string]any{}, "# H1 Title\nbody\n", "x.md"))
|
|
assert.Equal(t, "file-name", docTitle(map[string]any{}, "no heading\n", ".doc/ops/file-name.md"))
|
|
}
|
|
|
|
func TestTaxonomyOf(t *testing.T) {
|
|
assert.Equal(t, "ops", taxonomyOf(".doc/ops/CURRENT.md"))
|
|
assert.Equal(t, "ops", taxonomyOf(".doc/ops/short-term/x.md"))
|
|
assert.Equal(t, "", taxonomyOf(".doc/README.md"))
|
|
assert.Equal(t, "", taxonomyOf("other/x.md"))
|
|
}
|
|
|
|
func TestEstimateTokens(t *testing.T) {
|
|
assert.Equal(t, 1, estimateTokens(""))
|
|
assert.Equal(t, 2, estimateTokens("汉")) // CJK rune = 2
|
|
assert.Equal(t, 1, estimateTokens("abcd")) // 4 ascii / 4
|
|
assert.Equal(t, 2, estimateTokens("abcde")) // ceil(5/4)
|
|
}
|