weichen
63f1fa5f87
- internal/controlcenter/embed: OpenAI-compatible /embeddings client
(batch<=16, 30s timeout, 5xx/network retried once, 4xx not), Embedder
interface, deterministic offline Stub, pgvector literal formatting
- internal/controlcenter/docs: .doc/**/*.md sync into cc_documents with
sha256 diffing and vanished-file removal; YAML front matter -> jsonb,
title fallback, taxonomy from first .doc dir; heading-based chunking
with paragraph bisection over ~1800 est. tokens; chunk_hash idempotent
inserts (unchanged chunks keep embeddings); embedding failures degrade
to NULL vectors without failing the sync; IndexArtifact for
cc_stage_artifacts; StartPeriodic ticker (CC_DOC_SYNC_SECONDS)
- server: GET /api/v1/docs (taxonomy/q filters), /api/v1/docs/{id},
/api/v1/docs/search (semantic|keyword|hybrid, ts_rank+ILIKE keyword
fallback, 0.6/0.4 max-normalized fusion, degraded flag when embedding
unavailable); async artifact indexing hooked into agent artifacts and
external-submit handlers
- archtest module-boundaries allowlist updated for new packages
214 lines
8.3 KiB
Go
214 lines
8.3 KiB
Go
package server
|
|
|
|
import (
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"chat-agent-system/internal/controlcenter/config"
|
|
"chat-agent-system/internal/controlcenter/docs"
|
|
"chat-agent-system/internal/controlcenter/embed"
|
|
"chat-agent-system/internal/controlcenter/store"
|
|
)
|
|
|
|
// newDocTestServer builds a server wired to a docs.Syncer over a temp .doc
|
|
// tree. emb may be nil to exercise degraded search.
|
|
func newDocTestServer(t *testing.T, emb embed.Embedder, files map[string]string) (*httptest.Server, *store.Store, *docs.Syncer) {
|
|
t.Helper()
|
|
dsn := os.Getenv("TEST_CC_DB_DSN")
|
|
if dsn == "" {
|
|
t.Skip("TEST_CC_DB_DSN not set")
|
|
}
|
|
_, thisFile, _, _ := runtime.Caller(0)
|
|
migrationsDir := filepath.Join(filepath.Dir(thisFile), "..", "..", "..", "deploy", "control-center", "goose")
|
|
ctx := t.Context()
|
|
require.NoError(t, store.Migrate(ctx, dsn, migrationsDir))
|
|
st, err := store.Connect(ctx, dsn)
|
|
require.NoError(t, err)
|
|
t.Cleanup(st.Close)
|
|
|
|
repoRoot := t.TempDir()
|
|
for rel, content := range files {
|
|
full := filepath.Join(repoRoot, filepath.FromSlash(rel))
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
|
require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
|
|
}
|
|
|
|
syncer := docs.New(st, repoRoot, emb)
|
|
stats, err := syncer.Sync(ctx)
|
|
require.NoError(t, err)
|
|
|
|
srv := New(config.Config{ConsoleToken: testConsoleToken}, st)
|
|
srv.SetDocs(emb, syncer)
|
|
ts := httptest.NewServer(srv.Handler())
|
|
t.Cleanup(ts.Close)
|
|
t.Logf("sync stats: %+v", stats)
|
|
return ts, st, syncer
|
|
}
|
|
|
|
func docTestFiles(tag string) map[string]string {
|
|
return map[string]string{
|
|
".doc/ops/" + tag + "-pipeline.md": "---\ntitle: 流水线设计\nstatus: done\n---\n# 流水线设计\n\n八阶段流水线，任务从 proposal 到 confirmation。\n\n## 阶段\n\n提案、分析、设计、评审、实现、测试、评估、确认。\n",
|
|
".doc/audit/" + tag + "-notes.md": "# 审计笔记\n\n流水线租约过期后任务回到 assigned 状态。\n",
|
|
".doc/" + tag + "-root.md": "# 根文档\n\n没有 taxonomy 的根级文档。\n",
|
|
".doc/assets/" + tag + "-skip.md": "# 应被跳过\n\nassets 目录不入库。\n",
|
|
".doc/ops/" + tag + "-plain.txt": "not markdown, ignored",
|
|
}
|
|
}
|
|
|
|
func TestDocsSyncListAndGet(t *testing.T) {
|
|
tag := fmt.Sprintf("t%d", time.Now().UnixNano())
|
|
ts, st, syncer := newDocTestServer(t, embed.NewStub(embed.Dim), docTestFiles(tag))
|
|
|
|
code, raw := doReq(t, "GET", ts.URL+"/api/v1/docs", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
items := decodeBody(t, raw)["items"].([]any)
|
|
var paths []string
|
|
for _, it := range items {
|
|
paths = append(paths, it.(map[string]any)["path"].(string))
|
|
}
|
|
assert.Contains(t, paths, ".doc/ops/"+tag+"-pipeline.md")
|
|
assert.Contains(t, paths, ".doc/audit/"+tag+"-notes.md")
|
|
assert.Contains(t, paths, ".doc/"+tag+"-root.md")
|
|
assert.NotContains(t, paths, ".doc/assets/"+tag+"-skip.md", "assets/ must be skipped")
|
|
|
|
// taxonomy filter
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs?taxonomy=audit", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code)
|
|
for _, it := range decodeBody(t, raw)["items"].([]any) {
|
|
assert.Equal(t, "audit", it.(map[string]any)["taxonomy"])
|
|
}
|
|
|
|
// q filter + front matter title
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs?q=流水线设计", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code)
|
|
items = decodeBody(t, raw)["items"].([]any)
|
|
require.Len(t, items, 1)
|
|
it := items[0].(map[string]any)
|
|
assert.Equal(t, "流水线设计", it["title"])
|
|
assert.Equal(t, "ops", it["taxonomy"])
|
|
assert.Greater(t, it["chunks"].(float64), float64(0))
|
|
|
|
// detail
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs/"+it["id"].(string), testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
detail := decodeBody(t, raw)
|
|
assert.Equal(t, "done", detail["frontmatter"].(map[string]any)["status"])
|
|
|
|
code, _ = doReq(t, "GET", ts.URL+"/api/v1/docs/00000000-0000-0000-0000-000000000000", testConsoleToken, nil)
|
|
assert.Equal(t, http.StatusNotFound, code)
|
|
|
|
// re-sync is idempotent: chunk count unchanged
|
|
countChunks := func() int {
|
|
var n int
|
|
require.NoError(t, st.Pool.QueryRow(t.Context(),
|
|
`SELECT count(*) FROM cc_doc_chunks c JOIN cc_documents d ON d.id = c.doc_id
|
|
WHERE d.path LIKE '%' || $1 || '%'`, tag).Scan(&n))
|
|
return n
|
|
}
|
|
before := countChunks()
|
|
stats, err := syncer.Sync(t.Context())
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 0, stats.Added+stats.Updated, "unchanged files must be skipped")
|
|
assert.Equal(t, before, countChunks(), "unchanged docs must not duplicate chunks")
|
|
}
|
|
|
|
func TestDocsSearchModes(t *testing.T) {
|
|
tag := fmt.Sprintf("t%d", time.Now().UnixNano())
|
|
ts, _, _ := newDocTestServer(t, embed.NewStub(embed.Dim), docTestFiles(tag))
|
|
|
|
// q required
|
|
code, _ := doReq(t, "GET", ts.URL+"/api/v1/docs/search", testConsoleToken, nil)
|
|
assert.Equal(t, http.StatusBadRequest, code)
|
|
|
|
// keyword: ILIKE fallback hits Chinese content
|
|
code, raw := doReq(t, "GET", ts.URL+"/api/v1/docs/search?q=流水线&mode=keyword", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
m := decodeBody(t, raw)
|
|
assert.Equal(t, false, m["degraded"])
|
|
results := m["results"].([]any)
|
|
require.NotEmpty(t, results)
|
|
top := results[0].(map[string]any)
|
|
src := top["source"].(map[string]any)
|
|
assert.Equal(t, "doc", src["type"])
|
|
assert.Contains(t, src["path"], tag)
|
|
assert.LessOrEqual(t, len([]rune(top["snippet"].(string))), snippetRunes)
|
|
|
|
// semantic with stub embedder returns results
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs/search?q=pipeline&mode=semantic&limit=5", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
m = decodeBody(t, raw)
|
|
assert.Equal(t, false, m["degraded"])
|
|
assert.NotEmpty(t, m["results"])
|
|
|
|
// hybrid fuses both
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs/search?q=流水线&mode=hybrid", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
m = decodeBody(t, raw)
|
|
assert.Equal(t, false, m["degraded"])
|
|
assert.NotEmpty(t, m["results"])
|
|
|
|
// limit cap
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs/search?q=流水线&limit=500", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
assert.LessOrEqual(t, len(decodeBody(t, raw)["results"].([]any)), searchMaxLimit)
|
|
}
|
|
|
|
func TestDocsSearchDegradedWithoutEmbedder(t *testing.T) {
|
|
tag := fmt.Sprintf("t%d", time.Now().UnixNano())
|
|
ts, _, _ := newDocTestServer(t, nil, docTestFiles(tag)) // nil embedder
|
|
|
|
code, raw := doReq(t, "GET", ts.URL+"/api/v1/docs/search?q=流水线&mode=hybrid", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
m := decodeBody(t, raw)
|
|
assert.Equal(t, true, m["degraded"], "no embedder must degrade hybrid to keyword")
|
|
assert.NotEmpty(t, m["results"], "keyword fallback still returns hits")
|
|
}
|
|
|
|
func TestArtifactIndexingHook(t *testing.T) {
|
|
tag := fmt.Sprintf("t%d", time.Now().UnixNano())
|
|
dsn := os.Getenv("TEST_CC_DB_DSN")
|
|
if dsn == "" {
|
|
t.Skip("TEST_CC_DB_DSN not set")
|
|
}
|
|
ts, st, _ := newDocTestServer(t, embed.NewStub(embed.Dim), docTestFiles(tag))
|
|
require.NoError(t, RegisterAgent(t.Context(), st, "agent-a", "test", testAgentToken))
|
|
|
|
id := createAssignedTask(t, ts, true)
|
|
claimTask(t, ts, id)
|
|
code, raw := doReq(t, "POST", ts.URL+"/api/v1/agent/tasks/"+id+"/artifacts", testAgentToken, map[string]any{
|
|
"title": "分析产物", "content_md": "# 分析\n\n流水线分析产物 " + tag + " 内容。\n",
|
|
})
|
|
require.Equal(t, http.StatusCreated, code, string(raw))
|
|
artID := decodeBody(t, raw)["id"].(string)
|
|
|
|
// indexing is async; poll
|
|
var n int
|
|
for i := 0; i < 40; i++ {
|
|
require.NoError(t, st.Pool.QueryRow(t.Context(),
|
|
`SELECT count(*) FROM cc_doc_chunks WHERE artifact_id = $1`, artID).Scan(&n))
|
|
if n > 0 {
|
|
break
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
assert.Greater(t, n, 0, "artifact chunks indexed asynchronously")
|
|
|
|
// artifact shows up in search with its source
|
|
code, raw = doReq(t, "GET", ts.URL+"/api/v1/docs/search?q="+tag+"&mode=keyword", testConsoleToken, nil)
|
|
require.Equal(t, http.StatusOK, code, string(raw))
|
|
results := decodeBody(t, raw)["results"].([]any)
|
|
require.NotEmpty(t, results)
|
|
src := results[0].(map[string]any)["source"].(map[string]any)
|
|
assert.Equal(t, "artifact", src["type"])
|
|
assert.NotEmpty(t, src["task_slug"])
|
|
}
|