From 983b785cd6391e11d9700a115ce163a93d8b7c19 Mon Sep 17 00:00:00 2001 From: weichen Date: Mon, 21 Sep 2026 08:29:10 +0000 Subject: [PATCH] feat(config): wire MinerU and material RAG settings Add MINERU_* / MATERIAL_* to config loader, .env.example, and compose env passthrough. Secrets stay in local .env only. Co-authored-by: Cursor --- .env.example | 9 ++++++ docker-compose.dev.yml | 7 ++++ docker-compose.main.yml | 7 ++++ docker-compose.test.yml | 7 ++++ internal/config/config.go | 67 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+) diff --git a/.env.example b/.env.example index 39392f30..aa8ef6a2 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,15 @@ RERANKER_ENDPOINT_URL=https://www.dmxapi.cn/v1/rerank RERANKER_MODEL=bge-reranker-v2-m3-free RERANKER_API_KEY= +# --- 9b. MinerU 文档解析（上传 PDF/Office 异步解析；未配置 KEY 时降级本地提取）--- +MINERU_ENDPOINT_URL=https://mineru.net +MINERU_API_KEY= +MINERU_MODEL_VERSION=vlm +MINERU_LANGUAGE=ch +MINERU_TIMEOUT=90s +MATERIAL_FULLTEXT_MAX_BYTES=61440 +MATERIAL_RAG_TOP_K=8 + # --- 10. 专属业务/上下文库（Go 宿主：CONTEXT_DB_*；compose 同时认 SYSTEM_DB_* 别名）--- CONTEXT_DB_HOST=127.0.0.1 CONTEXT_DB_PORT=5433 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7fc43123..2bad3431 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -99,6 +99,13 @@ services: RERANKER_ENDPOINT_URL: ${RERANKER_ENDPOINT_URL:-} RERANKER_MODEL: ${RERANKER_MODEL:-bge-reranker-v2-m3-free} RERANKER_API_KEY: ${RERANKER_API_KEY:-} + MINERU_ENDPOINT_URL: ${MINERU_ENDPOINT_URL:-https://mineru.net} + MINERU_API_KEY: ${MINERU_API_KEY:-} + MINERU_MODEL_VERSION: ${MINERU_MODEL_VERSION:-vlm} + MINERU_LANGUAGE: ${MINERU_LANGUAGE:-ch} + MINERU_TIMEOUT: ${MINERU_TIMEOUT:-90s} + MATERIAL_FULLTEXT_MAX_BYTES: ${MATERIAL_FULLTEXT_MAX_BYTES:-61440} + MATERIAL_RAG_TOP_K: ${MATERIAL_RAG_TOP_K:-8} ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin-secret-password-123} AUTH_REGISTRATION_ENABLED: ${AUTH_REGISTRATION_ENABLED:-true} diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 402d4f32..a08bd1c3 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -104,6 +104,13 @@ services: RERANKER_ENDPOINT_URL: ${RERANKER_ENDPOINT_URL:-} RERANKER_MODEL: ${RERANKER_MODEL:-bge-reranker-v2-m3-free} RERANKER_API_KEY: ${RERANKER_API_KEY:-} + MINERU_ENDPOINT_URL: ${MINERU_ENDPOINT_URL:-https://mineru.net} + MINERU_API_KEY: ${MINERU_API_KEY:-} + MINERU_MODEL_VERSION: ${MINERU_MODEL_VERSION:-vlm} + MINERU_LANGUAGE: ${MINERU_LANGUAGE:-ch} + MINERU_TIMEOUT: ${MINERU_TIMEOUT:-90s} + MATERIAL_FULLTEXT_MAX_BYTES: ${MATERIAL_FULLTEXT_MAX_BYTES:-61440} + MATERIAL_RAG_TOP_K: ${MATERIAL_RAG_TOP_K:-8} CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-} ALLOW_IN_MEMORY: "false" CACHE_ENABLED: ${CACHE_ENABLED:-true} diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 5fba70ff..d8162095 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -97,6 +97,13 @@ services: RERANKER_ENDPOINT_URL: ${RERANKER_ENDPOINT_URL:-} RERANKER_MODEL: ${RERANKER_MODEL:-bge-reranker-v2-m3-free} RERANKER_API_KEY: ${RERANKER_API_KEY:-} + MINERU_ENDPOINT_URL: ${MINERU_ENDPOINT_URL:-https://mineru.net} + MINERU_API_KEY: ${MINERU_API_KEY:-} + MINERU_MODEL_VERSION: ${MINERU_MODEL_VERSION:-vlm} + MINERU_LANGUAGE: ${MINERU_LANGUAGE:-ch} + MINERU_TIMEOUT: ${MINERU_TIMEOUT:-90s} + MATERIAL_FULLTEXT_MAX_BYTES: ${MATERIAL_FULLTEXT_MAX_BYTES:-61440} + MATERIAL_RAG_TOP_K: ${MATERIAL_RAG_TOP_K:-8} ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} ADMIN_PASSWORD: ${ADMIN_PASSWORD:?required} AUTH_REGISTRATION_ENABLED: ${AUTH_REGISTRATION_ENABLED:-true} diff --git a/internal/config/config.go b/internal/config/config.go index 5e0b3110..98b9d548 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -52,6 +52,17 @@ type Config struct { RerankerModel string `json:"reranker_model"` RerankerAPIKey string `json:"reranker_api_key"` + // MinerU 文档解析（精准 API）；未配置 API Key 时整条链路降级为本地提取 + MineruEndpointURL string `json:"mineru_endpoint_url"` + MineruAPIKey string `json:"mineru_api_key"` + MineruModelVersion string `json:"mineru_model_version"` + MineruLanguage string `json:"mineru_language"` + MineruTimeout time.Duration `json:"mineru_timeout"` + + // 材料注入策略：小文档全文，大文档 RAG + MaterialFulltextMaxBytes int `json:"material_fulltext_max_bytes"` + MaterialRAGTopK int `json:"material_rag_top_k"` + // 系统管理员与准入认证配置 AdminUsername string `json:"admin_username"` AdminPassword string `json:"admin_password"` @@ -112,6 +123,15 @@ func Load(path string) (*Config, error) { RerankerModel: "bge-reranker-v2-m3-free", RerankerAPIKey: "", + MineruEndpointURL: "https://mineru.net", + MineruAPIKey: "", + MineruModelVersion: "vlm", + MineruLanguage: "ch", + MineruTimeout: 90 * time.Second, + + MaterialFulltextMaxBytes: 60 * 1024, + MaterialRAGTopK: 8, + AdminUsername: "admin", AdminPassword: "admin-secret-password-123", AuthRegistrationEnabled: true, @@ -235,6 +255,33 @@ func Load(path string) (*Config, error) { if v := os.Getenv("RERANKER_MODEL"); v != "" { cfg.RerankerModel = v } + if v := os.Getenv("MINERU_ENDPOINT_URL"); v != "" { + cfg.MineruEndpointURL = v + } + if v := os.Getenv("MINERU_API_KEY"); v != "" { + cfg.MineruAPIKey = v + } + if v := os.Getenv("MINERU_MODEL_VERSION"); v != "" { + cfg.MineruModelVersion = v + } + if v := os.Getenv("MINERU_LANGUAGE"); v != "" { + cfg.MineruLanguage = v + } + if v := os.Getenv("MINERU_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil { + cfg.MineruTimeout = d + } + } + if v := os.Getenv("MATERIAL_FULLTEXT_MAX_BYTES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.MaterialFulltextMaxBytes = n + } + } + if v := os.Getenv("MATERIAL_RAG_TOP_K"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + cfg.MaterialRAGTopK = n + } + } if v := os.Getenv("ADMIN_USERNAME"); v != "" { cfg.AdminUsername = v } @@ -403,6 +450,26 @@ func applyEnvFile(cfg *Config, path string) error { cfg.RerankerModel = val case "RERANKER_API_KEY": cfg.RerankerAPIKey = val + case "MINERU_ENDPOINT_URL": + cfg.MineruEndpointURL = val + case "MINERU_API_KEY": + cfg.MineruAPIKey = val + case "MINERU_MODEL_VERSION": + cfg.MineruModelVersion = val + case "MINERU_LANGUAGE": + cfg.MineruLanguage = val + case "MINERU_TIMEOUT": + if d, err := time.ParseDuration(val); err == nil { + cfg.MineruTimeout = d + } + case "MATERIAL_FULLTEXT_MAX_BYTES": + if n, err := strconv.Atoi(val); err == nil && n > 0 { + cfg.MaterialFulltextMaxBytes = n + } + case "MATERIAL_RAG_TOP_K": + if n, err := strconv.Atoi(val); err == nil && n > 0 { + cfg.MaterialRAGTopK = n + } case "ADMIN_USERNAME": cfg.AdminUsername = val case "ADMIN_PASSWORD":