From 93ab4802e9312637908197838d347ce40d88781f Mon Sep 17 00:00:00 2001 From: agent-bot Date: Tue, 29 Sep 2026 08:06:06 +0000 Subject: [PATCH] feat(config): field-level source attribution via layer snapshots (U10b core 1/2) - LoadWithSources: reflection snapshots at layer boundaries (default -> env-file per file -> process-env incl. DSN/CORS/KB normalization); diff yields exact per-field source - no manual key registry, no drift - Load unchanged signature/behavior (wrapper); ApplyCLIOverlay records the cli layer on a cloned config - 'config explain' upgraded to field table (59 fields; source + set flag only, values never printed per KTD17) - replaces the reporting-only key registry from PR #12 - tests: layer attribution (file/process override), cli overlay, Load regression --- cmd/server/main.go | 66 ++++++-------- internal/config/config.go | 66 +++++++++++++- internal/config/explain.go | 154 +++----------------------------- internal/config/explain_test.go | 81 +++++++++++------ internal/config/snapshot.go | 102 +++++++++++++++++++++ 5 files changed, 256 insertions(+), 213 deletions(-) create mode 100644 internal/config/snapshot.go diff --git a/cmd/server/main.go b/cmd/server/main.go index ea9a7cb3..c75a3aca 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -7,9 +7,6 @@ import ( "net/http" "os" "os/signal" - "path/filepath" - "strconv" - "strings" "syscall" "time" @@ -219,8 +216,8 @@ func configCheck(args []string) { fmt.Println("config check: 通过") } -// configExplain 解释每个配置键的最终来源层（KTD17 第一片）： -// cli > process-env > env-file > default；秘密值只显示 set 状态。 +// configExplain 输出字段级来源表（KTD17：只报告来源/状态，不打印值）。 +// 层序：cli > process-env > env-file（按加载顺序）> default。 func configExplain(args []string) { fs := flag.NewFlagSet("config explain", flag.ExitOnError) envPath := fs.String("env", "", "环境配置文件路径（默认根 .env + configs/.env.{ENV} 叠加）") @@ -229,44 +226,33 @@ func configExplain(args []string) { backend := fs.String("backend", "", "智能体后端覆盖") _ = fs.Parse(args) - var envFiles []string - if *envPath != "" { - envFiles = append(envFiles, *envPath) - } else { - envFiles = append(envFiles, ".env") - envName := "dev" - if keys, err := config.ParseEnvFileKeys(".env"); err == nil { - for _, k := range keys { - if k == "ENV" { - // 从文件行取值 - if b, rerr := os.ReadFile(".env"); rerr == nil { - for _, line := range strings.Split(string(b), "\n") { - if strings.HasPrefix(strings.TrimSpace(line), "ENV=") { - envName = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "ENV=")) - } - } - } - break - } - } + cfg, src, err := config.LoadWithSources(*envPath) + if err != nil { + fmt.Fprintf(os.Stderr, "config explain: 加载失败: %v\n", err) + os.Exit(1) + } + src, err = config.ApplyCLIOverrides(cfg, src, func(c *config.Config) { + if *port > 0 { + c.HTTPPort = *port } - if v := os.Getenv("ENV"); v != "" { - envName = v + if *host != "" { + c.HTTPHost = *host } - envFiles = append(envFiles, filepath.Join("configs", ".env."+envName)) + if *backend != "" { + c.AgentBackend = *backend + } + }) + if err != nil { + fmt.Fprintf(os.Stderr, "config explain: %v\n", err) + os.Exit(1) } - cli := map[string]string{} - if *port > 0 { - cli["HTTP_PORT"] = strconv.Itoa(*port) + fmt.Printf("config explain: 字段 %d 个（层序 cli > process-env > env-file > default）\n", len(src)) + fmt.Print(config.FormatFieldSources(config.SortedFieldSources(src))) + if issues := cfg.Check(); len(issues) > 0 { + fmt.Fprintln(os.Stderr, "附注：当前配置未通过 check：") + for _, it := range issues { + fmt.Fprintf(os.Stderr, " - %s\n", it) + } } - if *host != "" { - cli["HTTP_HOST"] = *host - } - if *backend != "" { - cli["AGENT_BACKEND"] = *backend - } - - fmt.Println("env files:", strings.Join(envFiles, ", ")) - fmt.Print(config.FormatExplanation(config.Explain(envFiles, cli))) } diff --git a/internal/config/config.go b/internal/config/config.go index 9cf970ff..4f937bc1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -100,7 +100,20 @@ func DefaultDataDir() string { return "./data/dev" } +// Load 加载配置（行为与历史版本一致）。 func Load(path string) (*Config, error) { + cfg, _, err := LoadWithSources(path) + return cfg, err +} + +// LoadWithSources 加载配置并给出字段级来源（KTD17 U10b）。 +// 层序：default → env-file（按文件顺序）→ process-env；CLI 覆盖由调用方 +// 在返回后应用并以 ApplyCLIOverrides 记录来源。 +func LoadWithSources(path string) (*Config, map[string]FieldSource, error) { + var phases []struct { + Layer string + Snap map[string]string + } cfg := &Config{ Env: "dev", DataDir: DefaultDataDir(), @@ -155,9 +168,18 @@ func Load(path string) (*Config, error) { MaxConcurrentAgents: 8, } + phases = append(phases, struct { + Layer string + Snap map[string]string + }{"default", snapshotFields(cfg)}) + if path == "" { // 与 Compose 一致：根 .env（公共密钥）+ configs/.env.{ENV}（环境差异 overlay） _ = applyEnvFile(cfg, ".env") + phases = append(phases, struct { + Layer string + Snap map[string]string + }{"env-file:.env", snapshotFields(cfg)}) envName := cfg.Env if v := os.Getenv("ENV"); v != "" { envName = v @@ -165,9 +187,18 @@ func Load(path string) (*Config, error) { if envName == "" { envName = "dev" } - _ = applyEnvFile(cfg, filepath.Join("configs", ".env."+envName)) + overlay := filepath.Join("configs", ".env."+envName) + _ = applyEnvFile(cfg, overlay) + phases = append(phases, struct { + Layer string + Snap map[string]string + }{"env-file:" + filepath.ToSlash(overlay), snapshotFields(cfg)}) } else { _ = applyEnvFile(cfg, path) + phases = append(phases, struct { + Layer string + Snap map[string]string + }{"env-file:" + filepath.ToSlash(path), snapshotFields(cfg)}) } // 允许系统环境变量覆盖 @@ -379,15 +410,42 @@ func Load(path string) (*Config, error) { applyDSNOverrides(cfg) applyCORSDefaults(cfg) NormalizeKBHostForRuntime(cfg) + phases = append(phases, struct { + Layer string + Snap map[string]string + }{"process-env", snapshotFields(cfg)}) if err := validateProductionSecrets(cfg); err != nil { - return nil, err + return nil, nil, err } if err := ValidateIndependentVault(cfg.VaultStateRoot, cfg.DataDir); err != nil { - return nil, err + return nil, nil, err } - return cfg, nil + return cfg, diffLayers(phases), nil +} + +// ApplyCLIOverrides 在 Load 之后应用显式 CLI 覆盖并更新字段来源层。 +// 返回应用后的配置副本来源表（不修改原 cfg）。 +func ApplyCLIOverrides(cfg *Config, src map[string]FieldSource, mutate func(*Config)) (map[string]FieldSource, error) { + if cfg == nil || mutate == nil { + return src, nil + } + clone := *cfg + before := snapshotFields(cfg) + mutate(&clone) + after := snapshotFields(&clone) + out := make(map[string]FieldSource, len(src)) + for k, v := range src { + out[k] = v + } + for field, val := range after { + if before[field] != val { + fs := FieldSource{Field: field, Layer: "cli", Set: true} + out[field] = fs + } + } + return out, nil } // applyEnvFile overlays KEY=VALUE pairs from path onto cfg. Missing file is a no-op. diff --git a/internal/config/explain.go b/internal/config/explain.go index 53ca0c04..e9bec8f7 100644 --- a/internal/config/explain.go +++ b/internal/config/explain.go @@ -4,99 +4,10 @@ import ( "bufio" "fmt" "os" - "path/filepath" "sort" "strings" ) -// knownEnvKeys 是配置环境键注册表（U10b 第一片：reporting-only）。 -// 仅用于 config explain 的来源解释，不参与取值——取值仍由 Load 的 -// 应用逻辑唯一决定；新增环境键时须同步登记，漂移由抽样测试约束。 -var knownEnvKeys = map[string]bool{ - "ADMIN_PASSWORD": true, - "ADMIN_USERNAME": true, - "AGENTX_API_BASE": true, - "AGENTX_API_KEY": true, - "AGENTX_MODEL": true, - "AGENTX_TIMEOUT": true, - "AGENT_BACKEND": true, - "AGENT_TIMEOUT": true, - "ALLOW_IN_MEMORY": true, - "APP_DB_DSN": true, - "APP_PUBLISH_HOST": true, - "AUTH_REGISTRATION_ENABLED": true, - "AUTH_REQUIRE_APPROVAL": true, - "CONTEXT_DB_HOST": true, - "CONTEXT_DB_NAME": true, - "CONTEXT_DB_PASSWORD": true, - "CONTEXT_DB_PORT": true, - "CONTEXT_DB_USER": true, - "CORS_ALLOWED_ORIGINS": true, - "DATA_DIR": true, - "EMBEDDING_API_KEY": true, - "EMBEDDING_ENDPOINT_URL": true, - "EMBEDDING_MODEL": true, - "EMBEDDING_PROVIDER": true, - "ENV": true, - "FAV_DEGRADED_READ": true, - "FAV_PUBLISHER_ENABLED": true, - "HTTP_ADDR": true, - "HTTP_HOST": true, - "HTTP_PORT": true, - "JEV_API_KEY": true, - "JEV_BASE_URL": true, - "JEV_DECIDE_MODE": true, - "KB_DB_HOST": true, - "KB_DB_NAME": true, - "KB_DB_PASSWORD": true, - "KB_DB_PORT": true, - "KB_DB_USER": true, - "MATERIAL_FULLTEXT_MAX_BYTES": true, - "MATERIAL_RAG_TOP_K": true, - "MAX_CONCURRENT_AGENTS": true, - "MINERU_API_KEY": true, - "MINERU_ENDPOINT_URL": true, - "MINERU_LANGUAGE": true, - "MINERU_MODEL_VERSION": true, - "MINERU_TIMEOUT": true, - "MINIMAX_API_KEY": true, - "MINIMAX_ENDPOINT_URL": true, - "PI_AGENT_BIN": true, - "PI_AGENT_BRIDGE_BIN": true, - "PI_AGENT_MODEL": true, - "PI_AGENT_PROVIDER": true, - "PI_AGENT_RUNNER_AUTH_TOKEN": true, - "PI_AGENT_RUNNER_URL": true, - "POSTGRES_SHARED_KB_DSN": true, - "RERANKER_API_KEY": true, - "RERANKER_ENDPOINT_URL": true, - "RERANKER_MODEL": true, - "RERANKER_PROVIDER": true, - "RUNNER_AUTH_TOKEN": true, - "VAULT_MASTER_KEY": true, - "VAULT_STATE_ROOT": true, - "VAULT_STORAGE_IDENTITY": true, - "WORKSPACE_QUOTA_BYTES": true, - "WORKSPACE_TTL": true, -} - -// ExplainLayer 是配置键的一个来源层。 -type ExplainLayer = string - -const ( - LayerDefault ExplainLayer = "default" - LayerEnvFile ExplainLayer = "env-file" - LayerProcessEnv ExplainLayer = "process-env" - LayerCLI ExplainLayer = "cli" -) - -// ExplainEntry 是单个配置键的解释行。 -type ExplainEntry struct { - Key string - Layer ExplainLayer - Value string // 秘密键只显示 /；env-file 层显示文件路径 -} - // ParseEnvFileKeys 用与 applyEnvFile 相同的解析规则列出文件中出现的键。 func ParseEnvFileKeys(path string) ([]string, error) { f, err := os.Open(path) @@ -120,59 +31,22 @@ func ParseEnvFileKeys(path string) ([]string, error) { return keys, scanner.Err() } -// Explain 生成各配置键的来源层解释（KTD17 config explain 第一片）。 -// envFiles 按加载顺序（后者覆盖前者）；cliOverrides 为 CLI 显式覆盖键。 -// 层序与生效优先级一致：cli > process-env > env-file > default。 -// 非秘密且来源为 env-file/process-env 的键展示脱敏值；秘密键只显示 set 状态。 -func Explain(envFiles []string, cliOverrides map[string]string) []ExplainEntry { - fileKeys := map[string]string{} // key -> 首个出现的文件 - for _, p := range envFiles { - keys, err := ParseEnvFileKeys(p) - if err != nil { - continue - } - for _, k := range keys { - fileKeys[k] = filepath.ToSlash(p) - } - } - - var out []ExplainEntry - for k := range knownEnvKeys { - e := ExplainEntry{Key: k, Layer: LayerDefault, Value: ""} - if src, ok := fileKeys[k]; ok { - e.Layer = LayerEnvFile - e.Value = src - if v := os.Getenv(k); v != "" { - e.Layer = LayerProcessEnv - e.Value = maskedValue(v) - } - } else if v := os.Getenv(k); v != "" { - e.Layer = LayerProcessEnv - e.Value = maskedValue(v) - } - if v, ok := cliOverrides[k]; ok { - e.Layer = LayerCLI - e.Value = maskedValue(v) - } - out = append(out, e) - } - sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) - return out -} - -func maskedValue(v string) string { - if strings.TrimSpace(v) == "" { - return "" - } - return "" -} - -// FormatExplanation 渲染 explain 表（供 CLI 输出与测试断言）。 -func FormatExplanation(entries []ExplainEntry) string { +// FormatFieldSources 渲染字段级来源表（KTD17：只报告来源/状态，不打印值）。 +func FormatFieldSources(entries []FieldSource) string { var b strings.Builder - b.WriteString("key layer detail\n") + b.WriteString("field layer set\n") for _, e := range entries { - fmt.Fprintf(&b, "%-37s %-12s %s\n", e.Key, e.Layer, e.Value) + fmt.Fprintf(&b, "%-37s %-19s %v\n", e.Field, e.Layer, e.Set) } return b.String() } + +// SortedFieldSourceNames 返回已知来源字段名（测试辅助）。 +func SortedFieldSourceNames(m map[string]FieldSource) []string { + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + sort.Strings(names) + return names +} diff --git a/internal/config/explain_test.go b/internal/config/explain_test.go index ab293a6f..b6cf9838 100644 --- a/internal/config/explain_test.go +++ b/internal/config/explain_test.go @@ -6,44 +6,67 @@ import ( "testing" ) -// 注册表键必须真实影响 Load（抽样防 reporting-only 注册表漂移）。 -func TestExplainRegistryKeysAreLive(t *testing.T) { - t.Setenv("HTTP_PORT", "9911") - t.Setenv("AGENT_BACKEND", "stub") - cfg, err := Load("") - if err != nil { - t.Fatalf("load: %v", err) - } - if cfg.HTTPPort != 9911 || cfg.AgentBackend != "stub" { - t.Fatalf("registry sample keys not applied: port=%d backend=%s", cfg.HTTPPort, cfg.AgentBackend) - } -} - -func TestExplainLayerAttribution(t *testing.T) { +func TestFieldSourcesLayerAttribution(t *testing.T) { dir := t.TempDir() envf := filepath.Join(dir, ".env") - if err := os.WriteFile(envf, []byte("HTTP_HOST=0.0.0.0\nKB_DB_PASSWORD=from-file\n"), 0o600); err != nil { + if err := os.WriteFile(envf, []byte("HTTP_HOST=0.0.0.0\nKB_DB_PASSWORD=from-file\nAGENT_BACKEND=stub\n"), 0o600); err != nil { t.Fatal(err) } t.Setenv("KB_DB_PASSWORD", "from-process") // 进程 env 覆盖 env-file - entries := Explain([]string{envf}, map[string]string{"AGENT_BACKEND": "stub"}) - by := map[string]ExplainEntry{} - for _, e := range entries { - by[e.Key] = e + + cfg, src, err := LoadWithSources(envf) + if err != nil { + t.Fatalf("load: %v", err) } - if by["HTTP_HOST"].Layer != LayerEnvFile { - t.Fatalf("HTTP_HOST layer=%s want env-file", by["HTTP_HOST"].Layer) + if cfg.HTTPHost != "0.0.0.0" || cfg.KbDbPassword != "from-process" || cfg.AgentBackend != "stub" { + t.Fatalf("values not applied: host=%s", cfg.HTTPHost) } - if by["KB_DB_PASSWORD"].Layer != LayerProcessEnv { - t.Fatalf("KB_DB_PASSWORD layer=%s want process-env", by["KB_DB_PASSWORD"].Layer) + if src["http_host"].Layer != "env-file:"+filepath.ToSlash(envf) { + t.Fatalf("http_host layer=%s", src["http_host"].Layer) } - if by["KB_DB_PASSWORD"].Value != "" { - t.Fatalf("secret value leaked: %q", by["KB_DB_PASSWORD"].Value) + if src["kb_db_password"].Layer != "process-env" { + t.Fatalf("kb_db_password layer=%s", src["kb_db_password"].Layer) } - if by["AGENT_BACKEND"].Layer != LayerCLI { - t.Fatalf("AGENT_BACKEND layer=%s want cli", by["AGENT_BACKEND"].Layer) + if src["agent_backend"].Layer != "env-file:"+filepath.ToSlash(envf) { + t.Fatalf("agent_backend layer=%s", src["agent_backend"].Layer) } - if by["ADMIN_USERNAME"].Layer != LayerDefault { - t.Fatalf("ADMIN_USERNAME layer=%s want default", by["ADMIN_USERNAME"].Layer) + // 未设置字段仍是 default 层 + if src["admin_username"].Layer != "default" || src["admin_username"].Set { + t.Fatalf("admin_username layer=%s set=%v", src["admin_username"].Layer, src["admin_username"].Set) + } +} + +func TestApplyCLIOverridesAttribution(t *testing.T) { + cfg, src, err := LoadWithSources("") + if err != nil { + t.Fatal(err) + } + src2, err := ApplyCLIOverrides(cfg, src, func(c *Config) { + c.HTTPPort = 9999 + c.AgentBackend = "stub" + }) + if err != nil { + t.Fatal(err) + } + if src2["http_port"].Layer != "cli" || !src2["http_port"].Set { + t.Fatalf("http_port layer=%s set=%v", src2["http_port"].Layer, src2["http_port"].Set) + } + if src2["agent_backend"].Layer != "cli" { + t.Fatalf("agent_backend layer=%s", src2["agent_backend"].Layer) + } + // 未覆盖字段来源不变 + if src2["http_host"].Layer != src["http_host"].Layer { + t.Fatalf("http_host layer changed unexpectedly") + } +} + +// Load 行为回归：老入口签名与结果不变。 +func TestLoadStillWorks(t *testing.T) { + cfg, err := Load("") + if err != nil { + t.Fatal(err) + } + if cfg.HTTPPort == 0 || cfg.DataDir == "" { + t.Fatal("defaults lost") } } diff --git a/internal/config/snapshot.go b/internal/config/snapshot.go new file mode 100644 index 00000000..88497379 --- /dev/null +++ b/internal/config/snapshot.go @@ -0,0 +1,102 @@ +package config + +import ( + "fmt" + "reflect" + "sort" + "time" +) + +// FieldSource 是单个配置字段的来源解释（KTD17：只报告来源/状态，不打印值）。 +type FieldSource struct { + Field string // json 标签名（无标签时为小写字段名） + Layer string // default | env-file: | process-env | cli + Set bool // 是否被显式设置（非默认值即视为被设置） +} + +// snapshotFields 用反射导出 Config 的全部字段值（格式化字符串）。 +// 字段均为基本类型/时间片，无指针与嵌套结构；新增字段自动纳入。 +func snapshotFields(c *Config) map[string]string { + out := map[string]string{} + v := reflect.ValueOf(*c) + t := v.Type() + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if f.PkgPath != "" { + continue // 未导出 + } + name := f.Tag.Get("json") + if idx := len(name); idx > 0 { + // 去 omitempty 等选项 + for j := 0; j < len(name); j++ { + if name[j] == ',' { + name = name[:j] + break + } + } + } + if name == "" { + name = lowerFirst(f.Name) + } + fv := v.Field(i) + var s string + if fv.Type() == reflect.TypeOf(time.Duration(0)) { + s = fv.Interface().(time.Duration).String() + } else { + s = fmt.Sprintf("%v", fv.Interface()) + } + out[name] = s + } + return out +} + +func lowerFirst(s string) string { + if s == "" { + return s + } + b := []byte(s) + if b[0] >= 'A' && b[0] <= 'Z' { + b[0] += 'a' - 'A' + } + return string(b) +} + +// diffLayers 依序比较各阶段快照，返回每个字段的最后变更层（即生效来源）。 +// phases 按应用顺序给出（层名, 快照）；字段未出现于任何快照时归为 default。 +func diffLayers(phases []struct { + Layer string + Snap map[string]string +}) map[string]FieldSource { + src := map[string]FieldSource{} + var prev map[string]string + for _, ph := range phases { + for field, val := range ph.Snap { + fs := src[field] + fs.Field = field + if prev == nil || prev[field] != val { + fs.Layer = ph.Layer + fs.Set = prev == nil || prev[field] != val && ph.Layer != phases[0].Layer + } + src[field] = fs + } + prev = ph.Snap + } + // default 层不算"显式设置" + for field, fs := range src { + if fs.Layer == phases[0].Layer { + fs.Set = false + src[field] = fs + } + } + return src +} + +// SortedFieldSources 排序输出（稳定渲染用）。 +func SortedFieldSources(m map[string]FieldSource) []FieldSource { + out := make([]FieldSource, 0, len(m)) + for _, fs := range m { + out = append(out, fs) + } + sort.Slice(out, func(i, j int) bool { return out[i].Field < out[j].Field }) + return out +}