package engineeringdashboard import ( "bufio" "os" "path/filepath" "regexp" "strings" "time" "gopkg.in/yaml.v3" ) const ( maxDocBodyBytes = 256 * 1024 maxTotalBodyBytes = 8 * 1024 * 1024 defaultDocsSource = "docs" ) var mdLinkRe = regexp.MustCompile(`\[([^\]]*)\]\(([^)]+)\)`) // DocsCollectResult is the docs collector output. type DocsCollectResult struct { Source SourceState Documents []DocumentNode Facts []Fact Modules []ModuleView Tasks []TaskState Warnings []string } // CollectDocs indexes allowlisted markdown under repoRoot. func CollectDocs(repoRoot string, now time.Time) DocsCollectResult { start := time.Now() res := DocsCollectResult{} totalBody := 0 var docs []DocumentNode byID := map[string]*DocumentNode{} err := filepath.WalkDir(filepath.Join(repoRoot, ".doc"), func(path string, d os.DirEntry, err error) error { if err != nil { res.Warnings = append(res.Warnings, err.Error()) return nil } rel, err := filepath.Rel(repoRoot, path) if err != nil { return nil } rel = filepath.ToSlash(rel) if d.IsDir() { if d.Name() == "assets" { return filepath.SkipDir } return nil } if !strings.HasSuffix(strings.ToLower(rel), ".md") { return nil } if IsAssetsPath(rel) { return nil } abs, err := SafeDocPath(repoRoot, rel) if err != nil { res.Warnings = append(res.Warnings, err.Error()) return nil } raw, err := os.ReadFile(abs) if err != nil { res.Warnings = append(res.Warnings, err.Error()) return nil } node := parseMarkdownDoc(rel, string(raw), &totalBody) docs = append(docs, node) byID[node.ID] = &docs[len(docs)-1] return nil }) if err != nil { res.Source = SourceState{ Name: defaultDocsSource, Status: SourceError, Freshness: FreshnessError, ObservedAt: now, Error: err.Error(), DurationMS: time.Since(start).Milliseconds(), } return res } // reverse links + broken idSet := map[string]bool{} for i := range docs { idSet[docs[i].ID] = true } for i := range docs { var broken []string for _, out := range docs[i].LinksOut { target := normalizeLinkTarget(docs[i].Path, out) if target == "" { continue } if !idSet[target] { broken = append(broken, out) continue } if t := byID[target]; t != nil { t.LinksIn = append(t.LinksIn, docs[i].ID) } } docs[i].BrokenLinks = broken } for i := range docs { if len(docs[i].LinksIn) == 0 && docs[i].Taxonomy != "ops" { // orphan heuristic: no inbound and not CURRENT-like if !strings.Contains(docs[i].Path, "CURRENT.md") && !strings.Contains(docs[i].Path, "README.md") { docs[i].Orphan = len(docs[i].LinksOut) == 0 } } } res.Documents = docs res.Modules = parseFeatureMatrixModules(docs) res.Tasks = parseTaskDocs(docs, now) res.Facts = append(res.Facts, factsFromTasks(res.Tasks, now)...) res.Source = SourceState{ Name: defaultDocsSource, Status: SourceOK, Freshness: FreshnessFresh, ObservedAt: now, LastSuccessAt: &now, DurationMS: time.Since(start).Milliseconds(), } return res } func parseMarkdownDoc(rel, raw string, totalBody *int) DocumentNode { node := DocumentNode{ ID: filepath.ToSlash(rel), Path: filepath.ToSlash(rel), } body := raw fm, rest, warn := splitFrontMatter(raw) if warn != "" { node.ParseWarnings = append(node.ParseWarnings, warn) } if fm != nil { node.FrontMatter = fm if s := fm["status"]; s != "" { node.Status = s } if t := fm["title"]; t != "" { node.Title = t } body = rest } if node.Title == "" { node.Title = firstHeading(body) } if node.Title == "" { node.Title = filepath.Base(rel) } node.Taxonomy = taxonomyOf(rel) for _, m := range mdLinkRe.FindAllStringSubmatch(body, -1) { href := strings.TrimSpace(m[2]) if href == "" || strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") || strings.HasPrefix(href, "mailto:") { continue } if strings.HasPrefix(href, "#") { continue } node.LinksOut = append(node.LinksOut, href) } remain := maxTotalBodyBytes - *totalBody if remain < 0 { remain = 0 } limit := maxDocBodyBytes if remain < limit { limit = remain } if len(body) > limit { node.Body = body[:limit] node.BodyTruncated = true } else { node.Body = body } *totalBody += len(node.Body) return node } func splitFrontMatter(raw string) (map[string]string, string, string) { if !strings.HasPrefix(raw, "---\n") && !strings.HasPrefix(raw, "---\r\n") { return nil, raw, "" } rest := raw[3:] if strings.HasPrefix(rest, "\r\n") { rest = rest[2:] } else if strings.HasPrefix(rest, "\n") { rest = rest[1:] } end := strings.Index(rest, "\n---") if end < 0 { return nil, raw, "front matter not closed" } block := rest[:end] body := rest[end+4:] body = strings.TrimPrefix(body, "\r") body = strings.TrimPrefix(body, "\n") var y map[string]any if err := yaml.Unmarshal([]byte(block), &y); err != nil { return nil, raw, "front matter yaml: " + err.Error() } out := map[string]string{} for k, v := range y { out[k] = stringifyYAML(v) } return out, body, "" } func stringifyYAML(v any) string { switch t := v.(type) { case string: return t case bool: if t { return "true" } return "false" default: b, _ := yaml.Marshal(v) return strings.TrimSpace(string(b)) } } func firstHeading(body string) string { sc := bufio.NewScanner(strings.NewReader(body)) for sc.Scan() { line := strings.TrimSpace(sc.Text()) if strings.HasPrefix(line, "# ") { return strings.TrimSpace(strings.TrimPrefix(line, "# ")) } } return "" } func taxonomyOf(rel string) string { rel = filepath.ToSlash(rel) parts := strings.Split(rel, "/") if len(parts) >= 2 && parts[0] == ".doc" { return parts[1] } return "" } func normalizeLinkTarget(fromPath, href string) string { href = strings.Split(href, "#")[0] href = strings.TrimSpace(href) if href == "" { return "" } dir := filepath.ToSlash(filepath.Dir(fromPath)) joined := filepath.ToSlash(filepath.Clean(filepath.Join(dir, href))) joined = strings.TrimPrefix(joined, "/") if !strings.HasSuffix(strings.ToLower(joined), ".md") { return "" } return joined } func parseFeatureMatrixModules(docs []DocumentNode) []ModuleView { var out []ModuleView for _, d := range docs { if !strings.HasSuffix(d.Path, "FEATURE-MATRIX.md") { continue } sc := bufio.NewScanner(strings.NewReader(d.Body)) for sc.Scan() { line := sc.Text() if !strings.HasPrefix(line, "|") || strings.Contains(line, "---") { continue } cols := splitTableRow(line) if len(cols) < 7 { continue } name := strings.TrimSpace(cols[0]) if name == "" || name == "模块" || strings.HasPrefix(name, "**") { continue } out = append(out, ModuleView{ Name: name, Status: strings.TrimSpace(cols[2]), Mutex: strings.Trim(strings.TrimSpace(cols[6]), "`"), Evidence: []string{d.Path}, }) } } return out } func splitTableRow(line string) []string { line = strings.Trim(line, "|") parts := strings.Split(line, "|") for i := range parts { parts[i] = strings.TrimSpace(parts[i]) } return parts } func parseTaskDocs(docs []DocumentNode, now time.Time) []TaskState { var out []TaskState for _, d := range docs { if !strings.Contains(d.Path, "/short-term/") && !strings.Contains(d.Path, "/long-term/") && !strings.Contains(d.Path, "/backlog/") { continue } st := DocStatusUnknown if d.Status != "" { st = DocStatus(d.Status) } claimed := false owner := d.FrontMatter["owner"] branch := d.FrontMatter["branch"] wt := d.FrontMatter["worktree"] mutex := d.FrontMatter["mutex"] if owner != "" && owner != "unassigned" { claimed = true } kind := AgentKindUnknown if strings.Contains(strings.ToLower(owner), "herdr") { kind = AgentKindHerdrEngineering } else if owner != "" && owner != "unassigned" { kind = AgentKindHuman } task := TaskState{ ID: "task:" + d.ID, Title: d.Title, Path: d.Path, DocStatus: st, Claim: AgentClaim{ Owner: owner, AgentKind: kind, Claimed: claimed, Mutex: mutex, Branch: branch, Worktree: wt, Activity: "unknown", Evidence: "declared in task front matter / CURRENT; activity unknown without runtime source", }, Git: GitFacets{Branch: branch, Worktree: wt, Cleanliness: "unknown", RemoteNote: "git facets filled by reconcile"}, Verification: VerificationFacets{Result: "unknown", Freshness: FreshnessUnknown}, Deployment: map[string]DeploymentFacets{ "dev": {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"}, "test": {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"}, "main": {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"}, }, EvidenceChain: append([]string{d.Path}, d.LinksOut...), } _ = now out = append(out, task) } return out } func factsFromTasks(tasks []TaskState, now time.Time) []Fact { var facts []Fact for _, t := range tasks { id := FactID(t.ID, "doc_status", "docs") facts = append(facts, Fact{ ID: id, Subject: t.ID, Field: "doc_status", Value: string(t.DocStatus), Provenance: ProvenanceDeclared, SourceRef: t.Path, ObservedAt: now, }) if t.Claim.Claimed { facts = append(facts, Fact{ ID: FactID(t.ID, "claim", "docs"), Subject: t.ID, Field: "claim", Value: t.Claim.Owner, Provenance: ProvenanceDeclared, SourceRef: t.Path, ObservedAt: now, }) } } return facts }