Agent-System/internal/engineeringdashboard/source_docs.go

package engineeringdashboard

import (
	"bufio"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"gopkg.in/yaml.v3"
)

const (
	maxDocBodyBytes     = 256 * 1024
	maxTotalBodyBytes   = 8 * 1024 * 1024
	defaultDocsSource   = "docs"
)

var mdLinkRe = regexp.MustCompile(`\[([^\]]*)\]\(([^)]+)\)`)

// DocsCollectResult is the docs collector output.
type DocsCollectResult struct {
	Source    SourceState
	Documents []DocumentNode
	Facts     []Fact
	Modules   []ModuleView
	Tasks     []TaskState
	Warnings  []string
}

// CollectDocs indexes allowlisted markdown under repoRoot.
func CollectDocs(repoRoot string, now time.Time) DocsCollectResult {
	start := time.Now()
	res := DocsCollectResult{}
	totalBody := 0
	var docs []DocumentNode
	byID := map[string]*DocumentNode{}

	err := filepath.WalkDir(filepath.Join(repoRoot, ".doc"), func(path string, d os.DirEntry, err error) error {
		if err != nil {
			res.Warnings = append(res.Warnings, err.Error())
			return nil
		}
		rel, err := filepath.Rel(repoRoot, path)
		if err != nil {
			return nil
		}
		rel = filepath.ToSlash(rel)
		if d.IsDir() {
			if d.Name() == "assets" {
				return filepath.SkipDir
			}
			return nil
		}
		if !strings.HasSuffix(strings.ToLower(rel), ".md") {
			return nil
		}
		if IsAssetsPath(rel) {
			return nil
		}
		abs, err := SafeDocPath(repoRoot, rel)
		if err != nil {
			res.Warnings = append(res.Warnings, err.Error())
			return nil
		}
		raw, err := os.ReadFile(abs)
		if err != nil {
			res.Warnings = append(res.Warnings, err.Error())
			return nil
		}
		node := parseMarkdownDoc(rel, string(raw), &totalBody)
		docs = append(docs, node)
		byID[node.ID] = &docs[len(docs)-1]
		return nil
	})
	if err != nil {
		res.Source = SourceState{
			Name: defaultDocsSource, Status: SourceError, Freshness: FreshnessError,
			ObservedAt: now, Error: err.Error(), DurationMS: time.Since(start).Milliseconds(),
		}
		return res
	}

	// reverse links + broken
	idSet := map[string]bool{}
	for i := range docs {
		idSet[docs[i].ID] = true
	}
	for i := range docs {
		var broken []string
		for _, out := range docs[i].LinksOut {
			target := normalizeLinkTarget(docs[i].Path, out)
			if target == "" {
				continue
			}
			if !idSet[target] {
				broken = append(broken, out)
				continue
			}
			if t := byID[target]; t != nil {
				t.LinksIn = append(t.LinksIn, docs[i].ID)
			}
		}
		docs[i].BrokenLinks = broken
	}
	for i := range docs {
		if len(docs[i].LinksIn) == 0 && docs[i].Taxonomy != "ops" {
			// orphan heuristic: no inbound and not CURRENT-like
			if !strings.Contains(docs[i].Path, "CURRENT.md") && !strings.Contains(docs[i].Path, "README.md") {
				docs[i].Orphan = len(docs[i].LinksOut) == 0
			}
		}
	}

	res.Documents = docs
	res.Modules = parseFeatureMatrixModules(docs)
	res.Tasks = parseTaskDocs(docs, now)
	res.Facts = append(res.Facts, factsFromTasks(res.Tasks, now)...)
	res.Source = SourceState{
		Name: defaultDocsSource, Status: SourceOK, Freshness: FreshnessFresh,
		ObservedAt: now, LastSuccessAt: &now, DurationMS: time.Since(start).Milliseconds(),
	}
	return res
}

func parseMarkdownDoc(rel, raw string, totalBody *int) DocumentNode {
	node := DocumentNode{
		ID:   filepath.ToSlash(rel),
		Path: filepath.ToSlash(rel),
	}
	body := raw
	fm, rest, warn := splitFrontMatter(raw)
	if warn != "" {
		node.ParseWarnings = append(node.ParseWarnings, warn)
	}
	if fm != nil {
		node.FrontMatter = fm
		if s := fm["status"]; s != "" {
			node.Status = s
		}
		if t := fm["title"]; t != "" {
			node.Title = t
		}
		body = rest
	}
	if node.Title == "" {
		node.Title = firstHeading(body)
	}
	if node.Title == "" {
		node.Title = filepath.Base(rel)
	}
	node.Taxonomy = taxonomyOf(rel)
	for _, m := range mdLinkRe.FindAllStringSubmatch(body, -1) {
		href := strings.TrimSpace(m[2])
		if href == "" || strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") || strings.HasPrefix(href, "mailto:") {
			continue
		}
		if strings.HasPrefix(href, "#") {
			continue
		}
		node.LinksOut = append(node.LinksOut, href)
	}
	remain := maxTotalBodyBytes - *totalBody
	if remain < 0 {
		remain = 0
	}
	limit := maxDocBodyBytes
	if remain < limit {
		limit = remain
	}
	if len(body) > limit {
		node.Body = body[:limit]
		node.BodyTruncated = true
	} else {
		node.Body = body
	}
	*totalBody += len(node.Body)
	return node
}

func splitFrontMatter(raw string) (map[string]string, string, string) {
	if !strings.HasPrefix(raw, "---\n") && !strings.HasPrefix(raw, "---\r\n") {
		return nil, raw, ""
	}
	rest := raw[3:]
	if strings.HasPrefix(rest, "\r\n") {
		rest = rest[2:]
	} else if strings.HasPrefix(rest, "\n") {
		rest = rest[1:]
	}
	end := strings.Index(rest, "\n---")
	if end < 0 {
		return nil, raw, "front matter not closed"
	}
	block := rest[:end]
	body := rest[end+4:]
	body = strings.TrimPrefix(body, "\r")
	body = strings.TrimPrefix(body, "\n")
	var y map[string]any
	if err := yaml.Unmarshal([]byte(block), &y); err != nil {
		return nil, raw, "front matter yaml: " + err.Error()
	}
	out := map[string]string{}
	for k, v := range y {
		out[k] = stringifyYAML(v)
	}
	return out, body, ""
}

func stringifyYAML(v any) string {
	switch t := v.(type) {
	case string:
		return t
	case bool:
		if t {
			return "true"
		}
		return "false"
	default:
		b, _ := yaml.Marshal(v)
		return strings.TrimSpace(string(b))
	}
}

func firstHeading(body string) string {
	sc := bufio.NewScanner(strings.NewReader(body))
	for sc.Scan() {
		line := strings.TrimSpace(sc.Text())
		if strings.HasPrefix(line, "# ") {
			return strings.TrimSpace(strings.TrimPrefix(line, "# "))
		}
	}
	return ""
}

func taxonomyOf(rel string) string {
	rel = filepath.ToSlash(rel)
	parts := strings.Split(rel, "/")
	if len(parts) >= 2 && parts[0] == ".doc" {
		return parts[1]
	}
	return ""
}

func normalizeLinkTarget(fromPath, href string) string {
	href = strings.Split(href, "#")[0]
	href = strings.TrimSpace(href)
	if href == "" {
		return ""
	}
	dir := filepath.ToSlash(filepath.Dir(fromPath))
	joined := filepath.ToSlash(filepath.Clean(filepath.Join(dir, href)))
	joined = strings.TrimPrefix(joined, "/")
	if !strings.HasSuffix(strings.ToLower(joined), ".md") {
		return ""
	}
	return joined
}

func parseFeatureMatrixModules(docs []DocumentNode) []ModuleView {
	var out []ModuleView
	for _, d := range docs {
		if !strings.HasSuffix(d.Path, "FEATURE-MATRIX.md") {
			continue
		}
		sc := bufio.NewScanner(strings.NewReader(d.Body))
		for sc.Scan() {
			line := sc.Text()
			if !strings.HasPrefix(line, "|") || strings.Contains(line, "---") {
				continue
			}
			cols := splitTableRow(line)
			if len(cols) < 7 {
				continue
			}
			name := strings.TrimSpace(cols[0])
			if name == "" || name == "模块" || strings.HasPrefix(name, "**") {
				continue
			}
			out = append(out, ModuleView{
				Name:     name,
				Status:   strings.TrimSpace(cols[2]),
				Mutex:    strings.Trim(strings.TrimSpace(cols[6]), "`"),
				Evidence: []string{d.Path},
			})
		}
	}
	return out
}

func splitTableRow(line string) []string {
	line = strings.Trim(line, "|")
	parts := strings.Split(line, "|")
	for i := range parts {
		parts[i] = strings.TrimSpace(parts[i])
	}
	return parts
}

func parseTaskDocs(docs []DocumentNode, now time.Time) []TaskState {
	var out []TaskState
	for _, d := range docs {
		if !strings.Contains(d.Path, "/short-term/") && !strings.Contains(d.Path, "/long-term/") && !strings.Contains(d.Path, "/backlog/") {
			continue
		}
		st := DocStatusUnknown
		if d.Status != "" {
			st = DocStatus(d.Status)
		}
		claimed := false
		owner := d.FrontMatter["owner"]
		branch := d.FrontMatter["branch"]
		wt := d.FrontMatter["worktree"]
		mutex := d.FrontMatter["mutex"]
		if owner != "" && owner != "unassigned" {
			claimed = true
		}
		kind := AgentKindUnknown
		if strings.Contains(strings.ToLower(owner), "herdr") {
			kind = AgentKindHerdrEngineering
		} else if owner != "" && owner != "unassigned" {
			kind = AgentKindHuman
		}
		task := TaskState{
			ID:        "task:" + d.ID,
			Title:     d.Title,
			Path:      d.Path,
			DocStatus: st,
			Claim: AgentClaim{
				Owner: owner, AgentKind: kind, Claimed: claimed,
				Mutex: mutex, Branch: branch, Worktree: wt,
				Activity: "unknown",
				Evidence: "declared in task front matter / CURRENT; activity unknown without runtime source",
			},
			Git:          GitFacets{Branch: branch, Worktree: wt, Cleanliness: "unknown", RemoteNote: "git facets filled by reconcile"},
			Verification: VerificationFacets{Result: "unknown", Freshness: FreshnessUnknown},
			Deployment: map[string]DeploymentFacets{
				"dev":  {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"},
				"test": {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"},
				"main": {ContainerStatus: "unknown", AppReadiness: "unknown", ServicesOK: "unknown"},
			},
			EvidenceChain: append([]string{d.Path}, d.LinksOut...),
		}
		_ = now
		out = append(out, task)
	}
	return out
}

func factsFromTasks(tasks []TaskState, now time.Time) []Fact {
	var facts []Fact
	for _, t := range tasks {
		id := FactID(t.ID, "doc_status", "docs")
		facts = append(facts, Fact{
			ID: id, Subject: t.ID, Field: "doc_status", Value: string(t.DocStatus),
			Provenance: ProvenanceDeclared, SourceRef: t.Path, ObservedAt: now,
		})
		if t.Claim.Claimed {
			facts = append(facts, Fact{
				ID: FactID(t.ID, "claim", "docs"), Subject: t.ID, Field: "claim",
				Value: t.Claim.Owner, Provenance: ProvenanceDeclared, SourceRef: t.Path, ObservedAt: now,
			})
		}
	}
	return facts
}