diff --git a/.doc/system-design/module-boundaries.yaml b/.doc/system-design/module-boundaries.yaml index cc3ee7c2..12d4819f 100644 --- a/.doc/system-design/module-boundaries.yaml +++ b/.doc/system-design/module-boundaries.yaml @@ -114,12 +114,14 @@ imports: - docrevision docgov: - app/uow + - doccontent - docduty - - docrevision - fsutil docrevision: - app/uow + - doccontent - fsutil + doccontent: [] freedoc: - app/uow - ctxmgr diff --git a/internal/doccontent/canonicalize.go b/internal/doccontent/canonicalize.go new file mode 100644 index 00000000..44261568 --- /dev/null +++ b/internal/doccontent/canonicalize.go @@ -0,0 +1,45 @@ +package doccontent + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// CanonicalizeMarkdown applies LF, UTF-8 NFC, and a single trailing newline. +// It must not trim trailing spaces or tabs: two trailing spaces are Markdown hard breaks. +func CanonicalizeMarkdown(raw string) string { + s := strings.ReplaceAll(raw, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + s = norm.NFC.String(s) + if s != "" && !strings.HasSuffix(s, "\n") { + s += "\n" + } + return s +} + +// LegacyDestructiveCanonicalizeMarkdown reproduces the pre-fix canonicalizer that +// stripped per-line trailing spaces/tabs. Used only to classify historical drift. +func LegacyDestructiveCanonicalizeMarkdown(raw string) string { + s := strings.ReplaceAll(raw, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + s = norm.NFC.String(s) + lines := strings.Split(s, "\n") + for i, line := range lines { + lines[i] = strings.TrimRightFunc(line, func(r rune) bool { + return r == ' ' || r == '\t' + }) + } + s = strings.Join(lines, "\n") + if s != "" && !strings.HasSuffix(s, "\n") { + s += "\n" + } + return s +} + +func SHA256Hex(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/doccontent/canonicalize_golden_test.go b/internal/doccontent/canonicalize_golden_test.go new file mode 100644 index 00000000..8c05c4f5 --- /dev/null +++ b/internal/doccontent/canonicalize_golden_test.go @@ -0,0 +1,70 @@ +package doccontent_test + +import ( + "strings" + "testing" + + "chat-agent-system/internal/doccontent" +) + +func TestCanonicalizeMarkdown_PreservesHardBreakDoubleSpaces(t *testing.T) { + raw := "第一行 \n第二行 \n普通行\n" + got := doccontent.CanonicalizeMarkdown(raw) + if !strings.Contains(got, "第一行 \n") || !strings.Contains(got, "第二行 \n") { + t.Fatalf("hard-break double spaces must be kept: %q", got) + } + if strings.Count(got, " \n") != 2 { + t.Fatalf("want 2 hard-breaks, got %q", got) + } + if got != raw { + t.Fatalf("already-canonical hard-break markdown must be identity: got %q", got) + } +} + +func TestCanonicalizeMarkdown_OnlyNormalizesLF_NFC_TrailingNewline(t *testing.T) { + in := "a\r\n b \t\n" + out := doccontent.CanonicalizeMarkdown(in) + if out != "a\n b \t\n" { + t.Fatalf("must keep trailing spaces/tabs and only normalize CRLF: got %q", out) + } +} + +func TestLegacyDestructiveCanonicalize_TrimsTrailingSpaces(t *testing.T) { + raw := "第一行 \n第二行 \n" + legacy := doccontent.LegacyDestructiveCanonicalizeMarkdown(raw) + if strings.Contains(legacy, " \n") { + t.Fatalf("legacy destructive canonicalizer must trim hard-breaks: %q", legacy) + } + if legacy != "第一行\n第二行\n" { + t.Fatalf("got %q", legacy) + } + if doccontent.SHA256Hex(legacy) == doccontent.SHA256Hex(doccontent.CanonicalizeMarkdown(raw)) { + t.Fatal("legacy and semantic-preserving hashes must differ for hard-break docs") + } +} + +func TestCanonicalizeMarkdown_Project143HardBreakGolden(t *testing.T) { + var b strings.Builder + for i := 0; i < 5; i++ { + b.WriteString("条目") + b.WriteByte('0' + byte(i)) + b.WriteString(" \n") + } + b.WriteString("结尾段落。\n") + raw := b.String() + canon := doccontent.CanonicalizeMarkdown(raw) + if canon != raw { + t.Fatalf("project143-shaped hard-breaks must survive: got %q", canon) + } + legacy := doccontent.LegacyDestructiveCanonicalizeMarkdown(raw) + if len(raw)-len(legacy) != 10 { + t.Fatalf("legacy must drop exactly 10 bytes (5×2 spaces), raw=%d legacy=%d", len(raw), len(legacy)) + } +} + +func TestDocrevisionWrappersShareCanonicalBytes(t *testing.T) { + raw := "第一行 \r\n第二行 \n" + if doccontent.CanonicalizeMarkdown(raw) == "" { + t.Fatal("empty canon") + } +} diff --git a/internal/docgov/lease.go b/internal/docgov/lease.go index 08f3d691..73f017a1 100644 --- a/internal/docgov/lease.go +++ b/internal/docgov/lease.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "chat-agent-system/internal/docrevision" + "chat-agent-system/internal/doccontent" "chat-agent-system/internal/fsutil" ) @@ -382,8 +382,8 @@ func (e *GovernanceEngine) SaveDocumentContent(ctx context.Context, req ContentS } now := time.Now() - canon := docrevision.CanonicalizeMarkdown(req.MarkdownBody) - shaStr := docrevision.SHA256Hex(canon) + canon := doccontent.CanonicalizeMarkdown(req.MarkdownBody) + shaStr := doccontent.SHA256Hex(canon) doc.ContentSHA256 = shaStr doc.IsTemplateOnly = false doc.AuthorClaimed = true diff --git a/internal/docrevision/canonicalize.go b/internal/docrevision/canonicalize.go index 967348ae..576256ad 100644 --- a/internal/docrevision/canonicalize.go +++ b/internal/docrevision/canonicalize.go @@ -1,49 +1,23 @@ package docrevision import ( - "crypto/sha256" - "encoding/hex" "strings" "unicode" "unicode/utf8" - "golang.org/x/text/unicode/norm" + "chat-agent-system/internal/doccontent" ) -// CanonicalizeMarkdown applies LF, UTF-8 NFC, and a single trailing newline. -// It must not trim trailing spaces or tabs: two trailing spaces are Markdown hard breaks. func CanonicalizeMarkdown(raw string) string { - s := strings.ReplaceAll(raw, "\r\n", "\n") - s = strings.ReplaceAll(s, "\r", "\n") - s = norm.NFC.String(s) - if s != "" && !strings.HasSuffix(s, "\n") { - s += "\n" - } - return s + return doccontent.CanonicalizeMarkdown(raw) } -// LegacyDestructiveCanonicalizeMarkdown reproduces the pre-fix canonicalizer that -// stripped per-line trailing spaces/tabs. Used only to classify historical drift. func LegacyDestructiveCanonicalizeMarkdown(raw string) string { - s := strings.ReplaceAll(raw, "\r\n", "\n") - s = strings.ReplaceAll(s, "\r", "\n") - s = norm.NFC.String(s) - lines := strings.Split(s, "\n") - for i, line := range lines { - lines[i] = strings.TrimRightFunc(line, func(r rune) bool { - return r == ' ' || r == '\t' - }) - } - s = strings.Join(lines, "\n") - if s != "" && !strings.HasSuffix(s, "\n") { - s += "\n" - } - return s + return doccontent.LegacyDestructiveCanonicalizeMarkdown(raw) } func SHA256Hex(content string) string { - sum := sha256.Sum256([]byte(content)) - return hex.EncodeToString(sum[:]) + return doccontent.SHA256Hex(content) } func ContentFingerprint(s string) string { diff --git a/internal/docrevision/canonicalize_golden_test.go b/internal/docrevision/canonicalize_golden_test.go index 4419eed2..9e8d4ea8 100644 --- a/internal/docrevision/canonicalize_golden_test.go +++ b/internal/docrevision/canonicalize_golden_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "chat-agent-system/internal/doccontent" "chat-agent-system/internal/docrevision" ) @@ -62,3 +63,16 @@ func TestCanonicalizeMarkdown_Project143HardBreakGolden(t *testing.T) { t.Fatalf("legacy must drop exactly 10 bytes (5×2 spaces), raw=%d legacy=%d", len(raw), len(legacy)) } } + +func TestCanonicalizeWrappersMatchDoccontent(t *testing.T) { + raw := "第一行 \r\n第二行 \n" + if docrevision.CanonicalizeMarkdown(raw) != doccontent.CanonicalizeMarkdown(raw) { + t.Fatal("CanonicalizeMarkdown wrapper drifted") + } + if docrevision.LegacyDestructiveCanonicalizeMarkdown(raw) != doccontent.LegacyDestructiveCanonicalizeMarkdown(raw) { + t.Fatal("legacy wrapper drifted") + } + if docrevision.SHA256Hex(raw) != doccontent.SHA256Hex(raw) { + t.Fatal("SHA256Hex wrapper drifted") + } +}