weichenandCursor
268a9bdb56
Add canonicalize, stable block map, diff, risk floor, memory ledger service, contract fixtures, concurrency tests, and backfill CLI. Co-authored-by: Cursor <cursoragent@cursor.com>
50 lines
1.0 KiB
Go
50 lines
1.0 KiB
Go
package docrevision
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
// CanonicalizeMarkdown applies LF, UTF-8 NFC, trailing newline, and trims trailing spaces per line.
|
|
func CanonicalizeMarkdown(raw string) string {
|
|
s := strings.ReplaceAll(raw, "\r\n", "\n")
|
|
s = strings.ReplaceAll(s, "\r", "\n")
|
|
s = norm.NFC.String(s)
|
|
lines := strings.Split(s, "\n")
|
|
for i, line := range lines {
|
|
lines[i] = strings.TrimRightFunc(line, func(r rune) bool {
|
|
return r == ' ' || r == '\t'
|
|
})
|
|
}
|
|
s = strings.Join(lines, "\n")
|
|
if s != "" && !strings.HasSuffix(s, "\n") {
|
|
s += "\n"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func SHA256Hex(content string) string {
|
|
sum := sha256.Sum256([]byte(content))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func ContentFingerprint(s string) string {
|
|
var b strings.Builder
|
|
for _, r := range strings.ToLower(s) {
|
|
if unicode.IsSpace(r) {
|
|
continue
|
|
}
|
|
b.WriteRune(r)
|
|
}
|
|
return SHA256Hex(b.String())[:16]
|
|
}
|
|
|
|
func RuneCount(s string) int {
|
|
return utf8.RuneCountInString(s)
|
|
}
|