Agent-System/internal/intent/segmenter.go

package intent

import (
	"regexp"
	"strings"
)

const maxPlanSegments = 4

var (
	numberedItemHead     = regexp.MustCompile(`(?:^|[\n；;])[\t ]*(?:[1-9]\s*[.、．)]|（[1-9]）|\([1-9]\)|[一二三四五]、)[\t ]*`)
	asciiVersionDot      = regexp.MustCompile(`(?i)(?:v?\d+)\.(?:\d)`)
	internalChainPhrases = []string{
		"查询并总结", "查询并汇总", "查询后总结", "查询后概括",
		"查一下并总结", "查阅并总结", "检索并总结",
		"阅读后概括", "阅读后总结", "阅读后汇总",
		"读完后概括", "读完后总结", "阅读并概括",
	}
	anaphoraMarkers = []string{"这个", "那个", "它", "其", "上述", "刚才的", "该结果", "这份"}
)

type splitMark struct {
	Token string
	Kind  string
}

var sequentialMarks = []splitMark{
	{Token: "，然后", Kind: "connector"},
	{Token: ",然后", Kind: "connector"},
	{Token: "，接着", Kind: "connector"},
	{Token: ",接着", Kind: "connector"},
	{Token: "；再", Kind: "connector"},
	{Token: ";再", Kind: "connector"},
	{Token: "，并且", Kind: "connector"},
	{Token: ",并且", Kind: "connector"},
	{Token: "，并", Kind: "connector"},
	{Token: ",并", Kind: "connector"},
	{Token: "然后", Kind: "connector"},
	{Token: "接着", Kind: "connector"},
	{Token: "；", Kind: "semicolon"},
	{Token: ";", Kind: "semicolon"},
	{Token: "。", Kind: "period"},
}

// SplitPlanSegments deterministically splits an utterance into at most
// independent candidate spans. It never truncates a >4 split; the caller rejects.
func SplitPlanSegments(input string) []InputSegment {
	origin := input
	raw := strings.TrimSpace(input)
	if raw == "" {
		return nil
	}
	if isInternalSingleActionChain(raw) {
		return []InputSegment{makeSegment(origin, 0, len(origin), "", "internal_chain", "high")}
	}
	if segs := splitNumbered(origin); len(segs) >= 2 {
		return segs
	}
	if segs := splitFirstThenFinally(origin); len(segs) >= 2 {
		return segs
	}
	if segs := splitNewlineClauses(origin); len(segs) >= 2 {
		return segs
	}
	if segs := splitSequential(origin); len(segs) >= 2 {
		return segs
	}
	return []InputSegment{makeSegment(origin, 0, len(origin), "", "single", "high")}
}

func isInternalSingleActionChain(input string) bool {
	for _, p := range internalChainPhrases {
		if strings.Contains(input, p) {
			if strings.Contains(input, "然后") || strings.Contains(input, "接着") ||
				strings.Contains(input, "最后") || numberedItemHead.MatchString(input) {
				return false
			}
			return true
		}
	}
	return false
}

func makeSegment(origin string, start, end int, connector, evidence, conf string) InputSegment {
	if start < 0 {
		start = 0
	}
	if end > len(origin) {
		end = len(origin)
	}
	if start > end {
		start, end = end, start
	}
	text := origin[start:end]
	return InputSegment{
		Text:                text,
		Normalized:          normalizeSegmentText(text),
		Span:                TextSpan{Start: start, End: end},
		Connector:           connector,
		Evidence:            evidence,
		StructureConfidence: conf,
	}
}

func normalizeSegmentText(s string) string {
	s = strings.TrimSpace(s)
	fields := strings.Fields(s)
	return strings.Join(fields, " ")
}

func splitNumbered(origin string) []InputSegment {
	locs := numberedItemHead.FindAllStringIndex(origin, -1)
	if len(locs) < 2 {
		return nil
	}
	var segs []InputSegment
	for i, loc := range locs {
		start := loc[1]
		end := len(origin)
		if i+1 < len(locs) {
			end = locs[i+1][0]
		}
		text := strings.TrimSpace(origin[start:end])
		if text == "" {
			return nil
		}
		conf := "high"
		connector := strings.TrimSpace(origin[loc[0]:loc[1]])
		segs = append(segs, makeSegment(origin, start, end, connector, "numbered", conf))
	}
	if len(segs) < 2 {
		return nil
	}
	return segs
}

func splitFirstThenFinally(origin string) []InputSegment {
	idxFirst := indexClauseToken(origin, "先")
	if idxFirst < 0 {
		return nil
	}
	type hit struct {
		idx   int
		token string
	}
	var hits []hit
	searchFrom := idxFirst + len("先")
	for searchFrom < len(origin) {
		next := earliestToken(origin[searchFrom:], []string{"最后", "然后再", "然后", "再"})
		if next.idx < 0 {
			break
		}
		abs := searchFrom + next.idx
		hits = append(hits, hit{idx: abs, token: next.token})
		searchFrom = abs + len(next.token)
	}
	if len(hits) == 0 {
		return nil
	}
	var segs []InputSegment
	firstEnd := hits[0].idx
	firstText := strings.TrimSpace(origin[idxFirst:firstEnd])
	if firstText == "" || firstText == "先" {
		return nil
	}
	segs = append(segs, makeSegment(origin, idxFirst, firstEnd, "先", "first_then", "high"))
	for i, h := range hits {
		start := h.idx + len(h.token)
		end := len(origin)
		if i+1 < len(hits) {
			end = hits[i+1].idx
		}
		if strings.TrimSpace(origin[start:end]) == "" {
			return nil
		}
		segs = append(segs, makeSegment(origin, start, end, h.token, "first_then", "high"))
	}
	if len(segs) < 2 {
		return nil
	}
	return segs
}

type tokenHit struct {
	idx   int
	token string
}

func earliestToken(s string, tokens []string) tokenHit {
	best := tokenHit{idx: -1}
	for _, tok := range tokens {
		i := strings.Index(s, tok)
		if i < 0 {
			continue
		}
		if best.idx < 0 || i < best.idx || (i == best.idx && len(tok) > len(best.token)) {
			best = tokenHit{idx: i, token: tok}
		}
	}
	return best
}

func indexClauseToken(s, tok string) int {
	from := 0
	for from < len(s) {
		rel := strings.Index(s[from:], tok)
		if rel < 0 {
			return -1
		}
		i := from + rel
		if tok == "先" && i >= len("优") && s[i-len("优"):i] == "优" {
			from = i + len(tok)
			continue
		}
		return i
	}
	return -1
}

func splitNewlineClauses(origin string) []InputSegment {
	if !strings.Contains(origin, "\n") {
		return nil
	}
	parts := strings.Split(origin, "\n")
	type piece struct {
		text       string
		start, end int
	}
	var pieces []piece
	cursor := 0
	for _, p := range parts {
		start := cursor
		end := cursor + len(p)
		cursor = end + 1
		trim := strings.TrimSpace(p)
		if trim == "" {
			continue
		}
		if !looksLikeIntentClause(trim) {
			return nil
		}
		pieces = append(pieces, piece{text: p, start: start, end: end})
	}
	if len(pieces) < 2 || len(pieces) > maxPlanSegments {
		if len(pieces) > maxPlanSegments {
			var segs []InputSegment
			for i, p := range pieces {
				conn := "newline"
				if i == 0 {
					conn = ""
				}
				segs = append(segs, makeSegment(origin, p.start, p.end, conn, "newline", "medium"))
			}
			return segs
		}
		return nil
	}
	var segs []InputSegment
	for i, p := range pieces {
		conn := "newline"
		if i == 0 {
			conn = ""
		}
		segs = append(segs, makeSegment(origin, p.start, p.end, conn, "newline", "medium"))
	}
	return segs
}

func looksLikeIntentClause(s string) bool {
	markers := []string{"查", "查询", "生成", "推进", "认领", "送审", "批准", "驳回", "总结", "看看", "帮我", "请"}
	for _, m := range markers {
		if strings.Contains(s, m) {
			return true
		}
	}
	return false
}

func splitSequential(origin string) []InputSegment {
	var segs []InputSegment
	start := 0
	incoming := ""
	for start < len(origin) {
		rel, mark := nextSequentialSplit(origin[start:])
		if rel < 0 {
			if strings.TrimSpace(origin[start:]) == "" {
				break
			}
			evidence := "sequential"
			conf := "high"
			if incoming == "" && len(segs) == 0 {
				evidence = "single"
			}
			if markKindOf(incoming) == "period" {
				conf = "medium"
			}
			segs = append(segs, makeSegment(origin, start, len(origin), incoming, evidence, conf))
			break
		}
		abs := start + rel
		if strings.TrimSpace(origin[start:abs]) == "" {
			start = abs + len(mark.Token)
			incoming = mark.Token
			continue
		}
		segs = append(segs, makeSegment(origin, start, abs, incoming, mark.Kind, confidenceForMark(mark.Kind)))
		incoming = mark.Token
		start = abs + len(mark.Token)
	}
	if len(segs) < 2 {
		return nil
	}
	return segs
}

func markKindOf(token string) string {
	for _, m := range sequentialMarks {
		if m.Token == token {
			return m.Kind
		}
	}
	return ""
}

func nextSequentialSplit(s string) (int, splitMark) {
	bestIdx := -1
	best := splitMark{}
	for _, m := range sequentialMarks {
		idx := 0
		for idx < len(s) {
			rel := strings.Index(s[idx:], m.Token)
			if rel < 0 {
				break
			}
			i := idx + rel
			if i <= 0 {
				idx = i + len(m.Token)
				continue
			}
			if m.Kind == "period" && asciiVersionDot.MatchString(windowAround(s, i, 4)) {
				idx = i + len(m.Token)
				continue
			}
			if isAndSplit(m.Token) && isInternalChainSplit(s, i, m.Token) {
				idx = i + len(m.Token)
				continue
			}
			if bestIdx < 0 || i < bestIdx || (i == bestIdx && len(m.Token) > len(best.Token)) {
				bestIdx = i
				best = m
			}
			break
		}
	}
	return bestIdx, best
}

func isAndSplit(token string) bool {
	switch token {
	case "，并", ",并", "，并且", ",并且":
		return true
	default:
		return false
	}
}

func confidenceForMark(kind string) string {
	switch kind {
	case "period":
		return "medium"
	default:
		return "high"
	}
}

func windowAround(s string, i, n int) string {
	start := i - n
	if start < 0 {
		start = 0
	}
	end := i + n
	if end > len(s) {
		end = len(s)
	}
	return s[start:end]
}

func isInternalChainSplit(s string, idx int, sep string) bool {
	start := idx - 12
	if start < 0 {
		start = 0
	}
	end := idx + len(sep) + 12
	if end > len(s) {
		end = len(s)
	}
	around := s[start:end]
	for _, p := range internalChainPhrases {
		if strings.Contains(around, p) {
			return true
		}
	}
	return false
}

func collectAnaphora(text string) []string {
	var out []string
	for _, m := range anaphoraMarkers {
		if strings.Contains(text, m) {
			out = append(out, m)
		}
	}
	return out
}