Agent-System/internal/ctxmgr/semantic_summary.go

package ctxmgr

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"sort"
	"strings"
	"time"
	"unicode/utf8"
)

// SemanticToolPermissions is the only allowed tool permission for summarizers.
// Summarizers must never receive tool callbacks or escalate auth.
const SemanticToolPermissions = "none"

const (
	ShadowStatusOff      = "off"
	ShadowStatusOK       = "ok"
	ShadowStatusFallback = "fallback"
)

const (
	semanticMaxMessages     = 40
	semanticMaxContentRunes = 240
	semanticProtocolV1      = "semantic-summary-v1"
	heuristicModelID        = "heuristic-semantic-v1"
)

// MinimizedMessage is the R13-minimized outbound unit for a summarizer.
// Only id/role/content are included; no metadata, stage, or auth claims.
type MinimizedMessage struct {
	ID      int64  `json:"id"`
	Role    string `json:"role"`
	Content string `json:"content"`
}

// SemanticSummaryRequest is the sole input to a no-tool summarizer.
type SemanticSummaryRequest struct {
	SessionID            string
	Messages             []MinimizedMessage
	InputDigest          string
	VisibilityDigest     string
	PriorNarrative       string
	PriorFacts           map[string]string
	WatermarkToMessageID int64
	ExpectedRevision     int
}

// SemanticSummaryResult is the summarizer output (advisory under shadow).
type SemanticSummaryResult struct {
	Narrative            string
	Model                string
	Protocol             string
	InputDigest          string // optional echo for server validation
	VisibilityDigest     string // optional echo
	WatermarkToMessageID int64  // optional echo
}

// SemanticShadowReport captures shadow evaluation. Under the fail-closed
// gray rollout a replaced report keeps the displaced deterministic narrative
// so later deterministic regeneration merges clean material (rollback-safe).
type SemanticShadowReport struct {
	Status             string    `json:"status"`
	Narrative          string    `json:"narrative,omitempty"`
	DisplacedNarrative string    `json:"displaced_narrative,omitempty"`
	InputDigest        string    `json:"input_digest,omitempty"`
	Model              string    `json:"model,omitempty"`
	Protocol           string    `json:"protocol,omitempty"`
	ToolPermissions    string    `json:"tool_permissions"`
	FreshnessNote      string    `json:"freshness_note,omitempty"`
	GeneratedAt        time.Time `json:"generated_at"`
}

// SemanticSummarizer produces a narrative from minimized messages.
// Implementations MUST NOT accept tools, callbacks, or auth mutation hooks.
type SemanticSummarizer interface {
	Summarize(ctx context.Context, req SemanticSummaryRequest) (*SemanticSummaryResult, error)
}

// StubSemanticSummarizer is a test double with fixed narrative or error.
type StubSemanticSummarizer struct {
	Narrative string
	Model     string
	Err       error
}

func (s *StubSemanticSummarizer) Summarize(_ context.Context, _ SemanticSummaryRequest) (*SemanticSummaryResult, error) {
	if s == nil {
		return nil, fmt.Errorf("semantic summarizer unavailable")
	}
	if s.Err != nil {
		return nil, s.Err
	}
	model := s.Model
	if model == "" {
		model = "stub-semantic-v1"
	}
	return &SemanticSummaryResult{
		Narrative: s.Narrative,
		Model:     model,
		Protocol:  semanticProtocolV1,
	}, nil
}

// HeuristicSemanticSummarizer is a no-network, no-tool fallback semantic engine
// used when shadow is on and no custom summarizer is injected.
type HeuristicSemanticSummarizer struct{}

func (h *HeuristicSemanticSummarizer) Summarize(_ context.Context, req SemanticSummaryRequest) (*SemanticSummaryResult, error) {
	if len(req.Messages) == 0 {
		return nil, fmt.Errorf("no authorized messages for semantic summary")
	}
	var parts []string
	for _, m := range req.Messages {
		if m.Role != "user" {
			continue
		}
		c := strings.TrimSpace(m.Content)
		if c == "" {
			continue
		}
		// Injection text is material only; never interpret as system/auth directives.
		c = stripInstructionishPrefix(c)
		if c == "" {
			continue
		}
		parts = append(parts, truncateRunes(c, 48))
		if len(parts) >= 4 {
			break
		}
	}
	if len(parts) == 0 {
		parts = append(parts, fmt.Sprintf("覆盖 %d 条授权可见对话", len(req.Messages)))
	}
	narrative := "语义要点（shadow）：" + strings.Join(parts, "；")
	if utf8.RuneCountInString(narrative) > 280 {
		narrative = truncateRunes(narrative, 280) + "…"
	}
	return &SemanticSummaryResult{
		Narrative: narrative,
		Model:     heuristicModelID,
		Protocol:  semanticProtocolV1,
	}, nil
}

func stripInstructionishPrefix(c string) string {
	lower := strings.ToLower(c)
	markers := []string{
		"ignore previous",
		"system:",
		"忽略以上",
		"忽略之前",
		"提权",
		"grant yourself",
		"override auth",
		"将所有阶段",
	}
	for _, m := range markers {
		if strings.Contains(lower, strings.ToLower(m)) || strings.Contains(c, m) {
			// Keep as opaque material; do not drop evidence, but neutralize directive framing.
			return "【用户材料】" + c
		}
	}
	return c
}

// MinimizeAuthorizedMessages builds R13-minimized summarizer input.
// Callers must pass only authorized-visible messages.
func MinimizeAuthorizedMessages(msgs []ChatMessageRecord) []MinimizedMessage {
	if len(msgs) == 0 {
		return nil
	}
	start := 0
	if len(msgs) > semanticMaxMessages {
		start = len(msgs) - semanticMaxMessages
	}
	out := make([]MinimizedMessage, 0, len(msgs)-start)
	for _, m := range msgs[start:] {
		out = append(out, MinimizedMessage{
			ID:      m.ID,
			Role:    m.Role,
			Content: truncateRunes(strings.TrimSpace(m.Content), semanticMaxContentRunes),
		})
	}
	return out
}

// DigestAuthorizedInput produces a stable digest over minimized messages.
func DigestAuthorizedInput(msgs []MinimizedMessage) string {
	h := sha256.New()
	for _, m := range msgs {
		_, _ = fmt.Fprintf(h, "%d|%s|%s\n", m.ID, m.Role, m.Content)
	}
	return hex.EncodeToString(h.Sum(nil))
}

// DigestAuthorizedVisibility digests the authorized-visible message identity set
// (id+role only). Used as chat_summaries.visibility_digest binding.
func DigestAuthorizedVisibility(msgs []MinimizedMessage) string {
	h := sha256.New()
	for _, m := range msgs {
		_, _ = fmt.Fprintf(h, "%d|%s\n", m.ID, m.Role)
	}
	return hex.EncodeToString(h.Sum(nil))
}

// MergePriorSummaryIntoInput folds the previous deterministic summary into the
// semantic request so shadow generation does not drop earlier covered facts.
func MergePriorSummaryIntoInput(msgs []MinimizedMessage, prior *ChatSummary) (merged []MinimizedMessage, priorNarrative string, priorFacts map[string]string) {
	merged = msgs
	if prior == nil {
		return merged, "", nil
	}
	priorNarrative = strings.TrimSpace(prior.Narrative)
	if len(prior.Facts) > 0 {
		priorFacts = make(map[string]string, len(prior.Facts))
		for k, v := range prior.Facts {
			priorFacts[k] = v
		}
	}
	return merged, priorNarrative, priorFacts
}

// DigestSemanticInput digests minimized messages plus merged prior summary material.
func DigestSemanticInput(msgs []MinimizedMessage, priorNarrative string, priorFacts map[string]string) string {
	h := sha256.New()
	for _, m := range msgs {
		_, _ = fmt.Fprintf(h, "%d|%s|%s\n", m.ID, m.Role, m.Content)
	}
	_, _ = fmt.Fprintf(h, "prior|%s\n", priorNarrative)
	if len(priorFacts) > 0 {
		keys := make([]string, 0, len(priorFacts))
		for k := range priorFacts {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			_, _ = fmt.Fprintf(h, "fact|%s=%s\n", k, priorFacts[k])
		}
	}
	return hex.EncodeToString(h.Sum(nil))
}

// scheduleSemanticShadow attaches a pending advisory report and enqueues background work.
// Served Narrative/Facts remain deterministic; REPLACE stays unwired.
func scheduleSemanticShadow(ctx context.Context, m *ContextManager, sum *ChatSummary, msgs []ChatMessageRecord, prior *ChatSummary) {
	if sum == nil || m == nil {
		return
	}
	sum.Source = "deterministic"
	if !SemanticSummaryShadowEnabled() {
		return
	}
	now := time.Now()
	minimized := MinimizeAuthorizedMessages(msgs)
	merged, priorNarrative, priorFacts := MergePriorSummaryIntoInput(minimized, prior)
	visDigest := DigestAuthorizedVisibility(merged)
	if sum.VisibilityDigest == "" {
		sum.VisibilityDigest = visDigest
	}
	inputDigest := DigestSemanticInput(merged, priorNarrative, priorFacts)
	report := &SemanticShadowReport{
		Status:          ShadowStatusPending,
		InputDigest:     inputDigest,
		ToolPermissions: SemanticToolPermissions,
		GeneratedAt:     now,
		FreshnessNote:   fmt.Sprintf("semantic_shadow_pending; deterministic served immediately at %s", now.UTC().Format(time.RFC3339)),
	}
	sum.Shadow = report
	sum.FreshnessNote = report.FreshnessNote
	job := SemanticShadowJob{
		SessionID:            sum.SessionID,
		SummaryRevision:      sum.Revision,
		InputDigest:          inputDigest,
		VisibilityDigest:     sum.VisibilityDigest,
		WatermarkToMessageID: sum.ToMessageID,
		PriorRevision:        sum.PreviousRevision,
		Messages:             merged,
		PriorNarrative:       priorNarrative,
		PriorFacts:           priorFacts,
	}
	if err := m.EnqueueSemanticShadow(ctx, job); err != nil {
		report.Status = ShadowStatusFallback
		report.FreshnessNote = fmt.Sprintf("semantic_shadow_fallback: enqueue_failed; deterministic retained at %s", now.UTC().Format(time.RFC3339))
		sum.Shadow = report
		sum.FreshnessNote = report.FreshnessNote
	}
}

func (m *ContextManager) SetSemanticSummarizer(s SemanticSummarizer) {
	if m == nil {
		return
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	m.semanticSummarizer = s
}

func (m *ContextManager) semanticSummarizerOrDefault() SemanticSummarizer {
	if m == nil {
		return &HeuristicSemanticSummarizer{}
	}
	m.mu.RLock()
	s := m.semanticSummarizer
	m.mu.RUnlock()
	if s != nil {
		return s
	}
	return &HeuristicSemanticSummarizer{}
}

// applySemanticShadow is retained for daily digest sync advisory evaluation.
// Rolling chat summaries use scheduleSemanticShadow (background reliable task).
// REPLACE is intentionally unwired: served narrative/facts stay deterministic.
func applySemanticShadow(ctx context.Context, sum *ChatSummary, msgs []ChatMessageRecord, summarizer SemanticSummarizer) {
	if sum == nil {
		return
	}
	sum.Source = "deterministic"
	if !SemanticSummaryShadowEnabled() {
		return
	}
	now := time.Now()
	minimized := MinimizeAuthorizedMessages(msgs)
	if sum.VisibilityDigest == "" {
		sum.VisibilityDigest = DigestAuthorizedVisibility(minimized)
	}
	digest := DigestAuthorizedInput(minimized)
	report := &SemanticShadowReport{
		Status:          ShadowStatusFallback,
		InputDigest:     digest,
		ToolPermissions: SemanticToolPermissions,
		GeneratedAt:     now,
	}
	if summarizer == nil {
		report.FreshnessNote = fmt.Sprintf("semantic_shadow_fallback: summarizer_unavailable; deterministic retained at %s", now.UTC().Format(time.RFC3339))
		sum.Shadow = report
		sum.FreshnessNote = report.FreshnessNote
		return
	}
	res, err := summarizer.Summarize(ctx, SemanticSummaryRequest{
		SessionID:            sum.SessionID,
		Messages:             minimized,
		InputDigest:          digest,
		VisibilityDigest:     sum.VisibilityDigest,
		WatermarkToMessageID: sum.ToMessageID,
		ExpectedRevision:     sum.Revision,
	})
	if err != nil || res == nil || strings.TrimSpace(res.Narrative) == "" {
		reason := "empty_or_error"
		if err != nil {
			reason = "model_error"
		}
		report.FreshnessNote = fmt.Sprintf("semantic_shadow_fallback: %s; deterministic retained at %s", reason, now.UTC().Format(time.RFC3339))
		sum.Shadow = report
		sum.FreshnessNote = report.FreshnessNote
		return
	}
	report.Status = ShadowStatusOK
	report.Narrative = res.Narrative
	report.Model = res.Model
	report.Protocol = res.Protocol
	if report.Protocol == "" {
		report.Protocol = semanticProtocolV1
	}
	// Explicitly do NOT swap sum.Narrative here: the rolling-chat replacement
	// path runs in persistShadowProvenance under the fail-closed gray rollout;
	// this advisory path (daily digest sync) stays deterministic.
	sum.Shadow = report
}

// applyDigestSemanticShadow attaches shadow metadata to a daily digest without replacing rule fields.
func applyDigestSemanticShadow(ctx context.Context, digest *DailyDigest, msgs []ChatMessageRecord, summarizer SemanticSummarizer) {
	if digest == nil || !SemanticSummaryShadowEnabled() {
		return
	}
	tmp := &ChatSummary{SessionID: digest.SessionID, Narrative: digest.Headline}
	applySemanticShadow(ctx, tmp, msgs, summarizer)
	if tmp.Shadow == nil {
		return
	}
	digest.Shadow = tmp.Shadow
	digest.FreshnessNote = tmp.FreshnessNote
	// Headline/KeyPoints remain rule/deterministic; shadow narrative is advisory only.
}