weichen
28ed7005dd
Extend agentctl SQLite with catalog tables, CJK FTS5 recall, hard/soft overlap gates, fail-open embeddings, backfill, and MCP/skills hooks so cross-conversation designs cannot silently conflict.
856 lines
30 KiB
Python
856 lines
30 KiB
Python
#!/usr/bin/env python3
|
|
"""Catalog governance layer: requirements, FTS5 CJK recall, overlap gates, embeddings.
|
|
|
|
Lives beside agentctl.py. Additive schema only; never touches product PostgreSQL.
|
|
Vector layer is fail-open: missing/failing embeddings degrade to FTS5 only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import struct
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from typing import Any, Callable, Optional
|
|
|
|
SCHEMA_SQL = '''
|
|
CREATE TABLE IF NOT EXISTS catalog_requirements(
|
|
id TEXT PRIMARY KEY,
|
|
title TEXT NOT NULL,
|
|
statement TEXT NOT NULL,
|
|
feature TEXT,
|
|
source_task TEXT,
|
|
source_doc TEXT,
|
|
owner TEXT NOT NULL,
|
|
state TEXT NOT NULL,
|
|
supersedes TEXT,
|
|
conversation_ref TEXT,
|
|
created REAL NOT NULL,
|
|
updated REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS catalog_req_state ON catalog_requirements(state, feature);
|
|
CREATE INDEX IF NOT EXISTS catalog_req_task ON catalog_requirements(source_task);
|
|
CREATE TABLE IF NOT EXISTS catalog_features(
|
|
id TEXT PRIMARY KEY,
|
|
title TEXT NOT NULL,
|
|
scope TEXT,
|
|
mutex TEXT,
|
|
matrix_status TEXT,
|
|
updated REAL NOT NULL
|
|
);
|
|
CREATE VIRTUAL TABLE IF NOT EXISTS catalog_req_text USING fts5(
|
|
req_id UNINDEXED, body, seg, tokenize='unicode61'
|
|
);
|
|
CREATE TABLE IF NOT EXISTS catalog_overlaps(
|
|
id TEXT PRIMARY KEY,
|
|
probe_task TEXT NOT NULL,
|
|
probe_req TEXT,
|
|
target_req TEXT,
|
|
target_task TEXT,
|
|
target_feature TEXT,
|
|
level TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
score REAL,
|
|
evidence TEXT NOT NULL,
|
|
created REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS catalog_overlaps_probe ON catalog_overlaps(probe_task, created);
|
|
CREATE TABLE IF NOT EXISTS catalog_decisions(
|
|
id TEXT PRIMARY KEY,
|
|
overlap_id TEXT NOT NULL,
|
|
task TEXT NOT NULL,
|
|
decision TEXT NOT NULL,
|
|
decided_by TEXT NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
authorization_ref TEXT,
|
|
created REAL NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS catalog_decisions_task ON catalog_decisions(task, overlap_id);
|
|
CREATE TABLE IF NOT EXISTS catalog_issues(
|
|
issue_id TEXT PRIMARY KEY,
|
|
severity TEXT,
|
|
issue_state TEXT,
|
|
owner TEXT,
|
|
module TEXT,
|
|
keywords TEXT,
|
|
main_record TEXT,
|
|
lesson_ids TEXT,
|
|
requirements TEXT,
|
|
tasks TEXT,
|
|
recurrence_count INTEGER DEFAULT 0,
|
|
evidence_basis TEXT,
|
|
created REAL,
|
|
updated REAL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS catalog_tests(
|
|
id TEXT PRIMARY KEY,
|
|
report_path TEXT NOT NULL,
|
|
task TEXT,
|
|
requirements TEXT,
|
|
scope TEXT,
|
|
verdict TEXT,
|
|
level TEXT,
|
|
real_agent INTEGER NOT NULL DEFAULT 0,
|
|
schema_version TEXT,
|
|
env TEXT,
|
|
sha TEXT,
|
|
created REAL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS catalog_releases(
|
|
id TEXT PRIMARY KEY,
|
|
env TEXT NOT NULL,
|
|
branch TEXT,
|
|
commit_sha TEXT,
|
|
tag TEXT,
|
|
deploy_seq INTEGER,
|
|
schema_version TEXT,
|
|
image_digest TEXT,
|
|
source_task TEXT,
|
|
phase TEXT,
|
|
receipt TEXT,
|
|
created REAL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS catalog_embeddings(
|
|
req_id TEXT PRIMARY KEY,
|
|
model TEXT NOT NULL,
|
|
dim INTEGER NOT NULL,
|
|
vec BLOB NOT NULL,
|
|
source_sha256 TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
created REAL NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS catalog_task_bindings(
|
|
task TEXT PRIMARY KEY,
|
|
requirement TEXT,
|
|
intent TEXT,
|
|
mutex TEXT,
|
|
feature TEXT,
|
|
updated REAL NOT NULL
|
|
);
|
|
'''
|
|
|
|
REQ_STATES = frozenset({
|
|
'proposed', 'accepted', 'in_progress', 'delivered', 'superseded', 'rejected',
|
|
})
|
|
HARD_KINDS = frozenset({'same_mutex', 'same_feature', 'same_scope', 'same_req_id'})
|
|
SOFT_KINDS = frozenset({'semantic'})
|
|
DECISIONS = frozenset({'proceed', 'merge_into', 'supersede', 'defer', 'reject'})
|
|
REQ_ID_RE = re.compile(r'^R-[A-Za-z0-9][A-Za-z0-9._-]{0,120}$')
|
|
|
|
DEFAULT_GATE = {
|
|
'hard_kinds': ['same_mutex', 'same_feature', 'same_scope', 'same_req_id'],
|
|
'semantic_top_k': 10,
|
|
'bm25_threshold': -8.0,
|
|
'ack_required': True,
|
|
}
|
|
DEFAULT_VECTOR = {
|
|
'enabled': False,
|
|
'timeout_seconds': 8,
|
|
'fail_open': True,
|
|
'cosine_threshold': 0.82,
|
|
}
|
|
|
|
CATALOG_GATE: dict = dict(DEFAULT_GATE)
|
|
CATALOG_VECTOR: dict = dict(DEFAULT_VECTOR)
|
|
|
|
# Deny callback injected by agentctl (deny(code, message, **details)).
|
|
_deny: Optional[Callable[..., Any]] = None
|
|
|
|
|
|
def bind_deny(fn: Callable[..., Any]) -> None:
|
|
global _deny
|
|
_deny = fn
|
|
|
|
|
|
def _fail(code: str, message: str, **details: Any) -> None:
|
|
if _deny is None:
|
|
raise RuntimeError(f'{code}: {message} {details}')
|
|
_deny(code, message, **details)
|
|
|
|
|
|
def apply_policy(policy: dict) -> None:
|
|
"""Optional catalog_gate / catalog_vector keys; missing → defaults."""
|
|
global CATALOG_GATE, CATALOG_VECTOR
|
|
gate = policy.get('catalog_gate')
|
|
if gate is None:
|
|
CATALOG_GATE = dict(DEFAULT_GATE)
|
|
elif not isinstance(gate, dict):
|
|
_fail('POLICY_INVALID', 'catalog_gate must be an object')
|
|
else:
|
|
merged = dict(DEFAULT_GATE)
|
|
merged.update(gate)
|
|
if not isinstance(merged.get('hard_kinds'), list):
|
|
_fail('POLICY_INVALID', 'catalog_gate.hard_kinds must be a list')
|
|
if not isinstance(merged.get('semantic_top_k'), int) or merged['semantic_top_k'] <= 0:
|
|
_fail('POLICY_INVALID', 'catalog_gate.semantic_top_k must be a positive int')
|
|
if not isinstance(merged.get('bm25_threshold'), (int, float)):
|
|
_fail('POLICY_INVALID', 'catalog_gate.bm25_threshold must be a number')
|
|
if not isinstance(merged.get('ack_required'), bool):
|
|
_fail('POLICY_INVALID', 'catalog_gate.ack_required must be a bool')
|
|
CATALOG_GATE = merged
|
|
vec = policy.get('catalog_vector')
|
|
if vec is None:
|
|
CATALOG_VECTOR = dict(DEFAULT_VECTOR)
|
|
elif not isinstance(vec, dict):
|
|
_fail('POLICY_INVALID', 'catalog_vector must be an object')
|
|
else:
|
|
merged = dict(DEFAULT_VECTOR)
|
|
merged.update(vec)
|
|
if not isinstance(merged.get('enabled'), bool):
|
|
_fail('POLICY_INVALID', 'catalog_vector.enabled must be a bool')
|
|
if not isinstance(merged.get('timeout_seconds'), int) or merged['timeout_seconds'] <= 0:
|
|
_fail('POLICY_INVALID', 'catalog_vector.timeout_seconds must be a positive int')
|
|
if not isinstance(merged.get('fail_open'), bool):
|
|
_fail('POLICY_INVALID', 'catalog_vector.fail_open must be a bool')
|
|
CATALOG_VECTOR = merged
|
|
|
|
|
|
def ensure_schema(db: sqlite3.Connection) -> None:
|
|
db.executescript(SCHEMA_SQL)
|
|
|
|
|
|
def _now() -> float:
|
|
return time.time()
|
|
|
|
|
|
def split_cjk(text: str) -> str:
|
|
"""Insert spaces around CJK chars so unicode61 FTS can phrase-match digrams."""
|
|
if not text:
|
|
return ''
|
|
spaced = re.sub(r'([\u3400-\u9fff\uf900-\ufaff])', r' \1 ', text)
|
|
return ' '.join(spaced.split())
|
|
|
|
|
|
def fts_phrase(text: str) -> str:
|
|
"""Build a double-quoted FTS5 phrase from CJK-split tokens."""
|
|
tokens = split_cjk(text).split()
|
|
if not tokens:
|
|
return '""'
|
|
# Escape embedded double quotes for FTS5.
|
|
safe = [t.replace('"', '""') for t in tokens]
|
|
return '"' + ' '.join(safe) + '"'
|
|
|
|
|
|
def validate_req_id(req_id: str) -> str:
|
|
if not REQ_ID_RE.fullmatch(req_id):
|
|
_fail('INVALID_REQUEST', 'Requirement id must match R-<slug>', id=req_id)
|
|
return req_id
|
|
|
|
|
|
def requirement_add(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
req_id: str,
|
|
title: str,
|
|
statement: str,
|
|
owner: str,
|
|
feature: Optional[str] = None,
|
|
source_task: Optional[str] = None,
|
|
source_doc: Optional[str] = None,
|
|
state: str = 'proposed',
|
|
supersedes: Optional[str] = None,
|
|
conversation_ref: Optional[str] = None,
|
|
) -> dict:
|
|
validate_req_id(req_id)
|
|
if state not in REQ_STATES:
|
|
_fail('INVALID_STATE', 'Unknown requirement state', state=state)
|
|
if not title.strip() or not statement.strip() or not owner.strip():
|
|
_fail('INVALID_REQUEST', 'title, statement, and owner are required')
|
|
existing = db.execute('SELECT id, source_task FROM catalog_requirements WHERE id=?', (req_id,)).fetchone()
|
|
if existing:
|
|
if source_task and existing['source_task'] and existing['source_task'] != source_task:
|
|
_fail(
|
|
'REQUIREMENT_EXISTS',
|
|
'Requirement id already owned by another task',
|
|
id=req_id,
|
|
holder=existing['source_task'],
|
|
)
|
|
_fail('REQUIREMENT_EXISTS', 'Requirement id already exists', id=req_id)
|
|
now = _now()
|
|
db.execute(
|
|
'INSERT INTO catalog_requirements VALUES(?,?,?,?,?,?,?,?,?,?,?,?)',
|
|
(
|
|
req_id, title.strip(), statement.strip(), feature, source_task, source_doc,
|
|
owner.strip(), state, supersedes, conversation_ref, now, now,
|
|
),
|
|
)
|
|
_index_requirement(db, req_id, title.strip() + '\n' + statement.strip())
|
|
return {'status': 'added', 'id': req_id}
|
|
|
|
|
|
def _index_requirement(db: sqlite3.Connection, req_id: str, body: str) -> None:
|
|
db.execute('DELETE FROM catalog_req_text WHERE req_id=?', (req_id,))
|
|
db.execute(
|
|
'INSERT INTO catalog_req_text(req_id, body, seg) VALUES(?,?,?)',
|
|
(req_id, body, split_cjk(body)),
|
|
)
|
|
|
|
|
|
def requirement_list(db: sqlite3.Connection, *, state: Optional[str] = None, limit: int = 100) -> list[dict]:
|
|
if state:
|
|
rows = db.execute(
|
|
'SELECT * FROM catalog_requirements WHERE state=? ORDER BY updated DESC LIMIT ?',
|
|
(state, limit),
|
|
).fetchall()
|
|
else:
|
|
rows = db.execute(
|
|
'SELECT * FROM catalog_requirements ORDER BY updated DESC LIMIT ?', (limit,)
|
|
).fetchall()
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
def requirement_show(db: sqlite3.Connection, req_id: str) -> dict:
|
|
row = db.execute('SELECT * FROM catalog_requirements WHERE id=?', (req_id,)).fetchone()
|
|
if not row:
|
|
_fail('UNKNOWN_REQUIREMENT', 'Requirement not found', id=req_id)
|
|
return dict(row)
|
|
|
|
|
|
def feature_upsert(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
feature_id: str,
|
|
title: str,
|
|
scope: Optional[str] = None,
|
|
mutex: Optional[str] = None,
|
|
matrix_status: Optional[str] = None,
|
|
) -> dict:
|
|
now = _now()
|
|
db.execute(
|
|
'''INSERT INTO catalog_features(id, title, scope, mutex, matrix_status, updated)
|
|
VALUES(?,?,?,?,?,?)
|
|
ON CONFLICT(id) DO UPDATE SET
|
|
title=excluded.title,
|
|
scope=COALESCE(excluded.scope, catalog_features.scope),
|
|
mutex=COALESCE(excluded.mutex, catalog_features.mutex),
|
|
matrix_status=COALESCE(excluded.matrix_status, catalog_features.matrix_status),
|
|
updated=excluded.updated''',
|
|
(feature_id, title, scope, mutex, matrix_status, now),
|
|
)
|
|
return {'status': 'upserted', 'id': feature_id}
|
|
|
|
|
|
def search(db: sqlite3.Connection, query: str, *, top: int = 10) -> list[dict]:
|
|
tokens = split_cjk(query).split()
|
|
if not tokens:
|
|
return []
|
|
# Soft recall: AND of char tokens (all must appear). Phrase is too strict for
|
|
# near-paraphrase overlap ("检测与治理" vs "检测治理"). Escape FTS keywords.
|
|
safe = []
|
|
for t in tokens:
|
|
t = t.replace('"', '""')
|
|
if t.upper() in {'AND', 'OR', 'NOT', 'NEAR'}:
|
|
t = '"' + t + '"'
|
|
else:
|
|
t = '"' + t + '"' # quote each token to avoid column/syntax issues
|
|
safe.append(t)
|
|
# Quoting each token and joining with spaces → AND of tokens in FTS5.
|
|
match_expr = ' '.join(safe)
|
|
try:
|
|
rows = db.execute(
|
|
'''SELECT r.id, r.title, r.statement, r.feature, r.source_task, r.state,
|
|
bm25(catalog_req_text) AS score
|
|
FROM catalog_req_text
|
|
JOIN catalog_requirements r ON r.id = catalog_req_text.req_id
|
|
WHERE catalog_req_text MATCH ?
|
|
ORDER BY score
|
|
LIMIT ?''',
|
|
(match_expr, top),
|
|
).fetchall()
|
|
except sqlite3.OperationalError:
|
|
return []
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
def bind_task(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
task: str,
|
|
requirement: Optional[str] = None,
|
|
intent: Optional[str] = None,
|
|
mutex: Optional[str] = None,
|
|
feature: Optional[str] = None,
|
|
) -> dict:
|
|
db.execute(
|
|
'''INSERT INTO catalog_task_bindings(task, requirement, intent, mutex, feature, updated)
|
|
VALUES(?,?,?,?,?,?)
|
|
ON CONFLICT(task) DO UPDATE SET
|
|
requirement=COALESCE(excluded.requirement, catalog_task_bindings.requirement),
|
|
intent=COALESCE(excluded.intent, catalog_task_bindings.intent),
|
|
mutex=COALESCE(excluded.mutex, catalog_task_bindings.mutex),
|
|
feature=COALESCE(excluded.feature, catalog_task_bindings.feature),
|
|
updated=excluded.updated''',
|
|
(task, requirement, intent, mutex, feature, _now()),
|
|
)
|
|
return {'status': 'bound', 'task': task}
|
|
|
|
|
|
def _active_tasks(db: sqlite3.Connection) -> list[dict]:
|
|
try:
|
|
rows = db.execute("SELECT * FROM tasks WHERE state='active'").fetchall()
|
|
except sqlite3.Error:
|
|
return []
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
def _path_overlaps(scopes_a: list[str], scopes_b: list[str], overlaps_fn: Callable) -> bool:
|
|
for a in scopes_a:
|
|
for b in scopes_b:
|
|
if overlaps_fn('path:' + a, 'path:' + b):
|
|
return True
|
|
return False
|
|
|
|
|
|
def probe_overlap(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
task: str,
|
|
statement: str,
|
|
requirement: Optional[str] = None,
|
|
feature: Optional[str] = None,
|
|
mutex: Optional[str] = None,
|
|
scopes: Optional[list[str]] = None,
|
|
top: Optional[int] = None,
|
|
overlaps_fn: Optional[Callable] = None,
|
|
) -> dict:
|
|
"""Run L1 hard + L2 soft (+ optional L3) overlap detection; persist soft/hard rows."""
|
|
top = top or int(CATALOG_GATE['semantic_top_k'])
|
|
hard_kinds = set(CATALOG_GATE.get('hard_kinds') or DEFAULT_GATE['hard_kinds'])
|
|
hard: list[dict] = []
|
|
soft: list[dict] = []
|
|
now = _now()
|
|
|
|
# L1: same_req_id
|
|
if requirement and 'same_req_id' in hard_kinds:
|
|
row = db.execute('SELECT * FROM catalog_requirements WHERE id=?', (requirement,)).fetchone()
|
|
if row and row['source_task'] and row['source_task'] != task:
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=requirement,
|
|
target_task=row['source_task'], target_feature=row['feature'],
|
|
level='hard', kind='same_req_id', score=1.0,
|
|
evidence=f'Requirement {requirement} owned by {row["source_task"]}',
|
|
created=now,
|
|
)
|
|
hard.append(hit)
|
|
|
|
# L1: same_feature — another active requirement on same feature (not superseded)
|
|
if feature and 'same_feature' in hard_kinds:
|
|
rows = db.execute(
|
|
'''SELECT * FROM catalog_requirements
|
|
WHERE feature=? AND state NOT IN ('superseded','rejected','delivered')
|
|
AND (source_task IS NULL OR source_task != ?)''',
|
|
(feature, task),
|
|
).fetchall()
|
|
for row in rows:
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=row['id'],
|
|
target_task=row['source_task'], target_feature=feature,
|
|
level='hard', kind='same_feature', score=1.0,
|
|
evidence=f'Feature {feature} already claimed by requirement {row["id"]}',
|
|
created=now,
|
|
)
|
|
hard.append(hit)
|
|
|
|
# L1: same_mutex via bindings + CURRENT-style mutex on active tasks
|
|
if mutex and 'same_mutex' in hard_kinds:
|
|
rows = db.execute(
|
|
'''SELECT * FROM catalog_task_bindings
|
|
WHERE mutex=? AND task != ?''',
|
|
(mutex, task),
|
|
).fetchall()
|
|
active_ids = {t['id'] for t in _active_tasks(db)}
|
|
for row in rows:
|
|
if row['task'] not in active_ids and row['task'] not in active_ids:
|
|
# Still hard if binding exists and peer task is active
|
|
pass
|
|
if row['task'] in active_ids:
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=row['requirement'],
|
|
target_task=row['task'], target_feature=row['feature'],
|
|
level='hard', kind='same_mutex', score=1.0,
|
|
evidence=f'Mutex {mutex} held by active task {row["task"]}',
|
|
created=now,
|
|
)
|
|
hard.append(hit)
|
|
|
|
# L1: same_scope — path overlaps with other active task scopes
|
|
if scopes and overlaps_fn and 'same_scope' in hard_kinds:
|
|
for peer in _active_tasks(db):
|
|
if peer['id'] == task:
|
|
continue
|
|
peer_scopes = json.loads(peer['scopes']) if isinstance(peer['scopes'], str) else peer['scopes']
|
|
if _path_overlaps(scopes, peer_scopes, overlaps_fn):
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=None,
|
|
target_task=peer['id'], target_feature=None,
|
|
level='hard', kind='same_scope', score=1.0,
|
|
evidence=f'Scope overlaps active task {peer["id"]} scopes={peer_scopes}',
|
|
created=now,
|
|
)
|
|
hard.append(hit)
|
|
|
|
# L2: semantic FTS5
|
|
threshold = float(CATALOG_GATE.get('bm25_threshold', DEFAULT_GATE['bm25_threshold']))
|
|
hits = search(db, statement, top=top)
|
|
for row in hits:
|
|
if requirement and row['id'] == requirement:
|
|
continue
|
|
if row.get('source_task') == task:
|
|
continue
|
|
score = float(row.get('score') or 0.0)
|
|
# bm25: lower (more negative) is better; accept scores <= threshold
|
|
if score > threshold and score >= 0:
|
|
# Positive bm25 is weak; still include if phrase matched and score is finite
|
|
pass
|
|
# Always record soft semantic hits that MATCH returned (already ranked)
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=row['id'],
|
|
target_task=row.get('source_task'), target_feature=row.get('feature'),
|
|
level='soft', kind='semantic', score=score,
|
|
evidence=f'FTS5 hit title={row["title"]!r} bm25={score:.4f}',
|
|
created=now,
|
|
)
|
|
soft.append(hit)
|
|
|
|
# L3: optional vector enhancement (fail-open)
|
|
vector_status = 'disabled'
|
|
if CATALOG_VECTOR.get('enabled'):
|
|
vector_status, vec_soft = _vector_soft_hits(
|
|
db, task=task, statement=statement, requirement=requirement, top=top, created=now,
|
|
)
|
|
# Merge unique soft hits
|
|
seen = {h['id'] for h in soft}
|
|
for h in vec_soft:
|
|
if h['id'] not in seen:
|
|
soft.append(h)
|
|
seen.add(h['id'])
|
|
|
|
# Soft hits need ack — filter already-acked
|
|
pending_soft = [h for h in soft if not _decision_exists(db, task, h['id'])]
|
|
status = 'conflict' if hard else ('ack_required' if pending_soft else 'none')
|
|
return {
|
|
'status': status,
|
|
'hard': hard,
|
|
'soft': soft,
|
|
'pending_soft': pending_soft,
|
|
'vector_layer': vector_status,
|
|
'task': task,
|
|
'requirement': requirement,
|
|
}
|
|
|
|
|
|
def _persist_overlap(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
probe_task: str,
|
|
probe_req: Optional[str],
|
|
target_req: Optional[str],
|
|
target_task: Optional[str],
|
|
target_feature: Optional[str],
|
|
level: str,
|
|
kind: str,
|
|
score: Optional[float],
|
|
evidence: str,
|
|
created: float,
|
|
) -> dict:
|
|
oid = str(uuid.uuid4())
|
|
db.execute(
|
|
'INSERT INTO catalog_overlaps VALUES(?,?,?,?,?,?,?,?,?,?,?)',
|
|
(
|
|
oid, probe_task, probe_req, target_req, target_task, target_feature,
|
|
level, kind, score, evidence, created,
|
|
),
|
|
)
|
|
return {
|
|
'id': oid, 'probe_task': probe_task, 'probe_req': probe_req,
|
|
'target_req': target_req, 'target_task': target_task,
|
|
'target_feature': target_feature, 'level': level, 'kind': kind,
|
|
'score': score, 'evidence': evidence, 'created': created,
|
|
}
|
|
|
|
|
|
def _decision_exists(db: sqlite3.Connection, task: str, overlap_id: str) -> bool:
|
|
row = db.execute(
|
|
'SELECT id FROM catalog_decisions WHERE task=? AND overlap_id=?',
|
|
(task, overlap_id),
|
|
).fetchone()
|
|
return bool(row)
|
|
|
|
|
|
def overlap_ack(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
task: str,
|
|
overlap_id: str,
|
|
decision: str,
|
|
decided_by: str,
|
|
reason: str,
|
|
authorization_ref: Optional[str] = None,
|
|
) -> dict:
|
|
if decision not in DECISIONS:
|
|
_fail('INVALID_REQUEST', 'Unknown decision', decision=decision)
|
|
if not reason.strip():
|
|
_fail('INVALID_REQUEST', 'reason is required for overlap ack')
|
|
ov = db.execute('SELECT * FROM catalog_overlaps WHERE id=?', (overlap_id,)).fetchone()
|
|
if not ov:
|
|
_fail('UNKNOWN_OVERLAP', 'Overlap id not found', id=overlap_id)
|
|
if ov['probe_task'] != task:
|
|
_fail('PATH_OUT_OF_SCOPE', 'Overlap belongs to another task', holder=ov['probe_task'])
|
|
if ov['level'] == 'hard' and decision == 'proceed':
|
|
_fail(
|
|
'REQUIREMENT_CONFLICT',
|
|
'Hard overlaps cannot be acknowledged as proceed; resolve mutex/feature/scope first',
|
|
overlap_id=overlap_id,
|
|
kind=ov['kind'],
|
|
holder=ov['target_task'],
|
|
)
|
|
if _decision_exists(db, task, overlap_id):
|
|
return {'status': 'replayed', 'overlap_id': overlap_id}
|
|
did = str(uuid.uuid4())
|
|
db.execute(
|
|
'INSERT INTO catalog_decisions VALUES(?,?,?,?,?,?,?,?)',
|
|
(did, overlap_id, task, decision, decided_by, reason.strip(), authorization_ref, _now()),
|
|
)
|
|
return {'status': 'acked', 'id': did, 'overlap_id': overlap_id, 'decision': decision}
|
|
|
|
|
|
def gate_task_register(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
task: str,
|
|
scopes: list[str],
|
|
requirement: Optional[str] = None,
|
|
intent: Optional[str] = None,
|
|
feature: Optional[str] = None,
|
|
mutex: Optional[str] = None,
|
|
overlaps_fn: Optional[Callable] = None,
|
|
statement: Optional[str] = None,
|
|
) -> dict:
|
|
"""Probe overlaps for task register. Caller must commit then enforce deny codes.
|
|
|
|
Returns probe result. Does not raise on soft/hard — register path commits first
|
|
so soft overlap rows survive OVERLAP_ACK_REQUIRED.
|
|
"""
|
|
stmt = statement or intent or ''
|
|
if not (requirement or stmt or feature or mutex):
|
|
return {'status': 'skipped', 'reason': 'no requirement/intent/feature/mutex provided'}
|
|
result = probe_overlap(
|
|
db, task=task, statement=stmt or (requirement or feature or mutex or task),
|
|
requirement=requirement, feature=feature, mutex=mutex,
|
|
scopes=scopes, overlaps_fn=overlaps_fn,
|
|
)
|
|
bind_task(db, task=task, requirement=requirement, intent=intent, mutex=mutex, feature=feature)
|
|
return result
|
|
|
|
|
|
def enforce_gate_result(result: dict) -> None:
|
|
"""Raise deny codes after the probe transaction has committed."""
|
|
if result.get('status') == 'skipped':
|
|
return
|
|
hard = result.get('hard') or []
|
|
if hard:
|
|
first = hard[0]
|
|
_fail(
|
|
'REQUIREMENT_CONFLICT',
|
|
first['evidence'],
|
|
kind=first['kind'],
|
|
holder=first.get('target_task'),
|
|
overlap_id=first['id'],
|
|
target_req=first.get('target_req'),
|
|
)
|
|
if CATALOG_GATE.get('ack_required', True) and result.get('pending_soft'):
|
|
_fail(
|
|
'OVERLAP_ACK_REQUIRED',
|
|
'Semantic overlaps require catalog overlap ack before coding',
|
|
pending=[h['id'] for h in result['pending_soft']],
|
|
count=len(result['pending_soft']),
|
|
)
|
|
|
|
|
|
def pending_acks_for_task(db: sqlite3.Connection, task: str) -> list[dict]:
|
|
rows = db.execute(
|
|
'''SELECT o.* FROM catalog_overlaps o
|
|
WHERE o.probe_task=? AND o.level='soft'
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM catalog_decisions d
|
|
WHERE d.overlap_id=o.id AND d.task=o.probe_task
|
|
)
|
|
ORDER BY o.created''',
|
|
(task,),
|
|
).fetchall()
|
|
return [dict(r) for r in rows]
|
|
|
|
|
|
def assert_acks_clear(db: sqlite3.Connection, task: str) -> None:
|
|
"""Block first write (with) when soft overlaps are unacked."""
|
|
if not CATALOG_GATE.get('ack_required', True):
|
|
return
|
|
pending = pending_acks_for_task(db, task)
|
|
if pending:
|
|
_fail(
|
|
'OVERLAP_ACK_REQUIRED',
|
|
'Unresolved soft overlaps; run catalog overlap ack',
|
|
pending=[p['id'] for p in pending],
|
|
count=len(pending),
|
|
)
|
|
|
|
|
|
def catalog_status(db: sqlite3.Connection) -> dict:
|
|
def count(table: str) -> int:
|
|
try:
|
|
return db.execute(f'SELECT COUNT(*) FROM {table}').fetchone()[0]
|
|
except sqlite3.Error:
|
|
return 0
|
|
|
|
return {
|
|
'status': 'ok',
|
|
'requirements': count('catalog_requirements'),
|
|
'features': count('catalog_features'),
|
|
'overlaps': count('catalog_overlaps'),
|
|
'decisions': count('catalog_decisions'),
|
|
'issues': count('catalog_issues'),
|
|
'tests': count('catalog_tests'),
|
|
'releases': count('catalog_releases'),
|
|
'embeddings': count('catalog_embeddings'),
|
|
'gate': dict(CATALOG_GATE),
|
|
'vector': {
|
|
'enabled': CATALOG_VECTOR.get('enabled'),
|
|
'fail_open': CATALOG_VECTOR.get('fail_open'),
|
|
},
|
|
}
|
|
|
|
|
|
def catalog_drift(db: sqlite3.Connection, *, matrix_states: dict[str, str]) -> dict:
|
|
"""Compare catalog_features.matrix_status to live MATRIX states; never writes docs."""
|
|
regressions = []
|
|
for fid, status in matrix_states.items():
|
|
row = db.execute('SELECT matrix_status FROM catalog_features WHERE id=?', (fid,)).fetchone()
|
|
if row and row['matrix_status'] == 'done' and status != 'done':
|
|
regressions.append({'record_id': fid, 'catalog': row['matrix_status'], 'matrix': status})
|
|
feature_upsert(db, feature_id=fid, title=fid, matrix_status=status)
|
|
open_caps = [k for k, v in matrix_states.items() if v not in ('done', 'removed')]
|
|
return {
|
|
'status': 'ok',
|
|
'record_count': len(matrix_states),
|
|
'regressions': regressions,
|
|
'open_capabilities': open_caps,
|
|
'note': 'catalog features refreshed from MATRIX; never auto-promotes done',
|
|
}
|
|
|
|
|
|
# --- Vector layer (fail-open) -------------------------------------------------
|
|
|
|
def _pack_vec(values: list[float]) -> bytes:
|
|
return struct.pack(f'<{len(values)}f', *values)
|
|
|
|
|
|
def _unpack_vec(blob: bytes) -> list[float]:
|
|
n = len(blob) // 4
|
|
return list(struct.unpack(f'<{n}f', blob))
|
|
|
|
|
|
def _cosine(a: list[float], b: list[float]) -> float:
|
|
if not a or not b or len(a) != len(b):
|
|
return 0.0
|
|
dot = sum(x * y for x, y in zip(a, b))
|
|
na = math.sqrt(sum(x * x for x in a))
|
|
nb = math.sqrt(sum(y * y for y in b))
|
|
if na == 0 or nb == 0:
|
|
return 0.0
|
|
return dot / (na * nb)
|
|
|
|
|
|
def _embed_text(text: str) -> tuple[str, Optional[list[float]], str]:
|
|
"""Return (status, vector_or_None, model). Never raises to caller for network errors."""
|
|
url = os.environ.get('EMBEDDING_ENDPOINT_URL', '').strip()
|
|
model = os.environ.get('EMBEDDING_MODEL', 'bge-m3').strip() or 'bge-m3'
|
|
key = os.environ.get('EMBEDDING_API_KEY', '').strip()
|
|
if not CATALOG_VECTOR.get('enabled'):
|
|
return 'disabled', None, model
|
|
if not url or not key:
|
|
return 'degraded', None, model
|
|
timeout = int(CATALOG_VECTOR.get('timeout_seconds', 8))
|
|
payload = json.dumps({'model': model, 'input': text}).encode('utf-8')
|
|
req = urllib.request.Request(
|
|
url, data=payload, method='POST',
|
|
headers={'Content-Type': 'application/json', 'Authorization': f'Bearer {key}'},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
if resp.status != 200:
|
|
return 'degraded', None, model
|
|
body = json.loads(resp.read().decode('utf-8'))
|
|
data = body.get('data') or []
|
|
if not data or 'embedding' not in data[0]:
|
|
return 'degraded', None, model
|
|
return 'ok', list(data[0]['embedding']), model
|
|
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, KeyError, IndexError, TypeError):
|
|
return 'degraded', None, model
|
|
|
|
|
|
def store_embedding(db: sqlite3.Connection, req_id: str, text: str) -> dict:
|
|
status, vec, model = _embed_text(text)
|
|
if status != 'ok' or vec is None:
|
|
return {'status': status, 'req_id': req_id, 'model': model}
|
|
blob = _pack_vec(vec)
|
|
sha = hashlib.sha256(text.encode('utf-8')).hexdigest()
|
|
db.execute(
|
|
'''INSERT INTO catalog_embeddings(req_id, model, dim, vec, source_sha256, status, created)
|
|
VALUES(?,?,?,?,?,?,?)
|
|
ON CONFLICT(req_id) DO UPDATE SET
|
|
model=excluded.model, dim=excluded.dim, vec=excluded.vec,
|
|
source_sha256=excluded.source_sha256, status=excluded.status,
|
|
created=excluded.created''',
|
|
(req_id, model, len(vec), blob, sha, 'ok', _now()),
|
|
)
|
|
return {'status': 'ok', 'req_id': req_id, 'dim': len(vec), 'model': model}
|
|
|
|
|
|
def _vector_soft_hits(
|
|
db: sqlite3.Connection,
|
|
*,
|
|
task: str,
|
|
statement: str,
|
|
requirement: Optional[str],
|
|
top: int,
|
|
created: float,
|
|
) -> tuple[str, list[dict]]:
|
|
status, probe, model = _embed_text(statement)
|
|
if status != 'ok' or probe is None:
|
|
return status, []
|
|
threshold = float(CATALOG_VECTOR.get('cosine_threshold', 0.82))
|
|
rows = db.execute(
|
|
"SELECT req_id, vec FROM catalog_embeddings WHERE status='ok'"
|
|
).fetchall()
|
|
scored: list[tuple[float, str]] = []
|
|
for row in rows:
|
|
if requirement and row['req_id'] == requirement:
|
|
continue
|
|
other = _unpack_vec(row['vec'])
|
|
sim = _cosine(probe, other)
|
|
if sim >= threshold:
|
|
scored.append((sim, row['req_id']))
|
|
scored.sort(reverse=True)
|
|
soft = []
|
|
for sim, rid in scored[:top]:
|
|
req = db.execute('SELECT * FROM catalog_requirements WHERE id=?', (rid,)).fetchone()
|
|
if not req:
|
|
continue
|
|
if req['source_task'] == task:
|
|
continue
|
|
hit = _persist_overlap(
|
|
db, probe_task=task, probe_req=requirement, target_req=rid,
|
|
target_task=req['source_task'], target_feature=req['feature'],
|
|
level='soft', kind='semantic', score=sim,
|
|
evidence=f'vector cosine={sim:.4f} model={model}',
|
|
created=created,
|
|
)
|
|
soft.append(hit)
|
|
return 'ok', soft
|