weichen
28ed7005dd
Extend agentctl SQLite with catalog tables, CJK FTS5 recall, hard/soft overlap gates, fail-open embeddings, backfill, and MCP/skills hooks so cross-conversation designs cannot silently conflict.
215 lines
7.7 KiB
Python
215 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
||
"""One-shot backfill of catalog tables from Markdown / SQLite sources.
|
||
|
||
Read-only against .doc and tasks table; writes only catalog_* tables.
|
||
Does not touch product PostgreSQL. Safe to re-run (upserts / replaces by PK).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
sys.dont_write_bytecode = True
|
||
import agentctl # noqa: E402
|
||
import agentctl_catalog as catalog # noqa: E402
|
||
|
||
|
||
def table_rows(content: str) -> list[list[str]]:
|
||
rows = []
|
||
for line in content.splitlines():
|
||
if not line.lstrip().startswith('|'):
|
||
continue
|
||
cells = [c.strip() for c in re.split(r'(?<!\\)\|', line.strip().strip('|'))]
|
||
if cells and all(re.fullmatch(r':?-+:?', c.replace(' ', '') or '-') for c in cells):
|
||
continue
|
||
rows.append(cells)
|
||
return rows
|
||
|
||
|
||
def backfill_matrix(db, repo: agentctl.Repo) -> int:
|
||
path = repo.main / '.doc/status/FEATURE-MATRIX.md'
|
||
if not path.is_file():
|
||
return 0
|
||
content = path.read_text(encoding='utf-8')
|
||
states = agentctl.matrix_states(content)
|
||
# Also capture mutex/title from table when possible.
|
||
n = 0
|
||
header = None
|
||
for cells in table_rows(content):
|
||
if header is None:
|
||
if cells and cells[0].casefold() in ('record_id', 'id'):
|
||
header = [c.casefold() for c in cells]
|
||
continue
|
||
if len(cells) != len(header):
|
||
continue
|
||
rid = cells[0]
|
||
title = cells[1] if len(cells) > 1 else rid
|
||
mutex = None
|
||
if 'mutex' in header:
|
||
mutex = cells[header.index('mutex')].strip('`')
|
||
status = states.get(rid)
|
||
catalog.feature_upsert(db, feature_id=rid, title=title, mutex=mutex, matrix_status=status)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def backfill_issues(db, repo: agentctl.Repo) -> int:
|
||
path = repo.main / '.doc/audit/ISSUES.md'
|
||
if not path.is_file():
|
||
return 0
|
||
content = path.read_text(encoding='utf-8')
|
||
n = 0
|
||
header = None
|
||
now = time.time()
|
||
for cells in table_rows(content):
|
||
if header is None:
|
||
if cells and 'issue_id' in cells[0].casefold():
|
||
header = [c.casefold() for c in cells]
|
||
continue
|
||
if not cells or not cells[0].startswith('ISSUE-'):
|
||
continue
|
||
issue_id = cells[0]
|
||
module = cells[1] if len(cells) > 1 else ''
|
||
severity = cells[3] if len(cells) > 3 else ''
|
||
state = cells[4] if len(cells) > 4 else ''
|
||
owner = cells[5] if len(cells) > 5 else ''
|
||
main_record = cells[7] if len(cells) > 7 else ''
|
||
lesson = cells[8] if len(cells) > 8 else ''
|
||
db.execute(
|
||
'''INSERT INTO catalog_issues(
|
||
issue_id, severity, issue_state, owner, module, keywords,
|
||
main_record, lesson_ids, requirements, tasks, recurrence_count,
|
||
evidence_basis, created, updated)
|
||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||
ON CONFLICT(issue_id) DO UPDATE SET
|
||
severity=excluded.severity, issue_state=excluded.issue_state,
|
||
owner=excluded.owner, module=excluded.module,
|
||
main_record=excluded.main_record, lesson_ids=excluded.lesson_ids,
|
||
updated=excluded.updated''',
|
||
(
|
||
issue_id, severity, state, owner, module, module,
|
||
main_record, lesson, None, None, 0, 'historical', now, now,
|
||
),
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def guess_verdict(text: str) -> str:
|
||
upper = text.upper()
|
||
if 'NOT REQUIRED' in upper or '不记成产品测试 PASS' in text:
|
||
return 'NOT_REQUIRED'
|
||
# Prefer explicit Evaluation / Verdict sections.
|
||
if re.search(r'\bFAIL\b', upper) and 'PASS' not in upper.split('FAIL')[0][-80:]:
|
||
if re.search(r'判定[：:].{0,40}FAIL|Verdict[：:].{0,20}FAIL|评价判定.{0,40}FAIL', text, re.I):
|
||
return 'FAIL'
|
||
if re.search(r'判定[：:].{0,40}PASS|Verdict[：:].{0,20}PASS|评价判定.{0,40}PASS', text, re.I):
|
||
return 'PASS'
|
||
if '**PASS**' in text or '| PASS |' in text:
|
||
return 'PASS'
|
||
return 'unknown'
|
||
|
||
|
||
def backfill_tests(db, repo: agentctl.Repo) -> int:
|
||
root = repo.main / '.doc/test-reports'
|
||
if not root.is_dir():
|
||
return 0
|
||
n = 0
|
||
now = time.time()
|
||
for path in sorted(root.glob('*.md')):
|
||
if path.name in ('INDEX.md', 'README.md'):
|
||
continue
|
||
text = path.read_text(encoding='utf-8', errors='replace')
|
||
rel = path.relative_to(repo.main).as_posix()
|
||
slug = path.stem
|
||
verdict = guess_verdict(text)
|
||
real = 1 if re.search(r'真实\s*PI|pi-agent|MiniMax', text, re.I) else 0
|
||
level = 'doc_only' if verdict == 'NOT_REQUIRED' else 'e2e'
|
||
db.execute(
|
||
'''INSERT INTO catalog_tests(
|
||
id, report_path, task, requirements, scope, verdict, level,
|
||
real_agent, schema_version, env, sha, created)
|
||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
|
||
ON CONFLICT(id) DO UPDATE SET
|
||
report_path=excluded.report_path, verdict=excluded.verdict,
|
||
real_agent=excluded.real_agent''',
|
||
(slug, rel, None, None, None, verdict, level, real, None, None, None, now),
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def backfill_tasks_as_features(db) -> int:
|
||
rows = db.execute('SELECT id, scopes, state FROM tasks').fetchall()
|
||
n = 0
|
||
for row in rows:
|
||
# Light projection: task id as feature hint only when looks like feature mutex.
|
||
scopes = json.loads(row['scopes']) if isinstance(row['scopes'], str) else row['scopes']
|
||
mutex = scopes[0] if scopes else row['id']
|
||
catalog.feature_upsert(
|
||
db,
|
||
feature_id=f'task:{row["id"]}',
|
||
title=row['id'],
|
||
scope=','.join(scopes),
|
||
mutex=mutex,
|
||
matrix_status=None,
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def backfill_deployments(db) -> int:
|
||
try:
|
||
rows = db.execute('SELECT * FROM deployments').fetchall()
|
||
except Exception:
|
||
return 0
|
||
n = 0
|
||
now = time.time()
|
||
for row in rows:
|
||
db.execute(
|
||
'''INSERT INTO catalog_releases(
|
||
id, env, branch, commit_sha, tag, deploy_seq, schema_version,
|
||
image_digest, source_task, phase, receipt, created)
|
||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
|
||
ON CONFLICT(id) DO UPDATE SET phase=excluded.phase, receipt=excluded.receipt''',
|
||
(
|
||
row['id'], row['env'], None, row['expected_commit'], None, None, None,
|
||
None, row['task'], row['phase'], row['receipt'], row['created'] or now,
|
||
),
|
||
)
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--repo', default='.')
|
||
args = parser.parse_args()
|
||
repo = agentctl.Repo(args.repo)
|
||
agentctl.require_policy(repo)
|
||
catalog.bind_deny(agentctl.deny)
|
||
with repo.db(True) as db:
|
||
counts = {
|
||
'features_matrix': backfill_matrix(db, repo),
|
||
'issues': backfill_issues(db, repo),
|
||
'tests': backfill_tests(db, repo),
|
||
'task_features': backfill_tasks_as_features(db),
|
||
'releases': backfill_deployments(db),
|
||
}
|
||
status = catalog.catalog_status(db)
|
||
agentctl.emit({'status': 'ok', 'backfill': counts, 'catalog': status})
|
||
return 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
try:
|
||
sys.exit(main())
|
||
except agentctl.Denied as exc:
|
||
agentctl.emit({'status': 'denied', 'code': exc.code, 'message': exc.message, **exc.details}, sys.stderr)
|
||
sys.exit(1)
|