#!/usr/bin/env python3 """One-shot backfill of catalog tables from Markdown / SQLite sources. Read-only against .doc and tasks table; writes only catalog_* tables. Does not touch product PostgreSQL. Safe to re-run (upserts / replaces by PK). """ from __future__ import annotations import argparse import json import re import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.dont_write_bytecode = True import agentctl # noqa: E402 import agentctl_catalog as catalog # noqa: E402 def table_rows(content: str) -> list[list[str]]: rows = [] for line in content.splitlines(): if not line.lstrip().startswith('|'): continue cells = [c.strip() for c in re.split(r'(? int: path = repo.main / '.doc/status/FEATURE-MATRIX.md' if not path.is_file(): return 0 content = path.read_text(encoding='utf-8') states = agentctl.matrix_states(content) # Also capture mutex/title from table when possible. n = 0 header = None for cells in table_rows(content): if header is None: if cells and cells[0].casefold() in ('record_id', 'id'): header = [c.casefold() for c in cells] continue if len(cells) != len(header): continue rid = cells[0] title = cells[1] if len(cells) > 1 else rid mutex = None if 'mutex' in header: mutex = cells[header.index('mutex')].strip('`') status = states.get(rid) catalog.feature_upsert(db, feature_id=rid, title=title, mutex=mutex, matrix_status=status) n += 1 return n def backfill_issues(db, repo: agentctl.Repo) -> int: path = repo.main / '.doc/audit/ISSUES.md' if not path.is_file(): return 0 content = path.read_text(encoding='utf-8') n = 0 header = None now = time.time() for cells in table_rows(content): if header is None: if cells and 'issue_id' in cells[0].casefold(): header = [c.casefold() for c in cells] continue if not cells or not cells[0].startswith('ISSUE-'): continue issue_id = cells[0] module = cells[1] if len(cells) > 1 else '' severity = cells[3] if len(cells) > 3 else '' state = cells[4] if len(cells) > 4 else '' owner = cells[5] if len(cells) > 5 else '' main_record = cells[7] if len(cells) > 7 else '' lesson = cells[8] if len(cells) > 8 else '' db.execute( '''INSERT INTO catalog_issues( issue_id, severity, issue_state, owner, module, keywords, main_record, lesson_ids, requirements, tasks, recurrence_count, evidence_basis, created, updated) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(issue_id) DO UPDATE SET severity=excluded.severity, issue_state=excluded.issue_state, owner=excluded.owner, module=excluded.module, main_record=excluded.main_record, lesson_ids=excluded.lesson_ids, updated=excluded.updated''', ( issue_id, severity, state, owner, module, module, main_record, lesson, None, None, 0, 'historical', now, now, ), ) n += 1 return n def guess_verdict(text: str) -> str: upper = text.upper() if 'NOT REQUIRED' in upper or '不记成产品测试 PASS' in text: return 'NOT_REQUIRED' # Prefer explicit Evaluation / Verdict sections. if re.search(r'\bFAIL\b', upper) and 'PASS' not in upper.split('FAIL')[0][-80:]: if re.search(r'判定[：:].{0,40}FAIL|Verdict[：:].{0,20}FAIL|评价判定.{0,40}FAIL', text, re.I): return 'FAIL' if re.search(r'判定[：:].{0,40}PASS|Verdict[：:].{0,20}PASS|评价判定.{0,40}PASS', text, re.I): return 'PASS' if '**PASS**' in text or '| PASS |' in text: return 'PASS' return 'unknown' def backfill_tests(db, repo: agentctl.Repo) -> int: root = repo.main / '.doc/test-reports' if not root.is_dir(): return 0 n = 0 now = time.time() for path in sorted(root.glob('*.md')): if path.name in ('INDEX.md', 'README.md'): continue text = path.read_text(encoding='utf-8', errors='replace') rel = path.relative_to(repo.main).as_posix() slug = path.stem verdict = guess_verdict(text) real = 1 if re.search(r'真实\s*PI|pi-agent|MiniMax', text, re.I) else 0 level = 'doc_only' if verdict == 'NOT_REQUIRED' else 'e2e' db.execute( '''INSERT INTO catalog_tests( id, report_path, task, requirements, scope, verdict, level, real_agent, schema_version, env, sha, created) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET report_path=excluded.report_path, verdict=excluded.verdict, real_agent=excluded.real_agent''', (slug, rel, None, None, None, verdict, level, real, None, None, None, now), ) n += 1 return n def backfill_tasks_as_features(db) -> int: rows = db.execute('SELECT id, scopes, state FROM tasks').fetchall() n = 0 for row in rows: # Light projection: task id as feature hint only when looks like feature mutex. scopes = json.loads(row['scopes']) if isinstance(row['scopes'], str) else row['scopes'] mutex = scopes[0] if scopes else row['id'] catalog.feature_upsert( db, feature_id=f'task:{row["id"]}', title=row['id'], scope=','.join(scopes), mutex=mutex, matrix_status=None, ) n += 1 return n def backfill_deployments(db) -> int: try: rows = db.execute('SELECT * FROM deployments').fetchall() except Exception: return 0 n = 0 now = time.time() for row in rows: db.execute( '''INSERT INTO catalog_releases( id, env, branch, commit_sha, tag, deploy_seq, schema_version, image_digest, source_task, phase, receipt, created) VALUES(?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET phase=excluded.phase, receipt=excluded.receipt''', ( row['id'], row['env'], None, row['expected_commit'], None, None, None, None, row['task'], row['phase'], row['receipt'], row['created'] or now, ), ) n += 1 return n def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--repo', default='.') args = parser.parse_args() repo = agentctl.Repo(args.repo) agentctl.require_policy(repo) catalog.bind_deny(agentctl.deny) with repo.db(True) as db: counts = { 'features_matrix': backfill_matrix(db, repo), 'issues': backfill_issues(db, repo), 'tests': backfill_tests(db, repo), 'task_features': backfill_tasks_as_features(db), 'releases': backfill_deployments(db), } status = catalog.catalog_status(db) agentctl.emit({'status': 'ok', 'backfill': counts, 'catalog': status}) return 0 if __name__ == '__main__': try: sys.exit(main()) except agentctl.Denied as exc: agentctl.emit({'status': 'denied', 'code': exc.code, 'message': exc.message, **exc.details}, sys.stderr) sys.exit(1)