124 lines
5.1 KiB
Python
124 lines
5.1 KiB
Python
"""Exercise check-evidence.py on card summaries and legacy full receipt blocks."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
|
|
|
|
|
class CheckEvidenceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.base = Path(self.temp.name)
|
|
self.repo = self.base / "repo"
|
|
self.repo.mkdir()
|
|
self.git("init", "-b", "main")
|
|
self.git("config", "user.name", "Evidence Test")
|
|
self.git("config", "user.email", "evidence@example.invalid")
|
|
(self.repo / "tracked").write_text("initial\n")
|
|
self.git("add", "tracked")
|
|
self.git("commit", "-qm", "Initial")
|
|
self.git("update-ref", "refs/remotes/origin/main", self.git("rev-parse", "HEAD"))
|
|
self.card = self.repo / ".task/main.md"
|
|
self.card.parent.mkdir()
|
|
self.card.write_text("# Test task\n")
|
|
self.env = os.environ.copy()
|
|
for key in ("CMD", "SRC", "RECEIPT", "IMPORT_RECEIPT", "VALIDATE_RECEIPT", "TAIL"):
|
|
self.env.pop(key, None)
|
|
self.env["AGENT_EVIDENCE_DIR"] = str(self.base / "evidence")
|
|
|
|
def git(self, *args):
|
|
return subprocess.check_output(["git", *args], cwd=self.repo, text=True,
|
|
stderr=subprocess.DEVNULL).strip()
|
|
|
|
def record(self, command="printf 'checked\\n'"):
|
|
result = subprocess.run(["bash", str(SCRIPTS / "evidence.sh")], cwd=self.repo,
|
|
env=self.env | {"CMD": command}, capture_output=True, text=True)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
text = self.card.read_text()
|
|
block = text.split("```json\n", 1)[1].split("\n```", 1)[0]
|
|
return json.loads(block)
|
|
|
|
def check(self):
|
|
result = subprocess.run(["python3", str(SCRIPTS / "check-evidence.py"), str(self.card)],
|
|
cwd=self.repo, capture_output=True, text=True)
|
|
return result.stdout + result.stderr
|
|
|
|
def replace_block(self, value):
|
|
prefix, rest = self.card.read_text().split("```json\n", 1)
|
|
suffix = rest.split("\n```", 1)[1]
|
|
self.card.write_text(prefix + "```json\n" + json.dumps(value, indent=2) + "\n```" + suffix)
|
|
|
|
def advance(self):
|
|
(self.repo / "tracked").write_text("changed\n")
|
|
self.git("commit", "-qam", "Change code")
|
|
|
|
def test_fresh_summary_passes(self):
|
|
summary = self.record()
|
|
self.assertNotIn("submodules", self.card.read_text())
|
|
self.assertEqual(self.check(), "")
|
|
self.assertTrue(Path(summary["receipt_path"]).is_file())
|
|
|
|
def test_summary_detects_stale_head_through_full_receipt(self):
|
|
self.record()
|
|
self.advance()
|
|
output = self.check()
|
|
self.assertIn("recorded HEAD predates non-card changes", output)
|
|
self.assertNotIn("unverified", output)
|
|
|
|
def test_tampered_summary_warns(self):
|
|
summary = self.record()
|
|
for field, value in (("command", "make test"), ("exit_code", 1), ("output_tail", ["forged"]),
|
|
("before_head", "0" * 40)):
|
|
with self.subTest(field=field):
|
|
self.replace_block(dict(summary, **{field: value}))
|
|
self.assertIn("unverified: summary does not match full receipt", self.check())
|
|
|
|
def test_malformed_summary_fields_warn(self):
|
|
summary = self.record()
|
|
for field, value in (("receipt_sha256", "zz"), ("exit_code", True), ("output_tail", "text"),
|
|
("receipt_path", "relative.json")):
|
|
with self.subTest(field=field):
|
|
self.replace_block(dict(summary, **{field: value}))
|
|
self.assertIn("unverified", self.check())
|
|
missing = dict(summary)
|
|
del missing["command"]
|
|
self.replace_block(missing)
|
|
self.assertIn("unverified: invalid summary fields", self.check())
|
|
|
|
def test_tampered_full_receipt_warns(self):
|
|
summary = self.record()
|
|
path = Path(summary["receipt_path"])
|
|
full = json.loads(path.read_text())
|
|
full["output"] = "changed\n"
|
|
path.write_text(json.dumps(full))
|
|
self.assertIn("unverified: receipt checksum mismatch", self.check())
|
|
|
|
def test_missing_full_receipt_warns_and_still_checks_summary_heads(self):
|
|
summary = self.record()
|
|
path = Path(summary["receipt_path"])
|
|
path.unlink()
|
|
output = self.check()
|
|
self.assertIn(f"unverified: full receipt not available locally ({path})", output)
|
|
self.assertNotIn("predates", output)
|
|
self.advance()
|
|
self.assertIn("recorded HEAD predates non-card changes", self.check())
|
|
|
|
def test_legacy_full_receipt_block_is_still_accepted(self):
|
|
summary = self.record()
|
|
full = json.loads(Path(summary["receipt_path"]).read_text())
|
|
self.replace_block(full)
|
|
self.assertEqual(self.check(), "")
|
|
full["output"] = "changed\n"
|
|
self.replace_block(full)
|
|
self.assertIn("unverified: receipt checksum mismatch", self.check())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|