Files
Agent-System-Dev/scripts/evidence.py
T

536 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Record local command evidence without upgrading its trust beyond SELF.
Receipts preserve sanitized full output and the exact command when it contains no
credentials. A hash of the original command remains when redaction is necessary.
Checksums detect accidental modification, not forgery by a local file's author.
"""
import datetime
import fcntl
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import tempfile
SCHEMA = 1
SUMMARY_SCHEMA = 1
DEFAULT_TAIL = 15
SECRET_NAME = re.compile(r"token|password|passwd|secret|api[_-]?key|credential|private[_-]?key", re.I)
SHA256 = re.compile(r"[0-9a-f]{64}\Z")
GIT_SHA = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z")
BRANCH = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,199}\Z")
# 可选的分支目标字段:两者须同时出现或同时缺省(旧收据没有)。
TARGET_FIELDS = ("root_target", "immediate_target")
PRE_MERGE_CLOSED = "已完成(合入前闭合)"
TASK_STATE = re.compile(r"^\[task-state\] registered=(.+) initial_base=([0-9a-f]+)$", re.M)
BASE_FIELDS = {"schema", "source", "timestamp", "command", "command_sha256", "cwd", "common_dir",
"before", "after", "after_error", "exit_code", "output", "output_sha256", "receipt_sha256"}
class EvidenceError(Exception):
pass
def digest(value):
return hashlib.sha256(value.encode() if isinstance(value, str) else value).hexdigest()
def canonical(value):
return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
def sanitize(value):
"""Redact known secret values and conventional credential syntax everywhere."""
for name, secret in sorted(os.environ.items(), key=lambda item: -len(item[1])):
if SECRET_NAME.search(name) and len(secret) >= 4:
value = value.replace(secret, "<redacted>")
value = re.sub(r"-----BEGIN [^-\r\n]*PRIVATE KEY-----.*?-----END [^-\r\n]*PRIVATE KEY-----",
"<redacted-private-key>", value, flags=re.S)
value = re.sub(r"(?i)(\bAuthorization['\"]?[ \t]*[=:][ \t]*['\"]?"
r"(?:Bearer|token|Basic)[ \t]+)[^\s'\";]+", r"\1<redacted>", value)
value = re.sub(r"(?i)\b(https?|ssh)://[^\s/@:]+:[^\s/@]+@", r"\1://<redacted>@", value)
credential = r"[\w-]*(?:token|password|passwd|secret|api[_-]?key|credential)[\w-]*"
prefix = r"(?:\b" + credential + r"['\"]?[ \t]*[=:][ \t]*|--" + credential + r"[ \t]+)"
value = re.sub(r"(?i)(" + prefix + r")(['\"])(.*?)\2",
r"\1\2<redacted>\2", value, flags=re.S)
value = re.sub(r"(?i)(--" + credential + r"[ \t]+)(?!<redacted>)[^\s'\";]+",
r"\1<redacted>", value)
value = re.sub(
r"(?i)(\b[\w-]*(?:token|password|passwd|secret|api[_-]?key|credential)[\w-]*"
r"['\"]?[ \t]*[=:][ \t]*['\"]?)(?!<redacted>)[^\s'\"&;,}]+",
r"\1<redacted>", value)
return value
def sanitize_tree(value):
if isinstance(value, str):
return sanitize(value)
if isinstance(value, list):
return [sanitize_tree(item) for item in value]
if isinstance(value, dict):
return {sanitize(key): sanitize_tree(item) for key, item in value.items()}
return value
def git(*args, required=True):
result = subprocess.run(["git", *args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode and required:
raise EvidenceError("Git object could not be recorded: " + " ".join(args[:2]))
return result.stdout if result.returncode == 0 else b""
def git_text(*args, **kwargs):
return git(*args, **kwargs).decode("utf-8", "replace").strip()
def common_dir():
return str(Path(git_text("rev-parse", "--git-common-dir")).resolve())
def snapshot():
untracked = {}
root = Path(git_text("rev-parse", "--show-toplevel"))
for raw in git("-C", str(root), "ls-files", "--full-name", "--others", "--exclude-standard", "-z").split(b"\0"):
if not raw:
continue
name = raw.decode("utf-8", "replace")
path = root / name
if path.is_symlink():
untracked[name] = digest(os.fsencode(os.readlink(path)))
elif path.is_file():
with path.open("rb") as stream:
untracked[name] = hashlib.file_digest(stream, "sha256").hexdigest()
else:
untracked[name] = None
target = os.environ.get("TASK_TARGET_BRANCH", "main")
return {
"head": git_text("rev-parse", "HEAD"),
"branch": git_text("rev-parse", "--abbrev-ref", "HEAD"),
"target_branch": target,
"target_head": git_text("rev-parse", "--verify", "refs/remotes/origin/" + target,
required=False) or None,
"status": git("status", "--porcelain=v1", "--untracked-files=all").decode("utf-8", "replace"),
"tracked_diff_sha256": digest(git("diff", "--binary", "HEAD", "--")),
"untracked": untracked,
"submodules": git("submodule", "status", "--recursive").decode("utf-8", "replace"),
}
def valid_branch(name):
return (isinstance(name, str) and BRANCH.match(name) is not None and ".." not in name
and "//" not in name and not name.endswith(("/", ".", ".lock")))
def receipt_keys_ok(keys):
keys = set(keys)
targets = keys & set(TARGET_FIELDS)
if targets not in (set(), set(TARGET_FIELDS)):
return False
rest = keys - targets
return rest in (BASE_FIELDS, BASE_FIELDS | {"capture_note"})
def task_targets():
"""Root and immediate targets named by the current branch card; no metadata means main/main."""
default = os.environ.get("TASK_TARGET_BRANCH", "main")
try:
card = find_card()
except EvidenceError:
return default, default
match = re.search(r"^- 分支类型:[^\n]*?;root_target:([^;\n]+);immediate_target:([^;\n]+)",
card.read_text(encoding="utf-8"), re.M)
if not match:
return default, default
root, immediate = match.group(1).strip(), match.group(2).strip()
if not (valid_branch(root) and valid_branch(immediate)):
raise EvidenceError("Task card target metadata is invalid")
return root, immediate
def create_receipt(command, output, exit_code, before, after, *, timestamp=None,
cwd=None, git_common_dir=None, after_error=None, capture_note=None, targets=None):
"""Package an actual captured run; callers must not reconstruct missing output."""
root_target, immediate_target = targets or task_targets()
receipt = sanitize_tree({
"schema": SCHEMA, "source": "SELF",
"timestamp": timestamp or datetime.datetime.now(datetime.timezone.utc).isoformat(),
"command": command, "command_sha256": digest(command),
"cwd": cwd or str(Path.cwd().resolve()),
"common_dir": git_common_dir or common_dir(),
"before": before, "after": after, "after_error": after_error,
"exit_code": exit_code, "output": output,
"root_target": root_target, "immediate_target": immediate_target,
})
if capture_note is not None:
receipt["capture_note"] = sanitize(capture_note)
receipt["output_sha256"] = digest(receipt["output"])
receipt["receipt_sha256"] = digest(canonical(receipt))
return receipt
def validate_snapshot(value):
if not isinstance(value, dict):
raise EvidenceError("Invalid receipt object snapshot")
expected = {"head", "branch", "target_branch", "target_head", "status",
"tracked_diff_sha256", "submodules", "untracked"}
if set(value) != expected:
raise EvidenceError("Invalid receipt object fields")
for key in ("head", "branch", "target_branch", "status", "tracked_diff_sha256", "submodules"):
if not isinstance(value.get(key), str):
raise EvidenceError("Invalid receipt object field: " + key)
if not GIT_SHA.fullmatch(value["head"]) or not SHA256.fullmatch(value["tracked_diff_sha256"]):
raise EvidenceError("Invalid receipt object digest")
if value.get("target_head") is not None and (
not isinstance(value["target_head"], str) or not GIT_SHA.fullmatch(value["target_head"])):
raise EvidenceError("Invalid receipt target identity")
if not isinstance(value.get("untracked"), dict):
raise EvidenceError("Invalid receipt untracked object")
for name, checksum in value["untracked"].items():
if not isinstance(name, str) or (checksum is not None and (
not isinstance(checksum, str) or not SHA256.fullmatch(checksum))):
raise EvidenceError("Invalid receipt untracked digest")
def read_receipt(filename):
if not Path(filename).is_absolute():
raise EvidenceError("Receipt path must be absolute")
value = json.loads(Path(filename).read_text())
if not isinstance(value, dict) or type(value.get("schema")) is not int or value["schema"] != SCHEMA:
raise EvidenceError("Unsupported receipt schema")
if not receipt_keys_ok(value):
raise EvidenceError("Invalid receipt fields")
for key in TARGET_FIELDS:
if key in value and not valid_branch(value[key]):
raise EvidenceError("Invalid receipt target: " + key)
for key in ("source", "timestamp", "command", "command_sha256", "cwd", "common_dir",
"output", "output_sha256", "receipt_sha256"):
if not isinstance(value.get(key), str):
raise EvidenceError("Invalid receipt field: " + key)
if value["source"] != "SELF" or type(value.get("exit_code")) is not int or not 0 <= value["exit_code"] <= 255:
raise EvidenceError("Invalid receipt source or exit code")
datetime.datetime.fromisoformat(value["timestamp"].replace("Z", "+00:00"))
for key in ("cwd", "common_dir"):
if not Path(value[key]).is_absolute():
raise EvidenceError("Receipt identity paths must be absolute")
for key in ("command_sha256", "output_sha256", "receipt_sha256"):
if not SHA256.fullmatch(value[key]):
raise EvidenceError("Invalid receipt digest: " + key)
validate_snapshot(value.get("before"))
if value.get("after") is not None:
validate_snapshot(value["after"])
elif not isinstance(value.get("after_error"), str) or not value["after_error"]:
raise EvidenceError("Missing after object or capture error")
if value.get("after_error") is not None and not isinstance(value["after_error"], str):
raise EvidenceError("Invalid after capture error")
if "capture_note" in value and not isinstance(value["capture_note"], str):
raise EvidenceError("Invalid capture note")
if digest(value["output"]) != value["output_sha256"]:
raise EvidenceError("Receipt output checksum mismatch")
payload = {key: item for key, item in value.items() if key != "receipt_sha256"}
if digest(canonical(payload)) != value["receipt_sha256"]:
raise EvidenceError("Receipt checksum mismatch")
if value != sanitize_tree(value):
raise EvidenceError("Receipt contains unredacted credential syntax")
if Path(value["common_dir"]).resolve() != Path(common_dir()):
raise EvidenceError("Receipt belongs to a different Git common directory")
return value
def task_context(current=None, require_target=True):
"""Verify an existing task checkout; this is not an approval identity."""
current = snapshot() if current is None else current
root = Path(git_text("rev-parse", "--show-toplevel")).resolve()
branch = current["branch"]
if not re.fullmatch(r"(?:feature|fix|hotfix|release)/[A-Za-z0-9._-]+", branch):
raise EvidenceError("Local recovery requires a task branch")
records = [row.split(b"\0") for row in git("worktree", "list", "--porcelain", "-z").split(b"\0\0") if row]
matches = [row for row in records if row[0].startswith(b"worktree ")
and Path(os.fsdecode(row[0][9:])).resolve() == root]
if len(matches) != 1 or not records or matches[0] == records[0]:
raise EvidenceError("Recovery requires a registered linked task worktree")
entry = matches[0]
if any(field.startswith((b"bare", b"detached", b"prunable")) for field in entry) or (
b"branch refs/heads/" + branch.encode() not in entry):
raise EvidenceError("Task worktree registration is not valid")
if (root / ".task").is_symlink():
raise EvidenceError("Task registration must belong to the current checkout")
card = find_card()
if card.is_symlink() or card.resolve().parent not in (root / ".task", root / ".task" / "archive"):
raise EvidenceError("Task registration must belong to the current checkout")
content = card.read_text()
locations = re.findall(r"^- 工作区:(.+?)\s*[((]worktree[))];分支:([^\s]+)\s*$", content, re.M)
bases = re.findall(r"^- 权威仓:.*;目标分支:([^;]+);R0(initial_base_sha)= ([0-9a-f]+)\s*$", content, re.M)
if len(locations) != 1 or Path(locations[0][0]).resolve() != root or locations[0][1] != branch:
raise EvidenceError("Task card belongs to a different worktree or branch")
if len(bases) != 1 or bases[0][0] != current["target_branch"] or not GIT_SHA.fullmatch(bases[0][1]):
raise EvidenceError("Task card has no valid target and initial baseline")
r0 = bases[0][1]
if not current["target_head"] and require_target:
raise EvidenceError("Task target is unverified; fetch it first")
for tip in (current["head"], current["target_head"]):
if tip:
git("merge-base", "--is-ancestor", r0, tip)
check = subprocess.run([sys.executable, str(Path(__file__).with_name("check-worktree.py")), "before-sync"],
capture_output=True, text=True)
if check.returncode:
raise EvidenceError("Task workspace is not ready: " + (check.stdout + check.stderr).strip())
return {"root": root, "r0": r0}
def validate_precheck(receipt, purpose="start"):
if purpose not in ("start", "resume-delivery", "resume-local"):
raise EvidenceError("Unknown precheck validation purpose")
after = receipt["after"]
current = snapshot()
if receipt["command"] not in ("make precheck", "bash scripts/task.sh precheck"):
raise EvidenceError("Task start requires a precheck receipt")
expected = "LOCAL_READY" if purpose == "resume-local" else "READY"
verdicts = re.findall(r"^\[verdict\] ([A-Z_]+)(?:\s|$)", receipt["output"], re.M)
if receipt["exit_code"] != 0 or verdicts != [expected]:
raise EvidenceError("Precheck receipt is not " + expected)
if after is None or receipt["after_error"] is not None:
raise EvidenceError("Precheck object capture is incomplete")
if receipt["before"]["head"] != current["head"] or after["head"] != current["head"]:
raise EvidenceError("Precheck source HEAD has changed")
if after["target_branch"] != current["target_branch"] or (
not after["target_head"] and purpose != "resume-local"):
raise EvidenceError("Precheck authoritative target has changed")
if purpose != "resume-local" and after["target_head"] != current["target_head"]:
raise EvidenceError("Precheck authoritative target has changed")
if purpose.startswith("resume-"):
context = task_context(current, require_target=purpose != "resume-local")
try:
Path(receipt["cwd"]).resolve().relative_to(context["root"])
except ValueError:
raise EvidenceError("Precheck belongs to a different task worktree")
if any(obj["branch"] != current["branch"] or obj["target_branch"] != current["target_branch"]
for obj in (receipt["before"], after)):
raise EvidenceError("Precheck belongs to a different task branch")
if after["target_head"]:
git("merge-base", "--is-ancestor", context["r0"], after["target_head"])
if purpose == "resume-local":
if after["target_head"] and current["target_head"]:
try:
git("merge-base", "--is-ancestor", after["target_head"], current["target_head"])
except EvidenceError:
raise EvidenceError("Precheck target history has rewound or diverged") from None
validate_local_scope(receipt, current, context)
def validate_local_scope(receipt, current, context):
"""LOCAL_READY continues one active registered task; it remains SELF local-work evidence only."""
card = find_card()
status = re.search(r";状态:\s*([^;\s][^;\n]*)", card.read_text(encoding="utf-8"))
if card.resolve().parent != context["root"] / ".task" or not status or status.group(1).startswith("已"):
raise EvidenceError("Local continuation requires an active registered task")
states = TASK_STATE.findall(receipt["output"])
if len(states) != 1 or Path(states[0][0]) != context["root"] or states[0][1] != context["r0"]:
raise EvidenceError("Precheck belongs to a different task identity or initial baseline")
for obj in (receipt["before"], receipt["after"]):
if obj["status"] or obj["untracked"] or obj["tracked_diff_sha256"] != digest(b""):
raise EvidenceError("Precheck workspace was not clean")
if obj["submodules"] != current["submodules"]:
raise EvidenceError("Precheck component state has changed")
if tuple(receipt.get(key) for key in TARGET_FIELDS) != task_targets():
raise EvidenceError("Precheck authorization scope differs from the registered task")
def find_card():
root = Path(git_text("rev-parse", "--show-toplevel"))
branch = git_text("rev-parse", "--abbrev-ref", "HEAD")
card = root / ".task" / (branch.replace("/", "-") + ".md")
if card.is_file():
return card
owner = re.compile(r"^- 工作区:.*分支:" + re.escape(branch) + r"(?:\s|$)", re.M)
for candidate in (root / ".task").glob("*.md"):
if owner.search(candidate.read_text()):
return candidate
# A card fold-closed before merge (task.sh close --self) lives in the archive until its PR lands.
for candidate in sorted((root / ".task" / "archive").glob("*.md")):
text = candidate.read_text()
status = re.search(r";状态:([^;\n]*)", text)
if owner.search(text) and status and status.group(1).startswith(PRE_MERGE_CLOSED):
return candidate
raise EvidenceError("No task card exists for the current branch")
def evidence_dir():
configured = os.environ.get("AGENT_EVIDENCE_DIR")
if configured:
base = Path(configured)
else:
base = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "agent-system-dev" / "evidence"
base.mkdir(mode=0o700, parents=True, exist_ok=True)
return base.resolve()
def store_receipt(receipt):
"""Write the full receipt once under its digest; identical existing content is accepted."""
directory = evidence_dir()
path = directory / (receipt["receipt_sha256"] + ".json")
encoded = json.dumps(receipt, ensure_ascii=False, indent=2) + "\n"
if path.exists() or path.is_symlink():
try:
same = not path.is_symlink() and json.loads(path.read_text(encoding="utf-8")) == receipt
except (OSError, ValueError):
same = False
if not same:
raise EvidenceError("Full receipt path holds different content: " + str(path))
return path
fd, temporary = tempfile.mkstemp(prefix=".receipt-", dir=directory)
try:
with os.fdopen(fd, "w", encoding="utf-8") as stream:
os.fchmod(stream.fileno(), 0o600)
stream.write(encoded)
stream.flush()
os.fsync(stream.fileno())
try:
os.link(temporary, path)
except FileExistsError:
return store_receipt(receipt)
finally:
os.unlink(temporary)
return path
def tail_lines():
value = os.environ.get("TAIL", "")
return int(value) if value.isdigit() else DEFAULT_TAIL
def build_summary(receipt, receipt_path, tail=None):
"""Card-sized view of a full receipt; every field is derived from the sanitized receipt."""
tail = tail_lines() if tail is None else tail
before, after = receipt["before"], receipt["after"]
summary = {
"summary_schema": SUMMARY_SCHEMA, "source": "SELF",
"timestamp": receipt["timestamp"], "command": receipt["command"],
"exit_code": receipt["exit_code"], "cwd": receipt["cwd"],
"before_head": before["head"], "after_head": after["head"] if after else None,
"branch": before["branch"], "target_branch": before["target_branch"],
"target_head": before["target_head"],
}
for key in TARGET_FIELDS:
if key in receipt:
summary[key] = receipt[key]
summary.update({
"before_tracked_diff_sha256": before["tracked_diff_sha256"],
"after_tracked_diff_sha256": after["tracked_diff_sha256"] if after else None,
"output_tail": receipt["output"].splitlines()[-tail:] if tail > 0 else [],
"output_sha256": receipt["output_sha256"],
"receipt_sha256": receipt["receipt_sha256"],
"receipt_path": str(receipt_path),
})
return summary
def append_card(card, receipt):
path = store_receipt(receipt)
encoded = json.dumps(build_summary(receipt, path), ensure_ascii=False, indent=2)
block = "\n## 证据(Evidence)\n\n" + (
"<!-- Local receipt: SELF. Digests detect corruption; they do not authenticate a reviewer or CI. -->\n"
+ "```json\n" + encoded + "\n```\n")
with card.open("r+", encoding="utf-8") as stream:
fcntl.flock(stream, fcntl.LOCK_EX)
existing = stream.read()
if receipt["receipt_sha256"] in existing:
return
if "## 证据(Evidence)" in existing:
block = block.replace("\n## 证据(Evidence)\n\n", "\n", 1)
stream.write(block)
stream.flush()
os.fsync(stream.fileno())
def write_receipt(path, receipt):
with path.open("x", encoding="utf-8") as stream:
os.chmod(path, 0o600)
json.dump(receipt, stream, ensure_ascii=False, indent=2)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def report(receipt, destination):
print(sanitize("[evidence] " + destination + " exit=" + str(receipt["exit_code"])
+ " src=SELF before=" + receipt["before"]["head"]
+ " receipt_sha256=" + receipt["receipt_sha256"]))
def main():
child_exit = None
try:
if os.environ.get("SRC", "SELF") not in ("", "SELF"):
raise EvidenceError("Local evidence is SELF; GATE/REPLAY require external execution identity")
if sys.argv[1:] == ["--local-context"]:
context = task_context(require_target=False)
print("[task-state] registered=" + str(context["root"]) + " initial_base=" + context["r0"])
return 0
if sys.argv[1:]:
raise EvidenceError("Unknown evidence arguments")
capture, imported, validated = (os.environ.get(key) for key in
("RECEIPT", "IMPORT_RECEIPT", "VALIDATE_RECEIPT"))
command = os.environ.get("CMD", "")
if sum(bool(item) for item in (capture, imported, validated)) > 1 or ((imported or validated) and command):
raise EvidenceError("Choose one capture/import/validate operation")
if imported or validated:
receipt = read_receipt(imported or validated)
if validated:
purpose = os.environ.get("VALIDATE_RECEIPT_PURPOSE", "start")
validate_precheck(receipt, purpose)
report(receipt, "local continuation validated (local work only; not delivery evidence)"
if purpose == "resume-local" else "precheck validated")
else:
card = find_card()
append_card(card, receipt)
report(receipt, "card=" + str(card))
return receipt["exit_code"]
if not command or "\n" in command or "\r" in command or "\0" in command:
raise EvidenceError("CMD must be a nonempty single-line command")
path = Path(capture) if capture else None
if path is not None and (not path.is_absolute() or path.exists() or path.is_symlink()):
raise EvidenceError("Capture requires an absolute, unused receipt path")
card = find_card() if path is None else None
before = snapshot()
targets = task_targets()
source_cwd, source_common = str(Path.cwd().resolve()), common_dir()
timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
child_env = os.environ.copy()
for key in ("CMD", "SRC", "RECEIPT", "IMPORT_RECEIPT", "VALIDATE_RECEIPT", "TAIL",
"VALIDATE_RECEIPT_PURPOSE"):
child_env.pop(key, None)
result = subprocess.run(["bash", "-o", "pipefail", "-c", command],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=child_env)
child_exit = result.returncode if result.returncode >= 0 else 128 - result.returncode
after, after_error = None, None
try:
after = snapshot()
except (EvidenceError, OSError) as error:
after_error = str(error)
receipt = create_receipt(command, result.stdout.decode("utf-8", "replace"), child_exit,
before, after, timestamp=timestamp, cwd=source_cwd,
git_common_dir=source_common, after_error=after_error,
targets=targets)
if path is not None:
write_receipt(path, receipt)
report(receipt, "receipt=" + str(path))
else:
append_card(card, receipt)
report(receipt, "card=" + str(card))
print(receipt["output"], end="" if receipt["output"].endswith("\n") else "\n")
if after_error:
print("[evidence] after object unavailable: " + sanitize(after_error), file=sys.stderr)
return child_exit
except (EvidenceError, OSError, ValueError, TypeError) as error:
print("[evidence] ERROR: " + sanitize(str(error)), file=sys.stderr)
return child_exit if child_exit else 2
if __name__ == "__main__":
sys.exit(main())