387 lines
13 KiB
Python
387 lines
13 KiB
Python
"""托管 shell 脚本：管道任一读取级不得是提前退出的读取器（grep -q/-m、head、sed q、awk exit、read）。
|
||
|
||
pipefail 下读取器先退出，上游生产者写满 64 KiB 管道缓冲后收到 SIGPIPE，管道状态报 141，
|
||
`if producer | grep -q PAT` 与 `if ! ...` 的判定被静默翻转；读取器位于管道中段
|
||
（`producer | head -1 | sed ...`）时同理。改用 here-string、进程替换或 sed 打印即退出，
|
||
读取器的结论才不会丢。
|
||
"""
|
||
import re
|
||
import subprocess
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SCRIPT_GLOBS = (":(glob)scripts/**/*.sh", ":(glob)scripts/hooks/*")
|
||
EXTRA_SCRIPTS = ("docker/verify-env.sh", "tests/load/collaboration/run_pgbench.sh")
|
||
# 发现到但检测器无法正确解析的脚本：路径 -> 理由。条目须仍被发现，否则视为过期。
|
||
EXEMPT = {}
|
||
|
||
HEREDOC = re.compile(r"<<(-?)\s*(['\"]?)([A-Za-z_][A-Za-z0-9_]*)\2")
|
||
SED_QUIT = re.compile(r"(?:^|[\s;{}'\"/$0-9])[qQ][0-9]*\s*(?:$|[\s;}'\"])")
|
||
AWK_END = re.compile(r"\bEND\s*\{")
|
||
AWK_EXIT = re.compile(r"\bexit\b")
|
||
ASSIGNMENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*=")
|
||
GREP_ARG_OPTS = set("efABCdDX")
|
||
BOUNDARY = set(" \t\n;&|()<>")
|
||
|
||
|
||
def managed_scripts():
|
||
"""git 跟踪的托管 shell 脚本；未跟踪副本（如 runtime/ 下的陈旧克隆）不在其中。"""
|
||
out = subprocess.run(
|
||
["git", "-C", str(ROOT), "ls-files", "-z", "--", *SCRIPT_GLOBS, *EXTRA_SCRIPTS],
|
||
check=True, capture_output=True, text=True,
|
||
).stdout
|
||
return sorted(p for p in out.split("\0") if p)
|
||
|
||
|
||
class _Frame:
|
||
"""一层命令上下文（顶层、$(...)、(...)、<(...)）内正在累积的管道。"""
|
||
|
||
def __init__(self, line):
|
||
self.stages, self.stage, self.start, self.cases = [], [], line, []
|
||
|
||
|
||
def pipelines(src):
|
||
"""词法扫描 bash 源码，返回每条多级管道各级的 [(起始行号, 文本), ...]；跳过注释与 heredoc 正文。"""
|
||
found, frames, modes, heredocs = [], [_Frame(1)], ["cmd"], []
|
||
line, word, i, n = 1, "", 0, len(src)
|
||
|
||
def emit(s):
|
||
for f in frames:
|
||
f.stage.append(s)
|
||
|
||
def finish(f):
|
||
if f.stages:
|
||
found.append(f.stages + [(f.start, "".join(f.stage).strip())])
|
||
f.stages, f.stage, f.start = [], [], line
|
||
|
||
def case_state():
|
||
cases = frames[-1].cases
|
||
return cases[-1] if cases else None
|
||
|
||
def keyword(w):
|
||
cases = frames[-1].cases
|
||
if w == "case":
|
||
cases.append("in")
|
||
elif w == "in" and case_state() == "in":
|
||
cases[-1] = "pattern"
|
||
elif w == "esac" and cases:
|
||
cases.pop()
|
||
|
||
def push_cmd(opener):
|
||
emit(opener)
|
||
modes.append("cmd")
|
||
frames.append(_Frame(line))
|
||
|
||
while i < n:
|
||
c, nxt = src[i], src[i + 1 : i + 2]
|
||
mode = modes[-1]
|
||
if mode in ("sq", "ansi"):
|
||
if c == "\\" and mode == "ansi":
|
||
emit(src[i : i + 2])
|
||
i += 2
|
||
continue
|
||
emit(c)
|
||
line += c == "\n"
|
||
if c == "'":
|
||
modes.pop()
|
||
i += 1
|
||
continue
|
||
if mode in ("dq", "brace"):
|
||
if c == "\\":
|
||
emit(src[i : i + 2])
|
||
line += nxt == "\n"
|
||
i += 2
|
||
elif c == "$" and nxt == "(":
|
||
push_cmd("$(")
|
||
i += 2
|
||
elif c == "$" and nxt == "{":
|
||
emit("${")
|
||
modes.append("brace")
|
||
i += 2
|
||
elif c == '"':
|
||
emit(c)
|
||
if mode == "dq":
|
||
modes.pop()
|
||
else:
|
||
modes.append("dq")
|
||
i += 1
|
||
elif c == "}" and mode == "brace":
|
||
emit(c)
|
||
modes.pop()
|
||
i += 1
|
||
else:
|
||
emit(c)
|
||
line += c == "\n"
|
||
i += 1
|
||
continue
|
||
|
||
if c in BOUNDARY and word:
|
||
keyword(word)
|
||
word = ""
|
||
top = frames[-1]
|
||
if c == "\\":
|
||
if nxt == "\n":
|
||
emit(" ")
|
||
line += 1
|
||
else:
|
||
emit(src[i : i + 2])
|
||
word += "\0"
|
||
i += 2
|
||
elif c == "#" and (i == 0 or src[i - 1] in " \t\n;&|("):
|
||
j = src.find("\n", i)
|
||
i = n if j < 0 else j
|
||
elif c == "'" or (c == "$" and nxt == "'"):
|
||
emit(src[i : i + 2] if c == "$" else c)
|
||
modes.append("ansi" if c == "$" else "sq")
|
||
word += "\0"
|
||
i += 2 if c == "$" else 1
|
||
elif c == '"':
|
||
emit(c)
|
||
modes.append("dq")
|
||
word += "\0"
|
||
i += 1
|
||
elif c == "$" and nxt == "{":
|
||
emit("${")
|
||
modes.append("brace")
|
||
word += "\0"
|
||
i += 2
|
||
elif c == "$" and nxt == "(":
|
||
push_cmd("$(")
|
||
word += "\0"
|
||
i += 2
|
||
elif c == "(":
|
||
if case_state() == "pattern":
|
||
emit(c)
|
||
else:
|
||
push_cmd(c)
|
||
i += 1
|
||
elif c == ")":
|
||
if case_state() == "pattern":
|
||
top.cases[-1] = "body"
|
||
elif len(frames) > 1:
|
||
finish(top)
|
||
frames.pop()
|
||
modes.pop()
|
||
emit(c)
|
||
i += 1
|
||
elif c == "|":
|
||
if nxt == "|":
|
||
finish(top)
|
||
emit("||")
|
||
i += 2
|
||
elif case_state() == "pattern":
|
||
emit(c)
|
||
i += 1
|
||
else:
|
||
top.stages.append((top.start, "".join(top.stage).strip()))
|
||
emit(c)
|
||
top.stage, top.start = [], line
|
||
i += 2 if nxt == "&" else 1
|
||
elif c == ";":
|
||
if src.startswith(";;", i) or src.startswith(";&", i):
|
||
if case_state() == "body":
|
||
top.cases[-1] = "pattern"
|
||
k = 3 if src.startswith(";;&", i) else 2
|
||
else:
|
||
k = 1
|
||
finish(top)
|
||
emit(src[i : i + k])
|
||
i += k
|
||
elif c == "&":
|
||
if nxt == "&":
|
||
finish(top)
|
||
emit("&&")
|
||
i += 2
|
||
elif nxt == ">" or (i > 0 and src[i - 1] in "<>"):
|
||
emit(c)
|
||
i += 1
|
||
else:
|
||
finish(top)
|
||
emit(c)
|
||
i += 1
|
||
elif c == "<" and src.startswith("<<", i):
|
||
m = None if src.startswith("<<<", i) else HEREDOC.match(src, i)
|
||
if m:
|
||
heredocs.append((m.group(3), m.group(1) == "-"))
|
||
emit(m.group(0))
|
||
i = m.end()
|
||
else:
|
||
k = 3 if src.startswith("<<<", i) else 2
|
||
emit(src[i : i + k])
|
||
i += k
|
||
elif c == "\n":
|
||
line += 1
|
||
i += 1
|
||
for term, dash in heredocs:
|
||
while i < n:
|
||
j = src.find("\n", i)
|
||
body = src[i : n if j < 0 else j]
|
||
i = n if j < 0 else j + 1
|
||
line += 1
|
||
if (body.lstrip("\t") if dash else body) == term:
|
||
break
|
||
heredocs = []
|
||
if not (top.stages and not "".join(top.stage).strip()):
|
||
finish(top)
|
||
emit("\n")
|
||
else:
|
||
emit(c)
|
||
if c not in BOUNDARY:
|
||
word += c
|
||
i += 1
|
||
|
||
if len(frames) != 1 or modes != ["cmd"]:
|
||
raise ValueError(f"unbalanced shell lexing at EOF: modes={modes}")
|
||
finish(frames[0])
|
||
return found
|
||
|
||
|
||
def _grep_exits_early(words):
|
||
for w in words[1:]:
|
||
w = w.strip("'\"")
|
||
if w == "--":
|
||
break
|
||
if w.startswith("--"):
|
||
if w.split("=")[0] in ("--quiet", "--silent", "--max-count"):
|
||
return True
|
||
elif w.startswith("-") and len(w) > 1:
|
||
for ch in w[1:]:
|
||
if ch in "qm":
|
||
return True
|
||
if ch in GREP_ARG_OPTS:
|
||
break
|
||
return False
|
||
|
||
|
||
def _awk_exits_early(prog):
|
||
"""awk 程序在 END 块之外含 exit（含 BEGIN 内），即可能在读完输入前退出。"""
|
||
while m := AWK_END.search(prog):
|
||
depth, j = 0, m.end() - 1
|
||
while j < len(prog):
|
||
depth += {"{": 1, "}": -1}.get(prog[j], 0)
|
||
j += 1
|
||
if depth == 0:
|
||
break
|
||
prog = prog[: m.start()] + prog[j:]
|
||
return bool(AWK_EXIT.search(prog))
|
||
|
||
|
||
def early_exit_reader(stage):
|
||
"""读取级读到结论后是否会提前关闭 stdin（grep -q/-m、head、sed q、awk exit、read）。"""
|
||
words = stage.split()
|
||
while words and (words[0] in ("{", "!", "time", "command", "env") or ASSIGNMENT.match(words[0])):
|
||
words.pop(0)
|
||
if not words:
|
||
return False
|
||
cmd, rest = words[0], " ".join(words[1:])
|
||
if cmd in ("grep", "egrep", "fgrep"):
|
||
return _grep_exits_early(words)
|
||
if cmd in ("head", "read"):
|
||
return True
|
||
if cmd == "sed":
|
||
return bool(SED_QUIT.search(rest))
|
||
if cmd in ("awk", "gawk", "mawk", "nawk"):
|
||
return _awk_exits_early(rest)
|
||
return False
|
||
|
||
|
||
def hazards(src):
|
||
"""各管道第二级起（读 pipe 的各级）中提前退出的读取器，返回 (行号, 该级文本)。"""
|
||
return [
|
||
(line, text)
|
||
for stages in pipelines(src)
|
||
for line, text in stages[1:]
|
||
if early_exit_reader(text)
|
||
]
|
||
|
||
|
||
class ShellPipeHazardTests(unittest.TestCase):
|
||
def test_detector_flags_synthetic_hazard(self):
|
||
found = hazards("set -euo pipefail\nseq 1 200000 | grep -q 1\n")
|
||
self.assertEqual([line for line, _ in found], [2], found)
|
||
|
||
def test_detector_flags_hazard_shapes(self):
|
||
for snippet in (
|
||
'if ! seq 1 200000 | grep -q 1; then echo miss; fi',
|
||
'x="$(sed -n \'s/^k=//p\' "$f" | head -1)"',
|
||
'if ! "$bin" serve -h 2>&1 | grep -Eq -- "(^|[[:space:]])-{1,2}config"; then :; fi',
|
||
'echo "${r:+($(printf \'%s\' "$r" | head -c 120))}"',
|
||
'find . -type f \\\n | grep -m1 x',
|
||
"seq 1 200000 | sed -n '/1/{p;q;}'",
|
||
'seq 1 200000 | grep -qiE "a|b" || exit 1',
|
||
):
|
||
with self.subTest(snippet=snippet):
|
||
self.assertTrue(hazards(snippet + "\n"), snippet)
|
||
|
||
def test_detector_flags_newly_covered_shapes(self):
|
||
for snippet, stage in (
|
||
("seq 1 200000 | head -1 | sed 's/^/x/'", "head -1"),
|
||
("x=\"$(seq 1 200000 | head -n 3 | tr -d ' ')\"", "head -n 3"),
|
||
("seq 1 200000 | awk '{print $1; exit}'", "awk '{print $1; exit}'"),
|
||
("seq 1 200000 | awk 'BEGIN { exit 1 }'", "awk 'BEGIN { exit 1 }'"),
|
||
("seq 1 200000 | awk '/7/{f=1; exit} END{print f}' | cat", "awk '/7/{f=1; exit} END{print f}'"),
|
||
("seq 1 200000 | read -r first", "read -r first"),
|
||
("seq 1 200000 | IFS= read -r first", "IFS= read -r first"),
|
||
("if seq 1 200000 | LC_ALL=C grep -q 1; then :; fi", "LC_ALL=C grep -q 1"),
|
||
("seq 1 200000 | env LC_ALL=C grep -m1 1 | wc -l", "env LC_ALL=C grep -m1 1"),
|
||
("seq 1 200000 | grep -q 1 | cat", "grep -q 1"),
|
||
):
|
||
with self.subTest(snippet=snippet):
|
||
self.assertEqual(hazards(snippet + "\n"), [(1, stage)], snippet)
|
||
|
||
def test_detector_accepts_safe_forms(self):
|
||
src = "\n".join((
|
||
"set -euo pipefail",
|
||
"# seq 1 200000 | grep -q 1",
|
||
"grep -q 1 < <(seq 1 200000)",
|
||
'grep -q x <<<"$v"',
|
||
"x=\"$(sed -n '/^k=/{s///p;q;}' \"$f\")\"",
|
||
"x=\"$(sed -n '/^k=/{s///p;q;}' < <(seq 1 200000))\"",
|
||
"awk '{print $1; exit}' < <(seq 1 200000)",
|
||
"head -c 160 < <(seq 1 200000)",
|
||
"ps -eo comm | sed -n '1,12p'",
|
||
"seq 1 200000 | sed -n '1,12p' | tr -d ' '",
|
||
"seq 1 200000 | awk '{n++} END {exit !n}'",
|
||
"seq 1 200000 | awk '$1 == 7 { f = 1 } END { exit !f }' && echo hit",
|
||
"seq 1 200000 | while read -r l; do :; done",
|
||
"seq 1 200000 | LC_ALL=C grep -c 1",
|
||
"case \"$w\" in --auth|--config) grep -q x <<<\"$w\" ;; esac",
|
||
"cat <<'EOF'",
|
||
"seq 1 200000 | grep -q 1",
|
||
"EOF",
|
||
"",
|
||
))
|
||
self.assertEqual(hazards(src), [])
|
||
|
||
def test_discovery_covers_managed_scripts(self):
|
||
found = managed_scripts()
|
||
for rel in (
|
||
"scripts/svc.sh",
|
||
"scripts/task.sh",
|
||
"scripts/e2e.sh",
|
||
"scripts/e2e-duties.sh",
|
||
"scripts/lint-boundaries.sh",
|
||
"scripts/hooks/pre-commit",
|
||
*EXTRA_SCRIPTS,
|
||
):
|
||
self.assertIn(rel, found)
|
||
self.assertFalse([p for p in found if p.startswith("runtime/")], found)
|
||
self.assertFalse(set(EXEMPT) - set(found), "stale EXEMPT entries")
|
||
|
||
def test_managed_scripts_have_no_early_exit_pipe_reader(self):
|
||
for rel in managed_scripts():
|
||
if rel in EXEMPT:
|
||
continue
|
||
with self.subTest(script=rel):
|
||
path = ROOT / rel
|
||
self.assertTrue(path.is_file(), f"{rel} missing")
|
||
found = hazards(path.read_text(encoding="utf-8"))
|
||
self.assertEqual(
|
||
found, [], "\n".join(f"{rel}:{line}: | {text}" for line, text in found)
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|