1#!/usr/bin/env python3 2# throwspec.py — find and remove C++ dynamic exception specifications. 3# 4# Usage: throwspec.py report classify every site, change nothing 5# throwspec.py apply delete the specifications with a type list 6# throwspec.py verify check a finished rewrite against HEAD 7# 8# --root <dir> repository root (default: two levels above this script) 9# 10# This is a ONE-SHOT SOURCE MIGRATION TOOL, not a build step. No Bazel rule 11# refers to it and it is deliberately absent from BUILD.bazel: it edits the 12# source tree in place and is run by hand. It lives here so the next sweep does 13# not have to re-derive the classification rules below, which took several 14# wrong attempts to get right. 15# 16# WHAT IT SEPARATES. A regular expression cannot tell these apart -- both are 17# the token `throw', whitespace, `(', a name, `)': 18# 19# void foo( int n ) throw (RuntimeException); // specification -> delete 20# if ( bad ) throw (UINT) ERROR_ALREADY_RUNNING; // statement -> keep 21# 22# A site counts as a specification only when ALL FOUR of these hold: 23# 24# 1. the argument parses as a comma-separated list of type names, allowing 25# `typename', template arguments, and a macro line-continuation INSIDE the 26# list; 27# 2. the previous significant token is `)', `const' or `volatile', where a 28# `\'+newline is skipped as whitespace -- without that, every specification 29# written inside a #define body looks unanchored, because the declarator's 30# closing paren is on the previous physical line; 31# 3. the argument is non-empty (an EMPTY specification is a separate task -- 32# MSVC implements throw() as __declspec(nothrow), so it means something); 33# 4. the FOLLOWING token is one of `{ ; = : , ) #'. This is the rule that 34# catches a throw statement whose operand carries a C-style cast; exactly 35# one site in the tree needs it (desktop/win32/source/setup/setup_main.cxx). 36# An identifier may follow only across a line break, which is where a macro 37# whose body ends in a specification meets the next line. 38# 39# Everything the classifier rejects is reported, never silently skipped. There 40# were 30 such sites when this removed 72,490 specifications; read all of them. 41# 42# WHY `verify' EXISTS. The rewrite only ever deletes, which gives a free 43# structural invariant: per file the counts of `{', `}' and `;' must be 44# unchanged, and `(' and `)' must drop by the same amount. That check covers all 45# of a five-thousand-file diff in seconds and is what makes a sweep this size 46# reviewable at all. Run it before asking anyone to build. 47# 48# It compares the working tree against HEAD, so any file you ALSO edited by hand 49# in the same change will appear in its output -- that is the point. Reconcile 50# the list against the files you touched deliberately; it should not be empty, 51# it should be exactly those. 52# 53# FOR THE NEXT SWEEP (throw() -> noexcept): the classifier already reports empty 54# specifications as kind EMPTY. Select those instead of SPEC, and replace 55# `rewrite()' with a substituting variant -- the span bookkeeping is the same, 56# only the replacement text differs. 57 58import collections 59import os 60import re 61import subprocess 62import sys 63import threading 64 65CODE, COMMENT, LITERAL = 1, 2, 3 66 67SOURCE_EXTS = ('.hxx', '.cxx', '.hpp', '.hdl', '.inl', '.h', '.c', '.cc', 68 '.cpp', '.mm') 69 70 71# ── lexical pass ───────────────────────────────────────────────────────── 72 73def classify(text): 74 """Per-character map: CODE / COMMENT / LITERAL (string or char literal).""" 75 n = len(text) 76 m = bytearray(n) 77 i = 0 78 while i < n: 79 c = text[i] 80 if c == '/' and i + 1 < n and text[i + 1] == '/': 81 start = i 82 i += 2 83 while i < n: 84 if text[i] == '\\': # continuation keeps the comment open 85 j = i + 1 86 if j < n and text[j] == '\r': 87 j += 1 88 if j < n and text[j] == '\n': 89 i = j + 1 90 continue 91 i += 1 92 continue 93 if text[i] == '\n': 94 break 95 i += 1 96 for k in range(start, i): 97 m[k] = COMMENT 98 continue 99 if c == '/' and i + 1 < n and text[i + 1] == '*': 100 start = i 101 i += 2 102 while i + 1 < n and not (text[i] == '*' and text[i + 1] == '/'): 103 i += 1 104 i = min(i + 2, n) 105 for k in range(start, i): 106 m[k] = COMMENT 107 continue 108 if c == '"' or c == "'": 109 start = i 110 q = c 111 i += 1 112 ok = False 113 while i < n: 114 if text[i] == '\\': 115 i += 2 116 continue 117 if text[i] == '\n': 118 break 119 if text[i] == q: 120 i += 1 121 ok = True 122 break 123 i += 1 124 if not ok: # unterminated on its line: an apostrophe, not a literal 125 i = start + 1 126 m[start] = CODE 127 continue 128 for k in range(start, i): 129 m[k] = LITERAL 130 continue 131 m[i] = CODE 132 i += 1 133 return m 134 135 136def is_continuation(text, i): 137 """True if text[i] is a macro line-continuation backslash.""" 138 if text[i] != '\\': 139 return False 140 j = i + 1 141 while j < len(text) and text[j] in ' \t\r': 142 j += 1 143 return j < len(text) and text[j] == '\n' 144 145 146def _skippable(text, m, i): 147 return (m[i] == COMMENT 148 or (m[i] == CODE and (text[i].isspace() or is_continuation(text, i)))) 149 150 151def skip_fwd(text, m, i): 152 """First significant position at or after i (whitespace, comments and 153 macro line-continuations are not significant).""" 154 n = len(text) 155 while i < n and _skippable(text, m, i): 156 i += 1 157 return i 158 159 160def skip_back(text, m, i): 161 """Last significant position at or before i.""" 162 while i >= 0 and _skippable(text, m, i): 163 i -= 1 164 return i 165 166 167def match_paren(text, m, open_pos): 168 """Index of the ')' matching the '(' at open_pos, or -1.""" 169 depth = 0 170 n = len(text) 171 i = open_pos 172 while i < n: 173 if m[i] == CODE: 174 if text[i] == '(': 175 depth += 1 176 elif text[i] == ')': 177 depth -= 1 178 if depth == 0: 179 return i 180 i += 1 181 return -1 182 183 184def strip_comments(text, m, a, b): 185 return ''.join(text[k] for k in range(a, b) if m[k] != COMMENT) 186 187 188# ── classification ─────────────────────────────────────────────────────── 189 190SEG = r'[A-Za-z_]\w*(?:\s*<[^<>()]*>)?' 191NAME = r'(?:\s*typename\b)?(?:\s*::)?\s*%s(?:\s*::\s*%s)*\s*' % (SEG, SEG) 192TYPELIST = re.compile(r'^%s(?:,%s)*$' % (NAME, NAME)) 193CONTINUATION = re.compile(r'\\[ \t]*\r?\n') 194KEYWORD = re.compile(r'\b(?:SAL_THROW_EXTERN_C|SAL_THROW_DTOR|SAL_THROW|throw)\b') 195IDENT_BACK = re.compile(r'[A-Za-z_]\w*$') 196 197 198def sites(path, text): 199 """Yield a dict per specification-shaped site. 200 201 kind is one of: 202 SPEC a specification with a type list -- rewrite this 203 EMPTY throw() / SAL_THROW( () ) / SAL_THROW_EXTERN_C() 204 STATEMENT a throw statement; the argument is an expression 205 REVIEW_UNANCHORED type list, but not attached to a declarator 206 REVIEW_FOLLOWED_BY_EXPR type list, but an expression follows it 207 ODD_SAL_THROW SAL_THROW without the inner parentheses 208 UNBALANCED unmatched parenthesis 209 """ 210 m = classify(text) 211 for mo in KEYWORD.finditer(text): 212 s = mo.start() 213 if m[s] != CODE: 214 continue 215 kw = mo.group(0) 216 op = skip_fwd(text, m, mo.end()) 217 if op >= len(text) or text[op] != '(': 218 continue 219 cl = match_paren(text, m, op) 220 line = text.count('\n', 0, s) + 1 221 if cl < 0: 222 yield dict(kind='UNBALANCED', kw=kw, start=s, end=mo.end(), 223 path=path, line=line, content='') 224 continue 225 226 if kw in ('throw', 'SAL_THROW_EXTERN_C'): 227 inner_a, inner_b = op + 1, cl 228 else: 229 # SAL_THROW( (...) ): unwrap the inner parentheses 230 a = skip_fwd(text, m, op + 1) 231 if a >= cl or text[a] != '(': 232 yield dict(kind='ODD_SAL_THROW', kw=kw, start=s, end=cl + 1, 233 path=path, line=line, content='') 234 continue 235 b = match_paren(text, m, a) 236 if b < 0 or skip_fwd(text, m, b + 1) != cl: 237 yield dict(kind='ODD_SAL_THROW', kw=kw, start=s, end=cl + 1, 238 path=path, line=line, content='') 239 continue 240 inner_a, inner_b = a + 1, b 241 242 content = strip_comments(text, m, inner_a, inner_b) 243 if not content.strip(): 244 yield dict(kind='EMPTY', kw=kw, start=s, end=cl + 1, path=path, 245 line=line, content='') 246 continue 247 content = CONTINUATION.sub(' ', content) # a spec may span macro lines 248 249 prev = skip_back(text, m, s - 1) 250 prev_ch = text[prev] if prev >= 0 else '' 251 ident = IDENT_BACK.search(text[max(0, prev - 32):prev + 1]) 252 prev_word = ident.group(0) if ident else '' 253 anchored = prev_ch == ')' or prev_word in ('const', 'volatile') 254 listish = bool(TYPELIST.match(content)) 255 256 after = skip_fwd(text, m, cl + 1) 257 followed_ok = (after >= len(text) or text[after] in '{;=:,)#' 258 or '\n' in text[cl + 1:after]) 259 260 if kw != 'throw': 261 kind = 'SPEC' if listish else 'ODD_SAL_THROW' 262 elif not listish: 263 kind = 'STATEMENT' 264 elif not anchored: 265 kind = 'REVIEW_UNANCHORED' 266 elif not followed_ok: 267 kind = 'REVIEW_FOLLOWED_BY_EXPR' 268 else: 269 kind = 'SPEC' 270 271 yield dict(kind=kind, kw=kw, start=s, end=cl + 1, path=path, line=line, 272 content=content.strip(), prev_ch=prev_ch, prev_word=prev_word) 273 274 275# ── rewriting ──────────────────────────────────────────────────────────── 276 277BLANK_LINE = re.compile(r'[ \t]*(\\?)[ \t]*\r?$') 278 279 280def _drop_emptied_line(out, pos): 281 """Drop the line at pos if deleting the specification left it blank. 282 283 A macro's continuation backslash must not change hands: remove the line only 284 when its own trailing backslash (if any) matches the previous line's, so a 285 macro body can never absorb the line that followed it. 286 """ 287 ls = out.rfind('\n', 0, pos) + 1 288 le = out.find('\n', pos) 289 if le < 0: 290 le = len(out) 291 mo = BLANK_LINE.fullmatch(out[ls:le]) 292 if not mo: 293 return out 294 ps = out.rfind('\n', 0, ls - 1) + 1 if ls else 0 295 prev_continues = out[ps:max(ls - 1, ps)].rstrip('\r').endswith('\\') 296 if bool(mo.group(1)) != prev_continues: 297 return out 298 return out[:ls] + out[le + 1:] if le < len(out) else out[:ls] 299 300 301def rewrite(text, spans): 302 """Delete the given (start, end) spans, closing up the line behind them.""" 303 m = classify(text) 304 out = text 305 joined = 0 306 for start, end in sorted(spans, reverse=True): 307 d = start 308 while d > 0 and out[d - 1] in ' \t\r\n': 309 d -= 1 310 # Never swallow whitespace that hangs off a comment (joining the rest of 311 # the line onto a // comment would comment it out) nor off a macro's 312 # line-continuation backslash (which would leave a stray one mid-line). 313 if d > 0 and (m[d - 1] != CODE or out[d - 1] == '\\'): 314 d = start 315 elif '\n' in out[d:start]: 316 joined += 1 317 out = _drop_emptied_line(out[:d] + out[end:], d) 318 return out, joined 319 320 321# ── drivers ────────────────────────────────────────────────────────────── 322 323def tracked_sources(root): 324 out = subprocess.run(['git', 'ls-files', 'main/'], cwd=root, 325 capture_output=True, text=True, check=True).stdout 326 for p in out.splitlines(): 327 if p.lower().endswith(SOURCE_EXTS): 328 yield p 329 330 331def read(root, rel): 332 # latin-1 is a byte<->char bijection, so line endings and any non-UTF-8 333 # bytes round-trip untouched. Every token this tool matches is ASCII. 334 with open(os.path.join(root, rel.replace('/', os.sep)), 335 encoding='latin-1', newline='') as f: 336 return f.read() 337 338 339def write(root, rel, text): 340 with open(os.path.join(root, rel.replace('/', os.sep)), 'w', 341 encoding='latin-1', newline='') as f: 342 f.write(text) 343 344 345def run_report(root, apply_changes): 346 counts = collections.Counter() 347 review = collections.defaultdict(list) 348 changed = joins = 0 349 for rel in tracked_sources(root): 350 text = read(root, rel) 351 if 'throw' not in text and 'SAL_THROW' not in text: 352 continue 353 spans = [] 354 for s in sites(rel, text): 355 counts[s['kind']] += 1 356 if s['kind'] == 'SPEC': 357 spans.append((s['start'], s['end'])) 358 elif s['kind'] != 'EMPTY': 359 review[s['kind']].append(s) 360 if spans and apply_changes: 361 new, j = rewrite(text, spans) 362 if new != text: 363 write(root, rel, new) 364 changed += 1 365 joins += j 366 for kind, n in counts.most_common(): 367 print('%-24s %7d' % (kind, n)) 368 if apply_changes: 369 print('\n%d files rewritten, %d lines closed up' % (changed, joins)) 370 print('\nEvery site below was rejected by the classifier. Read all of them:') 371 for kind in sorted(review): 372 for s in review[kind]: 373 print(' %-24s %s:%d <%s>' 374 % (kind, s['path'], s['line'], 375 ' '.join(s['content'].split())[:70])) 376 return 0 377 378 379def run_verify(root): 380 """Confirm a finished rewrite deleted only, and broke no nesting.""" 381 files = subprocess.run(['git', 'diff', '--name-only'], cwd=root, 382 capture_output=True, text=True, check=True).stdout.split() 383 files = [f for f in files if f.lower().endswith(SOURCE_EXTS)] 384 p = subprocess.Popen(['git', 'cat-file', '--batch'], cwd=root, 385 stdin=subprocess.PIPE, stdout=subprocess.PIPE) 386 387 # Feed from a thread: writing every request up front deadlocks once git's 388 # stdout fills the pipe while it is still blocked reading stdin. 389 def feed(): 390 for f in files: 391 p.stdin.write(('HEAD:%s\n' % f).encode()) 392 p.stdin.close() 393 threading.Thread(target=feed, daemon=True).start() 394 395 bad = [] 396 for rel in files: 397 size = int(p.stdout.readline().split()[2]) 398 old = p.stdout.read(size) 399 p.stdout.read(1) 400 with open(os.path.join(root, rel.replace('/', os.sep)), 'rb') as f: 401 new = f.read() 402 problems = ['%s %d->%d' % (c, old.count(c.encode()), new.count(c.encode())) 403 for c in '{};' 404 if old.count(c.encode()) != new.count(c.encode())] 405 if (old.count(b'(') - new.count(b'(')) != (old.count(b')') - new.count(b')')): 406 problems.append('parentheses removed unevenly') 407 if problems: 408 bad.append((rel, problems)) 409 print('checked %d files, %d with problems' % (len(files), len(bad))) 410 for rel, problems in bad: 411 print(' %s %s' % (rel, ', '.join(problems))) 412 return 1 if bad else 0 413 414 415def main(argv): 416 mode = argv[1] if len(argv) > 1 else '' 417 root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) 418 if '--root' in argv: 419 root = argv[argv.index('--root') + 1] 420 if mode == 'report': 421 return run_report(root, False) 422 if mode == 'apply': 423 return run_report(root, True) 424 if mode == 'verify': 425 return run_verify(root) 426 sys.stderr.write('usage: throwspec.py {report|apply|verify} [--root <dir>]\n' 427 ' report classify every site, change nothing\n' 428 ' apply delete the specifications with a type list\n' 429 ' verify check a finished rewrite against HEAD\n') 430 return 2 431 432 433if __name__ == '__main__': 434 sys.exit(main(sys.argv)) 435