1# usage: cmp7.py <old_run> <new_run> <mode:strict|ext> <out.json> [-v] 2# Masked structural comparison of two corpus output sets (<run>/out/*). 3# Layer 1: environment/noise masks only (RAND values, dates, ids, printer config, relative-href depth, auto-style numbering). 4# Layer 2: additionally applies the INTENDED edits of the fixes under test to both sides, so the remaining diff = unexplained. 5# Counts, per package, how often each fix's marker is present on the old side and on the new side (exercise map input). 6import zipfile, sys, re, os, hashlib, difflib, json, collections 7import xml.etree.ElementTree as ET 8old, new, mode, outjson = sys.argv[1:5] 9verbose = "-v" in sys.argv 10NS = dict(style="urn:oasis:names:tc:opendocument:xmlns:style:1.0", office="urn:oasis:names:tc:opendocument:xmlns:office:1.0", 11 text="urn:oasis:names:tc:opendocument:xmlns:text:1.0", draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0", 12 table="urn:oasis:names:tc:opendocument:xmlns:table:1.0", form="urn:oasis:names:tc:opendocument:xmlns:form:1.0", 13 chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0", meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0", 14 fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0", svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0", 15 xlink="http://www.w3.org/1999/xlink", config="urn:oasis:names:tc:opendocument:xmlns:config:1.0", 16 dc="http://purl.org/dc/elements/1.1/") 17def q(p, l): return "{%s}%s" % (NS[p], l) 18STYLE_NAME = q("style", "name") 19 20def mask_text(d): 21 d = re.sub(rb'Pictures/[0-9A-F]{24,40}', b'Pictures/H', d) 22 d = re.sub(rb'text:change-id="ct[0-9]+"', b'text:change-id="CT"', d) 23 d = re.sub(rb'<text:(time|date)\b([^>]*)>[^<]*</text:\1>', rb'<text:\1\2>T</text:\1>', d) 24 d = re.sub(rb'office:value="[-0-9.eE+]+"', b'office:value="#"', d) 25 d = re.sub(rb'<text:p>[0-9.]+</text:p>', b'<text:p>#</text:p>', d) 26 d = re.sub(rb'(text:time-value|text:date-value|text:time|text:date)="[^"]*"', rb'\1="T"', d) 27 d = re.sub(rb'<meta:(creation-date|editing-duration)>[^<]*</meta:\1>', rb'<meta:\1>T</meta:\1>', d) 28 d = re.sub(rb'<dc:date>[^<]*</dc:date>', b'<dc:date>T</dc:date>', d) 29 d = re.sub(rb'(xml:id|text:id|form:id|text:continue-list)="[^"]*"', rb'\1="ID"', d) 30 # environment: default printer of the host, relative links depend on the output directory depth 31 d = re.sub(rb'(<config:config-item config:name="(?:PrinterName|PrinterSetup)"[^>]*>)[^<]*', rb'\1PRN', d) 32 d = re.sub(rb'(xlink:href|form:image-data)="(?:\.\./)+', rb'\1="../*/', d) 33 # host calendar date moved from 2026-10-22 (baseline run) to 2026-10-23 (TODAY()-style cells) 34 d = re.sub(rb'2026-10-2[23]', b'DATE', d) 35 d = re.sub(rb'10/2[23]/26', b'DATE', d) 36 # fields.odt: weekday name length (Donnerstag vs Freitag) changes the document character count 37 d = re.sub(rb'(character-count[^>]*?>|meta:character-count=")\d+', rb'\1N', d) 38 return d 39def isref(k): 40 return k.endswith("style-name") or k.endswith("page-layout-name") or k.endswith("master-page-name") or k.endswith("class-names") or k.endswith("style-names") 41def ser(e, m, defn=False, out=None): 42 out = [] if out is None else out 43 attrs = sorted((k, m.get(v, v) if isref(k) and not (defn and k == STYLE_NAME) else v) for k, v in e.attrib.items() if not (defn and k == STYLE_NAME)) 44 out.append("<" + e.tag + " " + " ".join(f'{k}="{v}"' for k, v in attrs) + ">" + (e.text or "").strip()) 45 for c in e: ser(c, m, False, out); out.append((c.tail or "").strip()) 46 out.append("</>") 47 return "".join(out) 48 49def canon(root, dedupe): 50 autos = [c for ac in root.iter(q("office", "automatic-styles")) for c in ac if STYLE_NAME in c.attrib] 51 m = {} 52 for _ in range(4): 53 nm = {} 54 for c in autos: nm[c.get(STYLE_NAME)] = "S" + hashlib.md5(ser(c, m, True).encode()).hexdigest()[:10] 55 if nm == m: break 56 m = nm 57 lines = [] 58 def walk(e, depth): 59 if e.tag == q("office", "automatic-styles"): 60 lines.append("<automatic-styles>") 61 ss = sorted(ser(c, m, True) for c in e) 62 if dedupe: ss = sorted(set(ss)) 63 for s in ss: lines.append(" " + s) 64 lines.append("</automatic-styles>") 65 return 66 attrs = sorted((k, m.get(v, v) if isref(k) else v) for k, v in e.attrib.items()) 67 lines.append(" " * depth + "<" + e.tag + " " + " ".join(f'{k}="{v}"' for k, v in attrs) + ">" + ((e.text or "").strip())) 68 for c in e: walk(c, depth + 1) 69 walk(root, 0) 70 return lines 71 72def pct(v): 73 m = re.fullmatch(r'(-?[0-9.]+)%', v); return float(m.group(1)) if m else None 74def length_pos(v): 75 m = re.fullmatch(r'(-?[0-9.]+)([a-z%]*)', v); return float(m.group(1)) > 0 if m else True 76 77def normalise(root, cnt, strict, is_old): 78 """Apply the intended edits of the fixes (both sides). cnt[key] counts how often an edit was needed on this side.""" 79 parent = {c: p for p in root.iter() for c in p} 80 def bump(k): cnt[k] += 1 81 def drop_attr(e, name): del e.attrib[name] 82 for e in list(root.iter()): 83 t = e.tag; a = e.attrib 84 if t == q("draw", "connector") and q("svg", "viewBox") in a: drop_attr(e, q("svg", "viewBox")); bump("cc5a09edf8 viewBox") 85 if t in (q("text", "a"), q("text", "section-source"), q("meta", "auto-reload")): 86 k = {q("text", "a"): "4b1c1a1375 text:a xlink:type", q("text", "section-source"): "dd43dd10dc section-source xlink:type", q("meta", "auto-reload"): "60942d1a9c auto-reload xlink:type"}[t] 87 if q("xlink", "type") in a: drop_attr(e, q("xlink", "type")); bump(k) 88 elif q("xlink", "href") in a or t != q("text", "section-source"): bump(k + " MISSING") 89 if t in (q("office", "document-content"), q("office", "document-styles")) and q("office", "version") in a: 90 drop_attr(e, q("office", "version")); bump("c7615ea10d office:version present") 91 elif t in (q("office", "document-content"), q("office", "document-styles")): bump("c7615ea10d office:version MISSING") 92 if t == q("style", "page-layout-properties") and q("style", "layout-grid-standard-mode") in a: 93 drop_attr(e, q("style", "layout-grid-standard-mode")); bump("f18a0e638f layout-grid-standard-mode in page layouts") 94 if t == q("meta", "print-date"): 95 if not re.match(r'\d{4,}-\d\d-\d\dT', (e.text or "")): parent[e].remove(e); bump("150358e458 print-date bogus") 96 if t == q("form", "property") and q("office", "value") in a: 97 if a[q("office", "value")] == "": bump("e64781e76d form office:value empty") 98 a[q("office", "value")] = "#" 99 for atn in (q("draw", "opacity"), q("draw", "image-opacity")): 100 if atn in a: 101 p = pct(a[atn]) 102 if p is not None and (p > 100 or p < 0): bump("d47ac291e9 opacity out of range"); a[atn] = "%g%%" % max(0, min(100, p)) 103 if q("fo", "font-size") in a and not length_pos(a[q("fo", "font-size")]): drop_attr(e, q("fo", "font-size")); bump("efbe32fc4d font-size<=0") 104 for hn in ("hyphenation-push-char-count", "hyphenation-remain-char-count"): 105 if q("fo", hn) in a and a[q("fo", hn)] == "0": drop_attr(e, q("fo", hn)); bump("fe667bb0b4 hyphenation 0") 106 if q("office", "boolean-value") in a and a[q("office", "boolean-value")] not in ("true", "false"): 107 del a[q("office", "boolean-value")]; a[q("office", "value-type")] = "float"; a[q("office", "value")] = "#"; bump("4250607123 boolean non-0/1") 108 if q("table", "number-columns-spanned") in a and a[q("table", "number-columns-spanned")] == "0": drop_attr(e, q("table", "number-columns-spanned")); bump("895d935e38 colspan 0") 109 if q("text", "anchor-page-number") in a and a[q("text", "anchor-page-number")] == "0": drop_attr(e, q("text", "anchor-page-number")); bump("b070ffe226 anchor-page 0") 110 if t == q("text", "note-ref") and a.get(q("text", "reference-format")) in ("category-and-value", "caption", "value"): 111 a[q("text", "reference-format")] = "text"; bump("9eca93128d note-ref format") 112 if t == q("table", "dde-link"): 113 tabs = [c for c in e if c.tag == q("table", "table")] 114 if not tabs: bump("f8ad6c41e5 dde-link without table") 115 else: 116 tb = tabs[0] 117 kids = [c.tag for c in tb] 118 if kids == [q("table", "table-columns"), q("table", "table-rows")] and len(list(tb.iter(q("table", "table-cell")))) == 1 and not tb.attrib.get(q("table", "name")) is None and False: pass 119 if kids == [q("table", "table-columns"), q("table", "table-rows")] and len(list(tb.iter(q("table", "table-cell")))) == 1 and not "".join(tb.itertext()).strip() and len(list(tb.iter())) == 6 and not any(x.attrib for x in tb.iter(q("table", "table-cell"))): 120 e.remove(tb); bump("f8ad6c41e5 dde-link wrapper table"); 121 if t == q("chart", "symbol-image") and q("xlink", "href") not in a: parent[e].remove(e); bump("d6bd426c78 empty symbol-image") 122 if t in (q("table", "table-columns"), q("table", "table-rows")) and len(e) == 0 and parent.get(e) is not None and parent[e].tag == q("table", "table") and root.tag.endswith("document-content"): 123 # chart internal data (empty wrappers) 124 parent[e].remove(e); bump("26a3e0ceb0 empty chart table wrapper") 125 if strict: 126 norm_strict(root, cnt, parent) 127 128PARA_FAM = {"paragraph", "text"} 129def norm_strict(root, cnt, parent): 130 chart_shapes_dropped = False 131 for e in list(root.iter()): 132 t = e.tag; a = e.attrib 133 if t == q("style", "style") and a.get(q("style", "family")) == "paragraph": 134 for c in list(e): 135 if c.tag == q("style", "graphic-properties") and any(k.startswith("{%s}fill" % NS["draw"]) or k == q("draw", "opacity") or k.startswith("{%s}fill-" % NS["draw"]) for k in c.attrib): 136 e.remove(c); cnt["ef0a8aeea9 para draw:fill"] += 1 137 if t in (q("style", "page-layout-properties"), q("style", "header-footer-properties")): 138 for k in list(a): 139 if k.startswith("{%s}fill" % NS["draw"]) or k == q("draw", "opacity") or k.startswith("{%s}opacity" % NS["draw"]): 140 del a[k]; cnt["e86e06dbff page/hf draw:fill"] += 1 141 if t == q("style", "header-footer-properties"): 142 for c in list(e): 143 if c.tag == q("style", "background-image"): 144 if not c.attrib and len(c) == 0: e.remove(c); cnt["83b30299d0 hf background-image empty"] += 1 145 else: e.remove(c); cnt["83b30299d0 hf background-image with content"] += 1 146 if q("style", "layout-grid-snap-to-characters") in a: 147 a[q("style", "layout-grid-snap-to")] = a.pop(q("style", "layout-grid-snap-to-characters")); cnt["be94c9a190 grid-snap rename"] += 1 148 if t == q("style", "table-row-properties") and q("style", "keep-together") in a: del a[q("style", "keep-together")]; cnt["58c72c10be row keep-together"] += 1 149 if t in (q("draw", "object"), q("draw", "object-ole")): 150 for c in list(e): 151 if c.tag == q("text", "p"): 152 cnt["57b28c93a4 draw:object text:p " + ("with text" if "".join(c.itertext()).strip() else "empty")] += 1; e.remove(c) 153 if t == q("form", "combobox") and q("form", "xforms-list-source") in a: del a[q("form", "xforms-list-source")]; cnt["1028eb24a1 combobox xforms-list-source"] += 1 154 if t == q("chart", "chart"): 155 for c in list(e): 156 if c.tag.startswith("{%s}" % NS["draw"]): e.remove(c); cnt["edc67281bb chart additional shape " + c.tag.split("}")[1]] += 1; chart_shapes_dropped = True 157 if t.startswith("{%s}" % NS["style"]) and t.endswith("-properties") and q("text", "enable-numbering") in a: 158 del a[q("text", "enable-numbering")]; cnt["e9b28885e5 alien text:enable-numbering"] += 1 159 if t == q("text", "changed-region"): 160 for c in list(e): 161 if c.tag in (q("text", "insertion"),) and e.find(q("text", "insertion")) is not c: pass 162 # P3: successor change = an text:insertion/deletion nested in a changed-region that already carries a change; nested `office:change-info` with attribute form 163 for e in list(root.iter(q("office", "change-info"))): 164 if q("office", "chg-author") in e.attrib or q("office", "chg-date-time") in e.attrib: 165 p = parent.get(e) 166 if p is not None and p.tag in (q("text", "insertion"), q("text", "deletion"), q("text", "format-change")) and parent.get(p) is not None and len(p) == 1: 167 parent[p].remove(p); cnt["c8b13970b2 redline successor change-info"] += 1 168 elif p is not None: p.remove(e); cnt["c8b13970b2 redline successor change-info (other)"] += 1 169 # empty property elements left behind by dropped alien attributes 170 for e in list(root.iter()): 171 for c in list(e): 172 if c.tag.startswith("{%s}" % NS["style"]) and c.tag.endswith("-properties") and not c.attrib and len(c) == 0 and not (c.text or "").strip(): 173 e.remove(c); cnt["empty style:*-properties dropped"] += 1 174 if chart_shapes_dropped: 175 for _ in range(3): 176 used = {v for e in root.iter() for k, v in e.attrib.items() if isref(k)} 177 for ac in root.iter(q("office", "automatic-styles")): 178 for st in list(ac): 179 if st.get(STYLE_NAME) is not None and st.get(STYLE_NAME) not in used and st.tag == q("style", "style") and st.get(q("style", "family")) in ("graphic", "paragraph"): 180 ac.remove(st); cnt["orphan auto style after chart shapes dropped"] += 1 181 # drop empty paragraph auto styles and the references to them (merge / omission effect of row 1) 182 empty = set() 183 for ac in root.iter(q("office", "automatic-styles")): 184 for s in list(ac): 185 if s.tag == q("style", "style") and len(s) == 0 and set(s.attrib) <= {STYLE_NAME, q("style", "family"), q("style", "parent-style-name")} and q("style", "parent-style-name") not in s.attrib and s.get(q("style", "family")) in ("paragraph", "text", "graphic", "presentation"): 186 empty.add(s.get(STYLE_NAME)); ac.remove(s) 187 if empty: 188 for e in root.iter(): 189 for k in list(e.attrib): 190 if isref(k) and e.attrib[k] in empty and not (e.tag == q("style", "style")): del e.attrib[k] 191 192def load(path, name, strict, dedupe, cnt, is_old, raw): 193 z = zipfile.ZipFile(path); data = z.read(name) 194 d = mask_text(data) 195 try: root = ET.fromstring(d) 196 except ET.ParseError: return [d.decode("utf8", "replace")], [d.decode("utf8", "replace")] 197 raw_lines = canon(ET.fromstring(d), False) 198 if raw: return raw_lines, None 199 normalise(root, cnt, strict, is_old) 200 return raw_lines, canon(root, dedupe) 201 202strict = mode == "strict" 203res = {} 204tot = collections.Counter() 205for f in sorted(os.listdir(old + "/out")): 206 r = {"raw_diff_streams": [], "norm_diff_streams": [], "old": collections.Counter(), "new": collections.Counter()} 207 if not os.path.exists(new + "/out/" + f): res[f] = {"missing_in_new": True}; continue 208 pa, pb = old + "/out/" + f, new + "/out/" + f 209 za, zb = zipfile.ZipFile(pa), zipfile.ZipFile(pb) 210 nn = lambda z: sorted(re.sub(r'Pictures/[0-9A-F]{24,40}', 'Pictures/H', n) for n in z.namelist()) 211 if nn(za) != nn(zb): r["names_diff"] = sorted(set(nn(za)) ^ set(nn(zb))) 212 for n in sorted(za.namelist()): 213 if n.endswith("/") or not n.endswith(".xml") or n not in zb.namelist(): continue 214 ca, cb = collections.Counter(), collections.Counter() 215 ra, xa = load(pa, n, strict, strict, ca, True, False) 216 rb, xb = load(pb, n, strict, strict, cb, False, False) 217 r["old"].update(ca); r["new"].update(cb) 218 if ra != rb: r["raw_diff_streams"].append(n) 219 if xa != xb: 220 r["norm_diff_streams"].append(n) 221 diffl = [l[:300] for l in difflib.unified_diff(xa, xb, lineterm="", n=0) if not l.startswith(("---", "+++", "@@"))] 222 r.setdefault("diff_lines", {})[n] = diffl[:40] 223 if verbose: 224 print("UNEXPLAINED", f, n) 225 for l in diffl[:20]: print(" ", l) 226 r["old"] = dict(r["old"]); r["new"] = dict(r["new"]) 227 res[f] = r 228json.dump(res, open(outjson, "w"), indent=1) 229nr = sum(1 for v in res.values() if v.get("raw_diff_streams")); nn_ = sum(1 for v in res.values() if v.get("norm_diff_streams") or v.get("names_diff") or v.get("missing_in_new")) 230print(f"packages {len(res)}; differing after env masks: {nr}; differing after intended-edit normalisation: {nn_}") 231