xref: /trunk/test/odf-validate/cmp7.py (revision 0e16a31d78a652c56c5c0fc79c752f48b9048df9)

# usage: cmp7.py <old_run> <new_run> <mode:strict|ext> <out.json> [-v]
# Masked structural comparison of two corpus output sets (<run>/out/*).
# Layer 1: environment/noise masks only (RAND values, dates, ids, printer config, relative-href depth, auto-style numbering).
# Layer 2: additionally applies the INTENDED edits of the fixes under test to both sides, so the remaining diff = unexplained.
# Counts, per package, how often each fix's marker is present on the old side and on the new side (exercise map input).
import zipfile, sys, re, os, hashlib, difflib, json, collections
import xml.etree.ElementTree as ET
old, new, mode, outjson = sys.argv[1:5]
verbose = "-v" in sys.argv
NS = dict(style="urn:oasis:names:tc:opendocument:xmlns:style:1.0", office="urn:oasis:names:tc:opendocument:xmlns:office:1.0",
          text="urn:oasis:names:tc:opendocument:xmlns:text:1.0", draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0",
          table="urn:oasis:names:tc:opendocument:xmlns:table:1.0", form="urn:oasis:names:tc:opendocument:xmlns:form:1.0",
          chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0", meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0",
          fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0", svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0",
          xlink="http://www.w3.org/1999/xlink", config="urn:oasis:names:tc:opendocument:xmlns:config:1.0",
          dc="http://purl.org/dc/elements/1.1/")
def q(p, l): return "{%s}%s" % (NS[p], l)
STYLE_NAME = q("style", "name")

def mask_text(d):
    d = re.sub(rb'Pictures/[0-9A-F]{24,40}', b'Pictures/H', d)
    d = re.sub(rb'text:change-id="ct[0-9]+"', b'text:change-id="CT"', d)
    d = re.sub(rb'<text:(time|date)\b([^>]*)>[^<]*</text:\1>', rb'<text:\1\2>T</text:\1>', d)
    d = re.sub(rb'office:value="[-0-9.eE+]+"', b'office:value="#"', d)
    d = re.sub(rb'<text:p>[0-9.]+</text:p>', b'<text:p>#</text:p>', d)
    d = re.sub(rb'(text:time-value|text:date-value|text:time|text:date)="[^"]*"', rb'\1="T"', d)
    d = re.sub(rb'<meta:(creation-date|editing-duration)>[^<]*</meta:\1>', rb'<meta:\1>T</meta:\1>', d)
    d = re.sub(rb'<dc:date>[^<]*</dc:date>', b'<dc:date>T</dc:date>', d)
    d = re.sub(rb'(xml:id|text:id|form:id|text:continue-list)="[^"]*"', rb'\1="ID"', d)
    # environment: default printer of the host, relative links depend on the output directory depth
    d = re.sub(rb'(<config:config-item config:name="(?:PrinterName|PrinterSetup)"[^>]*>)[^<]*', rb'\1PRN', d)
    d = re.sub(rb'(xlink:href|form:image-data)="(?:\.\./)+', rb'\1="../*/', d)
    # host calendar date moved from 2026-10-22 (baseline run) to 2026-10-23 (TODAY()-style cells)
    d = re.sub(rb'2026-10-2[23]', b'DATE', d)
    d = re.sub(rb'10/2[23]/26', b'DATE', d)
    # fields.odt: weekday name length (Donnerstag vs Freitag) changes the document character count
    d = re.sub(rb'(character-count[^>]*?>|meta:character-count=")\d+', rb'\1N', d)
    return d
def isref(k):
    return k.endswith("style-name") or k.endswith("page-layout-name") or k.endswith("master-page-name") or k.endswith("class-names") or k.endswith("style-names")
def ser(e, m, defn=False, out=None):
    out = [] if out is None else out
    attrs = sorted((k, m.get(v, v) if isref(k) and not (defn and k == STYLE_NAME) else v) for k, v in e.attrib.items() if not (defn and k == STYLE_NAME))
    out.append("<" + e.tag + " " + " ".join(f'{k}="{v}"' for k, v in attrs) + ">" + (e.text or "").strip())
    for c in e: ser(c, m, False, out); out.append((c.tail or "").strip())
    out.append("</>")
    return "".join(out)

def canon(root, dedupe):
    autos = [c for ac in root.iter(q("office", "automatic-styles")) for c in ac if STYLE_NAME in c.attrib]
    m = {}
    for _ in range(4):
        nm = {}
        for c in autos: nm[c.get(STYLE_NAME)] = "S" + hashlib.md5(ser(c, m, True).encode()).hexdigest()[:10]
        if nm == m: break
        m = nm
    lines = []
    def walk(e, depth):
        if e.tag == q("office", "automatic-styles"):
            lines.append("<automatic-styles>")
            ss = sorted(ser(c, m, True) for c in e)
            if dedupe: ss = sorted(set(ss))
            for s in ss: lines.append("  " + s)
            lines.append("</automatic-styles>")
            return
        attrs = sorted((k, m.get(v, v) if isref(k) else v) for k, v in e.attrib.items())
        lines.append(" " * depth + "<" + e.tag + " " + " ".join(f'{k}="{v}"' for k, v in attrs) + ">" + ((e.text or "").strip()))
        for c in e: walk(c, depth + 1)
    walk(root, 0)
    return lines

def pct(v):
    m = re.fullmatch(r'(-?[0-9.]+)%', v); return float(m.group(1)) if m else None
def length_pos(v):
    m = re.fullmatch(r'(-?[0-9.]+)([a-z%]*)', v); return float(m.group(1)) > 0 if m else True

def normalise(root, cnt, strict, is_old):
    """Apply the intended edits of the fixes (both sides). cnt[key] counts how often an edit was needed on this side."""
    parent = {c: p for p in root.iter() for c in p}
    def bump(k): cnt[k] += 1
    def drop_attr(e, name): del e.attrib[name]
    for e in list(root.iter()):
        t = e.tag; a = e.attrib
        if t == q("draw", "connector") and q("svg", "viewBox") in a: drop_attr(e, q("svg", "viewBox")); bump("cc5a09edf8 viewBox")
        if t in (q("text", "a"), q("text", "section-source"), q("meta", "auto-reload")):
            k = {q("text", "a"): "4b1c1a1375 text:a xlink:type", q("text", "section-source"): "dd43dd10dc section-source xlink:type", q("meta", "auto-reload"): "60942d1a9c auto-reload xlink:type"}[t]
            if q("xlink", "type") in a: drop_attr(e, q("xlink", "type")); bump(k)
            elif q("xlink", "href") in a or t != q("text", "section-source"): bump(k + " MISSING")
        if t in (q("office", "document-content"), q("office", "document-styles")) and q("office", "version") in a:
            drop_attr(e, q("office", "version")); bump("c7615ea10d office:version present")
        elif t in (q("office", "document-content"), q("office", "document-styles")): bump("c7615ea10d office:version MISSING")
        if t == q("style", "page-layout-properties") and q("style", "layout-grid-standard-mode") in a:
            drop_attr(e, q("style", "layout-grid-standard-mode")); bump("f18a0e638f layout-grid-standard-mode in page layouts")
        if t == q("meta", "print-date"):
            if not re.match(r'\d{4,}-\d\d-\d\dT', (e.text or "")): parent[e].remove(e); bump("150358e458 print-date bogus")
        if t == q("form", "property") and q("office", "value") in a:
            if a[q("office", "value")] == "": bump("e64781e76d form office:value empty")
            a[q("office", "value")] = "#"
        for atn in (q("draw", "opacity"), q("draw", "image-opacity")):
            if atn in a:
                p = pct(a[atn])
                if p is not None and (p > 100 or p < 0): bump("d47ac291e9 opacity out of range"); a[atn] = "%g%%" % max(0, min(100, p))
        if q("fo", "font-size") in a and not length_pos(a[q("fo", "font-size")]): drop_attr(e, q("fo", "font-size")); bump("efbe32fc4d font-size<=0")
        for hn in ("hyphenation-push-char-count", "hyphenation-remain-char-count"):
            if q("fo", hn) in a and a[q("fo", hn)] == "0": drop_attr(e, q("fo", hn)); bump("fe667bb0b4 hyphenation 0")
        if q("office", "boolean-value") in a and a[q("office", "boolean-value")] not in ("true", "false"):
            del a[q("office", "boolean-value")]; a[q("office", "value-type")] = "float"; a[q("office", "value")] = "#"; bump("4250607123 boolean non-0/1")
        if q("table", "number-columns-spanned") in a and a[q("table", "number-columns-spanned")] == "0": drop_attr(e, q("table", "number-columns-spanned")); bump("895d935e38 colspan 0")
        if q("text", "anchor-page-number") in a and a[q("text", "anchor-page-number")] == "0": drop_attr(e, q("text", "anchor-page-number")); bump("b070ffe226 anchor-page 0")
        if t == q("text", "note-ref") and a.get(q("text", "reference-format")) in ("category-and-value", "caption", "value"):
            a[q("text", "reference-format")] = "text"; bump("9eca93128d note-ref format")
        if t == q("table", "dde-link"):
            tabs = [c for c in e if c.tag == q("table", "table")]
            if not tabs: bump("f8ad6c41e5 dde-link without table")
            else:
                tb = tabs[0]
                kids = [c.tag for c in tb]
                if kids == [q("table", "table-columns"), q("table", "table-rows")] and len(list(tb.iter(q("table", "table-cell")))) == 1 and not tb.attrib.get(q("table", "name")) is None and False: pass
                if kids == [q("table", "table-columns"), q("table", "table-rows")] and len(list(tb.iter(q("table", "table-cell")))) == 1 and not "".join(tb.itertext()).strip() and len(list(tb.iter())) == 6 and not any(x.attrib for x in tb.iter(q("table", "table-cell"))):
                    e.remove(tb); bump("f8ad6c41e5 dde-link wrapper table");
        if t == q("chart", "symbol-image") and q("xlink", "href") not in a: parent[e].remove(e); bump("d6bd426c78 empty symbol-image")
        if t in (q("table", "table-columns"), q("table", "table-rows")) and len(e) == 0 and parent.get(e) is not None and parent[e].tag == q("table", "table") and root.tag.endswith("document-content"):
            # chart internal data (empty wrappers)
            parent[e].remove(e); bump("26a3e0ceb0 empty chart table wrapper")
    if strict:
        norm_strict(root, cnt, parent)

PARA_FAM = {"paragraph", "text"}
def norm_strict(root, cnt, parent):
    chart_shapes_dropped = False
    for e in list(root.iter()):
        t = e.tag; a = e.attrib
        if t == q("style", "style") and a.get(q("style", "family")) == "paragraph":
            for c in list(e):
                if c.tag == q("style", "graphic-properties") and any(k.startswith("{%s}fill" % NS["draw"]) or k == q("draw", "opacity") or k.startswith("{%s}fill-" % NS["draw"]) for k in c.attrib):
                    e.remove(c); cnt["ef0a8aeea9 para draw:fill"] += 1
        if t in (q("style", "page-layout-properties"), q("style", "header-footer-properties")):
            for k in list(a):
                if k.startswith("{%s}fill" % NS["draw"]) or k == q("draw", "opacity") or k.startswith("{%s}opacity" % NS["draw"]):
                    del a[k]; cnt["e86e06dbff page/hf draw:fill"] += 1
        if t == q("style", "header-footer-properties"):
            for c in list(e):
                if c.tag == q("style", "background-image"):
                    if not c.attrib and len(c) == 0: e.remove(c); cnt["83b30299d0 hf background-image empty"] += 1
                    else: e.remove(c); cnt["83b30299d0 hf background-image with content"] += 1
        if q("style", "layout-grid-snap-to-characters") in a:
            a[q("style", "layout-grid-snap-to")] = a.pop(q("style", "layout-grid-snap-to-characters")); cnt["be94c9a190 grid-snap rename"] += 1
        if t == q("style", "table-row-properties") and q("style", "keep-together") in a: del a[q("style", "keep-together")]; cnt["58c72c10be row keep-together"] += 1
        if t in (q("draw", "object"), q("draw", "object-ole")):
            for c in list(e):
                if c.tag == q("text", "p"):
                    cnt["57b28c93a4 draw:object text:p " + ("with text" if "".join(c.itertext()).strip() else "empty")] += 1; e.remove(c)
        if t == q("form", "combobox") and q("form", "xforms-list-source") in a: del a[q("form", "xforms-list-source")]; cnt["1028eb24a1 combobox xforms-list-source"] += 1
        if t == q("chart", "chart"):
            for c in list(e):
                if c.tag.startswith("{%s}" % NS["draw"]): e.remove(c); cnt["edc67281bb chart additional shape " + c.tag.split("}")[1]] += 1; chart_shapes_dropped = True
        if t.startswith("{%s}" % NS["style"]) and t.endswith("-properties") and q("text", "enable-numbering") in a:
            del a[q("text", "enable-numbering")]; cnt["e9b28885e5 alien text:enable-numbering"] += 1
        if t == q("text", "changed-region"):
            for c in list(e):
                if c.tag in (q("text", "insertion"),) and e.find(q("text", "insertion")) is not c: pass
    # P3: successor change = an text:insertion/deletion nested in a changed-region that already carries a change; nested `office:change-info` with attribute form
    for e in list(root.iter(q("office", "change-info"))):
        if q("office", "chg-author") in e.attrib or q("office", "chg-date-time") in e.attrib:
            p = parent.get(e)
            if p is not None and p.tag in (q("text", "insertion"), q("text", "deletion"), q("text", "format-change")) and parent.get(p) is not None and len(p) == 1:
                parent[p].remove(p); cnt["c8b13970b2 redline successor change-info"] += 1
            elif p is not None: p.remove(e); cnt["c8b13970b2 redline successor change-info (other)"] += 1
    # empty property elements left behind by dropped alien attributes
    for e in list(root.iter()):
        for c in list(e):
            if c.tag.startswith("{%s}" % NS["style"]) and c.tag.endswith("-properties") and not c.attrib and len(c) == 0 and not (c.text or "").strip():
                e.remove(c); cnt["empty style:*-properties dropped"] += 1
    if chart_shapes_dropped:
        for _ in range(3):
            used = {v for e in root.iter() for k, v in e.attrib.items() if isref(k)}
            for ac in root.iter(q("office", "automatic-styles")):
                for st in list(ac):
                    if st.get(STYLE_NAME) is not None and st.get(STYLE_NAME) not in used and st.tag == q("style", "style") and st.get(q("style", "family")) in ("graphic", "paragraph"):
                        ac.remove(st); cnt["orphan auto style after chart shapes dropped"] += 1
    # drop empty paragraph auto styles and the references to them (merge / omission effect of row 1)
    empty = set()
    for ac in root.iter(q("office", "automatic-styles")):
        for s in list(ac):
            if s.tag == q("style", "style") and len(s) == 0 and set(s.attrib) <= {STYLE_NAME, q("style", "family"), q("style", "parent-style-name")} and q("style", "parent-style-name") not in s.attrib and s.get(q("style", "family")) in ("paragraph", "text", "graphic", "presentation"):
                empty.add(s.get(STYLE_NAME)); ac.remove(s)
    if empty:
        for e in root.iter():
            for k in list(e.attrib):
                if isref(k) and e.attrib[k] in empty and not (e.tag == q("style", "style")): del e.attrib[k]

def load(path, name, strict, dedupe, cnt, is_old, raw):
    z = zipfile.ZipFile(path); data = z.read(name)
    d = mask_text(data)
    try: root = ET.fromstring(d)
    except ET.ParseError: return [d.decode("utf8", "replace")], [d.decode("utf8", "replace")]
    raw_lines = canon(ET.fromstring(d), False)
    if raw: return raw_lines, None
    normalise(root, cnt, strict, is_old)
    return raw_lines, canon(root, dedupe)

strict = mode == "strict"
res = {}
tot = collections.Counter()
for f in sorted(os.listdir(old + "/out")):
    r = {"raw_diff_streams": [], "norm_diff_streams": [], "old": collections.Counter(), "new": collections.Counter()}
    if not os.path.exists(new + "/out/" + f): res[f] = {"missing_in_new": True}; continue
    pa, pb = old + "/out/" + f, new + "/out/" + f
    za, zb = zipfile.ZipFile(pa), zipfile.ZipFile(pb)
    nn = lambda z: sorted(re.sub(r'Pictures/[0-9A-F]{24,40}', 'Pictures/H', n) for n in z.namelist())
    if nn(za) != nn(zb): r["names_diff"] = sorted(set(nn(za)) ^ set(nn(zb)))
    for n in sorted(za.namelist()):
        if n.endswith("/") or not n.endswith(".xml") or n not in zb.namelist(): continue
        ca, cb = collections.Counter(), collections.Counter()
        ra, xa = load(pa, n, strict, strict, ca, True, False)
        rb, xb = load(pb, n, strict, strict, cb, False, False)
        r["old"].update(ca); r["new"].update(cb)
        if ra != rb: r["raw_diff_streams"].append(n)
        if xa != xb:
            r["norm_diff_streams"].append(n)
            diffl = [l[:300] for l in difflib.unified_diff(xa, xb, lineterm="", n=0) if not l.startswith(("---", "+++", "@@"))]
            r.setdefault("diff_lines", {})[n] = diffl[:40]
            if verbose:
                print("UNEXPLAINED", f, n)
                for l in diffl[:20]: print("    ", l)
    r["old"] = dict(r["old"]); r["new"] = dict(r["new"])
    res[f] = r
json.dump(res, open(outjson, "w"), indent=1)
nr = sum(1 for v in res.values() if v.get("raw_diff_streams")); nn_ = sum(1 for v in res.values() if v.get("norm_diff_streams") or v.get("names_diff") or v.get("missing_in_new"))
print(f"packages {len(res)}; differing after env masks: {nr}; differing after intended-edit normalisation: {nn_}")