1*0e16a31dSJim Jagielski# usage: diff7.py <old_dump_dir> <new_dump_dir> <out.json> 2*0e16a31dSJim Jagielski# Flattens the per-document reload dumps and lists every differing path with old/new value. 3*0e16a31dSJim Jagielskiimport sys, os, json, collections 4*0e16a31dSJim Jagielskiold, new, out = sys.argv[1:4] 5*0e16a31dSJim JagielskiIGN = ("props.creation_date", "props.gen", "props.edit_cycles") # none expected to differ; creation_date is kept from the input 6*0e16a31dSJim Jagielskidef flat(x, p=""): 7*0e16a31dSJim Jagielski if isinstance(x, dict): 8*0e16a31dSJim Jagielski for k in sorted(x): yield from flat(x[k], p + "." + k if p else k) 9*0e16a31dSJim Jagielski elif isinstance(x, list): 10*0e16a31dSJim Jagielski if len(x) > 40 and all(not isinstance(i, (dict, list)) for i in x): yield p, x 11*0e16a31dSJim Jagielski else: 12*0e16a31dSJim Jagielski for i, e in enumerate(x): yield from flat(e, f"{p}[{i}]") 13*0e16a31dSJim Jagielski else: yield p, x 14*0e16a31dSJim Jagielskires = {} 15*0e16a31dSJim Jagielskinames = sorted(f for f in os.listdir(old) if f.endswith(".json")) 16*0e16a31dSJim Jagielskifor f in names: 17*0e16a31dSJim Jagielski a = json.load(open(os.path.join(old, f))) 18*0e16a31dSJim Jagielski b = json.load(open(os.path.join(new, f))) if os.path.exists(os.path.join(new, f)) else {"error": "MISSING"} 19*0e16a31dSJim Jagielski fa, fb = dict(flat(a)), dict(flat(b)) 20*0e16a31dSJim Jagielski d = [(k, fa.get(k, "<absent>"), fb.get(k, "<absent>")) for k in sorted(set(fa) | set(fb)) if fa.get(k, "<absent>") != fb.get(k, "<absent>")] 21*0e16a31dSJim Jagielski res[f[:-5]] = {"type": a.get("type") or b.get("type") or ("error" if "error" in a else "?"), "errors": [a.get("error"), b.get("error"), a.get("body_error"), b.get("body_error")], "ndiff": len(d), "diffs": d, "nprops": len(fa)} 22*0e16a31dSJim Jagielskijson.dump(res, open(out, "w"), indent=0, default=str) 23*0e16a31dSJim Jagielskic = collections.Counter(); ident = collections.Counter(); diff = collections.Counter() 24*0e16a31dSJim Jagielskifor k, v in res.items(): 25*0e16a31dSJim Jagielski c[v["type"]] += 1 26*0e16a31dSJim Jagielski (diff if v["ndiff"] else ident)[v["type"]] += 1 27*0e16a31dSJim Jagielskifor t in sorted(c): print(f"{t:8s} docs={c[t]:3d} identical={ident[t]:3d} different={diff[t]:3d}") 28*0e16a31dSJim Jagielskiprint("total", len(res), "identical", sum(ident.values()), "different", sum(diff.values())) 29