1################################################################ 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20################################################################ 21 22# usage: diff7.py <old_dump_dir> <new_dump_dir> <out.json> 23# Flattens the per-document reload dumps and lists every differing path with old/new value. 24import sys, os, json, collections 25IGN = ("props.creation_date", "props.gen", "props.edit_cycles") # none expected to differ; creation_date is kept from the input 26def flat(x, p=""): 27 if isinstance(x, dict): 28 for k in sorted(x): yield from flat(x[k], p + "." + k if p else k) 29 elif isinstance(x, list): 30 if len(x) > 40 and all(not isinstance(i, (dict, list)) for i in x): yield p, x 31 else: 32 for i, e in enumerate(x): yield from flat(e, f"{p}[{i}]") 33 else: yield p, x 34 35 36def diff_dirs(old, new): 37 """Returns {document: {type, errors, ndiff, diffs, nprops}} for two dump directories.""" 38 res = {} 39 names = sorted(f for f in os.listdir(old) if f.endswith(".json")) 40 for f in names: 41 a = json.load(open(os.path.join(old, f))) 42 b = json.load(open(os.path.join(new, f))) if os.path.exists(os.path.join(new, f)) else {"error": "MISSING"} 43 fa, fb = dict(flat(a)), dict(flat(b)) 44 d = [(k, fa.get(k, "<absent>"), fb.get(k, "<absent>")) for k in sorted(set(fa) | set(fb)) if fa.get(k, "<absent>") != fb.get(k, "<absent>")] 45 res[f[:-5]] = {"type": a.get("type") or b.get("type") or ("error" if "error" in a else "?"), "errors": [a.get("error"), b.get("error"), a.get("body_error"), b.get("body_error")], "ndiff": len(d), "diffs": d, "nprops": len(fa)} 46 return res 47 48 49def main(argv=None): 50 old, new, out = sys.argv[1:4] if argv is None else argv 51 res = diff_dirs(old, new) 52 json.dump(res, open(out, "w"), indent=0, default=str) 53 c = collections.Counter(); ident = collections.Counter(); diff = collections.Counter() 54 for k, v in res.items(): 55 c[v["type"]] += 1 56 (diff if v["ndiff"] else ident)[v["type"]] += 1 57 for t in sorted(c): print(f"{t:8s} docs={c[t]:3d} identical={ident[t]:3d} different={diff[t]:3d}") 58 print("total", len(res), "identical", sum(ident.values()), "different", sum(diff.values())) 59 60 61if __name__ == "__main__": 62 main() 63