1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: smoke13.py <tag> [--root DIR] [--port N] [--stall SECONDS] [--known FILE] 23# Import smoke test for ODF 1.3: reloads every document of a foreign corpus (tools13/make-foreign-corpus.py) saved in 24# 1.3 and in 1.3 extended mode next to its 1.2 twin (same input, same producer), dumps a fixed property set of each 25# (dump7.py, with the gate's stall watchdog) and compares the dumps of every twin pair (diff7.py). 26# Fails on a document that hangs or kills the office, on a twin pair that differs without an entry in --known 27# (TSV: name<TAB>reason, like gate-volatile.tsv), and when the built-in controls misbehave: an unchanged copy must 28# compare equal and a copy with one changed word must compare different. 29import argparse, collections, importlib.util, json, os, re, shutil, subprocess, sys, zipfile 30 31D = os.path.dirname(os.path.abspath(__file__)) 32spec = importlib.util.spec_from_file_location("gate", D + "/gate.py") 33gate = importlib.util.module_from_spec(spec) 34spec.loader.exec_module(gate) 35PAIRS = (("v12", "v13"), ("v12ext", "v13ext")) 36 37 38def mutate(src, dst): 39 """Copy of an ODF text package with one word appended to the first paragraph that has text; False if none.""" 40 zi, zo = zipfile.ZipFile(src), zipfile.ZipFile(dst, "w") 41 done = False 42 for it in zi.infolist(): 43 d = zi.read(it.filename) 44 if it.filename == "content.xml" and not done: 45 t = d.decode("utf-8") 46 t2, n = re.subn(r">([A-Za-z][^<>]{2,})</text:p>", r">\1 mutated</text:p>", t, count=1) 47 if n: 48 d, done = t2.encode("utf-8"), True 49 zo.writestr(it, d, compress_type=it.compress_type) 50 zo.close() 51 return done 52 53 54def dumps(src, dest, profile, port, stall, log): 55 r = gate.dump_dir(src, dest, profile, port, stall, log) 56 if "error" in r: 57 sys.exit("dump of %s failed: %s" % (src, r)) 58 return r 59 60 61def diff(old, new, out): 62 subprocess.run([sys.executable, "-I", os.path.join(D, "diff7.py"), old, new, out], capture_output=True, check=True) 63 return json.load(open(out)) 64 65 66def main(): 67 ap = argparse.ArgumentParser() 68 ap.add_argument("tag") 69 ap.add_argument("--root", default=os.path.join(D, "runs", "phase3", "foreign")) 70 ap.add_argument("--port", type=int, default=2794) 71 ap.add_argument("--stall", type=int, default=300) 72 ap.add_argument("--known") 73 ap.add_argument("--resume", action="store_true", help="reuse the dumps already in the run directory of this tag") 74 a = ap.parse_args() 75 run = os.path.join(D, "runs", "phase3", "smoke-" + a.tag) 76 if os.path.exists(run) and not a.resume: 77 sys.exit("%s exists, use a new tag or --resume" % run) 78 os.makedirs(run, exist_ok=True) 79 log = os.path.join(run, "dump.log") 80 known = {} 81 if a.known: 82 known = {l.split("\t")[0]: l.rstrip("\n").split("\t")[1] for l in open(a.known) if l.strip() and not l.startswith("#")} 83 problems, report = [], {} 84 stats = {} 85 for mode in sorted({m for p in PAIRS for m in p}): 86 src = os.path.join(a.root, mode) 87 if not os.path.isdir(src): 88 sys.exit("no %s" % src) 89 stats[mode] = dumps(src, os.path.join(run, "dump", mode), "smoke13", a.port, a.stall, log) 90 for mode, r in stats.items(): 91 errs = [] 92 for f in os.listdir(os.path.join(run, "dump", mode)): 93 if f.endswith(".json"): 94 d = json.load(open(os.path.join(run, "dump", mode, f))) 95 if isinstance(d, dict) and (d.get("error") or d.get("body_error")): 96 errs.append((f[:-5], d.get("error") or d.get("body_error"))) 97 report[mode] = {"documents": r["documents"], "stalls": r["stalls"], "errors": errs} 98 problems += ["%s: %s: %s" % (mode, n, e) for n, e in errs] 99 # controls: pick a text document that dumped with characters in it 100 base = None 101 for f in sorted(x for x in os.listdir(os.path.join(run, "dump", "v12")) if x.endswith(".json")): 102 d = json.load(open(os.path.join(run, "dump", "v12", f))) 103 if d.get("type") == "writer" and (d.get("body") or {}).get("chars", 0) > 20 and f.endswith(".odt.json"): 104 base = os.path.join(a.root, "v12", f[:-5]) 105 break 106 ctl = {"old": os.path.join(run, "ctl", "old"), "new": os.path.join(run, "ctl", "new")} 107 for p in ctl.values(): 108 os.makedirs(p, exist_ok=True) 109 if base is None: 110 problems.append("controls: no text document with characters in the corpus") 111 else: 112 shutil.copy2(base, ctl["old"] + "/ctl-same.odt") 113 shutil.copy2(base, ctl["new"] + "/ctl-same.odt") 114 shutil.copy2(base, ctl["old"] + "/ctl-mut.odt") 115 if not mutate(base, ctl["new"] + "/ctl-mut.odt"): 116 problems.append("controls: could not change a word in " + base) 117 for k in ctl: 118 dumps(ctl[k], os.path.join(run, "dump", "ctl-" + k), "smoke13", a.port, a.stall, log) 119 c = diff(os.path.join(run, "dump", "ctl-old"), os.path.join(run, "dump", "ctl-new"), os.path.join(run, "diff-ctl.json")) 120 if c["ctl-same.odt"]["ndiff"] != 0: 121 problems.append("controls: an unchanged copy compares different (%d)" % c["ctl-same.odt"]["ndiff"]) 122 if c["ctl-mut.odt"]["ndiff"] == 0: 123 problems.append("controls: a copy with a changed word compares equal (the comparison cannot fail)") 124 report["controls"] = {"base": os.path.basename(base), "same_ndiff": c["ctl-same.odt"]["ndiff"], "mutated_ndiff": c["ctl-mut.odt"]["ndiff"]} 125 for old, new in PAIRS: 126 dd = diff(os.path.join(run, "dump", old), os.path.join(run, "dump", new), os.path.join(run, "diff-%s-%s.json" % (old, new))) 127 ident = sum(1 for v in dd.values() if not v["ndiff"]) 128 differing = sorted(k for k, v in dd.items() if v["ndiff"]) 129 unexplained = [k for k in differing if k not in known] 130 report["%s vs %s" % (old, new)] = {"compared": len(dd), "identical": ident, "different": len(differing), "unexplained": unexplained} 131 problems += ["%s vs %s: %s differs (%d paths, first: %s)" % (old, new, k, dd[k]["ndiff"], dd[k]["diffs"][0][:3]) for k in unexplained] 132 json.dump({"report": report, "problems": problems}, open(os.path.join(run, "result.json"), "w"), indent=1, default=str) 133 for k, v in report.items(): 134 print(k, {x: (y if not isinstance(y, list) else len(y)) for x, y in v.items()}) 135 for p in problems: 136 print("PROBLEM", p) 137 print("smoke13:", "FAIL" if problems else "PASS") 138 sys.exit(1 if problems else 0) 139 140 141main() 142