1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: python -m aootest.checkers.crosscheck <outdir> <val.json> <cross.json> (needs AOO_ODFVALIDATOR and AOO_SCHEMA_DIR; tests call check()) 23# Validates every package in <outdir> with the Apache ODF Toolkit odfvalidator (fetch-odfvalidator.sh; MSV engine, 24# the OASIS schemas given explicitly) in conformance mode (-c, compared with validate.py's "12") and extended 25# conformance mode (-e, compared with "12x"), and compares per stream with the jing results in <val.json>. 26# Error counts are not comparable between engines (different error recovery); the comparison is whether a stream 27# has any error and where its first error is. 28import sys, os, re, json, subprocess, collections 29from xml.sax.saxutils import escape 30 31STREAMS = {"content.xml", "styles.xml", "meta.xml", "settings.xml", "manifest.xml"} 32LINE = re.compile(r"^(?P<path>/.*?)(?:\[(?P<l>\d+),(?P<c>\d+)\])?:\s+(?P<lvl>Error|Fatal|Warning|Info):\s*(?P<msg>.*)$") 33# odfvalidator needs a MathML 2 schema for embedded formula objects and its jar has none; this accepts anything, 34# which is what validate.py does too (it skips MathML). 35ANY = ('<grammar xmlns="http://relaxng.org/ns/structure/1.0"><start><ref name="e"/></start>' 36 '<define name="e"><element><anyName/><interleave><zeroOrMore><attribute><anyName/></attribute></zeroOrMore>' 37 '<zeroOrMore><choice><ref name="e"/><text/></choice></zeroOrMore></interleave></element></define></grammar>') 38 39 40def run(flag, outdir, pkgs, work, tag, java, jar, schema_dir): 41 """Returns (events, crashed packages); events are (package, entry, line, col, level, message). 42 Raises if odfvalidator stops without a completion line for every package: its own failures are silent otherwise.""" 43 open(work + "/any.rng", "w").write(ANY) 44 events, crashed, todo, round_ = [], [], list(pkgs), 0 45 while todo: 46 round_ += 1 47 cfg = work + "/%s.%d.xml" % (tag, round_) 48 entries = {"schema": os.path.join(schema_dir, "OpenDocument-v1.2-os-schema.rng"), 49 "manifest-schema": os.path.join(schema_dir, "OpenDocument-v1.2-os-manifest-schema.rng"), 50 "dsig-schema": os.path.join(schema_dir, "OpenDocument-v1.2-os-dsig-schema.rng"), "mathml2-schema": work + "/any.rng"} 51 for i, p in enumerate(todo): 52 entries["path%06d" % i] = os.path.join(outdir, p) 53 open(cfg, "w").write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">\n<properties>\n' 54 + "".join('<entry key="%s">%s</entry>\n' % (k, escape(v)) for k, v in entries.items()) + "</properties>\n") 55 r = subprocess.run([java, "-jar", jar, "-v", flag, "-C", cfg], capture_output=True, text=True, timeout=3600) 56 text = r.stdout + r.stderr 57 done = set() 58 prefix = os.path.join(outdir, "") 59 for line in text.splitlines(): 60 m = LINE.match(line) 61 if not m or not m.group("path").startswith(prefix): 62 continue 63 rest = m.group("path")[len(prefix):] 64 pkg, _, entry = rest.partition("/") 65 if m.group("lvl") == "Info" and entry == "" and re.match(r"no errors|\d+ errors", m.group("msg")): 66 done.add(pkg) 67 if m.group("lvl") != "Info": 68 events.append((pkg, entry, int(m.group("l") or 0), int(m.group("c") or 0), m.group("lvl"), m.group("msg"))) 69 pending = [p for p in todo if p not in done] 70 if "Exception in thread" in text and pending: 71 crashed.append(pending[0]) 72 todo = todo[todo.index(pending[0]) + 1:] 73 elif pending: 74 raise RuntimeError("crosscheck: odfvalidator %s did not finish %d of %d packages (first: %s); output tail:\n%s" 75 % (flag, len(pending), len(todo), pending[0], "\n".join(text.splitlines()[-4:]))) 76 else: 77 todo = [] 78 return events, crashed 79 80 81def compare(events, jing_errors, pkgs): 82 odf = collections.defaultdict(list) 83 for pkg, entry, l, c, lvl, msg in events: 84 if lvl in ("Error", "Fatal") and entry.rsplit("/", 1)[-1] in STREAMS: 85 odf[pkg + "/" + entry].append((l, c, msg)) 86 jing = collections.defaultdict(list) 87 for f, l, c, msg in jing_errors: 88 if f != "?": 89 jing[f].append((l, c, msg)) 90 both, same_pos, only_jing, only_odf = 0, 0, [], [] 91 diffpos = [] 92 for k in sorted(set(odf) | set(jing)): 93 if k in odf and k in jing: 94 both += 1 95 if min(odf[k])[:2] == min(jing[k])[:2]: 96 same_pos += 1 97 else: 98 diffpos.append([k, min(jing[k])[:2], min(odf[k])[:2], min(jing[k])[2][:100], min(odf[k])[2][:100]]) 99 elif k in jing: 100 only_jing.append([k, min(jing[k])[:2], min(jing[k])[2][:140]]) 101 else: 102 only_odf.append([k, min(odf[k])[:2], min(odf[k])[2][:140]]) 103 pkgmsgs = collections.Counter() 104 for pkg, entry, l, c, lvl, msg in events: 105 if entry.rsplit("/", 1)[-1] not in STREAMS and entry.rsplit("/", 1)[-1] != "": 106 pkgmsgs["%s %s: %s" % (lvl, entry.rsplit("/", 1)[-1], re.sub(r"\d+", "N", msg)[:120])] += 1 107 elif entry == "": 108 pkgmsgs["%s (package): %s" % (lvl, re.sub(r"\d+", "N", msg)[:120])] += 1 109 return {"odfvalidator_error_streams": sorted(odf), "streams_both_with_errors": both, "first_error_same_position": same_pos, "only_jing": only_jing, "only_odfvalidator": only_odf, 110 "different_first_error": diffpos, "package_level": dict(pkgmsgs)} 111 112 113def check(outdir, val, workdir, jar, schema_dir, java="java"): 114 """Returns the result that main() writes as <cross.json>. `val` is the validate.check() result.""" 115 outdir = os.path.abspath(outdir) 116 if not os.path.isfile(jar): 117 raise FileNotFoundError(jar) 118 work = os.path.abspath(workdir) 119 os.makedirs(work, exist_ok=True) 120 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".") and p not in val.get("bad_zip", [])) 121 res = {} 122 for flag, jv, name in (("-c", "12", "conformance"), ("-e", "12x", "extended")): 123 events, crashed = run(flag, outdir, pkgs, work, name, java, jar, schema_dir) 124 res[name] = compare(events, val["errors"][jv], pkgs) 125 res[name]["odfvalidator_crashed_on"] = crashed 126 return res 127 128 129def main(argv=None): 130 outdir, valjson, crossjson = sys.argv[1:4] if argv is None else argv 131 res = check(outdir, json.load(open(valjson)), crossjson[:-5] + ".d", os.environ["AOO_ODFVALIDATOR"], os.environ["AOO_SCHEMA_DIR"], 132 os.environ.get("AOO_JAVA", "java")) 133 for (flag, jv, name) in (("-c", "12", "conformance"), ("-e", "12x", "extended")): 134 r = res[name] 135 print(f"{name} (odfvalidator {flag} vs jing {jv}): both {r['streams_both_with_errors']} (same first position {r['first_error_same_position']}), " 136 f"only jing {len(r['only_jing'])}, only odfvalidator {len(r['only_odfvalidator'])}, odfvalidator crashed on {len(r['odfvalidator_crashed_on'])} packages") 137 for k, v in sorted(r["package_level"].items()): 138 print(f" {v:5d} {k}") 139 json.dump(res, open(crossjson, "w"), indent=1) 140 141 142if __name__ == "__main__": 143 main() 144