1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: crosscheck.py <outdir> <val.json> <cross.json> 23# Validates every package in <outdir> with the Apache ODF Toolkit odfvalidator (fetch-odfvalidator.sh; MSV engine, 24# the OASIS schemas given explicitly) in conformance mode (-c, compared with validate.py's "12") and extended 25# conformance mode (-e, compared with "12x"), and compares per stream with the jing results in <val.json>. 26# Error counts are not comparable between engines (different error recovery); the comparison is whether a stream 27# has any error and where its first error is. 28import sys, os, re, json, subprocess, collections 29from xml.sax.saxutils import escape 30 31D = os.path.dirname(os.path.abspath(__file__)) 32JAR = os.environ.get("ODFV_ODFVALIDATOR", D + "/tools/odfvalidator-1.1.8-incubating-jar-with-dependencies.jar") 33JAVA = os.environ.get("ODFV_JAVA", "java") 34SCH = D + "/schema/" 35STREAMS = {"content.xml", "styles.xml", "meta.xml", "settings.xml", "manifest.xml"} 36LINE = re.compile(r"^(?P<path>/.*?)(?:\[(?P<l>\d+),(?P<c>\d+)\])?:\s+(?P<lvl>Error|Fatal|Warning|Info):\s*(?P<msg>.*)$") 37# odfvalidator needs a MathML 2 schema for embedded formula objects and its jar has none; this accepts anything, 38# which is what validate.py does too (it skips MathML). 39ANY = ('<grammar xmlns="http://relaxng.org/ns/structure/1.0"><start><ref name="e"/></start>' 40 '<define name="e"><element><anyName/><interleave><zeroOrMore><attribute><anyName/></attribute></zeroOrMore>' 41 '<zeroOrMore><choice><ref name="e"/><text/></choice></zeroOrMore></interleave></element></define></grammar>') 42 43 44def run(flag, outdir, pkgs, work, tag): 45 """Returns (events, crashed packages); events are (package, entry, line, col, level, message). 46 Exits if odfvalidator stops without a completion line for every package: its own failures are silent otherwise.""" 47 open(work + "/any.rng", "w").write(ANY) 48 events, crashed, todo, round_ = [], [], list(pkgs), 0 49 while todo: 50 round_ += 1 51 cfg = work + "/%s.%d.xml" % (tag, round_) 52 entries = {"schema": SCH + "OpenDocument-v1.2-os-schema.rng", "manifest-schema": SCH + "OpenDocument-v1.2-os-manifest-schema.rng", 53 "dsig-schema": SCH + "OpenDocument-v1.2-os-dsig-schema.rng", "mathml2-schema": work + "/any.rng"} 54 for i, p in enumerate(todo): 55 entries["path%06d" % i] = os.path.join(outdir, p) 56 open(cfg, "w").write('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">\n<properties>\n' 57 + "".join('<entry key="%s">%s</entry>\n' % (k, escape(v)) for k, v in entries.items()) + "</properties>\n") 58 r = subprocess.run([JAVA, "-jar", JAR, "-v", flag, "-C", cfg], capture_output=True, text=True, timeout=3600) 59 text = r.stdout + r.stderr 60 done = set() 61 prefix = os.path.join(outdir, "") 62 for line in text.splitlines(): 63 m = LINE.match(line) 64 if not m or not m.group("path").startswith(prefix): 65 continue 66 rest = m.group("path")[len(prefix):] 67 pkg, _, entry = rest.partition("/") 68 if m.group("lvl") == "Info" and entry == "" and re.match(r"no errors|\d+ errors", m.group("msg")): 69 done.add(pkg) 70 if m.group("lvl") != "Info": 71 events.append((pkg, entry, int(m.group("l") or 0), int(m.group("c") or 0), m.group("lvl"), m.group("msg"))) 72 pending = [p for p in todo if p not in done] 73 if "Exception in thread" in text and pending: 74 crashed.append(pending[0]) 75 todo = todo[todo.index(pending[0]) + 1:] 76 elif pending: 77 sys.exit("crosscheck.py: odfvalidator %s did not finish %d of %d packages (first: %s); output tail:\n%s" 78 % (flag, len(pending), len(todo), pending[0], "\n".join(text.splitlines()[-4:]))) 79 else: 80 todo = [] 81 return events, crashed 82 83 84def compare(events, jing_errors, pkgs): 85 odf = collections.defaultdict(list) 86 for pkg, entry, l, c, lvl, msg in events: 87 if lvl in ("Error", "Fatal") and entry.rsplit("/", 1)[-1] in STREAMS: 88 odf[pkg + "/" + entry].append((l, c, msg)) 89 jing = collections.defaultdict(list) 90 for f, l, c, msg in jing_errors: 91 if f != "?": 92 jing[f].append((l, c, msg)) 93 both, same_pos, only_jing, only_odf = 0, 0, [], [] 94 diffpos = [] 95 for k in sorted(set(odf) | set(jing)): 96 if k in odf and k in jing: 97 both += 1 98 if min(odf[k])[:2] == min(jing[k])[:2]: 99 same_pos += 1 100 else: 101 diffpos.append([k, min(jing[k])[:2], min(odf[k])[:2], min(jing[k])[2][:100], min(odf[k])[2][:100]]) 102 elif k in jing: 103 only_jing.append([k, min(jing[k])[:2], min(jing[k])[2][:140]]) 104 else: 105 only_odf.append([k, min(odf[k])[:2], min(odf[k])[2][:140]]) 106 pkgmsgs = collections.Counter() 107 for pkg, entry, l, c, lvl, msg in events: 108 if entry.rsplit("/", 1)[-1] not in STREAMS and entry.rsplit("/", 1)[-1] != "": 109 pkgmsgs["%s %s: %s" % (lvl, entry.rsplit("/", 1)[-1], re.sub(r"\d+", "N", msg)[:120])] += 1 110 elif entry == "": 111 pkgmsgs["%s (package): %s" % (lvl, re.sub(r"\d+", "N", msg)[:120])] += 1 112 return {"odfvalidator_error_streams": sorted(odf), "streams_both_with_errors": both, "first_error_same_position": same_pos, "only_jing": only_jing, "only_odfvalidator": only_odf, 113 "different_first_error": diffpos, "package_level": dict(pkgmsgs)} 114 115 116def main(): 117 outdir, valjson, crossjson = os.path.abspath(sys.argv[1]), sys.argv[2], sys.argv[3] 118 if not os.path.abspath(crossjson).startswith(os.path.join(D, "runs") + os.sep): 119 sys.exit("crosscheck.py: <cross.json> must live under " + os.path.join(D, "runs")) 120 if not os.path.isfile(JAR): 121 sys.exit("crosscheck.py: " + JAR + " not found; run fetch-odfvalidator.sh or set ODFV_ODFVALIDATOR") 122 work = os.path.abspath(crossjson[:-5] + ".d") 123 os.makedirs(work, exist_ok=True) 124 val = json.load(open(valjson)) 125 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".") and p not in val.get("bad_zip", [])) 126 res = {} 127 for flag, jv, name in (("-c", "12", "conformance"), ("-e", "12x", "extended")): 128 events, crashed = run(flag, outdir, pkgs, work, name) 129 res[name] = compare(events, val["errors"][jv], pkgs) 130 res[name]["odfvalidator_crashed_on"] = crashed 131 r = res[name] 132 print(f"{name} (odfvalidator {flag} vs jing {jv}): both {r['streams_both_with_errors']} (same first position {r['first_error_same_position']}), " 133 f"only jing {len(r['only_jing'])}, only odfvalidator {len(r['only_odfvalidator'])}, odfvalidator crashed on {len(crashed)} packages") 134 for k, v in sorted(r["package_level"].items()): 135 print(f" {v:5d} {k}") 136 json.dump(res, open(crossjson, "w"), indent=1) 137 138 139main() 140