1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: doctypecheck.py <outdir> <dt.json> 23# ODF 1.2 Part 1 2.2.1 A.2/E and 2.2.3-2.2.10 on every package (or flat XML file) in <outdir>: 24# the mimetype file (or office:mimetype), the root element and the office:body child must agree 25# on one document type. Findings are "shall: [rule] ..." or "should: [rule] ...". 26import sys, os, io, json, zipfile, collections 27from xml.etree import ElementTree as ET 28 29D = os.path.dirname(os.path.abspath(__file__)) 30OFFICE = "urn:oasis:names:tc:opendocument:xmlns:office:1.0" 31MATH = "http://www.w3.org/1998/Math/MathML" 32P = "application/vnd.oasis.opendocument." 33# type -> (mimetype values, office:body child). 2.2.5 spells the drawing values "application.vnd...graphics" 34# and "...graphic-template"; the registered values in Part 1 appendix C are used instead. 35TYPES = { 36 "text": ([P + "text", P + "text-template", P + "text-master"], "text"), 37 "spreadsheet": ([P + "spreadsheet", P + "spreadsheet-template"], "spreadsheet"), 38 "drawing": ([P + "graphics", P + "graphics-template"], "drawing"), 39 "presentation": ([P + "presentation", P + "presentation-template"], "presentation"), 40 "chart": ([P + "chart", P + "chart-template"], "chart"), 41 "image": ([P + "image", P + "image-template"], "image"), 42 "formula": ([P + "formula"], None), 43 "database": ([P + "base"], "database"), 44} 45BY_MIME = {m: t for t, (ms, _) in TYPES.items() for m in ms} 46# registered in Part 1 appendix C, but not a value any of 2.2.3-2.2.10 allows 47UNLISTED = {P + "text-web", P + "formula-template"} 48 49 50def local(tag): 51 if not tag.startswith("{"): 52 return tag 53 ns, name = tag[1:].split("}", 1) 54 return {OFFICE: "office:", MATH: "math:"}.get(ns, "?:") + name 55 56 57def scan(data): 58 """Returns (root tag, [office:body child tags]); stops reading after office:body.""" 59 root, body, depth, in_body = None, [], 0, False 60 for ev, el in ET.iterparse(io.BytesIO(data), events=("start", "end")): 61 if ev == "start": 62 depth += 1 63 if depth == 1: 64 root = el.tag 65 elif depth == 2 and el.tag == "{%s}body" % OFFICE: 66 in_body = True 67 elif depth == 3 and in_body: 68 body.append(el.tag) 69 else: 70 depth -= 1 71 if depth == 1 and in_body: 72 break 73 return root, body 74 75 76def check_content(add, entry, data, T, flat_mime=None): 77 """Checks one content.xml (or a flat file) against document type T.""" 78 try: 79 root, body = scan(data) 80 except ET.ParseError as e: 81 add(entry, "shall", "doctype-malformed", "not well-formed: %s" % e) 82 return 83 if T == "formula": 84 if root != "{%s}math" % MATH: 85 add(entry, "shall", "doctype-root", "formula document whose root is %s, not math:math (2.2.9 B)" % local(root)) 86 return 87 want = "{%s}document%s" % (OFFICE, "" if flat_mime is not None else "-content") 88 if root != want: 89 add(entry, "shall", "doctype-root", "root is %s, not %s (2.2.1 B.2.1, C.2)" % (local(root), local(want))) 90 return 91 child = TYPES[T][1] 92 if "{%s}%s" % (OFFICE, child) not in body: 93 add(entry, "shall", "doctype-body", "%s document without office:%s in office:body; has %s (2.2.x C)" % 94 (T, child, ", ".join(local(b) for b in body) or "no body child")) 95 96 97def check_package(path): 98 out = [] 99 100 def add(entry, level, rule, msg): 101 out.append((entry, level, rule, msg)) 102 with open(path, "rb") as f: 103 magic = f.read(4) 104 if magic[:2] != b"PK": 105 return check_flat(path, out, add) 106 z = zipfile.ZipFile(path) 107 files = set(z.namelist()) 108 if "content.xml" not in files and "styles.xml" not in files: 109 add("", "shall", "doctype-no-content", "neither content.xml nor styles.xml (2.2.1 A.2)") 110 mime = z.read("mimetype").decode("utf-8", "replace") if "mimetype" in files else None 111 T = None 112 if mime is None: 113 add("mimetype", "shall", "doctype-no-mimetype", "no mimetype file (2.2.3-2.2.10 B)") 114 elif mime in UNLISTED: 115 add("mimetype", "shall", "doctype-unlisted-mimetype", '"%s" is registered but no document type in 2.2.3-2.2.10 allows it' % mime) 116 elif mime not in BY_MIME: 117 add("mimetype", "shall", "doctype-unknown-mimetype", 'mimetype "%s" is not a value of any document type (2.2.1 E)' % mime) 118 else: 119 T = BY_MIME[mime] 120 if T and "content.xml" not in files: 121 add("content.xml", "shall", "doctype-no-body", "no content.xml, so no office:body child (2.2.x C)") 122 elif T: 123 check_content(add, "content.xml", z.read("content.xml"), T) 124 # sub documents: the manifest media type of a directory against that directory's content.xml 125 if "META-INF/manifest.xml" in files: 126 try: 127 root = ET.fromstring(z.read("META-INF/manifest.xml")) 128 except ET.ParseError: 129 root = [] 130 mns = "{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}" 131 for e in root: 132 p, mt = e.get(mns + "full-path"), e.get(mns + "media-type") 133 if p and p.endswith("/") and p != "/" and p + "content.xml" in files and mt in BY_MIME: 134 sub = [] 135 check_content(lambda en, lv, ru, ms: sub.append((en, ru, ms)), p + "content.xml", z.read(p + "content.xml"), BY_MIME[mt]) 136 for en, ru, ms in sub: 137 add(en, "should", "sub-" + ru, "sub document typed %s: %s" % (mt, ms)) 138 return out, mime or "(none)" 139 140 141def check_flat(path, out, add): 142 data = open(path, "rb").read() 143 144 try: 145 root = ET.fromstring(data) 146 except ET.ParseError as e: 147 raise zipfile.BadZipFile(str(e)) 148 mime = root.get("{%s}mimetype" % OFFICE) 149 if root.tag == "{%s}math" % MATH: 150 return out, "(flat math)" 151 if mime is None: 152 add("", "shall", "doctype-no-mimetype", "no office:mimetype attribute (2.2.3-2.2.10 A)") 153 return out, "(none)" 154 T = BY_MIME.get(mime) 155 if mime in UNLISTED: 156 add("", "shall", "doctype-unlisted-mimetype", '"%s" is registered but no document type in 2.2.3-2.2.10 allows it' % mime) 157 elif T is None or T == "formula": 158 add("", "shall", "doctype-unknown-mimetype", 'office:mimetype "%s" is not a value of any flat document type (2.2.1 E)' % mime) 159 else: 160 check_content(add, "", data, T, flat_mime=mime) 161 return out, mime 162 163 164def main(): 165 outdir, dtjson = sys.argv[1], sys.argv[2] 166 if not os.path.abspath(dtjson).startswith(os.path.join(D, "runs") + os.sep): 167 sys.exit("doctypecheck.py: <dt.json> must live under " + os.path.join(D, "runs")) 168 errors, bad, types = [], [], collections.Counter() 169 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".")) 170 for pkg in pkgs: 171 try: 172 found, mime = check_package(os.path.join(outdir, pkg)) 173 except zipfile.BadZipFile: 174 bad.append(pkg); continue 175 types[mime] += 1 176 for entry, level, rule, msg in found: 177 errors.append([pkg + "/" + entry, 0, 0, "%s: [%s] %s" % (level, rule, msg)]) 178 json.dump({"packages": pkgs, "bad": bad, "types": types, "errors": errors}, open(dtjson, "w")) 179 shall = [e for e in errors if e[3].startswith("shall:")] 180 print(f"document type check: {len(shall)} shall and {len(errors) - len(shall)} should findings in " 181 f"{len({e[0].split('/')[0] for e in errors})} of {len(pkgs)} documents, unreadable: {bad}") 182 for k, v in sorted(types.items()): 183 print(f" {v:6d} {k}") 184 rules = collections.Counter(e[3].split("]")[0] + "]" for e in errors) 185 for k, v in sorted(rules.items()): 186 print(f" {v:6d} {k}") 187 188 189main() 190