1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20#************************************************************** 21 22# usage: pkgcheck.py <outdir> <pkg.json> 23# ODF 1.2 Part 3 2.2.1, 3.2, 3.3 and 3.4.1 on every package in <outdir>. Each finding is 24# "shall: [rule] ..." or "should: [rule] ...". Schema validity of the manifest is validate.py's job. 25import sys, os, re, zipfile, json, struct, subprocess, shutil, warnings 26from xml.etree import ElementTree as ET 27 28D = os.path.dirname(os.path.abspath(__file__)) 29JING = ["java", "-jar", os.environ.get("ODFV_JING", "/Applications/XMLauthor/lib/jing.jar"), "-i"] 30DSIG_SCHEMA = D + "/schema/OpenDocument-v1.2-os-dsig-schema.rng" 31MANI = "{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}" 32DSIG = "{urn:oasis:names:tc:opendocument:xmlns:digitalsignature:1.0}" 33ERR = re.compile(r"^(.*?):(\d+):(\d+): (error|fatal|warning): (.*)$") 34 35 36def check_package(path, work): 37 """Returns [(entry, level, rule, message)] for one package.""" 38 out = [] 39 40 def add(entry, level, rule, msg): 41 out.append((entry, level, rule, msg)) 42 z = zipfile.ZipFile(path) 43 infos = z.infolist() 44 names = [i.filename for i in infos] 45 for n in sorted({n for n in names if names.count(n) > 1}): 46 add(n, "shall", "zip-duplicate", "zip entry name occurs more than once") 47 for i in infos: 48 if i.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): 49 add(i.filename, "shall", "zip-method", "compression method %d is neither STORED nor DEFLATED (2.2.1 A)" % i.compress_type) 50 if i.filename.endswith("/"): 51 add(i.filename, "should", "zip-dir-entry", "directories have no zip entries (4.3)") 52 files = {n for n in names if not n.endswith("/")} 53 54 # mimetype (3.3) 55 mime = None 56 if "mimetype" not in files: 57 add("mimetype", "should", "mimetype-missing", "no mimetype file (2.2.1 C)") 58 else: 59 mi = z.getinfo("mimetype") 60 first = min(infos, key=lambda i: i.header_offset) 61 if first.filename != "mimetype" or mi.header_offset != 0: 62 add("mimetype", "shall", "mimetype-first", "mimetype is not the first file of the zip (3.3)") 63 with open(path, "rb") as f: 64 f.seek(mi.header_offset) 65 hdr = f.read(30) 66 if len(hdr) == 30 and hdr[:4] == b"PK\x03\x04": 67 method, extralen = struct.unpack("<H", hdr[8:10])[0], struct.unpack("<H", hdr[28:30])[0] 68 if method != zipfile.ZIP_STORED: 69 add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, local header method %d (3.3)" % method) 70 if extralen: 71 add("mimetype", "shall", "mimetype-extra", "mimetype has a %d byte extra field in its local header (3.3)" % extralen) 72 else: 73 add("mimetype", "shall", "mimetype-first", "no local file header at the recorded offset") 74 if mi.compress_type != zipfile.ZIP_STORED: 75 add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, central directory method %d (3.3)" % mi.compress_type) 76 raw = z.read("mimetype") 77 try: 78 mime = raw.decode("ascii") 79 except UnicodeDecodeError: 80 add("mimetype", "shall", "mimetype-ascii", "mimetype content is not ASCII (3.3)") 81 82 # META-INF contents (2.2.1 D, E) 83 for n in sorted(files): 84 if n.startswith("META-INF/") and n != "META-INF/manifest.xml" and "signatures" not in n.rsplit("/", 1)[-1]: 85 add(n, "shall", "meta-inf-file", "file other than manifest.xml and signature files in META-INF (2.2.1 E)") 86 87 # manifest (2.2.1 B, 3.2) 88 entries = None 89 if "META-INF/manifest.xml" not in files: 90 add("META-INF/manifest.xml", "shall", "manifest-missing", "no META-INF/manifest.xml (2.2.1 B)") 91 else: 92 try: 93 root = ET.fromstring(z.read("META-INF/manifest.xml")) 94 if root.tag != MANI + "manifest": 95 add("META-INF/manifest.xml", "shall", "manifest-root", "root element is not manifest:manifest (2.2.1 B.2)") 96 else: 97 entries = [e for e in root if e.tag == MANI + "file-entry"] 98 except ET.ParseError as e: 99 add("META-INF/manifest.xml", "shall", "manifest-malformed", "not well-formed: %s (2.2.1 B.1)" % e) 100 if entries is not None: 101 paths = [e.get(MANI + "full-path") for e in entries] 102 for p in sorted({p for p in paths if paths.count(p) > 1}, key=str): 103 add("META-INF/manifest.xml", "shall", "manifest-duplicate", "more than one file-entry for %s (3.2)" % p) 104 for p in paths: 105 if p in ("META-INF/manifest.xml", "mimetype"): 106 add("META-INF/manifest.xml", "shall", "manifest-self", "file-entry for %s (3.2)" % p) 107 elif p and p.endswith("/") and p != "/": 108 if p + "content.xml" not in files and p + "styles.xml" not in files: 109 add("META-INF/manifest.xml", "should", "manifest-dir-entry", "file-entry for %s, a directory without a document (4.3)" % p) 110 elif p not in files and p != "/": 111 add("META-INF/manifest.xml", "should", "manifest-dangling", "file-entry for %s, which is not in the package (3.2)" % p) 112 for n in sorted(files): 113 if n != "mimetype" and not n.startswith("META-INF/") and n not in paths: 114 add(n, "shall", "manifest-coverage", "no file-entry in the manifest (3.2)") 115 root_entry = [e for e in entries if e.get(MANI + "full-path") == "/"] 116 if "mimetype" in files and not root_entry: 117 add("META-INF/manifest.xml", "shall", "manifest-root-entry", 'no file-entry for "/" although the package has a mimetype file (3.2)') 118 for e in root_entry: 119 mt = e.get(MANI + "media-type") 120 if "mimetype" not in files: 121 add("mimetype", "shall", "mimetype-missing", 'file-entry for "/" but no mimetype file (3.3)') 122 elif mime is not None and mt != mime: 123 add("mimetype", "shall", "mimetype-mismatch", 'mimetype is "%s" but the "/" media-type is "%s" (3.3)' % (mime, mt)) 124 # encrypted entries (3.4.1) 125 for e in entries: 126 if e.find(MANI + "encryption-data") is None: 127 continue 128 p = e.get(MANI + "full-path") 129 size = e.get(MANI + "size") 130 if size is None or not re.fullmatch(r"[0-9]+", size): 131 add(p, "shall", "encrypted-size", "encrypted entry without a valid manifest:size (3.4.1, 4.8.13)") 132 if p in files and z.getinfo(p).compress_type != zipfile.ZIP_STORED: 133 add(p, "shall", "encrypted-method", "encrypted entry is not flagged STORED in the zip (3.4.1)") 134 135 # signature files (2.2.1 D) 136 sigs = sorted(n for n in files if n.startswith("META-INF/") and "signatures" in n.rsplit("/", 1)[-1]) 137 for n in sigs: 138 try: 139 root = ET.fromstring(z.read(n)) 140 except ET.ParseError as e: 141 add(n, "shall", "signature-malformed", "not well-formed: %s (2.2.1 D.1)" % e) 142 continue 143 if root.tag != DSIG + "document-signatures": 144 add(n, "shall", "signature-root", "root element is not dsig:document-signatures (2.2.1 D.2)") 145 continue 146 p = os.path.join(work, os.path.basename(path), n) 147 os.makedirs(os.path.dirname(p), exist_ok=True) 148 open(p, "wb").write(z.read(n)) 149 r = subprocess.run(JING + [DSIG_SCHEMA, p], capture_output=True, text=True) 150 for line in (r.stdout + r.stderr).splitlines(): 151 m = ERR.match(line) 152 if m: 153 add(n, "shall", "signature-schema", "%s:%s %s (2.2.1 D.3)" % (m.group(2), m.group(3), m.group(5))) 154 elif line.strip(): 155 add(n, "shall", "signature-schema", line.strip()) 156 return out 157 158 159def main(): 160 outdir, pkgjson = sys.argv[1], sys.argv[2] 161 if not os.path.abspath(pkgjson).startswith(os.path.join(D, "runs") + os.sep): 162 sys.exit("pkgcheck.py: <pkg.json> must live under " + os.path.join(D, "runs")) 163 work = os.path.abspath(pkgjson[:-5] + ".xml") 164 shutil.rmtree(work, ignore_errors=True) 165 errors, bad_zip = [], [] 166 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".")) 167 for pkg in pkgs: 168 try: 169 found = check_package(os.path.join(outdir, pkg), work) 170 except zipfile.BadZipFile: 171 bad_zip.append(pkg); continue 172 for entry, level, rule, msg in found: 173 errors.append([pkg + "/" + entry, 0, 0, "%s: [%s] %s" % (level, rule, msg)]) 174 json.dump({"packages": pkgs, "bad_zip": bad_zip, "errors": errors}, open(pkgjson, "w")) 175 shall = [e for e in errors if e[3].startswith("shall:")] 176 print(f"package check: {len(shall)} shall and {len(errors) - len(shall)} should findings in " 177 f"{len({e[0].split('/')[0] for e in errors})} of {len(pkgs)} packages, bad zips: {bad_zip}") 178 rules = {} 179 for e in errors: 180 k = e[3].split("]")[0] + "]" 181 rules[k] = rules.get(k, 0) + 1 182 for k in sorted(rules): 183 print(f" {rules[k]:6d} {k}") 184 185 186main() 187