xref: /trunk/test/odf-validate/pkgcheck.py (revision 0e16a31d78a652c56c5c0fc79c752f48b9048df9)

#**************************************************************
#
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.
#
#**************************************************************

# usage: pkgcheck.py <outdir> <pkg.json>
# ODF 1.2 Part 3 2.2.1, 3.2, 3.3 and 3.4.1 on every package in <outdir>. Each finding is
# "shall: [rule] ..." or "should: [rule] ...". Schema validity of the manifest is validate.py's job.
import sys, os, re, zipfile, json, struct, subprocess, shutil, warnings
from xml.etree import ElementTree as ET

D = os.path.dirname(os.path.abspath(__file__))
JING = ["java", "-jar", os.environ.get("ODFV_JING", "/Applications/XMLauthor/lib/jing.jar"), "-i"]
DSIG_SCHEMA = D + "/schema/OpenDocument-v1.2-os-dsig-schema.rng"
MANI = "{urn:oasis:names:tc:opendocument:xmlns:manifest:1.0}"
DSIG = "{urn:oasis:names:tc:opendocument:xmlns:digitalsignature:1.0}"
ERR = re.compile(r"^(.*?):(\d+):(\d+): (error|fatal|warning): (.*)$")


def check_package(path, work):
    """Returns [(entry, level, rule, message)] for one package."""
    out = []

    def add(entry, level, rule, msg):
        out.append((entry, level, rule, msg))
    z = zipfile.ZipFile(path)
    infos = z.infolist()
    names = [i.filename for i in infos]
    for n in sorted({n for n in names if names.count(n) > 1}):
        add(n, "shall", "zip-duplicate", "zip entry name occurs more than once")
    for i in infos:
        if i.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
            add(i.filename, "shall", "zip-method", "compression method %d is neither STORED nor DEFLATED (2.2.1 A)" % i.compress_type)
        if i.filename.endswith("/"):
            add(i.filename, "should", "zip-dir-entry", "directories have no zip entries (4.3)")
    files = {n for n in names if not n.endswith("/")}

    # mimetype (3.3)
    mime = None
    if "mimetype" not in files:
        add("mimetype", "should", "mimetype-missing", "no mimetype file (2.2.1 C)")
    else:
        mi = z.getinfo("mimetype")
        first = min(infos, key=lambda i: i.header_offset)
        if first.filename != "mimetype" or mi.header_offset != 0:
            add("mimetype", "shall", "mimetype-first", "mimetype is not the first file of the zip (3.3)")
        with open(path, "rb") as f:
            f.seek(mi.header_offset)
            hdr = f.read(30)
        if len(hdr) == 30 and hdr[:4] == b"PK\x03\x04":
            method, extralen = struct.unpack("<H", hdr[8:10])[0], struct.unpack("<H", hdr[28:30])[0]
            if method != zipfile.ZIP_STORED:
                add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, local header method %d (3.3)" % method)
            if extralen:
                add("mimetype", "shall", "mimetype-extra", "mimetype has a %d byte extra field in its local header (3.3)" % extralen)
        else:
            add("mimetype", "shall", "mimetype-first", "no local file header at the recorded offset")
        if mi.compress_type != zipfile.ZIP_STORED:
            add("mimetype", "shall", "mimetype-compressed", "mimetype is compressed, central directory method %d (3.3)" % mi.compress_type)
        raw = z.read("mimetype")
        try:
            mime = raw.decode("ascii")
        except UnicodeDecodeError:
            add("mimetype", "shall", "mimetype-ascii", "mimetype content is not ASCII (3.3)")

    # META-INF contents (2.2.1 D, E)
    for n in sorted(files):
        if n.startswith("META-INF/") and n != "META-INF/manifest.xml" and "signatures" not in n.rsplit("/", 1)[-1]:
            add(n, "shall", "meta-inf-file", "file other than manifest.xml and signature files in META-INF (2.2.1 E)")

    # manifest (2.2.1 B, 3.2)
    entries = None
    if "META-INF/manifest.xml" not in files:
        add("META-INF/manifest.xml", "shall", "manifest-missing", "no META-INF/manifest.xml (2.2.1 B)")
    else:
        try:
            root = ET.fromstring(z.read("META-INF/manifest.xml"))
            if root.tag != MANI + "manifest":
                add("META-INF/manifest.xml", "shall", "manifest-root", "root element is not manifest:manifest (2.2.1 B.2)")
            else:
                entries = [e for e in root if e.tag == MANI + "file-entry"]
        except ET.ParseError as e:
            add("META-INF/manifest.xml", "shall", "manifest-malformed", "not well-formed: %s (2.2.1 B.1)" % e)
    if entries is not None:
        paths = [e.get(MANI + "full-path") for e in entries]
        for p in sorted({p for p in paths if paths.count(p) > 1}, key=str):
            add("META-INF/manifest.xml", "shall", "manifest-duplicate", "more than one file-entry for %s (3.2)" % p)
        for p in paths:
            if p in ("META-INF/manifest.xml", "mimetype"):
                add("META-INF/manifest.xml", "shall", "manifest-self", "file-entry for %s (3.2)" % p)
            elif p and p.endswith("/") and p != "/":
                if p + "content.xml" not in files and p + "styles.xml" not in files:
                    add("META-INF/manifest.xml", "should", "manifest-dir-entry", "file-entry for %s, a directory without a document (4.3)" % p)
            elif p not in files and p != "/":
                add("META-INF/manifest.xml", "should", "manifest-dangling", "file-entry for %s, which is not in the package (3.2)" % p)
        for n in sorted(files):
            if n != "mimetype" and not n.startswith("META-INF/") and n not in paths:
                add(n, "shall", "manifest-coverage", "no file-entry in the manifest (3.2)")
        root_entry = [e for e in entries if e.get(MANI + "full-path") == "/"]
        if "mimetype" in files and not root_entry:
            add("META-INF/manifest.xml", "shall", "manifest-root-entry", 'no file-entry for "/" although the package has a mimetype file (3.2)')
        for e in root_entry:
            mt = e.get(MANI + "media-type")
            if "mimetype" not in files:
                add("mimetype", "shall", "mimetype-missing", 'file-entry for "/" but no mimetype file (3.3)')
            elif mime is not None and mt != mime:
                add("mimetype", "shall", "mimetype-mismatch", 'mimetype is "%s" but the "/" media-type is "%s" (3.3)' % (mime, mt))
        # encrypted entries (3.4.1)
        for e in entries:
            if e.find(MANI + "encryption-data") is None:
                continue
            p = e.get(MANI + "full-path")
            size = e.get(MANI + "size")
            if size is None or not re.fullmatch(r"[0-9]+", size):
                add(p, "shall", "encrypted-size", "encrypted entry without a valid manifest:size (3.4.1, 4.8.13)")
            if p in files and z.getinfo(p).compress_type != zipfile.ZIP_STORED:
                add(p, "shall", "encrypted-method", "encrypted entry is not flagged STORED in the zip (3.4.1)")

    # signature files (2.2.1 D)
    sigs = sorted(n for n in files if n.startswith("META-INF/") and "signatures" in n.rsplit("/", 1)[-1])
    for n in sigs:
        try:
            root = ET.fromstring(z.read(n))
        except ET.ParseError as e:
            add(n, "shall", "signature-malformed", "not well-formed: %s (2.2.1 D.1)" % e)
            continue
        if root.tag != DSIG + "document-signatures":
            add(n, "shall", "signature-root", "root element is not dsig:document-signatures (2.2.1 D.2)")
            continue
        p = os.path.join(work, os.path.basename(path), n)
        os.makedirs(os.path.dirname(p), exist_ok=True)
        open(p, "wb").write(z.read(n))
        r = subprocess.run(JING + [DSIG_SCHEMA, p], capture_output=True, text=True)
        for line in (r.stdout + r.stderr).splitlines():
            m = ERR.match(line)
            if m:
                add(n, "shall", "signature-schema", "%s:%s %s (2.2.1 D.3)" % (m.group(2), m.group(3), m.group(5)))
            elif line.strip():
                add(n, "shall", "signature-schema", line.strip())
    return out


def main():
    outdir, pkgjson = sys.argv[1], sys.argv[2]
    if not os.path.abspath(pkgjson).startswith(os.path.join(D, "runs") + os.sep):
        sys.exit("pkgcheck.py: <pkg.json> must live under " + os.path.join(D, "runs"))
    work = os.path.abspath(pkgjson[:-5] + ".xml")
    shutil.rmtree(work, ignore_errors=True)
    errors, bad_zip = [], []
    pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith("."))
    for pkg in pkgs:
        try:
            found = check_package(os.path.join(outdir, pkg), work)
        except zipfile.BadZipFile:
            bad_zip.append(pkg); continue
        for entry, level, rule, msg in found:
            errors.append([pkg + "/" + entry, 0, 0, "%s: [%s] %s" % (level, rule, msg)])
    json.dump({"packages": pkgs, "bad_zip": bad_zip, "errors": errors}, open(pkgjson, "w"))
    shall = [e for e in errors if e[3].startswith("shall:")]
    print(f"package check: {len(shall)} shall and {len(errors) - len(shall)} should findings in "
          f"{len({e[0].split('/')[0] for e in errors})} of {len(pkgs)} packages, bad zips: {bad_zip}")
    rules = {}
    for e in errors:
        k = e[3].split("]")[0] + "]"
        rules[k] = rules.get(k, 0) + 1
    for k in sorted(rules):
        print(f"  {rules[k]:6d} {k}")


main()