1################################################################ 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20################################################################ 21 22"""Negative-control documents for the checkers: ports of make-controls.py, make-pkg-controls.py and make-doctype-controls.py.""" 23import io, json, os, zipfile, warnings 24 25 26def _prepare(path): 27 """A control directory must be empty or absent: nothing is ever deleted here.""" 28 if path is None: 29 return 30 if os.path.isdir(path) and os.listdir(path): 31 raise ValueError("control directory is not empty: " + path) 32 os.makedirs(path, exist_ok=True) 33 34 35def _target(path, name): 36 """A file to write, or a throwaway buffer when only the expectations are wanted.""" 37 return io.BytesIO() if path is None else os.path.join(path, name) 38 39 40def _write(path, name, text): 41 if path is not None: 42 open(os.path.join(path, name), "w").write(text) 43 44 45def make_odf_controls(outdir): 46 """Writes the negative-control packages for validate.check() and idcheck.check() into `outdir` (empty or absent).""" 47 C = os.fspath(outdir) 48 MANI = ('<?xml version="1.0" encoding="UTF-8"?>\n<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">' 49 '<manifest:file-entry manifest:full-path="/" manifest:version="1.2" manifest:media-type="application/vnd.oasis.opendocument.text"/>' 50 '<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/></manifest:manifest>\n') 51 MATH = ('<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE math:math PUBLIC "-//OpenOffice.org//DTD Modified W3C MathML 1.01//EN" "math.dtd">\n' 52 '<math:math xmlns:math="http://www.w3.org/1998/Math/MathML"><math:semantics><math:mi>x</math:mi></math:semantics></math:math>\n') 53 54 55 def mk(name, ver, body, extra=None): 56 content = ('<?xml version="1.0" encoding="UTF-8"?>\n<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" ' 57 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" office:version="%s"><office:body><office:text>%s</office:text></office:body></office:document-content>\n') % (ver, body) 58 with zipfile.ZipFile(_target(C, name + ".odt"), "w") as z: 59 z.writestr(zipfile.ZipInfo("mimetype"), "application/vnd.oasis.opendocument.text", compress_type=zipfile.ZIP_STORED) 60 z.writestr("content.xml", content, compress_type=zipfile.ZIP_DEFLATED) 61 z.writestr("META-INF/manifest.xml", MANI, compress_type=zipfile.ZIP_DEFLATED) 62 for n, t in (extra or {}).items(): 63 z.writestr(n, t, compress_type=zipfile.ZIP_DEFLATED) 64 65 66 OFFNS = ('xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" ' 67 'xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" xmlns:chart="urn:oasis:names:tc:opendocument:xmlns:chart:1.0" ' 68 'xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0" xmlns:dc="http://purl.org/dc/elements/1.1/" ' 69 'xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"') 70 71 72 def mkdoc(name, ext, mime, streams): 73 """A package whose streams are given whole: {"content.xml": (root element, body)}, written with office:version 1.2.""" 74 mani = MANI.replace("application/vnd.oasis.opendocument.text", mime) 75 with zipfile.ZipFile(_target(C, name + "." + ext), "w") as z: 76 z.writestr(zipfile.ZipInfo("mimetype"), mime, compress_type=zipfile.ZIP_STORED) 77 for n, (root, body) in streams.items(): 78 z.writestr(n, '<?xml version="1.0" encoding="UTF-8"?>\n<%s %s office:version="1.2">%s</%s>\n' % (root, OFFNS, body, root), compress_type=zipfile.ZIP_DEFLATED) 79 z.writestr("META-INF/manifest.xml", mani, compress_type=zipfile.ZIP_DEFLATED) 80 81 82 _prepare(C) 83 mk("good", "1.2", "<text:p>hi</text:p>") 84 mk("bad", "1.2", "<text:bogus/>") 85 mk("wrongver", "1.1", "<text:p>hi</text:p>") 86 # text:drop-down is new in ODF 1.3 (OFFICE-3881): invalid in 1.2, valid in 1.3 and 1.4 87 mk("new13", "1.2", '<text:p><text:drop-down text:name="d"><text:label text:value="a"/></text:drop-down></text:p>') 88 # ODF 1.2 Part 1 2.2.1 B.4: an embedded object whose content.xml is MathML 89 mk("mathobj", "1.2", "<text:p>hi</text:p>", {"Object 1/content.xml": MATH}) 90 # ODF 1.2 Part 1 2.2.1 D.1/D.2: checked by idcheck.py, not by the schema 91 mk("goodid", "1.2", '<text:p xml:id="a1">x</text:p><text:p xml:id=" b-2.c ">y</text:p>') 92 mk("dupid", "1.2", '<text:p xml:id="a1">x</text:p><text:p xml:id=" a1 ">y</text:p>') 93 mk("badid", "1.2", '<text:p xml:id="1abc">x</text:p>') 94 mk("badid2", "1.2", '<text:p xml:id="a b">x</text:p>') 95 mk("badns", "1.2", "<text:p><zz:span/></text:p>") 96 # Part 3 2.2.1 F: signature files are held to the same rules (validate.py does not look at them) 97 mk("badsig", "1.2", "<text:p>hi</text:p>", {"META-INF/documentsignatures.xml": '<a><b xml:id="x"/><b xml:id="x"/></a>'}) 98 # Part 1 2.2.2 B.1: valid as an extended document only (foreign attribute, foreign element in a paragraph, foreign element elsewhere) 99 mk("extforeign", "1.2", '<text:p xmlns:foo="urn:foo" foo:x="1">a<foo:y>b</foo:y>c</text:p><foo:w xmlns:foo="urn:foo">z<text:p>q</text:p></foo:w>') 100 TEXT = "application/vnd.oasis.opendocument.text" 101 # meta:creator-initials in office:annotation is new in ODF 1.3; baseinit is the same note without it 102 ANN = '<office:body><office:text><text:p><office:annotation><dc:creator>A</dc:creator>%s<text:p>n</text:p></office:annotation>x</text:p></office:text></office:body>' 103 for n, ini in (("baseinit", ""), ("newinit", "<meta:creator-initials>AB</meta:creator-initials>")): 104 mkdoc(n, "odt", TEXT, {"content.xml": ("office:document-content", ANN % ini)}) 105 # chart:coordinate-region (inner plot position) is new in 1.3; baseregion is the same chart without it 106 CHART = "application/vnd.oasis.opendocument.chart" 107 for n, region in (("baseregion", ""), ("newregion", '<chart:coordinate-region svg:x="1cm" svg:y="1cm" svg:width="5cm" svg:height="4cm"/>')): 108 mkdoc(n, "odc", CHART, {"content.xml": ("office:document-content", '<office:body><office:chart><chart:chart chart:class="chart:bar"><chart:plot-area>%s</chart:plot-area></chart:chart></office:chart></office:body>' % region)}) 109 # 1.3 makes style:scale-to, style:scale-to-pages and the pair scale-to-X/scale-to-Y mutually exclusive (OFFICE-3857); 1.2 has no scale-to-X/Y 110 for n, attrs in (("scaleone", 'style:scale-to="80%"'), ("scalexy", 'style:scale-to-X="2" style:scale-to-Y="3"'), ("scaleboth", 'style:scale-to-X="2" style:scale-to-Y="3" style:scale-to="80%"')): 111 mkdoc(n, "odt", TEXT, {"content.xml": ("office:document-content", "<office:body><office:text><text:p>hi</text:p></office:text></office:body>"), 112 "styles.xml": ("office:document-styles", '<office:automatic-styles><style:page-layout style:name="pm1"><style:page-layout-properties %s/></style:page-layout></office:automatic-styles>' % attrs)}) 113 # 1.4 (OFFICE-4142): draw:decorative on style:graphic-properties is new, invalid in 1.2 and 1.3; basedeco is the same style without it 114 for n, attrs in (("basedeco", ""), ("newdeco", ' draw:decorative="true"')): 115 mkdoc(n, "odt", TEXT, {"content.xml": ("office:document-content", "<office:body><office:text><text:p>hi</text:p></office:text></office:body>"), 116 "styles.xml": ("office:document-styles", '<office:automatic-styles><style:style style:name="gr1" style:family="graphic"><style:graphic-properties%s/></style:style></office:automatic-styles>' % attrs)}) 117 # 1.4 (OFFICE-4110): draw:handle needs the new position-x/-y or polar-pole pair; the old combined attribute alone is valid in 1.2 and 1.3 only. 118 # The pair itself is invalid in 1.2 and 1.3. 119 for n, attrs in (("handleold", 'draw:handle-position="0 0"'), ("handlenew", 'draw:handle-position-x="0" draw:handle-position-y="0"'), 120 ("handlepolar", 'draw:handle-polar-pole-x="0" draw:handle-polar-pole-y="0"')): 121 mkdoc(n, "odt", TEXT, {"content.xml": ("office:document-content", '<office:body><office:text><text:p><draw:custom-shape><draw:enhanced-geometry><draw:handle %s/></draw:enhanced-geometry></draw:custom-shape></text:p></office:text></office:body>' % attrs)}) 122 123 124def make_pkg_controls(outdir): 125 """Writes pkgcheck's negative-control packages and .expected.json into `outdir` (empty or absent), or only builds the table when 126 `outdir` is None. Returns {package: sorted [(level, rule)]}: the findings pkgcheck.check() must report, exactly.""" 127 C = None if outdir is None else os.fspath(outdir) 128 MIME = "application/vnd.oasis.opendocument.text" 129 S, Z = zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED 130 SIG = ('<?xml version="1.0" encoding="UTF-8"?>\n<dsig:document-signatures xmlns:dsig="urn:oasis:names:tc:opendocument:xmlns:digitalsignature:1.0" ' 131 'dsig:version="%s" xmlns:ds="http://www.w3.org/2000/09/xmldsig#"><ds:Signature Id="s1"/></dsig:document-signatures>\n') 132 ENC = ('<manifest:encryption-data manifest:checksum-type="SHA1/1K" manifest:checksum="AAAA">' 133 '<manifest:algorithm manifest:algorithm-name="Blowfish CFB" manifest:initialisation-vector="AAAA"/>' 134 '<manifest:key-derivation manifest:key-derivation-name="PBKDF2" manifest:key-size="16" manifest:iteration-count="1024" manifest:salt="AAAA"/>' 135 '</manifest:encryption-data>') 136 137 138 def entry(path, mt="text/xml", inner="", **attrs): 139 a = "".join(' manifest:%s="%s"' % kv for kv in attrs.items()) 140 return '<manifest:file-entry manifest:full-path="%s" manifest:media-type="%s"%s>%s</manifest:file-entry>' % (path, mt, a, inner) 141 142 143 def manifest(*entries, root="manifest:manifest"): 144 return ('<?xml version="1.0" encoding="UTF-8"?>\n<%s xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">%s</%s>\n' 145 % (root, "".join(entries), root)) 146 147 148 ROOT = entry("/", MIME, version="1.2") 149 CONTENT = entry("content.xml") 150 GOOD_MANI = manifest(ROOT, CONTENT) 151 EXPECTED = {} 152 153 154 def build(name, files, expect=()): 155 """files: (zip name, data, method[, extra]) in physical order.""" 156 with zipfile.ZipFile(_target(C, name + ".odt"), "w") as z, warnings.catch_warnings(): 157 warnings.simplefilter("ignore") 158 for f in files: 159 zi = zipfile.ZipInfo(f[0]) 160 if len(f) > 3: 161 zi.extra = f[3] 162 z.writestr(zi, f[1], compress_type=f[2]) 163 EXPECTED[name + ".odt"] = sorted(expect) 164 165 166 def std(mani=GOOD_MANI, mime=MIME, extra=()): 167 return [("mimetype", mime, S), ("content.xml", "<c/>", Z)] + list(extra) + [("META-INF/manifest.xml", mani, Z)] 168 169 170 _prepare(C) 171 build("good", std()) 172 sub = [("Object 1/content.xml", "<c/>", Z), ("Pictures/a.png", "x", S), ("secret.xml", "0123456789", S), 173 ("META-INF/documentsignatures.xml", SIG % "1.2", Z)] 174 build("good-full", std(manifest(ROOT, CONTENT, entry("Object 1/", MIME, version="1.2"), entry("Object 1/content.xml"), 175 entry("Pictures/a.png", "image/png"), entry("secret.xml", inner=ENC, size="10")), extra=sub)) 176 build("mime-deflated", [("mimetype", MIME, Z), ("content.xml", "<c/>", Z), ("META-INF/manifest.xml", GOOD_MANI, Z)], [("shall", "mimetype-compressed")]) 177 build("mime-extra", [("mimetype", MIME, S, b"\xfe\xca\x00\x00"), ("content.xml", "<c/>", Z), ("META-INF/manifest.xml", GOOD_MANI, Z)], [("shall", "mimetype-extra")]) 178 build("mime-notfirst", [("content.xml", "<c/>", Z), ("mimetype", MIME, S), ("META-INF/manifest.xml", GOOD_MANI, Z)], [("shall", "mimetype-first")]) 179 build("mime-mismatch", std(mime="application/x-bogus"), [("shall", "mimetype-mismatch")]) 180 build("mime-nonascii", std(mime=MIME + "é"), [("shall", "mimetype-ascii")]) 181 build("mime-missing", std()[1:], [("should", "mimetype-missing"), ("shall", "mimetype-missing")]) 182 build("mime-missing-noroot", std(manifest(CONTENT))[1:], [("should", "mimetype-missing")]) 183 build("no-root-entry", std(manifest(CONTENT)), [("shall", "manifest-root-entry")]) 184 build("uncovered", std(extra=[("extra.xml", "<e/>", Z)]), [("shall", "manifest-coverage")]) 185 build("dup-entry", std(manifest(ROOT, CONTENT, CONTENT)), [("shall", "manifest-duplicate")]) 186 build("self-entry", std(manifest(ROOT, CONTENT, entry("mimetype", "text/plain"))), [("shall", "manifest-self")]) 187 build("dangling", std(manifest(ROOT, CONTENT, entry("gone.xml"))), [("should", "manifest-dangling")]) 188 build("dir-entry-manifest", std(manifest(ROOT, CONTENT, entry("Configurations2/", "application/vnd.sun.xml.ui.configuration"))), [("should", "manifest-dir-entry")]) 189 build("stray-meta-inf", std(extra=[("META-INF/foo.xml", "<f/>", Z)]), [("shall", "meta-inf-file")]) 190 build("no-manifest", [("mimetype", MIME, S), ("content.xml", "<c/>", Z)], [("shall", "manifest-missing")]) 191 build("manifest-malformed", std("<manifest:manifest"), [("shall", "manifest-malformed")]) 192 build("manifest-wrongroot", std(manifest(root="manifest:other")), [("shall", "manifest-root")]) 193 build("zip-bzip2", std(extra=[("b.xml", "<b/>", zipfile.ZIP_BZIP2)], mani=manifest(ROOT, CONTENT, entry("b.xml"))), [("shall", "zip-method")]) 194 build("zip-dup-name", std(extra=[("content.xml", "<d/>", Z)]), [("shall", "zip-duplicate")]) 195 build("zip-dir-entry", std(extra=[("Pictures/", "", S)]), [("should", "zip-dir-entry")]) 196 build("enc-nosize", std(manifest(ROOT, CONTENT, entry("secret.xml", inner=ENC)), extra=[("secret.xml", "0123456789", S)]), [("shall", "encrypted-size")]) 197 build("enc-deflated", std(manifest(ROOT, CONTENT, entry("secret.xml", inner=ENC, size="10")), extra=[("secret.xml", "0123456789", Z)]), [("shall", "encrypted-method")]) 198 build("sig-malformed", std(extra=[("META-INF/documentsignatures.xml", "<dsig", Z)]), [("shall", "signature-malformed")]) 199 build("sig-wrongroot", std(extra=[("META-INF/documentsignatures.xml", "<a/>", Z)]), [("shall", "signature-root")]) 200 build("sig-badversion", std(extra=[("META-INF/documentsignatures.xml", SIG % "1.1", Z)]), [("shall", "signature-schema")]) 201 if C: 202 json.dump(EXPECTED, open(os.path.join(C, ".expected.json"), "w"), indent=1) 203 return EXPECTED 204 205 206def make_doctype_controls(outdir): 207 """Writes doctypecheck's negative-control documents and .expected.json into `outdir` (empty or absent), or only builds the table 208 when `outdir` is None. Returns {document: sorted [(level, rule)]}.""" 209 C = None if outdir is None else os.fspath(outdir) 210 P = "application/vnd.oasis.opendocument." 211 NS = ('xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:math="http://www.w3.org/1998/Math/MathML"') 212 S, Z = zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED 213 EXPECTED = {} 214 215 216 def content(child="text", root="office:document-content", extra=""): 217 body = "<office:body>%s</office:body>" % ("<office:%s/>" % child if child else "") 218 return '<?xml version="1.0" encoding="UTF-8"?>\n<%s %s office:version="1.2"%s>%s</%s>\n' % (root, NS, extra, body, root) 219 220 221 MATHDOC = '<?xml version="1.0" encoding="UTF-8"?>\n<math:math %s><math:mi>x</math:mi></math:math>\n' % NS 222 223 224 def pkg(name, mime, body, expect=(), files=None, manifest=""): 225 """files: replaces the default content.xml; manifest: extra file-entry elements.""" 226 with zipfile.ZipFile(_target(C, name + ".odt"), "w") as z: 227 if mime is not None: 228 z.writestr(zipfile.ZipInfo("mimetype"), mime, compress_type=S) 229 for n, d in (files if files is not None else {"content.xml": body}).items(): 230 z.writestr(n, d, compress_type=Z) 231 z.writestr("META-INF/manifest.xml", '<?xml version="1.0" encoding="UTF-8"?>\n<manifest:manifest xmlns:manifest=' 232 '"urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">%s</manifest:manifest>\n' % manifest, compress_type=Z) 233 EXPECTED[name + ".odt"] = sorted(expect) 234 235 236 def flat(name, mime, child="text", expect=()): 237 attr = "" if mime is None else ' office:mimetype="%s"' % mime 238 _write(C, name + ".fodt", content(child, "office:document", attr)) 239 EXPECTED[name + ".fodt"] = sorted(expect) 240 241 242 _prepare(C) 243 for t, ms, child in (("text", ["text", "text-template", "text-master"], "text"), ("spreadsheet", ["spreadsheet", "spreadsheet-template"], "spreadsheet"), 244 ("drawing", ["graphics", "graphics-template"], "drawing"), ("presentation", ["presentation", "presentation-template"], "presentation"), 245 ("chart", ["chart", "chart-template"], "chart"), ("image", ["image", "image-template"], "image"), ("database", ["base"], "database")): 246 for m in ms: 247 pkg("good-" + m, P + m, content(child)) 248 pkg("good-formula", P + "formula", MATHDOC) 249 sub = lambda mt, c: {"content.xml": content(), "Object 1/content.xml": c} 250 ent = lambda mt: '<manifest:file-entry manifest:full-path="Object 1/" manifest:media-type="%s"/>' % mt 251 pkg("good-embedded", P + "text", None, files=sub(P + "chart", content("chart")), manifest=ent(P + "chart")) 252 pkg("good-embedded-math", P + "text", None, files=sub("", MATHDOC), manifest=ent(P + "formula")) 253 flat("good-flat", P + "text") 254 for m, ru in (("application/x-bogus", "doctype-unknown-mimetype"), (P + "text\n", "doctype-unknown-mimetype"), 255 ("application.vnd.oasis.opendocument.graphics", "doctype-unknown-mimetype"), 256 (P + "text-web", "doctype-unlisted-mimetype"), (P + "formula-template", "doctype-unlisted-mimetype")): 257 pkg("bad-mime-" + str(len(EXPECTED)), m, content(), [("shall", ru)]) 258 pkg("no-mimetype", None, content(), [("shall", "doctype-no-mimetype")]) 259 pkg("body-mismatch", P + "text", content("spreadsheet"), [("shall", "doctype-body")]) 260 pkg("body-empty", P + "text", content(None), [("shall", "doctype-body")]) 261 pkg("root-styles", P + "text", content(root="office:document-styles"), [("shall", "doctype-root")]) 262 pkg("formula-wrong-root", P + "formula", content(), [("shall", "doctype-root")]) 263 pkg("math-in-text", P + "text", MATHDOC, [("shall", "doctype-root")]) 264 pkg("only-styles", P + "text", None, [("shall", "doctype-no-body")], files={"styles.xml": content(None, "office:document-styles")}) 265 pkg("only-meta", P + "text", None, [("shall", "doctype-no-content"), ("shall", "doctype-no-body")], files={"meta.xml": "<a/>"}) 266 pkg("malformed", P + "text", "<office:document-content", [("shall", "doctype-malformed")]) 267 pkg("sub-body", P + "text", None, [("should", "sub-doctype-body")], files=sub(P + "chart", content("spreadsheet")), manifest=ent(P + "chart")) 268 pkg("sub-root", P + "text", None, [("should", "sub-doctype-root")], files=sub(P + "formula", content()), manifest=ent(P + "formula")) 269 flat("flat-no-mimetype", None, expect=[("shall", "doctype-no-mimetype")]) 270 flat("flat-unknown", "application/x-bogus", expect=[("shall", "doctype-unknown-mimetype")]) 271 flat("flat-formula-mime", P + "formula", expect=[("shall", "doctype-unknown-mimetype")]) 272 flat("flat-body", P + "text", "presentation", [("shall", "doctype-body")]) 273 if C: 274 json.dump(EXPECTED, open(os.path.join(C, ".expected.json"), "w"), indent=1) 275 return EXPECTED 276
