1#!/usr/bin/env python3 2#************************************************************** 3# 4# Licensed to the Apache Software Foundation (ASF) under one 5# or more contributor license agreements. See the NOTICE file 6# distributed with this work for additional information 7# regarding copyright ownership. The ASF licenses this file 8# to you under the Apache License, Version 2.0 (the 9# "License"); you may not use this file except in compliance 10# with the License. You may obtain a copy of the License at 11# 12# http://www.apache.org/licenses/LICENSE-2.0 13# 14# Unless required by applicable law or agreed to in writing, 15# software distributed under the License is distributed on an 16# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17# KIND, either express or implied. See the License for the 18# specific language governing permissions and limitations 19# under the License. 20# 21#************************************************************** 22 23# Black-box corpus builder: converts the same inputs with another office suite's headless binary once per output 24# ODF version (1.2, 1.2 extended, 1.3, 1.3 extended), so an import smoke test can load each 1.3 file next to its 25# 1.2 twin. Only observes output files; never reads that suite's source or installed configuration. 26# 27# python3 -I make-foreign-corpus.py <soffice binary> [--inputs LIST] [--root DIR] [--modes v12,v13,...] 28# python3 -I make-foreign-corpus.py <soffice binary> --probe WORKDIR [--probe-range 0-20] (value table only) 29import argparse, os, re, signal, subprocess, sys, time, zipfile 30from urllib.parse import quote 31 32HERE = os.path.dirname(os.path.abspath(__file__)) 33TEST = os.path.dirname(HERE) # test/odf-validate 34JOBS = os.path.join(TEST, "runs", "gate-t16f", "jobs.tsv") 35DEFAULT_ROOT = os.path.join(TEST, "runs", "phase3", "foreign") 36 37# Observed black-box (--probe, one small .odt converted per value; office:version and manifest:version read back 38# from the result, extension namespaces = loext/calcext/officeooo/drawooo/tableooo/field declared on the root): 39# 1 -> 1.0 (no version attribute) 2 -> 1.1 4 -> 1.2 plain 8 -> 1.2 + extensions 9 -> 1.2 + extensions 40# 10 -> 1.3 plain 11 -> 1.3 + extensions 12 -> 1.4 plain 13 -> 1.4 + extensions 41# 0, 3, 5, 6, 7 and 14.. all wrote 1.4 + extensions (the default). 42DEFAULT_VERSION_V12 = 4 43DEFAULT_VERSION_V12EXT = 8 44DEFAULT_VERSION_V13 = 10 45DEFAULT_VERSION_V13EXT = 11 46 47# mode -> (DefaultVersion value, expected office:version, whether extension namespaces are expected) 48MODES = { 49 "v12": (DEFAULT_VERSION_V12, "1.2", False), 50 "v12ext": (DEFAULT_VERSION_V12EXT, "1.2", True), 51 "v13": (DEFAULT_VERSION_V13, "1.3", False), 52 "v13ext": (DEFAULT_VERSION_V13EXT, "1.3", True), 53} 54OUT_EXT = {"odt": "odt", "sxw": "odt", "doc": "odt", "docx": "odt", "rtf": "odt", 55 "ods": "ods", "sxc": "ods", "xls": "ods", "xlsx": "ods", 56 "odp": "odp", "sxi": "odp", "ppt": "odp", "pptx": "odp", "odg": "odg", "sxd": "odg"} 57 58# Selection of the built-in list: per source extension a quota, filled round-robin across source directories 59# (largest file first within a directory, 2 KB..2 MB) so the spread covers many components and not one folder. 60# Quotas total 56 (<= 60); 55 are filled on this tree (the two testinput.odp share a name, so one is dropped): 61# native ODF odt 14, ods 9, odp 3, odg 3 (every ods/odp/odg source in the list), legacy 62# doc 6, xls 6, ppt 2, sxw 5, sxc 5, sxi 2. docx/rtf are left out: the aim is ODF 1.2/1.3 twins of ODF/legacy 63# office sources, and OOXML import adds noise unrelated to ODF 1.3 import. The xmlsecurity signed-document 64# variants are near duplicates, so the per-directory round-robin lets at most a few of them in. 65QUOTA = [("odt", 14), ("ods", 9), ("odp", 4), ("odg", 3), ("doc", 6), ("xls", 6), ("ppt", 2), 66 ("sxw", 5), ("sxc", 5), ("sxi", 2)] 67MIN_SIZE, MAX_SIZE = 2048, 2 * 1024 * 1024 68 69EXT_URI_RX = re.compile(r"documentfoundation|openoffice\.org/20(?:09|10)/|ooo-ms-interop:xmlns:field") 70 71 72def builtin_selection(jobs=JOBS): 73 seen, byext = set(), {} 74 with open(jobs, encoding="utf8") as fh: 75 for line in fh: 76 p = line.split("\t", 1)[0].strip() 77 if not p or p in seen or not os.path.isfile(p): 78 continue 79 seen.add(p) 80 ext = p.rsplit(".", 1)[-1].lower() 81 sz = os.path.getsize(p) 82 if ext in OUT_EXT and MIN_SIZE <= sz <= MAX_SIZE: 83 byext.setdefault(ext, []).append((p, sz)) 84 chosen, used = [], set() 85 for ext, quota in QUOTA: 86 dirs = {} 87 for p, sz in byext.get(ext, []): 88 dirs.setdefault(os.path.dirname(p), []).append((-sz, p)) 89 queues = [[p for _, p in sorted(v)] for _, v in sorted(dirs.items())] 90 got = 0 91 while got < quota and any(queues): 92 for q in queues: 93 if not q or got >= quota: 94 continue 95 p = q.pop(0) 96 key = (os.path.splitext(os.path.basename(p))[0], OUT_EXT[ext]) 97 if key in used: 98 continue 99 used.add(key) 100 chosen.append(p) 101 got += 1 102 return chosen 103 104 105def read_list(path): 106 out = [] 107 with open(path, encoding="utf8") as fh: 108 for line in fh: 109 p = line.split("\t", 1)[0].strip() 110 if p and not p.startswith("#"): 111 out.append(os.path.abspath(p)) 112 return out 113 114 115def profile_url(d): 116 return "file://" + quote(d) 117 118 119def write_profile(d, value): 120 os.makedirs(os.path.join(d, "user"), exist_ok=True) 121 with open(os.path.join(d, "user", "registrymodifications.xcu"), "w", encoding="utf8") as fh: 122 fh.write('<?xml version="1.0" encoding="UTF-8"?>\n' 123 '<oor:items xmlns:oor="http://openoffice.org/2001/registry" xmlns:xs="http://www.w3.org/2001/XMLSchema"' 124 ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n' 125 '<item oor:path="/org.openoffice.Office.Common/Save/ODF"><prop oor:name="DefaultVersion" oor:op="fuse">' 126 '<value>%d</value></prop></item>\n</oor:items>\n' % value) 127 128 129def own_pids(profile_dirs): 130 # Processes whose command line carries exactly one of our profile URLs (as a whole argument). 131 toks = ["-env:UserInstallation=" + profile_url(d) for d in profile_dirs] 132 r = subprocess.run(["ps", "-axww", "-o", "pid=,command="], capture_output=True, text=True) 133 pids = [] 134 for line in r.stdout.splitlines(): 135 line = line.strip() 136 if not line: 137 continue 138 pid, _, cmd = line.partition(" ") 139 pid = int(pid) 140 if pid == os.getpid(): 141 continue 142 for t in toks: 143 if re.search(re.escape(t) + r"(?:\s|$)", cmd): 144 pids.append(pid) 145 break 146 return pids 147 148 149def kill_own(profile_dirs): 150 pids = own_pids(profile_dirs) 151 for sig in (signal.SIGTERM, signal.SIGKILL): 152 for pid in pids: 153 try: 154 os.kill(pid, sig) 155 except ProcessLookupError: 156 pass 157 deadline = time.time() + 8 158 while time.time() < deadline and own_pids(profile_dirs): 159 time.sleep(0.5) 160 pids = own_pids(profile_dirs) 161 if not pids: 162 return 163 164 165def run_convert(soffice, prof, outext, outdir, files, timeout, log): 166 cmd = [soffice, "-env:UserInstallation=" + profile_url(prof), "--headless", "--norestore", 167 "--convert-to", outext, "--outdir", outdir] + files 168 with open(log, "ab") as lf: 169 lf.write(("\n$ " + " ".join(cmd) + "\n").encode()) 170 lf.flush() 171 p = subprocess.Popen(cmd, stdout=lf, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL) 172 try: 173 p.wait(timeout=timeout) 174 timed_out = False 175 except subprocess.TimeoutExpired: 176 timed_out = True 177 # soffice.bin may outlive the launcher briefly; only our own profile is touched 178 t0 = time.time() 179 while own_pids([prof]) and time.time() - t0 < 20 and not timed_out: 180 time.sleep(0.5) 181 if timed_out or own_pids([prof]): 182 kill_own([prof]) 183 return timed_out 184 185 186def inspect(path): 187 # -> (office:version of content.xml, manifest:version, declared extension namespace prefixes, problems) 188 problems, vers, ext_ns, mv = [], {}, set(), None 189 try: 190 z = zipfile.ZipFile(path) 191 for n in ("content.xml", "styles.xml", "meta.xml", "settings.xml"): 192 if n not in z.namelist(): 193 continue 194 head = z.read(n)[:20000].decode("utf8", "replace") 195 m = re.search(r'office:version="([^"]*)"', head) 196 vers[n] = m.group(1) if m else None 197 if n in ("content.xml", "styles.xml"): 198 for pre, uri in re.findall(r'xmlns:([\w.-]+)="([^"]*)"', head): 199 if EXT_URI_RX.search(uri): 200 ext_ns.add(pre) 201 m = re.search(r'manifest:version="([^"]*)"', z.read("META-INF/manifest.xml").decode("utf8", "replace")) 202 mv = m.group(1) if m else None 203 except Exception as e: # unreadable output is a failure, reported by the caller 204 return None, None, set(), ["unreadable: %s" % e] 205 cv = vers.get("content.xml") 206 if len(set(vers.values())) > 1: 207 problems.append("office:version differs between parts: %s" % vers) 208 return cv, mv, ext_ns, problems 209 210 211def probe(soffice, work, rng): 212 lo, hi = rng 213 src = os.path.join(work, "probe-input") 214 os.makedirs(src, exist_ok=True) 215 doc = os.path.join(src, "probe.odt") 216 # minimal ODF 1.2 text document built here, so the probe needs no outside file 217 with zipfile.ZipFile(doc, "w") as z: 218 z.writestr("mimetype", "application/vnd.oasis.opendocument.text", compress_type=zipfile.ZIP_STORED) 219 z.writestr("META-INF/manifest.xml", 220 '<?xml version="1.0" encoding="UTF-8"?>\n<manifest:manifest xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" manifest:version="1.2">' 221 '<manifest:file-entry manifest:full-path="/" manifest:media-type="application/vnd.oasis.opendocument.text"/>' 222 '<manifest:file-entry manifest:full-path="content.xml" manifest:media-type="text/xml"/></manifest:manifest>') 223 z.writestr("content.xml", 224 '<?xml version="1.0" encoding="UTF-8"?>\n<office:document-content xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"' 225 ' xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" office:version="1.2"><office:body><office:text>' 226 '<text:p>probe</text:p></office:text></office:body></office:document-content>') 227 dirs = [os.path.join(work, "probe-prof-%d" % n) for n in range(lo, hi + 1)] 228 if own_pids(dirs): 229 sys.exit("refusing to start: an office using one of the probe profiles is running") 230 print("value\toffice:version\tmanifest:version\textension namespaces") 231 for n, prof in zip(range(lo, hi + 1), dirs): 232 write_profile(prof, n) 233 out = os.path.join(work, "probe-out-%d" % n) 234 os.makedirs(out, exist_ok=True) 235 f = os.path.join(out, "probe.odt") 236 if os.path.exists(f): 237 os.remove(f) 238 run_convert(soffice, prof, "odt", out, [doc], 300, os.path.join(work, "probe.log")) 239 if not os.path.exists(f): 240 print("%d\tNO OUTPUT" % n) 241 continue 242 cv, mv, ext, _ = inspect(f) 243 print("%d\t%s\t%s\t%s" % (n, cv, mv, ",".join(sorted(ext)) or "-")) 244 245 246def main(): 247 ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) 248 ap.add_argument("soffice", help="the other suite's soffice binary") 249 ap.add_argument("--inputs", help="file with one input path per line (first tab column used; '#' comments)") 250 ap.add_argument("--root", default=DEFAULT_ROOT, help="output root (default %(default)s)") 251 ap.add_argument("--modes", default=",".join(MODES), help="comma list of modes (default all)") 252 ap.add_argument("--batch", type=int, default=10, help="files per soffice call (default %(default)s)") 253 ap.add_argument("--timeout", type=int, default=300, help="seconds per soffice call (default %(default)s)") 254 ap.add_argument("--list-selection", action="store_true", help="print the built-in selection and exit") 255 ap.add_argument("--probe", metavar="WORKDIR", help="only print the DefaultVersion value table, using WORKDIR") 256 ap.add_argument("--probe-range", default="0-20") 257 a = ap.parse_args() 258 soffice = os.path.abspath(a.soffice) 259 if a.list_selection: 260 print("\n".join(builtin_selection())) 261 return 0 262 if not os.access(soffice, os.X_OK): 263 sys.exit("not executable: %s" % soffice) 264 if a.probe: 265 lo, hi = map(int, a.probe_range.split("-")) 266 probe(soffice, os.path.abspath(a.probe), (lo, hi)) 267 return 0 268 269 modes = a.modes.split(",") 270 for m in modes: 271 if m not in MODES: 272 sys.exit("unknown mode %s" % m) 273 root = os.path.abspath(a.root) 274 profs = {m: os.path.join(root, "foreign-prof-" + m) for m in modes} 275 running = own_pids(list(profs.values())) 276 if running: 277 sys.exit("refusing to start: office process(es) %s already use one of our profile directories" % running) 278 279 inputs = read_list(a.inputs) if a.inputs else builtin_selection() 280 inputs = [p for p in inputs if os.path.isfile(p)] 281 keys = {} 282 for p in inputs: 283 ext = p.rsplit(".", 1)[-1].lower() 284 if ext not in OUT_EXT: 285 sys.exit("unsupported input type: %s" % p) 286 k = (os.path.splitext(os.path.basename(p))[0], OUT_EXT[ext]) 287 if k in keys: 288 sys.exit("output name collision %s.%s: %s and %s" % (k[0], k[1], keys[k], p)) 289 keys[k] = p 290 if not inputs: 291 sys.exit("no input files") 292 os.makedirs(os.path.join(root, "logs"), exist_ok=True) 293 print("%d inputs, modes %s, root %s" % (len(inputs), ",".join(modes), root), flush=True) 294 295 rows, short = [], False 296 for m in modes: 297 value, want, want_ext = MODES[m] 298 write_profile(profs[m], value) 299 outdir = os.path.join(root, m) 300 os.makedirs(outdir, exist_ok=True) 301 log = os.path.join(root, "logs", m + ".log") 302 todo = {} # outext -> [(input, expected output)] 303 for p in inputs: 304 oe = OUT_EXT[p.rsplit(".", 1)[-1].lower()] 305 o = os.path.join(outdir, os.path.splitext(os.path.basename(p))[0] + "." + oe) 306 if os.path.exists(o): 307 os.remove(o) # a stale file must not pass for a fresh conversion 308 todo.setdefault(oe, []).append((p, o)) 309 for oe, items in todo.items(): 310 for i in range(0, len(items), a.batch): 311 chunk = items[i:i + a.batch] 312 to = run_convert(soffice, profs[m], oe, outdir, [p for p, _ in chunk], a.timeout, log) 313 missing = [(p, o) for p, o in chunk if not os.path.exists(o)] 314 if (to or missing) and len(chunk) > 1: 315 print(" %s: %d of %d missing after batch%s; retrying one by one" % 316 (m, len(missing), len(chunk), " (timeout)" if to else ""), flush=True) 317 for p, o in missing: 318 run_convert(soffice, profs[m], oe, outdir, [p], a.timeout, log) 319 n_out = n_ok = 0 320 for p in inputs: 321 oe = OUT_EXT[p.rsplit(".", 1)[-1].lower()] 322 o = os.path.join(outdir, os.path.splitext(os.path.basename(p))[0] + "." + oe) 323 if not os.path.exists(o): 324 rows.append((p, m, o, "", "", "fail", "no output")) 325 continue 326 n_out += 1 327 cv, mv, ext, probs = inspect(o) 328 if cv != want: 329 probs.append("office:version %s, expected %s" % (cv, want)) 330 if mv != want: 331 probs.append("manifest:version %s, expected %s" % (mv, want)) 332 if want_ext and not ext: 333 probs.append("no extension namespaces declared") 334 if not want_ext and ext: 335 probs.append("unexpected extension namespaces %s" % ",".join(sorted(ext))) 336 if not probs: 337 n_ok += 1 338 rows.append((p, m, o, cv or "", mv or "", "fail" if probs else "ok", "; ".join(probs))) 339 print("%s (DefaultVersion %d): %d/%d outputs, %d verified as office:version %s" % 340 (m, value, n_out, len(inputs), n_ok, want), flush=True) 341 if n_out < len(inputs) or n_ok < n_out: 342 short = True 343 with open(os.path.join(root, "manifest.tsv"), "w", encoding="utf8") as fh: 344 fh.write("input\tmode\toutput\toffice:version\tmanifest:version\tstatus\tnote\n") 345 for r in rows: 346 fh.write("\t".join(r) + "\n") 347 bad = [r for r in rows if r[5] != "ok"] 348 for r in bad[:20]: 349 print("FAIL %s [%s]: %s" % (r[0], r[1], r[6])) 350 if own_pids(list(profs.values())): 351 print("warning: office processes of ours still running, killing them") 352 kill_own(list(profs.values())) 353 print("manifest: %s (%d rows, %d not ok)" % (os.path.join(root, "manifest.tsv"), len(rows), len(bad))) 354 return 1 if (short or bad) else 0 355 356 357if __name__ == "__main__": 358 sys.exit(main()) 359