1################################################################ 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20################################################################ 21 22"""One save mode over the source documents: convert every document once, then run the checkers once over the output 23directory (one JVM per mode). Everything is computed on first use and kept.""" 24from pathlib import Path 25 26from aootest import documents, tools 27from aootest.checkers import crosscheck, doctypecheck, formulacheck, idcheck, pkgcheck, validate 28from aootest.client import WorkerError, WorkerFailure, WorkerHang 29 30FILTERS = documents.FILTERS 31 32 33def by_package(errors): 34 """{package: [entries]} for a checker's error list (the first item of an entry is '<package>/<part>').""" 35 out = {} 36 for e in errors: 37 out.setdefault(e[0].split("/", 1)[0], []).append(e) 38 return out 39 40 41def save_one(session, src, dst, timeout=180): 42 """Converts `src` to `dst` with the filter for its extension. Returns {"status": ok|FAIL|HANG|CRASH, "detail": str}.""" 43 try: 44 session.call("convert", {"src": src, "dst": dst, "filter": FILTERS[dst.rsplit(".", 1)[1]]}, timeout=timeout) 45 return {"status": "ok", "detail": ""} 46 except WorkerError as e: 47 return {"status": "FAIL", "detail": "%s: %s" % (e.kind, e.message)} 48 except WorkerHang as e: 49 return {"status": "HANG", "detail": str(e)} 50 except WorkerFailure as e: 51 return {"status": "CRASH", "detail": str(e)} 52 53 54class ModeRun: 55 def __init__(self, name, session, workdir, sources, prepared): 56 self.name, self.session, self.sources, self.prepared = name, session, sources, prepared 57 self.dir = Path(workdir) / name 58 self._saved = self._again = self._validation = self._cross = self._checks = None 59 60 @property 61 def outdir(self): 62 return self.dir / "out" 63 64 def saved(self): 65 """{source id: {"status", "detail", "out"}}; status is ok, FAIL, HANG or CRASH, and "out" is the saved file when ok.""" 66 if self._saved is None: 67 self.outdir.mkdir(parents=True, exist_ok=True) 68 res = {} 69 for source in self.sources: 70 dst = str(self.outdir / documents.out_name(source)) 71 try: 72 src = self.prepared.file(source) 73 except WorkerFailure as e: 74 status = "FAIL" if isinstance(e, WorkerError) else "HANG" if isinstance(e, WorkerHang) else "CRASH" 75 res[source.id] = {"status": status, "detail": str(e), "out": None} 76 continue 77 r = save_one(self.session, src, dst) 78 r["out"] = dst if r["status"] == "ok" else None 79 res[source.id] = r 80 self._saved = res 81 return self._saved 82 83 def saved_again(self): 84 """The same conversions as saved(), a second time, into out2/: {source id: {"status", "detail", "out"}}.""" 85 if self._again is None: 86 self.saved() 87 out2 = self.dir / "out2" 88 out2.mkdir(parents=True, exist_ok=True) 89 res = {} 90 for source in self.sources: 91 if self._saved[source.id]["out"] is None: 92 res[source.id] = {"status": "skipped", "detail": "the first save failed", "out": None} 93 continue 94 dst = str(out2 / documents.out_name(source)) 95 r = save_one(self.session, self.prepared.file(source), dst) 96 r["out"] = dst if r["status"] == "ok" else None 97 res[source.id] = r 98 self._again = res 99 return self._again 100 101 def validation(self): 102 """{"val", "pkg", "doctype", "ids", "formulas"}: each checker's result over the output directory; the Java ones are 103 None when java, jing or the schemas are missing.""" 104 if self._validation is None: 105 self.saved() 106 res = {"ids": idcheck.check(str(self.outdir)), "doctype": doctypecheck.check(str(self.outdir)), 107 "formulas": formulacheck.check(str(self.outdir)), "val": None, "pkg": None} 108 jing, schemas = tools.jing_command(), tools.schema_dir() 109 if jing and schemas: 110 res["val"] = validate.check(str(self.outdir), str(self.dir / "w-validate"), jing, schemas) 111 res["pkg"] = pkgcheck.check(str(self.outdir), str(self.dir / "w-pkg"), jing, schemas) 112 self._validation = res 113 return self._validation 114 115 def crosscheck(self): 116 """The odfvalidator comparison with jing over the output directory (aootest.checkers.crosscheck), or None when java, 117 the odfvalidator jar, jing or the schemas are missing.""" 118 if self._cross is None: 119 val = self.validation()["val"] 120 jar, java, schemas = tools.odfvalidator(), tools.java(), tools.schema_dir() 121 if val is None or not (jar and java and schemas): 122 return None 123 self._cross = crosscheck.check(str(self.outdir), val, str(self.dir / "w-cross"), jar, str(schemas), java) 124 return self._cross 125 126 def reload_checks(self): 127 """{source id: {check name: value} | {"__error__": text}} for the saved copies of the sources that have reload checks.""" 128 if self._checks is None: 129 saved, res = self.saved(), {} 130 for source in self.sources: 131 if not source.checks or saved[source.id]["out"] is None: 132 continue 133 checks = [{"name": c["name"], "expr": c["expr"]} for f in source.checks for c in documents.load_checks(f)] 134 try: 135 res[source.id] = self.session.call("roundtrip", {"path": saved[source.id]["out"], "checks": checks}, timeout=300) 136 except WorkerFailure as e: 137 res[source.id] = {"__error__": "%s: %s" % (type(e).__name__, e)} 138 self._checks = res 139 return self._checks 140