1#************************************************************** 2# 3# Licensed to the Apache Software Foundation (ASF) under one 4# or more contributor license agreements. See the NOTICE file 5# distributed with this work for additional information 6# regarding copyright ownership. The ASF licenses this file 7# to you under the Apache License, Version 2.0 (the 8# "License"); you may not use this file except in compliance 9# with the License. You may obtain a copy of the License at 10# 11# http://www.apache.org/licenses/LICENSE-2.0 12# 13# Unless required by applicable law or agreed to in writing, 14# software distributed under the License is distributed on an 15# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16# KIND, either express or implied. See the License for the 17# specific language governing permissions and limitations 18# under the License. 19# 20 21# usage: cov_corpus.py <outdir> <cov-dir> 22# What a corpus of saved packages exercises: every element, every (element, attribute) pair and the 23# distinct attribute values (capped), per namespace URI, counted over the XML streams of all packages. 24import sys, os, zipfile, collections 25from xml.parsers import expat 26 27D = os.path.dirname(os.path.abspath(__file__)) 28STREAMS = {"content.xml", "styles.xml", "meta.xml", "settings.xml", "manifest.xml"} 29VALUE_CAP = 200 30 31 32def esc(v): 33 return v.replace("\\", "\\\\").replace("\t", "\\t").replace("\n", "\\n").replace("\r", "\\r") 34 35 36def split(name): 37 ns, _, local = name.rpartition(" ") 38 return ns, local 39 40 41def scan(data, pkg, elems, pairs, values): 42 stack = [] 43 p = expat.ParserCreate(namespace_separator=" ") 44 45 def start(name, attrs): 46 e = split(name) 47 elems[e].add(pkg) 48 for a, v in attrs.items(): 49 k = (e, split(a)) 50 pairs[k].add(pkg) 51 vs = values[k] 52 if len(vs) < VALUE_CAP or v in vs: 53 vs[v] += 1 54 p.StartElementHandler = start 55 p.Parse(data, True) 56 57 58def main(): 59 outdir, covdir = sys.argv[1], sys.argv[2] 60 if not os.path.abspath(covdir).startswith(os.path.join(D, "runs") + os.sep): 61 sys.exit("cov_corpus.py: <cov-dir> must live under " + os.path.join(D, "runs")) 62 os.makedirs(covdir, exist_ok=True) 63 elems, pairs = collections.defaultdict(set), collections.defaultdict(set) 64 values = collections.defaultdict(collections.Counter) 65 pkgs = sorted(p for p in os.listdir(outdir) if not p.startswith(".")) 66 bad, nstreams = [], 0 67 for pkg in pkgs: 68 try: 69 z = zipfile.ZipFile(os.path.join(outdir, pkg)) 70 except Exception: 71 bad.append(pkg); continue 72 for name in z.namelist(): 73 if name.rsplit("/", 1)[-1] not in STREAMS: 74 continue 75 nstreams += 1 76 try: 77 scan(z.read(name), pkg, elems, pairs, values) 78 except expat.ExpatError as e: 79 bad.append(pkg + "/" + name) 80 with open(os.path.join(covdir, "elements.tsv"), "w") as f: 81 for (ns, loc), s in sorted(elems.items()): 82 f.write(f"{ns}\t{loc}\t{len(s)}\n") 83 with open(os.path.join(covdir, "attributes.tsv"), "w") as f: 84 for ((ens, eloc), (ans, aloc)), s in sorted(pairs.items()): 85 f.write(f"{ens}\t{eloc}\t{ans}\t{aloc}\t{len(s)}\n") 86 with open(os.path.join(covdir, "values.tsv"), "w") as f: 87 for ((ens, eloc), (ans, aloc)), c in sorted(values.items()): 88 for v, n in sorted(c.items()): 89 f.write(f"{ens}\t{eloc}\t{ans}\t{aloc}\t{esc(v)}\t{n}\n") 90 print(f"{len(pkgs)} packages, {nstreams} streams, {len(elems)} elements, {len(pairs)} element/attribute pairs, " 91 f"unreadable: {bad}") 92 93 94main() 95