xref: /trunk/test/odf-validate/cov_source.py (revision 0e16a31d78a652c56c5c0fc79c752f48b9048df9)

#**************************************************************
#
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.
#
#**************************************************************

"""Statically extract the (namespace, local name) pairs that the ODF export
code can emit.  Usage: python3 -I cov_source.py <repo-main-dir> <out-dir>

Writes emit.tsv, emit-unresolved.tsv and emit-skipped.tsv into <out-dir>.
"""

import collections
import os
import re
import sys

SRC_EXT = (".cxx", ".hxx", ".h")
SKIP_DIRS = {"test", "qa", "workben", "unotest", "transform", "unoxml", "solver"}
SKIP_PATHS = ("filter/source/svg/",)
KEEP_NAMES = {"impastp4.cxx"}
IMPORT_NAME = re.compile(r"import|imp", re.I)

NS_RE = re.compile(r"XML_NAMESPACE_([A-Z][A-Z0-9_]*)")
TOK_RE = re.compile(r"\bXML_[A-Z_][A-Z0-9_]*\b")
STR_RE = re.compile(r'"((?:[^"\\\n]|\\.)*)"')
NO_EXPORT = re.compile(r"\bMID_FLAG_NO_PROPERTY(_EXPORT)?\b|\bMID_SW_FLAG_NO_ITEM(_EXPORT)?\b")
ELEMENT_ROW = re.compile(r"\bMID_(SW_)?FLAG_ELEMENT_ITEM(_EXPORT)?\b")
PROP_ROW = re.compile(r"\bMID_\w+|\bXML_(?:\w+_)?TYPE_|\bsizeof\s*\(")
EVENT_ROW = re.compile(r'^\{\s*"[^"]*"\s*,\s*XML_NAMESPACE_\w+\s*,\s*"')
EMIT_ATTR = ("AddAttribute", "AddAttributeASCII")
EMIT_FUNCS = set(EMIT_ATTR) | {"AddAttributeIdLegacy", "SvXMLElementExport", "GetQNameByKey"}
DECL_ARG = re.compile(r"^(?:const\s+|enum\s+|sal_uInt16\b|SvXMLExport\b|SvXMLAttributeList\b)")


def read(path):
    with open(path, errors="replace") as f:
        return f.read()


def mask(text):
    """Blank comments and string contents (same length, newlines kept)."""
    out = list(text)
    i, n = 0, len(text)
    while i < n:
        c = text[i]
        if c == "/" and text[i:i + 2] == "//":
            j = text.find("\n", i)
            j = n if j < 0 else j
            for k in range(i, j):
                out[k] = " "
            i = j
        elif c == "/" and text[i:i + 2] == "/*":
            j = text.find("*/", i + 2)
            j = n if j < 0 else j + 2
            for k in range(i, j):
                if text[k] != "\n":
                    out[k] = " "
            i = j
        elif c in "\"'":
            j = i + 1
            while j < n and text[j] != c and text[j] != "\n":
                j += 2 if text[j] == "\\" else 1
            for k in range(i + 1, min(j, n)):
                out[k] = " "
            i = j + 1
        else:
            i += 1
    return "".join(out)


def directive_spans(text):
    """Character ranges of preprocessor lines, with backslash continuations."""
    spans = []
    for m in re.finditer(r"^[ \t]*#(?:[^\n]*\\\n)*[^\n]*", text, re.M):
        spans.append((m.start(), m.end()))
    return spans


def blank(masked, spans):
    out = list(masked)
    for a, b in spans:
        for k in range(a, b):
            if out[k] != "\n":
                out[k] = " "
    return "".join(out)


def match_close(m, i):
    """Index of the bracket closing the one at m[i]."""
    depth = 0
    for k in range(i, len(m)):
        c = m[k]
        if c in "({[":
            depth += 1
        elif c in ")}]":
            depth -= 1
            if depth == 0:
                return k
    return len(m) - 1


def split_args(m, open_i, close_i):
    """(start, end) of each top-level argument between two brackets."""
    args, depth, start = [], 0, open_i + 1
    for k in range(open_i + 1, close_i):
        c = m[k]
        if c in "({[":
            depth += 1
        elif c in ")}]":
            depth -= 1
        elif c == "," and depth == 0:
            args.append((start, k))
            start = k + 1
    args.append((start, close_i))
    return args


def enclosing_call(m, pos):
    """(function name, argument index, argument count) of the innermost call around pos."""
    depth = 0
    for k in range(pos - 1, -1, -1):
        c = m[k]
        if c in ")}]":
            depth += 1
        elif c in "({[":
            if depth == 0:
                if c != "(":
                    return None
                name = re.search(r"(\w+)\s*$", m[max(0, k - 60):k])
                a = split_args(m, k, match_close(m, k))
                idx = next((n for n, (x, y) in enumerate(a) if x <= pos < y), None)
                return (name.group(1) if name else None), idx, len(a)
            depth -= 1
    return None


def parse_tokens(path):
    toks = {}
    pat = re.compile(r'TOKEN\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*(XML_\w+)\s*(?:/\*.*?\*/)?\s*\)')
    for line in read(path).splitlines():
        if line.lstrip().startswith("//"):
            continue
        m = pat.search(line)
        if m:
            toks[m.group(2)] = m.group(1).replace('\\"', '"')
    return toks


ADD_RE = re.compile(r"(?:\b_GetNamespaceMap\s*\(\s*\)\s*\.|\bmpNamespaceMap\s*->)\s*Add(?:AtIndex)?\s*\(")


def ns_adds(masked, clean, toks):
    """(key, URI) pairs from export-side namespace map Add/AddAtIndex calls."""
    defs = dict(re.findall(r'^[ \t]*#[ \t]*define[ \t]+(\w+)[ \t]+"([^"]*)"', clean, re.M))
    out = []
    for m in ADD_RE.finditer(masked):
        o = m.end() - 1
        a = split_args(masked, o, match_close(masked, o))
        if len(a) not in (3, 4):
            continue
        key = NS_RE.fullmatch(masked[a[-1][0]:a[-1][1]].strip())
        arg = clean[a[-2][0]:a[-2][1]]
        tok = re.search(r"\b(XML_N_\w+)", arg)
        lit = next((defs[w] for w in re.findall(r"\w+", arg) if w in defs), None)
        uri = toks.get(tok.group(1)) if tok else lit
        if key and uri is not None:
            out.append((key.group(1), uri))
    return out


class Macro:
    def __init__(self, ns_i, ns_fixed, tok_i, tok_prefix):
        self.ns_i, self.ns_fixed, self.tok_i, self.tok_prefix = ns_i, ns_fixed, tok_i, tok_prefix


def parse_macros(text):
    """Function-like macros that carry a namespace key and a token parameter."""
    macros = {}
    for m in re.finditer(r"^[ \t]*#[ \t]*define[ \t]+(\w+)\(([^)]*)\)((?:[^\n]|\\\n)*)", text, re.M):
        name, params, body = m.group(1), [p.strip() for p in m.group(2).split(",")], m.group(3)
        ns = re.search(r"XML_NAMESPACE_##(\w+)", body)
        fixed = re.search(r"XML_NAMESPACE_([A-Z][A-Z0-9_]*)\s*,\s*(\w+)", body)
        if ns and ns.group(1) in params:
            ns_i = params.index(ns.group(1))
            tok = re.search(r"(?:xmloff::token::)?XML_##(\w+)", body)
            if tok and tok.group(1) in params:
                macros[name] = Macro(ns_i, None, params.index(tok.group(1)), "XML_")
                continue
            tok = re.search(r"xmloff::token::(\w+)", body)
            if tok and tok.group(1) in params:
                macros[name] = Macro(ns_i, None, params.index(tok.group(1)), "")
        elif fixed and fixed.group(2) in params:
            macros[name] = Macro(None, fixed.group(1), params.index(fixed.group(2)), "")
    return macros


def included(text):
    return re.findall(r'^[ \t]*#[ \t]*include[ \t]*["<]([^">]+)[">]', text, re.M)


KEYWORDS = {"if", "for", "while", "switch", "catch", "return", "sizeof", "else"}
DEF_TAIL = re.compile(r"\s*(?:const\s*)?(?:throw\s*\([^)]*\)\s*)?(?::[^{;]*)?\{")


def func_defs(m):
    """(name, parameter identifiers, body start, body end) of function definitions."""
    out = []
    for mm in re.finditer(r"\b([A-Za-z_]\w*)\s*\(", m):
        if mm.group(1) in KEYWORDS:
            continue
        o = mm.end() - 1
        c = match_close(m, o)
        tail = DEF_TAIL.match(m, c + 1)
        if not tail:
            continue
        params = []
        for a, b in split_args(m, o, c):
            w = re.findall(r"\w+", re.sub(r"=.*", "", m[a:b], flags=re.S))
            params.append(w[-1] if len(w) > 1 else None)
        b0 = tail.end() - 1
        out.append((mm.group(1), params, b0, match_close(m, b0)))
    return out


class Wrapper:
    def __init__(self, tok_i, nparams, keys, ns_param, role):
        self.tok_i, self.nparams, self.keys, self.ns_param, self.role = tok_i, nparams, keys, ns_param, role


class Scanner:
    def __init__(self, main):
        self.main = main
        self.toks = parse_tokens(os.path.join(main, "xmloff/source/core/xmltoken.cxx"))
        self.pairs = collections.defaultdict(lambda: {"roles": set(), "mods": set(), "ex": []})
        self.unres, self.skipped, self.files = [], [], []
        self.header_macros = {}
        self.nskey_uri_global = {}
        self.ns_local = {}
        self.wrappers = collections.defaultdict(list)
        self.cache = {}
        self.funcs = {}

    def walk(self):
        out = []
        for d, ds, fs in os.walk(self.main):
            rel = os.path.relpath(d, self.main).replace(os.sep, "/") + "/"
            ds[:] = [x for x in ds if x not in SKIP_DIRS]
            if any(p in rel for p in SKIP_PATHS):
                continue
            for f in fs:
                if f.endswith(SRC_EXT):
                    out.append(os.path.join(d, f))
        return sorted(out)

    @staticmethod
    def eligible(path):
        base = os.path.basename(path)
        return base.lower() in KEEP_NAMES or not IMPORT_NAME.search(base)

    def uri(self, key, path):
        key = re.sub(r"_IDX$", "", key)
        uri = self.ns_local.get(path, {}).get(key) or self.nskey_uri_global.get(key)
        if uri:
            return uri
        if key == "NONE":
            return ""
        if key == "XML":
            return self.toks["XML_N_XML"]
        return None

    def prepare(self):
        texts = {}
        for p in self.walk():
            t = read(p)
            if "XML_NAMESPACE_" not in t:
                continue
            texts[p] = t
        for p, t in texts.items():
            if p.endswith((".hxx", ".h")):
                mc = parse_macros(t)
                if mc:
                    self.header_macros[os.path.basename(p)] = mc
        first = {}
        for p, t in sorted(texts.items(), key=lambda kv: kv[0]):
            if not self.eligible(p):
                continue
            self.files.append(p)
            m = mask(t)
            for key, tok in ns_adds(m, t, self.toks):
                self.ns_local.setdefault(p, {})[key] = tok
                if p.endswith("xmloff/source/core/xmlexp.cxx"):
                    self.nskey_uri_global[key] = tok
                else:
                    first.setdefault(key, tok)
        for key, tok in first.items():
            self.nskey_uri_global.setdefault(key, tok)
        self.texts = texts

    def names(self, arg_masked, arg_clean):
        """Names in a name argument, and whether anything was left unresolved."""
        found, rest = [], arg_masked
        for m in STR_RE.finditer(arg_clean):
            if m.group(1):
                found.append(m.group(1))
        for m in TOK_RE.finditer(arg_masked):
            if m.group(0) in self.toks and not m.group(0).startswith(("XML_NP_", "XML_N_")):
                found.append(self.toks[m.group(0)])
        if not found and re.search(r"\bXML_NP?_[A-Z]", arg_masked):
            return [], False
        rest = TOK_RE.sub(" ", STR_RE.sub(" ", arg_clean))
        rest = re.sub(r"(?:::)?(?:xmloff::token::|token::|GetXMLToken|RTL_CONSTASCII_USTRINGPARAM|"
                      r"OUString|createFromAscii|rtl::|sal_Char|static_cast<[^>]*>)", " ", rest)
        rest = re.sub(r"[\s()?:]|\bsal_True\b|\bsal_False\b", "", rest)
        return found, bool(rest) or not found

    def record(self, path, line, key, names, role, row):
        uri = self.uri(key, path)
        if uri is None:
            self.unres.append((path, line, "unmapped namespace key " + key, row))
            return
        mod = os.path.relpath(path, self.main).split(os.sep)[0]
        for n in names:
            e = self.pairs[(uri, n)]
            e["roles"].add(role)
            e["mods"].add(mod)
            e["ex"].append((os.path.relpath(path, self.main), line, role))

    def scan_file(self, path):
        text = self.texts[path]
        spans = directive_spans(text)
        m = blank(mask(text), spans)
        clean = blank(text, spans)
        macros = {}
        for inc in included(text):
            macros.update(self.header_macros.get(os.path.basename(inc), {}))
        macros.update(parse_macros(text))
        self.cache[path] = (m, clean, text)
        line_of = lambda i: text.count("\n", 0, i) + 1
        snip = lambda a, b: " ".join(clean[a:b].split())[:160]
        handled = set()
        opener = self.openers(m)

        funcs = self.funcs[path] = func_defs(m)
        tokmaps = [(t.end(), match_close(m, t.end() - 1))
                   for t in re.finditer(r"SvXMLTokenMapEntry\s+\w+\s*\[[^\]]*\]\s*=\s*\{", m)]
        in_tokmap = lambda i: any(a <= i < b for a, b in tokmaps)

        def wrap(ns_a, ns_b, nm_a, nm_b, role, i):
            """Register the enclosing function as a wrapper when it forwards its parameters."""
            inner = [f for f in funcs if f[2] < i < f[3]]
            if not inner:
                return False
            name, params, b0, b1 = min(inner, key=lambda f: f[3] - f[2])
            nm, ns = m[nm_a:nm_b].strip(), m[ns_a:ns_b].strip()
            if nm not in params or name in EMIT_FUNCS or role == "either":
                return False
            if NS_RE.fullmatch(ns):
                keys, ns_param = [NS_RE.fullmatch(ns).group(1)], None
            elif ns in params:
                keys, ns_param = None, ns
            else:
                return False
            module = os.path.relpath(path, self.main).split(os.sep)[0]
            self.wrappers[(module, name)].append(
                Wrapper(params.index(nm), len(params), keys, params.index(ns) if ns_param else None, role))
            return True

        def pair_from(ns_a, ns_b, nm_a, nm_b, role, ctx_a, ctx_b, i, fixed=None, wrapok=True):
            if in_tokmap(i):
                return
            keys = NS_RE.findall(m[ns_a:ns_b])
            names, bad = (fixed, False) if fixed else self.names(m[nm_a:nm_b], clean[nm_a:nm_b])
            row = snip(ctx_a, ctx_b)
            for k in NS_RE.finditer(m[ns_a:ns_b]):
                handled.add(ns_a + k.start())
            if (not keys or bad) and wrapok and not fixed and wrap(ns_a, ns_b, nm_a, nm_b, role, i):
                return
            if not keys:
                self.unres.append((path, line_of(i), "namespace not a literal key", row))
                return
            if bad:
                self.unres.append((path, line_of(i), "name not a literal token or string", row))
            if not names:
                return
            if NO_EXPORT.search(clean[ctx_a:ctx_b]):
                self.skipped.append((path, line_of(i), "not exported (flags)", row))
                return
            for k in keys:
                self.record(path, line_of(i), k, names, role, row)

        for name, params, b0, b1 in funcs:
            ns_fn = [f for f in funcs if f[0] == re.sub(r"Name$", "Namespace", name) and name.endswith("AttributeName")]
            if not ns_fn:
                continue
            ns_body = clean[ns_fn[0][2]:ns_fn[0][3]]
            default = re.findall(r"return\s+XML_NAMESPACE_([A-Z][A-Z0-9_]*)\s*;", ns_body)[-1:]
            special = {(y if x in ns_fn[0][1] else x): k for x, y, k in re.findall(
                r"if\s*\(\s*(\w+)\s*==\s*(\w+)\s*\)\s*return\s+XML_NAMESPACE_(\w+)", ns_body)}
            for cm in re.finditer(r'case\s+(\w+)\s*:\s*return\s+"([^"]+)"', clean[b0:b1]):
                key = special.get(cm.group(1)) or default and default[0]
                if key:
                    self.record(path, line_of(b0 + cm.start()), key, [cm.group(2)], "attribute", cm.group(0))

        for mm in re.finditer(r"\b(%s)\s*\(" % "|".join(EMIT_ATTR), m):
            o = mm.end() - 1
            c = match_close(m, o)
            a = split_args(m, o, c)
            if len(a) < 2 or (len(a) == 2 and "XML_NAMESPACE_" not in m[a[0][0]:a[0][1]]):
                continue
            if DECL_ARG.match(m[a[0][0]:a[0][1]].strip()):
                continue
            pair_from(a[0][0], a[0][1], a[1][0], a[1][1], "attribute", mm.start(), c + 1, mm.start())

        for mm in re.finditer(r"\bSvXMLElementExport\b\s*(?:\w+\s*)?\(", m):
            o = mm.end() - 1
            c = match_close(m, o)
            a = split_args(m, o, c)
            k = 2 if len(a) >= 6 else 1
            if len(a) < 3 or len(a) <= k + 1 or DECL_ARG.match(m[a[k][0]:a[k][1]].strip()):
                continue
            pair_from(a[k][0], a[k][1], a[k + 1][0], a[k + 1][1], "element", mm.start(), c + 1, mm.start())

        for mm in re.finditer(r"\bAddAttributeIdLegacy\s*\(", m):
            o = mm.end() - 1
            c = match_close(m, o)
            a = split_args(m, o, c)
            if len(a) == 2 and not DECL_ARG.match(m[a[0][0]:a[0][1]].strip()):
                pair_from(a[0][0], a[0][1], 0, 0, "attribute", mm.start(), c + 1, mm.start(), ["id"])

        for mm in re.finditer(r"\bGetQNameByKey\s*\(", m):
            o = mm.end() - 1
            c = match_close(m, o)
            a = split_args(m, o, c)
            up = enclosing_call(m, mm.start())
            if up and up[0] in EMIT_ATTR and up[1] >= (1 if up[2] == 2 else 2):
                self.skipped.append((path, line_of(mm.start()), "QName used as an attribute value", snip(mm.start(), c + 1)))
                for k in NS_RE.finditer(m[a[0][0]:a[0][1]]):
                    handled.add(a[0][0] + k.start())
            elif len(a) == 2 and STR_RE.search(clean[a[1][0]:a[1][1]]) and not TOK_RE.search(m[a[1][0]:a[1][1]]):
                self.skipped.append((path, line_of(mm.start()), "QName with literal local name (a value)", snip(mm.start(), c + 1)))
            elif len(a) == 2:
                pair_from(a[0][0], a[0][1], a[1][0], a[1][1], "either", mm.start(), c + 1, mm.start(), wrapok=False)

        for name, mac in macros.items():
            for mm in re.finditer(r"\b%s\s*\(" % re.escape(name), m):
                o = mm.end() - 1
                c = match_close(m, o)
                a = split_args(m, o, c)
                if len(a) <= max(mac.ns_i or 0, mac.tok_i):
                    continue
                if in_tokmap(mm.start()):
                    continue
                row = snip(mm.start(), c + 1)
                ns = mac.ns_fixed or m[a[mac.ns_i][0]:a[mac.ns_i][1]].strip()
                tk = mac.tok_prefix + clean[a[mac.tok_i][0]:a[mac.tok_i][1]].strip()
                names, bad = self.names(tk, tk)
                if not re.fullmatch(r"[A-Z][A-Z0-9_]*", ns) or bad:
                    self.unres.append((path, line_of(mm.start()), "macro argument not a known key/token", row))
                    continue
                body = clean[mm.start():c]
                if NO_EXPORT.search(body):
                    self.skipped.append((path, line_of(mm.start()), "not exported (flags)", row))
                    continue
                role = "element" if ELEMENT_ROW.search(body) else "attribute" if PROP_ROW.search(body) else "either"
                self.record(path, line_of(mm.start()), ns, names, role, row)

        for mm in NS_RE.finditer(m):
            if mm.start() in handled:
                continue
            j = mm.end()
            while j < len(m) and m[j] in " \t\r\n":
                j += 1
            if j >= len(m) or m[j] != ",":
                continue
            br = opener.get(mm.start())
            if br is None:
                continue
            bi, bc = br
            before = re.search(r"(\w+)\s*(\w*)\s*$", m[max(0, bi - 80):bi])
            if m[bi] == "(" and before and (before.group(1) in EMIT_FUNCS or before.group(1) == "SvXMLElementExport"
                                            or before.group(2) and before.group(1) == "SvXMLElementExport"):
                continue
            a = split_args(m, bi, bc)
            idx = next(n for n, (s, e) in enumerate(a) if s <= mm.start() < e)
            if idx + 1 >= len(a) or m[a[idx][0]:a[idx][1]].strip() != mm.group(0):
                continue
            row_clean = clean[bi:bc + 1]
            if m[bi] == "{" and re.search(r"\bXML_TOK_\w+", row_clean):
                continue
            if EVENT_ROW.match(row_clean):
                self.skipped.append((path, line_of(mm.start()), "event-name translation (a value, not a name)", snip(bi, bc + 1)))
                continue
            if m[bi] == "{" or PROP_ROW.search(row_clean):
                role = "element" if ELEMENT_ROW.search(row_clean) else "attribute"
                if not PROP_ROW.search(row_clean):
                    role = "either"
            else:
                role = "either"
            pair_from(a[idx][0], a[idx][1], a[idx + 1][0], a[idx + 1][1], role, bi, bc + 1, mm.start(), wrapok=False)

    def ns_default(self, module, fname, ns_param_i):
        """Default namespace key(s) of the ns parameter of fname, from its declarations."""
        out = set()
        for p, (m, clean, text) in self.cache.items():
            if os.path.relpath(p, self.main).split(os.sep)[0] != module:
                continue
            for mm in re.finditer(r"\b%s\s*\(" % re.escape(fname), m):
                o = mm.end() - 1
                a = split_args(m, o, match_close(m, o))
                if ns_param_i < len(a):
                    d = re.search(r"=\s*XML_NAMESPACE_([A-Z][A-Z0-9_]*)", m[a[ns_param_i][0]:a[ns_param_i][1]])
                    if d:
                        out.add(d.group(1))
        return out

    def scan_wrapper_calls(self, path, discover):
        """Resolve calls of wrapper functions; with discover, only register forwarders."""
        m, clean, text = self.cache[path]
        module = os.path.relpath(path, self.main).split(os.sep)[0]
        line_of = lambda i: text.count("\n", 0, i) + 1
        found = 0
        for (mod, fname), ws in list(self.wrappers.items()):
            if mod != module:
                continue
            for mm in re.finditer(r"\b%s\s*\(" % re.escape(fname), m):
                o = mm.end() - 1
                c = match_close(m, o)
                a = split_args(m, o, c)
                a0 = m[a[0][0]:a[0][1]].strip()
                if DEF_TAIL.match(m, c + 1) or re.fullmatch(r"[\w:]+(?:\s+[\w:*&]+)+\s*(=.*)?", a0, re.S):
                    continue
                row = " ".join(clean[mm.start():c + 1].split())[:160]
                res, fwd = set(), []
                for w in ws:
                    if not (w.tok_i < len(a) <= w.nparams):
                        continue
                    ta, tb = a[w.tok_i]
                    names, bad = self.names(m[ta:tb], clean[ta:tb])
                    if w.keys:
                        keys = tuple(w.keys)
                    elif w.ns_param < len(a):
                        k = NS_RE.fullmatch(m[a[w.ns_param][0]:a[w.ns_param][1]].strip())
                        keys = (k.group(1),) if k else None
                    else:
                        d = self.ns_default(module, fname, w.ns_param)
                        keys = tuple(d) if len(d) == 1 else None
                    if bad and keys:
                        fwd.append((m[ta:tb].strip(), keys, w.role))
                    res.add(None if bad or not keys else (tuple(names), keys, w.role))
                if discover:
                    inner = [f for f in self.funcs[path] if f[2] < mm.start() < f[3]]
                    if inner and len(fwd) == 1 and len(res) == 1:
                        name, params, b0, b1 = min(inner, key=lambda f: f[3] - f[2])
                        if fwd[0][0] in params and name not in EMIT_FUNCS:
                            nw = Wrapper(params.index(fwd[0][0]), len(params), list(fwd[0][1]), None, fwd[0][2])
                            old = self.wrappers[(module, name)]
                            if not any(x.tok_i == nw.tok_i and x.nparams == nw.nparams and x.keys == nw.keys for x in old):
                                old.append(nw)
                                found += 1
                    continue
                if len(res) != 1 or None in res:
                    if res and not (len(fwd) == 1 and any(f[2] < mm.start() < f[3] and fwd[0][0] in f[1] for f in self.funcs[path])):
                        self.unres.append((path, line_of(mm.start()), "wrapper %s() call not resolvable" % fname, row))
                    continue
                names, keys, role = res.pop()
                for k in keys:
                    self.record(path, line_of(mm.start()), k, list(names), role, row)
        return found

    @staticmethod
    def openers(m):
        """Map every XML_NAMESPACE_ match start to its innermost enclosing bracket."""
        starts = {x.start() for x in NS_RE.finditer(m)}
        out, stack = {}, []
        pairs = {}
        for i, c in enumerate(m):
            if c in "({":
                stack.append(i)
            elif c in ")}":
                if stack:
                    pairs[stack.pop()] = i
            elif i in starts:
                out[i] = stack[-1] if stack else None
        res = {}
        for s, o in out.items():
            if o is not None and o in pairs:
                res[s] = (o, pairs[o])
        return res

    def write(self, outdir):
        os.makedirs(outdir, exist_ok=True)
        rows = []
        for (uri, name), e in self.pairs.items():
            known = e["roles"] - {"either"}
            role = known.pop() if len(known) == 1 else "either"
            ex = sorted(e["ex"], key=lambda x: (x[2] == "either", x[0], x[1]))[0]
            rows.append((uri, name, role, ",".join(sorted(e["mods"])), "%s:%d" % ex[:2]))
        rows.sort()
        with open(os.path.join(outdir, "emit.tsv"), "w") as f:
            f.write("# ODF export names found statically in %d export-side C++ files under main/.\n" % len(self.files))
            f.write("# Rule: .cxx/.hxx/.h under main/ whose name has no 'imp'/'import' (impastp4.cxx kept), outside test/qa/workben/unotest/unoxml/solver,"
                    " xmloff/source/transform and filter/source/svg; shared tables (property maps, handlers) included;"
                    " import token-map rows (XML_TOK_) and rows flagged no-export dropped.\n")
            f.write("# role: element/attribute from the emitting call or property-map flags; either = unknown or seen as both."
                    " Empty ns_uri = XML_NAMESPACE_NONE.\n")
            f.write("# ns_uri\tlocal\trole\tmodules\texample_file:line\n")
            for r in rows:
                f.write("\t".join(r) + "\n")
        for name, lst in (("emit-unresolved.tsv", self.unres), ("emit-skipped.tsv", self.skipped)):
            with open(os.path.join(outdir, name), "w") as f:
                f.write("# file:line\treason\tsnippet\n")
                for p, ln, why, row in sorted(set(lst)):
                    f.write("%s:%d\t%s\t%s\n" % (os.path.relpath(p, self.main), ln, why, row))
        return rows


def main(argv):
    if len(argv) != 3:
        sys.exit(__doc__)
    sc = Scanner(argv[1])
    sc.prepare()
    for p in sc.files:
        sc.scan_file(p)
    for _ in range(5):
        if not sum(sc.scan_wrapper_calls(p, True) for p in sc.files):
            break
    for p in sc.files:
        sc.scan_wrapper_calls(p, False)
    rows = sc.write(argv[2])
    roles = collections.Counter(r[2] for r in rows)
    print("tokens in xmltoken.cxx: %d; files scanned: %d" % (len(sc.toks), len(sc.files)))
    print("distinct pairs: %d %s" % (len(rows), dict(roles)))
    print("unresolved sites: %d; skipped rows: %d" % (len(set(sc.unres)), len(set(sc.skipped))))


if __name__ == "__main__":
    main(sys.argv)