1#!/usr/bin/env python3
2# SPDX-License-Identifier: LGPL-2.1-or-later
3
4import argparse
5import collections
6import sys
7import os
8import subprocess
9import io
10
11try:
12    from lxml import etree
13except ModuleNotFoundError as e:
14    etree = e
15
16try:
17    from shlex import join as shlex_join
18except ImportError as e:
19    shlex_join = e
20
21try:
22    from shlex import quote as shlex_quote
23except ImportError as e:
24    shlex_quote = e
25
26class NoCommand(Exception):
27    pass
28
29BORING_INTERFACES = [
30    'org.freedesktop.DBus.Peer',
31    'org.freedesktop.DBus.Introspectable',
32    'org.freedesktop.DBus.Properties',
33]
34RED = '\x1b[31m'
35GREEN = '\x1b[32m'
36YELLOW = '\x1b[33m'
37RESET = '\x1b[39m'
38
39def xml_parser():
40    return etree.XMLParser(no_network=True,
41                           remove_comments=False,
42                           strip_cdata=False,
43                           resolve_entities=False)
44
45def print_method(declarations, elem, *, prefix, file, is_signal=False):
46    name = elem.get('name')
47    klass = 'signal' if is_signal else 'method'
48    declarations[klass].append(name)
49
50    # @org.freedesktop.systemd1.Privileged("true")
51    # SetShowStatus(in  s mode);
52
53    for anno in elem.findall('./annotation'):
54        anno_name = anno.get('name')
55        anno_value = anno.get('value')
56        print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
57
58    print(f'''{prefix}{name}(''', file=file, end='')
59    lead = ',\n' + prefix + ' ' * len(name) + ' '
60
61    for num, arg in enumerate(elem.findall('./arg')):
62        argname = arg.get('name')
63
64        if argname is None:
65            if opts.print_errors:
66                print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
67            argname = 'UNNAMED'
68
69        type = arg.get('type')
70        if not is_signal:
71            direction = arg.get('direction')
72            print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
73        else:
74            print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')
75
76    print(f');', file=file)
77
78ACCESS_MAP = {
79    'read' : 'readonly',
80    'write' : 'readwrite',
81}
82
83def value_ellipsis(type):
84    if type == 's':
85        return "'...'";
86    if type[0] == 'a':
87        inner = value_ellipsis(type[1:])
88        return f"[{inner}{', ...' if inner != '...' else ''}]";
89    return '...'
90
91def print_property(declarations, elem, *, prefix, file):
92    name = elem.get('name')
93    type = elem.get('type')
94    access = elem.get('access')
95
96    declarations['property'].append(name)
97
98    # @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
99    # @org.freedesktop.systemd1.Privileged("true")
100    # readwrite b EnableWallMessages = false;
101
102    for anno in elem.findall('./annotation'):
103        anno_name = anno.get('name')
104        anno_value = anno.get('value')
105        print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
106
107    access = ACCESS_MAP.get(access, access)
108    print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)
109
110def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
111    name = iface.get('name')
112
113    is_boring = (name in BORING_INTERFACES or
114                 only_interface is not None and name != only_interface)
115
116    if is_boring and print_boring:
117        print(f'''{prefix}interface {name} {{ ... }};''', file=file)
118
119    elif not is_boring and not print_boring:
120        print(f'''{prefix}interface {name} {{''', file=file)
121        prefix2 = prefix + '  '
122
123        for num, elem in enumerate(iface.findall('./method')):
124            if num == 0:
125                print(f'''{prefix2}methods:''', file=file)
126            print_method(declarations, elem, prefix=prefix2 + '  ', file=file)
127
128        for num, elem in enumerate(iface.findall('./signal')):
129            if num == 0:
130                print(f'''{prefix2}signals:''', file=file)
131            print_method(declarations, elem, prefix=prefix2 + '  ', file=file, is_signal=True)
132
133        for num, elem in enumerate(iface.findall('./property')):
134            if num == 0:
135                print(f'''{prefix2}properties:''', file=file)
136            print_property(declarations, elem, prefix=prefix2 + '  ', file=file)
137
138        print(f'''{prefix}}};''', file=file)
139
140def document_has_elem_with_text(document, elem, item_repr):
141    predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
142    for loc in document.findall(predicate):
143        if loc.text == item_repr:
144            return True
145    return False
146
147def check_documented(document, declarations, stats):
148    missing = []
149    for klass, items in declarations.items():
150        stats['total'] += len(items)
151
152        for item in items:
153            if klass == 'method':
154                elem = 'function'
155                item_repr = f'{item}()'
156            elif klass == 'signal':
157                elem = 'function'
158                item_repr = item
159            elif klass == 'property':
160                elem = 'varname'
161                item_repr = item
162            else:
163                assert False, (klass, item)
164
165            if not document_has_elem_with_text(document, elem, item_repr):
166                if opts.print_errors:
167                    print(f'{klass} {item} is not documented :(')
168                missing.append((klass, item))
169
170    stats['missing'] += len(missing)
171
172    return missing
173
174def xml_to_text(destination, xml, *, only_interface=None):
175    file = io.StringIO()
176
177    declarations = collections.defaultdict(list)
178    interfaces = []
179
180    print(f'''node {destination} {{''', file=file)
181
182    for print_boring in [False, True]:
183        for iface in xml.findall('./interface'):
184            print_interface(iface, prefix='  ', file=file,
185                            print_boring=print_boring,
186                            only_interface=only_interface,
187                            declarations=declarations)
188            name = iface.get('name')
189            if not name in BORING_INTERFACES:
190                interfaces.append(name)
191
192    print(f'''}};''', file=file)
193
194    return file.getvalue(), declarations, interfaces
195
196def subst_output(document, programlisting, stats):
197    executable = programlisting.get('executable', None)
198    if executable is None:
199        # Not our thing
200        return
201    executable = programlisting.get('executable')
202    node = programlisting.get('node')
203    interface = programlisting.get('interface')
204
205    argv = [f'{opts.build_dir}/{executable}', f'--bus-introspect={interface}']
206    if isinstance(shlex_join, Exception):
207        print(f'COMMAND: {" ".join(shlex_quote(arg) for arg in argv)}')
208    else:
209        print(f'COMMAND: {shlex_join(argv)}')
210
211    try:
212        out = subprocess.check_output(argv, universal_newlines=True)
213    except FileNotFoundError:
214        print(f'{executable} not found, ignoring', file=sys.stderr)
215        return
216
217    xml = etree.fromstring(out, parser=xml_parser())
218
219    new_text, declarations, interfaces = xml_to_text(node, xml, only_interface=interface)
220    programlisting.text = '\n' + new_text + '    '
221
222    if declarations:
223        missing = check_documented(document, declarations, stats)
224        parent = programlisting.getparent()
225
226        # delete old comments
227        for child in parent:
228            if (child.tag == etree.Comment
229                and 'Autogenerated' in child.text):
230                parent.remove(child)
231            if (child.tag == etree.Comment
232                and 'not documented' in child.text):
233                parent.remove(child)
234            if (child.tag == "variablelist"
235                and child.attrib.get("generated",False) == "True"):
236                parent.remove(child)
237
238        # insert pointer for systemd-directives generation
239        the_tail = programlisting.tail #tail is erased by addnext, so save it here.
240        prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
241        programlisting.addnext(prev_element)
242        programlisting.tail = the_tail
243
244        for interface in interfaces:
245            variablelist = etree.Element("variablelist")
246            variablelist.attrib['class'] = 'dbus-interface'
247            variablelist.attrib['generated'] = 'True'
248            variablelist.attrib['extra-ref'] = interface
249
250            prev_element.addnext(variablelist)
251            prev_element.tail = the_tail
252            prev_element = variablelist
253
254        for decl_type,decl_list in declarations.items():
255            for declaration in decl_list:
256                variablelist = etree.Element("variablelist")
257                variablelist.attrib['class'] = 'dbus-'+decl_type
258                variablelist.attrib['generated'] = 'True'
259                if decl_type == 'method' :
260                    variablelist.attrib['extra-ref'] = declaration + '()'
261                else:
262                    variablelist.attrib['extra-ref'] = declaration
263
264                prev_element.addnext(variablelist)
265                prev_element.tail = the_tail
266                prev_element = variablelist
267
268        last_element = etree.Comment("End of Autogenerated section")
269        prev_element.addnext(last_element)
270        prev_element.tail = the_tail
271        last_element.tail = the_tail
272
273        # insert comments for undocumented items
274        for item in reversed(missing):
275            comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
276            comment.tail = programlisting.tail
277            parent.insert(parent.index(programlisting) + 1, comment)
278
279def process(page):
280    src = open(page).read()
281    xml = etree.fromstring(src, parser=xml_parser())
282
283    # print('parsing {}'.format(name), file=sys.stderr)
284    if xml.tag != 'refentry':
285        return
286
287    stats = collections.Counter()
288
289    pls = xml.findall('.//programlisting')
290    for pl in pls:
291        subst_output(xml, pl, stats)
292
293    out_text = etree.tostring(xml, encoding='unicode')
294    # massage format to avoid some lxml whitespace handling idiosyncrasies
295    # https://bugs.launchpad.net/lxml/+bug/526799
296    out_text = (src[:src.find('<refentryinfo')] +
297                out_text[out_text.find('<refentryinfo'):] +
298                '\n')
299
300    if not opts.test:
301        with open(page, 'w') as out:
302            out.write(out_text)
303
304    return dict(stats=stats, modified=(out_text != src))
305
306def parse_args():
307    p = argparse.ArgumentParser()
308    p.add_argument('--test', action='store_true',
309                   help='only verify that everything is up2date')
310    p.add_argument('--build-dir', default='build')
311    p.add_argument('pages', nargs='+')
312    opts = p.parse_args()
313    opts.print_errors = not opts.test
314    return opts
315
316if __name__ == '__main__':
317    opts = parse_args()
318
319    for item in (etree, shlex_quote):
320        if isinstance(item, Exception):
321            print(item, file=sys.stderr)
322            exit(77 if opts.test else 1)
323
324    if not os.path.exists(f'{opts.build_dir}/systemd'):
325        exit(f"{opts.build_dir}/systemd doesn't exist. Use --build-dir=.")
326
327    stats = {page.split('/')[-1] : process(page) for page in opts.pages}
328
329    # Let's print all statistics at the end
330    mlen = max(len(page) for page in stats)
331    total = sum((item['stats'] for item in stats.values()), collections.Counter())
332    total = 'total', dict(stats=total, modified=False)
333    modified = []
334    classification = 'OUTDATED' if opts.test else 'MODIFIED'
335    for page, info in sorted(stats.items()) + [total]:
336        m = info['stats']['missing']
337        t = info['stats']['total']
338        p = page + ':'
339        c = classification if info['modified'] else ''
340        if c:
341            modified.append(page)
342        color = RED if m > t/2 else (YELLOW if m else GREEN)
343        print(f'{color}{p:{mlen + 1}} {t - m}/{t} {c}{RESET}')
344
345    if opts.test and modified:
346        exit(f'Outdated pages: {", ".join(modified)}\n'
347             f'Hint: ninja -C {opts.build_dir} update-dbus-docs')
348