1#!/usr/bin/python
2# Copyright (C) 2017-2022 Free Software Foundation, Inc.
3# This file is part of the GNU C Library.
4#
5# The GNU C Library is free software; you can redistribute it and/or
6# modify it under the terms of the GNU Lesser General Public
7# License as published by the Free Software Foundation; either
8# version 2.1 of the License, or (at your option) any later version.
9#
10# The GNU C Library is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13# Lesser General Public License for more details.
14#
15# You should have received a copy of the GNU Lesser General Public
16# License along with the GNU C Library; if not, see
17# <https://www.gnu.org/licenses/>.
18"""Compare results of string functions
19
20Given a string benchmark result file, print a table with comparisons with a
21baseline.  The baseline is the first function, which typically is the builtin
22function.
23"""
24import matplotlib as mpl
25mpl.use('Agg')
26
27import sys
28import os
29import json
30import pylab
31import argparse
32import traceback
33
34try:
35    import jsonschema as validator
36except ImportError:
37    print('Could not find jsonschema module.')
38    raise
39
40
41def parse_file(filename, schema_filename):
42    try:
43        with open(schema_filename, 'r') as schemafile:
44            schema = json.load(schemafile)
45            with open(filename, 'r') as benchfile:
46                bench = json.load(benchfile)
47                validator.validate(bench, schema)
48        return bench
49    except:
50        print(traceback.format_exc(limit=1))
51        sys.exit(os.EX_NOINPUT)
52
53def draw_graph(f, v, ifuncs, results):
54    """Plot graphs for functions
55
56    Plot line graphs for each of the ifuncs
57
58    Args:
59        f: Function name
60        v: Benchmark variant for the function.
61        ifuncs: List of ifunc names
62        results: Dictionary of results for each test criterion
63    """
64    print('Generating graph for %s, variant \'%s\'' % (f, v))
65    xkeys = results.keys()
66
67    pylab.clf()
68    fig = pylab.figure(frameon=False)
69    fig.set_size_inches(32, 18)
70    pylab.ylabel('Performance improvement from base')
71    X = range(len(xkeys))
72    pylab.xticks(X, xkeys)
73
74    i = 0
75
76    while i < len(ifuncs):
77        Y = [results[k][i] for k in xkeys]
78        lines = pylab.plot(X, Y, label=':'+ifuncs[i])
79        i = i + 1
80
81    pylab.legend()
82    pylab.grid()
83    pylab.savefig('%s-%s.png' % (f, v), bbox_inches='tight')
84
85
86def process_results(results, attrs, funcs, base_func, graph, no_diff, no_header):
87    """ Process results and print them
88
89    Args:
90        results: JSON dictionary of results
91        attrs: Attributes that form the test criteria
92        funcs: Functions that are selected
93    """
94
95    for f in results['functions'].keys():
96
97        v = results['functions'][f]['bench-variant']
98
99        selected = {}
100        index = 0
101        base_index = 0
102        if funcs:
103            ifuncs = []
104            first_func = True
105            for i in results['functions'][f]['ifuncs']:
106                if i in funcs:
107                    if first_func:
108                        base_index = index
109                        first_func = False
110                    selected[index] = 1
111                    ifuncs.append(i)
112                else:
113                    selected[index] = 0
114                index += 1
115        else:
116            ifuncs = results['functions'][f]['ifuncs']
117            for i in ifuncs:
118                selected[index] = 1
119                index += 1
120
121        if base_func:
122            try:
123                base_index = results['functions'][f]['ifuncs'].index(base_func)
124            except ValueError:
125                sys.stderr.write('Invalid -b "%s" parameter. Options: %s.\n' %
126                                 (base_func, ', '.join(results['functions'][f]['ifuncs'])))
127                sys.exit(os.EX_DATAERR)
128
129        if not no_header:
130            print('Function: %s' % f)
131            print('Variant: %s' % v)
132            print("%36s%s" % (' ', '\t'.join(ifuncs)))
133            print("=" * 120)
134
135        graph_res = {}
136        for res in results['functions'][f]['results']:
137            try:
138                attr_list = ['%s=%s' % (a, res[a]) for a in attrs]
139            except KeyError as ke:
140                sys.stderr.write('Invalid -a %s parameter. Options: %s.\n'
141                                 % (ke, ', '.join([a for a in res.keys() if a != 'timings'])))
142                sys.exit(os.EX_DATAERR)
143            i = 0
144            key = ', '.join(attr_list)
145            sys.stdout.write('%36s: ' % key)
146            graph_res[key] = res['timings']
147            for t in res['timings']:
148                if selected[i]:
149                    sys.stdout.write ('%12.2f' % t)
150                    if not no_diff:
151                        if i != base_index:
152                            base = res['timings'][base_index]
153                            diff = (base - t) * 100 / base
154                            sys.stdout.write (' (%6.2f%%)' % diff)
155                    sys.stdout.write('\t')
156                i = i + 1
157            print('')
158
159        if graph:
160            draw_graph(f, v, results['functions'][f]['ifuncs'], graph_res)
161
162
163def main(args):
164    """Program Entry Point
165
166    Take a string benchmark output file and compare timings.
167    """
168
169    base_func = None
170    filename = args.input
171    schema_filename = args.schema
172    base_func = args.base
173    attrs = args.attributes.split(',')
174    if args.functions:
175        funcs = args.functions.split(',')
176        if base_func and not base_func in funcs:
177            print('Baseline function (%s) not found.' % base_func)
178            sys.exit(os.EX_DATAERR)
179    else:
180        funcs = None
181
182    results = parse_file(args.input, args.schema)
183    process_results(results, attrs, funcs, base_func, args.graph, args.no_diff, args.no_header)
184    return os.EX_OK
185
186
187if __name__ == '__main__':
188    parser = argparse.ArgumentParser()
189
190    # The required arguments.
191    req = parser.add_argument_group(title='required arguments')
192    req.add_argument('-a', '--attributes', required=True,
193                        help='Comma separated list of benchmark attributes.')
194    req.add_argument('-i', '--input', required=True,
195                        help='Input JSON benchmark result file.')
196    req.add_argument('-s', '--schema', required=True,
197                        help='Schema file to validate the result file.')
198
199    # Optional arguments.
200    parser.add_argument('-f', '--functions',
201                        help='Comma separated list of functions.')
202    parser.add_argument('-b', '--base',
203                        help='IFUNC variant to set as baseline.')
204    parser.add_argument('-g', '--graph', action='store_true',
205                        help='Generate a graph from results.')
206    parser.add_argument('--no-diff', action='store_true',
207                        help='Do not print the difference from baseline.')
208    parser.add_argument('--no-header', action='store_true',
209                        help='Do not print the header.')
210
211    args = parser.parse_args()
212    sys.exit(main(args))
213