1# SPDX-License-Identifier: GPL-2.0
2
3from __future__ import print_function
4
5import os
6import sys
7import glob
8import optparse
9import tempfile
10import logging
11import shutil
12
13try:
14    import configparser
15except ImportError:
16    import ConfigParser as configparser
17
18def data_equal(a, b):
19    # Allow multiple values in assignment separated by '|'
20    a_list = a.split('|')
21    b_list = b.split('|')
22
23    for a_item in a_list:
24        for b_item in b_list:
25            if (a_item == b_item):
26                return True
27            elif (a_item == '*') or (b_item == '*'):
28                return True
29
30    return False
31
32class Fail(Exception):
33    def __init__(self, test, msg):
34        self.msg = msg
35        self.test = test
36    def getMsg(self):
37        return '\'%s\' - %s' % (self.test.path, self.msg)
38
39class Notest(Exception):
40    def __init__(self, test, arch):
41        self.arch = arch
42        self.test = test
43    def getMsg(self):
44        return '[%s] \'%s\'' % (self.arch, self.test.path)
45
46class Unsup(Exception):
47    def __init__(self, test):
48        self.test = test
49    def getMsg(self):
50        return '\'%s\'' % self.test.path
51
52class Event(dict):
53    terms = [
54        'cpu',
55        'flags',
56        'type',
57        'size',
58        'config',
59        'sample_period',
60        'sample_type',
61        'read_format',
62        'disabled',
63        'inherit',
64        'pinned',
65        'exclusive',
66        'exclude_user',
67        'exclude_kernel',
68        'exclude_hv',
69        'exclude_idle',
70        'mmap',
71        'comm',
72        'freq',
73        'inherit_stat',
74        'enable_on_exec',
75        'task',
76        'watermark',
77        'precise_ip',
78        'mmap_data',
79        'sample_id_all',
80        'exclude_host',
81        'exclude_guest',
82        'exclude_callchain_kernel',
83        'exclude_callchain_user',
84        'wakeup_events',
85        'bp_type',
86        'config1',
87        'config2',
88        'branch_sample_type',
89        'sample_regs_user',
90        'sample_stack_user',
91    ]
92
93    def add(self, data):
94        for key, val in data:
95            log.debug("      %s = %s" % (key, val))
96            self[key] = val
97
98    def __init__(self, name, data, base):
99        log.debug("    Event %s" % name);
100        self.name  = name;
101        self.group = ''
102        self.add(base)
103        self.add(data)
104
105    def equal(self, other):
106        for t in Event.terms:
107            log.debug("      [%s] %s %s" % (t, self[t], other[t]));
108            if t not in self or t not in other:
109                return False
110            if not data_equal(self[t], other[t]):
111                return False
112        return True
113
114    def optional(self):
115        if 'optional' in self and self['optional'] == '1':
116            return True
117        return False
118
119    def diff(self, other):
120        for t in Event.terms:
121            if t not in self or t not in other:
122                continue
123            if not data_equal(self[t], other[t]):
124                log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
125
126# Test file description needs to have following sections:
127# [config]
128#   - just single instance in file
129#   - needs to specify:
130#     'command' - perf command name
131#     'args'    - special command arguments
132#     'ret'     - expected command return value (0 by default)
133#     'arch'    - architecture specific test (optional)
134#                 comma separated list, ! at the beginning
135#                 negates it.
136#
137# [eventX:base]
138#   - one or multiple instances in file
139#   - expected values assignments
140class Test(object):
141    def __init__(self, path, options):
142        parser = configparser.SafeConfigParser()
143        parser.read(path)
144
145        log.warning("running '%s'" % path)
146
147        self.path     = path
148        self.test_dir = options.test_dir
149        self.perf     = options.perf
150        self.command  = parser.get('config', 'command')
151        self.args     = parser.get('config', 'args')
152
153        try:
154            self.ret  = parser.get('config', 'ret')
155        except:
156            self.ret  = 0
157
158        try:
159            self.arch  = parser.get('config', 'arch')
160            log.warning("test limitation '%s'" % self.arch)
161        except:
162            self.arch  = ''
163
164        self.expect   = {}
165        self.result   = {}
166        log.debug("  loading expected events");
167        self.load_events(path, self.expect)
168
169    def is_event(self, name):
170        if name.find("event") == -1:
171            return False
172        else:
173            return True
174
175    def skip_test(self, myarch):
176        # If architecture not set always run test
177        if self.arch == '':
178            # log.warning("test for arch %s is ok" % myarch)
179            return False
180
181        # Allow multiple values in assignment separated by ','
182        arch_list = self.arch.split(',')
183
184        # Handle negated list such as !s390x,ppc
185        if arch_list[0][0] == '!':
186            arch_list[0] = arch_list[0][1:]
187            log.warning("excluded architecture list %s" % arch_list)
188            for arch_item in arch_list:
189                # log.warning("test for %s arch is %s" % (arch_item, myarch))
190                if arch_item == myarch:
191                    return True
192            return False
193
194        for arch_item in arch_list:
195            # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
196            if arch_item == myarch:
197                return False
198        return True
199
200    def load_events(self, path, events):
201        parser_event = configparser.SafeConfigParser()
202        parser_event.read(path)
203
204        # The event record section header contains 'event' word,
205        # optionaly followed by ':' allowing to load 'parent
206        # event' first as a base
207        for section in filter(self.is_event, parser_event.sections()):
208
209            parser_items = parser_event.items(section);
210            base_items   = {}
211
212            # Read parent event if there's any
213            if (':' in section):
214                base = section[section.index(':') + 1:]
215                parser_base = configparser.SafeConfigParser()
216                parser_base.read(self.test_dir + '/' + base)
217                base_items = parser_base.items('event')
218
219            e = Event(section, parser_items, base_items)
220            events[section] = e
221
222    def run_cmd(self, tempdir):
223        junk1, junk2, junk3, junk4, myarch = (os.uname())
224
225        if self.skip_test(myarch):
226            raise Notest(self, myarch)
227
228        cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
229              self.perf, self.command, tempdir, self.args)
230        ret = os.WEXITSTATUS(os.system(cmd))
231
232        log.info("  '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
233
234        if not data_equal(str(ret), str(self.ret)):
235            raise Unsup(self)
236
237    def compare(self, expect, result):
238        match = {}
239
240        log.debug("  compare");
241
242        # For each expected event find all matching
243        # events in result. Fail if there's not any.
244        for exp_name, exp_event in expect.items():
245            exp_list = []
246            res_event = {}
247            log.debug("    matching [%s]" % exp_name)
248            for res_name, res_event in result.items():
249                log.debug("      to [%s]" % res_name)
250                if (exp_event.equal(res_event)):
251                    exp_list.append(res_name)
252                    log.debug("    ->OK")
253                else:
254                    log.debug("    ->FAIL");
255
256            log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
257
258            # we did not any matching event - fail
259            if not exp_list:
260                if exp_event.optional():
261                    log.debug("    %s does not match, but is optional" % exp_name)
262                else:
263                    if not res_event:
264                        log.debug("    res_event is empty");
265                    else:
266                        exp_event.diff(res_event)
267                    raise Fail(self, 'match failure');
268
269            match[exp_name] = exp_list
270
271        # For each defined group in the expected events
272        # check we match the same group in the result.
273        for exp_name, exp_event in expect.items():
274            group = exp_event.group
275
276            if (group == ''):
277                continue
278
279            for res_name in match[exp_name]:
280                res_group = result[res_name].group
281                if res_group not in match[group]:
282                    raise Fail(self, 'group failure')
283
284                log.debug("    group: [%s] matches group leader %s" %
285                         (exp_name, str(match[group])))
286
287        log.debug("  matched")
288
289    def resolve_groups(self, events):
290        for name, event in events.items():
291            group_fd = event['group_fd'];
292            if group_fd == '-1':
293                continue;
294
295            for iname, ievent in events.items():
296                if (ievent['fd'] == group_fd):
297                    event.group = iname
298                    log.debug('[%s] has group leader [%s]' % (name, iname))
299                    break;
300
301    def run(self):
302        tempdir = tempfile.mkdtemp();
303
304        try:
305            # run the test script
306            self.run_cmd(tempdir);
307
308            # load events expectation for the test
309            log.debug("  loading result events");
310            for f in glob.glob(tempdir + '/event*'):
311                self.load_events(f, self.result);
312
313            # resolve group_fd to event names
314            self.resolve_groups(self.expect);
315            self.resolve_groups(self.result);
316
317            # do the expectation - results matching - both ways
318            self.compare(self.expect, self.result)
319            self.compare(self.result, self.expect)
320
321        finally:
322            # cleanup
323            shutil.rmtree(tempdir)
324
325
326def run_tests(options):
327    for f in glob.glob(options.test_dir + '/' + options.test):
328        try:
329            Test(f, options).run()
330        except Unsup as obj:
331            log.warning("unsupp  %s" % obj.getMsg())
332        except Notest as obj:
333            log.warning("skipped %s" % obj.getMsg())
334
335def setup_log(verbose):
336    global log
337    level = logging.CRITICAL
338
339    if verbose == 1:
340        level = logging.WARNING
341    if verbose == 2:
342        level = logging.INFO
343    if verbose >= 3:
344        level = logging.DEBUG
345
346    log = logging.getLogger('test')
347    log.setLevel(level)
348    ch  = logging.StreamHandler()
349    ch.setLevel(level)
350    formatter = logging.Formatter('%(message)s')
351    ch.setFormatter(formatter)
352    log.addHandler(ch)
353
354USAGE = '''%s [OPTIONS]
355  -d dir  # tests dir
356  -p path # perf binary
357  -t test # single test
358  -v      # verbose level
359''' % sys.argv[0]
360
361def main():
362    parser = optparse.OptionParser(usage=USAGE)
363
364    parser.add_option("-t", "--test",
365                      action="store", type="string", dest="test")
366    parser.add_option("-d", "--test-dir",
367                      action="store", type="string", dest="test_dir")
368    parser.add_option("-p", "--perf",
369                      action="store", type="string", dest="perf")
370    parser.add_option("-v", "--verbose",
371                      default=0, action="count", dest="verbose")
372
373    options, args = parser.parse_args()
374    if args:
375        parser.error('FAILED wrong arguments %s' %  ' '.join(args))
376        return -1
377
378    setup_log(options.verbose)
379
380    if not options.test_dir:
381        print('FAILED no -d option specified')
382        sys.exit(-1)
383
384    if not options.test:
385        options.test = 'test*'
386
387    try:
388        run_tests(options)
389
390    except Fail as obj:
391        print("FAILED %s" % obj.getMsg())
392        sys.exit(-1)
393
394    sys.exit(0)
395
396if __name__ == '__main__':
397    main()
398