1 // SPDX-License-Identifier: GPL-2.0
2 /* Manage affinity to optimize IPIs inside the kernel perf API. */
3 #define _GNU_SOURCE 1
4 #include <sched.h>
5 #include <stdlib.h>
6 #include <linux/bitmap.h>
7 #include <linux/zalloc.h>
8 #include "perf.h"
9 #include "cpumap.h"
10 #include "affinity.h"
11
get_cpu_set_size(void)12 static int get_cpu_set_size(void)
13 {
14 int sz = cpu__max_cpu().cpu + 8 - 1;
15 /*
16 * sched_getaffinity doesn't like masks smaller than the kernel.
17 * Hopefully that's big enough.
18 */
19 if (sz < 4096)
20 sz = 4096;
21 return sz / 8;
22 }
23
affinity__setup(struct affinity * a)24 int affinity__setup(struct affinity *a)
25 {
26 int cpu_set_size = get_cpu_set_size();
27
28 a->orig_cpus = bitmap_zalloc(cpu_set_size * 8);
29 if (!a->orig_cpus)
30 return -1;
31 sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
32 a->sched_cpus = bitmap_zalloc(cpu_set_size * 8);
33 if (!a->sched_cpus) {
34 zfree(&a->orig_cpus);
35 return -1;
36 }
37 bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size);
38 a->changed = false;
39 return 0;
40 }
41
42 /*
43 * perf_event_open does an IPI internally to the target CPU.
44 * It is more efficient to change perf's affinity to the target
45 * CPU and then set up all events on that CPU, so we amortize
46 * CPU communication.
47 */
affinity__set(struct affinity * a,int cpu)48 void affinity__set(struct affinity *a, int cpu)
49 {
50 int cpu_set_size = get_cpu_set_size();
51
52 /*
53 * Return:
54 * - if cpu is -1
55 * - restrict out of bound access to sched_cpus
56 */
57 if (cpu == -1 || ((cpu >= (cpu_set_size * 8))))
58 return;
59
60 a->changed = true;
61 set_bit(cpu, a->sched_cpus);
62 /*
63 * We ignore errors because affinity is just an optimization.
64 * This could happen for example with isolated CPUs or cpusets.
65 * In this case the IPIs inside the kernel's perf API still work.
66 */
67 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus);
68 clear_bit(cpu, a->sched_cpus);
69 }
70
__affinity__cleanup(struct affinity * a)71 static void __affinity__cleanup(struct affinity *a)
72 {
73 int cpu_set_size = get_cpu_set_size();
74
75 if (a->changed)
76 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
77 zfree(&a->sched_cpus);
78 zfree(&a->orig_cpus);
79 }
80
affinity__cleanup(struct affinity * a)81 void affinity__cleanup(struct affinity *a)
82 {
83 if (a != NULL)
84 __affinity__cleanup(a);
85 }
86