1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2022, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17 #include <linux/cpu.h>
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/suspend.h>
21 #include <asm/cpu_device_id.h>
22 #include <asm/intel-family.h>
23
24 #include "uncore-frequency-common.h"
25
26 /* Max instances for uncore data, one for each die */
27 static int uncore_max_entries __read_mostly;
28 /* Storage for uncore data for all instances */
29 static struct uncore_data *uncore_instances;
30 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
31 static cpumask_t uncore_cpu_mask;
32 /* CPU online callback register instance */
33 static enum cpuhp_state uncore_hp_state __read_mostly;
34
35 #define MSR_UNCORE_RATIO_LIMIT 0x620
36 #define MSR_UNCORE_PERF_STATUS 0x621
37 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
38
uncore_read_control_freq(struct uncore_data * data,unsigned int * min,unsigned int * max)39 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
40 unsigned int *max)
41 {
42 u64 cap;
43 int ret;
44
45 if (data->control_cpu < 0)
46 return -ENXIO;
47
48 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
49 if (ret)
50 return ret;
51
52 *max = (cap & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
53 *min = ((cap & GENMASK(14, 8)) >> 8) * UNCORE_FREQ_KHZ_MULTIPLIER;
54
55 return 0;
56 }
57
uncore_write_control_freq(struct uncore_data * data,unsigned int input,unsigned int min_max)58 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
59 unsigned int min_max)
60 {
61 int ret;
62 u64 cap;
63
64 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
65 if (!input || input > 0x7F)
66 return -EINVAL;
67
68 if (data->control_cpu < 0)
69 return -ENXIO;
70
71 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
72 if (ret)
73 return ret;
74
75 if (min_max) {
76 cap &= ~0x7F;
77 cap |= input;
78 } else {
79 cap &= ~GENMASK(14, 8);
80 cap |= (input << 8);
81 }
82
83 ret = wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
84 if (ret)
85 return ret;
86
87 data->stored_uncore_data = cap;
88
89 return 0;
90 }
91
uncore_read_freq(struct uncore_data * data,unsigned int * freq)92 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
93 {
94 u64 ratio;
95 int ret;
96
97 if (data->control_cpu < 0)
98 return -ENXIO;
99
100 ret = rdmsrl_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
101 if (ret)
102 return ret;
103
104 *freq = (ratio & 0x7F) * UNCORE_FREQ_KHZ_MULTIPLIER;
105
106 return 0;
107 }
108
109 /* Caller provides protection */
uncore_get_instance(unsigned int cpu)110 static struct uncore_data *uncore_get_instance(unsigned int cpu)
111 {
112 int id = topology_logical_die_id(cpu);
113
114 if (id >= 0 && id < uncore_max_entries)
115 return &uncore_instances[id];
116
117 return NULL;
118 }
119
uncore_event_cpu_online(unsigned int cpu)120 static int uncore_event_cpu_online(unsigned int cpu)
121 {
122 struct uncore_data *data;
123 int target;
124
125 /* Check if there is an online cpu in the package for uncore MSR */
126 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
127 if (target < nr_cpu_ids)
128 return 0;
129
130 /* Use this CPU on this die as a control CPU */
131 cpumask_set_cpu(cpu, &uncore_cpu_mask);
132
133 data = uncore_get_instance(cpu);
134 if (!data)
135 return 0;
136
137 data->package_id = topology_physical_package_id(cpu);
138 data->die_id = topology_die_id(cpu);
139
140 return uncore_freq_add_entry(data, cpu);
141 }
142
uncore_event_cpu_offline(unsigned int cpu)143 static int uncore_event_cpu_offline(unsigned int cpu)
144 {
145 struct uncore_data *data;
146 int target;
147
148 data = uncore_get_instance(cpu);
149 if (!data)
150 return 0;
151
152 /* Check if existing cpu is used for uncore MSRs */
153 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
154 return 0;
155
156 /* Find a new cpu to set uncore MSR */
157 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
158
159 if (target < nr_cpu_ids) {
160 cpumask_set_cpu(target, &uncore_cpu_mask);
161 uncore_freq_add_entry(data, target);
162 } else {
163 uncore_freq_remove_die_entry(data);
164 }
165
166 return 0;
167 }
168
uncore_pm_notify(struct notifier_block * nb,unsigned long mode,void * _unused)169 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
170 void *_unused)
171 {
172 int i;
173
174 switch (mode) {
175 case PM_POST_HIBERNATION:
176 case PM_POST_RESTORE:
177 case PM_POST_SUSPEND:
178 for (i = 0; i < uncore_max_entries; ++i) {
179 struct uncore_data *data = &uncore_instances[i];
180
181 if (!data || !data->valid || !data->stored_uncore_data)
182 return 0;
183
184 wrmsrl_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
185 data->stored_uncore_data);
186 }
187 break;
188 default:
189 break;
190 }
191 return 0;
192 }
193
194 static struct notifier_block uncore_pm_nb = {
195 .notifier_call = uncore_pm_notify,
196 };
197
198 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
199 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, NULL),
200 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, NULL),
201 X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, NULL),
202 X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, NULL),
203 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
204 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
205 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
206 X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, NULL),
207 {}
208 };
209 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
210
intel_uncore_init(void)211 static int __init intel_uncore_init(void)
212 {
213 const struct x86_cpu_id *id;
214 int ret;
215
216 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
217 return -ENODEV;
218
219 id = x86_match_cpu(intel_uncore_cpu_ids);
220 if (!id)
221 return -ENODEV;
222
223 uncore_max_entries = topology_max_packages() *
224 topology_max_die_per_package();
225 uncore_instances = kcalloc(uncore_max_entries,
226 sizeof(*uncore_instances), GFP_KERNEL);
227 if (!uncore_instances)
228 return -ENOMEM;
229
230 ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
231 uncore_read_freq);
232 if (ret)
233 goto err_free;
234
235 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
236 "platform/x86/uncore-freq:online",
237 uncore_event_cpu_online,
238 uncore_event_cpu_offline);
239 if (ret < 0)
240 goto err_rem_kobj;
241
242 uncore_hp_state = ret;
243
244 ret = register_pm_notifier(&uncore_pm_nb);
245 if (ret)
246 goto err_rem_state;
247
248 return 0;
249
250 err_rem_state:
251 cpuhp_remove_state(uncore_hp_state);
252 err_rem_kobj:
253 uncore_freq_common_exit();
254 err_free:
255 kfree(uncore_instances);
256
257 return ret;
258 }
module_init(intel_uncore_init)259 module_init(intel_uncore_init)
260
261 static void __exit intel_uncore_exit(void)
262 {
263 int i;
264
265 unregister_pm_notifier(&uncore_pm_nb);
266 cpuhp_remove_state(uncore_hp_state);
267 for (i = 0; i < uncore_max_entries; ++i)
268 uncore_freq_remove_die_entry(&uncore_instances[i]);
269 uncore_freq_common_exit();
270 kfree(uncore_instances);
271 }
272 module_exit(intel_uncore_exit)
273
274 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
275 MODULE_LICENSE("GPL v2");
276 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
277