1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Routines to identify caches on Intel CPU.
4 *
5 * Changes:
6 * Venkatesh Pallipadi : Adding cache identification through cpuid(4)
7 * Ashok Raj <ashok.raj@intel.com>: Work with CPU hotplug infrastructure.
8 * Andi Kleen / Andreas Herrmann : CPUID4 emulation on AMD.
9 */
10
11 #include <linux/slab.h>
12 #include <linux/cacheinfo.h>
13 #include <linux/cpu.h>
14 #include <linux/cpuhotplug.h>
15 #include <linux/sched.h>
16 #include <linux/capability.h>
17 #include <linux/sysfs.h>
18 #include <linux/pci.h>
19 #include <linux/stop_machine.h>
20
21 #include <asm/cpufeature.h>
22 #include <asm/cacheinfo.h>
23 #include <asm/amd_nb.h>
24 #include <asm/smp.h>
25 #include <asm/mtrr.h>
26 #include <asm/tlbflush.h>
27
28 #include "cpu.h"
29
30 #define LVL_1_INST 1
31 #define LVL_1_DATA 2
32 #define LVL_2 3
33 #define LVL_3 4
34 #define LVL_TRACE 5
35
36 /* Shared last level cache maps */
37 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
38
39 /* Shared L2 cache maps */
40 DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_l2c_shared_map);
41
42 static cpumask_var_t cpu_cacheinfo_mask;
43
44 /* Kernel controls MTRR and/or PAT MSRs. */
45 unsigned int memory_caching_control __ro_after_init;
46
47 struct _cache_table {
48 unsigned char descriptor;
49 char cache_type;
50 short size;
51 };
52
53 #define MB(x) ((x) * 1024)
54
55 /* All the cache descriptor types we care about (no TLB or
56 trace cache entries) */
57
58 static const struct _cache_table cache_table[] =
59 {
60 { 0x06, LVL_1_INST, 8 }, /* 4-way set assoc, 32 byte line size */
61 { 0x08, LVL_1_INST, 16 }, /* 4-way set assoc, 32 byte line size */
62 { 0x09, LVL_1_INST, 32 }, /* 4-way set assoc, 64 byte line size */
63 { 0x0a, LVL_1_DATA, 8 }, /* 2 way set assoc, 32 byte line size */
64 { 0x0c, LVL_1_DATA, 16 }, /* 4-way set assoc, 32 byte line size */
65 { 0x0d, LVL_1_DATA, 16 }, /* 4-way set assoc, 64 byte line size */
66 { 0x0e, LVL_1_DATA, 24 }, /* 6-way set assoc, 64 byte line size */
67 { 0x21, LVL_2, 256 }, /* 8-way set assoc, 64 byte line size */
68 { 0x22, LVL_3, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */
69 { 0x23, LVL_3, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */
70 { 0x25, LVL_3, MB(2) }, /* 8-way set assoc, sectored cache, 64 byte line size */
71 { 0x29, LVL_3, MB(4) }, /* 8-way set assoc, sectored cache, 64 byte line size */
72 { 0x2c, LVL_1_DATA, 32 }, /* 8-way set assoc, 64 byte line size */
73 { 0x30, LVL_1_INST, 32 }, /* 8-way set assoc, 64 byte line size */
74 { 0x39, LVL_2, 128 }, /* 4-way set assoc, sectored cache, 64 byte line size */
75 { 0x3a, LVL_2, 192 }, /* 6-way set assoc, sectored cache, 64 byte line size */
76 { 0x3b, LVL_2, 128 }, /* 2-way set assoc, sectored cache, 64 byte line size */
77 { 0x3c, LVL_2, 256 }, /* 4-way set assoc, sectored cache, 64 byte line size */
78 { 0x3d, LVL_2, 384 }, /* 6-way set assoc, sectored cache, 64 byte line size */
79 { 0x3e, LVL_2, 512 }, /* 4-way set assoc, sectored cache, 64 byte line size */
80 { 0x3f, LVL_2, 256 }, /* 2-way set assoc, 64 byte line size */
81 { 0x41, LVL_2, 128 }, /* 4-way set assoc, 32 byte line size */
82 { 0x42, LVL_2, 256 }, /* 4-way set assoc, 32 byte line size */
83 { 0x43, LVL_2, 512 }, /* 4-way set assoc, 32 byte line size */
84 { 0x44, LVL_2, MB(1) }, /* 4-way set assoc, 32 byte line size */
85 { 0x45, LVL_2, MB(2) }, /* 4-way set assoc, 32 byte line size */
86 { 0x46, LVL_3, MB(4) }, /* 4-way set assoc, 64 byte line size */
87 { 0x47, LVL_3, MB(8) }, /* 8-way set assoc, 64 byte line size */
88 { 0x48, LVL_2, MB(3) }, /* 12-way set assoc, 64 byte line size */
89 { 0x49, LVL_3, MB(4) }, /* 16-way set assoc, 64 byte line size */
90 { 0x4a, LVL_3, MB(6) }, /* 12-way set assoc, 64 byte line size */
91 { 0x4b, LVL_3, MB(8) }, /* 16-way set assoc, 64 byte line size */
92 { 0x4c, LVL_3, MB(12) }, /* 12-way set assoc, 64 byte line size */
93 { 0x4d, LVL_3, MB(16) }, /* 16-way set assoc, 64 byte line size */
94 { 0x4e, LVL_2, MB(6) }, /* 24-way set assoc, 64 byte line size */
95 { 0x60, LVL_1_DATA, 16 }, /* 8-way set assoc, sectored cache, 64 byte line size */
96 { 0x66, LVL_1_DATA, 8 }, /* 4-way set assoc, sectored cache, 64 byte line size */
97 { 0x67, LVL_1_DATA, 16 }, /* 4-way set assoc, sectored cache, 64 byte line size */
98 { 0x68, LVL_1_DATA, 32 }, /* 4-way set assoc, sectored cache, 64 byte line size */
99 { 0x70, LVL_TRACE, 12 }, /* 8-way set assoc */
100 { 0x71, LVL_TRACE, 16 }, /* 8-way set assoc */
101 { 0x72, LVL_TRACE, 32 }, /* 8-way set assoc */
102 { 0x73, LVL_TRACE, 64 }, /* 8-way set assoc */
103 { 0x78, LVL_2, MB(1) }, /* 4-way set assoc, 64 byte line size */
104 { 0x79, LVL_2, 128 }, /* 8-way set assoc, sectored cache, 64 byte line size */
105 { 0x7a, LVL_2, 256 }, /* 8-way set assoc, sectored cache, 64 byte line size */
106 { 0x7b, LVL_2, 512 }, /* 8-way set assoc, sectored cache, 64 byte line size */
107 { 0x7c, LVL_2, MB(1) }, /* 8-way set assoc, sectored cache, 64 byte line size */
108 { 0x7d, LVL_2, MB(2) }, /* 8-way set assoc, 64 byte line size */
109 { 0x7f, LVL_2, 512 }, /* 2-way set assoc, 64 byte line size */
110 { 0x80, LVL_2, 512 }, /* 8-way set assoc, 64 byte line size */
111 { 0x82, LVL_2, 256 }, /* 8-way set assoc, 32 byte line size */
112 { 0x83, LVL_2, 512 }, /* 8-way set assoc, 32 byte line size */
113 { 0x84, LVL_2, MB(1) }, /* 8-way set assoc, 32 byte line size */
114 { 0x85, LVL_2, MB(2) }, /* 8-way set assoc, 32 byte line size */
115 { 0x86, LVL_2, 512 }, /* 4-way set assoc, 64 byte line size */
116 { 0x87, LVL_2, MB(1) }, /* 8-way set assoc, 64 byte line size */
117 { 0xd0, LVL_3, 512 }, /* 4-way set assoc, 64 byte line size */
118 { 0xd1, LVL_3, MB(1) }, /* 4-way set assoc, 64 byte line size */
119 { 0xd2, LVL_3, MB(2) }, /* 4-way set assoc, 64 byte line size */
120 { 0xd6, LVL_3, MB(1) }, /* 8-way set assoc, 64 byte line size */
121 { 0xd7, LVL_3, MB(2) }, /* 8-way set assoc, 64 byte line size */
122 { 0xd8, LVL_3, MB(4) }, /* 12-way set assoc, 64 byte line size */
123 { 0xdc, LVL_3, MB(2) }, /* 12-way set assoc, 64 byte line size */
124 { 0xdd, LVL_3, MB(4) }, /* 12-way set assoc, 64 byte line size */
125 { 0xde, LVL_3, MB(8) }, /* 12-way set assoc, 64 byte line size */
126 { 0xe2, LVL_3, MB(2) }, /* 16-way set assoc, 64 byte line size */
127 { 0xe3, LVL_3, MB(4) }, /* 16-way set assoc, 64 byte line size */
128 { 0xe4, LVL_3, MB(8) }, /* 16-way set assoc, 64 byte line size */
129 { 0xea, LVL_3, MB(12) }, /* 24-way set assoc, 64 byte line size */
130 { 0xeb, LVL_3, MB(18) }, /* 24-way set assoc, 64 byte line size */
131 { 0xec, LVL_3, MB(24) }, /* 24-way set assoc, 64 byte line size */
132 { 0x00, 0, 0}
133 };
134
135
136 enum _cache_type {
137 CTYPE_NULL = 0,
138 CTYPE_DATA = 1,
139 CTYPE_INST = 2,
140 CTYPE_UNIFIED = 3
141 };
142
143 union _cpuid4_leaf_eax {
144 struct {
145 enum _cache_type type:5;
146 unsigned int level:3;
147 unsigned int is_self_initializing:1;
148 unsigned int is_fully_associative:1;
149 unsigned int reserved:4;
150 unsigned int num_threads_sharing:12;
151 unsigned int num_cores_on_die:6;
152 } split;
153 u32 full;
154 };
155
156 union _cpuid4_leaf_ebx {
157 struct {
158 unsigned int coherency_line_size:12;
159 unsigned int physical_line_partition:10;
160 unsigned int ways_of_associativity:10;
161 } split;
162 u32 full;
163 };
164
165 union _cpuid4_leaf_ecx {
166 struct {
167 unsigned int number_of_sets:32;
168 } split;
169 u32 full;
170 };
171
172 struct _cpuid4_info_regs {
173 union _cpuid4_leaf_eax eax;
174 union _cpuid4_leaf_ebx ebx;
175 union _cpuid4_leaf_ecx ecx;
176 unsigned int id;
177 unsigned long size;
178 struct amd_northbridge *nb;
179 };
180
181 static unsigned short num_cache_leaves;
182
183 /* AMD doesn't have CPUID4. Emulate it here to report the same
184 information to the user. This makes some assumptions about the machine:
185 L2 not shared, no SMT etc. that is currently true on AMD CPUs.
186
187 In theory the TLBs could be reported as fake type (they are in "dummy").
188 Maybe later */
189 union l1_cache {
190 struct {
191 unsigned line_size:8;
192 unsigned lines_per_tag:8;
193 unsigned assoc:8;
194 unsigned size_in_kb:8;
195 };
196 unsigned val;
197 };
198
199 union l2_cache {
200 struct {
201 unsigned line_size:8;
202 unsigned lines_per_tag:4;
203 unsigned assoc:4;
204 unsigned size_in_kb:16;
205 };
206 unsigned val;
207 };
208
209 union l3_cache {
210 struct {
211 unsigned line_size:8;
212 unsigned lines_per_tag:4;
213 unsigned assoc:4;
214 unsigned res:2;
215 unsigned size_encoded:14;
216 };
217 unsigned val;
218 };
219
220 static const unsigned short assocs[] = {
221 [1] = 1,
222 [2] = 2,
223 [4] = 4,
224 [6] = 8,
225 [8] = 16,
226 [0xa] = 32,
227 [0xb] = 48,
228 [0xc] = 64,
229 [0xd] = 96,
230 [0xe] = 128,
231 [0xf] = 0xffff /* fully associative - no way to show this currently */
232 };
233
234 static const unsigned char levels[] = { 1, 1, 2, 3 };
235 static const unsigned char types[] = { 1, 2, 3, 3 };
236
237 static const enum cache_type cache_type_map[] = {
238 [CTYPE_NULL] = CACHE_TYPE_NOCACHE,
239 [CTYPE_DATA] = CACHE_TYPE_DATA,
240 [CTYPE_INST] = CACHE_TYPE_INST,
241 [CTYPE_UNIFIED] = CACHE_TYPE_UNIFIED,
242 };
243
244 static void
amd_cpuid4(int leaf,union _cpuid4_leaf_eax * eax,union _cpuid4_leaf_ebx * ebx,union _cpuid4_leaf_ecx * ecx)245 amd_cpuid4(int leaf, union _cpuid4_leaf_eax *eax,
246 union _cpuid4_leaf_ebx *ebx,
247 union _cpuid4_leaf_ecx *ecx)
248 {
249 unsigned dummy;
250 unsigned line_size, lines_per_tag, assoc, size_in_kb;
251 union l1_cache l1i, l1d;
252 union l2_cache l2;
253 union l3_cache l3;
254 union l1_cache *l1 = &l1d;
255
256 eax->full = 0;
257 ebx->full = 0;
258 ecx->full = 0;
259
260 cpuid(0x80000005, &dummy, &dummy, &l1d.val, &l1i.val);
261 cpuid(0x80000006, &dummy, &dummy, &l2.val, &l3.val);
262
263 switch (leaf) {
264 case 1:
265 l1 = &l1i;
266 fallthrough;
267 case 0:
268 if (!l1->val)
269 return;
270 assoc = assocs[l1->assoc];
271 line_size = l1->line_size;
272 lines_per_tag = l1->lines_per_tag;
273 size_in_kb = l1->size_in_kb;
274 break;
275 case 2:
276 if (!l2.val)
277 return;
278 assoc = assocs[l2.assoc];
279 line_size = l2.line_size;
280 lines_per_tag = l2.lines_per_tag;
281 /* cpu_data has errata corrections for K7 applied */
282 size_in_kb = __this_cpu_read(cpu_info.x86_cache_size);
283 break;
284 case 3:
285 if (!l3.val)
286 return;
287 assoc = assocs[l3.assoc];
288 line_size = l3.line_size;
289 lines_per_tag = l3.lines_per_tag;
290 size_in_kb = l3.size_encoded * 512;
291 if (boot_cpu_has(X86_FEATURE_AMD_DCM)) {
292 size_in_kb = size_in_kb >> 1;
293 assoc = assoc >> 1;
294 }
295 break;
296 default:
297 return;
298 }
299
300 eax->split.is_self_initializing = 1;
301 eax->split.type = types[leaf];
302 eax->split.level = levels[leaf];
303 eax->split.num_threads_sharing = 0;
304 eax->split.num_cores_on_die = __this_cpu_read(cpu_info.x86_max_cores) - 1;
305
306
307 if (assoc == 0xffff)
308 eax->split.is_fully_associative = 1;
309 ebx->split.coherency_line_size = line_size - 1;
310 ebx->split.ways_of_associativity = assoc - 1;
311 ebx->split.physical_line_partition = lines_per_tag - 1;
312 ecx->split.number_of_sets = (size_in_kb * 1024) / line_size /
313 (ebx->split.ways_of_associativity + 1) - 1;
314 }
315
316 #if defined(CONFIG_AMD_NB) && defined(CONFIG_SYSFS)
317
318 /*
319 * L3 cache descriptors
320 */
amd_calc_l3_indices(struct amd_northbridge * nb)321 static void amd_calc_l3_indices(struct amd_northbridge *nb)
322 {
323 struct amd_l3_cache *l3 = &nb->l3_cache;
324 unsigned int sc0, sc1, sc2, sc3;
325 u32 val = 0;
326
327 pci_read_config_dword(nb->misc, 0x1C4, &val);
328
329 /* calculate subcache sizes */
330 l3->subcaches[0] = sc0 = !(val & BIT(0));
331 l3->subcaches[1] = sc1 = !(val & BIT(4));
332
333 if (boot_cpu_data.x86 == 0x15) {
334 l3->subcaches[0] = sc0 += !(val & BIT(1));
335 l3->subcaches[1] = sc1 += !(val & BIT(5));
336 }
337
338 l3->subcaches[2] = sc2 = !(val & BIT(8)) + !(val & BIT(9));
339 l3->subcaches[3] = sc3 = !(val & BIT(12)) + !(val & BIT(13));
340
341 l3->indices = (max(max3(sc0, sc1, sc2), sc3) << 10) - 1;
342 }
343
344 /*
345 * check whether a slot used for disabling an L3 index is occupied.
346 * @l3: L3 cache descriptor
347 * @slot: slot number (0..1)
348 *
349 * @returns: the disabled index if used or negative value if slot free.
350 */
amd_get_l3_disable_slot(struct amd_northbridge * nb,unsigned slot)351 static int amd_get_l3_disable_slot(struct amd_northbridge *nb, unsigned slot)
352 {
353 unsigned int reg = 0;
354
355 pci_read_config_dword(nb->misc, 0x1BC + slot * 4, ®);
356
357 /* check whether this slot is activated already */
358 if (reg & (3UL << 30))
359 return reg & 0xfff;
360
361 return -1;
362 }
363
show_cache_disable(struct cacheinfo * this_leaf,char * buf,unsigned int slot)364 static ssize_t show_cache_disable(struct cacheinfo *this_leaf, char *buf,
365 unsigned int slot)
366 {
367 int index;
368 struct amd_northbridge *nb = this_leaf->priv;
369
370 index = amd_get_l3_disable_slot(nb, slot);
371 if (index >= 0)
372 return sprintf(buf, "%d\n", index);
373
374 return sprintf(buf, "FREE\n");
375 }
376
377 #define SHOW_CACHE_DISABLE(slot) \
378 static ssize_t \
379 cache_disable_##slot##_show(struct device *dev, \
380 struct device_attribute *attr, char *buf) \
381 { \
382 struct cacheinfo *this_leaf = dev_get_drvdata(dev); \
383 return show_cache_disable(this_leaf, buf, slot); \
384 }
385 SHOW_CACHE_DISABLE(0)
386 SHOW_CACHE_DISABLE(1)
387
amd_l3_disable_index(struct amd_northbridge * nb,int cpu,unsigned slot,unsigned long idx)388 static void amd_l3_disable_index(struct amd_northbridge *nb, int cpu,
389 unsigned slot, unsigned long idx)
390 {
391 int i;
392
393 idx |= BIT(30);
394
395 /*
396 * disable index in all 4 subcaches
397 */
398 for (i = 0; i < 4; i++) {
399 u32 reg = idx | (i << 20);
400
401 if (!nb->l3_cache.subcaches[i])
402 continue;
403
404 pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
405
406 /*
407 * We need to WBINVD on a core on the node containing the L3
408 * cache which indices we disable therefore a simple wbinvd()
409 * is not sufficient.
410 */
411 wbinvd_on_cpu(cpu);
412
413 reg |= BIT(31);
414 pci_write_config_dword(nb->misc, 0x1BC + slot * 4, reg);
415 }
416 }
417
418 /*
419 * disable a L3 cache index by using a disable-slot
420 *
421 * @l3: L3 cache descriptor
422 * @cpu: A CPU on the node containing the L3 cache
423 * @slot: slot number (0..1)
424 * @index: index to disable
425 *
426 * @return: 0 on success, error status on failure
427 */
amd_set_l3_disable_slot(struct amd_northbridge * nb,int cpu,unsigned slot,unsigned long index)428 static int amd_set_l3_disable_slot(struct amd_northbridge *nb, int cpu,
429 unsigned slot, unsigned long index)
430 {
431 int ret = 0;
432
433 /* check if @slot is already used or the index is already disabled */
434 ret = amd_get_l3_disable_slot(nb, slot);
435 if (ret >= 0)
436 return -EEXIST;
437
438 if (index > nb->l3_cache.indices)
439 return -EINVAL;
440
441 /* check whether the other slot has disabled the same index already */
442 if (index == amd_get_l3_disable_slot(nb, !slot))
443 return -EEXIST;
444
445 amd_l3_disable_index(nb, cpu, slot, index);
446
447 return 0;
448 }
449
store_cache_disable(struct cacheinfo * this_leaf,const char * buf,size_t count,unsigned int slot)450 static ssize_t store_cache_disable(struct cacheinfo *this_leaf,
451 const char *buf, size_t count,
452 unsigned int slot)
453 {
454 unsigned long val = 0;
455 int cpu, err = 0;
456 struct amd_northbridge *nb = this_leaf->priv;
457
458 if (!capable(CAP_SYS_ADMIN))
459 return -EPERM;
460
461 cpu = cpumask_first(&this_leaf->shared_cpu_map);
462
463 if (kstrtoul(buf, 10, &val) < 0)
464 return -EINVAL;
465
466 err = amd_set_l3_disable_slot(nb, cpu, slot, val);
467 if (err) {
468 if (err == -EEXIST)
469 pr_warn("L3 slot %d in use/index already disabled!\n",
470 slot);
471 return err;
472 }
473 return count;
474 }
475
476 #define STORE_CACHE_DISABLE(slot) \
477 static ssize_t \
478 cache_disable_##slot##_store(struct device *dev, \
479 struct device_attribute *attr, \
480 const char *buf, size_t count) \
481 { \
482 struct cacheinfo *this_leaf = dev_get_drvdata(dev); \
483 return store_cache_disable(this_leaf, buf, count, slot); \
484 }
485 STORE_CACHE_DISABLE(0)
486 STORE_CACHE_DISABLE(1)
487
subcaches_show(struct device * dev,struct device_attribute * attr,char * buf)488 static ssize_t subcaches_show(struct device *dev,
489 struct device_attribute *attr, char *buf)
490 {
491 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
492 int cpu = cpumask_first(&this_leaf->shared_cpu_map);
493
494 return sprintf(buf, "%x\n", amd_get_subcaches(cpu));
495 }
496
subcaches_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)497 static ssize_t subcaches_store(struct device *dev,
498 struct device_attribute *attr,
499 const char *buf, size_t count)
500 {
501 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
502 int cpu = cpumask_first(&this_leaf->shared_cpu_map);
503 unsigned long val;
504
505 if (!capable(CAP_SYS_ADMIN))
506 return -EPERM;
507
508 if (kstrtoul(buf, 16, &val) < 0)
509 return -EINVAL;
510
511 if (amd_set_subcaches(cpu, val))
512 return -EINVAL;
513
514 return count;
515 }
516
517 static DEVICE_ATTR_RW(cache_disable_0);
518 static DEVICE_ATTR_RW(cache_disable_1);
519 static DEVICE_ATTR_RW(subcaches);
520
521 static umode_t
cache_private_attrs_is_visible(struct kobject * kobj,struct attribute * attr,int unused)522 cache_private_attrs_is_visible(struct kobject *kobj,
523 struct attribute *attr, int unused)
524 {
525 struct device *dev = kobj_to_dev(kobj);
526 struct cacheinfo *this_leaf = dev_get_drvdata(dev);
527 umode_t mode = attr->mode;
528
529 if (!this_leaf->priv)
530 return 0;
531
532 if ((attr == &dev_attr_subcaches.attr) &&
533 amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
534 return mode;
535
536 if ((attr == &dev_attr_cache_disable_0.attr ||
537 attr == &dev_attr_cache_disable_1.attr) &&
538 amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
539 return mode;
540
541 return 0;
542 }
543
544 static struct attribute_group cache_private_group = {
545 .is_visible = cache_private_attrs_is_visible,
546 };
547
init_amd_l3_attrs(void)548 static void init_amd_l3_attrs(void)
549 {
550 int n = 1;
551 static struct attribute **amd_l3_attrs;
552
553 if (amd_l3_attrs) /* already initialized */
554 return;
555
556 if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE))
557 n += 2;
558 if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
559 n += 1;
560
561 amd_l3_attrs = kcalloc(n, sizeof(*amd_l3_attrs), GFP_KERNEL);
562 if (!amd_l3_attrs)
563 return;
564
565 n = 0;
566 if (amd_nb_has_feature(AMD_NB_L3_INDEX_DISABLE)) {
567 amd_l3_attrs[n++] = &dev_attr_cache_disable_0.attr;
568 amd_l3_attrs[n++] = &dev_attr_cache_disable_1.attr;
569 }
570 if (amd_nb_has_feature(AMD_NB_L3_PARTITIONING))
571 amd_l3_attrs[n++] = &dev_attr_subcaches.attr;
572
573 cache_private_group.attrs = amd_l3_attrs;
574 }
575
576 const struct attribute_group *
cache_get_priv_group(struct cacheinfo * this_leaf)577 cache_get_priv_group(struct cacheinfo *this_leaf)
578 {
579 struct amd_northbridge *nb = this_leaf->priv;
580
581 if (this_leaf->level < 3 || !nb)
582 return NULL;
583
584 if (nb && nb->l3_cache.indices)
585 init_amd_l3_attrs();
586
587 return &cache_private_group;
588 }
589
amd_init_l3_cache(struct _cpuid4_info_regs * this_leaf,int index)590 static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index)
591 {
592 int node;
593
594 /* only for L3, and not in virtualized environments */
595 if (index < 3)
596 return;
597
598 node = topology_die_id(smp_processor_id());
599 this_leaf->nb = node_to_amd_nb(node);
600 if (this_leaf->nb && !this_leaf->nb->l3_cache.indices)
601 amd_calc_l3_indices(this_leaf->nb);
602 }
603 #else
604 #define amd_init_l3_cache(x, y)
605 #endif /* CONFIG_AMD_NB && CONFIG_SYSFS */
606
607 static int
cpuid4_cache_lookup_regs(int index,struct _cpuid4_info_regs * this_leaf)608 cpuid4_cache_lookup_regs(int index, struct _cpuid4_info_regs *this_leaf)
609 {
610 union _cpuid4_leaf_eax eax;
611 union _cpuid4_leaf_ebx ebx;
612 union _cpuid4_leaf_ecx ecx;
613 unsigned edx;
614
615 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
616 if (boot_cpu_has(X86_FEATURE_TOPOEXT))
617 cpuid_count(0x8000001d, index, &eax.full,
618 &ebx.full, &ecx.full, &edx);
619 else
620 amd_cpuid4(index, &eax, &ebx, &ecx);
621 amd_init_l3_cache(this_leaf, index);
622 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
623 cpuid_count(0x8000001d, index, &eax.full,
624 &ebx.full, &ecx.full, &edx);
625 amd_init_l3_cache(this_leaf, index);
626 } else {
627 cpuid_count(4, index, &eax.full, &ebx.full, &ecx.full, &edx);
628 }
629
630 if (eax.split.type == CTYPE_NULL)
631 return -EIO; /* better error ? */
632
633 this_leaf->eax = eax;
634 this_leaf->ebx = ebx;
635 this_leaf->ecx = ecx;
636 this_leaf->size = (ecx.split.number_of_sets + 1) *
637 (ebx.split.coherency_line_size + 1) *
638 (ebx.split.physical_line_partition + 1) *
639 (ebx.split.ways_of_associativity + 1);
640 return 0;
641 }
642
find_num_cache_leaves(struct cpuinfo_x86 * c)643 static int find_num_cache_leaves(struct cpuinfo_x86 *c)
644 {
645 unsigned int eax, ebx, ecx, edx, op;
646 union _cpuid4_leaf_eax cache_eax;
647 int i = -1;
648
649 if (c->x86_vendor == X86_VENDOR_AMD ||
650 c->x86_vendor == X86_VENDOR_HYGON)
651 op = 0x8000001d;
652 else
653 op = 4;
654
655 do {
656 ++i;
657 /* Do cpuid(op) loop to find out num_cache_leaves */
658 cpuid_count(op, i, &eax, &ebx, &ecx, &edx);
659 cache_eax.full = eax;
660 } while (cache_eax.split.type != CTYPE_NULL);
661 return i;
662 }
663
cacheinfo_amd_init_llc_id(struct cpuinfo_x86 * c,int cpu)664 void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu)
665 {
666 /*
667 * We may have multiple LLCs if L3 caches exist, so check if we
668 * have an L3 cache by looking at the L3 cache CPUID leaf.
669 */
670 if (!cpuid_edx(0x80000006))
671 return;
672
673 if (c->x86 < 0x17) {
674 /* LLC is at the node level. */
675 per_cpu(cpu_llc_id, cpu) = c->cpu_die_id;
676 } else if (c->x86 == 0x17 && c->x86_model <= 0x1F) {
677 /*
678 * LLC is at the core complex level.
679 * Core complex ID is ApicId[3] for these processors.
680 */
681 per_cpu(cpu_llc_id, cpu) = c->apicid >> 3;
682 } else {
683 /*
684 * LLC ID is calculated from the number of threads sharing the
685 * cache.
686 * */
687 u32 eax, ebx, ecx, edx, num_sharing_cache = 0;
688 u32 llc_index = find_num_cache_leaves(c) - 1;
689
690 cpuid_count(0x8000001d, llc_index, &eax, &ebx, &ecx, &edx);
691 if (eax)
692 num_sharing_cache = ((eax >> 14) & 0xfff) + 1;
693
694 if (num_sharing_cache) {
695 int bits = get_count_order(num_sharing_cache);
696
697 per_cpu(cpu_llc_id, cpu) = c->apicid >> bits;
698 }
699 }
700 }
701
cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 * c,int cpu)702 void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu)
703 {
704 /*
705 * We may have multiple LLCs if L3 caches exist, so check if we
706 * have an L3 cache by looking at the L3 cache CPUID leaf.
707 */
708 if (!cpuid_edx(0x80000006))
709 return;
710
711 /*
712 * LLC is at the core complex level.
713 * Core complex ID is ApicId[3] for these processors.
714 */
715 per_cpu(cpu_llc_id, cpu) = c->apicid >> 3;
716 }
717
init_amd_cacheinfo(struct cpuinfo_x86 * c)718 void init_amd_cacheinfo(struct cpuinfo_x86 *c)
719 {
720
721 if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
722 num_cache_leaves = find_num_cache_leaves(c);
723 } else if (c->extended_cpuid_level >= 0x80000006) {
724 if (cpuid_edx(0x80000006) & 0xf000)
725 num_cache_leaves = 4;
726 else
727 num_cache_leaves = 3;
728 }
729 }
730
init_hygon_cacheinfo(struct cpuinfo_x86 * c)731 void init_hygon_cacheinfo(struct cpuinfo_x86 *c)
732 {
733 num_cache_leaves = find_num_cache_leaves(c);
734 }
735
init_intel_cacheinfo(struct cpuinfo_x86 * c)736 void init_intel_cacheinfo(struct cpuinfo_x86 *c)
737 {
738 /* Cache sizes */
739 unsigned int l1i = 0, l1d = 0, l2 = 0, l3 = 0;
740 unsigned int new_l1d = 0, new_l1i = 0; /* Cache sizes from cpuid(4) */
741 unsigned int new_l2 = 0, new_l3 = 0, i; /* Cache sizes from cpuid(4) */
742 unsigned int l2_id = 0, l3_id = 0, num_threads_sharing, index_msb;
743 #ifdef CONFIG_SMP
744 unsigned int cpu = c->cpu_index;
745 #endif
746
747 if (c->cpuid_level > 3) {
748 static int is_initialized;
749
750 if (is_initialized == 0) {
751 /* Init num_cache_leaves from boot CPU */
752 num_cache_leaves = find_num_cache_leaves(c);
753 is_initialized++;
754 }
755
756 /*
757 * Whenever possible use cpuid(4), deterministic cache
758 * parameters cpuid leaf to find the cache details
759 */
760 for (i = 0; i < num_cache_leaves; i++) {
761 struct _cpuid4_info_regs this_leaf = {};
762 int retval;
763
764 retval = cpuid4_cache_lookup_regs(i, &this_leaf);
765 if (retval < 0)
766 continue;
767
768 switch (this_leaf.eax.split.level) {
769 case 1:
770 if (this_leaf.eax.split.type == CTYPE_DATA)
771 new_l1d = this_leaf.size/1024;
772 else if (this_leaf.eax.split.type == CTYPE_INST)
773 new_l1i = this_leaf.size/1024;
774 break;
775 case 2:
776 new_l2 = this_leaf.size/1024;
777 num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
778 index_msb = get_count_order(num_threads_sharing);
779 l2_id = c->apicid & ~((1 << index_msb) - 1);
780 break;
781 case 3:
782 new_l3 = this_leaf.size/1024;
783 num_threads_sharing = 1 + this_leaf.eax.split.num_threads_sharing;
784 index_msb = get_count_order(num_threads_sharing);
785 l3_id = c->apicid & ~((1 << index_msb) - 1);
786 break;
787 default:
788 break;
789 }
790 }
791 }
792 /*
793 * Don't use cpuid2 if cpuid4 is supported. For P4, we use cpuid2 for
794 * trace cache
795 */
796 if ((num_cache_leaves == 0 || c->x86 == 15) && c->cpuid_level > 1) {
797 /* supports eax=2 call */
798 int j, n;
799 unsigned int regs[4];
800 unsigned char *dp = (unsigned char *)regs;
801 int only_trace = 0;
802
803 if (num_cache_leaves != 0 && c->x86 == 15)
804 only_trace = 1;
805
806 /* Number of times to iterate */
807 n = cpuid_eax(2) & 0xFF;
808
809 for (i = 0 ; i < n ; i++) {
810 cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]);
811
812 /* If bit 31 is set, this is an unknown format */
813 for (j = 0 ; j < 3 ; j++)
814 if (regs[j] & (1 << 31))
815 regs[j] = 0;
816
817 /* Byte 0 is level count, not a descriptor */
818 for (j = 1 ; j < 16 ; j++) {
819 unsigned char des = dp[j];
820 unsigned char k = 0;
821
822 /* look up this descriptor in the table */
823 while (cache_table[k].descriptor != 0) {
824 if (cache_table[k].descriptor == des) {
825 if (only_trace && cache_table[k].cache_type != LVL_TRACE)
826 break;
827 switch (cache_table[k].cache_type) {
828 case LVL_1_INST:
829 l1i += cache_table[k].size;
830 break;
831 case LVL_1_DATA:
832 l1d += cache_table[k].size;
833 break;
834 case LVL_2:
835 l2 += cache_table[k].size;
836 break;
837 case LVL_3:
838 l3 += cache_table[k].size;
839 break;
840 }
841
842 break;
843 }
844
845 k++;
846 }
847 }
848 }
849 }
850
851 if (new_l1d)
852 l1d = new_l1d;
853
854 if (new_l1i)
855 l1i = new_l1i;
856
857 if (new_l2) {
858 l2 = new_l2;
859 #ifdef CONFIG_SMP
860 per_cpu(cpu_llc_id, cpu) = l2_id;
861 per_cpu(cpu_l2c_id, cpu) = l2_id;
862 #endif
863 }
864
865 if (new_l3) {
866 l3 = new_l3;
867 #ifdef CONFIG_SMP
868 per_cpu(cpu_llc_id, cpu) = l3_id;
869 #endif
870 }
871
872 #ifdef CONFIG_SMP
873 /*
874 * If cpu_llc_id is not yet set, this means cpuid_level < 4 which in
875 * turns means that the only possibility is SMT (as indicated in
876 * cpuid1). Since cpuid2 doesn't specify shared caches, and we know
877 * that SMT shares all caches, we can unconditionally set cpu_llc_id to
878 * c->phys_proc_id.
879 */
880 if (per_cpu(cpu_llc_id, cpu) == BAD_APICID)
881 per_cpu(cpu_llc_id, cpu) = c->phys_proc_id;
882 #endif
883
884 c->x86_cache_size = l3 ? l3 : (l2 ? l2 : (l1i+l1d));
885
886 if (!l2)
887 cpu_detect_cache_sizes(c);
888 }
889
__cache_amd_cpumap_setup(unsigned int cpu,int index,struct _cpuid4_info_regs * base)890 static int __cache_amd_cpumap_setup(unsigned int cpu, int index,
891 struct _cpuid4_info_regs *base)
892 {
893 struct cpu_cacheinfo *this_cpu_ci;
894 struct cacheinfo *this_leaf;
895 int i, sibling;
896
897 /*
898 * For L3, always use the pre-calculated cpu_llc_shared_mask
899 * to derive shared_cpu_map.
900 */
901 if (index == 3) {
902 for_each_cpu(i, cpu_llc_shared_mask(cpu)) {
903 this_cpu_ci = get_cpu_cacheinfo(i);
904 if (!this_cpu_ci->info_list)
905 continue;
906 this_leaf = this_cpu_ci->info_list + index;
907 for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) {
908 if (!cpu_online(sibling))
909 continue;
910 cpumask_set_cpu(sibling,
911 &this_leaf->shared_cpu_map);
912 }
913 }
914 } else if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
915 unsigned int apicid, nshared, first, last;
916
917 nshared = base->eax.split.num_threads_sharing + 1;
918 apicid = cpu_data(cpu).apicid;
919 first = apicid - (apicid % nshared);
920 last = first + nshared - 1;
921
922 for_each_online_cpu(i) {
923 this_cpu_ci = get_cpu_cacheinfo(i);
924 if (!this_cpu_ci->info_list)
925 continue;
926
927 apicid = cpu_data(i).apicid;
928 if ((apicid < first) || (apicid > last))
929 continue;
930
931 this_leaf = this_cpu_ci->info_list + index;
932
933 for_each_online_cpu(sibling) {
934 apicid = cpu_data(sibling).apicid;
935 if ((apicid < first) || (apicid > last))
936 continue;
937 cpumask_set_cpu(sibling,
938 &this_leaf->shared_cpu_map);
939 }
940 }
941 } else
942 return 0;
943
944 return 1;
945 }
946
__cache_cpumap_setup(unsigned int cpu,int index,struct _cpuid4_info_regs * base)947 static void __cache_cpumap_setup(unsigned int cpu, int index,
948 struct _cpuid4_info_regs *base)
949 {
950 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
951 struct cacheinfo *this_leaf, *sibling_leaf;
952 unsigned long num_threads_sharing;
953 int index_msb, i;
954 struct cpuinfo_x86 *c = &cpu_data(cpu);
955
956 if (c->x86_vendor == X86_VENDOR_AMD ||
957 c->x86_vendor == X86_VENDOR_HYGON) {
958 if (__cache_amd_cpumap_setup(cpu, index, base))
959 return;
960 }
961
962 this_leaf = this_cpu_ci->info_list + index;
963 num_threads_sharing = 1 + base->eax.split.num_threads_sharing;
964
965 cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map);
966 if (num_threads_sharing == 1)
967 return;
968
969 index_msb = get_count_order(num_threads_sharing);
970
971 for_each_online_cpu(i)
972 if (cpu_data(i).apicid >> index_msb == c->apicid >> index_msb) {
973 struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i);
974
975 if (i == cpu || !sib_cpu_ci->info_list)
976 continue;/* skip if itself or no cacheinfo */
977 sibling_leaf = sib_cpu_ci->info_list + index;
978 cpumask_set_cpu(i, &this_leaf->shared_cpu_map);
979 cpumask_set_cpu(cpu, &sibling_leaf->shared_cpu_map);
980 }
981 }
982
ci_leaf_init(struct cacheinfo * this_leaf,struct _cpuid4_info_regs * base)983 static void ci_leaf_init(struct cacheinfo *this_leaf,
984 struct _cpuid4_info_regs *base)
985 {
986 this_leaf->id = base->id;
987 this_leaf->attributes = CACHE_ID;
988 this_leaf->level = base->eax.split.level;
989 this_leaf->type = cache_type_map[base->eax.split.type];
990 this_leaf->coherency_line_size =
991 base->ebx.split.coherency_line_size + 1;
992 this_leaf->ways_of_associativity =
993 base->ebx.split.ways_of_associativity + 1;
994 this_leaf->size = base->size;
995 this_leaf->number_of_sets = base->ecx.split.number_of_sets + 1;
996 this_leaf->physical_line_partition =
997 base->ebx.split.physical_line_partition + 1;
998 this_leaf->priv = base->nb;
999 }
1000
init_cache_level(unsigned int cpu)1001 int init_cache_level(unsigned int cpu)
1002 {
1003 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
1004
1005 if (!num_cache_leaves)
1006 return -ENOENT;
1007 if (!this_cpu_ci)
1008 return -EINVAL;
1009 this_cpu_ci->num_levels = 3;
1010 this_cpu_ci->num_leaves = num_cache_leaves;
1011 return 0;
1012 }
1013
1014 /*
1015 * The max shared threads number comes from CPUID.4:EAX[25-14] with input
1016 * ECX as cache index. Then right shift apicid by the number's order to get
1017 * cache id for this cache node.
1018 */
get_cache_id(int cpu,struct _cpuid4_info_regs * id4_regs)1019 static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4_regs)
1020 {
1021 struct cpuinfo_x86 *c = &cpu_data(cpu);
1022 unsigned long num_threads_sharing;
1023 int index_msb;
1024
1025 num_threads_sharing = 1 + id4_regs->eax.split.num_threads_sharing;
1026 index_msb = get_count_order(num_threads_sharing);
1027 id4_regs->id = c->apicid >> index_msb;
1028 }
1029
populate_cache_leaves(unsigned int cpu)1030 int populate_cache_leaves(unsigned int cpu)
1031 {
1032 unsigned int idx, ret;
1033 struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
1034 struct cacheinfo *this_leaf = this_cpu_ci->info_list;
1035 struct _cpuid4_info_regs id4_regs = {};
1036
1037 for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) {
1038 ret = cpuid4_cache_lookup_regs(idx, &id4_regs);
1039 if (ret)
1040 return ret;
1041 get_cache_id(cpu, &id4_regs);
1042 ci_leaf_init(this_leaf++, &id4_regs);
1043 __cache_cpumap_setup(cpu, idx, &id4_regs);
1044 }
1045 this_cpu_ci->cpu_map_populated = true;
1046
1047 return 0;
1048 }
1049
1050 /*
1051 * Disable and enable caches. Needed for changing MTRRs and the PAT MSR.
1052 *
1053 * Since we are disabling the cache don't allow any interrupts,
1054 * they would run extremely slow and would only increase the pain.
1055 *
1056 * The caller must ensure that local interrupts are disabled and
1057 * are reenabled after cache_enable() has been called.
1058 */
1059 static unsigned long saved_cr4;
1060 static DEFINE_RAW_SPINLOCK(cache_disable_lock);
1061
cache_disable(void)1062 void cache_disable(void) __acquires(cache_disable_lock)
1063 {
1064 unsigned long cr0;
1065
1066 /*
1067 * Note that this is not ideal
1068 * since the cache is only flushed/disabled for this CPU while the
1069 * MTRRs are changed, but changing this requires more invasive
1070 * changes to the way the kernel boots
1071 */
1072
1073 raw_spin_lock(&cache_disable_lock);
1074
1075 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
1076 cr0 = read_cr0() | X86_CR0_CD;
1077 write_cr0(cr0);
1078
1079 /*
1080 * Cache flushing is the most time-consuming step when programming
1081 * the MTRRs. Fortunately, as per the Intel Software Development
1082 * Manual, we can skip it if the processor supports cache self-
1083 * snooping.
1084 */
1085 if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
1086 wbinvd();
1087
1088 /* Save value of CR4 and clear Page Global Enable (bit 7) */
1089 if (cpu_feature_enabled(X86_FEATURE_PGE)) {
1090 saved_cr4 = __read_cr4();
1091 __write_cr4(saved_cr4 & ~X86_CR4_PGE);
1092 }
1093
1094 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
1095 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
1096 flush_tlb_local();
1097
1098 if (cpu_feature_enabled(X86_FEATURE_MTRR))
1099 mtrr_disable();
1100
1101 /* Again, only flush caches if we have to. */
1102 if (!static_cpu_has(X86_FEATURE_SELFSNOOP))
1103 wbinvd();
1104 }
1105
cache_enable(void)1106 void cache_enable(void) __releases(cache_disable_lock)
1107 {
1108 /* Flush TLBs (no need to flush caches - they are disabled) */
1109 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
1110 flush_tlb_local();
1111
1112 if (cpu_feature_enabled(X86_FEATURE_MTRR))
1113 mtrr_enable();
1114
1115 /* Enable caches */
1116 write_cr0(read_cr0() & ~X86_CR0_CD);
1117
1118 /* Restore value of CR4 */
1119 if (cpu_feature_enabled(X86_FEATURE_PGE))
1120 __write_cr4(saved_cr4);
1121
1122 raw_spin_unlock(&cache_disable_lock);
1123 }
1124
cache_cpu_init(void)1125 static void cache_cpu_init(void)
1126 {
1127 unsigned long flags;
1128
1129 local_irq_save(flags);
1130 cache_disable();
1131
1132 if (memory_caching_control & CACHE_MTRR)
1133 mtrr_generic_set_state();
1134
1135 if (memory_caching_control & CACHE_PAT)
1136 pat_cpu_init();
1137
1138 cache_enable();
1139 local_irq_restore(flags);
1140 }
1141
1142 static bool cache_aps_delayed_init = true;
1143
set_cache_aps_delayed_init(bool val)1144 void set_cache_aps_delayed_init(bool val)
1145 {
1146 cache_aps_delayed_init = val;
1147 }
1148
get_cache_aps_delayed_init(void)1149 bool get_cache_aps_delayed_init(void)
1150 {
1151 return cache_aps_delayed_init;
1152 }
1153
cache_rendezvous_handler(void * unused)1154 static int cache_rendezvous_handler(void *unused)
1155 {
1156 if (get_cache_aps_delayed_init() || !cpu_online(smp_processor_id()))
1157 cache_cpu_init();
1158
1159 return 0;
1160 }
1161
cache_bp_init(void)1162 void __init cache_bp_init(void)
1163 {
1164 mtrr_bp_init();
1165 pat_bp_init();
1166
1167 if (memory_caching_control)
1168 cache_cpu_init();
1169 }
1170
cache_bp_restore(void)1171 void cache_bp_restore(void)
1172 {
1173 if (memory_caching_control)
1174 cache_cpu_init();
1175 }
1176
cache_ap_online(unsigned int cpu)1177 static int cache_ap_online(unsigned int cpu)
1178 {
1179 cpumask_set_cpu(cpu, cpu_cacheinfo_mask);
1180
1181 if (!memory_caching_control || get_cache_aps_delayed_init())
1182 return 0;
1183
1184 /*
1185 * Ideally we should hold mtrr_mutex here to avoid MTRR entries
1186 * changed, but this routine will be called in CPU boot time,
1187 * holding the lock breaks it.
1188 *
1189 * This routine is called in two cases:
1190 *
1191 * 1. very early time of software resume, when there absolutely
1192 * isn't MTRR entry changes;
1193 *
1194 * 2. CPU hotadd time. We let mtrr_add/del_page hold cpuhotplug
1195 * lock to prevent MTRR entry changes
1196 */
1197 stop_machine_from_inactive_cpu(cache_rendezvous_handler, NULL,
1198 cpu_cacheinfo_mask);
1199
1200 return 0;
1201 }
1202
cache_ap_offline(unsigned int cpu)1203 static int cache_ap_offline(unsigned int cpu)
1204 {
1205 cpumask_clear_cpu(cpu, cpu_cacheinfo_mask);
1206 return 0;
1207 }
1208
1209 /*
1210 * Delayed cache initialization for all AP's
1211 */
cache_aps_init(void)1212 void cache_aps_init(void)
1213 {
1214 if (!memory_caching_control || !get_cache_aps_delayed_init())
1215 return;
1216
1217 stop_machine(cache_rendezvous_handler, NULL, cpu_online_mask);
1218 set_cache_aps_delayed_init(false);
1219 }
1220
cache_ap_register(void)1221 static int __init cache_ap_register(void)
1222 {
1223 zalloc_cpumask_var(&cpu_cacheinfo_mask, GFP_KERNEL);
1224 cpumask_set_cpu(smp_processor_id(), cpu_cacheinfo_mask);
1225
1226 cpuhp_setup_state_nocalls(CPUHP_AP_CACHECTRL_STARTING,
1227 "x86/cachectrl:starting",
1228 cache_ap_online, cache_ap_offline);
1229 return 0;
1230 }
1231 early_initcall(cache_ap_register);
1232