1 /* Generic MTRR (Memory Type Range Register) driver.
2
3 Copyright (C) 1997-2000 Richard Gooch
4 Copyright (c) 2002 Patrick Mochel
5
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Library General Public
8 License as published by the Free Software Foundation; either
9 version 2 of the License, or (at your option) any later version.
10
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Library General Public License for more details.
15
16 You should have received a copy of the GNU Library General Public
17 License along with this library; if not, write to the Free
18 Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19
20 Richard Gooch may be reached by email at rgooch@atnf.csiro.au
21 The postal address is:
22 Richard Gooch, c/o ATNF, P. O. Box 76, Epping, N.S.W., 2121, Australia.
23
24 Source: "Pentium Pro Family Developer's Manual, Volume 3:
25 Operating System Writer's Guide" (Intel document number 242692),
26 section 11.11.7
27
28 This was cleaned and made readable by Patrick Mochel <mochel@osdl.org>
29 on 6-7 March 2002.
30 Source: Intel Architecture Software Developers Manual, Volume 3:
31 System Programming Guide; Section 9.11. (1997 edition - PPro).
32 */
33
34 #define DEBUG
35
36 #include <linux/types.h> /* FIXME: kvm_para.h needs this */
37
38 #include <linux/stop_machine.h>
39 #include <linux/kvm_para.h>
40 #include <linux/uaccess.h>
41 #include <linux/module.h>
42 #include <linux/mutex.h>
43 #include <linux/init.h>
44 #include <linux/sort.h>
45 #include <linux/cpu.h>
46 #include <linux/pci.h>
47 #include <linux/smp.h>
48 #include <linux/syscore_ops.h>
49
50 #include <asm/processor.h>
51 #include <asm/e820.h>
52 #include <asm/mtrr.h>
53 #include <asm/msr.h>
54
55 #include "mtrr.h"
56
57 u32 num_var_ranges;
58
59 unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
60 static DEFINE_MUTEX(mtrr_mutex);
61
62 u64 size_or_mask, size_and_mask;
63 static bool mtrr_aps_delayed_init;
64
65 static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
66
67 const struct mtrr_ops *mtrr_if;
68
69 static void set_mtrr(unsigned int reg, unsigned long base,
70 unsigned long size, mtrr_type type);
71
set_mtrr_ops(const struct mtrr_ops * ops)72 void set_mtrr_ops(const struct mtrr_ops *ops)
73 {
74 if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
75 mtrr_ops[ops->vendor] = ops;
76 }
77
78 /* Returns non-zero if we have the write-combining memory type */
have_wrcomb(void)79 static int have_wrcomb(void)
80 {
81 struct pci_dev *dev;
82 u8 rev;
83
84 dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
85 if (dev != NULL) {
86 /*
87 * ServerWorks LE chipsets < rev 6 have problems with
88 * write-combining. Don't allow it and leave room for other
89 * chipsets to be tagged
90 */
91 if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
92 dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) {
93 pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev);
94 if (rev <= 5) {
95 pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
96 pci_dev_put(dev);
97 return 0;
98 }
99 }
100 /*
101 * Intel 450NX errata # 23. Non ascending cacheline evictions to
102 * write combining memory may resulting in data corruption
103 */
104 if (dev->vendor == PCI_VENDOR_ID_INTEL &&
105 dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
106 pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
107 pci_dev_put(dev);
108 return 0;
109 }
110 pci_dev_put(dev);
111 }
112 return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
113 }
114
115 /* This function returns the number of variable MTRRs */
set_num_var_ranges(void)116 static void __init set_num_var_ranges(void)
117 {
118 unsigned long config = 0, dummy;
119
120 if (use_intel())
121 rdmsr(MSR_MTRRcap, config, dummy);
122 else if (is_cpu(AMD))
123 config = 2;
124 else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
125 config = 8;
126
127 num_var_ranges = config & 0xff;
128 }
129
init_table(void)130 static void __init init_table(void)
131 {
132 int i, max;
133
134 max = num_var_ranges;
135 for (i = 0; i < max; i++)
136 mtrr_usage_table[i] = 1;
137 }
138
139 struct set_mtrr_data {
140 atomic_t count;
141 atomic_t gate;
142 unsigned long smp_base;
143 unsigned long smp_size;
144 unsigned int smp_reg;
145 mtrr_type smp_type;
146 };
147
148 static DEFINE_PER_CPU(struct cpu_stop_work, mtrr_work);
149
150 /**
151 * mtrr_work_handler - Synchronisation handler. Executed by "other" CPUs.
152 * @info: pointer to mtrr configuration data
153 *
154 * Returns nothing.
155 */
mtrr_work_handler(void * info)156 static int mtrr_work_handler(void *info)
157 {
158 #ifdef CONFIG_SMP
159 struct set_mtrr_data *data = info;
160 unsigned long flags;
161
162 atomic_dec(&data->count);
163 while (!atomic_read(&data->gate))
164 cpu_relax();
165
166 local_irq_save(flags);
167
168 atomic_dec(&data->count);
169 while (atomic_read(&data->gate))
170 cpu_relax();
171
172 /* The master has cleared me to execute */
173 if (data->smp_reg != ~0U) {
174 mtrr_if->set(data->smp_reg, data->smp_base,
175 data->smp_size, data->smp_type);
176 } else if (mtrr_aps_delayed_init) {
177 /*
178 * Initialize the MTRRs inaddition to the synchronisation.
179 */
180 mtrr_if->set_all();
181 }
182
183 atomic_dec(&data->count);
184 while (!atomic_read(&data->gate))
185 cpu_relax();
186
187 atomic_dec(&data->count);
188 local_irq_restore(flags);
189 #endif
190 return 0;
191 }
192
types_compatible(mtrr_type type1,mtrr_type type2)193 static inline int types_compatible(mtrr_type type1, mtrr_type type2)
194 {
195 return type1 == MTRR_TYPE_UNCACHABLE ||
196 type2 == MTRR_TYPE_UNCACHABLE ||
197 (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
198 (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
199 }
200
201 /**
202 * set_mtrr - update mtrrs on all processors
203 * @reg: mtrr in question
204 * @base: mtrr base
205 * @size: mtrr size
206 * @type: mtrr type
207 *
208 * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly:
209 *
210 * 1. Queue work to do the following on all processors:
211 * 2. Disable Interrupts
212 * 3. Wait for all procs to do so
213 * 4. Enter no-fill cache mode
214 * 5. Flush caches
215 * 6. Clear PGE bit
216 * 7. Flush all TLBs
217 * 8. Disable all range registers
218 * 9. Update the MTRRs
219 * 10. Enable all range registers
220 * 11. Flush all TLBs and caches again
221 * 12. Enter normal cache mode and reenable caching
222 * 13. Set PGE
223 * 14. Wait for buddies to catch up
224 * 15. Enable interrupts.
225 *
226 * What does that mean for us? Well, first we set data.count to the number
227 * of CPUs. As each CPU announces that it started the rendezvous handler by
228 * decrementing the count, We reset data.count and set the data.gate flag
229 * allowing all the cpu's to proceed with the work. As each cpu disables
230 * interrupts, it'll decrement data.count once. We wait until it hits 0 and
231 * proceed. We clear the data.gate flag and reset data.count. Meanwhile, they
232 * are waiting for that flag to be cleared. Once it's cleared, each
233 * CPU goes through the transition of updating MTRRs.
234 * The CPU vendors may each do it differently,
235 * so we call mtrr_if->set() callback and let them take care of it.
236 * When they're done, they again decrement data->count and wait for data.gate
237 * to be set.
238 * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag
239 * Everyone then enables interrupts and we all continue on.
240 *
241 * Note that the mechanism is the same for UP systems, too; all the SMP stuff
242 * becomes nops.
243 */
244 static void
set_mtrr(unsigned int reg,unsigned long base,unsigned long size,mtrr_type type)245 set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
246 {
247 struct set_mtrr_data data;
248 unsigned long flags;
249 int cpu;
250
251 preempt_disable();
252
253 data.smp_reg = reg;
254 data.smp_base = base;
255 data.smp_size = size;
256 data.smp_type = type;
257 atomic_set(&data.count, num_booting_cpus() - 1);
258
259 /* Make sure data.count is visible before unleashing other CPUs */
260 smp_wmb();
261 atomic_set(&data.gate, 0);
262
263 /* Start the ball rolling on other CPUs */
264 for_each_online_cpu(cpu) {
265 struct cpu_stop_work *work = &per_cpu(mtrr_work, cpu);
266
267 if (cpu == smp_processor_id())
268 continue;
269
270 stop_one_cpu_nowait(cpu, mtrr_work_handler, &data, work);
271 }
272
273
274 while (atomic_read(&data.count))
275 cpu_relax();
276
277 /* Ok, reset count and toggle gate */
278 atomic_set(&data.count, num_booting_cpus() - 1);
279 smp_wmb();
280 atomic_set(&data.gate, 1);
281
282 local_irq_save(flags);
283
284 while (atomic_read(&data.count))
285 cpu_relax();
286
287 /* Ok, reset count and toggle gate */
288 atomic_set(&data.count, num_booting_cpus() - 1);
289 smp_wmb();
290 atomic_set(&data.gate, 0);
291
292 /* Do our MTRR business */
293
294 /*
295 * HACK!
296 *
297 * We use this same function to initialize the mtrrs during boot,
298 * resume, runtime cpu online and on an explicit request to set a
299 * specific MTRR.
300 *
301 * During boot or suspend, the state of the boot cpu's mtrrs has been
302 * saved, and we want to replicate that across all the cpus that come
303 * online (either at the end of boot or resume or during a runtime cpu
304 * online). If we're doing that, @reg is set to something special and on
305 * this cpu we still do mtrr_if->set_all(). During boot/resume, this
306 * is unnecessary if at this point we are still on the cpu that started
307 * the boot/resume sequence. But there is no guarantee that we are still
308 * on the same cpu. So we do mtrr_if->set_all() on this cpu aswell to be
309 * sure that we are in sync with everyone else.
310 */
311 if (reg != ~0U)
312 mtrr_if->set(reg, base, size, type);
313 else
314 mtrr_if->set_all();
315
316 /* Wait for the others */
317 while (atomic_read(&data.count))
318 cpu_relax();
319
320 atomic_set(&data.count, num_booting_cpus() - 1);
321 smp_wmb();
322 atomic_set(&data.gate, 1);
323
324 /*
325 * Wait here for everyone to have seen the gate change
326 * So we're the last ones to touch 'data'
327 */
328 while (atomic_read(&data.count))
329 cpu_relax();
330
331 local_irq_restore(flags);
332 preempt_enable();
333 }
334
335 /**
336 * mtrr_add_page - Add a memory type region
337 * @base: Physical base address of region in pages (in units of 4 kB!)
338 * @size: Physical size of region in pages (4 kB)
339 * @type: Type of MTRR desired
340 * @increment: If this is true do usage counting on the region
341 *
342 * Memory type region registers control the caching on newer Intel and
343 * non Intel processors. This function allows drivers to request an
344 * MTRR is added. The details and hardware specifics of each processor's
345 * implementation are hidden from the caller, but nevertheless the
346 * caller should expect to need to provide a power of two size on an
347 * equivalent power of two boundary.
348 *
349 * If the region cannot be added either because all regions are in use
350 * or the CPU cannot support it a negative value is returned. On success
351 * the register number for this entry is returned, but should be treated
352 * as a cookie only.
353 *
354 * On a multiprocessor machine the changes are made to all processors.
355 * This is required on x86 by the Intel processors.
356 *
357 * The available types are
358 *
359 * %MTRR_TYPE_UNCACHABLE - No caching
360 *
361 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
362 *
363 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
364 *
365 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
366 *
367 * BUGS: Needs a quiet flag for the cases where drivers do not mind
368 * failures and do not wish system log messages to be sent.
369 */
mtrr_add_page(unsigned long base,unsigned long size,unsigned int type,bool increment)370 int mtrr_add_page(unsigned long base, unsigned long size,
371 unsigned int type, bool increment)
372 {
373 unsigned long lbase, lsize;
374 int i, replace, error;
375 mtrr_type ltype;
376
377 if (!mtrr_if)
378 return -ENXIO;
379
380 error = mtrr_if->validate_add_page(base, size, type);
381 if (error)
382 return error;
383
384 if (type >= MTRR_NUM_TYPES) {
385 pr_warning("mtrr: type: %u invalid\n", type);
386 return -EINVAL;
387 }
388
389 /* If the type is WC, check that this processor supports it */
390 if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
391 pr_warning("mtrr: your processor doesn't support write-combining\n");
392 return -ENOSYS;
393 }
394
395 if (!size) {
396 pr_warning("mtrr: zero sized request\n");
397 return -EINVAL;
398 }
399
400 if (base & size_or_mask || size & size_or_mask) {
401 pr_warning("mtrr: base or size exceeds the MTRR width\n");
402 return -EINVAL;
403 }
404
405 error = -EINVAL;
406 replace = -1;
407
408 /* No CPU hotplug when we change MTRR entries */
409 get_online_cpus();
410
411 /* Search for existing MTRR */
412 mutex_lock(&mtrr_mutex);
413 for (i = 0; i < num_var_ranges; ++i) {
414 mtrr_if->get(i, &lbase, &lsize, <ype);
415 if (!lsize || base > lbase + lsize - 1 ||
416 base + size - 1 < lbase)
417 continue;
418 /*
419 * At this point we know there is some kind of
420 * overlap/enclosure
421 */
422 if (base < lbase || base + size - 1 > lbase + lsize - 1) {
423 if (base <= lbase &&
424 base + size - 1 >= lbase + lsize - 1) {
425 /* New region encloses an existing region */
426 if (type == ltype) {
427 replace = replace == -1 ? i : -2;
428 continue;
429 } else if (types_compatible(type, ltype))
430 continue;
431 }
432 pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
433 " 0x%lx000,0x%lx000\n", base, size, lbase,
434 lsize);
435 goto out;
436 }
437 /* New region is enclosed by an existing region */
438 if (ltype != type) {
439 if (types_compatible(type, ltype))
440 continue;
441 pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
442 base, size, mtrr_attrib_to_str(ltype),
443 mtrr_attrib_to_str(type));
444 goto out;
445 }
446 if (increment)
447 ++mtrr_usage_table[i];
448 error = i;
449 goto out;
450 }
451 /* Search for an empty MTRR */
452 i = mtrr_if->get_free_region(base, size, replace);
453 if (i >= 0) {
454 set_mtrr(i, base, size, type);
455 if (likely(replace < 0)) {
456 mtrr_usage_table[i] = 1;
457 } else {
458 mtrr_usage_table[i] = mtrr_usage_table[replace];
459 if (increment)
460 mtrr_usage_table[i]++;
461 if (unlikely(replace != i)) {
462 set_mtrr(replace, 0, 0, 0);
463 mtrr_usage_table[replace] = 0;
464 }
465 }
466 } else {
467 pr_info("mtrr: no more MTRRs available\n");
468 }
469 error = i;
470 out:
471 mutex_unlock(&mtrr_mutex);
472 put_online_cpus();
473 return error;
474 }
475
mtrr_check(unsigned long base,unsigned long size)476 static int mtrr_check(unsigned long base, unsigned long size)
477 {
478 if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
479 pr_warning("mtrr: size and base must be multiples of 4 kiB\n");
480 pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base);
481 dump_stack();
482 return -1;
483 }
484 return 0;
485 }
486
487 /**
488 * mtrr_add - Add a memory type region
489 * @base: Physical base address of region
490 * @size: Physical size of region
491 * @type: Type of MTRR desired
492 * @increment: If this is true do usage counting on the region
493 *
494 * Memory type region registers control the caching on newer Intel and
495 * non Intel processors. This function allows drivers to request an
496 * MTRR is added. The details and hardware specifics of each processor's
497 * implementation are hidden from the caller, but nevertheless the
498 * caller should expect to need to provide a power of two size on an
499 * equivalent power of two boundary.
500 *
501 * If the region cannot be added either because all regions are in use
502 * or the CPU cannot support it a negative value is returned. On success
503 * the register number for this entry is returned, but should be treated
504 * as a cookie only.
505 *
506 * On a multiprocessor machine the changes are made to all processors.
507 * This is required on x86 by the Intel processors.
508 *
509 * The available types are
510 *
511 * %MTRR_TYPE_UNCACHABLE - No caching
512 *
513 * %MTRR_TYPE_WRBACK - Write data back in bursts whenever
514 *
515 * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
516 *
517 * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
518 *
519 * BUGS: Needs a quiet flag for the cases where drivers do not mind
520 * failures and do not wish system log messages to be sent.
521 */
mtrr_add(unsigned long base,unsigned long size,unsigned int type,bool increment)522 int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
523 bool increment)
524 {
525 if (mtrr_check(base, size))
526 return -EINVAL;
527 return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
528 increment);
529 }
530 EXPORT_SYMBOL(mtrr_add);
531
532 /**
533 * mtrr_del_page - delete a memory type region
534 * @reg: Register returned by mtrr_add
535 * @base: Physical base address
536 * @size: Size of region
537 *
538 * If register is supplied then base and size are ignored. This is
539 * how drivers should call it.
540 *
541 * Releases an MTRR region. If the usage count drops to zero the
542 * register is freed and the region returns to default state.
543 * On success the register is returned, on failure a negative error
544 * code.
545 */
mtrr_del_page(int reg,unsigned long base,unsigned long size)546 int mtrr_del_page(int reg, unsigned long base, unsigned long size)
547 {
548 int i, max;
549 mtrr_type ltype;
550 unsigned long lbase, lsize;
551 int error = -EINVAL;
552
553 if (!mtrr_if)
554 return -ENXIO;
555
556 max = num_var_ranges;
557 /* No CPU hotplug when we change MTRR entries */
558 get_online_cpus();
559 mutex_lock(&mtrr_mutex);
560 if (reg < 0) {
561 /* Search for existing MTRR */
562 for (i = 0; i < max; ++i) {
563 mtrr_if->get(i, &lbase, &lsize, <ype);
564 if (lbase == base && lsize == size) {
565 reg = i;
566 break;
567 }
568 }
569 if (reg < 0) {
570 pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
571 base, size);
572 goto out;
573 }
574 }
575 if (reg >= max) {
576 pr_warning("mtrr: register: %d too big\n", reg);
577 goto out;
578 }
579 mtrr_if->get(reg, &lbase, &lsize, <ype);
580 if (lsize < 1) {
581 pr_warning("mtrr: MTRR %d not used\n", reg);
582 goto out;
583 }
584 if (mtrr_usage_table[reg] < 1) {
585 pr_warning("mtrr: reg: %d has count=0\n", reg);
586 goto out;
587 }
588 if (--mtrr_usage_table[reg] < 1)
589 set_mtrr(reg, 0, 0, 0);
590 error = reg;
591 out:
592 mutex_unlock(&mtrr_mutex);
593 put_online_cpus();
594 return error;
595 }
596
597 /**
598 * mtrr_del - delete a memory type region
599 * @reg: Register returned by mtrr_add
600 * @base: Physical base address
601 * @size: Size of region
602 *
603 * If register is supplied then base and size are ignored. This is
604 * how drivers should call it.
605 *
606 * Releases an MTRR region. If the usage count drops to zero the
607 * register is freed and the region returns to default state.
608 * On success the register is returned, on failure a negative error
609 * code.
610 */
mtrr_del(int reg,unsigned long base,unsigned long size)611 int mtrr_del(int reg, unsigned long base, unsigned long size)
612 {
613 if (mtrr_check(base, size))
614 return -EINVAL;
615 return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
616 }
617 EXPORT_SYMBOL(mtrr_del);
618
619 /*
620 * HACK ALERT!
621 * These should be called implicitly, but we can't yet until all the initcall
622 * stuff is done...
623 */
init_ifs(void)624 static void __init init_ifs(void)
625 {
626 #ifndef CONFIG_X86_64
627 amd_init_mtrr();
628 cyrix_init_mtrr();
629 centaur_init_mtrr();
630 #endif
631 }
632
633 /* The suspend/resume methods are only for CPU without MTRR. CPU using generic
634 * MTRR driver doesn't require this
635 */
636 struct mtrr_value {
637 mtrr_type ltype;
638 unsigned long lbase;
639 unsigned long lsize;
640 };
641
642 static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
643
mtrr_save(void)644 static int mtrr_save(void)
645 {
646 int i;
647
648 for (i = 0; i < num_var_ranges; i++) {
649 mtrr_if->get(i, &mtrr_value[i].lbase,
650 &mtrr_value[i].lsize,
651 &mtrr_value[i].ltype);
652 }
653 return 0;
654 }
655
mtrr_restore(void)656 static void mtrr_restore(void)
657 {
658 int i;
659
660 for (i = 0; i < num_var_ranges; i++) {
661 if (mtrr_value[i].lsize) {
662 set_mtrr(i, mtrr_value[i].lbase,
663 mtrr_value[i].lsize,
664 mtrr_value[i].ltype);
665 }
666 }
667 }
668
669
670
671 static struct syscore_ops mtrr_syscore_ops = {
672 .suspend = mtrr_save,
673 .resume = mtrr_restore,
674 };
675
676 int __initdata changed_by_mtrr_cleanup;
677
678 /**
679 * mtrr_bp_init - initialize mtrrs on the boot CPU
680 *
681 * This needs to be called early; before any of the other CPUs are
682 * initialized (i.e. before smp_init()).
683 *
684 */
mtrr_bp_init(void)685 void __init mtrr_bp_init(void)
686 {
687 u32 phys_addr;
688
689 init_ifs();
690
691 phys_addr = 32;
692
693 if (cpu_has_mtrr) {
694 mtrr_if = &generic_mtrr_ops;
695 size_or_mask = 0xff000000; /* 36 bits */
696 size_and_mask = 0x00f00000;
697 phys_addr = 36;
698
699 /*
700 * This is an AMD specific MSR, but we assume(hope?) that
701 * Intel will implement it to when they extend the address
702 * bus of the Xeon.
703 */
704 if (cpuid_eax(0x80000000) >= 0x80000008) {
705 phys_addr = cpuid_eax(0x80000008) & 0xff;
706 /* CPUID workaround for Intel 0F33/0F34 CPU */
707 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
708 boot_cpu_data.x86 == 0xF &&
709 boot_cpu_data.x86_model == 0x3 &&
710 (boot_cpu_data.x86_mask == 0x3 ||
711 boot_cpu_data.x86_mask == 0x4))
712 phys_addr = 36;
713
714 size_or_mask = ~((1ULL << (phys_addr - PAGE_SHIFT)) - 1);
715 size_and_mask = ~size_or_mask & 0xfffff00000ULL;
716 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
717 boot_cpu_data.x86 == 6) {
718 /*
719 * VIA C* family have Intel style MTRRs,
720 * but don't support PAE
721 */
722 size_or_mask = 0xfff00000; /* 32 bits */
723 size_and_mask = 0;
724 phys_addr = 32;
725 }
726 } else {
727 switch (boot_cpu_data.x86_vendor) {
728 case X86_VENDOR_AMD:
729 if (cpu_has_k6_mtrr) {
730 /* Pre-Athlon (K6) AMD CPU MTRRs */
731 mtrr_if = mtrr_ops[X86_VENDOR_AMD];
732 size_or_mask = 0xfff00000; /* 32 bits */
733 size_and_mask = 0;
734 }
735 break;
736 case X86_VENDOR_CENTAUR:
737 if (cpu_has_centaur_mcr) {
738 mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
739 size_or_mask = 0xfff00000; /* 32 bits */
740 size_and_mask = 0;
741 }
742 break;
743 case X86_VENDOR_CYRIX:
744 if (cpu_has_cyrix_arr) {
745 mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
746 size_or_mask = 0xfff00000; /* 32 bits */
747 size_and_mask = 0;
748 }
749 break;
750 default:
751 break;
752 }
753 }
754
755 if (mtrr_if) {
756 set_num_var_ranges();
757 init_table();
758 if (use_intel()) {
759 get_mtrr_state();
760
761 if (mtrr_cleanup(phys_addr)) {
762 changed_by_mtrr_cleanup = 1;
763 mtrr_if->set_all();
764 }
765 }
766 }
767 }
768
mtrr_ap_init(void)769 void mtrr_ap_init(void)
770 {
771 if (!use_intel() || mtrr_aps_delayed_init)
772 return;
773 /*
774 * Ideally we should hold mtrr_mutex here to avoid mtrr entries
775 * changed, but this routine will be called in cpu boot time,
776 * holding the lock breaks it.
777 *
778 * This routine is called in two cases:
779 *
780 * 1. very earily time of software resume, when there absolutely
781 * isn't mtrr entry changes;
782 *
783 * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug
784 * lock to prevent mtrr entry changes
785 */
786 set_mtrr(~0U, 0, 0, 0);
787 }
788
789 /**
790 * Save current fixed-range MTRR state of the BSP
791 */
mtrr_save_state(void)792 void mtrr_save_state(void)
793 {
794 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
795 }
796
set_mtrr_aps_delayed_init(void)797 void set_mtrr_aps_delayed_init(void)
798 {
799 if (!use_intel())
800 return;
801
802 mtrr_aps_delayed_init = true;
803 }
804
805 /*
806 * Delayed MTRR initialization for all AP's
807 */
mtrr_aps_init(void)808 void mtrr_aps_init(void)
809 {
810 if (!use_intel())
811 return;
812
813 /*
814 * Check if someone has requested the delay of AP MTRR initialization,
815 * by doing set_mtrr_aps_delayed_init(), prior to this point. If not,
816 * then we are done.
817 */
818 if (!mtrr_aps_delayed_init)
819 return;
820
821 set_mtrr(~0U, 0, 0, 0);
822 mtrr_aps_delayed_init = false;
823 }
824
mtrr_bp_restore(void)825 void mtrr_bp_restore(void)
826 {
827 if (!use_intel())
828 return;
829
830 mtrr_if->set_all();
831 }
832
mtrr_init_finialize(void)833 static int __init mtrr_init_finialize(void)
834 {
835 if (!mtrr_if)
836 return 0;
837
838 if (use_intel()) {
839 if (!changed_by_mtrr_cleanup)
840 mtrr_state_warn();
841 return 0;
842 }
843
844 /*
845 * The CPU has no MTRR and seems to not support SMP. They have
846 * specific drivers, we use a tricky method to support
847 * suspend/resume for them.
848 *
849 * TBD: is there any system with such CPU which supports
850 * suspend/resume? If no, we should remove the code.
851 */
852 register_syscore_ops(&mtrr_syscore_ops);
853
854 return 0;
855 }
856 subsys_initcall(mtrr_init_finialize);
857