1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6 * and Cort Dougan (PReP) (cort@cs.nmt.edu)
7 * Copyright (C) 1996 Paul Mackerras
8 * Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk).
9 *
10 * Derived from "arch/i386/mm/init.c"
11 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
12 *
13 * This program is free software; you can redistribute it and/or
14 * modify it under the terms of the GNU General Public License
15 * as published by the Free Software Foundation; either version
16 * 2 of the License, or (at your option) any later version.
17 *
18 */
19
20 #include <linux/config.h>
21 #include <linux/sched.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/string.h>
25 #include <linux/types.h>
26 #include <linux/mm.h>
27 #include <linux/stddef.h>
28 #include <linux/init.h>
29 #include <linux/bootmem.h>
30 #include <linux/highmem.h>
31 #ifdef CONFIG_BLK_DEV_INITRD
32 #include <linux/blk.h> /* for initrd_* */
33 #endif
34
35 #include <asm/pgalloc.h>
36 #include <asm/prom.h>
37 #include <asm/io.h>
38 #include <asm/mmu_context.h>
39 #include <asm/pgtable.h>
40 #include <asm/mmu.h>
41 #include <asm/smp.h>
42 #include <asm/machdep.h>
43 #include <asm/btext.h>
44 #include <asm/tlb.h>
45
46 #include "mem_pieces.h"
47 #include "mmu_decl.h"
48
49 /*
50 * Just any arbitrary offset to the start of the vmalloc VM area: the
51 * current 64MB value just means that there will be a 64MB "hole" after the
52 * physical memory until the kernel virtual memory starts. That means that
53 * any out-of-bounds memory accesses will hopefully be caught.
54 * The vmalloc() routines leaves a hole of 4kB between each vmalloced
55 * area for the same reason. ;)
56 *
57 * We no longer map larger than phys RAM with the BATs so we don't have
58 * to worry about the VMALLOC_OFFSET causing problems. We do have to worry
59 * about clashes between our early calls to ioremap() that start growing down
60 * from ioremap_base being run into the VM area allocations (growing upwards
61 * from VMALLOC_START). For this reason we have ioremap_bot to check when
62 * we actually run into our mappings setup in the early boot with the VM
63 * system. This really does become a problem for machines with good amounts
64 * of RAM. -- Cort
65 */
66 #ifdef CONFIG_PIN_TLB
67 #define VMALLOC_OFFSET (0x2000000) /* 32M */
68 #else
69 #define VMALLOC_OFFSET (0x1000000) /* 16M */
70 #endif
71
72 unsigned long vmalloc_start;
73
74 mmu_gather_t mmu_gathers[NR_CPUS];
75
76 unsigned long total_memory;
77 unsigned long total_lowmem;
78
79 unsigned long ppc_memstart;
80 unsigned long ppc_memoffset = PAGE_OFFSET;
81
82 int mem_init_done;
83 int init_bootmem_done;
84 int boot_mapsize;
85 unsigned long totalram_pages;
86 unsigned long totalhigh_pages;
87 #ifdef CONFIG_ALL_PPC
88 unsigned long agp_special_page;
89 #endif
90
91 extern char _end[];
92 extern char etext[], _stext[];
93 extern char __init_begin, __init_end;
94 extern char __prep_begin, __prep_end;
95 extern char __chrp_begin, __chrp_end;
96 extern char __pmac_begin, __pmac_end;
97 extern char __openfirmware_begin, __openfirmware_end;
98
99 #ifdef CONFIG_HIGHMEM
100 pte_t *kmap_pte;
101 pgprot_t kmap_prot;
102 #endif
103
104 void MMU_init(void);
105 void set_phys_avail(unsigned long total_ram);
106
107 /* XXX should be in current.h -- paulus */
108 extern struct task_struct *current_set[NR_CPUS];
109
110 char *klimit = _end;
111 struct mem_pieces phys_avail;
112
113 extern char *sysmap;
114 extern unsigned long sysmap_size;
115
116 /*
117 * this tells the system to map all of ram with the segregs
118 * (i.e. page tables) instead of the bats.
119 * -- Cort
120 */
121 int __map_without_bats;
122
123 /* max amount of RAM to use */
124 unsigned long __max_memory;
125
do_check_pgt_cache(int low,int high)126 int do_check_pgt_cache(int low, int high)
127 {
128 int freed = 0;
129 if (pgtable_cache_size > high) {
130 do {
131 if (pgd_quicklist) {
132 free_pgd_slow(get_pgd_fast());
133 freed++;
134 }
135 if (pte_quicklist) {
136 pte_free_slow(pte_alloc_one_fast(NULL, 0));
137 freed++;
138 }
139 } while (pgtable_cache_size > low);
140 }
141 return freed;
142 }
143
show_mem(void)144 void show_mem(void)
145 {
146 int i,free = 0,total = 0,reserved = 0;
147 int shared = 0, cached = 0;
148 struct task_struct *p;
149 int highmem = 0;
150
151 printk("Mem-info:\n");
152 show_free_areas();
153 printk("Free swap: %6dkB\n",nr_swap_pages<<(PAGE_SHIFT-10));
154 i = max_mapnr;
155 while (i-- > 0) {
156 total++;
157 if (PageHighMem(mem_map+i))
158 highmem++;
159 if (PageReserved(mem_map+i))
160 reserved++;
161 else if (PageSwapCache(mem_map+i))
162 cached++;
163 else if (!page_count(mem_map+i))
164 free++;
165 else
166 shared += atomic_read(&mem_map[i].count) - 1;
167 }
168 printk("%d pages of RAM\n",total);
169 printk("%d pages of HIGHMEM\n", highmem);
170 printk("%d free pages\n",free);
171 printk("%d reserved pages\n",reserved);
172 printk("%d pages shared\n",shared);
173 printk("%d pages swap cached\n",cached);
174 printk("%d pages in page table cache\n",(int)pgtable_cache_size);
175 show_buffers();
176 printk("%-8s %3s %8s %8s %8s %9s %8s", "Process", "Pid",
177 "Ctx", "Ctx<<4", "Last Sys", "pc", "task");
178 #ifdef CONFIG_SMP
179 printk(" %3s", "CPU");
180 #endif /* CONFIG_SMP */
181 printk("\n");
182 for_each_task(p)
183 {
184 printk("%-8.8s %3d %8ld %8ld %8ld %c%08lx %08lx ",
185 p->comm,p->pid,
186 (p->mm)?p->mm->context:0,
187 (p->mm)?(p->mm->context<<4):0,
188 p->thread.last_syscall,
189 (p->thread.regs)?user_mode(p->thread.regs) ? 'u' : 'k' : '?',
190 (p->thread.regs)?p->thread.regs->nip:0,
191 (ulong)p);
192 {
193 int iscur = 0;
194 #ifdef CONFIG_SMP
195 printk("%3d ", p->processor);
196 if ( (p->processor != NO_PROC_ID) &&
197 (p == current_set[p->processor]) )
198 {
199 iscur = 1;
200 printk("current");
201 }
202 #else
203 if ( p == current )
204 {
205 iscur = 1;
206 printk("current");
207 }
208
209 if ( p == last_task_used_math )
210 {
211 if ( iscur )
212 printk(",");
213 printk("last math");
214 }
215 #endif /* CONFIG_SMP */
216 printk("\n");
217 }
218 }
219 }
220
si_meminfo(struct sysinfo * val)221 void si_meminfo(struct sysinfo *val)
222 {
223 val->totalram = totalram_pages;
224 val->sharedram = 0;
225 val->freeram = nr_free_pages();
226 val->bufferram = atomic_read(&buffermem_pages);
227 val->totalhigh = totalhigh_pages;
228 val->freehigh = nr_free_highpages();
229 val->mem_unit = PAGE_SIZE;
230 }
231
232 /* Free up now-unused memory */
free_sec(unsigned long start,unsigned long end,const char * name)233 static void free_sec(unsigned long start, unsigned long end, const char *name)
234 {
235 unsigned long cnt = 0;
236
237 while (start < end) {
238 ClearPageReserved(virt_to_page(start));
239 set_page_count(virt_to_page(start), 1);
240 free_page(start);
241 cnt++;
242 start += PAGE_SIZE;
243 }
244 if (cnt) {
245 printk(" %ldk %s", cnt << (PAGE_SHIFT - 10), name);
246 totalram_pages += cnt;
247 }
248 }
249
free_initmem(void)250 void free_initmem(void)
251 {
252 #define FREESEC(TYPE) \
253 free_sec((unsigned long)(&__ ## TYPE ## _begin), \
254 (unsigned long)(&__ ## TYPE ## _end), \
255 #TYPE);
256
257 printk (KERN_INFO "Freeing unused kernel memory:");
258 FREESEC(init);
259 if (_machine != _MACH_Pmac)
260 FREESEC(pmac);
261 if (_machine != _MACH_chrp)
262 FREESEC(chrp);
263 if (_machine != _MACH_prep)
264 FREESEC(prep);
265 if (!have_of)
266 FREESEC(openfirmware);
267 printk("\n");
268 #undef FREESEC
269 }
270
271 #ifdef CONFIG_BLK_DEV_INITRD
free_initrd_mem(unsigned long start,unsigned long end)272 void free_initrd_mem(unsigned long start, unsigned long end)
273 {
274 printk (KERN_INFO "Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
275
276 for (; start < end; start += PAGE_SIZE) {
277 ClearPageReserved(virt_to_page(start));
278 set_page_count(virt_to_page(start), 1);
279 free_page(start);
280 totalram_pages++;
281 }
282 }
283 #endif
284
285 /*
286 * Check for command-line options that affect what MMU_init will do.
287 */
MMU_setup(void)288 void MMU_setup(void)
289 {
290 /* Check for nobats option (used in mapin_ram). */
291 if (strstr(cmd_line, "nobats")) {
292 __map_without_bats = 1;
293 }
294
295 /* Look for mem= option on command line */
296 if (strstr(cmd_line, "mem=")) {
297 char *p, *q;
298 unsigned long maxmem = 0;
299
300 for (q = cmd_line; (p = strstr(q, "mem=")) != 0; ) {
301 q = p + 4;
302 if (p > cmd_line && p[-1] != ' ')
303 continue;
304 maxmem = simple_strtoul(q, &q, 0);
305 if (*q == 'k' || *q == 'K') {
306 maxmem <<= 10;
307 ++q;
308 } else if (*q == 'm' || *q == 'M') {
309 maxmem <<= 20;
310 ++q;
311 }
312 }
313 __max_memory = maxmem;
314 }
315 }
316
317 /*
318 * MMU_init sets up the basic memory mappings for the kernel,
319 * including both RAM and possibly some I/O regions,
320 * and sets up the page tables and the MMU hardware ready to go.
321 */
MMU_init(void)322 void __init MMU_init(void)
323 {
324 if (ppc_md.progress)
325 ppc_md.progress("MMU:enter", 0x111);
326
327 /* parse args from command line */
328 MMU_setup();
329
330 /*
331 * Figure out how much memory we have, how much
332 * is lowmem, and how much is highmem.
333 */
334 total_memory = ppc_md.find_end_of_memory();
335
336 if (__max_memory && total_memory > __max_memory)
337 total_memory = __max_memory;
338 total_lowmem = total_memory;
339 adjust_total_lowmem();
340 set_phys_avail(total_lowmem);
341 vmalloc_start = KERNELBASE + total_lowmem;
342
343 /* Initialize the MMU hardware */
344 if (ppc_md.progress)
345 ppc_md.progress("MMU:hw init", 0x300);
346 MMU_init_hw();
347
348 /* Map in all of RAM starting at KERNELBASE */
349 if (ppc_md.progress)
350 ppc_md.progress("MMU:mapin", 0x301);
351 mapin_ram();
352
353 #ifdef CONFIG_HIGHMEM
354 ioremap_base = PKMAP_BASE;
355 #else
356 ioremap_base = 0xfe000000UL; /* for now, could be 0xfffff000 */
357 #endif /* CONFIG_HIGHMEM */
358 ioremap_bot = ioremap_base;
359
360 /* Map in I/O resources */
361 if (ppc_md.progress)
362 ppc_md.progress("MMU:setio", 0x302);
363 if (ppc_md.setup_io_mappings)
364 ppc_md.setup_io_mappings();
365
366 /* Initialize the context management stuff */
367 mmu_context_init();
368
369 if (ppc_md.progress)
370 ppc_md.progress("MMU:exit", 0x211);
371
372 #ifdef CONFIG_BOOTX_TEXT
373 /* By default, we are no longer mapped */
374 boot_text_mapped = 0;
375 /* Must be done last, or ppc_md.progress will die. */
376 map_boot_text();
377 #endif
378 }
379
380 /* This is only called until mem_init is done. */
early_get_page(void)381 void __init *early_get_page(void)
382 {
383 void *p;
384
385 if (init_bootmem_done) {
386 p = alloc_bootmem_pages(PAGE_SIZE);
387 } else {
388 p = mem_pieces_find(PAGE_SIZE, PAGE_SIZE);
389 }
390 return p;
391 }
392
393 /*
394 * Initialize the bootmem system and give it all the memory we
395 * have available.
396 */
do_init_bootmem(void)397 void __init do_init_bootmem(void)
398 {
399 unsigned long start, size;
400 int i;
401
402 /*
403 * Find an area to use for the bootmem bitmap.
404 * We look for the first area which is at least
405 * 128kB in length (128kB is enough for a bitmap
406 * for 4GB of memory, using 4kB pages), plus 1 page
407 * (in case the address isn't page-aligned).
408 */
409 start = 0;
410 size = 0;
411 for (i = 0; i < phys_avail.n_regions; ++i) {
412 unsigned long a = phys_avail.regions[i].address;
413 unsigned long s = phys_avail.regions[i].size;
414 if (s <= size)
415 continue;
416 start = a;
417 size = s;
418 if (s >= 33 * PAGE_SIZE)
419 break;
420 }
421 start = PAGE_ALIGN(start);
422
423 min_low_pfn = start >> PAGE_SHIFT;
424 max_low_pfn = (PPC_MEMSTART + total_lowmem) >> PAGE_SHIFT;
425 boot_mapsize = init_bootmem_node(&contig_page_data, min_low_pfn,
426 PPC_MEMSTART >> PAGE_SHIFT,
427 max_low_pfn);
428
429 /* remove the bootmem bitmap from the available memory */
430 mem_pieces_remove(&phys_avail, start, boot_mapsize, 1);
431
432 /* add everything in phys_avail into the bootmem map */
433 for (i = 0; i < phys_avail.n_regions; ++i)
434 free_bootmem(phys_avail.regions[i].address,
435 phys_avail.regions[i].size);
436
437 init_bootmem_done = 1;
438 }
439
440 /*
441 * paging_init() sets up the page tables - in fact we've already done this.
442 */
paging_init(void)443 void __init paging_init(void)
444 {
445 unsigned long zones_size[MAX_NR_ZONES], i;
446
447 #ifdef CONFIG_HIGHMEM
448 map_page(PKMAP_BASE, 0, 0); /* XXX gross */
449 pkmap_page_table = pte_offset(pmd_offset(pgd_offset_k(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
450 map_page(KMAP_FIX_BEGIN, 0, 0); /* XXX gross */
451 kmap_pte = pte_offset(pmd_offset(pgd_offset_k(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
452 kmap_prot = PAGE_KERNEL;
453 #endif /* CONFIG_HIGHMEM */
454
455 /*
456 * All pages are DMA-able so we put them all in the DMA zone.
457 */
458 zones_size[ZONE_DMA] = total_lowmem >> PAGE_SHIFT;
459 for (i = 1; i < MAX_NR_ZONES; i++)
460 zones_size[i] = 0;
461
462 #ifdef CONFIG_HIGHMEM
463 zones_size[ZONE_HIGHMEM] = (total_memory - total_lowmem) >> PAGE_SHIFT;
464 #endif /* CONFIG_HIGHMEM */
465
466 free_area_init(zones_size);
467 }
468
mem_init(void)469 void __init mem_init(void)
470 {
471 unsigned long addr;
472 int codepages = 0;
473 int datapages = 0;
474 int initpages = 0;
475 #ifdef CONFIG_HIGHMEM
476 unsigned long highmem_mapnr;
477
478 highmem_mapnr = total_lowmem >> PAGE_SHIFT;
479 highmem_start_page = mem_map + highmem_mapnr;
480 #endif /* CONFIG_HIGHMEM */
481 max_mapnr = total_memory >> PAGE_SHIFT;
482
483 high_memory = (void *) __va(PPC_MEMSTART + total_lowmem);
484 num_physpages = max_mapnr; /* RAM is assumed contiguous */
485
486 totalram_pages += free_all_bootmem();
487
488 /* adjust vmalloc_start */
489 vmalloc_start = (vmalloc_start + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1);
490
491 #ifdef CONFIG_BLK_DEV_INITRD
492 /* if we are booted from BootX with an initial ramdisk,
493 make sure the ramdisk pages aren't reserved. */
494 if (initrd_start) {
495 for (addr = initrd_start; addr < initrd_end; addr += PAGE_SIZE)
496 ClearPageReserved(virt_to_page(addr));
497 }
498 #endif /* CONFIG_BLK_DEV_INITRD */
499
500 #if defined(CONFIG_ALL_PPC)
501 /* mark the RTAS pages as reserved */
502 if ( rtas_data )
503 for (addr = (ulong)__va(rtas_data);
504 addr < PAGE_ALIGN((ulong)__va(rtas_data)+rtas_size) ;
505 addr += PAGE_SIZE)
506 SetPageReserved(virt_to_page(addr));
507 if (agp_special_page)
508 SetPageReserved(virt_to_page(agp_special_page));
509 #endif /* defined(CONFIG_ALL_PPC) */
510 if ( sysmap )
511 for (addr = (unsigned long)sysmap;
512 addr < PAGE_ALIGN((unsigned long)sysmap+sysmap_size) ;
513 addr += PAGE_SIZE)
514 SetPageReserved(virt_to_page(addr));
515
516 for (addr = PAGE_OFFSET; addr < (unsigned long)high_memory;
517 addr += PAGE_SIZE) {
518 if (!PageReserved(virt_to_page(addr)))
519 continue;
520 if (addr < (ulong) etext)
521 codepages++;
522 else if (addr >= (unsigned long)&__init_begin
523 && addr < (unsigned long)&__init_end)
524 initpages++;
525 else if (addr < (ulong) klimit)
526 datapages++;
527 }
528
529 #ifdef CONFIG_HIGHMEM
530 {
531 unsigned long pfn;
532
533 for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
534 struct page *page = mem_map + pfn;
535
536 ClearPageReserved(page);
537 set_bit(PG_highmem, &page->flags);
538 atomic_set(&page->count, 1);
539 __free_page(page);
540 totalhigh_pages++;
541 }
542 totalram_pages += totalhigh_pages;
543 }
544 #endif /* CONFIG_HIGHMEM */
545
546 printk(KERN_INFO "Memory: %luk available (%dk kernel code, %dk data, %dk init, %ldk highmem)\n",
547 (unsigned long)nr_free_pages()<< (PAGE_SHIFT-10),
548 codepages<< (PAGE_SHIFT-10), datapages<< (PAGE_SHIFT-10),
549 initpages<< (PAGE_SHIFT-10),
550 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
551 if (sysmap)
552 printk("System.map loaded at 0x%08x for debugger, size: %ld bytes\n",
553 (unsigned int)sysmap, sysmap_size);
554 #if defined(CONFIG_ALL_PPC)
555 if (agp_special_page)
556 printk(KERN_INFO "AGP special page: 0x%08lx\n", agp_special_page);
557 #endif /* defined(CONFIG_ALL_PPC) */
558 mem_init_done = 1;
559 }
560
561 /*
562 * Set phys_avail to the amount of physical memory,
563 * less the kernel text/data/bss.
564 */
565 void __init
set_phys_avail(unsigned long total_memory)566 set_phys_avail(unsigned long total_memory)
567 {
568 unsigned long kstart, ksize;
569
570 /*
571 * Initially, available physical memory is equivalent to all
572 * physical memory.
573 */
574
575 phys_avail.regions[0].address = PPC_MEMSTART;
576 phys_avail.regions[0].size = total_memory;
577 phys_avail.n_regions = 1;
578
579 /*
580 * Map out the kernel text/data/bss from the available physical
581 * memory.
582 */
583
584 kstart = __pa(_stext); /* should be 0 */
585 ksize = PAGE_ALIGN(klimit - _stext);
586
587 mem_pieces_remove(&phys_avail, kstart, ksize, 0);
588 mem_pieces_remove(&phys_avail, 0, 0x4000, 0);
589
590 #if defined(CONFIG_BLK_DEV_INITRD)
591 /* Remove the init RAM disk from the available memory. */
592 if (initrd_start) {
593 mem_pieces_remove(&phys_avail, __pa(initrd_start),
594 initrd_end - initrd_start, 1);
595 }
596 #endif /* CONFIG_BLK_DEV_INITRD */
597 #ifdef CONFIG_ALL_PPC
598 /* remove the RTAS pages from the available memory */
599 if (rtas_data)
600 mem_pieces_remove(&phys_avail, rtas_data, rtas_size, 1);
601 /* Because of some uninorth weirdness, we need a page of
602 * memory as high as possible (it must be outside of the
603 * bus address seen as the AGP aperture). It will be used
604 * by the r128 DRM driver
605 *
606 * FIXME: We need to make sure that page doesn't overlap any of the\
607 * above. This could be done by improving mem_pieces_find to be able
608 * to do a backward search from the end of the list.
609 */
610 if (_machine == _MACH_Pmac && find_devices("uni-north-agp")) {
611 agp_special_page = (total_memory - PAGE_SIZE);
612 mem_pieces_remove(&phys_avail, agp_special_page, PAGE_SIZE, 0);
613 agp_special_page = (unsigned long)__va(agp_special_page);
614 }
615 #endif /* CONFIG_ALL_PPC */
616 /* remove the sysmap pages from the available memory */
617 if (sysmap)
618 mem_pieces_remove(&phys_avail, __pa(sysmap), sysmap_size, 1);
619 }
620
621 /* Mark some memory as reserved by removing it from phys_avail. */
reserve_phys_mem(unsigned long start,unsigned long size)622 void __init reserve_phys_mem(unsigned long start, unsigned long size)
623 {
624 mem_pieces_remove(&phys_avail, start, size, 1);
625 }
626
627 /*
628 * This is called when a page has been modified by the kernel.
629 * It just marks the page as not i-cache clean. We do the i-cache
630 * flush later when the page is given to a user process, if necessary.
631 */
flush_dcache_page(struct page * page)632 void flush_dcache_page(struct page *page)
633 {
634 clear_bit(PG_arch_1, &page->flags);
635 }
636
flush_icache_page(struct vm_area_struct * vma,struct page * page)637 void flush_icache_page(struct vm_area_struct *vma, struct page *page)
638 {
639 if (page->mapping && !PageReserved(page)
640 && !test_bit(PG_arch_1, &page->flags)) {
641 __flush_dcache_icache(kmap(page));
642 kunmap(page);
643 set_bit(PG_arch_1, &page->flags);
644 }
645 }
646
clear_user_page(void * page,unsigned long vaddr)647 void clear_user_page(void *page, unsigned long vaddr)
648 {
649 clear_page(page);
650 __flush_dcache_icache(page);
651 }
652
copy_user_page(void * vto,void * vfrom,unsigned long vaddr)653 void copy_user_page(void *vto, void *vfrom, unsigned long vaddr)
654 {
655 copy_page(vto, vfrom);
656 __flush_dcache_icache(vto);
657 }
658
flush_icache_user_range(struct vm_area_struct * vma,struct page * page,unsigned long addr,int len)659 void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
660 unsigned long addr, int len)
661 {
662 unsigned long maddr;
663
664 maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
665 flush_icache_range(maddr, maddr + len);
666 kunmap(page);
667 }
668