/* * SN Platform FetchOp Support * * This driver exports the SN fetchop facility to user processes. * Fetchops are atomic memory operations that are implemented in the * memory controller on SGI SN hardware. */ /* * Copyright (C) 1999,2001-2003 Silicon Graphics, Inc. All rights * reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of version 2 of the GNU General Public License * as published by the Free Software Foundation. * * This program is distributed in the hope that it would be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * * Further, this software is distributed without any warranty that it is * free of the rightful claim of any third person regarding infringement * or the like. Any license provided herein, whether implied or * otherwise, applies only to this software file. Patent licenses, if * any, provided herein do not apply to combinations of this program with * other software, or any other product whatsoever. * * You should have received a copy of the GNU General Public License * along with this program; if not, write the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston MA * 02111-1307, USA. * * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy, * Mountain View, CA 94043, or: * * http://www.sgi.com * * For further information regarding this notice, see: * * http://oss.sgi.com/projects/GenInfo/NoticeExplan */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define DRIVER_ID_STR "SGI Fetchop Device Driver" #define REVISION "1.03" #define MSPEC_TO_NID(maddr) nasid_to_cnodeid(NASID_GET(maddr)) static int fetchop_mmap(struct file *file, struct vm_area_struct *vma); static void fetchop_open(struct vm_area_struct *vma); static void fetchop_close(struct vm_area_struct *vma); static struct file_operations fetchop_fops = { owner: THIS_MODULE, mmap: fetchop_mmap, }; static struct miscdevice fetchop_miscdev = { MISC_DYNAMIC_MINOR, "fetchop", &fetchop_fops }; static struct vm_operations_struct fetchop_vm_ops = { open: fetchop_open, close: fetchop_close, }; /* * There is one of these structs per node. It is used to manage the fetchop * space that is available on the node. Current assumption is that there is * only 1 fetchop block of memory per node. */ struct node_fetchops { long maddr; /* MSPEC address of start of fetchops. */ int count; /* Total number of fetchop pages. */ atomic_t free; /* Number of pages currently free. */ unsigned long bits[1]; /* Bitmap for managing pages. */ }; /* * One of these structures is allocated when a fetchop region is mmaped. The * structure is pointed to by the vma->vm_private_data field in the vma struct. * This structure is used to record the addresses of the fetchop pages. */ struct vma_data { int count; /* Number of pages allocated. */ atomic_t refcnt; /* Number of vmas sharing the data. */ unsigned long maddr[1]; /* Array of MSPEC addresses. */ }; /* * Fetchop statistics. */ struct fetchop_stats { unsigned long map_count; /* Number of active mmap's */ unsigned long pages_in_use; /* Number of fetchop pages in use */ unsigned long pages_total; /* Total number of fetchop pages */ }; static struct fetchop_stats fetchop_stats; static struct node_fetchops *node_fetchops[MAX_COMPACT_NODES]; static spinlock_t fetchop_lock = SPIN_LOCK_UNLOCKED; /* * NOTE: This is included here simply because the kernel doesn't have * a generally acceptable UC memory allocator. See PV: 896479 for * more details. --cw * * efi_memmap_walk_uc * * This function walks the EFI memory map and calls 'callback' once * for each EFI memory descriptor that has memory that marked as only * EFI_MEMORY_UC. */ static void efi_memmap_walk_uc (efi_freemem_callback_t callback, void *arg) { void *efi_map_start, *efi_map_end, *p; efi_memory_desc_t *md; u64 efi_desc_size, start, end; efi_map_start = __va(ia64_boot_param->efi_memmap); efi_map_end = efi_map_start + ia64_boot_param->efi_memmap_size; efi_desc_size = ia64_boot_param->efi_memdesc_size; for (p = efi_map_start; p < efi_map_end; p += efi_desc_size) { md = p; if (md->attribute == EFI_MEMORY_UC) { start = PAGE_ALIGN(md->phys_addr); end = PAGE_ALIGN((md->phys_addr+(md->num_pages << EFI_PAGE_SHIFT)) & PAGE_MASK); if ((*callback)(start, end, arg) < 0) return; } } } /* * fetchop_initialize_page * * Initial a page that is about to be used for fetchops. * All fetchop variables in the page are set to 0. * */ static void fetchop_initialize_page(unsigned long maddr) { unsigned long p, pe; for (p=FETCHOP_KADDR_TO_MSPEC_ADDR(maddr), pe=p+PAGE_SIZE; p= numnodes) nid = numa_node_id(); for (i=0; ibits, fops->count)) < fops->count) { if (test_and_set_bit(bit, fops->bits) == 0) { atomic_dec(&node_fetchops[nid]->free); maddr = fops->maddr + (bit<maddr) >> PAGE_SHIFT; clear_bit(bit, node_fetchops[nid]->bits); atomic_inc(&node_fetchops[nid]->free); } static void fetchop_free_pages(struct vma_data *vdata) { int i; for (i=0; icount; i++) fetchop_free_page(vdata->maddr[i]); } /* * fetchop_update_stats * * Update statistics of the number of fetchop mappings & pages. * If creating a new mapping, ensure that we don't exceed the maximum allowed * number of fetchop pages. */ static int fetchop_update_stats(int mmap, long count) { int ret = 0; spin_lock(&fetchop_lock); if (count > 0 && fetchop_stats.pages_in_use + count > fetchop_stats.pages_total) { ret = -1; } else { fetchop_stats.map_count += mmap; fetchop_stats.pages_in_use += count; } spin_unlock(&fetchop_lock); return ret; } /* * fetchop_mmap * * Called when mmaping the device. Creates fetchop pages and map them * to user space. */ static int fetchop_mmap(struct file *file, struct vm_area_struct *vma) { unsigned long vm_start; unsigned long maddr; int pages; struct vma_data *vdata; if (vma->vm_pgoff != 0) return -EINVAL; if ((vma->vm_flags&VM_WRITE) == 0) return -EPERM; pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; if (fetchop_update_stats(1, pages) < 0) return -ENOSPC; if (!(vdata=vmalloc(sizeof(struct vma_data)+(pages-1)*sizeof(long)))) { fetchop_update_stats(-1, -pages); return -ENOMEM; } vdata->count = 0; vdata->refcnt = ATOMIC_INIT(1); vma->vm_private_data = vdata; vma->vm_flags |= (VM_IO | VM_SHM | VM_LOCKED | VM_NONCACHED); vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); vma->vm_ops = &fetchop_vm_ops; vm_start = vma->vm_start; while (vm_start < vma->vm_end) { maddr = fetchop_alloc_page(numa_node_id()); if (maddr == 0) BUG(); vdata->maddr[vdata->count++] = maddr; if (remap_page_range(vm_start, __pa(maddr), PAGE_SIZE, vma->vm_page_prot)) { fetchop_free_pages(vma->vm_private_data); vfree(vdata); fetchop_update_stats(-1, -pages); return -EAGAIN; } vm_start += PAGE_SIZE; } return 0; } /* * fetchop_open * * Called when a device mapping is created by a means other than mmap * (via fork, etc.). Increments the reference count on the underlying * fetchop data so it is not freed prematurely. */ static void fetchop_open(struct vm_area_struct *vma) { struct vma_data *vdata; vdata = vma->vm_private_data; if (vdata && vdata->count) { atomic_inc(&vdata->refcnt); } } /* * fetchop_close * * Called when unmapping a device mapping. Frees all fetchop pages * belonging to the vma. */ static void fetchop_close(struct vm_area_struct *vma) { struct vma_data *vdata; vdata = vma->vm_private_data; if (vdata && vdata->count && !atomic_dec(&vdata->refcnt)) { fetchop_free_pages(vdata); fetchop_update_stats(-1, -vdata->count); vfree(vdata); } } #ifdef CONFIG_PROC_FS static struct proc_dir_entry *proc_fetchop; /* * fetchop_read_proc * * Implements /proc/fetchop. Return statistics about fetchops. */ static int fetchop_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data) { struct node_fetchops *fops; int len = 0, nid; len += sprintf(page + len, "mappings : %lu\n", fetchop_stats.map_count); len += sprintf(page + len, "current fetchop pages : %lu\n", fetchop_stats.pages_in_use); len += sprintf(page + len, "maximum fetchop pages : %lu\n", fetchop_stats.pages_total); len += sprintf(page + len, "%4s %7s %7s\n", "node", "total", "free"); for (nid = 0; nid < numnodes; nid++) { fops = node_fetchops[nid]; len += sprintf(page + len, "%4d %7d %7d\n", nid, fops ? fops->count : 0, fops ? atomic_read(&fops->free) : 0); } if (len <= off+count) *eof = 1; *start = page + off; len -= off; if (len>count) len = count; if (len<0) len = 0; return len; } static int fetchop_write_proc (struct file *file, const char *userbuf, unsigned long count, void *data) { char buf[80]; if (copy_from_user(buf, userbuf, count < sizeof(buf) ? count : sizeof(buf))) return -EFAULT; return count; } #endif /* CONFIG_PROC_FS */ /* * fetchop_build_memmap, * * Called at boot time to build a map of pages that can be used for * fetchops. */ static int __init fetchop_build_memmap(unsigned long start, unsigned long end, void *arg) { struct node_fetchops *fops; long count, bytes; count = (end - start) >> PAGE_SHIFT; bytes = sizeof(struct node_fetchops) + count/8; fops = vmalloc(bytes); memset(fops, 0, bytes); fops->maddr = FETCHOP_KADDR_TO_MSPEC_ADDR(start); fops->count = count; atomic_add(count, &fops->free); fetchop_stats.pages_total += count; node_fetchops[MSPEC_TO_NID(start)] = fops; sn_flush_all_caches((long)__va(start), end - start); return 0; } /* * fetchop_init * * Called at boot time to initialize the fetchop facility. */ static int __init fetchop_init(void) { int ret; devfs_handle_t hnd; if (!ia64_platform_is("sn2")) return -ENODEV; #ifdef CONFIG_DEVFS_FS if (!devfs_register(NULL, FETCHOP_BASENAME, DEVFS_FL_AUTO_DEVNUM, 0, 0, S_IFCHR | S_IRUGO | S_IWUGO, &fetchop_fops, NULL)) { printk(KERN_ERR "%s: failed to register device\n", DRIVER_ID_STR); return -ENODEV; } #endif if ((ret = misc_register(&fetchop_miscdev))) { printk(KERN_ERR "%s: failed to register device\n", DRIVER_ID_STR); return ret; } printk(KERN_DEBUG "%s: registered misc-device with minor %d\n", DRIVER_ID_STR, fetchop_miscdev.minor); if ((proc_fetchop = create_proc_entry(FETCHOP_BASENAME, 0644, NULL)) == NULL) { printk(KERN_ERR "%s: unable to create proc entry", DRIVER_ID_STR); devfs_unregister(hnd); return -EINVAL; } #ifdef CONFIG_PROC_FS if ((proc_fetchop = create_proc_entry(FETCHOP_BASENAME, 0644, NULL)) == NULL) { printk(KERN_ERR "%s: unable to create proc entry", DRIVER_ID_STR); devfs_unregister(hnd); return -EINVAL; } proc_fetchop->read_proc = fetchop_read_proc; proc_fetchop->write_proc = fetchop_write_proc; #endif /* CONFIG_PROC_FS */ efi_memmap_walk_uc(fetchop_build_memmap, 0); printk(KERN_INFO "%s: v%s\n", DRIVER_ID_STR, REVISION); return 0; } /*----------------------------------------------------------------------------- * KERNEL APIs * Note: right now, these APIs return a full page of fetchops. If these * interfaces are used often for tasks which do not require a full page of * fetchops, new APIs should be added to suballocate out of a single page. */ unsigned long fetchop_kalloc_page(int nid) { if (fetchop_update_stats(1, 1) < 0) return 0; return fetchop_alloc_page(nid); } EXPORT_SYMBOL(fetchop_kalloc_page); void fetchop_kfree_page(unsigned long maddr) { fetchop_free_page(maddr); fetchop_update_stats(-1, -1); } EXPORT_SYMBOL(fetchop_kfree_page); module_init(fetchop_init); MODULE_AUTHOR("Silicon Graphics Inc."); MODULE_DESCRIPTION("Driver for SGI SN 'fetchop' atomic memory operations"); MODULE_LICENSE("GPL");