67 static void unmap_grant_pages(struct gntdev_grant_map *map,
83 	struct gntdev_grant_map *map;  in gntdev_print_maps()  local
86 	list_for_each_entry(map, &priv->maps, next)  in gntdev_print_maps()
88 		       map->index, map->count,  in gntdev_print_maps()
89 		       map->index == text_index && text ? text : "");  in gntdev_print_maps()
93 static void gntdev_free_map(struct gntdev_grant_map *map)  in gntdev_free_map()  argument
95 	if (map == NULL)  in gntdev_free_map()
99 	if (map->dma_vaddr) {  in gntdev_free_map()
102 		args.dev = map->dma_dev;  in gntdev_free_map()
103 		args.coherent = !!(map->dma_flags & GNTDEV_DMA_FLAG_COHERENT);  in gntdev_free_map()
104 		args.nr_pages = map->count;  in gntdev_free_map()
105 		args.pages = map->pages;  in gntdev_free_map()
106 		args.frames = map->frames;  in gntdev_free_map()
107 		args.vaddr = map->dma_vaddr;  in gntdev_free_map()
108 		args.dev_bus_addr = map->dma_bus_addr;  in gntdev_free_map()
113 	if (map->pages)  in gntdev_free_map()
114 		gnttab_free_pages(map->count, map->pages);  in gntdev_free_map()
117 	kvfree(map->frames);  in gntdev_free_map()
119 	kvfree(map->pages);  in gntdev_free_map()
120 	kvfree(map->grants);  in gntdev_free_map()
121 	kvfree(map->map_ops);  in gntdev_free_map()
122 	kvfree(map->unmap_ops);  in gntdev_free_map()
123 	kvfree(map->kmap_ops);  in gntdev_free_map()
124 	kvfree(map->kunmap_ops);  in gntdev_free_map()
125 	kvfree(map->being_removed);  in gntdev_free_map()
126 	kfree(map);  in gntdev_free_map()
221 	struct gntdev_grant_map *map;  in gntdev_add_map()  local
223 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_add_map()
224 		if (add->index + add->count < map->index) {  in gntdev_add_map()
225 			list_add_tail(&add->next, &map->next);  in gntdev_add_map()
228 		add->index = map->index + map->count;  in gntdev_add_map()
239 	struct gntdev_grant_map *map;  in gntdev_find_map_index()  local
241 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_find_map_index()
242 		if (map->index != index)  in gntdev_find_map_index()
244 		if (count && map->count != count)  in gntdev_find_map_index()
246 		return map;  in gntdev_find_map_index()
251 void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)  in gntdev_put_map()  argument
253 	if (!map)  in gntdev_put_map()
256 	if (!refcount_dec_and_test(&map->users))  in gntdev_put_map()
259 	if (map->pages && !use_ptemod) {  in gntdev_put_map()
270 		refcount_set(&map->users, 1);  in gntdev_put_map()
277 		unmap_grant_pages(map, 0, map->count);  in gntdev_put_map()
280 		if (!refcount_dec_and_test(&map->users))  in gntdev_put_map()
289 	if (use_ptemod && map->notifier_init)  in gntdev_put_map()
290 		mmu_interval_notifier_remove(&map->notifier);  in gntdev_put_map()
292 	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {  in gntdev_put_map()
293 		notify_remote_via_evtchn(map->notify.event);  in gntdev_put_map()
294 		evtchn_put(map->notify.event);  in gntdev_put_map()
296 	gntdev_free_map(map);  in gntdev_put_map()
303 	struct gntdev_grant_map *map = data;  in find_grant_ptes()  local
304 	unsigned int pgnr = (addr - map->pages_vm_start) >> PAGE_SHIFT;  in find_grant_ptes()
305 	int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte |  in find_grant_ptes()
309 	BUG_ON(pgnr >= map->count);  in find_grant_ptes()
312 	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
313 			  map->grants[pgnr].ref,  in find_grant_ptes()
314 			  map->grants[pgnr].domid);  in find_grant_ptes()
315 	gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
320 int gntdev_map_grant_pages(struct gntdev_grant_map *map)  in gntdev_map_grant_pages()  argument
327 		if (map->map_ops[0].handle != INVALID_GRANT_HANDLE)  in gntdev_map_grant_pages()
329 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
331 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
332 			gnttab_set_map_op(&map->map_ops[i], addr, map->flags,  in gntdev_map_grant_pages()
333 				map->grants[i].ref,  in gntdev_map_grant_pages()
334 				map->grants[i].domid);  in gntdev_map_grant_pages()
335 			gnttab_set_unmap_op(&map->unmap_ops[i], addr,  in gntdev_map_grant_pages()
336 				map->flags, INVALID_GRANT_HANDLE);  in gntdev_map_grant_pages()
350 		unsigned int flags = (map->flags & ~GNTMAP_device_map) |  in gntdev_map_grant_pages()
353 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
355 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
356 			BUG_ON(PageHighMem(map->pages[i]));  in gntdev_map_grant_pages()
358 			gnttab_set_map_op(&map->kmap_ops[i], address, flags,  in gntdev_map_grant_pages()
359 				map->grants[i].ref,  in gntdev_map_grant_pages()
360 				map->grants[i].domid);  in gntdev_map_grant_pages()
361 			gnttab_set_unmap_op(&map->kunmap_ops[i], address,  in gntdev_map_grant_pages()
366 	pr_debug("map %d+%d\n", map->index, map->count);  in gntdev_map_grant_pages()
367 	err = gnttab_map_refs(map->map_ops, map->kmap_ops, map->pages,  in gntdev_map_grant_pages()
368 			map->count);  in gntdev_map_grant_pages()
370 	for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
371 		if (map->map_ops[i].status == GNTST_okay) {  in gntdev_map_grant_pages()
372 			map->unmap_ops[i].handle = map->map_ops[i].handle;  in gntdev_map_grant_pages()
377 		if (map->flags & GNTMAP_device_map)  in gntdev_map_grant_pages()
378 			map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr;  in gntdev_map_grant_pages()
381 			if (map->kmap_ops[i].status == GNTST_okay) {  in gntdev_map_grant_pages()
383 				map->kunmap_ops[i].handle = map->kmap_ops[i].handle;  in gntdev_map_grant_pages()
388 	atomic_add(alloced, &map->live_grants);  in gntdev_map_grant_pages()
396 	struct gntdev_grant_map *map = data->data;  in __unmap_grant_pages_done()  local
397 	unsigned int offset = data->unmap_ops - map->unmap_ops;  in __unmap_grant_pages_done()
402 		if (map->unmap_ops[offset + i].status == GNTST_okay &&  in __unmap_grant_pages_done()
403 		    map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE)  in __unmap_grant_pages_done()
406 		WARN_ON(map->unmap_ops[offset + i].status != GNTST_okay &&  in __unmap_grant_pages_done()
407 			map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE);  in __unmap_grant_pages_done()
409 			map->unmap_ops[offset+i].handle,  in __unmap_grant_pages_done()
410 			map->unmap_ops[offset+i].status);  in __unmap_grant_pages_done()
411 		map->unmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;  in __unmap_grant_pages_done()
413 			if (map->kunmap_ops[offset + i].status == GNTST_okay &&  in __unmap_grant_pages_done()
414 			    map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE)  in __unmap_grant_pages_done()
417 			WARN_ON(map->kunmap_ops[offset + i].status != GNTST_okay &&  in __unmap_grant_pages_done()
418 				map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE);  in __unmap_grant_pages_done()
420 				 map->kunmap_ops[offset+i].handle,  in __unmap_grant_pages_done()
421 				 map->kunmap_ops[offset+i].status);  in __unmap_grant_pages_done()
422 			map->kunmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;  in __unmap_grant_pages_done()
430 	live_grants = atomic_sub_return(successful_unmaps, &map->live_grants);  in __unmap_grant_pages_done()
436 	gntdev_put_map(NULL, map);  in __unmap_grant_pages_done()
439 static void __unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in __unmap_grant_pages()  argument
442 	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {  in __unmap_grant_pages()
443 		int pgno = (map->notify.addr >> PAGE_SHIFT);  in __unmap_grant_pages()
447 			uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));  in __unmap_grant_pages()
449 			tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;  in __unmap_grant_pages()
450 			map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;  in __unmap_grant_pages()
454 	map->unmap_data.unmap_ops = map->unmap_ops + offset;  in __unmap_grant_pages()
455 	map->unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;  in __unmap_grant_pages()
456 	map->unmap_data.pages = map->pages + offset;  in __unmap_grant_pages()
457 	map->unmap_data.count = pages;  in __unmap_grant_pages()
458 	map->unmap_data.done = __unmap_grant_pages_done;  in __unmap_grant_pages()
459 	map->unmap_data.data = map;  in __unmap_grant_pages()
460 	refcount_inc(&map->users); /* to keep map alive during async call below */  in __unmap_grant_pages()
462 	gnttab_unmap_refs_async(&map->unmap_data);  in __unmap_grant_pages()
465 static void unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in unmap_grant_pages()  argument
470 	if (atomic_read(&map->live_grants) == 0)  in unmap_grant_pages()
473 	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);  in unmap_grant_pages()
479 		while (pages && map->being_removed[offset]) {  in unmap_grant_pages()
485 			if (map->being_removed[offset + range])  in unmap_grant_pages()
487 			map->being_removed[offset + range] = true;  in unmap_grant_pages()
491 			__unmap_grant_pages(map, offset, range);  in unmap_grant_pages()
501 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_open()  local
504 	refcount_inc(&map->users);  in gntdev_vma_open()
509 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_close()  local
516 	gntdev_put_map(priv, map);  in gntdev_vma_close()
522 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_find_special_page()  local
524 	return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];  in gntdev_vma_find_special_page()
539 	struct gntdev_grant_map *map =  in gntdev_invalidate()  local
547 	map_start = map->pages_vm_start;  in gntdev_invalidate()
548 	map_end = map->pages_vm_start + (map->count << PAGE_SHIFT);  in gntdev_invalidate()
562 		 map->index, map->count, map_start, map_end,  in gntdev_invalidate()
564 	unmap_grant_pages(map, (mstart - map_start) >> PAGE_SHIFT,  in gntdev_invalidate()
610 	struct gntdev_grant_map *map;  in gntdev_release()  local
616 		map = list_entry(priv->maps.next,  in gntdev_release()
618 		list_del(&map->next);  in gntdev_release()
619 		gntdev_put_map(NULL /* already removed */, map);  in gntdev_release()
635 	struct gntdev_grant_map *map;  in gntdev_ioctl_map_grant_ref()  local
645 	map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */);  in gntdev_ioctl_map_grant_ref()
646 	if (!map)  in gntdev_ioctl_map_grant_ref()
649 	if (copy_from_user(map->grants, &u->refs,  in gntdev_ioctl_map_grant_ref()
650 			   sizeof(map->grants[0]) * op.count) != 0) {  in gntdev_ioctl_map_grant_ref()
651 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
656 	gntdev_add_map(priv, map);  in gntdev_ioctl_map_grant_ref()
657 	op.index = map->index << PAGE_SHIFT;  in gntdev_ioctl_map_grant_ref()
670 	struct gntdev_grant_map *map;  in gntdev_ioctl_unmap_grant_ref()  local
678 	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);  in gntdev_ioctl_unmap_grant_ref()
679 	if (map) {  in gntdev_ioctl_unmap_grant_ref()
680 		list_del(&map->next);  in gntdev_ioctl_unmap_grant_ref()
684 	if (map)  in gntdev_ioctl_unmap_grant_ref()
685 		gntdev_put_map(priv, map);  in gntdev_ioctl_unmap_grant_ref()
694 	struct gntdev_grant_map *map;  in gntdev_ioctl_get_offset_for_vaddr()  local
706 	map = vma->vm_private_data;  in gntdev_ioctl_get_offset_for_vaddr()
707 	if (!map)  in gntdev_ioctl_get_offset_for_vaddr()
710 	op.offset = map->index << PAGE_SHIFT;  in gntdev_ioctl_get_offset_for_vaddr()
711 	op.count = map->count;  in gntdev_ioctl_get_offset_for_vaddr()
725 	struct gntdev_grant_map *map;  in gntdev_ioctl_notify()  local
753 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_ioctl_notify()
754 		uint64_t begin = map->index << PAGE_SHIFT;  in gntdev_ioctl_notify()
755 		uint64_t end = (map->index + map->count) << PAGE_SHIFT;  in gntdev_ioctl_notify()
764 			(map->flags & GNTMAP_readonly)) {  in gntdev_ioctl_notify()
769 	out_flags = map->notify.flags;  in gntdev_ioctl_notify()
770 	out_event = map->notify.event;  in gntdev_ioctl_notify()
772 	map->notify.flags = op.action;  in gntdev_ioctl_notify()
773 	map->notify.addr = op.index - (map->index << PAGE_SHIFT);  in gntdev_ioctl_notify()
774 	map->notify.event = op.event_channel_port;  in gntdev_ioctl_notify()
1038 	struct gntdev_grant_map *map;  in gntdev_mmap()  local
1048 	map = gntdev_find_map_index(priv, index, count);  in gntdev_mmap()
1049 	if (!map)  in gntdev_mmap()
1051 	if (!atomic_add_unless(&map->in_use, 1, 1))  in gntdev_mmap()
1054 	refcount_inc(&map->users);  in gntdev_mmap()
1063 	vma->vm_private_data = map;  in gntdev_mmap()
1064 	if (map->flags) {  in gntdev_mmap()
1066 				(map->flags & GNTMAP_readonly))  in gntdev_mmap()
1069 		map->flags = GNTMAP_host_map;  in gntdev_mmap()
1071 			map->flags |= GNTMAP_readonly;  in gntdev_mmap()
1074 	map->pages_vm_start = vma->vm_start;  in gntdev_mmap()
1078 			&map->notifier, vma->vm_mm, vma->vm_start,  in gntdev_mmap()
1083 		map->notifier_init = true;  in gntdev_mmap()
1098 		mmu_interval_read_begin(&map->notifier);  in gntdev_mmap()
1102 					  find_grant_ptes, map);  in gntdev_mmap()
1109 	err = gntdev_map_grant_pages(map);  in gntdev_mmap()
1114 		err = vm_map_pages_zero(vma, map->pages, map->count);  in gntdev_mmap()
1129 		unmap_grant_pages(map, 0, map->count);  in gntdev_mmap()
1130 	gntdev_put_map(priv, map);  in gntdev_mmap()