70 static int unmap_grant_pages(struct gntdev_grant_map *map,
86 	struct gntdev_grant_map *map;  in gntdev_print_maps()  local
89 	list_for_each_entry(map, &priv->maps, next)  in gntdev_print_maps()
91 		       map->index, map->count,  in gntdev_print_maps()
92 		       map->index == text_index && text ? text : "");  in gntdev_print_maps()
96 static void gntdev_free_map(struct gntdev_grant_map *map)  in gntdev_free_map()  argument
98 	if (map == NULL)  in gntdev_free_map()
102 	if (map->dma_vaddr) {  in gntdev_free_map()
105 		args.dev = map->dma_dev;  in gntdev_free_map()
106 		args.coherent = !!(map->dma_flags & GNTDEV_DMA_FLAG_COHERENT);  in gntdev_free_map()
107 		args.nr_pages = map->count;  in gntdev_free_map()
108 		args.pages = map->pages;  in gntdev_free_map()
109 		args.frames = map->frames;  in gntdev_free_map()
110 		args.vaddr = map->dma_vaddr;  in gntdev_free_map()
111 		args.dev_bus_addr = map->dma_bus_addr;  in gntdev_free_map()
116 	if (map->pages)  in gntdev_free_map()
117 		gnttab_free_pages(map->count, map->pages);  in gntdev_free_map()
120 	kfree(map->frames);  in gntdev_free_map()
122 	kfree(map->pages);  in gntdev_free_map()
123 	kfree(map->grants);  in gntdev_free_map()
124 	kfree(map->map_ops);  in gntdev_free_map()
125 	kfree(map->unmap_ops);  in gntdev_free_map()
126 	kfree(map->kmap_ops);  in gntdev_free_map()
127 	kfree(map->kunmap_ops);  in gntdev_free_map()
128 	kfree(map);  in gntdev_free_map()
209 	struct gntdev_grant_map *map;  in gntdev_add_map()  local
211 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_add_map()
212 		if (add->index + add->count < map->index) {  in gntdev_add_map()
213 			list_add_tail(&add->next, &map->next);  in gntdev_add_map()
216 		add->index = map->index + map->count;  in gntdev_add_map()
227 	struct gntdev_grant_map *map;  in gntdev_find_map_index()  local
229 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_find_map_index()
230 		if (map->index != index)  in gntdev_find_map_index()
232 		if (count && map->count != count)  in gntdev_find_map_index()
234 		return map;  in gntdev_find_map_index()
239 void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)  in gntdev_put_map()  argument
241 	if (!map)  in gntdev_put_map()
244 	if (!refcount_dec_and_test(&map->users))  in gntdev_put_map()
247 	atomic_sub(map->count, &pages_mapped);  in gntdev_put_map()
249 	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {  in gntdev_put_map()
250 		notify_remote_via_evtchn(map->notify.event);  in gntdev_put_map()
251 		evtchn_put(map->notify.event);  in gntdev_put_map()
256 		list_del(&map->next);  in gntdev_put_map()
260 	if (map->pages && !use_ptemod)  in gntdev_put_map()
261 		unmap_grant_pages(map, 0, map->count);  in gntdev_put_map()
262 	gntdev_free_map(map);  in gntdev_put_map()
270 	struct gntdev_grant_map *map = data;  in find_grant_ptes()  local
271 	unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;  in find_grant_ptes()
272 	int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte;  in find_grant_ptes()
275 	BUG_ON(pgnr >= map->count);  in find_grant_ptes()
286 	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
287 			  map->grants[pgnr].ref,  in find_grant_ptes()
288 			  map->grants[pgnr].domid);  in find_grant_ptes()
289 	gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
303 int gntdev_map_grant_pages(struct gntdev_grant_map *map)  in gntdev_map_grant_pages()  argument
309 		if (map->map_ops[0].handle != -1)  in gntdev_map_grant_pages()
311 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
313 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
314 			gnttab_set_map_op(&map->map_ops[i], addr, map->flags,  in gntdev_map_grant_pages()
315 				map->grants[i].ref,  in gntdev_map_grant_pages()
316 				map->grants[i].domid);  in gntdev_map_grant_pages()
317 			gnttab_set_unmap_op(&map->unmap_ops[i], addr,  in gntdev_map_grant_pages()
318 				map->flags, -1 /* handle */);  in gntdev_map_grant_pages()
327 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
329 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
330 			BUG_ON(PageHighMem(map->pages[i]));  in gntdev_map_grant_pages()
332 			gnttab_set_map_op(&map->kmap_ops[i], address,  in gntdev_map_grant_pages()
333 				map->flags | GNTMAP_host_map,  in gntdev_map_grant_pages()
334 				map->grants[i].ref,  in gntdev_map_grant_pages()
335 				map->grants[i].domid);  in gntdev_map_grant_pages()
336 			gnttab_set_unmap_op(&map->kunmap_ops[i], address,  in gntdev_map_grant_pages()
337 				map->flags | GNTMAP_host_map, -1);  in gntdev_map_grant_pages()
341 	pr_debug("map %d+%d\n", map->index, map->count);  in gntdev_map_grant_pages()
342 	err = gnttab_map_refs(map->map_ops, use_ptemod ? map->kmap_ops : NULL,  in gntdev_map_grant_pages()
343 			map->pages, map->count);  in gntdev_map_grant_pages()
347 	for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
348 		if (map->map_ops[i].status) {  in gntdev_map_grant_pages()
353 		map->unmap_ops[i].handle = map->map_ops[i].handle;  in gntdev_map_grant_pages()
355 			map->kunmap_ops[i].handle = map->kmap_ops[i].handle;  in gntdev_map_grant_pages()
357 		else if (map->dma_vaddr) {  in gntdev_map_grant_pages()
360 			bfn = pfn_to_bfn(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
361 			map->unmap_ops[i].dev_bus_addr = __pfn_to_phys(bfn);  in gntdev_map_grant_pages()
368 static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in __unmap_grant_pages()  argument
374 	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {  in __unmap_grant_pages()
375 		int pgno = (map->notify.addr >> PAGE_SHIFT);  in __unmap_grant_pages()
378 			uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));  in __unmap_grant_pages()
379 			tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;  in __unmap_grant_pages()
380 			map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;  in __unmap_grant_pages()
384 	unmap_data.unmap_ops = map->unmap_ops + offset;  in __unmap_grant_pages()
385 	unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;  in __unmap_grant_pages()
386 	unmap_data.pages = map->pages + offset;  in __unmap_grant_pages()
394 		if (map->unmap_ops[offset+i].status)  in __unmap_grant_pages()
397 			map->unmap_ops[offset+i].handle,  in __unmap_grant_pages()
398 			map->unmap_ops[offset+i].status);  in __unmap_grant_pages()
399 		map->unmap_ops[offset+i].handle = -1;  in __unmap_grant_pages()
404 static int unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in unmap_grant_pages()  argument
409 	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);  in unmap_grant_pages()
415 		while (pages && map->unmap_ops[offset].handle == -1) {  in unmap_grant_pages()
421 			if (map->unmap_ops[offset+range].handle == -1)  in unmap_grant_pages()
425 		err = __unmap_grant_pages(map, offset, range);  in unmap_grant_pages()
437 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_open()  local
440 	refcount_inc(&map->users);  in gntdev_vma_open()
445 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_close()  local
459 		map->vma = NULL;  in gntdev_vma_close()
463 	gntdev_put_map(priv, map);  in gntdev_vma_close()
469 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_find_special_page()  local
471 	return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];  in gntdev_vma_find_special_page()
482 static bool in_range(struct gntdev_grant_map *map,  in in_range()  argument
485 	if (!map->vma)  in in_range()
487 	if (map->vma->vm_start >= end)  in in_range()
489 	if (map->vma->vm_end <= start)  in in_range()
495 static int unmap_if_in_range(struct gntdev_grant_map *map,  in unmap_if_in_range()  argument
502 	if (!in_range(map, start, end))  in unmap_if_in_range()
508 	mstart = max(start, map->vma->vm_start);  in unmap_if_in_range()
509 	mend   = min(end,   map->vma->vm_end);  in unmap_if_in_range()
511 			map->index, map->count,  in unmap_if_in_range()
512 			map->vma->vm_start, map->vma->vm_end,  in unmap_if_in_range()
514 	err = unmap_grant_pages(map,  in unmap_if_in_range()
515 				(mstart - map->vma->vm_start) >> PAGE_SHIFT,  in unmap_if_in_range()
528 	struct gntdev_grant_map *map;  in mn_invl_range_start()  local
536 	list_for_each_entry(map, &priv->maps, next) {  in mn_invl_range_start()
537 		ret = unmap_if_in_range(map, start, end, blockable);  in mn_invl_range_start()
541 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_invl_range_start()
542 		ret = unmap_if_in_range(map, start, end, blockable);  in mn_invl_range_start()
557 	struct gntdev_grant_map *map;  in mn_release()  local
561 	list_for_each_entry(map, &priv->maps, next) {  in mn_release()
562 		if (!map->vma)  in mn_release()
565 				map->index, map->count,  in mn_release()
566 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
567 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
570 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_release()
571 		if (!map->vma)  in mn_release()
574 				map->index, map->count,  in mn_release()
575 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
576 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
647 	struct gntdev_grant_map *map;  in gntdev_release()  local
653 		map = list_entry(priv->maps.next,  in gntdev_release()
655 		list_del(&map->next);  in gntdev_release()
656 		gntdev_put_map(NULL /* already removed */, map);  in gntdev_release()
676 	struct gntdev_grant_map *map;  in gntdev_ioctl_map_grant_ref()  local
686 	map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */);  in gntdev_ioctl_map_grant_ref()
687 	if (!map)  in gntdev_ioctl_map_grant_ref()
692 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
696 	if (copy_from_user(map->grants, &u->refs,  in gntdev_ioctl_map_grant_ref()
697 			   sizeof(map->grants[0]) * op.count) != 0) {  in gntdev_ioctl_map_grant_ref()
698 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
703 	gntdev_add_map(priv, map);  in gntdev_ioctl_map_grant_ref()
704 	op.index = map->index << PAGE_SHIFT;  in gntdev_ioctl_map_grant_ref()
717 	struct gntdev_grant_map *map;  in gntdev_ioctl_unmap_grant_ref()  local
725 	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);  in gntdev_ioctl_unmap_grant_ref()
726 	if (map) {  in gntdev_ioctl_unmap_grant_ref()
727 		list_del(&map->next);  in gntdev_ioctl_unmap_grant_ref()
729 			list_add_tail(&map->next, &priv->freeable_maps);  in gntdev_ioctl_unmap_grant_ref()
733 	if (map)  in gntdev_ioctl_unmap_grant_ref()
734 		gntdev_put_map(priv, map);  in gntdev_ioctl_unmap_grant_ref()
743 	struct gntdev_grant_map *map;  in gntdev_ioctl_get_offset_for_vaddr()  local
755 	map = vma->vm_private_data;  in gntdev_ioctl_get_offset_for_vaddr()
756 	if (!map)  in gntdev_ioctl_get_offset_for_vaddr()
759 	op.offset = map->index << PAGE_SHIFT;  in gntdev_ioctl_get_offset_for_vaddr()
760 	op.count = map->count;  in gntdev_ioctl_get_offset_for_vaddr()
774 	struct gntdev_grant_map *map;  in gntdev_ioctl_notify()  local
802 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_ioctl_notify()
803 		uint64_t begin = map->index << PAGE_SHIFT;  in gntdev_ioctl_notify()
804 		uint64_t end = (map->index + map->count) << PAGE_SHIFT;  in gntdev_ioctl_notify()
813 			(map->flags & GNTMAP_readonly)) {  in gntdev_ioctl_notify()
818 	out_flags = map->notify.flags;  in gntdev_ioctl_notify()
819 	out_event = map->notify.event;  in gntdev_ioctl_notify()
821 	map->notify.flags = op.action;  in gntdev_ioctl_notify()
822 	map->notify.addr = op.index - (map->index << PAGE_SHIFT);  in gntdev_ioctl_notify()
823 	map->notify.event = op.event_channel_port;  in gntdev_ioctl_notify()
1086 	struct gntdev_grant_map *map;  in gntdev_mmap()  local
1096 	map = gntdev_find_map_index(priv, index, count);  in gntdev_mmap()
1097 	if (!map)  in gntdev_mmap()
1099 	if (use_ptemod && map->vma)  in gntdev_mmap()
1106 	refcount_inc(&map->users);  in gntdev_mmap()
1115 	vma->vm_private_data = map;  in gntdev_mmap()
1118 		map->vma = vma;  in gntdev_mmap()
1120 	if (map->flags) {  in gntdev_mmap()
1122 				(map->flags & GNTMAP_readonly))  in gntdev_mmap()
1125 		map->flags = GNTMAP_host_map;  in gntdev_mmap()
1127 			map->flags |= GNTMAP_readonly;  in gntdev_mmap()
1133 		map->pages_vm_start = vma->vm_start;  in gntdev_mmap()
1136 					  find_grant_ptes, map);  in gntdev_mmap()
1143 	err = gntdev_map_grant_pages(map);  in gntdev_mmap()
1150 				map->pages[i]);  in gntdev_mmap()
1183 		map->vma = NULL;  in gntdev_mmap()
1184 		unmap_grant_pages(map, 0, map->count);  in gntdev_mmap()
1186 	gntdev_put_map(priv, map);  in gntdev_mmap()