68 static int unmap_grant_pages(struct gntdev_grant_map *map,
84 	struct gntdev_grant_map *map;  in gntdev_print_maps()  local
87 	list_for_each_entry(map, &priv->maps, next)  in gntdev_print_maps()
89 		       map->index, map->count,  in gntdev_print_maps()
90 		       map->index == text_index && text ? text : "");  in gntdev_print_maps()
94 static void gntdev_free_map(struct gntdev_grant_map *map)  in gntdev_free_map()  argument
96 	if (map == NULL)  in gntdev_free_map()
100 	if (map->dma_vaddr) {  in gntdev_free_map()
103 		args.dev = map->dma_dev;  in gntdev_free_map()
104 		args.coherent = !!(map->dma_flags & GNTDEV_DMA_FLAG_COHERENT);  in gntdev_free_map()
105 		args.nr_pages = map->count;  in gntdev_free_map()
106 		args.pages = map->pages;  in gntdev_free_map()
107 		args.frames = map->frames;  in gntdev_free_map()
108 		args.vaddr = map->dma_vaddr;  in gntdev_free_map()
109 		args.dev_bus_addr = map->dma_bus_addr;  in gntdev_free_map()
114 	if (map->pages)  in gntdev_free_map()
115 		gnttab_free_pages(map->count, map->pages);  in gntdev_free_map()
118 	kfree(map->frames);  in gntdev_free_map()
120 	kfree(map->pages);  in gntdev_free_map()
121 	kfree(map->grants);  in gntdev_free_map()
122 	kfree(map->map_ops);  in gntdev_free_map()
123 	kfree(map->unmap_ops);  in gntdev_free_map()
124 	kfree(map->kmap_ops);  in gntdev_free_map()
125 	kfree(map->kunmap_ops);  in gntdev_free_map()
126 	kfree(map);  in gntdev_free_map()
207 	struct gntdev_grant_map *map;  in gntdev_add_map()  local
209 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_add_map()
210 		if (add->index + add->count < map->index) {  in gntdev_add_map()
211 			list_add_tail(&add->next, &map->next);  in gntdev_add_map()
214 		add->index = map->index + map->count;  in gntdev_add_map()
225 	struct gntdev_grant_map *map;  in gntdev_find_map_index()  local
227 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_find_map_index()
228 		if (map->index != index)  in gntdev_find_map_index()
230 		if (count && map->count != count)  in gntdev_find_map_index()
232 		return map;  in gntdev_find_map_index()
237 void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)  in gntdev_put_map()  argument
239 	if (!map)  in gntdev_put_map()
242 	if (!refcount_dec_and_test(&map->users))  in gntdev_put_map()
245 	atomic_sub(map->count, &pages_mapped);  in gntdev_put_map()
247 	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {  in gntdev_put_map()
248 		notify_remote_via_evtchn(map->notify.event);  in gntdev_put_map()
249 		evtchn_put(map->notify.event);  in gntdev_put_map()
254 		list_del(&map->next);  in gntdev_put_map()
258 	if (map->pages && !use_ptemod)  in gntdev_put_map()
259 		unmap_grant_pages(map, 0, map->count);  in gntdev_put_map()
260 	gntdev_free_map(map);  in gntdev_put_map()
267 	struct gntdev_grant_map *map = data;  in find_grant_ptes()  local
268 	unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;  in find_grant_ptes()
269 	int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte;  in find_grant_ptes()
272 	BUG_ON(pgnr >= map->count);  in find_grant_ptes()
283 	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
284 			  map->grants[pgnr].ref,  in find_grant_ptes()
285 			  map->grants[pgnr].domid);  in find_grant_ptes()
286 	gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
299 int gntdev_map_grant_pages(struct gntdev_grant_map *map)  in gntdev_map_grant_pages()  argument
305 		if (map->map_ops[0].handle != -1)  in gntdev_map_grant_pages()
307 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
309 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
310 			gnttab_set_map_op(&map->map_ops[i], addr, map->flags,  in gntdev_map_grant_pages()
311 				map->grants[i].ref,  in gntdev_map_grant_pages()
312 				map->grants[i].domid);  in gntdev_map_grant_pages()
313 			gnttab_set_unmap_op(&map->unmap_ops[i], addr,  in gntdev_map_grant_pages()
314 				map->flags, -1 /* handle */);  in gntdev_map_grant_pages()
323 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
325 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
326 			BUG_ON(PageHighMem(map->pages[i]));  in gntdev_map_grant_pages()
328 			gnttab_set_map_op(&map->kmap_ops[i], address,  in gntdev_map_grant_pages()
329 				map->flags | GNTMAP_host_map,  in gntdev_map_grant_pages()
330 				map->grants[i].ref,  in gntdev_map_grant_pages()
331 				map->grants[i].domid);  in gntdev_map_grant_pages()
332 			gnttab_set_unmap_op(&map->kunmap_ops[i], address,  in gntdev_map_grant_pages()
333 				map->flags | GNTMAP_host_map, -1);  in gntdev_map_grant_pages()
337 	pr_debug("map %d+%d\n", map->index, map->count);  in gntdev_map_grant_pages()
338 	err = gnttab_map_refs(map->map_ops, use_ptemod ? map->kmap_ops : NULL,  in gntdev_map_grant_pages()
339 			map->pages, map->count);  in gntdev_map_grant_pages()
343 	for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
344 		if (map->map_ops[i].status) {  in gntdev_map_grant_pages()
349 		map->unmap_ops[i].handle = map->map_ops[i].handle;  in gntdev_map_grant_pages()
351 			map->kunmap_ops[i].handle = map->kmap_ops[i].handle;  in gntdev_map_grant_pages()
353 		else if (map->dma_vaddr) {  in gntdev_map_grant_pages()
356 			bfn = pfn_to_bfn(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
357 			map->unmap_ops[i].dev_bus_addr = __pfn_to_phys(bfn);  in gntdev_map_grant_pages()
364 static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in __unmap_grant_pages()  argument
370 	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {  in __unmap_grant_pages()
371 		int pgno = (map->notify.addr >> PAGE_SHIFT);  in __unmap_grant_pages()
374 			uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));  in __unmap_grant_pages()
375 			tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;  in __unmap_grant_pages()
376 			map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;  in __unmap_grant_pages()
380 	unmap_data.unmap_ops = map->unmap_ops + offset;  in __unmap_grant_pages()
381 	unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;  in __unmap_grant_pages()
382 	unmap_data.pages = map->pages + offset;  in __unmap_grant_pages()
390 		if (map->unmap_ops[offset+i].status)  in __unmap_grant_pages()
393 			map->unmap_ops[offset+i].handle,  in __unmap_grant_pages()
394 			map->unmap_ops[offset+i].status);  in __unmap_grant_pages()
395 		map->unmap_ops[offset+i].handle = -1;  in __unmap_grant_pages()
400 static int unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in unmap_grant_pages()  argument
405 	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);  in unmap_grant_pages()
411 		while (pages && map->unmap_ops[offset].handle == -1) {  in unmap_grant_pages()
417 			if (map->unmap_ops[offset+range].handle == -1)  in unmap_grant_pages()
421 		err = __unmap_grant_pages(map, offset, range);  in unmap_grant_pages()
433 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_open()  local
436 	refcount_inc(&map->users);  in gntdev_vma_open()
441 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_close()  local
455 		map->vma = NULL;  in gntdev_vma_close()
459 	gntdev_put_map(priv, map);  in gntdev_vma_close()
465 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_find_special_page()  local
467 	return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];  in gntdev_vma_find_special_page()
478 static bool in_range(struct gntdev_grant_map *map,  in in_range()  argument
481 	if (!map->vma)  in in_range()
483 	if (map->vma->vm_start >= end)  in in_range()
485 	if (map->vma->vm_end <= start)  in in_range()
491 static int unmap_if_in_range(struct gntdev_grant_map *map,  in unmap_if_in_range()  argument
498 	if (!in_range(map, start, end))  in unmap_if_in_range()
504 	mstart = max(start, map->vma->vm_start);  in unmap_if_in_range()
505 	mend   = min(end,   map->vma->vm_end);  in unmap_if_in_range()
507 			map->index, map->count,  in unmap_if_in_range()
508 			map->vma->vm_start, map->vma->vm_end,  in unmap_if_in_range()
510 	err = unmap_grant_pages(map,  in unmap_if_in_range()
511 				(mstart - map->vma->vm_start) >> PAGE_SHIFT,  in unmap_if_in_range()
522 	struct gntdev_grant_map *map;  in mn_invl_range_start()  local
530 	list_for_each_entry(map, &priv->maps, next) {  in mn_invl_range_start()
531 		ret = unmap_if_in_range(map, range->start, range->end,  in mn_invl_range_start()
536 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_invl_range_start()
537 		ret = unmap_if_in_range(map, range->start, range->end,  in mn_invl_range_start()
553 	struct gntdev_grant_map *map;  in mn_release()  local
557 	list_for_each_entry(map, &priv->maps, next) {  in mn_release()
558 		if (!map->vma)  in mn_release()
561 				map->index, map->count,  in mn_release()
562 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
563 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
566 	list_for_each_entry(map, &priv->freeable_maps, next) {  in mn_release()
567 		if (!map->vma)  in mn_release()
570 				map->index, map->count,  in mn_release()
571 				map->vma->vm_start, map->vma->vm_end);  in mn_release()
572 		err = unmap_grant_pages(map, /* offset */ 0, map->count);  in mn_release()
636 	struct gntdev_grant_map *map;  in gntdev_release()  local
642 		map = list_entry(priv->maps.next,  in gntdev_release()
644 		list_del(&map->next);  in gntdev_release()
645 		gntdev_put_map(NULL /* already removed */, map);  in gntdev_release()
665 	struct gntdev_grant_map *map;  in gntdev_ioctl_map_grant_ref()  local
675 	map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */);  in gntdev_ioctl_map_grant_ref()
676 	if (!map)  in gntdev_ioctl_map_grant_ref()
681 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
685 	if (copy_from_user(map->grants, &u->refs,  in gntdev_ioctl_map_grant_ref()
686 			   sizeof(map->grants[0]) * op.count) != 0) {  in gntdev_ioctl_map_grant_ref()
687 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
692 	gntdev_add_map(priv, map);  in gntdev_ioctl_map_grant_ref()
693 	op.index = map->index << PAGE_SHIFT;  in gntdev_ioctl_map_grant_ref()
706 	struct gntdev_grant_map *map;  in gntdev_ioctl_unmap_grant_ref()  local
714 	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);  in gntdev_ioctl_unmap_grant_ref()
715 	if (map) {  in gntdev_ioctl_unmap_grant_ref()
716 		list_del(&map->next);  in gntdev_ioctl_unmap_grant_ref()
718 			list_add_tail(&map->next, &priv->freeable_maps);  in gntdev_ioctl_unmap_grant_ref()
722 	if (map)  in gntdev_ioctl_unmap_grant_ref()
723 		gntdev_put_map(priv, map);  in gntdev_ioctl_unmap_grant_ref()
732 	struct gntdev_grant_map *map;  in gntdev_ioctl_get_offset_for_vaddr()  local
744 	map = vma->vm_private_data;  in gntdev_ioctl_get_offset_for_vaddr()
745 	if (!map)  in gntdev_ioctl_get_offset_for_vaddr()
748 	op.offset = map->index << PAGE_SHIFT;  in gntdev_ioctl_get_offset_for_vaddr()
749 	op.count = map->count;  in gntdev_ioctl_get_offset_for_vaddr()
763 	struct gntdev_grant_map *map;  in gntdev_ioctl_notify()  local
791 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_ioctl_notify()
792 		uint64_t begin = map->index << PAGE_SHIFT;  in gntdev_ioctl_notify()
793 		uint64_t end = (map->index + map->count) << PAGE_SHIFT;  in gntdev_ioctl_notify()
802 			(map->flags & GNTMAP_readonly)) {  in gntdev_ioctl_notify()
807 	out_flags = map->notify.flags;  in gntdev_ioctl_notify()
808 	out_event = map->notify.event;  in gntdev_ioctl_notify()
810 	map->notify.flags = op.action;  in gntdev_ioctl_notify()
811 	map->notify.addr = op.index - (map->index << PAGE_SHIFT);  in gntdev_ioctl_notify()
812 	map->notify.event = op.event_channel_port;  in gntdev_ioctl_notify()
1075 	struct gntdev_grant_map *map;  in gntdev_mmap()  local
1085 	map = gntdev_find_map_index(priv, index, count);  in gntdev_mmap()
1086 	if (!map)  in gntdev_mmap()
1088 	if (use_ptemod && map->vma)  in gntdev_mmap()
1095 	refcount_inc(&map->users);  in gntdev_mmap()
1104 	vma->vm_private_data = map;  in gntdev_mmap()
1107 		map->vma = vma;  in gntdev_mmap()
1109 	if (map->flags) {  in gntdev_mmap()
1111 				(map->flags & GNTMAP_readonly))  in gntdev_mmap()
1114 		map->flags = GNTMAP_host_map;  in gntdev_mmap()
1116 			map->flags |= GNTMAP_readonly;  in gntdev_mmap()
1122 		map->pages_vm_start = vma->vm_start;  in gntdev_mmap()
1125 					  find_grant_ptes, map);  in gntdev_mmap()
1132 	err = gntdev_map_grant_pages(map);  in gntdev_mmap()
1137 		err = vm_map_pages_zero(vma, map->pages, map->count);  in gntdev_mmap()
1169 		map->vma = NULL;  in gntdev_mmap()
1170 		unmap_grant_pages(map, 0, map->count);  in gntdev_mmap()
1172 	gntdev_put_map(priv, map);  in gntdev_mmap()