65 static int unmap_grant_pages(struct gntdev_grant_map *map,
81 	struct gntdev_grant_map *map;  in gntdev_print_maps()  local
84 	list_for_each_entry(map, &priv->maps, next)  in gntdev_print_maps()
86 		       map->index, map->count,  in gntdev_print_maps()
87 		       map->index == text_index && text ? text : "");  in gntdev_print_maps()
91 static void gntdev_free_map(struct gntdev_grant_map *map)  in gntdev_free_map()  argument
93 	if (map == NULL)  in gntdev_free_map()
97 	if (map->dma_vaddr) {  in gntdev_free_map()
100 		args.dev = map->dma_dev;  in gntdev_free_map()
101 		args.coherent = !!(map->dma_flags & GNTDEV_DMA_FLAG_COHERENT);  in gntdev_free_map()
102 		args.nr_pages = map->count;  in gntdev_free_map()
103 		args.pages = map->pages;  in gntdev_free_map()
104 		args.frames = map->frames;  in gntdev_free_map()
105 		args.vaddr = map->dma_vaddr;  in gntdev_free_map()
106 		args.dev_bus_addr = map->dma_bus_addr;  in gntdev_free_map()
111 	if (map->pages)  in gntdev_free_map()
112 		gnttab_free_pages(map->count, map->pages);  in gntdev_free_map()
115 	kvfree(map->frames);  in gntdev_free_map()
117 	kvfree(map->pages);  in gntdev_free_map()
118 	kvfree(map->grants);  in gntdev_free_map()
119 	kvfree(map->map_ops);  in gntdev_free_map()
120 	kvfree(map->unmap_ops);  in gntdev_free_map()
121 	kvfree(map->kmap_ops);  in gntdev_free_map()
122 	kvfree(map->kunmap_ops);  in gntdev_free_map()
123 	kfree(map);  in gntdev_free_map()
215 	struct gntdev_grant_map *map;  in gntdev_add_map()  local
217 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_add_map()
218 		if (add->index + add->count < map->index) {  in gntdev_add_map()
219 			list_add_tail(&add->next, &map->next);  in gntdev_add_map()
222 		add->index = map->index + map->count;  in gntdev_add_map()
233 	struct gntdev_grant_map *map;  in gntdev_find_map_index()  local
235 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_find_map_index()
236 		if (map->index != index)  in gntdev_find_map_index()
238 		if (count && map->count != count)  in gntdev_find_map_index()
240 		return map;  in gntdev_find_map_index()
245 void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map)  in gntdev_put_map()  argument
247 	if (!map)  in gntdev_put_map()
250 	if (!refcount_dec_and_test(&map->users))  in gntdev_put_map()
253 	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {  in gntdev_put_map()
254 		notify_remote_via_evtchn(map->notify.event);  in gntdev_put_map()
255 		evtchn_put(map->notify.event);  in gntdev_put_map()
258 	if (map->pages && !use_ptemod)  in gntdev_put_map()
259 		unmap_grant_pages(map, 0, map->count);  in gntdev_put_map()
260 	gntdev_free_map(map);  in gntdev_put_map()
267 	struct gntdev_grant_map *map = data;  in find_grant_ptes()  local
268 	unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;  in find_grant_ptes()
269 	int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte |  in find_grant_ptes()
273 	BUG_ON(pgnr >= map->count);  in find_grant_ptes()
276 	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
277 			  map->grants[pgnr].ref,  in find_grant_ptes()
278 			  map->grants[pgnr].domid);  in find_grant_ptes()
279 	gnttab_set_unmap_op(&map->unmap_ops[pgnr], pte_maddr, flags,  in find_grant_ptes()
284 int gntdev_map_grant_pages(struct gntdev_grant_map *map)  in gntdev_map_grant_pages()  argument
290 		if (map->map_ops[0].handle != INVALID_GRANT_HANDLE)  in gntdev_map_grant_pages()
292 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
294 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
295 			gnttab_set_map_op(&map->map_ops[i], addr, map->flags,  in gntdev_map_grant_pages()
296 				map->grants[i].ref,  in gntdev_map_grant_pages()
297 				map->grants[i].domid);  in gntdev_map_grant_pages()
298 			gnttab_set_unmap_op(&map->unmap_ops[i], addr,  in gntdev_map_grant_pages()
299 				map->flags, INVALID_GRANT_HANDLE);  in gntdev_map_grant_pages()
313 		unsigned int flags = (map->flags & ~GNTMAP_device_map) |  in gntdev_map_grant_pages()
316 		for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
318 				pfn_to_kaddr(page_to_pfn(map->pages[i]));  in gntdev_map_grant_pages()
319 			BUG_ON(PageHighMem(map->pages[i]));  in gntdev_map_grant_pages()
321 			gnttab_set_map_op(&map->kmap_ops[i], address, flags,  in gntdev_map_grant_pages()
322 				map->grants[i].ref,  in gntdev_map_grant_pages()
323 				map->grants[i].domid);  in gntdev_map_grant_pages()
324 			gnttab_set_unmap_op(&map->kunmap_ops[i], address,  in gntdev_map_grant_pages()
329 	pr_debug("map %d+%d\n", map->index, map->count);  in gntdev_map_grant_pages()
330 	err = gnttab_map_refs(map->map_ops, map->kmap_ops, map->pages,  in gntdev_map_grant_pages()
331 			map->count);  in gntdev_map_grant_pages()
333 	for (i = 0; i < map->count; i++) {  in gntdev_map_grant_pages()
334 		if (map->map_ops[i].status == GNTST_okay)  in gntdev_map_grant_pages()
335 			map->unmap_ops[i].handle = map->map_ops[i].handle;  in gntdev_map_grant_pages()
339 		if (map->flags & GNTMAP_device_map)  in gntdev_map_grant_pages()
340 			map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr;  in gntdev_map_grant_pages()
343 			if (map->kmap_ops[i].status == GNTST_okay)  in gntdev_map_grant_pages()
344 				map->kunmap_ops[i].handle = map->kmap_ops[i].handle;  in gntdev_map_grant_pages()
352 static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in __unmap_grant_pages()  argument
358 	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {  in __unmap_grant_pages()
359 		int pgno = (map->notify.addr >> PAGE_SHIFT);  in __unmap_grant_pages()
362 			uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));  in __unmap_grant_pages()
363 			tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;  in __unmap_grant_pages()
364 			map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;  in __unmap_grant_pages()
368 	unmap_data.unmap_ops = map->unmap_ops + offset;  in __unmap_grant_pages()
369 	unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;  in __unmap_grant_pages()
370 	unmap_data.pages = map->pages + offset;  in __unmap_grant_pages()
378 		if (map->unmap_ops[offset+i].status)  in __unmap_grant_pages()
381 			map->unmap_ops[offset+i].handle,  in __unmap_grant_pages()
382 			map->unmap_ops[offset+i].status);  in __unmap_grant_pages()
383 		map->unmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;  in __unmap_grant_pages()
385 			if (map->kunmap_ops[offset+i].status)  in __unmap_grant_pages()
388 				 map->kunmap_ops[offset+i].handle,  in __unmap_grant_pages()
389 				 map->kunmap_ops[offset+i].status);  in __unmap_grant_pages()
390 			map->kunmap_ops[offset+i].handle = INVALID_GRANT_HANDLE;  in __unmap_grant_pages()
396 static int unmap_grant_pages(struct gntdev_grant_map *map, int offset,  in unmap_grant_pages()  argument
401 	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);  in unmap_grant_pages()
408 		       map->unmap_ops[offset].handle == INVALID_GRANT_HANDLE) {  in unmap_grant_pages()
414 			if (map->unmap_ops[offset + range].handle ==  in unmap_grant_pages()
419 		err = __unmap_grant_pages(map, offset, range);  in unmap_grant_pages()
431 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_open()  local
434 	refcount_inc(&map->users);  in gntdev_vma_open()
439 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_close()  local
445 		WARN_ON(map->vma != vma);  in gntdev_vma_close()
446 		mmu_interval_notifier_remove(&map->notifier);  in gntdev_vma_close()
447 		map->vma = NULL;  in gntdev_vma_close()
450 	gntdev_put_map(priv, map);  in gntdev_vma_close()
456 	struct gntdev_grant_map *map = vma->vm_private_data;  in gntdev_vma_find_special_page()  local
458 	return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];  in gntdev_vma_find_special_page()
473 	struct gntdev_grant_map *map =  in gntdev_invalidate()  local
487 	if (map->vma->vm_start >= range->end ||  in gntdev_invalidate()
488 	    map->vma->vm_end <= range->start)  in gntdev_invalidate()
491 	mstart = max(range->start, map->vma->vm_start);  in gntdev_invalidate()
492 	mend = min(range->end, map->vma->vm_end);  in gntdev_invalidate()
493 	pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",  in gntdev_invalidate()
494 			map->index, map->count,  in gntdev_invalidate()
495 			map->vma->vm_start, map->vma->vm_end,  in gntdev_invalidate()
497 	err = unmap_grant_pages(map,  in gntdev_invalidate()
498 				(mstart - map->vma->vm_start) >> PAGE_SHIFT,  in gntdev_invalidate()
545 	struct gntdev_grant_map *map;  in gntdev_release()  local
551 		map = list_entry(priv->maps.next,  in gntdev_release()
553 		list_del(&map->next);  in gntdev_release()
554 		gntdev_put_map(NULL /* already removed */, map);  in gntdev_release()
570 	struct gntdev_grant_map *map;  in gntdev_ioctl_map_grant_ref()  local
580 	map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */);  in gntdev_ioctl_map_grant_ref()
581 	if (!map)  in gntdev_ioctl_map_grant_ref()
584 	if (copy_from_user(map->grants, &u->refs,  in gntdev_ioctl_map_grant_ref()
585 			   sizeof(map->grants[0]) * op.count) != 0) {  in gntdev_ioctl_map_grant_ref()
586 		gntdev_put_map(NULL, map);  in gntdev_ioctl_map_grant_ref()
591 	gntdev_add_map(priv, map);  in gntdev_ioctl_map_grant_ref()
592 	op.index = map->index << PAGE_SHIFT;  in gntdev_ioctl_map_grant_ref()
605 	struct gntdev_grant_map *map;  in gntdev_ioctl_unmap_grant_ref()  local
613 	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);  in gntdev_ioctl_unmap_grant_ref()
614 	if (map) {  in gntdev_ioctl_unmap_grant_ref()
615 		list_del(&map->next);  in gntdev_ioctl_unmap_grant_ref()
619 	if (map)  in gntdev_ioctl_unmap_grant_ref()
620 		gntdev_put_map(priv, map);  in gntdev_ioctl_unmap_grant_ref()
629 	struct gntdev_grant_map *map;  in gntdev_ioctl_get_offset_for_vaddr()  local
641 	map = vma->vm_private_data;  in gntdev_ioctl_get_offset_for_vaddr()
642 	if (!map)  in gntdev_ioctl_get_offset_for_vaddr()
645 	op.offset = map->index << PAGE_SHIFT;  in gntdev_ioctl_get_offset_for_vaddr()
646 	op.count = map->count;  in gntdev_ioctl_get_offset_for_vaddr()
660 	struct gntdev_grant_map *map;  in gntdev_ioctl_notify()  local
688 	list_for_each_entry(map, &priv->maps, next) {  in gntdev_ioctl_notify()
689 		uint64_t begin = map->index << PAGE_SHIFT;  in gntdev_ioctl_notify()
690 		uint64_t end = (map->index + map->count) << PAGE_SHIFT;  in gntdev_ioctl_notify()
699 			(map->flags & GNTMAP_readonly)) {  in gntdev_ioctl_notify()
704 	out_flags = map->notify.flags;  in gntdev_ioctl_notify()
705 	out_event = map->notify.event;  in gntdev_ioctl_notify()
707 	map->notify.flags = op.action;  in gntdev_ioctl_notify()
708 	map->notify.addr = op.index - (map->index << PAGE_SHIFT);  in gntdev_ioctl_notify()
709 	map->notify.event = op.event_channel_port;  in gntdev_ioctl_notify()
716 	/* Drop the reference to the event channel we did not save in the map */  in gntdev_ioctl_notify()
973 	struct gntdev_grant_map *map;  in gntdev_mmap()  local
979 	pr_debug("map %d+%d at %lx (pgoff %lx)\n",  in gntdev_mmap()
983 	map = gntdev_find_map_index(priv, index, count);  in gntdev_mmap()
984 	if (!map)  in gntdev_mmap()
986 	if (use_ptemod && map->vma)  in gntdev_mmap()
988 	refcount_inc(&map->users);  in gntdev_mmap()
997 	vma->vm_private_data = map;  in gntdev_mmap()
998 	if (map->flags) {  in gntdev_mmap()
1000 				(map->flags & GNTMAP_readonly))  in gntdev_mmap()
1003 		map->flags = GNTMAP_host_map;  in gntdev_mmap()
1005 			map->flags |= GNTMAP_readonly;  in gntdev_mmap()
1009 		map->vma = vma;  in gntdev_mmap()
1011 			&map->notifier, vma->vm_mm, vma->vm_start,  in gntdev_mmap()
1014 			map->vma = NULL;  in gntdev_mmap()
1031 		mmu_interval_read_begin(&map->notifier);  in gntdev_mmap()
1033 		map->pages_vm_start = vma->vm_start;  in gntdev_mmap()
1036 					  find_grant_ptes, map);  in gntdev_mmap()
1043 	err = gntdev_map_grant_pages(map);  in gntdev_mmap()
1048 		err = vm_map_pages_zero(vma, map->pages, map->count);  in gntdev_mmap()
1063 		unmap_grant_pages(map, 0, map->count);  in gntdev_mmap()
1064 		if (map->vma) {  in gntdev_mmap()
1065 			mmu_interval_notifier_remove(&map->notifier);  in gntdev_mmap()
1066 			map->vma = NULL;  in gntdev_mmap()
1069 	gntdev_put_map(priv, map);  in gntdev_mmap()