47 static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,  in ttm_bo_vm_fault_idle()  argument
53 	if (likely(!bo->moving))  in ttm_bo_vm_fault_idle()
59 	if (dma_fence_is_signaled(bo->moving))  in ttm_bo_vm_fault_idle()
71 		ttm_bo_get(bo);  in ttm_bo_vm_fault_idle()
73 		(void) dma_fence_wait(bo->moving, true);  in ttm_bo_vm_fault_idle()
74 		dma_resv_unlock(bo->base.resv);  in ttm_bo_vm_fault_idle()
75 		ttm_bo_put(bo);  in ttm_bo_vm_fault_idle()
82 	err = dma_fence_wait(bo->moving, true);  in ttm_bo_vm_fault_idle()
90 	dma_fence_put(bo->moving);  in ttm_bo_vm_fault_idle()
91 	bo->moving = NULL;  in ttm_bo_vm_fault_idle()
97 static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo,  in ttm_bo_io_mem_pfn()  argument
100 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_io_mem_pfn()
103 		return bdev->driver->io_mem_pfn(bo, page_offset);  in ttm_bo_io_mem_pfn()
105 	return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT)  in ttm_bo_io_mem_pfn()
112 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)  in ttm_bo_vm_fault()  local
114 	struct ttm_bo_device *bdev = bo->bdev;  in ttm_bo_vm_fault()
125 		&bdev->man[bo->mem.mem_type];  in ttm_bo_vm_fault()
134 	if (unlikely(!dma_resv_trylock(bo->base.resv))) {  in ttm_bo_vm_fault()
137 				ttm_bo_get(bo);  in ttm_bo_vm_fault()
139 				(void) ttm_bo_wait_unreserved(bo);  in ttm_bo_vm_fault()
140 				ttm_bo_put(bo);  in ttm_bo_vm_fault()
158 	if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {  in ttm_bo_vm_fault()
164 		struct dma_fence *moving = dma_fence_get(bo->moving);  in ttm_bo_vm_fault()
166 		err = bdev->driver->fault_reserve_notify(bo);  in ttm_bo_vm_fault()
179 		if (bo->moving != moving) {  in ttm_bo_vm_fault()
181 			ttm_bo_move_to_lru_tail(bo, NULL);  in ttm_bo_vm_fault()
191 	ret = ttm_bo_vm_fault_idle(bo, vmf);  in ttm_bo_vm_fault()
207 	err = ttm_mem_io_reserve_vm(bo);  in ttm_bo_vm_fault()
214 		vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node);  in ttm_bo_vm_fault()
216 		drm_vma_node_start(&bo->base.vma_node);  in ttm_bo_vm_fault()
218 	if (unlikely(page_offset >= bo->num_pages)) {  in ttm_bo_vm_fault()
231 	if (bo->mem.bus.is_iomem) {  in ttm_bo_vm_fault()
232 		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,  in ttm_bo_vm_fault()
242 		ttm = bo->ttm;  in ttm_bo_vm_fault()
243 		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,  in ttm_bo_vm_fault()
258 		if (bo->mem.bus.is_iomem) {  in ttm_bo_vm_fault()
261 			pfn = ttm_bo_io_mem_pfn(bo, page_offset);  in ttm_bo_vm_fault()
270 			page->index = drm_vma_node_start(&bo->base.vma_node) +  in ttm_bo_vm_fault()
297 	dma_resv_unlock(bo->base.resv);  in ttm_bo_vm_fault()
303 	struct ttm_buffer_object *bo =  in ttm_bo_vm_open()  local
306 	WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping);  in ttm_bo_vm_open()
308 	ttm_bo_get(bo);  in ttm_bo_vm_open()
313 	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;  in ttm_bo_vm_close()  local
315 	ttm_bo_put(bo);  in ttm_bo_vm_close()
319 static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,  in ttm_bo_vm_access_kmap()  argument
337 		ret = ttm_bo_kmap(bo, page, 1, &map);  in ttm_bo_vm_access_kmap()
362 	struct ttm_buffer_object *bo = vma->vm_private_data;  in ttm_bo_vm_access()  local
365 	if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages)  in ttm_bo_vm_access()
368 	ret = ttm_bo_reserve(bo, true, false, NULL);  in ttm_bo_vm_access()
372 	switch (bo->mem.mem_type) {  in ttm_bo_vm_access()
374 		if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {  in ttm_bo_vm_access()
375 			ret = ttm_tt_swapin(bo->ttm);  in ttm_bo_vm_access()
381 		ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);  in ttm_bo_vm_access()
384 		if (bo->bdev->driver->access_memory)  in ttm_bo_vm_access()
385 			ret = bo->bdev->driver->access_memory(  in ttm_bo_vm_access()
386 				bo, offset, buf, len, write);  in ttm_bo_vm_access()
391 	ttm_bo_unreserve(bo);  in ttm_bo_vm_access()
408 	struct ttm_buffer_object *bo = NULL;  in ttm_bo_vm_lookup()  local
414 		bo = container_of(node, struct ttm_buffer_object,  in ttm_bo_vm_lookup()
416 		bo = ttm_bo_get_unless_zero(bo);  in ttm_bo_vm_lookup()
421 	if (!bo)  in ttm_bo_vm_lookup()
424 	return bo;  in ttm_bo_vm_lookup()
431 	struct ttm_buffer_object *bo;  in ttm_bo_mmap()  local
437 	bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma));  in ttm_bo_mmap()
438 	if (unlikely(!bo))  in ttm_bo_mmap()
441 	driver = bo->bdev->driver;  in ttm_bo_mmap()
446 	ret = driver->verify_access(bo, filp);  in ttm_bo_mmap()
457 	vma->vm_private_data = bo;  in ttm_bo_mmap()
470 	ttm_bo_put(bo);  in ttm_bo_mmap()
475 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)  in ttm_fbdev_mmap()  argument
480 	ttm_bo_get(bo);  in ttm_fbdev_mmap()
483 	vma->vm_private_data = bo;  in ttm_fbdev_mmap()