137 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_del_ttm()  local
139 	WARN_ON(nvbo->pin_refcnt > 0);  in nouveau_bo_del_ttm()
140 	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);  in nouveau_bo_del_ttm()
149 	kfree(nvbo);  in nouveau_bo_del_ttm()
161 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,  in nouveau_bo_fixup_align()  argument
164 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_fixup_align()
168 		if (nvbo->mode) {  in nouveau_bo_fixup_align()
171 				*size = roundup_64(*size, 64 * nvbo->mode);  in nouveau_bo_fixup_align()
175 				*size = roundup_64(*size, 64 * nvbo->mode);  in nouveau_bo_fixup_align()
179 				*size = roundup_64(*size, 64 * nvbo->mode);  in nouveau_bo_fixup_align()
183 				*size = roundup_64(*size, 32 * nvbo->mode);  in nouveau_bo_fixup_align()
187 		*size = roundup_64(*size, (1 << nvbo->page));  in nouveau_bo_fixup_align()
188 		*align = max((1 <<  nvbo->page), *align);  in nouveau_bo_fixup_align()
199 	struct nouveau_bo *nvbo;  in nouveau_bo_alloc()  local
209 	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);  in nouveau_bo_alloc()
210 	if (!nvbo)  in nouveau_bo_alloc()
212 	INIT_LIST_HEAD(&nvbo->head);  in nouveau_bo_alloc()
213 	INIT_LIST_HEAD(&nvbo->entry);  in nouveau_bo_alloc()
214 	INIT_LIST_HEAD(&nvbo->vma_list);  in nouveau_bo_alloc()
215 	nvbo->bo.bdev = &drm->ttm.bdev;  in nouveau_bo_alloc()
226 			nvbo->force_coherent = true;  in nouveau_bo_alloc()
230 		nvbo->kind = (tile_flags & 0x0000ff00) >> 8;  in nouveau_bo_alloc()
231 		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {  in nouveau_bo_alloc()
232 			kfree(nvbo);  in nouveau_bo_alloc()
236 		nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;  in nouveau_bo_alloc()
239 		nvbo->kind = (tile_flags & 0x00007f00) >> 8;  in nouveau_bo_alloc()
240 		nvbo->comp = (tile_flags & 0x00030000) >> 16;  in nouveau_bo_alloc()
241 		if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {  in nouveau_bo_alloc()
242 			kfree(nvbo);  in nouveau_bo_alloc()
246 		nvbo->zeta = (tile_flags & 0x00000007);  in nouveau_bo_alloc()
248 	nvbo->mode = tile_mode;  in nouveau_bo_alloc()
249 	nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);  in nouveau_bo_alloc()
271 		if (pi < 0 || !nvbo->comp || vmm->page[i].comp)  in nouveau_bo_alloc()
283 	if (nvbo->comp && !vmm->page[pi].comp) {  in nouveau_bo_alloc()
285 			nvbo->kind = mmu->kind[nvbo->kind];  in nouveau_bo_alloc()
286 		nvbo->comp = 0;  in nouveau_bo_alloc()
288 	nvbo->page = vmm->page[pi].shift;  in nouveau_bo_alloc()
290 	nouveau_bo_fixup_align(nvbo, flags, align, size);  in nouveau_bo_alloc()
292 	return nvbo;  in nouveau_bo_alloc()
296 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 flags,  in nouveau_bo_init()  argument
303 	acc_size = ttm_bo_dma_acc_size(nvbo->bo.bdev, size, sizeof(*nvbo));  in nouveau_bo_init()
305 	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;  in nouveau_bo_init()
306 	nouveau_bo_placement_set(nvbo, flags, 0);  in nouveau_bo_init()
308 	ret = ttm_bo_init(nvbo->bo.bdev, &nvbo->bo, size, type,  in nouveau_bo_init()
309 			  &nvbo->placement, align >> PAGE_SHIFT, false,  in nouveau_bo_init()
325 	struct nouveau_bo *nvbo;  in nouveau_bo_new()  local
328 	nvbo = nouveau_bo_alloc(cli, &size, &align, flags, tile_mode,  in nouveau_bo_new()
330 	if (IS_ERR(nvbo))  in nouveau_bo_new()
331 		return PTR_ERR(nvbo);  in nouveau_bo_new()
333 	ret = nouveau_bo_init(nvbo, size, align, flags, sg, robj);  in nouveau_bo_new()
337 	*pnvbo = nvbo;  in nouveau_bo_new()
355 set_placement_range(struct nouveau_bo *nvbo, uint32_t type)  in set_placement_range()  argument
357 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in set_placement_range()
362 	    nvbo->mode && (type & TTM_PL_FLAG_VRAM) &&  in set_placement_range()
363 	    nvbo->bo.mem.num_pages < vram_pages / 4) {  in set_placement_range()
370 		if (nvbo->zeta) {  in set_placement_range()
377 		for (i = 0; i < nvbo->placement.num_placement; ++i) {  in set_placement_range()
378 			nvbo->placements[i].fpfn = fpfn;  in set_placement_range()
379 			nvbo->placements[i].lpfn = lpfn;  in set_placement_range()
381 		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {  in set_placement_range()
382 			nvbo->busy_placements[i].fpfn = fpfn;  in set_placement_range()
383 			nvbo->busy_placements[i].lpfn = lpfn;  in set_placement_range()
389 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)  in nouveau_bo_placement_set()  argument
391 	struct ttm_placement *pl = &nvbo->placement;  in nouveau_bo_placement_set()
392 	uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :  in nouveau_bo_placement_set()
394 			 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);  in nouveau_bo_placement_set()
396 	pl->placement = nvbo->placements;  in nouveau_bo_placement_set()
397 	set_placement_list(nvbo->placements, &pl->num_placement,  in nouveau_bo_placement_set()
400 	pl->busy_placement = nvbo->busy_placements;  in nouveau_bo_placement_set()
401 	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,  in nouveau_bo_placement_set()
404 	set_placement_range(nvbo, type);  in nouveau_bo_placement_set()
408 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)  in nouveau_bo_pin()  argument
410 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_pin()
411 	struct ttm_buffer_object *bo = &nvbo->bo;  in nouveau_bo_pin()
421 		if (!nvbo->contig) {  in nouveau_bo_pin()
422 			nvbo->contig = true;  in nouveau_bo_pin()
428 	if (nvbo->pin_refcnt) {  in nouveau_bo_pin()
435 		nvbo->pin_refcnt++;  in nouveau_bo_pin()
440 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);  in nouveau_bo_pin()
441 		ret = nouveau_bo_validate(nvbo, false, false);  in nouveau_bo_pin()
446 	nvbo->pin_refcnt++;  in nouveau_bo_pin()
447 	nouveau_bo_placement_set(nvbo, memtype, 0);  in nouveau_bo_pin()
453 	nvbo->pin_refcnt--;  in nouveau_bo_pin()
454 	ret = nouveau_bo_validate(nvbo, false, false);  in nouveau_bo_pin()
457 	nvbo->pin_refcnt++;  in nouveau_bo_pin()
472 		nvbo->contig = false;  in nouveau_bo_pin()
478 nouveau_bo_unpin(struct nouveau_bo *nvbo)  in nouveau_bo_unpin()  argument
480 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_unpin()
481 	struct ttm_buffer_object *bo = &nvbo->bo;  in nouveau_bo_unpin()
488 	ref = --nvbo->pin_refcnt;  in nouveau_bo_unpin()
493 	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);  in nouveau_bo_unpin()
495 	ret = nouveau_bo_validate(nvbo, false, false);  in nouveau_bo_unpin()
515 nouveau_bo_map(struct nouveau_bo *nvbo)  in nouveau_bo_map()  argument
519 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);  in nouveau_bo_map()
523 	ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);  in nouveau_bo_map()
525 	ttm_bo_unreserve(&nvbo->bo);  in nouveau_bo_map()
530 nouveau_bo_unmap(struct nouveau_bo *nvbo)  in nouveau_bo_unmap()  argument
532 	if (!nvbo)  in nouveau_bo_unmap()
535 	ttm_bo_kunmap(&nvbo->kmap);  in nouveau_bo_unmap()
539 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)  in nouveau_bo_sync_for_device()  argument
541 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_sync_for_device()
542 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;  in nouveau_bo_sync_for_device()
549 	if (nvbo->force_coherent)  in nouveau_bo_sync_for_device()
559 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)  in nouveau_bo_sync_for_cpu()  argument
561 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_sync_for_cpu()
562 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;  in nouveau_bo_sync_for_cpu()
569 	if (nvbo->force_coherent)  in nouveau_bo_sync_for_cpu()
578 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,  in nouveau_bo_validate()  argument
584 	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);  in nouveau_bo_validate()
588 	nouveau_bo_sync_for_device(nvbo);  in nouveau_bo_validate()
594 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)  in nouveau_bo_wr16()  argument
597 	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);  in nouveau_bo_wr16()
608 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)  in nouveau_bo_rd32()  argument
611 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);  in nouveau_bo_rd32()
622 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)  in nouveau_bo_wr32()  argument
625 	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);  in nouveau_bo_wr32()
722 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_evict_flags()  local
726 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,  in nouveau_bo_evict_flags()
730 		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);  in nouveau_bo_evict_flags()
734 	*pl = nvbo->placement;  in nouveau_bo_evict_flags()
1308 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_move_ntfy()  local
1316 	    mem->mem.page == nvbo->page) {  in nouveau_bo_move_ntfy()
1317 		list_for_each_entry(vma, &nvbo->vma_list, head) {  in nouveau_bo_move_ntfy()
1321 		list_for_each_entry(vma, &nvbo->vma_list, head) {  in nouveau_bo_move_ntfy()
1334 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_vm_bind()  local
1343 					       nvbo->mode, nvbo->zeta);  in nouveau_bo_vm_bind()
1368 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_move()  local
1377 	if (nvbo->pin_refcnt)  in nouveau_bo_move()
1378 		NV_WARN(drm, "Moving pinned object %p!\n", nvbo);  in nouveau_bo_move()
1422 			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);  in nouveau_bo_move()
1431 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_verify_access()  local
1433 	return drm_vma_node_verify_access(&nvbo->bo.base.vma_node,  in nouveau_bo_verify_access()
1541 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_ttm_fault_reserve_notify()  local
1551 		    !nvbo->kind)  in nouveau_ttm_fault_reserve_notify()
1555 			nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);  in nouveau_ttm_fault_reserve_notify()
1557 			ret = nouveau_bo_validate(nvbo, false, false);  in nouveau_ttm_fault_reserve_notify()
1569 	for (i = 0; i < nvbo->placement.num_placement; ++i) {  in nouveau_ttm_fault_reserve_notify()
1570 		nvbo->placements[i].fpfn = 0;  in nouveau_ttm_fault_reserve_notify()
1571 		nvbo->placements[i].lpfn = mappable;  in nouveau_ttm_fault_reserve_notify()
1574 	for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {  in nouveau_ttm_fault_reserve_notify()
1575 		nvbo->busy_placements[i].fpfn = 0;  in nouveau_ttm_fault_reserve_notify()
1576 		nvbo->busy_placements[i].lpfn = mappable;  in nouveau_ttm_fault_reserve_notify()
1579 	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);  in nouveau_ttm_fault_reserve_notify()
1580 	return nouveau_bo_validate(nvbo, false, false);  in nouveau_ttm_fault_reserve_notify()
1685 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)  in nouveau_bo_fence()  argument
1687 	struct dma_resv *resv = nvbo->bo.base.resv;  in nouveau_bo_fence()