1 /* SPDX-License-Identifier: MIT */
19 #include <linux/io-mapping.h>
49 #define I915_GTT_PAGE_MASK -I915_GTT_PAGE_SIZE
53 #define I915_FENCE_REG_NONE -1
61 #define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT)
64 #define I915_PTE_MASK(pte_len)		(I915_PTES(pte_len) - 1)
66 #define I915_PDE_MASK			(I915_PDES - 1)
68 /* gen6-hsw has bit 11-4 for physical addr bit 39-32 */
81 #define NUM_PTE(pde_shift)     (1 << (pde_shift - PAGE_SHIFT))
89  * Cacheability Control is a 4-bit value. The low three bits are stored in bits
93 					 (((bits) & 0x8) << (11 - 3)))
171 	__px_choose_expr(px, struct i915_page_table *, __x->base, \
172 	__px_choose_expr(px, struct i915_page_directory *, __x->pt.base, \
181 	__px_choose_expr(px, struct i915_page_directory *, &__x->pt, \
183 #define px_used(px) (&px_pt(px)->used)
191 	/* Map an object into an address space with the given cache flags. */
192 	void (*bind_vma)(struct i915_address_space *vm,
201 	void (*unbind_vma)(struct i915_address_space *vm,
217 	 * Every address space belongs to a struct file - except for the global
221 	 * owner. Tracking the owner is important in order to free up per-file
232 	 * Each active user context has its own address space (in full-ppgtt).
233 	 * Since the vm may be shared between multiple contexts, we count how
253 	/* Some systems support read-only mappings for GGTT and/or PPGTT */
261 		(*alloc_pt_dma)(struct i915_address_space *vm, int sz);
268 	void (*allocate_va_range)(struct i915_address_space *vm,
271 	void (*clear_range)(struct i915_address_space *vm,
273 	void (*insert_page)(struct i915_address_space *vm,
278 	void (*insert_entries)(struct i915_address_space *vm,
282 	void (*cleanup)(struct i915_address_space *vm);
293  * collateral associated with any va->pa translations GEN hardware also has a
299 	struct i915_address_space vm;  member
303 	resource_size_t mappable_end;	/* End offset that we can CPU map */
342 	struct i915_address_space vm;  member
347 #define i915_is_ggtt(vm) ((vm)->is_ggtt)  argument
350 i915_vm_is_4lvl(const struct i915_address_space *vm)  in i915_vm_is_4lvl()  argument
352 	return (vm->total - 1) >> 32;  in i915_vm_is_4lvl()
356 i915_vm_has_scratch_64K(struct i915_address_space *vm)  in i915_vm_has_scratch_64K()  argument
358 	return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K);  in i915_vm_has_scratch_64K()
362 i915_vm_has_cache_coloring(struct i915_address_space *vm)  in i915_vm_has_cache_coloring()  argument
364 	return i915_is_ggtt(vm) && vm->mm.color_adjust;  in i915_vm_has_cache_coloring()
368 i915_vm_to_ggtt(struct i915_address_space *vm)  in i915_vm_to_ggtt()  argument
370 	BUILD_BUG_ON(offsetof(struct i915_ggtt, vm));  in i915_vm_to_ggtt()
371 	GEM_BUG_ON(!i915_is_ggtt(vm));  in i915_vm_to_ggtt()
372 	return container_of(vm, struct i915_ggtt, vm);  in i915_vm_to_ggtt()
376 i915_vm_to_ppgtt(struct i915_address_space *vm)  in i915_vm_to_ppgtt()  argument
378 	BUILD_BUG_ON(offsetof(struct i915_ppgtt, vm));  in i915_vm_to_ppgtt()
379 	GEM_BUG_ON(i915_is_ggtt(vm));  in i915_vm_to_ppgtt()
380 	return container_of(vm, struct i915_ppgtt, vm);  in i915_vm_to_ppgtt()
384 i915_vm_get(struct i915_address_space *vm)  in i915_vm_get()  argument
386 	kref_get(&vm->ref);  in i915_vm_get()
387 	return vm;  in i915_vm_get()
392 static inline void i915_vm_put(struct i915_address_space *vm)  in i915_vm_put()  argument
394 	kref_put(&vm->ref, i915_vm_release);  in i915_vm_put()
398 i915_vm_open(struct i915_address_space *vm)  in i915_vm_open()  argument
400 	GEM_BUG_ON(!atomic_read(&vm->open));  in i915_vm_open()
401 	atomic_inc(&vm->open);  in i915_vm_open()
402 	return i915_vm_get(vm);  in i915_vm_open()
406 i915_vm_tryopen(struct i915_address_space *vm)  in i915_vm_tryopen()  argument
408 	if (atomic_add_unless(&vm->open, 1, 0))  in i915_vm_tryopen()
409 		return i915_vm_get(vm);  in i915_vm_tryopen()
414 void __i915_vm_close(struct i915_address_space *vm);
417 i915_vm_close(struct i915_address_space *vm)  in i915_vm_close()  argument
419 	GEM_BUG_ON(!atomic_read(&vm->open));  in i915_vm_close()
420 	__i915_vm_close(vm);  in i915_vm_close()
422 	i915_vm_put(vm);  in i915_vm_close()
425 void i915_address_space_init(struct i915_address_space *vm, int subclass);
426 void i915_address_space_fini(struct i915_address_space *vm);
430 	const u32 mask = NUM_PTE(pde_shift) - 1;  in i915_pte_index()
442 	const u64 mask = ~((1ULL << pde_shift) - 1);  in i915_pte_count()
451 		return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift);  in i915_pte_count()
453 	return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift);  in i915_pte_count()
465 	return pd->entry[n];  in i915_pt_entry()
472 	return pdp->entry[n];  in i915_pd_entry()
478 	struct i915_page_table *pt = ppgtt->pd->entry[n];  in i915_page_dir_dma_addr()
480 	return __px_dma(pt ? px_base(pt) : ppgtt->vm.scratch[ppgtt->vm.top]);  in i915_page_dir_dma_addr()
495 	return ggtt->mappable_end > 0;  in i915_ggtt_has_aperture()
516 int setup_scratch_page(struct i915_address_space *vm);
517 void free_scratch(struct i915_address_space *vm);
519 struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz);
520 struct i915_page_table *alloc_pt(struct i915_address_space *vm);
521 struct i915_page_directory *alloc_pd(struct i915_address_space *vm);
524 int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
526 void free_px(struct i915_address_space *vm,
528 #define free_pt(vm, px) free_px(vm, px, 0)  argument
529 #define free_pd(vm, px) free_px(vm, px_pt(px), 1)  argument
556 void ppgtt_bind_vma(struct i915_address_space *vm,
561 void ppgtt_unbind_vma(struct i915_address_space *vm,
568 int i915_vm_alloc_pt_stash(struct i915_address_space *vm,
571 int i915_vm_pin_pt_stash(struct i915_address_space *vm,
573 void i915_vm_free_pt_stash(struct i915_address_space *vm,
580 	struct scatterlist *sg = vma->pages->sgl;  in sgt_dma()
583 	return (struct sgt_dma){ sg, addr, addr + sg->length };  in sgt_dma()