60 	struct drm_i915_private *i915 = ppgtt->vm.i915;  in gen8_ppgtt_notify_vgt()
61 	struct intel_uncore *uncore = ppgtt->vm.gt->uncore;  in gen8_ppgtt_notify_vgt()
72 	if (i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_notify_vgt()
150 static unsigned int gen8_pd_top_count(const struct i915_address_space *vm)  in gen8_pd_top_count()  argument
152 	unsigned int shift = __gen8_pte_shift(vm->top);  in gen8_pd_top_count()
154 	return (vm->total + (1ull << shift) - 1) >> shift;  in gen8_pd_top_count()
158 gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx)  in gen8_pdp_for_page_index()  argument
160 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_pdp_for_page_index()
162 	if (vm->top == 2)  in gen8_pdp_for_page_index()
165 		return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top));  in gen8_pdp_for_page_index()
169 gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr)  in gen8_pdp_for_page_address()  argument
171 	return gen8_pdp_for_page_index(vm, addr >> GEN8_PTE_SHIFT);  in gen8_pdp_for_page_address()
174 static void __gen8_ppgtt_cleanup(struct i915_address_space *vm,  in __gen8_ppgtt_cleanup()  argument
185 			__gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1);  in __gen8_ppgtt_cleanup()
189 	free_px(vm, &pd->pt, lvl);  in __gen8_ppgtt_cleanup()
192 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)  in gen8_ppgtt_cleanup()  argument
194 	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_cleanup()
196 	if (intel_vgpu_active(vm->i915))  in gen8_ppgtt_cleanup()
199 	__gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top);  in gen8_ppgtt_cleanup()
200 	free_scratch(vm);  in gen8_ppgtt_cleanup()
203 static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm,  in __gen8_ppgtt_clear()  argument
207 	const struct drm_i915_gem_object * const scratch = vm->scratch[lvl];  in __gen8_ppgtt_clear()
210 	GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);  in __gen8_ppgtt_clear()
214 	    __func__, vm, lvl + 1, start, end,  in __gen8_ppgtt_clear()
224 			    __func__, vm, lvl + 1, idx, start, end);  in __gen8_ppgtt_clear()
226 			__gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl);  in __gen8_ppgtt_clear()
232 			start = __gen8_ppgtt_clear(vm, as_pd(pt),  in __gen8_ppgtt_clear()
240 			    __func__, vm, lvl, start, end,  in __gen8_ppgtt_clear()
247 				 vm->scratch[0]->encode,  in __gen8_ppgtt_clear()
255 			free_px(vm, pt, lvl);  in __gen8_ppgtt_clear()
261 static void gen8_ppgtt_clear(struct i915_address_space *vm,  in gen8_ppgtt_clear()  argument
266 	GEM_BUG_ON(range_overflows(start, length, vm->total));  in gen8_ppgtt_clear()
272 	__gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_clear()
273 			   start, start + length, vm->top);  in gen8_ppgtt_clear()
276 static void __gen8_ppgtt_alloc(struct i915_address_space * const vm,  in __gen8_ppgtt_alloc()  argument
283 	GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);  in __gen8_ppgtt_alloc()
287 	    __func__, vm, lvl + 1, *start, end,  in __gen8_ppgtt_alloc()
300 			    __func__, vm, lvl + 1, idx);  in __gen8_ppgtt_alloc()
306 			fill_px(pt, vm->scratch[lvl]->encode);  in __gen8_ppgtt_alloc()
322 			__gen8_ppgtt_alloc(vm, stash,  in __gen8_ppgtt_alloc()
332 			    __func__, vm, lvl, *start, end,  in __gen8_ppgtt_alloc()
345 static void gen8_ppgtt_alloc(struct i915_address_space *vm,  in gen8_ppgtt_alloc()  argument
351 	GEM_BUG_ON(range_overflows(start, length, vm->total));  in gen8_ppgtt_alloc()
357 	__gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_alloc()
358 			   &start, start + length, vm->top);  in gen8_ppgtt_alloc()
361 static void __gen8_ppgtt_foreach(struct i915_address_space *vm,  in __gen8_ppgtt_foreach()  argument
364 				 void (*fn)(struct i915_address_space *vm,  in __gen8_ppgtt_foreach()  argument
381 			__gen8_ppgtt_foreach(vm, as_pd(pt), start, end, lvl,  in __gen8_ppgtt_foreach()
384 			fn(vm, pt, data);  in __gen8_ppgtt_foreach()
394 static void gen8_ppgtt_foreach(struct i915_address_space *vm,  in gen8_ppgtt_foreach()  argument
396 			       void (*fn)(struct i915_address_space *vm,  in gen8_ppgtt_foreach()  argument
404 	__gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_foreach()
405 			     &start, start + length, vm->top,  in gen8_ppgtt_foreach()
466 	GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm));  in gen8_ppgtt_insert_huge()
470 			gen8_pdp_for_page_address(vma->vm, start);  in gen8_ppgtt_insert_huge()
545 		     (i915_vm_has_scratch_64K(vma->vm) &&  in gen8_ppgtt_insert_huge()
562 			if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {  in gen8_ppgtt_insert_huge()
565 				encode = vma->vm->scratch[0]->encode;  in gen8_ppgtt_insert_huge()
578 static void gen8_ppgtt_insert(struct i915_address_space *vm,  in gen8_ppgtt_insert()  argument
583 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_insert()
593 				gen8_pdp_for_page_index(vm, idx);  in gen8_ppgtt_insert()
603 static void gen8_ppgtt_insert_entry(struct i915_address_space *vm,  in gen8_ppgtt_insert_entry()  argument
611 		gen8_pdp_for_page_index(vm, idx);  in gen8_ppgtt_insert_entry()
621 static int gen8_init_scratch(struct i915_address_space *vm)  in gen8_init_scratch()  argument
629 	 * we can reuse it for all vm, keeping contexts and processes separate.  in gen8_init_scratch()
631 	if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) {  in gen8_init_scratch()
632 		struct i915_address_space *clone = vm->gt->vm;  in gen8_init_scratch()
636 		vm->scratch_order = clone->scratch_order;  in gen8_init_scratch()
637 		for (i = 0; i <= vm->top; i++)  in gen8_init_scratch()
638 			vm->scratch[i] = i915_gem_object_get(clone->scratch[i]);  in gen8_init_scratch()
643 	ret = setup_scratch_page(vm);  in gen8_init_scratch()
647 	pte_flags = vm->has_read_only;  in gen8_init_scratch()
648 	if (i915_gem_object_is_lmem(vm->scratch[0]))  in gen8_init_scratch()
651 	vm->scratch[0]->encode =  in gen8_init_scratch()
652 		gen8_pte_encode(px_dma(vm->scratch[0]),  in gen8_init_scratch()
655 	for (i = 1; i <= vm->top; i++) {  in gen8_init_scratch()
658 		obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_init_scratch()
662 		ret = map_pt_dma(vm, obj);  in gen8_init_scratch()
668 		fill_px(obj, vm->scratch[i - 1]->encode);  in gen8_init_scratch()
671 		vm->scratch[i] = obj;  in gen8_init_scratch()
678 		i915_gem_object_put(vm->scratch[i]);  in gen8_init_scratch()
684 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_preallocate_top_level_pdp()  local
688 	GEM_BUG_ON(vm->top != 2);  in gen8_preallocate_top_level_pdp()
689 	GEM_BUG_ON(gen8_pd_top_count(vm) != GEN8_3LVL_PDPES);  in gen8_preallocate_top_level_pdp()
695 		pde = alloc_pd(vm);  in gen8_preallocate_top_level_pdp()
699 		err = map_pt_dma(vm, pde->pt.base);  in gen8_preallocate_top_level_pdp()
701 			free_pd(vm, pde);  in gen8_preallocate_top_level_pdp()
705 		fill_px(pde, vm->scratch[1]->encode);  in gen8_preallocate_top_level_pdp()
715 gen8_alloc_top_pd(struct i915_address_space *vm)  in gen8_alloc_top_pd()  argument
717 	const unsigned int count = gen8_pd_top_count(vm);  in gen8_alloc_top_pd()
727 	pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_alloc_top_pd()
734 	err = map_pt_dma(vm, pd->pt.base);  in gen8_alloc_top_pd()
738 	fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count);  in gen8_alloc_top_pd()
743 	free_pd(vm, pd);  in gen8_alloc_top_pd()
764 	ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;  in gen8_ppgtt_create()
765 	ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t));  in gen8_ppgtt_create()
775 	ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12);  in gen8_ppgtt_create()
778 		ppgtt->vm.alloc_pt_dma = alloc_pt_lmem;  in gen8_ppgtt_create()
780 		ppgtt->vm.alloc_pt_dma = alloc_pt_dma;  in gen8_ppgtt_create()
782 	err = gen8_init_scratch(&ppgtt->vm);  in gen8_ppgtt_create()
786 	ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm);  in gen8_ppgtt_create()
792 	if (!i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_create()
798 	ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;  in gen8_ppgtt_create()
799 	ppgtt->vm.insert_entries = gen8_ppgtt_insert;  in gen8_ppgtt_create()
800 	ppgtt->vm.insert_page = gen8_ppgtt_insert_entry;  in gen8_ppgtt_create()
801 	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;  in gen8_ppgtt_create()
802 	ppgtt->vm.clear_range = gen8_ppgtt_clear;  in gen8_ppgtt_create()
803 	ppgtt->vm.foreach = gen8_ppgtt_foreach;  in gen8_ppgtt_create()
805 	ppgtt->vm.pte_encode = gen8_pte_encode;  in gen8_ppgtt_create()
810 	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;  in gen8_ppgtt_create()
815 	__gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd,  in gen8_ppgtt_create()
816 			     gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top);  in gen8_ppgtt_create()
818 	free_scratch(&ppgtt->vm);  in gen8_ppgtt_create()