55 	struct drm_i915_private *i915 = ppgtt->vm.i915;  in gen8_ppgtt_notify_vgt()
56 	struct intel_uncore *uncore = ppgtt->vm.gt->uncore;  in gen8_ppgtt_notify_vgt()
67 	if (i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_notify_vgt()
146 gen8_pd_top_count(const struct i915_address_space *vm)  in gen8_pd_top_count()  argument
148 	unsigned int shift = __gen8_pte_shift(vm->top);  in gen8_pd_top_count()
149 	return (vm->total + (1ull << shift) - 1) >> shift;  in gen8_pd_top_count()
153 gen8_pdp_for_page_index(struct i915_address_space * const vm, const u64 idx)  in gen8_pdp_for_page_index()  argument
155 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_pdp_for_page_index()
157 	if (vm->top == 2)  in gen8_pdp_for_page_index()
160 		return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top));  in gen8_pdp_for_page_index()
164 gen8_pdp_for_page_address(struct i915_address_space * const vm, const u64 addr)  in gen8_pdp_for_page_address()  argument
166 	return gen8_pdp_for_page_index(vm, addr >> GEN8_PTE_SHIFT);  in gen8_pdp_for_page_address()
169 static void __gen8_ppgtt_cleanup(struct i915_address_space *vm,  in __gen8_ppgtt_cleanup()  argument
180 			__gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1);  in __gen8_ppgtt_cleanup()
184 	free_px(vm, &pd->pt, lvl);  in __gen8_ppgtt_cleanup()
187 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)  in gen8_ppgtt_cleanup()  argument
189 	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_cleanup()
191 	if (intel_vgpu_active(vm->i915))  in gen8_ppgtt_cleanup()
194 	__gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top);  in gen8_ppgtt_cleanup()
195 	free_scratch(vm);  in gen8_ppgtt_cleanup()
198 static u64 __gen8_ppgtt_clear(struct i915_address_space * const vm,  in __gen8_ppgtt_clear()  argument
202 	const struct drm_i915_gem_object * const scratch = vm->scratch[lvl];  in __gen8_ppgtt_clear()
205 	GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);  in __gen8_ppgtt_clear()
209 	    __func__, vm, lvl + 1, start, end,  in __gen8_ppgtt_clear()
219 			    __func__, vm, lvl + 1, idx, start, end);  in __gen8_ppgtt_clear()
221 			__gen8_ppgtt_cleanup(vm, as_pd(pt), I915_PDES, lvl);  in __gen8_ppgtt_clear()
227 			start = __gen8_ppgtt_clear(vm, as_pd(pt),  in __gen8_ppgtt_clear()
235 			    __func__, vm, lvl, start, end,  in __gen8_ppgtt_clear()
242 				 vm->scratch[0]->encode,  in __gen8_ppgtt_clear()
251 			free_px(vm, pt, lvl);  in __gen8_ppgtt_clear()
257 static void gen8_ppgtt_clear(struct i915_address_space *vm,  in gen8_ppgtt_clear()  argument
262 	GEM_BUG_ON(range_overflows(start, length, vm->total));  in gen8_ppgtt_clear()
268 	__gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_clear()
269 			   start, start + length, vm->top);  in gen8_ppgtt_clear()
272 static void __gen8_ppgtt_alloc(struct i915_address_space * const vm,  in __gen8_ppgtt_alloc()  argument
279 	GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT);  in __gen8_ppgtt_alloc()
283 	    __func__, vm, lvl + 1, *start, end,  in __gen8_ppgtt_alloc()
296 			    __func__, vm, lvl + 1, idx);  in __gen8_ppgtt_alloc()
304 			    intel_vgpu_active(vm->i915))  in __gen8_ppgtt_alloc()
305 				fill_px(pt, vm->scratch[lvl]->encode);  in __gen8_ppgtt_alloc()
321 			__gen8_ppgtt_alloc(vm, stash,  in __gen8_ppgtt_alloc()
331 			    __func__, vm, lvl, *start, end,  in __gen8_ppgtt_alloc()
344 static void gen8_ppgtt_alloc(struct i915_address_space *vm,  in gen8_ppgtt_alloc()  argument
350 	GEM_BUG_ON(range_overflows(start, length, vm->total));  in gen8_ppgtt_alloc()
356 	__gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd,  in gen8_ppgtt_alloc()
357 			   &start, start + length, vm->top);  in gen8_ppgtt_alloc()
419 	GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm));  in gen8_ppgtt_insert_huge()
423 			gen8_pdp_for_page_address(vma->vm, start);  in gen8_ppgtt_insert_huge()
496 		     (i915_vm_has_scratch_64K(vma->vm) &&  in gen8_ppgtt_insert_huge()
514 			if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {  in gen8_ppgtt_insert_huge()
517 				encode = vma->vm->scratch[0]->encode;  in gen8_ppgtt_insert_huge()
531 static void gen8_ppgtt_insert(struct i915_address_space *vm,  in gen8_ppgtt_insert()  argument
536 	struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_insert()
546 				gen8_pdp_for_page_index(vm, idx);  in gen8_ppgtt_insert()
556 static int gen8_init_scratch(struct i915_address_space *vm)  in gen8_init_scratch()  argument
563 	 * we can reuse it for all vm, keeping contexts and processes separate.  in gen8_init_scratch()
565 	if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) {  in gen8_init_scratch()
566 		struct i915_address_space *clone = vm->gt->vm;  in gen8_init_scratch()
570 		vm->scratch_order = clone->scratch_order;  in gen8_init_scratch()
571 		for (i = 0; i <= vm->top; i++)  in gen8_init_scratch()
572 			vm->scratch[i] = i915_gem_object_get(clone->scratch[i]);  in gen8_init_scratch()
577 	ret = setup_scratch_page(vm);  in gen8_init_scratch()
581 	vm->scratch[0]->encode =  in gen8_init_scratch()
582 		gen8_pte_encode(px_dma(vm->scratch[0]),  in gen8_init_scratch()
583 				I915_CACHE_LLC, vm->has_read_only);  in gen8_init_scratch()
585 	for (i = 1; i <= vm->top; i++) {  in gen8_init_scratch()
588 		obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_init_scratch()
592 		ret = pin_pt_dma(vm, obj);  in gen8_init_scratch()
598 		fill_px(obj, vm->scratch[i - 1]->encode);  in gen8_init_scratch()
601 		vm->scratch[i] = obj;  in gen8_init_scratch()
608 		i915_gem_object_put(vm->scratch[i]);  in gen8_init_scratch()
614 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_preallocate_top_level_pdp()  local
618 	GEM_BUG_ON(vm->top != 2);  in gen8_preallocate_top_level_pdp()
619 	GEM_BUG_ON(gen8_pd_top_count(vm) != GEN8_3LVL_PDPES);  in gen8_preallocate_top_level_pdp()
625 		pde = alloc_pd(vm);  in gen8_preallocate_top_level_pdp()
629 		err = pin_pt_dma(vm, pde->pt.base);  in gen8_preallocate_top_level_pdp()
632 			free_pd(vm, pde);  in gen8_preallocate_top_level_pdp()
636 		fill_px(pde, vm->scratch[1]->encode);  in gen8_preallocate_top_level_pdp()
646 gen8_alloc_top_pd(struct i915_address_space *vm)  in gen8_alloc_top_pd()  argument
648 	const unsigned int count = gen8_pd_top_count(vm);  in gen8_alloc_top_pd()
658 	pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen8_alloc_top_pd()
665 	err = pin_pt_dma(vm, pd->pt.base);  in gen8_alloc_top_pd()
669 	fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count);  in gen8_alloc_top_pd()
674 	free_pd(vm, pd);  in gen8_alloc_top_pd()
695 	ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2;  in gen8_ppgtt_create()
696 	ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t));  in gen8_ppgtt_create()
706 	ppgtt->vm.has_read_only = !IS_GEN_RANGE(gt->i915, 11, 12);  in gen8_ppgtt_create()
708 	ppgtt->vm.alloc_pt_dma = alloc_pt_dma;  in gen8_ppgtt_create()
710 	err = gen8_init_scratch(&ppgtt->vm);  in gen8_ppgtt_create()
714 	ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm);  in gen8_ppgtt_create()
720 	if (!i915_vm_is_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_create()
726 	ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;  in gen8_ppgtt_create()
727 	ppgtt->vm.insert_entries = gen8_ppgtt_insert;  in gen8_ppgtt_create()
728 	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;  in gen8_ppgtt_create()
729 	ppgtt->vm.clear_range = gen8_ppgtt_clear;  in gen8_ppgtt_create()
731 	ppgtt->vm.pte_encode = gen8_pte_encode;  in gen8_ppgtt_create()
736 	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;  in gen8_ppgtt_create()
741 	__gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd,  in gen8_ppgtt_create()
742 			     gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top);  in gen8_ppgtt_create()
744 	free_scratch(&ppgtt->vm);  in gen8_ppgtt_create()