23 	dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);  in gen6_write_pde()
74 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,  in gen6_ppgtt_clear_range()  argument
77 	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_ppgtt_clear_range()
79 	const gen6_pte_t scratch_pte = vm->scratch[0]->encode;  in gen6_ppgtt_clear_range()
110 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,  in gen6_ppgtt_insert_entries()  argument
115 	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen6_ppgtt_insert_entries()
120 	const u32 pte_encode = vm->pte_encode(0, cache_level, flags);  in gen6_ppgtt_insert_entries()
166 	gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);  in gen6_flush_pd()
172 static void gen6_alloc_va_range(struct i915_address_space *vm,  in gen6_alloc_va_range()  argument
176 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_alloc_va_range()
193 			fill32_px(pt, vm->scratch[0]->encode);  in gen6_alloc_va_range()
214 		with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)  in gen6_alloc_va_range()
221 	struct i915_address_space * const vm = &ppgtt->base.vm;  in gen6_ppgtt_init_scratch()  local
224 	ret = setup_scratch_page(vm);  in gen6_ppgtt_init_scratch()
228 	vm->scratch[0]->encode =  in gen6_ppgtt_init_scratch()
229 		vm->pte_encode(px_dma(vm->scratch[0]),  in gen6_ppgtt_init_scratch()
232 	vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);  in gen6_ppgtt_init_scratch()
233 	if (IS_ERR(vm->scratch[1])) {  in gen6_ppgtt_init_scratch()
234 		ret = PTR_ERR(vm->scratch[1]);  in gen6_ppgtt_init_scratch()
238 	ret = map_pt_dma(vm, vm->scratch[1]);  in gen6_ppgtt_init_scratch()
242 	fill32_px(vm->scratch[1], vm->scratch[0]->encode);  in gen6_ppgtt_init_scratch()
247 	i915_gem_object_put(vm->scratch[1]);  in gen6_ppgtt_init_scratch()
249 	i915_gem_object_put(vm->scratch[0]);  in gen6_ppgtt_init_scratch()
250 	vm->scratch[0] = NULL;  in gen6_ppgtt_init_scratch()
262 			free_pt(&ppgtt->base.vm, pt);  in gen6_ppgtt_free_pd()
265 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)  in gen6_ppgtt_cleanup()  argument
267 	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_ppgtt_cleanup()
270 	free_scratch(vm);  in gen6_ppgtt_cleanup()
273 		free_pd(&ppgtt->base.vm, ppgtt->base.pd);  in gen6_ppgtt_cleanup()
278 static void pd_vma_bind(struct i915_address_space *vm,  in pd_vma_bind()  argument
284 	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);  in pd_vma_bind()
291 	gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);  in pd_vma_bind()
294 static void pd_vma_unbind(struct i915_address_space *vm,  in pd_vma_unbind()  argument
310 		free_pt(&ppgtt->base.vm, pt);  in pd_vma_unbind()
327 	GEM_BUG_ON(!kref_read(&ppgtt->base.vm.ref));  in gen6_ppgtt_pin()
339 	err = i915_vm_lock_objects(&ppgtt->base.vm, ww);  in gen6_ppgtt_pin()
380 	struct i915_ggtt * const ggtt = ppgtt->base.vm.gt->ggtt;  in gen6_alloc_top_pd()
388 	pd->pt.base = __i915_gem_object_create_internal(ppgtt->base.vm.gt->i915,  in gen6_alloc_top_pd()
397 	pd->pt.base->base.resv = i915_vm_resv_get(&ppgtt->base.vm);  in gen6_alloc_top_pd()
398 	pd->pt.base->shares_resv_from = &ppgtt->base.vm;  in gen6_alloc_top_pd()
400 	ppgtt->vma = i915_vma_instance(pd->pt.base, &ggtt->vm, NULL);  in gen6_alloc_top_pd()
413 	free_pd(&ppgtt->base.vm, pd);  in gen6_alloc_top_pd()
439 	ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));  in gen6_ppgtt_create()
440 	ppgtt->base.vm.top = 1;  in gen6_ppgtt_create()
442 	ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;  in gen6_ppgtt_create()
443 	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;  in gen6_ppgtt_create()
444 	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;  in gen6_ppgtt_create()
445 	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;  in gen6_ppgtt_create()
446 	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;  in gen6_ppgtt_create()
448 	ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;  in gen6_ppgtt_create()
449 	ppgtt->base.vm.alloc_scratch_dma = alloc_pt_dma;  in gen6_ppgtt_create()
450 	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;  in gen6_ppgtt_create()
465 	i915_vm_put(&ppgtt->base.vm);  in gen6_ppgtt_create()