730 static void gen6_initialize_pt(struct gen6_hw_ppgtt *ppgtt,  in gen6_initialize_pt()  argument
733 	fill32_px(&ppgtt->base.vm, pt, ppgtt->scratch_pte);  in gen6_initialize_pt()
859 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)  in mark_tlbs_dirty()  argument
861 	ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->vm.i915)->ring_mask;  in mark_tlbs_dirty()
997 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_clear_4lvl()  local
998 	struct i915_pml4 *pml4 = &ppgtt->pml4;  in gen8_ppgtt_clear_4lvl()
1043 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,  in gen8_ppgtt_insert_pte_entries()  argument
1055 	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));  in gen8_ppgtt_insert_pte_entries()
1086 				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));  in gen8_ppgtt_insert_pte_entries()
1104 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_insert_3lvl()  local
1108 	gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,  in gen8_ppgtt_insert_3lvl()
1238 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_insert_4lvl()  local
1240 	struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;  in gen8_ppgtt_insert_4lvl()
1248 		while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],  in gen8_ppgtt_insert_4lvl()
1316 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)  in gen8_ppgtt_notify_vgt()  argument
1318 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_ppgtt_notify_vgt()
1324 		const u64 daddr = px_dma(&ppgtt->pml4);  in gen8_ppgtt_notify_vgt()
1333 			const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);  in gen8_ppgtt_notify_vgt()
1374 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)  in gen8_ppgtt_cleanup_4lvl()  argument
1379 		if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)  in gen8_ppgtt_cleanup_4lvl()
1382 		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);  in gen8_ppgtt_cleanup_4lvl()
1385 	cleanup_px(&ppgtt->vm, &ppgtt->pml4);  in gen8_ppgtt_cleanup_4lvl()
1391 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_cleanup()  local
1394 		gen8_ppgtt_notify_vgt(ppgtt, false);  in gen8_ppgtt_cleanup()
1397 		gen8_ppgtt_cleanup_4lvl(ppgtt);  in gen8_ppgtt_cleanup()
1399 		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);  in gen8_ppgtt_cleanup()
1495 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen8_ppgtt_alloc_4lvl()  local
1496 	struct i915_pml4 *pml4 = &ppgtt->pml4;  in gen8_ppgtt_alloc_4lvl()
1529 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,  in gen8_dump_pdp()  argument
1535 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_dump_pdp()
1545 		if (pdp->page_directory[pdpe] == ppgtt->vm.scratch_pd)  in gen8_dump_pdp()
1553 			if (pd->page_table[pde] == ppgtt->vm.scratch_pt)  in gen8_dump_pdp()
1584 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)  in gen8_dump_ppgtt()  argument
1586 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_dump_ppgtt()
1589 	u64 start = 0, length = ppgtt->vm.total;  in gen8_dump_ppgtt()
1593 		struct i915_pml4 *pml4 = &ppgtt->pml4;  in gen8_dump_ppgtt()
1597 			if (pml4->pdps[pml4e] == ppgtt->vm.scratch_pdp)  in gen8_dump_ppgtt()
1601 			gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);  in gen8_dump_ppgtt()
1604 		gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);  in gen8_dump_ppgtt()
1608 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)  in gen8_preallocate_top_level_pdp()  argument
1610 	struct i915_address_space *vm = &ppgtt->vm;  in gen8_preallocate_top_level_pdp()
1611 	struct i915_page_directory_pointer *pdp = &ppgtt->pdp;  in gen8_preallocate_top_level_pdp()
1613 	u64 start = 0, length = ppgtt->vm.total;  in gen8_preallocate_top_level_pdp()
1649 	struct i915_hw_ppgtt *ppgtt;  in gen8_ppgtt_create()  local
1652 	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);  in gen8_ppgtt_create()
1653 	if (!ppgtt)  in gen8_ppgtt_create()
1656 	kref_init(&ppgtt->ref);  in gen8_ppgtt_create()
1658 	ppgtt->vm.i915 = i915;  in gen8_ppgtt_create()
1659 	ppgtt->vm.dma = &i915->drm.pdev->dev;  in gen8_ppgtt_create()
1661 	ppgtt->vm.total = USES_FULL_48BIT_PPGTT(i915) ?  in gen8_ppgtt_create()
1670 	ppgtt->vm.has_read_only = !intel_vgpu_active(i915);  in gen8_ppgtt_create()
1672 	i915_address_space_init(&ppgtt->vm, i915);  in gen8_ppgtt_create()
1678 		ppgtt->vm.pt_kmap_wc = true;  in gen8_ppgtt_create()
1680 	err = gen8_init_scratch(&ppgtt->vm);  in gen8_ppgtt_create()
1684 	if (use_4lvl(&ppgtt->vm)) {  in gen8_ppgtt_create()
1685 		err = setup_px(&ppgtt->vm, &ppgtt->pml4);  in gen8_ppgtt_create()
1689 		gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4);  in gen8_ppgtt_create()
1691 		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;  in gen8_ppgtt_create()
1692 		ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;  in gen8_ppgtt_create()
1693 		ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;  in gen8_ppgtt_create()
1695 		err = __pdp_init(&ppgtt->vm, &ppgtt->pdp);  in gen8_ppgtt_create()
1700 			err = gen8_preallocate_top_level_pdp(ppgtt);  in gen8_ppgtt_create()
1702 				__pdp_fini(&ppgtt->pdp);  in gen8_ppgtt_create()
1707 		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;  in gen8_ppgtt_create()
1708 		ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;  in gen8_ppgtt_create()
1709 		ppgtt->vm.clear_range = gen8_ppgtt_clear_3lvl;  in gen8_ppgtt_create()
1713 		gen8_ppgtt_notify_vgt(ppgtt, true);  in gen8_ppgtt_create()
1715 	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;  in gen8_ppgtt_create()
1716 	ppgtt->debug_dump = gen8_dump_ppgtt;  in gen8_ppgtt_create()
1718 	ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;  in gen8_ppgtt_create()
1719 	ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;  in gen8_ppgtt_create()
1720 	ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;  in gen8_ppgtt_create()
1721 	ppgtt->vm.vma_ops.clear_pages = clear_pages;  in gen8_ppgtt_create()
1723 	return ppgtt;  in gen8_ppgtt_create()
1726 	gen8_free_scratch(&ppgtt->vm);  in gen8_ppgtt_create()
1728 	kfree(ppgtt);  in gen8_ppgtt_create()
1734 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);  in gen6_dump_ppgtt()  local
1735 	const gen6_pte_t scratch_pte = ppgtt->scratch_pte;  in gen6_dump_ppgtt()
1745 		if (i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {  in gen6_dump_ppgtt()
1749 			u32 pd_entry = readl(ppgtt->pd_addr + pde);  in gen6_dump_ppgtt()
1787 static inline void gen6_write_pde(const struct gen6_hw_ppgtt *ppgtt,  in gen6_write_pde()  argument
1793 		  ppgtt->pd_addr + pde);  in gen6_write_pde()
1855 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_ppgtt_clear_range()  local
1860 	const gen6_pte_t scratch_pte = ppgtt->scratch_pte;  in gen6_ppgtt_clear_range()
1863 		struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];  in gen6_ppgtt_clear_range()
1875 			ppgtt->scan_for_unused_pt = true;  in gen6_ppgtt_clear_range()
1899 	struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);  in gen6_ppgtt_insert_entries()  local
1907 	GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);  in gen6_ppgtt_insert_entries()
1909 	vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);  in gen6_ppgtt_insert_entries()
1925 			vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);  in gen6_ppgtt_insert_entries()
1937 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_alloc_va_range()  local
1943 	gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {  in gen6_alloc_va_range()
1951 			gen6_initialize_pt(ppgtt, pt);  in gen6_alloc_va_range()
1952 			ppgtt->base.pd.page_table[pde] = pt;  in gen6_alloc_va_range()
1954 			if (i915_vma_is_bound(ppgtt->vma,  in gen6_alloc_va_range()
1956 				gen6_write_pde(ppgtt, pde, pt);  in gen6_alloc_va_range()
1967 		mark_tlbs_dirty(&ppgtt->base);  in gen6_alloc_va_range()
1968 		gen6_ggtt_invalidate(ppgtt->base.vm.i915);  in gen6_alloc_va_range()
1978 static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)  in gen6_ppgtt_init_scratch()  argument
1980 	struct i915_address_space * const vm = &ppgtt->base.vm;  in gen6_ppgtt_init_scratch()
1989 	ppgtt->scratch_pte =  in gen6_ppgtt_init_scratch()
1999 	gen6_initialize_pt(ppgtt, vm->scratch_pt);  in gen6_ppgtt_init_scratch()
2000 	gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)  in gen6_ppgtt_init_scratch()
2001 		ppgtt->base.pd.page_table[pde] = vm->scratch_pt;  in gen6_ppgtt_init_scratch()
2012 static void gen6_ppgtt_free_pd(struct gen6_hw_ppgtt *ppgtt)  in gen6_ppgtt_free_pd()  argument
2017 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)  in gen6_ppgtt_free_pd()
2018 		if (pt != ppgtt->base.vm.scratch_pt)  in gen6_ppgtt_free_pd()
2019 			free_pt(&ppgtt->base.vm, pt);  in gen6_ppgtt_free_pd()
2024 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));  in gen6_ppgtt_cleanup()  local
2026 	i915_vma_destroy(ppgtt->vma);  in gen6_ppgtt_cleanup()
2028 	gen6_ppgtt_free_pd(ppgtt);  in gen6_ppgtt_cleanup()
2050 	struct gen6_hw_ppgtt *ppgtt = vma->private;  in pd_vma_bind()  local
2055 	ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);  in pd_vma_bind()
2056 	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;  in pd_vma_bind()
2058 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)  in pd_vma_bind()
2059 		gen6_write_pde(ppgtt, pde, pt);  in pd_vma_bind()
2061 	mark_tlbs_dirty(&ppgtt->base);  in pd_vma_bind()
2062 	gen6_ggtt_invalidate(ppgtt->base.vm.i915);  in pd_vma_bind()
2069 	struct gen6_hw_ppgtt *ppgtt = vma->private;  in pd_vma_unbind()  local
2070 	struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;  in pd_vma_unbind()
2074 	if (!ppgtt->scan_for_unused_pt)  in pd_vma_unbind()
2078 	gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {  in pd_vma_unbind()
2082 		free_pt(&ppgtt->base.vm, pt);  in pd_vma_unbind()
2083 		ppgtt->base.pd.page_table[pde] = scratch_pt;  in pd_vma_unbind()
2086 	ppgtt->scan_for_unused_pt = false;  in pd_vma_unbind()
2096 static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)  in pd_vma_create()  argument
2098 	struct drm_i915_private *i915 = ppgtt->base.vm.i915;  in pd_vma_create()
2113 	vma->private = ppgtt;  in pd_vma_create()
2130 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);  in gen6_ppgtt_pin()  local
2138 	if (ppgtt->pin_count++)  in gen6_ppgtt_pin()
2146 	return i915_vma_pin(ppgtt->vma,  in gen6_ppgtt_pin()
2153 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);  in gen6_ppgtt_unpin()  local
2155 	GEM_BUG_ON(!ppgtt->pin_count);  in gen6_ppgtt_unpin()
2156 	if (--ppgtt->pin_count)  in gen6_ppgtt_unpin()
2159 	i915_vma_unpin(ppgtt->vma);  in gen6_ppgtt_unpin()
2165 	struct gen6_hw_ppgtt *ppgtt;  in gen6_ppgtt_create()  local
2168 	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);  in gen6_ppgtt_create()
2169 	if (!ppgtt)  in gen6_ppgtt_create()
2172 	kref_init(&ppgtt->base.ref);  in gen6_ppgtt_create()
2174 	ppgtt->base.vm.i915 = i915;  in gen6_ppgtt_create()
2175 	ppgtt->base.vm.dma = &i915->drm.pdev->dev;  in gen6_ppgtt_create()
2177 	ppgtt->base.vm.total = I915_PDES * GEN6_PTES * PAGE_SIZE;  in gen6_ppgtt_create()
2179 	i915_address_space_init(&ppgtt->base.vm, i915);  in gen6_ppgtt_create()
2181 	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;  in gen6_ppgtt_create()
2182 	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;  in gen6_ppgtt_create()
2183 	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;  in gen6_ppgtt_create()
2184 	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;  in gen6_ppgtt_create()
2185 	ppgtt->base.debug_dump = gen6_dump_ppgtt;  in gen6_ppgtt_create()
2187 	ppgtt->base.vm.vma_ops.bind_vma    = ppgtt_bind_vma;  in gen6_ppgtt_create()
2188 	ppgtt->base.vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;  in gen6_ppgtt_create()
2189 	ppgtt->base.vm.vma_ops.set_pages   = ppgtt_set_pages;  in gen6_ppgtt_create()
2190 	ppgtt->base.vm.vma_ops.clear_pages = clear_pages;  in gen6_ppgtt_create()
2192 	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;  in gen6_ppgtt_create()
2194 	err = gen6_ppgtt_init_scratch(ppgtt);  in gen6_ppgtt_create()
2198 	ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);  in gen6_ppgtt_create()
2199 	if (IS_ERR(ppgtt->vma)) {  in gen6_ppgtt_create()
2200 		err = PTR_ERR(ppgtt->vma);  in gen6_ppgtt_create()
2204 	return &ppgtt->base;  in gen6_ppgtt_create()
2207 	gen6_ppgtt_free_scratch(&ppgtt->base.vm);  in gen6_ppgtt_create()
2209 	kfree(ppgtt);  in gen6_ppgtt_create()
2285 	struct i915_hw_ppgtt *ppgtt;  in i915_ppgtt_create()  local
2287 	ppgtt = __hw_ppgtt_create(i915);  in i915_ppgtt_create()
2288 	if (IS_ERR(ppgtt))  in i915_ppgtt_create()
2289 		return ppgtt;  in i915_ppgtt_create()
2291 	ppgtt->vm.file = fpriv;  in i915_ppgtt_create()
2293 	trace_i915_ppgtt_create(&ppgtt->vm);  in i915_ppgtt_create()
2295 	return ppgtt;  in i915_ppgtt_create()
2324 	struct i915_hw_ppgtt *ppgtt =  in i915_ppgtt_release()  local
2327 	trace_i915_ppgtt_release(&ppgtt->vm);  in i915_ppgtt_release()
2329 	ppgtt_destroy_vma(&ppgtt->vm);  in i915_ppgtt_release()
2331 	GEM_BUG_ON(!list_empty(&ppgtt->vm.active_list));  in i915_ppgtt_release()
2332 	GEM_BUG_ON(!list_empty(&ppgtt->vm.inactive_list));  in i915_ppgtt_release()
2333 	GEM_BUG_ON(!list_empty(&ppgtt->vm.unbound_list));  in i915_ppgtt_release()
2335 	ppgtt->vm.cleanup(&ppgtt->vm);  in i915_ppgtt_release()
2336 	i915_address_space_fini(&ppgtt->vm);  in i915_ppgtt_release()
2337 	kfree(ppgtt);  in i915_ppgtt_release()
2872 	struct i915_hw_ppgtt *ppgtt;  in i915_gem_init_aliasing_ppgtt()  local
2875 	ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM));  in i915_gem_init_aliasing_ppgtt()
2876 	if (IS_ERR(ppgtt))  in i915_gem_init_aliasing_ppgtt()
2877 		return PTR_ERR(ppgtt);  in i915_gem_init_aliasing_ppgtt()
2879 	if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {  in i915_gem_init_aliasing_ppgtt()
2890 	err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total);  in i915_gem_init_aliasing_ppgtt()
2894 	i915->mm.aliasing_ppgtt = ppgtt;  in i915_gem_init_aliasing_ppgtt()
2905 	i915_ppgtt_put(ppgtt);  in i915_gem_init_aliasing_ppgtt()
2912 	struct i915_hw_ppgtt *ppgtt;  in i915_gem_fini_aliasing_ppgtt()  local
2914 	ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);  in i915_gem_fini_aliasing_ppgtt()
2915 	if (!ppgtt)  in i915_gem_fini_aliasing_ppgtt()
2918 	i915_ppgtt_put(ppgtt);  in i915_gem_fini_aliasing_ppgtt()