Lines Matching +full:top +full:- +full:level
1 // SPDX-License-Identifier: MIT
19 const enum i915_cache_level level) in gen8_pde_encode() argument
23 if (level != I915_CACHE_NONE) in gen8_pde_encode()
32 enum i915_cache_level level, in gen8_pte_encode() argument
43 switch (level) { in gen8_pte_encode()
60 struct drm_i915_private *i915 = ppgtt->vm.i915; in gen8_ppgtt_notify_vgt()
61 struct intel_uncore *uncore = ppgtt->vm.gt->uncore; in gen8_ppgtt_notify_vgt()
66 atomic_inc(px_used(ppgtt->pd)); /* never remove */ in gen8_ppgtt_notify_vgt()
68 atomic_dec(px_used(ppgtt->pd)); in gen8_ppgtt_notify_vgt()
70 mutex_lock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
72 if (i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_notify_vgt()
73 const u64 daddr = px_dma(ppgtt->pd); in gen8_ppgtt_notify_vgt()
103 mutex_unlock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
107 #define GEN8_PAGE_SIZE (SZ_4K) /* page and page-directory sizes are the same */
128 return GEN8_PDES - *idx; in gen8_pd_range()
130 return i915_pde_index(end, shift) - *idx; in gen8_pd_range()
145 return GEN8_PDES - (start & (GEN8_PDES - 1)); in gen8_pt_count()
147 return end - start; in gen8_pt_count()
152 unsigned int shift = __gen8_pte_shift(vm->top); in gen8_pd_top_count()
154 return (vm->total + (1ull << shift) - 1) >> shift; in gen8_pd_top_count()
162 if (vm->top == 2) in gen8_pdp_for_page_index()
163 return ppgtt->pd; in gen8_pdp_for_page_index()
165 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top)); in gen8_pdp_for_page_index()
179 void **pde = pd->entry; in __gen8_ppgtt_cleanup()
185 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1); in __gen8_ppgtt_cleanup()
186 } while (pde++, --count); in __gen8_ppgtt_cleanup()
189 free_px(vm, &pd->pt, lvl); in __gen8_ppgtt_cleanup()
196 if (intel_vgpu_active(vm->i915)) in gen8_ppgtt_cleanup()
199 __gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top); in gen8_ppgtt_cleanup()
207 const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; in __gen8_ppgtt_clear()
210 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_clear()
212 len = gen8_pd_range(start, end, lvl--, &idx); in __gen8_ppgtt_clear()
219 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_clear()
221 if (atomic_fetch_inc(&pt->used) >> gen8_pd_shift(1) && in __gen8_ppgtt_clear()
242 atomic_read(&pt->used)); in __gen8_ppgtt_clear()
243 GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); in __gen8_ppgtt_clear()
247 vm->scratch[0]->encode, in __gen8_ppgtt_clear()
250 atomic_sub(count, &pt->used); in __gen8_ppgtt_clear()
256 } while (idx++, --len); in __gen8_ppgtt_clear()
266 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_clear()
272 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_clear()
273 start, start + length, vm->top); in gen8_ppgtt_clear()
283 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_alloc()
285 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_alloc()
289 GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1)); in __gen8_ppgtt_alloc()
291 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
294 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
297 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
302 pt = stash->pt[!!lvl]; in __gen8_ppgtt_alloc()
303 __i915_gem_object_pin_pages(pt->base); in __gen8_ppgtt_alloc()
304 i915_gem_object_make_unshrinkable(pt->base); in __gen8_ppgtt_alloc()
306 fill_px(pt, vm->scratch[lvl]->encode); in __gen8_ppgtt_alloc()
308 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
309 if (likely(!pd->entry[idx])) { in __gen8_ppgtt_alloc()
310 stash->pt[!!lvl] = pt->stash; in __gen8_ppgtt_alloc()
311 atomic_set(&pt->used, 0); in __gen8_ppgtt_alloc()
314 pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
319 atomic_inc(&pt->used); in __gen8_ppgtt_alloc()
320 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
325 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
326 atomic_dec(&pt->used); in __gen8_ppgtt_alloc()
327 GEM_BUG_ON(!atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
334 atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
336 atomic_add(count, &pt->used); in __gen8_ppgtt_alloc()
338 GEM_BUG_ON(atomic_read(&pt->used) > NALLOC * I915_PDES); in __gen8_ppgtt_alloc()
341 } while (idx++, --len); in __gen8_ppgtt_alloc()
342 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
351 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_alloc()
357 __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_alloc()
358 &start, start + length, vm->top); in gen8_ppgtt_alloc()
371 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_foreach()
373 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
375 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_foreach()
377 atomic_inc(&pt->used); in __gen8_ppgtt_foreach()
378 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
388 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
389 atomic_dec(&pt->used); in __gen8_ppgtt_foreach()
390 } while (idx++, --len); in __gen8_ppgtt_foreach()
391 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
404 __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_foreach()
405 &start, start + length, vm->top, in gen8_ppgtt_foreach()
424 GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); in gen8_ppgtt_insert_pte()
425 vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; in gen8_ppgtt_insert_pte()
427 iter->dma += I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert_pte()
428 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_pte()
429 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_pte()
430 if (!iter->sg || sg_dma_len(iter->sg) == 0) { in gen8_ppgtt_insert_pte()
435 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_pte()
436 iter->max = iter->dma + sg_dma_len(iter->sg); in gen8_ppgtt_insert_pte()
445 pd = pdp->entry[gen8_pd_index(idx, 2)]; in gen8_ppgtt_insert_pte()
463 unsigned int rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
464 u64 start = vma->node.start; in gen8_ppgtt_insert_huge()
466 GEM_BUG_ON(!i915_vm_is_4lvl(vma->vm)); in gen8_ppgtt_insert_huge()
470 gen8_pdp_for_page_address(vma->vm, start); in gen8_ppgtt_insert_huge()
474 unsigned int maybe_64K = -1; in gen8_ppgtt_insert_huge()
479 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M && in gen8_ppgtt_insert_huge()
480 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && in gen8_ppgtt_insert_huge()
496 vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K && in gen8_ppgtt_insert_huge()
497 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
499 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) in gen8_ppgtt_insert_huge()
506 GEM_BUG_ON(sg_dma_len(iter->sg) < page_size); in gen8_ppgtt_insert_huge()
507 vaddr[index++] = encode | iter->dma; in gen8_ppgtt_insert_huge()
510 iter->dma += page_size; in gen8_ppgtt_insert_huge()
511 rem -= page_size; in gen8_ppgtt_insert_huge()
512 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_huge()
513 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_huge()
514 if (!iter->sg) in gen8_ppgtt_insert_huge()
517 rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
521 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_huge()
522 iter->max = iter->dma + rem; in gen8_ppgtt_insert_huge()
524 if (maybe_64K != -1 && index < I915_PDES && in gen8_ppgtt_insert_huge()
525 !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
527 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))) in gen8_ppgtt_insert_huge()
528 maybe_64K = -1; in gen8_ppgtt_insert_huge()
530 if (unlikely(!IS_ALIGNED(iter->dma, page_size))) in gen8_ppgtt_insert_huge()
538 * Is it safe to mark the 2M block as 64K? -- Either we have in gen8_ppgtt_insert_huge()
539 * filled whole page-table with 64K entries, or filled part of in gen8_ppgtt_insert_huge()
543 if (maybe_64K != -1 && in gen8_ppgtt_insert_huge()
545 (i915_vm_has_scratch_64K(vma->vm) && in gen8_ppgtt_insert_huge()
546 !iter->sg && IS_ALIGNED(vma->node.start + in gen8_ppgtt_insert_huge()
547 vma->node.size, in gen8_ppgtt_insert_huge()
559 * instead - which we detect as missing results during in gen8_ppgtt_insert_huge()
562 if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) { in gen8_ppgtt_insert_huge()
565 encode = vma->vm->scratch[0]->encode; in gen8_ppgtt_insert_huge()
574 vma->page_sizes.gtt |= page_size; in gen8_ppgtt_insert_huge()
575 } while (iter->sg && sg_dma_len(iter->sg)); in gen8_ppgtt_insert_huge()
586 if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) { in gen8_ppgtt_insert()
589 u64 idx = vma->node.start >> GEN8_PTE_SHIFT; in gen8_ppgtt_insert()
599 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert()
606 enum i915_cache_level level, in gen8_ppgtt_insert_entry() argument
617 vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); in gen8_ppgtt_insert_entry()
631 if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) { in gen8_init_scratch()
632 struct i915_address_space *clone = vm->gt->vm; in gen8_init_scratch()
634 GEM_BUG_ON(!clone->has_read_only); in gen8_init_scratch()
636 vm->scratch_order = clone->scratch_order; in gen8_init_scratch()
637 for (i = 0; i <= vm->top; i++) in gen8_init_scratch()
638 vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); in gen8_init_scratch()
647 pte_flags = vm->has_read_only; in gen8_init_scratch()
648 if (i915_gem_object_is_lmem(vm->scratch[0])) in gen8_init_scratch()
651 vm->scratch[0]->encode = in gen8_init_scratch()
652 gen8_pte_encode(px_dma(vm->scratch[0]), in gen8_init_scratch()
655 for (i = 1; i <= vm->top; i++) { in gen8_init_scratch()
658 obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_init_scratch()
668 fill_px(obj, vm->scratch[i - 1]->encode); in gen8_init_scratch()
669 obj->encode = gen8_pde_encode(px_dma(obj), I915_CACHE_LLC); in gen8_init_scratch()
671 vm->scratch[i] = obj; in gen8_init_scratch()
677 while (i--) in gen8_init_scratch()
678 i915_gem_object_put(vm->scratch[i]); in gen8_init_scratch()
679 return -ENOMEM; in gen8_init_scratch()
684 struct i915_address_space *vm = &ppgtt->vm; in gen8_preallocate_top_level_pdp()
685 struct i915_page_directory *pd = ppgtt->pd; in gen8_preallocate_top_level_pdp()
688 GEM_BUG_ON(vm->top != 2); in gen8_preallocate_top_level_pdp()
699 err = map_pt_dma(vm, pde->pt.base); in gen8_preallocate_top_level_pdp()
705 fill_px(pde, vm->scratch[1]->encode); in gen8_preallocate_top_level_pdp()
725 return ERR_PTR(-ENOMEM); in gen8_alloc_top_pd()
727 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_alloc_top_pd()
728 if (IS_ERR(pd->pt.base)) { in gen8_alloc_top_pd()
729 err = PTR_ERR(pd->pt.base); in gen8_alloc_top_pd()
730 pd->pt.base = NULL; in gen8_alloc_top_pd()
734 err = map_pt_dma(vm, pd->pt.base); in gen8_alloc_top_pd()
738 fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); in gen8_alloc_top_pd()
749 * with a net effect resembling a 2-level page table in normal x86 terms. Each
761 return ERR_PTR(-ENOMEM); in gen8_ppgtt_create()
764 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; in gen8_ppgtt_create()
765 ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); in gen8_ppgtt_create()
768 * From bdw, there is hw support for read-only pages in the PPGTT. in gen8_ppgtt_create()
773 * Gen12 has inherited the same read-only fault issue from gen11. in gen8_ppgtt_create()
775 ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12); in gen8_ppgtt_create()
777 if (HAS_LMEM(gt->i915)) in gen8_ppgtt_create()
778 ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; in gen8_ppgtt_create()
780 ppgtt->vm.alloc_pt_dma = alloc_pt_dma; in gen8_ppgtt_create()
782 err = gen8_init_scratch(&ppgtt->vm); in gen8_ppgtt_create()
786 ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm); in gen8_ppgtt_create()
787 if (IS_ERR(ppgtt->pd)) { in gen8_ppgtt_create()
788 err = PTR_ERR(ppgtt->pd); in gen8_ppgtt_create()
792 if (!i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_create()
798 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; in gen8_ppgtt_create()
799 ppgtt->vm.insert_entries = gen8_ppgtt_insert; in gen8_ppgtt_create()
800 ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; in gen8_ppgtt_create()
801 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; in gen8_ppgtt_create()
802 ppgtt->vm.clear_range = gen8_ppgtt_clear; in gen8_ppgtt_create()
803 ppgtt->vm.foreach = gen8_ppgtt_foreach; in gen8_ppgtt_create()
805 ppgtt->vm.pte_encode = gen8_pte_encode; in gen8_ppgtt_create()
807 if (intel_vgpu_active(gt->i915)) in gen8_ppgtt_create()
810 ppgtt->vm.cleanup = gen8_ppgtt_cleanup; in gen8_ppgtt_create()
815 __gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd, in gen8_ppgtt_create()
816 gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top); in gen8_ppgtt_create()
818 free_scratch(&ppgtt->vm); in gen8_ppgtt_create()