Lines Matching +full:top +full:- +full:level
1 // SPDX-License-Identifier: MIT
19 const enum i915_cache_level level) in gen8_pde_encode() argument
23 if (level != I915_CACHE_NONE) in gen8_pde_encode()
32 enum i915_cache_level level, in gen8_pte_encode() argument
43 switch (level) { in gen8_pte_encode()
60 struct drm_i915_private *i915 = ppgtt->vm.i915; in gen8_ppgtt_notify_vgt()
61 struct intel_uncore *uncore = ppgtt->vm.gt->uncore; in gen8_ppgtt_notify_vgt()
66 atomic_inc(px_used(ppgtt->pd)); /* never remove */ in gen8_ppgtt_notify_vgt()
68 atomic_dec(px_used(ppgtt->pd)); in gen8_ppgtt_notify_vgt()
70 mutex_lock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
72 if (i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_notify_vgt()
73 const u64 daddr = px_dma(ppgtt->pd); in gen8_ppgtt_notify_vgt()
103 mutex_unlock(&i915->vgpu.lock); in gen8_ppgtt_notify_vgt()
107 #define GEN8_PAGE_SIZE (SZ_4K) /* page and page-directory sizes are the same */
128 return GEN8_PDES - *idx; in gen8_pd_range()
130 return i915_pde_index(end, shift) - *idx; in gen8_pd_range()
145 return GEN8_PDES - (start & (GEN8_PDES - 1)); in gen8_pt_count()
147 return end - start; in gen8_pt_count()
152 unsigned int shift = __gen8_pte_shift(vm->top); in gen8_pd_top_count()
154 return (vm->total + (1ull << shift) - 1) >> shift; in gen8_pd_top_count()
162 if (vm->top == 2) in gen8_pdp_for_page_index()
163 return ppgtt->pd; in gen8_pdp_for_page_index()
165 return i915_pd_entry(ppgtt->pd, gen8_pd_index(idx, vm->top)); in gen8_pdp_for_page_index()
179 void **pde = pd->entry; in __gen8_ppgtt_cleanup()
185 __gen8_ppgtt_cleanup(vm, *pde, GEN8_PDES, lvl - 1); in __gen8_ppgtt_cleanup()
186 } while (pde++, --count); in __gen8_ppgtt_cleanup()
189 free_px(vm, &pd->pt, lvl); in __gen8_ppgtt_cleanup()
196 if (intel_vgpu_active(vm->i915)) in gen8_ppgtt_cleanup()
199 if (ppgtt->pd) in gen8_ppgtt_cleanup()
200 __gen8_ppgtt_cleanup(vm, ppgtt->pd, in gen8_ppgtt_cleanup()
201 gen8_pd_top_count(vm), vm->top); in gen8_ppgtt_cleanup()
210 const struct drm_i915_gem_object * const scratch = vm->scratch[lvl]; in __gen8_ppgtt_clear()
213 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_clear()
215 len = gen8_pd_range(start, end, lvl--, &idx); in __gen8_ppgtt_clear()
222 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_clear()
224 if (atomic_fetch_inc(&pt->used) >> gen8_pd_shift(1) && in __gen8_ppgtt_clear()
247 atomic_read(&pt->used)); in __gen8_ppgtt_clear()
248 GEM_BUG_ON(!count || count >= atomic_read(&pt->used)); in __gen8_ppgtt_clear()
251 if (pt->is_compact) { in __gen8_ppgtt_clear()
260 vm->scratch[0]->encode, in __gen8_ppgtt_clear()
263 atomic_sub(count, &pt->used); in __gen8_ppgtt_clear()
269 } while (idx++, --len); in __gen8_ppgtt_clear()
279 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_clear()
285 __gen8_ppgtt_clear(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_clear()
286 start, start + length, vm->top); in gen8_ppgtt_clear()
296 GEM_BUG_ON(end > vm->total >> GEN8_PTE_SHIFT); in __gen8_ppgtt_alloc()
298 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_alloc()
302 GEM_BUG_ON(!len || (idx + len - 1) >> gen8_pd_shift(1)); in __gen8_ppgtt_alloc()
304 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
307 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
310 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
315 pt = stash->pt[!!lvl]; in __gen8_ppgtt_alloc()
316 __i915_gem_object_pin_pages(pt->base); in __gen8_ppgtt_alloc()
318 fill_px(pt, vm->scratch[lvl]->encode); in __gen8_ppgtt_alloc()
320 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
321 if (likely(!pd->entry[idx])) { in __gen8_ppgtt_alloc()
322 stash->pt[!!lvl] = pt->stash; in __gen8_ppgtt_alloc()
323 atomic_set(&pt->used, 0); in __gen8_ppgtt_alloc()
326 pt = pd->entry[idx]; in __gen8_ppgtt_alloc()
331 atomic_inc(&pt->used); in __gen8_ppgtt_alloc()
332 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
337 spin_lock(&pd->lock); in __gen8_ppgtt_alloc()
338 atomic_dec(&pt->used); in __gen8_ppgtt_alloc()
339 GEM_BUG_ON(!atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
346 atomic_read(&pt->used)); in __gen8_ppgtt_alloc()
348 atomic_add(count, &pt->used); in __gen8_ppgtt_alloc()
350 GEM_BUG_ON(atomic_read(&pt->used) > NALLOC * I915_PDES); in __gen8_ppgtt_alloc()
353 } while (idx++, --len); in __gen8_ppgtt_alloc()
354 spin_unlock(&pd->lock); in __gen8_ppgtt_alloc()
363 GEM_BUG_ON(range_overflows(start, length, vm->total)); in gen8_ppgtt_alloc()
369 __gen8_ppgtt_alloc(vm, stash, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_alloc()
370 &start, start + length, vm->top); in gen8_ppgtt_alloc()
383 len = gen8_pd_range(*start, end, lvl--, &idx); in __gen8_ppgtt_foreach()
385 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
387 struct i915_page_table *pt = pd->entry[idx]; in __gen8_ppgtt_foreach()
389 atomic_inc(&pt->used); in __gen8_ppgtt_foreach()
390 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
400 spin_lock(&pd->lock); in __gen8_ppgtt_foreach()
401 atomic_dec(&pt->used); in __gen8_ppgtt_foreach()
402 } while (idx++, --len); in __gen8_ppgtt_foreach()
403 spin_unlock(&pd->lock); in __gen8_ppgtt_foreach()
416 __gen8_ppgtt_foreach(vm, i915_vm_to_ppgtt(vm)->pd, in gen8_ppgtt_foreach()
417 &start, start + length, vm->top, in gen8_ppgtt_foreach()
436 GEM_BUG_ON(sg_dma_len(iter->sg) < I915_GTT_PAGE_SIZE); in gen8_ppgtt_insert_pte()
437 vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; in gen8_ppgtt_insert_pte()
439 iter->dma += I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert_pte()
440 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_pte()
441 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_pte()
442 if (!iter->sg || sg_dma_len(iter->sg) == 0) { in gen8_ppgtt_insert_pte()
447 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_pte()
448 iter->max = iter->dma + sg_dma_len(iter->sg); in gen8_ppgtt_insert_pte()
457 pd = pdp->entry[gen8_pd_index(idx, 2)]; in gen8_ppgtt_insert_pte()
476 const gen8_pte_t pte_encode = vm->pte_encode(0, cache_level, flags); in xehpsdv_ppgtt_insert_huge()
477 unsigned int rem = sg_dma_len(iter->sg); in xehpsdv_ppgtt_insert_huge()
478 u64 start = vma_res->start; in xehpsdv_ppgtt_insert_huge()
496 if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && in xehpsdv_ppgtt_insert_huge()
497 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && in xehpsdv_ppgtt_insert_huge()
509 GEM_BUG_ON(!IS_ALIGNED(iter->dma, in xehpsdv_ppgtt_insert_huge()
520 pt->is_compact = true; in xehpsdv_ppgtt_insert_huge()
522 GEM_BUG_ON(pt->is_compact); in xehpsdv_ppgtt_insert_huge()
532 vaddr[index++] = encode | iter->dma; in xehpsdv_ppgtt_insert_huge()
535 iter->dma += page_size; in xehpsdv_ppgtt_insert_huge()
536 rem -= page_size; in xehpsdv_ppgtt_insert_huge()
537 if (iter->dma >= iter->max) { in xehpsdv_ppgtt_insert_huge()
538 iter->sg = __sg_next(iter->sg); in xehpsdv_ppgtt_insert_huge()
539 if (!iter->sg) in xehpsdv_ppgtt_insert_huge()
542 rem = sg_dma_len(iter->sg); in xehpsdv_ppgtt_insert_huge()
546 iter->dma = sg_dma_address(iter->sg); in xehpsdv_ppgtt_insert_huge()
547 iter->max = iter->dma + rem; in xehpsdv_ppgtt_insert_huge()
549 if (unlikely(!IS_ALIGNED(iter->dma, page_size))) in xehpsdv_ppgtt_insert_huge()
554 vma_res->page_sizes_gtt |= page_size; in xehpsdv_ppgtt_insert_huge()
555 } while (iter->sg && sg_dma_len(iter->sg)); in xehpsdv_ppgtt_insert_huge()
565 unsigned int rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
566 u64 start = vma_res->start; in gen8_ppgtt_insert_huge()
576 unsigned int maybe_64K = -1; in gen8_ppgtt_insert_huge()
581 if (vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_2M && in gen8_ppgtt_insert_huge()
582 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) && in gen8_ppgtt_insert_huge()
598 vma_res->bi.page_sizes.sg & I915_GTT_PAGE_SIZE_64K && in gen8_ppgtt_insert_huge()
599 IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
601 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE)) in gen8_ppgtt_insert_huge()
608 GEM_BUG_ON(sg_dma_len(iter->sg) < page_size); in gen8_ppgtt_insert_huge()
609 vaddr[index++] = encode | iter->dma; in gen8_ppgtt_insert_huge()
612 iter->dma += page_size; in gen8_ppgtt_insert_huge()
613 rem -= page_size; in gen8_ppgtt_insert_huge()
614 if (iter->dma >= iter->max) { in gen8_ppgtt_insert_huge()
615 iter->sg = __sg_next(iter->sg); in gen8_ppgtt_insert_huge()
616 if (!iter->sg) in gen8_ppgtt_insert_huge()
619 rem = sg_dma_len(iter->sg); in gen8_ppgtt_insert_huge()
623 iter->dma = sg_dma_address(iter->sg); in gen8_ppgtt_insert_huge()
624 iter->max = iter->dma + rem; in gen8_ppgtt_insert_huge()
626 if (maybe_64K != -1 && index < I915_PDES && in gen8_ppgtt_insert_huge()
627 !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) && in gen8_ppgtt_insert_huge()
629 rem >= (I915_PDES - index) * I915_GTT_PAGE_SIZE))) in gen8_ppgtt_insert_huge()
630 maybe_64K = -1; in gen8_ppgtt_insert_huge()
632 if (unlikely(!IS_ALIGNED(iter->dma, page_size))) in gen8_ppgtt_insert_huge()
640 * Is it safe to mark the 2M block as 64K? -- Either we have in gen8_ppgtt_insert_huge()
641 * filled whole page-table with 64K entries, or filled part of in gen8_ppgtt_insert_huge()
645 if (maybe_64K != -1 && in gen8_ppgtt_insert_huge()
648 !iter->sg && IS_ALIGNED(vma_res->start + in gen8_ppgtt_insert_huge()
649 vma_res->node_size, in gen8_ppgtt_insert_huge()
662 * instead - which we detect as missing results during in gen8_ppgtt_insert_huge()
665 if (I915_SELFTEST_ONLY(vm->scrub_64K)) { in gen8_ppgtt_insert_huge()
668 encode = vm->scratch[0]->encode; in gen8_ppgtt_insert_huge()
678 vma_res->page_sizes_gtt |= page_size; in gen8_ppgtt_insert_huge()
679 } while (iter->sg && sg_dma_len(iter->sg)); in gen8_ppgtt_insert_huge()
690 if (vma_res->bi.page_sizes.sg > I915_GTT_PAGE_SIZE) { in gen8_ppgtt_insert()
691 if (HAS_64K_PAGES(vm->i915)) in gen8_ppgtt_insert()
696 u64 idx = vma_res->start >> GEN8_PTE_SHIFT; in gen8_ppgtt_insert()
706 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; in gen8_ppgtt_insert()
713 enum i915_cache_level level, in gen8_ppgtt_insert_entry() argument
724 GEM_BUG_ON(pt->is_compact); in gen8_ppgtt_insert_entry()
727 vaddr[gen8_pd_index(idx, 0)] = gen8_pte_encode(addr, level, flags); in gen8_ppgtt_insert_entry()
734 enum i915_cache_level level, in __xehpsdv_ppgtt_insert_entry_lm() argument
748 if (!pt->is_compact) { in __xehpsdv_ppgtt_insert_entry_lm()
751 pt->is_compact = true; in __xehpsdv_ppgtt_insert_entry_lm()
755 vaddr[gen8_pd_index(idx, 0) / 16] = gen8_pte_encode(addr, level, flags); in __xehpsdv_ppgtt_insert_entry_lm()
761 enum i915_cache_level level, in xehpsdv_ppgtt_insert_entry() argument
766 level, flags); in xehpsdv_ppgtt_insert_entry()
768 return gen8_ppgtt_insert_entry(vm, addr, offset, level, flags); in xehpsdv_ppgtt_insert_entry()
781 if (vm->has_read_only && vm->gt->vm && !i915_is_ggtt(vm->gt->vm)) { in gen8_init_scratch()
782 struct i915_address_space *clone = vm->gt->vm; in gen8_init_scratch()
784 GEM_BUG_ON(!clone->has_read_only); in gen8_init_scratch()
786 vm->scratch_order = clone->scratch_order; in gen8_init_scratch()
787 for (i = 0; i <= vm->top; i++) in gen8_init_scratch()
788 vm->scratch[i] = i915_gem_object_get(clone->scratch[i]); in gen8_init_scratch()
797 pte_flags = vm->has_read_only; in gen8_init_scratch()
798 if (i915_gem_object_is_lmem(vm->scratch[0])) in gen8_init_scratch()
801 vm->scratch[0]->encode = in gen8_init_scratch()
802 gen8_pte_encode(px_dma(vm->scratch[0]), in gen8_init_scratch()
805 for (i = 1; i <= vm->top; i++) { in gen8_init_scratch()
808 obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_init_scratch()
820 fill_px(obj, vm->scratch[i - 1]->encode); in gen8_init_scratch()
821 obj->encode = gen8_pde_encode(px_dma(obj), I915_CACHE_NONE); in gen8_init_scratch()
823 vm->scratch[i] = obj; in gen8_init_scratch()
829 while (i--) in gen8_init_scratch()
830 i915_gem_object_put(vm->scratch[i]); in gen8_init_scratch()
831 vm->scratch[0] = NULL; in gen8_init_scratch()
837 struct i915_address_space *vm = &ppgtt->vm; in gen8_preallocate_top_level_pdp()
838 struct i915_page_directory *pd = ppgtt->pd; in gen8_preallocate_top_level_pdp()
841 GEM_BUG_ON(vm->top != 2); in gen8_preallocate_top_level_pdp()
852 err = map_pt_dma(vm, pde->pt.base); in gen8_preallocate_top_level_pdp()
858 fill_px(pde, vm->scratch[1]->encode); in gen8_preallocate_top_level_pdp()
878 return ERR_PTR(-ENOMEM); in gen8_alloc_top_pd()
880 pd->pt.base = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); in gen8_alloc_top_pd()
881 if (IS_ERR(pd->pt.base)) { in gen8_alloc_top_pd()
882 err = PTR_ERR(pd->pt.base); in gen8_alloc_top_pd()
883 pd->pt.base = NULL; in gen8_alloc_top_pd()
887 err = map_pt_dma(vm, pd->pt.base); in gen8_alloc_top_pd()
891 fill_page_dma(px_base(pd), vm->scratch[vm->top]->encode, count); in gen8_alloc_top_pd()
902 * with a net effect resembling a 2-level page table in normal x86 terms. Each
916 return ERR_PTR(-ENOMEM); in gen8_ppgtt_create()
919 ppgtt->vm.top = i915_vm_is_4lvl(&ppgtt->vm) ? 3 : 2; in gen8_ppgtt_create()
920 ppgtt->vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen8_pte_t)); in gen8_ppgtt_create()
923 * From bdw, there is hw support for read-only pages in the PPGTT. in gen8_ppgtt_create()
928 * Gen12 has inherited the same read-only fault issue from gen11. in gen8_ppgtt_create()
930 ppgtt->vm.has_read_only = !IS_GRAPHICS_VER(gt->i915, 11, 12); in gen8_ppgtt_create()
932 if (HAS_LMEM(gt->i915)) { in gen8_ppgtt_create()
933 ppgtt->vm.alloc_pt_dma = alloc_pt_lmem; in gen8_ppgtt_create()
938 * pages in the hw, we can only mark the *entire* page-table as in gen8_ppgtt_create()
942 * page-table with scratch pointing to LMEM, since that's in gen8_ppgtt_create()
947 if (HAS_64K_PAGES(gt->i915)) in gen8_ppgtt_create()
948 ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; in gen8_ppgtt_create()
950 ppgtt->vm.alloc_scratch_dma = alloc_pt_lmem; in gen8_ppgtt_create()
952 ppgtt->vm.alloc_pt_dma = alloc_pt_dma; in gen8_ppgtt_create()
953 ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; in gen8_ppgtt_create()
956 ppgtt->vm.pte_encode = gen8_pte_encode; in gen8_ppgtt_create()
958 ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; in gen8_ppgtt_create()
959 ppgtt->vm.insert_entries = gen8_ppgtt_insert; in gen8_ppgtt_create()
960 if (HAS_64K_PAGES(gt->i915)) in gen8_ppgtt_create()
961 ppgtt->vm.insert_page = xehpsdv_ppgtt_insert_entry; in gen8_ppgtt_create()
963 ppgtt->vm.insert_page = gen8_ppgtt_insert_entry; in gen8_ppgtt_create()
964 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; in gen8_ppgtt_create()
965 ppgtt->vm.clear_range = gen8_ppgtt_clear; in gen8_ppgtt_create()
966 ppgtt->vm.foreach = gen8_ppgtt_foreach; in gen8_ppgtt_create()
967 ppgtt->vm.cleanup = gen8_ppgtt_cleanup; in gen8_ppgtt_create()
969 err = gen8_init_scratch(&ppgtt->vm); in gen8_ppgtt_create()
973 pd = gen8_alloc_top_pd(&ppgtt->vm); in gen8_ppgtt_create()
978 ppgtt->pd = pd; in gen8_ppgtt_create()
980 if (!i915_vm_is_4lvl(&ppgtt->vm)) { in gen8_ppgtt_create()
986 if (intel_vgpu_active(gt->i915)) in gen8_ppgtt_create()
992 i915_vm_put(&ppgtt->vm); in gen8_ppgtt_create()