Lines Matching +full:data +full:- +full:addr
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2.
52 u64 addr; member
67 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) in kvm_block_mapping_supported() argument
73 * 52-bit PAs. in kvm_block_mapping_supported()
78 if (granule > (end - addr)) in kvm_block_mapping_supported()
81 return IS_ALIGNED(addr, granule) && IS_ALIGNED(phys, granule); in kvm_block_mapping_supported()
84 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) in kvm_pgtable_idx() argument
87 u64 mask = BIT(PAGE_SHIFT - 3) - 1; in kvm_pgtable_idx()
89 return (data->addr >> shift) & mask; in kvm_pgtable_idx()
92 static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) in __kvm_pgd_page_idx() argument
94 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */ in __kvm_pgd_page_idx()
95 u64 mask = BIT(pgt->ia_bits) - 1; in __kvm_pgd_page_idx()
97 return (addr & mask) >> shift; in __kvm_pgd_page_idx()
100 static u32 kvm_pgd_page_idx(struct kvm_pgtable_walk_data *data) in kvm_pgd_page_idx() argument
102 return __kvm_pgd_page_idx(data->pgt, data->addr); in kvm_pgd_page_idx()
112 return __kvm_pgd_page_idx(&pgt, -1ULL) + 1; in kvm_pgd_pages()
122 if (level == KVM_PGTABLE_MAX_LEVELS - 1) in kvm_pte_table()
177 u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE : in kvm_set_valid_leaf_pte()
192 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr, in kvm_pgtable_visitor_cb() argument
196 struct kvm_pgtable_walker *walker = data->walker; in kvm_pgtable_visitor_cb()
197 return walker->cb(addr, data->end, level, ptep, flag, walker->arg); in kvm_pgtable_visitor_cb()
200 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
203 static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_visit() argument
207 u64 addr = data->addr; in __kvm_pgtable_visit() local
210 enum kvm_pgtable_walk_flags flags = data->walker->flags; in __kvm_pgtable_visit()
213 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
218 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
228 data->addr += kvm_granule_size(level); in __kvm_pgtable_visit()
233 ret = __kvm_pgtable_walk(data, childp, level + 1); in __kvm_pgtable_visit()
238 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
246 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_walk() argument
253 return -EINVAL; in __kvm_pgtable_walk()
255 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { in __kvm_pgtable_walk()
258 if (data->addr >= data->end) in __kvm_pgtable_walk()
261 ret = __kvm_pgtable_visit(data, ptep, level); in __kvm_pgtable_walk()
269 static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data) in _kvm_pgtable_walk() argument
273 struct kvm_pgtable *pgt = data->pgt; in _kvm_pgtable_walk()
274 u64 limit = BIT(pgt->ia_bits); in _kvm_pgtable_walk()
276 if (data->addr > limit || data->end > limit) in _kvm_pgtable_walk()
277 return -ERANGE; in _kvm_pgtable_walk()
279 if (!pgt->pgd) in _kvm_pgtable_walk()
280 return -EINVAL; in _kvm_pgtable_walk()
282 for (idx = kvm_pgd_page_idx(data); data->addr < data->end; ++idx) { in _kvm_pgtable_walk()
283 kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE]; in _kvm_pgtable_walk()
285 ret = __kvm_pgtable_walk(data, ptep, pgt->start_level); in _kvm_pgtable_walk()
293 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_walk() argument
298 .addr = ALIGN_DOWN(addr, PAGE_SIZE), in kvm_pgtable_walk()
299 .end = PAGE_ALIGN(walk_data.addr + size), in kvm_pgtable_walk()
312 struct hyp_map_data *data) in hyp_map_set_prot_attr() argument
322 return -EINVAL; in hyp_map_set_prot_attr()
326 return -EINVAL; in hyp_map_set_prot_attr()
329 return -EINVAL; in hyp_map_set_prot_attr()
337 data->attr = attr; in hyp_map_set_prot_attr()
341 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, in hyp_map_walker_try_leaf() argument
342 kvm_pte_t *ptep, struct hyp_map_data *data) in hyp_map_walker_try_leaf() argument
344 u64 granule = kvm_granule_size(level), phys = data->phys; in hyp_map_walker_try_leaf()
346 if (!kvm_block_mapping_supported(addr, end, phys, level)) in hyp_map_walker_try_leaf()
349 WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level)); in hyp_map_walker_try_leaf()
350 data->phys += granule; in hyp_map_walker_try_leaf()
354 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_map_walker() argument
359 if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg)) in hyp_map_walker()
362 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in hyp_map_walker()
363 return -EINVAL; in hyp_map_walker()
367 return -ENOMEM; in hyp_map_walker()
373 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, in kvm_pgtable_hyp_map() argument
390 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_hyp_map()
400 pgt->pgd = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL); in kvm_pgtable_hyp_init()
401 if (!pgt->pgd) in kvm_pgtable_hyp_init()
402 return -ENOMEM; in kvm_pgtable_hyp_init()
404 pgt->ia_bits = va_bits; in kvm_pgtable_hyp_init()
405 pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels; in kvm_pgtable_hyp_init()
406 pgt->mmu = NULL; in kvm_pgtable_hyp_init()
410 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_free_walker() argument
424 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_hyp_destroy()
425 free_page((unsigned long)pgt->pgd); in kvm_pgtable_hyp_destroy()
426 pgt->pgd = NULL; in kvm_pgtable_hyp_destroy()
440 struct stage2_map_data *data) in stage2_map_set_prot_attr() argument
450 return -EINVAL; in stage2_map_set_prot_attr()
460 data->attr = attr; in stage2_map_set_prot_attr()
464 static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, in stage2_map_walker_try_leaf() argument
466 struct stage2_map_data *data) in stage2_map_walker_try_leaf() argument
468 u64 granule = kvm_granule_size(level), phys = data->phys; in stage2_map_walker_try_leaf()
470 if (!kvm_block_mapping_supported(addr, end, phys, level)) in stage2_map_walker_try_leaf()
475 * early, as it will be bumped-up again in stage2_map_walk_leaf(). in stage2_map_walker_try_leaf()
482 if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level)) in stage2_map_walker_try_leaf()
485 /* There's an existing valid leaf entry, so perform break-before-make */ in stage2_map_walker_try_leaf()
487 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); in stage2_map_walker_try_leaf()
488 kvm_set_valid_leaf_pte(ptep, phys, data->attr, level); in stage2_map_walker_try_leaf()
490 data->phys += granule; in stage2_map_walker_try_leaf()
494 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, in stage2_map_walk_table_pre() argument
496 struct stage2_map_data *data) in stage2_map_walk_table_pre() argument
498 if (data->anchor) in stage2_map_walk_table_pre()
501 if (!kvm_block_mapping_supported(addr, end, data->phys, level)) in stage2_map_walk_table_pre()
507 * Invalidate the whole stage-2, as we may have numerous leaf in stage2_map_walk_table_pre()
511 kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu); in stage2_map_walk_table_pre()
512 data->anchor = ptep; in stage2_map_walk_table_pre()
516 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walk_leaf() argument
517 struct stage2_map_data *data) in stage2_map_walk_leaf() argument
522 if (data->anchor) { in stage2_map_walk_leaf()
529 if (stage2_map_walker_try_leaf(addr, end, level, ptep, data)) in stage2_map_walk_leaf()
532 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in stage2_map_walk_leaf()
533 return -EINVAL; in stage2_map_walk_leaf()
535 if (!data->memcache) in stage2_map_walk_leaf()
536 return -ENOMEM; in stage2_map_walk_leaf()
538 childp = kvm_mmu_memory_cache_alloc(data->memcache); in stage2_map_walk_leaf()
540 return -ENOMEM; in stage2_map_walk_leaf()
549 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); in stage2_map_walk_leaf()
560 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level, in stage2_map_walk_table_post() argument
562 struct stage2_map_data *data) in stage2_map_walk_table_post() argument
566 if (!data->anchor) in stage2_map_walk_table_post()
572 if (data->anchor == ptep) { in stage2_map_walk_table_post()
573 data->anchor = NULL; in stage2_map_walk_table_post()
574 ret = stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walk_table_post()
595 * been set, but otherwise frees the page-table pages while walking back up
596 * the page-table, installing the block entry when it revisits the anchor
599 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walker() argument
602 struct stage2_map_data *data = arg; in stage2_map_walker() local
606 return stage2_map_walk_table_pre(addr, end, level, ptep, data); in stage2_map_walker()
608 return stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walker()
610 return stage2_map_walk_table_post(addr, end, level, ptep, data); in stage2_map_walker()
613 return -EINVAL; in stage2_map_walker()
616 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_stage2_map() argument
623 .mmu = pgt->mmu, in kvm_pgtable_stage2_map()
638 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_map()
643 static void stage2_flush_dcache(void *addr, u64 size) in stage2_flush_dcache() argument
648 __flush_dcache_area(addr, size); in stage2_flush_dcache()
657 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_unmap_walker() argument
683 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); in stage2_unmap_walker()
697 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_unmap() argument
701 .arg = pgt->mmu, in kvm_pgtable_stage2_unmap()
705 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_unmap()
715 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_attr_walker() argument
720 struct stage2_attr_data *data = arg; in stage2_attr_walker() local
725 data->level = level; in stage2_attr_walker()
726 data->pte = pte; in stage2_attr_walker()
727 pte &= ~data->attr_clr; in stage2_attr_walker()
728 pte |= data->attr_set; in stage2_attr_walker()
732 * but worst-case the access flag update gets lost and will be in stage2_attr_walker()
735 if (data->pte != pte) in stage2_attr_walker()
741 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, in stage2_update_leaf_attrs() argument
748 struct stage2_attr_data data = { in stage2_update_leaf_attrs() local
754 .arg = &data, in stage2_update_leaf_attrs()
758 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in stage2_update_leaf_attrs()
763 *orig_pte = data.pte; in stage2_update_leaf_attrs()
766 *level = data.level; in stage2_update_leaf_attrs()
770 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_wrprotect() argument
772 return stage2_update_leaf_attrs(pgt, addr, size, 0, in kvm_pgtable_stage2_wrprotect()
777 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkyoung() argument
780 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0, in kvm_pgtable_stage2_mkyoung()
786 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkold() argument
789 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF, in kvm_pgtable_stage2_mkold()
795 * See the '->clear_flush_young()' callback on the KVM mmu notifier. in kvm_pgtable_stage2_mkold()
800 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_is_young() argument
803 stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL); in kvm_pgtable_stage2_is_young()
807 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, in kvm_pgtable_stage2_relax_perms() argument
823 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level); in kvm_pgtable_stage2_relax_perms()
825 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level); in kvm_pgtable_stage2_relax_perms()
829 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_flush_walker() argument
842 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_flush() argument
852 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_flush()
858 u64 vtcr = kvm->arch.vtcr; in kvm_pgtable_stage2_init()
861 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; in kvm_pgtable_stage2_init()
864 pgt->pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); in kvm_pgtable_stage2_init()
865 if (!pgt->pgd) in kvm_pgtable_stage2_init()
866 return -ENOMEM; in kvm_pgtable_stage2_init()
868 pgt->ia_bits = ia_bits; in kvm_pgtable_stage2_init()
869 pgt->start_level = start_level; in kvm_pgtable_stage2_init()
870 pgt->mmu = &kvm->arch.mmu; in kvm_pgtable_stage2_init()
877 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_free_walker() argument
903 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_stage2_destroy()
904 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE; in kvm_pgtable_stage2_destroy()
905 free_pages_exact(pgt->pgd, pgd_sz); in kvm_pgtable_stage2_destroy()
906 pgt->pgd = NULL; in kvm_pgtable_stage2_destroy()