Lines Matching refs:gmap
35 static struct gmap *gmap_alloc(unsigned long limit) in gmap_alloc()
37 struct gmap *gmap; in gmap_alloc() local
59 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL); in gmap_alloc()
60 if (!gmap) in gmap_alloc()
62 INIT_LIST_HEAD(&gmap->crst_list); in gmap_alloc()
63 INIT_LIST_HEAD(&gmap->children); in gmap_alloc()
64 INIT_LIST_HEAD(&gmap->pt_list); in gmap_alloc()
65 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL); in gmap_alloc()
66 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC); in gmap_alloc()
67 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC); in gmap_alloc()
68 spin_lock_init(&gmap->guest_table_lock); in gmap_alloc()
69 spin_lock_init(&gmap->shadow_lock); in gmap_alloc()
70 atomic_set(&gmap->ref_count, 1); in gmap_alloc()
75 list_add(&page->lru, &gmap->crst_list); in gmap_alloc()
78 gmap->table = table; in gmap_alloc()
79 gmap->asce = atype | _ASCE_TABLE_LENGTH | in gmap_alloc()
81 gmap->asce_end = limit; in gmap_alloc()
82 return gmap; in gmap_alloc()
85 kfree(gmap); in gmap_alloc()
97 struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) in gmap_create()
99 struct gmap *gmap; in gmap_create() local
102 gmap = gmap_alloc(limit); in gmap_create()
103 if (!gmap) in gmap_create()
105 gmap->mm = mm; in gmap_create()
107 list_add_rcu(&gmap->list, &mm->context.gmap_list); in gmap_create()
109 gmap_asce = gmap->asce; in gmap_create()
114 return gmap; in gmap_create()
118 static void gmap_flush_tlb(struct gmap *gmap) in gmap_flush_tlb() argument
121 __tlb_flush_idte(gmap->asce); in gmap_flush_tlb()
183 static void gmap_free(struct gmap *gmap) in gmap_free() argument
188 if (!(gmap_is_shadow(gmap) && gmap->removed)) in gmap_free()
189 gmap_flush_tlb(gmap); in gmap_free()
191 list_for_each_entry_safe(page, next, &gmap->crst_list, lru) in gmap_free()
193 gmap_radix_tree_free(&gmap->guest_to_host); in gmap_free()
194 gmap_radix_tree_free(&gmap->host_to_guest); in gmap_free()
197 if (gmap_is_shadow(gmap)) { in gmap_free()
199 list_for_each_entry_safe(page, next, &gmap->pt_list, lru) in gmap_free()
201 gmap_rmap_radix_tree_free(&gmap->host_to_rmap); in gmap_free()
203 gmap_put(gmap->parent); in gmap_free()
206 kfree(gmap); in gmap_free()
215 struct gmap *gmap_get(struct gmap *gmap) in gmap_get() argument
217 atomic_inc(&gmap->ref_count); in gmap_get()
218 return gmap; in gmap_get()
228 void gmap_put(struct gmap *gmap) in gmap_put() argument
230 if (atomic_dec_return(&gmap->ref_count) == 0) in gmap_put()
231 gmap_free(gmap); in gmap_put()
239 void gmap_remove(struct gmap *gmap) in gmap_remove() argument
241 struct gmap *sg, *next; in gmap_remove()
245 if (!list_empty(&gmap->children)) { in gmap_remove()
246 spin_lock(&gmap->shadow_lock); in gmap_remove()
247 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
251 spin_unlock(&gmap->shadow_lock); in gmap_remove()
254 spin_lock(&gmap->mm->context.lock); in gmap_remove()
255 list_del_rcu(&gmap->list); in gmap_remove()
256 if (list_empty(&gmap->mm->context.gmap_list)) in gmap_remove()
258 else if (list_is_singular(&gmap->mm->context.gmap_list)) in gmap_remove()
259 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, in gmap_remove()
260 struct gmap, list)->asce; in gmap_remove()
263 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); in gmap_remove()
264 spin_unlock(&gmap->mm->context.lock); in gmap_remove()
267 gmap_put(gmap); in gmap_remove()
275 void gmap_enable(struct gmap *gmap) in gmap_enable() argument
277 S390_lowcore.gmap = (unsigned long) gmap; in gmap_enable()
285 void gmap_disable(struct gmap *gmap) in gmap_disable() argument
287 S390_lowcore.gmap = 0UL; in gmap_disable()
296 struct gmap *gmap_get_enabled(void) in gmap_get_enabled()
298 return (struct gmap *) S390_lowcore.gmap; in gmap_get_enabled()
305 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, in gmap_alloc_table() argument
317 spin_lock(&gmap->guest_table_lock); in gmap_alloc_table()
319 list_add(&page->lru, &gmap->crst_list); in gmap_alloc_table()
325 spin_unlock(&gmap->guest_table_lock); in gmap_alloc_table()
356 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) in __gmap_unlink_by_vmaddr() argument
361 BUG_ON(gmap_is_shadow(gmap)); in __gmap_unlink_by_vmaddr()
362 spin_lock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
363 entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in __gmap_unlink_by_vmaddr()
368 spin_unlock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
379 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr) in __gmap_unmap_by_gaddr() argument
383 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, in __gmap_unmap_by_gaddr()
385 return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0; in __gmap_unmap_by_gaddr()
396 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) in gmap_unmap_segment() argument
401 BUG_ON(gmap_is_shadow(gmap)); in gmap_unmap_segment()
408 down_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
410 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_unmap_segment()
411 up_write(&gmap->mm->mmap_sem); in gmap_unmap_segment()
413 gmap_flush_tlb(gmap); in gmap_unmap_segment()
427 int gmap_map_segment(struct gmap *gmap, unsigned long from, in gmap_map_segment() argument
433 BUG_ON(gmap_is_shadow(gmap)); in gmap_map_segment()
437 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) in gmap_map_segment()
441 down_write(&gmap->mm->mmap_sem); in gmap_map_segment()
444 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_map_segment()
446 if (radix_tree_insert(&gmap->guest_to_host, in gmap_map_segment()
451 up_write(&gmap->mm->mmap_sem); in gmap_map_segment()
453 gmap_flush_tlb(gmap); in gmap_map_segment()
456 gmap_unmap_segment(gmap, to, len); in gmap_map_segment()
474 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr) in __gmap_translate() argument
479 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); in __gmap_translate()
494 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr) in gmap_translate() argument
498 down_read(&gmap->mm->mmap_sem); in gmap_translate()
499 rc = __gmap_translate(gmap, gaddr); in gmap_translate()
500 up_read(&gmap->mm->mmap_sem); in gmap_translate()
514 struct gmap *gmap; in gmap_unlink() local
518 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_unlink()
519 flush = __gmap_unlink_by_vmaddr(gmap, vmaddr); in gmap_unlink()
521 gmap_flush_tlb(gmap); in gmap_unlink()
526 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new,
540 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) in __gmap_link() argument
552 BUG_ON(gmap_is_shadow(gmap)); in __gmap_link()
554 table = gmap->table; in __gmap_link()
555 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { in __gmap_link()
558 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, in __gmap_link()
563 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { in __gmap_link()
566 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, in __gmap_link()
571 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { in __gmap_link()
574 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, in __gmap_link()
581 mm = gmap->mm; in __gmap_link()
594 if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) in __gmap_link()
601 spin_lock(&gmap->guest_table_lock); in __gmap_link()
603 rc = radix_tree_insert(&gmap->host_to_guest, in __gmap_link()
619 gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr); in __gmap_link()
621 spin_unlock(&gmap->guest_table_lock); in __gmap_link()
636 int gmap_fault(struct gmap *gmap, unsigned long gaddr, in gmap_fault() argument
643 down_read(&gmap->mm->mmap_sem); in gmap_fault()
647 vmaddr = __gmap_translate(gmap, gaddr); in gmap_fault()
652 if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags, in gmap_fault()
664 rc = __gmap_link(gmap, gaddr, vmaddr); in gmap_fault()
666 up_read(&gmap->mm->mmap_sem); in gmap_fault()
674 void __gmap_zap(struct gmap *gmap, unsigned long gaddr) in __gmap_zap() argument
681 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, in __gmap_zap()
686 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl); in __gmap_zap()
688 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0); in __gmap_zap()
694 void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) in gmap_discard() argument
699 down_read(&gmap->mm->mmap_sem); in gmap_discard()
704 radix_tree_lookup(&gmap->guest_to_host, in gmap_discard()
710 vma = find_vma(gmap->mm, vmaddr); in gmap_discard()
722 up_read(&gmap->mm->mmap_sem); in gmap_discard()
760 static void gmap_call_notifier(struct gmap *gmap, unsigned long start, in gmap_call_notifier() argument
766 nb->notifier_call(gmap, start, end); in gmap_call_notifier()
787 static inline unsigned long *gmap_table_walk(struct gmap *gmap, in gmap_table_walk() argument
792 if ((gmap->asce & _ASCE_TYPE_MASK) + 4 < (level * 4)) in gmap_table_walk()
794 if (gmap_is_shadow(gmap) && gmap->removed) in gmap_table_walk()
796 if (gaddr & (-1UL << (31 + ((gmap->asce & _ASCE_TYPE_MASK) >> 2)*11))) in gmap_table_walk()
798 table = gmap->table; in gmap_table_walk()
799 switch (gmap->asce & _ASCE_TYPE_MASK) { in gmap_table_walk()
845 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_walk() argument
850 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_walk()
852 table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */ in gmap_pte_op_walk()
855 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); in gmap_pte_op_walk()
869 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_fixup() argument
872 struct mm_struct *mm = gmap->mm; in gmap_pte_op_fixup()
876 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_fixup()
884 return __gmap_link(gmap, gaddr, vmaddr); in gmap_pte_op_fixup()
905 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr) in gmap_pmd_op_walk() argument
909 BUG_ON(gmap_is_shadow(gmap)); in gmap_pmd_op_walk()
910 spin_lock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
911 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); in gmap_pmd_op_walk()
914 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
920 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
929 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) in gmap_pmd_op_end() argument
932 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_end()
949 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pmd() argument
962 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
968 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
994 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pte() argument
1005 ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); in gmap_protect_pte()
1012 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); in gmap_protect_pte()
1030 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, in gmap_protect_range() argument
1037 BUG_ON(gmap_is_shadow(gmap)); in gmap_protect_range()
1040 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_protect_range()
1043 rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1050 rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1058 gmap_pmd_op_end(gmap, pmdp); in gmap_protect_range()
1065 vmaddr = __gmap_translate(gmap, gaddr); in gmap_protect_range()
1068 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot); in gmap_protect_range()
1090 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, in gmap_mprotect_notify() argument
1095 if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap)) in gmap_mprotect_notify()
1099 down_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
1100 rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT); in gmap_mprotect_notify()
1101 up_read(&gmap->mm->mmap_sem); in gmap_mprotect_notify()
1119 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) in gmap_read_table() argument
1126 if (gmap_is_shadow(gmap)) in gmap_read_table()
1131 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); in gmap_read_table()
1146 vmaddr = __gmap_translate(gmap, gaddr); in gmap_read_table()
1151 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ); in gmap_read_table()
1167 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap()
1195 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap()
1198 struct gmap *parent; in gmap_protect_rmap()
1276 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page()
1296 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt()
1313 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt()
1342 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt()
1370 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt()
1399 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t()
1427 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t()
1456 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t()
1484 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t()
1513 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t()
1543 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow()
1580 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, in gmap_find_shadow()
1583 struct gmap *sg; in gmap_find_shadow()
1609 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid()
1632 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, in gmap_shadow()
1635 struct gmap *sg, *new; in gmap_shadow()
1724 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t()
1808 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t()
1891 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt()
1975 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup()
2015 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt()
2094 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page()
2096 struct gmap *parent; in gmap_shadow_page()
2162 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify()
2229 struct gmap *gmap, *sg, *next; in ptep_notify() local
2234 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in ptep_notify()
2235 spin_lock(&gmap->guest_table_lock); in ptep_notify()
2236 table = radix_tree_lookup(&gmap->host_to_guest, in ptep_notify()
2240 spin_unlock(&gmap->guest_table_lock); in ptep_notify()
2244 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { in ptep_notify()
2245 spin_lock(&gmap->shadow_lock); in ptep_notify()
2247 &gmap->children, list) in ptep_notify()
2249 spin_unlock(&gmap->shadow_lock); in ptep_notify()
2252 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); in ptep_notify()
2258 static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp, in pmdp_notify_gmap() argument
2262 gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); in pmdp_notify_gmap()
2275 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, in gmap_pmdp_xchg() argument
2279 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_xchg()
2282 __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, in gmap_pmdp_xchg()
2295 struct gmap *gmap; in gmap_pmdp_clear() local
2299 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_clear()
2300 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2301 pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_clear()
2305 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_clear()
2312 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2348 struct gmap *gmap; in gmap_pmdp_idte_local() local
2352 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_local()
2353 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2354 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_local()
2359 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_local()
2364 gmap->asce, IDTE_LOCAL); in gmap_pmdp_idte_local()
2369 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2383 struct gmap *gmap; in gmap_pmdp_idte_global() local
2387 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_global()
2388 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2389 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_global()
2394 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_global()
2399 gmap->asce, IDTE_GLOBAL); in gmap_pmdp_idte_global()
2406 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2421 bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp, in gmap_test_and_clear_dirty_pmd() argument
2434 gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0); in gmap_test_and_clear_dirty_pmd()
2448 void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4], in gmap_sync_dirty_log_pmd() argument
2456 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_sync_dirty_log_pmd()
2461 if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr)) in gmap_sync_dirty_log_pmd()
2465 ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); in gmap_sync_dirty_log_pmd()
2468 if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) in gmap_sync_dirty_log_pmd()
2473 gmap_pmd_op_end(gmap, pmdp); in gmap_sync_dirty_log_pmd()