46 	struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER);  in crst_table_alloc()  local
48 	if (!ptdesc)  in crst_table_alloc()
50 	arch_set_page_dat(ptdesc_page(ptdesc), CRST_ALLOC_ORDER);  in crst_table_alloc()
51 	return (unsigned long *) ptdesc_to_virt(ptdesc);  in crst_table_alloc()
143 	struct ptdesc *ptdesc;  in page_table_alloc_pgste()  local
146 	ptdesc = pagetable_alloc(GFP_KERNEL, 0);  in page_table_alloc_pgste()
147 	if (ptdesc) {  in page_table_alloc_pgste()
148 		table = (u64 *)ptdesc_to_virt(ptdesc);  in page_table_alloc_pgste()
152 	return ptdesc_page(ptdesc);  in page_table_alloc_pgste()
245 	struct ptdesc *ptdesc;  in page_table_alloc()  local
253 			ptdesc = list_first_entry(&mm->context.pgtable_list,  in page_table_alloc()
254 						struct ptdesc, pt_list);  in page_table_alloc()
255 			mask = atomic_read(&ptdesc->_refcount) >> 24;  in page_table_alloc()
267 				table = (unsigned long *) ptdesc_to_virt(ptdesc);  in page_table_alloc()
271 				atomic_xor_bits(&ptdesc->_refcount,  in page_table_alloc()
273 				list_del_init(&ptdesc->pt_list);  in page_table_alloc()
281 	ptdesc = pagetable_alloc(GFP_KERNEL, 0);  in page_table_alloc()
282 	if (!ptdesc)  in page_table_alloc()
284 	if (!pagetable_pte_ctor(ptdesc)) {  in page_table_alloc()
285 		pagetable_free(ptdesc);  in page_table_alloc()
288 	arch_set_page_dat(ptdesc_page(ptdesc), 0);  in page_table_alloc()
290 	table = (unsigned long *) ptdesc_to_virt(ptdesc);  in page_table_alloc()
293 		INIT_LIST_HEAD(&ptdesc->pt_list);  in page_table_alloc()
294 		atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_alloc()
299 		atomic_xor_bits(&ptdesc->_refcount, 0x01U << 24);  in page_table_alloc()
302 		list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_alloc()
325 	struct ptdesc *ptdesc;  in pte_free_now()  local
327 	ptdesc = container_of(head, struct ptdesc, pt_rcu_head);  in pte_free_now()
328 	pagetable_pte_dtor(ptdesc);  in pte_free_now()
329 	pagetable_free(ptdesc);  in pte_free_now()
335 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free()  local
346 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free()
348 		if ((mask & 0x03U) && !folio_test_active(ptdesc_folio(ptdesc))) {  in page_table_free()
354 			list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free()
357 			list_del_init(&ptdesc->pt_list);  in page_table_free()
360 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x10U << (bit + 24));  in page_table_free()
367 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_free()
371 	page_table_release_check(ptdesc_page(ptdesc), table, half, mask);  in page_table_free()
372 	if (folio_test_clear_active(ptdesc_folio(ptdesc)))  in page_table_free()
373 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in page_table_free()
375 		pte_free_now(&ptdesc->pt_rcu_head);  in page_table_free()
383 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free_rcu()  local
399 	mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free_rcu()
401 	if ((mask & 0x03U) && !folio_test_active(ptdesc_folio(ptdesc))) {  in page_table_free_rcu()
408 		list_add_tail(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free_rcu()
411 		list_del_init(&ptdesc->pt_list);  in page_table_free_rcu()
422 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in __tlb_remove_table()  local
426 		pagetable_free(ptdesc);  in __tlb_remove_table()
430 		mask = atomic_xor_bits(&ptdesc->_refcount, mask << (4 + 24));  in __tlb_remove_table()
436 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in __tlb_remove_table()
441 	page_table_release_check(ptdesc_page(ptdesc), table, half, mask);  in __tlb_remove_table()
442 	if (folio_test_clear_active(ptdesc_folio(ptdesc)))  in __tlb_remove_table()
443 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in __tlb_remove_table()
445 		pte_free_now(&ptdesc->pt_rcu_head);  in __tlb_remove_table()
489 	struct ptdesc *ptdesc;  in base_crst_alloc()  local
491 	ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, CRST_ALLOC_ORDER);  in base_crst_alloc()
492 	if (!ptdesc)  in base_crst_alloc()
494 	table = ptdesc_address(ptdesc);  in base_crst_alloc()