54 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
56 	pagetable_free(virt_to_ptdesc(table));  in crst_table_free()
144 	u64 *table;  in page_table_alloc_pgste()  local
148 		table = (u64 *)ptdesc_to_virt(ptdesc);  in page_table_alloc_pgste()
149 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_pgste()
150 		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_pgste()
244 	unsigned long *table;  in page_table_alloc()  local
250 		table = NULL;  in page_table_alloc()
267 				table = (unsigned long *) ptdesc_to_virt(ptdesc);  in page_table_alloc()
270 					table += PTRS_PER_PTE;  in page_table_alloc()
277 		if (table)  in page_table_alloc()
278 			return table;  in page_table_alloc()
290 	table = (unsigned long *) ptdesc_to_virt(ptdesc);  in page_table_alloc()
295 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc()
296 		memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc()
300 		memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);  in page_table_alloc()
305 	return table;  in page_table_alloc()
308 static void page_table_release_check(struct page *page, void *table,  in page_table_release_check()  argument
319 		 table, half, mask);  in page_table_release_check()
332 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
335 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free()
339 		bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));  in page_table_free()
371 	page_table_release_check(ptdesc_page(ptdesc), table, half, mask);  in page_table_free()
378 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
383 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free_rcu()
387 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
388 		table = (unsigned long *) ((unsigned long)table | 0x03U);  in page_table_free_rcu()
389 		tlb_remove_ptdesc(tlb, table);  in page_table_free_rcu()
392 	bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));  in page_table_free_rcu()
414 	table = (unsigned long *) ((unsigned long) table | (0x01U << bit));  in page_table_free_rcu()
415 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
421 	void *table = (void *)((unsigned long) _table ^ mask);  in __tlb_remove_table()  local
422 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in __tlb_remove_table()
441 	page_table_release_check(ptdesc_page(ptdesc), table, half, mask);  in __tlb_remove_table()
473 	unsigned long *table;  in base_pgt_alloc()  local
475 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
476 	if (table)  in base_pgt_alloc()
477 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
478 	return table;  in base_pgt_alloc()
481 static void base_pgt_free(unsigned long *table)  in base_pgt_free()  argument
483 	kmem_cache_free(base_pgt_cache, table);  in base_pgt_free()
488 	unsigned long *table;  in base_crst_alloc()  local
494 	table = ptdesc_address(ptdesc);  in base_crst_alloc()
496 	crst_table_init(table, val);  in base_crst_alloc()
497 	return table;  in base_crst_alloc()
500 static void base_crst_free(unsigned long *table)  in base_crst_free()  argument
502 	pagetable_free(virt_to_ptdesc(table));  in base_crst_free()
549 	unsigned long *ste, next, *table;  in base_segment_walk()  local
559 			table = base_pgt_alloc();  in base_segment_walk()
560 			if (!table)  in base_segment_walk()
562 			*ste = __pa(table) | _SEGMENT_ENTRY;  in base_segment_walk()
564 		table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);  in base_segment_walk()
565 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
569 			base_pgt_free(table);  in base_segment_walk()
578 	unsigned long *rtte, next, *table;  in base_region3_walk()  local
588 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
589 			if (!table)  in base_region3_walk()
591 			*rtte = __pa(table) | _REGION3_ENTRY;  in base_region3_walk()
593 		table = __va(*rtte & _REGION_ENTRY_ORIGIN);  in base_region3_walk()
594 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
598 			base_crst_free(table);  in base_region3_walk()
606 	unsigned long *rste, next, *table;  in base_region2_walk()  local
616 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
617 			if (!table)  in base_region2_walk()
619 			*rste = __pa(table) | _REGION2_ENTRY;  in base_region2_walk()
621 		table = __va(*rste & _REGION_ENTRY_ORIGIN);  in base_region2_walk()
622 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
626 			base_crst_free(table);  in base_region2_walk()
634 	unsigned long *rfte, next, *table;  in base_region1_walk()  local
644 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
645 			if (!table)  in base_region1_walk()
647 			*rfte = __pa(table) | _REGION1_ENTRY;  in base_region1_walk()
649 		table = __va(*rfte & _REGION_ENTRY_ORIGIN);  in base_region1_walk()
650 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
654 			base_crst_free(table);  in base_region1_walk()
668 	unsigned long *table = __va(asce & _ASCE_ORIGIN);  in base_asce_free()  local
674 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
677 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
680 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
683 		base_region1_walk(table, 0, TASK_SIZE_MAX, 0);  in base_asce_free()
686 	base_crst_free(table);  in base_asce_free()
720 	unsigned long asce, *table, end;  in base_asce_alloc()  local
727 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
728 		if (!table)  in base_asce_alloc()
730 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
731 		asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
733 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
734 		if (!table)  in base_asce_alloc()
736 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
737 		asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
739 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
740 		if (!table)  in base_asce_alloc()
742 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
743 		asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
745 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
746 		if (!table)  in base_asce_alloc()
748 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
749 		asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()