66 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
68 	free_pages((unsigned long) table, 2);  in crst_table_free()
82 	unsigned long *table, *pgd;  in crst_table_upgrade()  local
90 		table = crst_table_alloc(mm);  in crst_table_upgrade()
91 		if (!table) {  in crst_table_upgrade()
98 			crst_table_init(table, _REGION2_ENTRY_EMPTY);  in crst_table_upgrade()
99 			p4d_populate(mm, (p4d_t *) table, (pud_t *) pgd);  in crst_table_upgrade()
100 			mm->pgd = (pgd_t *) table;  in crst_table_upgrade()
105 			crst_table_init(table, _REGION1_ENTRY_EMPTY);  in crst_table_upgrade()
106 			pgd_populate(mm, (pgd_t *) table, (p4d_t *) pgd);  in crst_table_upgrade()
107 			mm->pgd = (pgd_t *) table;  in crst_table_upgrade()
159 	u64 *table;  in page_table_alloc_pgste()  local
163 		table = (u64 *)page_to_phys(page);  in page_table_alloc_pgste()
164 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_pgste()
165 		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_pgste()
182 	unsigned long *table;  in page_table_alloc()  local
188 		table = NULL;  in page_table_alloc()
196 				table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
199 					table += PTRS_PER_PTE;  in page_table_alloc()
206 		if (table)  in page_table_alloc()
207 			return table;  in page_table_alloc()
219 	table = (unsigned long *) page_to_phys(page);  in page_table_alloc()
223 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc()
224 		memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc()
228 		memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);  in page_table_alloc()
233 	return table;  in page_table_alloc()
236 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
241 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free()
244 		bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));  in page_table_free()
263 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
271 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in page_table_free_rcu()
273 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
274 		table = (unsigned long *) (__pa(table) | 3);  in page_table_free_rcu()
275 		tlb_remove_table(tlb, table);  in page_table_free_rcu()
278 	bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));  in page_table_free_rcu()
287 	table = (unsigned long *) (__pa(table) | (1U << bit));  in page_table_free_rcu()
288 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
294 	void *table = (void *)((unsigned long) _table ^ mask);  in __tlb_remove_table()  local
295 	struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT);  in __tlb_remove_table()
299 		free_pages((unsigned long) table, 2);  in __tlb_remove_table()
322 static void tlb_remove_table_one(void *table)  in tlb_remove_table_one()  argument
332 	__tlb_remove_table(table);  in tlb_remove_table_one()
358 void tlb_remove_table(struct mmu_gather *tlb, void *table)  in tlb_remove_table()  argument
368 			tlb_remove_table_one(table);  in tlb_remove_table()
373 	(*batch)->tables[(*batch)->nr++] = table;  in tlb_remove_table()
387 	u64 *table;  in base_pgt_alloc()  local
389 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
390 	if (table)  in base_pgt_alloc()
391 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
392 	return (unsigned long) table;  in base_pgt_alloc()
395 static void base_pgt_free(unsigned long table)  in base_pgt_free()  argument
397 	kmem_cache_free(base_pgt_cache, (void *) table);  in base_pgt_free()
402 	unsigned long table;  in base_crst_alloc()  local
404 	table =	 __get_free_pages(GFP_KERNEL, CRST_ALLOC_ORDER);  in base_crst_alloc()
405 	if (table)  in base_crst_alloc()
406 		crst_table_init((unsigned long *)table, val);  in base_crst_alloc()
407 	return table;  in base_crst_alloc()
410 static void base_crst_free(unsigned long table)  in base_crst_free()  argument
412 	free_pages(table, CRST_ALLOC_ORDER);  in base_crst_free()
459 	unsigned long *ste, next, table;  in base_segment_walk()  local
469 			table = base_pgt_alloc();  in base_segment_walk()
470 			if (!table)  in base_segment_walk()
472 			*ste = table | _SEGMENT_ENTRY;  in base_segment_walk()
474 		table = *ste & _SEGMENT_ENTRY_ORIGIN;  in base_segment_walk()
475 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
479 			base_pgt_free(table);  in base_segment_walk()
488 	unsigned long *rtte, next, table;  in base_region3_walk()  local
498 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
499 			if (!table)  in base_region3_walk()
501 			*rtte = table | _REGION3_ENTRY;  in base_region3_walk()
503 		table = *rtte & _REGION_ENTRY_ORIGIN;  in base_region3_walk()
504 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
508 			base_crst_free(table);  in base_region3_walk()
516 	unsigned long *rste, next, table;  in base_region2_walk()  local
526 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
527 			if (!table)  in base_region2_walk()
529 			*rste = table | _REGION2_ENTRY;  in base_region2_walk()
531 		table = *rste & _REGION_ENTRY_ORIGIN;  in base_region2_walk()
532 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
536 			base_crst_free(table);  in base_region2_walk()
544 	unsigned long *rfte, next, table;  in base_region1_walk()  local
554 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
555 			if (!table)  in base_region1_walk()
557 			*rfte = table | _REGION1_ENTRY;  in base_region1_walk()
559 		table = *rfte & _REGION_ENTRY_ORIGIN;  in base_region1_walk()
560 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
564 			base_crst_free(table);  in base_region1_walk()
578 	unsigned long table = asce & _ASCE_ORIGIN;  in base_asce_free()  local
584 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
587 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
590 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
593 		base_region1_walk(table, 0, -_PAGE_SIZE, 0);  in base_asce_free()
596 	base_crst_free(table);  in base_asce_free()
630 	unsigned long asce, table, end;  in base_asce_alloc()  local
637 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
638 		if (!table)  in base_asce_alloc()
640 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
641 		asce = table | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
643 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
644 		if (!table)  in base_asce_alloc()
646 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
647 		asce = table | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
649 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
650 		if (!table)  in base_asce_alloc()
652 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
653 		asce = table | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
655 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
656 		if (!table)  in base_asce_alloc()
658 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
659 		asce = table | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()