3  *  Page table allocation functions
64 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
66 	free_pages((unsigned long)table, CRST_ALLOC_ORDER);  in crst_table_free()
160 	u64 *table;  in page_table_alloc_pgste()  local
164 		table = (u64 *)page_to_virt(page);  in page_table_alloc_pgste()
165 		memset64(table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_pgste()
166 		memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_pgste()
251 	unsigned long *table;  in page_table_alloc()  local
255 	/* Try to get a fragment of a 4K page as a 2K page table */  in page_table_alloc()
257 		table = NULL;  in page_table_alloc()
274 				table = (unsigned long *) page_to_virt(page);  in page_table_alloc()
277 					table += PTRS_PER_PTE;  in page_table_alloc()
284 		if (table)  in page_table_alloc()
285 			return table;  in page_table_alloc()
296 	/* Initialize page table */  in page_table_alloc()
297 	table = (unsigned long *) page_to_virt(page);  in page_table_alloc()
299 		/* Return 4K page table with PGSTEs */  in page_table_alloc()
301 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc()
302 		memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc()
306 		memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);  in page_table_alloc()
311 	return table;  in page_table_alloc()
314 static void page_table_release_check(struct page *page, void *table,  in page_table_release_check()  argument
323 		 table, half, mask);  in page_table_release_check()
327 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
332 	page = virt_to_page(table);  in page_table_free()
334 		/* Free 2K page table fragment of a 4K page */  in page_table_free()
335 		bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));  in page_table_free()
360 	page_table_release_check(page, table, half, mask);  in page_table_free()
365 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,  in page_table_free_rcu()  argument
373 	page = virt_to_page(table);  in page_table_free_rcu()
375 		gmap_unlink(mm, table, vmaddr);  in page_table_free_rcu()
376 		table = (unsigned long *) ((unsigned long)table | 0x03U);  in page_table_free_rcu()
377 		tlb_remove_table(tlb, table);  in page_table_free_rcu()
380 	bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));  in page_table_free_rcu()
394 	table = (unsigned long *) ((unsigned long) table | (0x01U << bit));  in page_table_free_rcu()
395 	tlb_remove_table(tlb, table);  in page_table_free_rcu()
401 	void *table = (void *)((unsigned long) _table ^ mask);  in __tlb_remove_table()  local
402 	struct page *page = virt_to_page(table);  in __tlb_remove_table()
406 		free_pages((unsigned long)table, CRST_ALLOC_ORDER);  in __tlb_remove_table()
408 	case 0x01U:	/* lower 2K of a 4K page table */  in __tlb_remove_table()
409 	case 0x02U:	/* higher 2K of a 4K page table */  in __tlb_remove_table()
415 	case 0x03U:	/* 4K page table with pgstes */  in __tlb_remove_table()
421 	page_table_release_check(page, table, half, mask);  in __tlb_remove_table()
435 	unsigned long *table;  in base_pgt_alloc()  local
437 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
438 	if (table)  in base_pgt_alloc()
439 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
440 	return table;  in base_pgt_alloc()
443 static void base_pgt_free(unsigned long *table)  in base_pgt_free()  argument
445 	kmem_cache_free(base_pgt_cache, table);  in base_pgt_free()
450 	unsigned long *table;  in base_crst_alloc()  local
452 	table =	(unsigned long *)__get_free_pages(GFP_KERNEL, CRST_ALLOC_ORDER);  in base_crst_alloc()
453 	if (table)  in base_crst_alloc()
454 		crst_table_init(table, val);  in base_crst_alloc()
455 	return table;  in base_crst_alloc()
458 static void base_crst_free(unsigned long *table)  in base_crst_free()  argument
460 	free_pages((unsigned long)table, CRST_ALLOC_ORDER);  in base_crst_free()
507 	unsigned long *ste, next, *table;  in base_segment_walk()  local
517 			table = base_pgt_alloc();  in base_segment_walk()
518 			if (!table)  in base_segment_walk()
520 			*ste = __pa(table) | _SEGMENT_ENTRY;  in base_segment_walk()
522 		table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);  in base_segment_walk()
523 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
527 			base_pgt_free(table);  in base_segment_walk()
536 	unsigned long *rtte, next, *table;  in base_region3_walk()  local
546 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
547 			if (!table)  in base_region3_walk()
549 			*rtte = __pa(table) | _REGION3_ENTRY;  in base_region3_walk()
551 		table = __va(*rtte & _REGION_ENTRY_ORIGIN);  in base_region3_walk()
552 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
556 			base_crst_free(table);  in base_region3_walk()
564 	unsigned long *rste, next, *table;  in base_region2_walk()  local
574 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
575 			if (!table)  in base_region2_walk()
577 			*rste = __pa(table) | _REGION2_ENTRY;  in base_region2_walk()
579 		table = __va(*rste & _REGION_ENTRY_ORIGIN);  in base_region2_walk()
580 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
584 			base_crst_free(table);  in base_region2_walk()
592 	unsigned long *rfte, next, *table;  in base_region1_walk()  local
602 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
603 			if (!table)  in base_region1_walk()
605 			*rfte = __pa(table) | _REGION1_ENTRY;  in base_region1_walk()
607 		table = __va(*rfte & _REGION_ENTRY_ORIGIN);  in base_region1_walk()
608 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
612 			base_crst_free(table);  in base_region1_walk()
626 	unsigned long *table = __va(asce & _ASCE_ORIGIN);  in base_asce_free()  local
632 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
635 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
638 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
641 		base_region1_walk(table, 0, TASK_SIZE_MAX, 0);  in base_asce_free()
644 	base_crst_free(table);  in base_asce_free()
678 	unsigned long asce, *table, end;  in base_asce_alloc()  local
685 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
686 		if (!table)  in base_asce_alloc()
688 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
689 		asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
691 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
692 		if (!table)  in base_asce_alloc()
694 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
695 		asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
697 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
698 		if (!table)  in base_asce_alloc()
700 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
701 		asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
703 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
704 		if (!table)  in base_asce_alloc()
706 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
707 		asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()