39 	unsigned long *table;  in gmap_alloc()  local
76 	table = (unsigned long *) page_to_phys(page);  in gmap_alloc()
77 	crst_table_init(table, etype);  in gmap_alloc()
78 	gmap->table = table;  in gmap_alloc()
80 		_ASCE_USER_BITS | __pa(table);  in gmap_alloc()
305 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,  in gmap_alloc_table()  argument
311 	/* since we dont free the gmap table until gmap_free we can unlock */  in gmap_alloc_table()
318 	if (*table & _REGION_ENTRY_INVALID) {  in gmap_alloc_table()
320 		*table = (unsigned long) new | _REGION_ENTRY_LENGTH |  in gmap_alloc_table()
321 			(*table & _REGION_ENTRY_TYPE_MASK);  in gmap_alloc_table()
333  * @entry: pointer to a segment table entry in the guest address space
468  * This function does not establish potentially missing page table entries.
492  * This function does not establish potentially missing page table entries.
506  * gmap_unlink - disconnect a page table from the gmap shadow tables
508  * @table: pointer to the host page table
509  * @vmaddr: vm address associated with the host page table
511 void gmap_unlink(struct mm_struct *mm, unsigned long *table,  in gmap_unlink()  argument
543 	unsigned long *table;  in __gmap_link()  local
553 	/* Create higher level tables in the gmap page table */  in __gmap_link()
554 	table = gmap->table;  in __gmap_link()
556 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in __gmap_link()
557 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
558 		    gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,  in __gmap_link()
561 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
564 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in __gmap_link()
565 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
566 		    gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,  in __gmap_link()
569 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
572 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in __gmap_link()
573 		if ((*table & _REGION_ENTRY_INVALID) &&  in __gmap_link()
574 		    gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,  in __gmap_link()
577 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in __gmap_link()
579 	table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in __gmap_link()
580 	/* Walk the parent mm page table */  in __gmap_link()
596 	/* Link gmap segment table entry location to page table. */  in __gmap_link()
602 	if (*table == _SEGMENT_ENTRY_EMPTY) {  in __gmap_link()
604 				       vmaddr >> PMD_SHIFT, table);  in __gmap_link()
607 				*table = (pmd_val(*pmd) &  in __gmap_link()
611 				*table = pmd_val(*pmd) &  in __gmap_link()
614 	} else if (*table & _SEGMENT_ENTRY_PROTECT &&  in __gmap_link()
616 		unprot = (u64)*table;  in __gmap_link()
619 		gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);  in __gmap_link()
685 		/* Get pointer to the page table entry */  in __gmap_zap()
773  * @level: page table level to stop at
775  * Returns a table entry pointer for the given guest address and @level
776  * @level=0 : returns a pointer to a page table table entry (or NULL)
777  * @level=1 : returns a pointer to a segment table entry (or NULL)
778  * @level=2 : returns a pointer to a region-3 table entry (or NULL)
779  * @level=3 : returns a pointer to a region-2 table entry (or NULL)
780  * @level=4 : returns a pointer to a region-1 table entry (or NULL)
790 	unsigned long *table;  in gmap_table_walk()  local
798 	table = gmap->table;  in gmap_table_walk()
801 		table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;  in gmap_table_walk()
804 		if (*table & _REGION_ENTRY_INVALID)  in gmap_table_walk()
806 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in gmap_table_walk()
809 		table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;  in gmap_table_walk()
812 		if (*table & _REGION_ENTRY_INVALID)  in gmap_table_walk()
814 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in gmap_table_walk()
817 		table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;  in gmap_table_walk()
820 		if (*table & _REGION_ENTRY_INVALID)  in gmap_table_walk()
822 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);  in gmap_table_walk()
825 		table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;  in gmap_table_walk()
828 		if (*table & _REGION_ENTRY_INVALID)  in gmap_table_walk()
830 		table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);  in gmap_table_walk()
831 		table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;  in gmap_table_walk()
833 	return table;  in gmap_table_walk()
837  * gmap_pte_op_walk - walk the gmap page table, get the page table lock
848 	unsigned long *table;  in gmap_pte_op_walk()  local
851 	/* Walk the gmap page table, lock and get pte pointer */  in gmap_pte_op_walk()
852 	table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */  in gmap_pte_op_walk()
853 	if (!table || *table & _SEGMENT_ENTRY_INVALID)  in gmap_pte_op_walk()
855 	return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);  in gmap_pte_op_walk()
859  * gmap_pte_op_fixup - force a page in and connect the gmap page table
867  * up or connecting the gmap page table.
888  * gmap_pte_op_end - release the page table lock
898  * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
914 	/* without huge pages, there is no need to take the table lock */  in gmap_pmd_op_walk()
924 	/* 4k page table entries are locked via the pte (pte_alloc_map_lock). */  in gmap_pmd_op_walk()
1094  * This function establishes missing page table entries.
1113  * gmap_read_table - get an unsigned long value from a guest page table using
1259  * gmap_idte_one - invalidate a single region or segment table entry
1260  * @asce: region or segment table *origin* + table-type bits
1261  * @vaddr: virtual address to identify the table entry to flush
1263  * The invalid bit of a single region or segment table entry is set
1265  * The table-type of the @asce identifies the portion of the @vaddr
1276  * gmap_unshadow_page - remove a page from a shadow page table
1284 	unsigned long *table;  in gmap_unshadow_page()  local
1287 	table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */  in gmap_unshadow_page()
1288 	if (!table || *table & _PAGE_INVALID)  in gmap_unshadow_page()
1291 	ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);  in gmap_unshadow_page()
1295  * __gmap_unshadow_pgt - remove all entries from a shadow page table
1298  * @pgt: pointer to the start of a shadow page table
1313  * gmap_unshadow_pgt - remove a shadow page table from a segment entry
1334 	/* Free page table */  in gmap_unshadow_pgt()
1341  * __gmap_unshadow_sgt - remove all entries from a shadow segment table
1344  * @sgt: pointer to the start of a shadow segment table
1362 		/* Free page table */  in __gmap_unshadow_sgt()
1370  * gmap_unshadow_sgt - remove a shadow segment table from a region-3 entry
1391 	/* Free segment table */  in gmap_unshadow_sgt()
1398  * __gmap_unshadow_r3t - remove all entries from a shadow region-3 table
1401  * @r3t: pointer to the start of a shadow region-3 table
1419 		/* Free segment table */  in __gmap_unshadow_r3t()
1427  * gmap_unshadow_r3t - remove a shadow region-3 table from a region-2 entry
1448 	/* Free region 3 table */  in gmap_unshadow_r3t()
1455  * __gmap_unshadow_r2t - remove all entries from a shadow region-2 table
1458  * @r2t: pointer to the start of a shadow region-2 table
1476 		/* Free region 3 table */  in __gmap_unshadow_r2t()
1484  * gmap_unshadow_r2t - remove a shadow region-2 table from a region-1 entry
1505 	/* Free region 2 table */  in gmap_unshadow_r2t()
1512  * __gmap_unshadow_r1t - remove all entries from a shadow region-1 table
1515  * @r1t: pointer to the start of a shadow region-1 table
1536 		/* Free region 2 table */  in __gmap_unshadow_r1t()
1544  * gmap_unshadow - remove a shadow page table completely
1551 	unsigned long *table;  in gmap_unshadow()  local
1559 	table = (unsigned long *)(sg->asce & _ASCE_ORIGIN);  in gmap_unshadow()
1562 		__gmap_unshadow_r1t(sg, 0, table);  in gmap_unshadow()
1565 		__gmap_unshadow_r2t(sg, 0, table);  in gmap_unshadow()
1568 		__gmap_unshadow_r3t(sg, 0, table);  in gmap_unshadow()
1571 		__gmap_unshadow_sgt(sg, 0, table);  in gmap_unshadow()
1579  * @asce: ASCE for which the shadow table is created
1582  * Returns the pointer to a gmap if a shadow table with the given asce is
1607  * @asce: ASCE for which the shadow table is requested
1626  * @asce: ASCE for which the shadow table is created
1629  * The pages of the top level page table referred by the asce parameter
1631  * The shadow table will be removed automatically on any change to the
1632  * PTE mapping for the source table.
1636  * parent gmap table could not be protected.
1713  * gmap_shadow_r2t - create an empty shadow region 2 table
1716  * @r2t: parent gmap address of the region 2 table to get shadowed
1719  * The r2t parameter specifies the address of the source table. The
1720  * four pages of the source table are made read-only in the parent gmap
1721  * address space. A write to the source table area @r2t will automatically
1722  * remove the shadow r2 table and all of its decendents.
1725  * shadow table structure is incomplete, -ENOMEM if out of memory and
1734 	unsigned long *s_r2t, *table;  in gmap_shadow_r2t()  local
1739 	/* Allocate a shadow region second table */  in gmap_shadow_r2t()
1747 	/* Install shadow region second table */  in gmap_shadow_r2t()
1749 	table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */  in gmap_shadow_r2t()
1750 	if (!table) {  in gmap_shadow_r2t()
1754 	if (!(*table & _REGION_ENTRY_INVALID)) {  in gmap_shadow_r2t()
1757 	} else if (*table & _REGION_ENTRY_ORIGIN) {  in gmap_shadow_r2t()
1762 	/* mark as invalid as long as the parent table is not protected */  in gmap_shadow_r2t()
1763 	*table = (unsigned long) s_r2t | _REGION_ENTRY_LENGTH |  in gmap_shadow_r2t()
1766 		*table |= (r2t & _REGION_ENTRY_PROTECT);  in gmap_shadow_r2t()
1770 		*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_r2t()
1775 	/* Make r2t read-only in parent gmap page table */  in gmap_shadow_r2t()
1783 		table = gmap_table_walk(sg, saddr, 4);  in gmap_shadow_r2t()
1784 		if (!table || (*table & _REGION_ENTRY_ORIGIN) !=  in gmap_shadow_r2t()
1788 			*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_r2t()
1802  * gmap_shadow_r3t - create a shadow region 3 table
1805  * @r3t: parent gmap address of the region 3 table to get shadowed
1809  * shadow table structure is incomplete, -ENOMEM if out of memory and
1818 	unsigned long *s_r3t, *table;  in gmap_shadow_r3t()  local
1823 	/* Allocate a shadow region second table */  in gmap_shadow_r3t()
1831 	/* Install shadow region second table */  in gmap_shadow_r3t()
1833 	table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */  in gmap_shadow_r3t()
1834 	if (!table) {  in gmap_shadow_r3t()
1838 	if (!(*table & _REGION_ENTRY_INVALID)) {  in gmap_shadow_r3t()
1841 	} else if (*table & _REGION_ENTRY_ORIGIN) {  in gmap_shadow_r3t()
1845 	/* mark as invalid as long as the parent table is not protected */  in gmap_shadow_r3t()
1846 	*table = (unsigned long) s_r3t | _REGION_ENTRY_LENGTH |  in gmap_shadow_r3t()
1849 		*table |= (r3t & _REGION_ENTRY_PROTECT);  in gmap_shadow_r3t()
1853 		*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_r3t()
1858 	/* Make r3t read-only in parent gmap page table */  in gmap_shadow_r3t()
1866 		table = gmap_table_walk(sg, saddr, 3);  in gmap_shadow_r3t()
1867 		if (!table || (*table & _REGION_ENTRY_ORIGIN) !=  in gmap_shadow_r3t()
1871 			*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_r3t()
1885  * gmap_shadow_sgt - create a shadow segment table
1888  * @sgt: parent gmap address of the segment table to get shadowed
1892  * shadow table structure is incomplete, -ENOMEM if out of memory and
1901 	unsigned long *s_sgt, *table;  in gmap_shadow_sgt()  local
1906 	/* Allocate a shadow segment table */  in gmap_shadow_sgt()
1914 	/* Install shadow region second table */  in gmap_shadow_sgt()
1916 	table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */  in gmap_shadow_sgt()
1917 	if (!table) {  in gmap_shadow_sgt()
1921 	if (!(*table & _REGION_ENTRY_INVALID)) {  in gmap_shadow_sgt()
1924 	} else if (*table & _REGION_ENTRY_ORIGIN) {  in gmap_shadow_sgt()
1929 	/* mark as invalid as long as the parent table is not protected */  in gmap_shadow_sgt()
1930 	*table = (unsigned long) s_sgt | _REGION_ENTRY_LENGTH |  in gmap_shadow_sgt()
1933 		*table |= sgt & _REGION_ENTRY_PROTECT;  in gmap_shadow_sgt()
1937 		*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_sgt()
1942 	/* Make sgt read-only in parent gmap page table */  in gmap_shadow_sgt()
1950 		table = gmap_table_walk(sg, saddr, 2);  in gmap_shadow_sgt()
1951 		if (!table || (*table & _REGION_ENTRY_ORIGIN) !=  in gmap_shadow_sgt()
1955 			*table &= ~_REGION_ENTRY_INVALID;  in gmap_shadow_sgt()
1969  * gmap_shadow_lookup_pgtable - find a shadow page table
1972  * @pgt: parent gmap address of the page table to get shadowed
1976  * Returns 0 if the shadow page table was found and -EAGAIN if the page
1977  * table was not found.
1985 	unsigned long *table;  in gmap_shadow_pgt_lookup()  local
1991 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt_lookup()
1992 	if (table && !(*table & _SEGMENT_ENTRY_INVALID)) {  in gmap_shadow_pgt_lookup()
1994 		page = pfn_to_page(*table >> PAGE_SHIFT);  in gmap_shadow_pgt_lookup()
1996 		*dat_protection = !!(*table & _SEGMENT_ENTRY_PROTECT);  in gmap_shadow_pgt_lookup()
2009  * gmap_shadow_pgt - instantiate a shadow page table
2012  * @pgt: parent gmap address of the page table to get shadowed
2016  * shadow table structure is incomplete, -ENOMEM if out of memory,
2025 	unsigned long *s_pgt, *table;  in gmap_shadow_pgt()  local
2030 	/* Allocate a shadow page table */  in gmap_shadow_pgt()
2038 	/* Install shadow page table */  in gmap_shadow_pgt()
2040 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt()
2041 	if (!table) {  in gmap_shadow_pgt()
2045 	if (!(*table & _SEGMENT_ENTRY_INVALID)) {  in gmap_shadow_pgt()
2048 	} else if (*table & _SEGMENT_ENTRY_ORIGIN) {  in gmap_shadow_pgt()
2052 	/* mark as invalid as long as the parent table is not protected */  in gmap_shadow_pgt()
2053 	*table = (unsigned long) s_pgt | _SEGMENT_ENTRY |  in gmap_shadow_pgt()
2058 		*table &= ~_SEGMENT_ENTRY_INVALID;  in gmap_shadow_pgt()
2063 	/* Make pgt read-only in parent gmap page table (not the pgste) */  in gmap_shadow_pgt()
2069 		table = gmap_table_walk(sg, saddr, 1);  in gmap_shadow_pgt()
2070 		if (!table || (*table & _SEGMENT_ENTRY_ORIGIN) !=  in gmap_shadow_pgt()
2074 			*table &= ~_SEGMENT_ENTRY_INVALID;  in gmap_shadow_pgt()
2095  * shadow table structure is incomplete, -ENOMEM if out of memory and
2133 			/* Get page table pointer */  in gmap_shadow_page()
2181 	/* Check for top level table */  in gmap_shadow_notify()
2186 		/* The complete shadow table has to go */  in gmap_shadow_notify()
2193 	/* Remove the page table tree from on specific entry */  in gmap_shadow_notify()
2224  * @pte: pointer to the page table entry
2227  * This function is assumed to be called with the page table lock held
2234 	unsigned long *table;  in ptep_notify()  local
2242 		table = radix_tree_lookup(&gmap->host_to_guest,  in ptep_notify()
2244 		if (table)  in ptep_notify()
2245 			gaddr = __gmap_segment_gaddr(table) + offset;  in ptep_notify()
2247 		if (!table)  in ptep_notify()