240 	struct gmap *sg, *next;  in gmap_remove()  local
246 		list_for_each_entry_safe(sg, next, &gmap->children, list) {  in gmap_remove()
247 			list_del(&sg->list);  in gmap_remove()
248 			gmap_put(sg);  in gmap_remove()
962  * Expected to be called with sg->mm->mmap_lock in read and
1008  * Expected to be called with sg->mm->mmap_lock in read
1044  * Called with sg->mm->mmap_lock in read.
1177  * @sg: pointer to the shadow guest address space structure
1181  * Called with the sg->guest_table_lock
1183 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,  in gmap_insert_rmap()  argument
1189 	BUG_ON(!gmap_is_shadow(sg));  in gmap_insert_rmap()
1190 	slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_insert_rmap()
1193 							&sg->guest_table_lock);  in gmap_insert_rmap()
1200 		radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);  in gmap_insert_rmap()
1203 		radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,  in gmap_insert_rmap()
1210  * @sg: pointer to the shadow guest address space structure
1218 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,  in gmap_protect_rmap()  argument
1228 	BUG_ON(!gmap_is_shadow(sg));  in gmap_protect_rmap()
1229 	parent = sg->parent;  in gmap_protect_rmap()
1246 			spin_lock(&sg->guest_table_lock);  in gmap_protect_rmap()
1250 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_protect_rmap()
1251 			spin_unlock(&sg->guest_table_lock);  in gmap_protect_rmap()
1294  * @sg: pointer to the shadow guest address space structure
1297  * Called with the sg->guest_table_lock
1299 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_page()  argument
1303 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_page()
1304 	table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */  in gmap_unshadow_page()
1307 	gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);  in gmap_unshadow_page()
1308 	ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);  in gmap_unshadow_page()
1313  * @sg: pointer to the shadow guest address space structure
1317  * Called with the sg->guest_table_lock
1319 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_pgt()  argument
1324 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_pgt()
1331  * @sg: pointer to the shadow guest address space structure
1334  * Called with the sg->guest_table_lock
1336 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_pgt()  argument
1341 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_pgt()
1342 	ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */  in gmap_unshadow_pgt()
1345 	gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);  in gmap_unshadow_pgt()
1350 	__gmap_unshadow_pgt(sg, raddr, pgt);  in gmap_unshadow_pgt()
1359  * @sg: pointer to the shadow guest address space structure
1363  * Called with the sg->guest_table_lock
1365 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_sgt()  argument
1372 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_sgt()
1378 		__gmap_unshadow_pgt(sg, raddr, pgt);  in __gmap_unshadow_sgt()
1388  * @sg: pointer to the shadow guest address space structure
1393 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_sgt()  argument
1398 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_sgt()
1399 	r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */  in gmap_unshadow_sgt()
1402 	gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);  in gmap_unshadow_sgt()
1407 	__gmap_unshadow_sgt(sg, raddr, sgt);  in gmap_unshadow_sgt()
1416  * @sg: pointer to the shadow guest address space structure
1420  * Called with the sg->guest_table_lock
1422 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r3t()  argument
1429 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r3t()
1435 		__gmap_unshadow_sgt(sg, raddr, sgt);  in __gmap_unshadow_r3t()
1445  * @sg: pointer to the shadow guest address space structure
1448  * Called with the sg->guest_table_lock
1450 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r3t()  argument
1455 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r3t()
1456 	r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */  in gmap_unshadow_r3t()
1459 	gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);  in gmap_unshadow_r3t()
1464 	__gmap_unshadow_r3t(sg, raddr, r3t);  in gmap_unshadow_r3t()
1473  * @sg: pointer to the shadow guest address space structure
1477  * Called with the sg->guest_table_lock
1479 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r2t()  argument
1486 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r2t()
1492 		__gmap_unshadow_r3t(sg, raddr, r3t);  in __gmap_unshadow_r2t()
1502  * @sg: pointer to the shadow guest address space structure
1505  * Called with the sg->guest_table_lock
1507 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r2t()  argument
1512 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r2t()
1513 	r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */  in gmap_unshadow_r2t()
1516 	gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);  in gmap_unshadow_r2t()
1521 	__gmap_unshadow_r2t(sg, raddr, r2t);  in gmap_unshadow_r2t()
1530  * @sg: pointer to the shadow guest address space structure
1536 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r1t()  argument
1543 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r1t()
1549 		__gmap_unshadow_r2t(sg, raddr, r2t);  in __gmap_unshadow_r1t()
1562  * @sg: pointer to the shadow guest address space structure
1564  * Called with sg->guest_table_lock
1566 static void gmap_unshadow(struct gmap *sg)  in gmap_unshadow()  argument
1570 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow()
1571 	if (sg->removed)  in gmap_unshadow()
1573 	sg->removed = 1;  in gmap_unshadow()
1574 	gmap_call_notifier(sg, 0, -1UL);  in gmap_unshadow()
1575 	gmap_flush_tlb(sg);  in gmap_unshadow()
1576 	table = (unsigned long *)(sg->asce & _ASCE_ORIGIN);  in gmap_unshadow()
1577 	switch (sg->asce & _ASCE_TYPE_MASK) {  in gmap_unshadow()
1579 		__gmap_unshadow_r1t(sg, 0, table);  in gmap_unshadow()
1582 		__gmap_unshadow_r2t(sg, 0, table);  in gmap_unshadow()
1585 		__gmap_unshadow_r3t(sg, 0, table);  in gmap_unshadow()
1588 		__gmap_unshadow_sgt(sg, 0, table);  in gmap_unshadow()
1606 	struct gmap *sg;  in gmap_find_shadow()  local
1608 	list_for_each_entry(sg, &parent->children, list) {  in gmap_find_shadow()
1609 		if (sg->orig_asce != asce || sg->edat_level != edat_level ||  in gmap_find_shadow()
1610 		    sg->removed)  in gmap_find_shadow()
1612 		if (!sg->initialized)  in gmap_find_shadow()
1614 		refcount_inc(&sg->ref_count);  in gmap_find_shadow()
1615 		return sg;  in gmap_find_shadow()
1623  * @sg: pointer to the shadow guest address space structure
1632 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)  in gmap_shadow_valid()  argument
1634 	if (sg->removed)  in gmap_shadow_valid()
1636 	return sg->orig_asce == asce && sg->edat_level == edat_level;  in gmap_shadow_valid()
1658 	struct gmap *sg, *new;  in gmap_shadow()  local
1665 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1667 	if (sg)  in gmap_shadow()
1668 		return sg;  in gmap_shadow()
1683 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1684 	if (sg) {  in gmap_shadow()
1687 		return sg;  in gmap_shadow()
1691 		list_for_each_entry(sg, &parent->children, list) {  in gmap_shadow()
1692 			if (sg->orig_asce & _ASCE_REAL_SPACE) {  in gmap_shadow()
1693 				spin_lock(&sg->guest_table_lock);  in gmap_shadow()
1694 				gmap_unshadow(sg);  in gmap_shadow()
1695 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow()
1696 				list_del(&sg->list);  in gmap_shadow()
1697 				gmap_put(sg);  in gmap_shadow()
1731  * @sg: pointer to the shadow guest address space structure
1745  * Called with sg->mm->mmap_lock in read.
1747 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,  in gmap_shadow_r2t()  argument
1755 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r2t()
1765 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1766 	table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */  in gmap_shadow_r2t()
1782 	if (sg->edat_level >= 1)  in gmap_shadow_r2t()
1784 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r2t()
1788 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1791 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1797 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r2t()
1798 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1800 		table = gmap_table_walk(sg, saddr, 4);  in gmap_shadow_r2t()
1807 		gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_r2t()
1809 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1812 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1820  * @sg: pointer to the shadow guest address space structure
1829  * Called with sg->mm->mmap_lock in read.
1831 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,  in gmap_shadow_r3t()  argument
1839 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r3t()
1849 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1850 	table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */  in gmap_shadow_r3t()
1866 	if (sg->edat_level >= 1)  in gmap_shadow_r3t()
1868 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r3t()
1872 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1875 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1881 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r3t()
1882 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1884 		table = gmap_table_walk(sg, saddr, 3);  in gmap_shadow_r3t()
1891 		gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_r3t()
1893 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1896 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1904  * @sg: pointer to the shadow guest address space structure
1913  * Called with sg->mm->mmap_lock in read.
1915 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,  in gmap_shadow_sgt()  argument
1923 	BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));  in gmap_shadow_sgt()
1933 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1934 	table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */  in gmap_shadow_sgt()
1950 	if (sg->edat_level >= 1)  in gmap_shadow_sgt()
1952 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_sgt()
1956 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1959 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1965 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_sgt()
1966 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1968 		table = gmap_table_walk(sg, saddr, 2);  in gmap_shadow_sgt()
1975 		gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_sgt()
1977 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1980 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1988  * @sg: pointer to the shadow guest address space structure
1997  * Called with sg->mm->mmap_lock in read.
1999 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,  in gmap_shadow_pgt_lookup()  argument
2007 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_pgt_lookup()
2008 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
2009 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt_lookup()
2020 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
2028  * @sg: pointer to the shadow guest address space structure
2039 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,  in gmap_shadow_pgt()  argument
2047 	BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE));  in gmap_shadow_pgt()
2049 	page = page_table_alloc_pgste(sg->mm);  in gmap_shadow_pgt()
2057 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2058 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt()
2073 	list_add(&page->lru, &sg->pt_list);  in gmap_shadow_pgt()
2077 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2080 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2084 	rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);  in gmap_shadow_pgt()
2085 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2087 		table = gmap_table_walk(sg, saddr, 1);  in gmap_shadow_pgt()
2094 		gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_pgt()
2096 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2099 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2108  * @sg: pointer to the shadow guest address space structure
2116  * Called with sg->mm->mmap_lock in read.
2118 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)  in gmap_shadow_page()  argument
2128 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_page()
2129 	parent = sg->parent;  in gmap_shadow_page()
2150 			spin_lock(&sg->guest_table_lock);  in gmap_shadow_page()
2152 			tptep = (pte_t *) gmap_table_walk(sg, saddr, 0);  in gmap_shadow_page()
2154 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2159 			rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);  in gmap_shadow_page()
2162 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_shadow_page()
2167 			spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2184  * Called with sg->parent->shadow_lock.
2186 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,  in gmap_shadow_notify()  argument
2192 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_notify()
2194 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_notify()
2195 	if (sg->removed) {  in gmap_shadow_notify()
2196 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2200 	start = sg->orig_asce & _ASCE_ORIGIN;  in gmap_shadow_notify()
2201 	end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;  in gmap_shadow_notify()
2202 	if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&  in gmap_shadow_notify()
2205 		gmap_unshadow(sg);  in gmap_shadow_notify()
2206 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2207 		list_del(&sg->list);  in gmap_shadow_notify()
2208 		gmap_put(sg);  in gmap_shadow_notify()
2212 	head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_shadow_notify()
2218 			gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_notify()
2221 			gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_notify()
2224 			gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_notify()
2227 			gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_notify()
2230 			gmap_unshadow_page(sg, raddr);  in gmap_shadow_notify()
2235 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2253 	struct gmap *gmap, *sg, *next;  in ptep_notify()  local
2270 			list_for_each_entry_safe(sg, next,  in ptep_notify()
2272 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()