89 static inline bool is_table_desc(uint64_t desc, unsigned int level)  in is_table_desc()  argument
91 	return level != XLAT_LAST_LEVEL &&  in is_table_desc()
108 				    unsigned int level)  in is_desc_superset()  argument
110 	uint64_t mask = DESC_ATTRS_MASK | GENMASK(47, LEVEL_TO_VA_SIZE_SHIFT(level));  in is_desc_superset()
116 static void debug_show_pte(uint64_t *pte, unsigned int level)  in debug_show_pte()  argument
118 	MMU_DEBUG("%.*s", level * 2U, ". . . ");  in debug_show_pte()
126 	if (is_table_desc(*pte, level)) {  in debug_show_pte()
151 static inline void debug_show_pte(uint64_t *pte, unsigned int level) { }  in debug_show_pte()  argument
154 static void set_pte_table_desc(uint64_t *pte, uint64_t *table, unsigned int level)  in set_pte_table_desc()  argument
158 	debug_show_pte(pte, level);  in set_pte_table_desc()
161 static void set_pte_block_desc(uint64_t *pte, uint64_t desc, unsigned int level)  in set_pte_block_desc()  argument
164 		desc |= (level == XLAT_LAST_LEVEL) ? PTE_PAGE_DESC : PTE_BLOCK_DESC;  in set_pte_block_desc()
167 	debug_show_pte(pte, level);  in set_pte_block_desc()
170 static uint64_t *expand_to_table(uint64_t *pte, unsigned int level)  in expand_to_table()  argument
174 	__ASSERT(level < XLAT_LAST_LEVEL, "can't expand last level");  in expand_to_table()
193 		if (level + 1 == XLAT_LAST_LEVEL) {  in expand_to_table()
197 		stride_shift = LEVEL_TO_VA_SIZE_SHIFT(level + 1);  in expand_to_table()
211 	set_pte_table_desc(pte, table, level);  in expand_to_table()
224 	unsigned int level = BASE_XLAT_LEVEL;  in set_mapping()  local
228 		__ASSERT(level <= XLAT_LAST_LEVEL,  in set_mapping()
232 		pte = &table[XLAT_TABLE_VA_IDX(virt, level)];  in set_mapping()
233 		ptes[level] = pte;  in set_mapping()
235 		if (is_table_desc(*pte, level)) {  in set_mapping()
237 			level++;  in set_mapping()
246 				level, pte, *pte);  in set_mapping()
251 		level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level);  in set_mapping()
253 		if (is_desc_superset(*pte, desc, level)) {  in set_mapping()
264 			table = expand_to_table(pte, level);  in set_mapping()
269 			level++;  in set_mapping()
281 		set_pte_block_desc(pte, desc, level);  in set_mapping()
284 		while (level != BASE_XLAT_LEVEL &&  in set_mapping()
287 			pte = ptes[--level];  in set_mapping()
288 			set_pte_block_desc(pte, 0, level);  in set_mapping()
299 		level = BASE_XLAT_LEVEL;  in set_mapping()
307 static uint64_t *dup_table(uint64_t *src_table, unsigned int level)  in dup_table()  argument
316 	MMU_DEBUG("dup (level %d) [%d]%p to [%d]%p\n", level,  in dup_table()
322 		if (is_table_desc(src_table[i], level)) {  in dup_table()
334 			   uintptr_t virt, size_t size, unsigned int level)  in privatize_table()  argument
336 	size_t step, level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level);  in privatize_table()
345 		i = XLAT_TABLE_VA_IDX(virt, level);  in privatize_table()
347 		if (!is_table_desc(dst_table[i], level) ||  in privatize_table()
348 		    !is_table_desc(src_table[i], level)) {  in privatize_table()
358 			dst_subtable = dup_table(src_subtable, level + 1);  in privatize_table()
362 			set_pte_table_desc(&dst_table[i], dst_subtable, level);  in privatize_table()
368 				      virt, step, level + 1);  in privatize_table()
405 static void discard_table(uint64_t *table, unsigned int level)  in discard_table()  argument
410 		if (is_table_desc(table[i], level)) {  in discard_table()
412 			discard_table(pte_desc_table(table[i]), level + 1);  in discard_table()
423 			   uintptr_t virt, size_t size, unsigned int level)  in globalize_table()  argument
425 	size_t step, level_size = 1ULL << LEVEL_TO_VA_SIZE_SHIFT(level);  in globalize_table()
434 		i = XLAT_TABLE_VA_IDX(virt, level);  in globalize_table()
443 			__ASSERT(is_table_desc(src_table[i], level),  in globalize_table()
445 			if (!is_table_desc(dst_table[i], level)) {  in globalize_table()
447 				if (!expand_to_table(&dst_table[i], level)) {  in globalize_table()
453 					      virt, step, level + 1);  in globalize_table()
462 		uint64_t *old_table = is_table_desc(dst_table[i], level) ?  in globalize_table()
466 		debug_show_pte(&dst_table[i], level);  in globalize_table()
467 		if (is_table_desc(src_table[i], level)) {  in globalize_table()
474 			discard_table(old_table, level + 1);  in globalize_table()