61 static inline unsigned int table_index(uint64_t *pte)  in table_index()  argument
63 	unsigned int i = (pte - xlat_tables) / Ln_XLAT_NUM_ENTRIES;  in table_index()
65 	__ASSERT(i < CONFIG_MAX_XLAT_TABLES, "table %p out of range", pte);  in table_index()
194 static void debug_show_pte(uint64_t *pte, unsigned int level)  in debug_show_pte()  argument
197 	MMU_DEBUG("[%d]%p: ", table_index(pte), pte);  in debug_show_pte()
199 	if (is_free_desc(*pte)) {  in debug_show_pte()
204 	MMU_DEBUG("0x%016llx ", *pte);  in debug_show_pte()
206 	if (is_table_desc(*pte, level)) {  in debug_show_pte()
207 		uint64_t *table = pte_desc_table(*pte);  in debug_show_pte()
213 	if (is_block_desc(*pte)) {  in debug_show_pte()
215 	} else if (!is_inval_desc(*pte)) {  in debug_show_pte()
221 	uint8_t mem_type = (*pte >> 2) & MT_TYPE_MASK;  in debug_show_pte()
225 	MMU_DEBUG((*pte & PTE_BLOCK_DESC_AP_RO) ? "-RO" : "-RW");  in debug_show_pte()
226 	MMU_DEBUG((*pte & PTE_BLOCK_DESC_NS) ? "-NS" : "-S");  in debug_show_pte()
227 	MMU_DEBUG((*pte & PTE_BLOCK_DESC_AP_ELx) ? "-ELx" : "-ELh");  in debug_show_pte()
228 	MMU_DEBUG((*pte & PTE_BLOCK_DESC_PXN) ? "-PXN" : "-PX");  in debug_show_pte()
229 	MMU_DEBUG((*pte & PTE_BLOCK_DESC_UXN) ? "-UXN" : "-UX");  in debug_show_pte()
230 	MMU_DEBUG((*pte & PTE_SW_WRITABLE) ? "-WRITABLE" : "");  in debug_show_pte()
234 static inline void debug_show_pte(uint64_t *pte, unsigned int level) { }  in debug_show_pte()  argument
237 static void set_pte_table_desc(uint64_t *pte, uint64_t *table, unsigned int level)  in set_pte_table_desc()  argument
240 	*pte = PTE_TABLE_DESC | (uint64_t)table;  in set_pte_table_desc()
241 	debug_show_pte(pte, level);  in set_pte_table_desc()
244 static void set_pte_block_desc(uint64_t *pte, uint64_t desc, unsigned int level)  in set_pte_block_desc()  argument
256 	*pte = desc;  in set_pte_block_desc()
257 	debug_show_pte(pte, level);  in set_pte_block_desc()
260 static uint64_t *expand_to_table(uint64_t *pte, unsigned int level)  in expand_to_table()  argument
271 	if (!is_free_desc(*pte)) {  in expand_to_table()
276 		uint64_t desc = *pte;  in expand_to_table()
297 		table_usage(pte, 1);  in expand_to_table()
301 	set_pte_table_desc(pte, table, level);  in expand_to_table()
310 	uint64_t *pte;  in set_mapping()  local
319 		pte = &table[XLAT_TABLE_VA_IDX(virt, level)];  in set_mapping()
321 		if (is_table_desc(*pte, level)) {  in set_mapping()
324 			table = pte_desc_table(*pte);  in set_mapping()
328 		if (!may_overwrite && !is_free_desc(*pte)) {  in set_mapping()
332 				level, pte, *pte);  in set_mapping()
338 		if (is_desc_superset(*pte, desc, level)) {  in set_mapping()
350 			table = expand_to_table(pte, level);  in set_mapping()
359 		if (is_free_desc(*pte)) {  in set_mapping()
360 			table_usage(pte, 1);  in set_mapping()
363 		set_pte_block_desc(pte, desc, level);  in set_mapping()
382 	uint64_t *pte, *subtable;  in del_mapping()  local
389 		pte = &table[XLAT_TABLE_VA_IDX(virt, level)];  in del_mapping()
391 		if (is_free_desc(*pte)) {  in del_mapping()
395 		if (step != level_size && is_block_desc(*pte)) {  in del_mapping()
397 			expand_to_table(pte, level);  in del_mapping()
400 		if (is_table_desc(*pte, level)) {  in del_mapping()
401 			subtable = pte_desc_table(*pte);  in del_mapping()
410 		*pte = 0;  in del_mapping()
411 		table_usage(pte, -1);  in del_mapping()
1395 	uint64_t *pte;  in get_pte_location()  local
1400 		pte = &table[XLAT_TABLE_VA_IDX(virt, level)];  in get_pte_location()
1402 			return pte;  in get_pte_location()
1405 		if (is_table_desc(*pte, level)) {  in get_pte_location()
1407 			table = pte_desc_table(*pte);  in get_pte_location()
1419 	uint64_t *pte = get_pte_location(&kernel_ptables, virt);  in arch_mem_page_out()  local
1422 	__ASSERT(pte != NULL, "");  in arch_mem_page_out()
1423 	desc = *pte;  in arch_mem_page_out()
1440 	*pte = desc;  in arch_mem_page_out()
1442 	debug_show_pte(pte, XLAT_LAST_LEVEL);  in arch_mem_page_out()
1451 	uint64_t *pte = get_pte_location(&kernel_ptables, virt);  in arch_mem_page_in()  local
1456 	__ASSERT(pte != NULL, "");  in arch_mem_page_in()
1457 	desc = *pte;  in arch_mem_page_in()
1474 	*pte = desc;  in arch_mem_page_in()
1476 	debug_show_pte(pte, XLAT_LAST_LEVEL);  in arch_mem_page_in()
1485 	uint64_t *pte = get_pte_location(&kernel_ptables, virt);  in arch_page_location_get()  local
1489 	if (!pte) {  in arch_page_location_get()
1492 	desc = *pte;  in arch_page_location_get()
1515 	uint64_t *pte = get_pte_location(&kernel_ptables, virt);  in arch_page_info_get()  local
1519 	if (!pte) {  in arch_page_info_get()
1522 	desc = *pte;  in arch_page_info_get()
1556 		*pte = desc;  in arch_page_info_get()
1558 		debug_show_pte(pte, XLAT_LAST_LEVEL);  in arch_page_info_get()
1602 	uint64_t *pte, desc;  in z_arm64_do_demand_paging()  local
1623 	pte = get_pte_location(&kernel_ptables, virt);  in z_arm64_do_demand_paging()
1624 	if (!pte) {  in z_arm64_do_demand_paging()
1628 	desc = *pte;  in z_arm64_do_demand_paging()
1668 		*pte = desc;  in z_arm64_do_demand_paging()
1683 		*pte = desc;  in z_arm64_do_demand_paging()