223 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)  in psb_mmu_free_pt()  argument
225 	__free_page(pt->p);  in psb_mmu_free_pt()
226 	kfree(pt);  in psb_mmu_free_pt()
234 	struct psb_mmu_pt *pt;  in psb_mmu_free_pagedir()  local
247 		pt = pd->tables[i];  in psb_mmu_free_pagedir()
248 		if (pt)  in psb_mmu_free_pagedir()
249 			psb_mmu_free_pt(pt);  in psb_mmu_free_pagedir()
262 	struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);  in psb_mmu_alloc_pt()  local
271 	if (!pt)  in psb_mmu_alloc_pt()
274 	pt->p = alloc_page(GFP_DMA32);  in psb_mmu_alloc_pt()
275 	if (!pt->p) {  in psb_mmu_alloc_pt()
276 		kfree(pt);  in psb_mmu_alloc_pt()
282 	v = kmap_atomic(pt->p);  in psb_mmu_alloc_pt()
299 	pt->count = 0;  in psb_mmu_alloc_pt()
300 	pt->pd = pd;  in psb_mmu_alloc_pt()
301 	pt->index = 0;  in psb_mmu_alloc_pt()
303 	return pt;  in psb_mmu_alloc_pt()
310 	struct psb_mmu_pt *pt;  in psb_mmu_pt_alloc_map_lock()  local
315 	pt = pd->tables[index];  in psb_mmu_pt_alloc_map_lock()
316 	while (!pt) {  in psb_mmu_pt_alloc_map_lock()
318 		pt = psb_mmu_alloc_pt(pd);  in psb_mmu_pt_alloc_map_lock()
319 		if (!pt)  in psb_mmu_pt_alloc_map_lock()
325 			psb_mmu_free_pt(pt);  in psb_mmu_pt_alloc_map_lock()
327 			pt = pd->tables[index];  in psb_mmu_pt_alloc_map_lock()
332 		pd->tables[index] = pt;  in psb_mmu_pt_alloc_map_lock()
333 		v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;  in psb_mmu_pt_alloc_map_lock()
334 		pt->index = index;  in psb_mmu_pt_alloc_map_lock()
342 	pt->v = kmap_atomic(pt->p);  in psb_mmu_pt_alloc_map_lock()
343 	return pt;  in psb_mmu_pt_alloc_map_lock()
350 	struct psb_mmu_pt *pt;  in psb_mmu_pt_map_lock()  local
354 	pt = pd->tables[index];  in psb_mmu_pt_map_lock()
355 	if (!pt) {  in psb_mmu_pt_map_lock()
359 	pt->v = kmap_atomic(pt->p);  in psb_mmu_pt_map_lock()
360 	return pt;  in psb_mmu_pt_map_lock()
363 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)  in psb_mmu_pt_unmap_unlock()  argument
365 	struct psb_mmu_pd *pd = pt->pd;  in psb_mmu_pt_unmap_unlock()
368 	kunmap_atomic(pt->v);  in psb_mmu_pt_unmap_unlock()
369 	if (pt->count == 0) {  in psb_mmu_pt_unmap_unlock()
371 		v[pt->index] = pd->invalid_pde;  in psb_mmu_pt_unmap_unlock()
372 		pd->tables[pt->index] = NULL;  in psb_mmu_pt_unmap_unlock()
375 			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);  in psb_mmu_pt_unmap_unlock()
380 		psb_mmu_free_pt(pt);  in psb_mmu_pt_unmap_unlock()
386 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,  in psb_mmu_set_pte()  argument
389 	pt->v[psb_mmu_pt_index(addr)] = pte;  in psb_mmu_set_pte()
392 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,  in psb_mmu_invalidate_pte()  argument
395 	pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;  in psb_mmu_invalidate_pte()
481 	struct psb_mmu_pt *pt;  in psb_mmu_flush_ptes()  local
510 			pt = psb_mmu_pt_map_lock(pd, addr);  in psb_mmu_flush_ptes()
511 			if (!pt)  in psb_mmu_flush_ptes()
514 				psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);  in psb_mmu_flush_ptes()
518 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_flush_ptes()
528 	struct psb_mmu_pt *pt;  in psb_mmu_remove_pfn_sequence()  local
541 		pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_remove_pfn_sequence()
542 		if (!pt)  in psb_mmu_remove_pfn_sequence()
545 			psb_mmu_invalidate_pte(pt, addr);  in psb_mmu_remove_pfn_sequence()
546 			--pt->count;  in psb_mmu_remove_pfn_sequence()
548 		psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_remove_pfn_sequence()
568 	struct psb_mmu_pt *pt;  in psb_mmu_remove_pages()  local
597 			pt = psb_mmu_pt_map_lock(pd, addr);  in psb_mmu_remove_pages()
598 			if (!pt)  in psb_mmu_remove_pages()
601 				psb_mmu_invalidate_pte(pt, addr);  in psb_mmu_remove_pages()
602 				--pt->count;  in psb_mmu_remove_pages()
605 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_remove_pages()
624 	struct psb_mmu_pt *pt;  in psb_mmu_insert_pfn_sequence()  local
639 		pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_insert_pfn_sequence()
640 		if (!pt) {  in psb_mmu_insert_pfn_sequence()
646 			psb_mmu_set_pte(pt, addr, pte);  in psb_mmu_insert_pfn_sequence()
647 			pt->count++;  in psb_mmu_insert_pfn_sequence()
649 		psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_insert_pfn_sequence()
671 	struct psb_mmu_pt *pt;  in psb_mmu_insert_pages()  local
703 			pt = psb_mmu_pt_alloc_map_lock(pd, addr);  in psb_mmu_insert_pages()
704 			if (!pt)  in psb_mmu_insert_pages()
709 				psb_mmu_set_pte(pt, addr, pte);  in psb_mmu_insert_pages()
710 				pt->count++;  in psb_mmu_insert_pages()
712 			psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_insert_pages()
737 	struct psb_mmu_pt *pt;  in psb_mmu_virtual_to_pfn()  local
742 	pt = psb_mmu_pt_map_lock(pd, virtual);  in psb_mmu_virtual_to_pfn()
743 	if (!pt) {  in psb_mmu_virtual_to_pfn()
761 	tmp = pt->v[psb_mmu_pt_index(virtual)];  in psb_mmu_virtual_to_pfn()
768 	psb_mmu_pt_unmap_unlock(pt);  in psb_mmu_virtual_to_pfn()