190 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
219 extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
280 void arch_tlb_gather_mmu(struct mmu_gather *tlb,
282 void tlb_flush_mmu(struct mmu_gather *tlb);
283 void arch_tlb_finish_mmu(struct mmu_gather *tlb,
286 static inline void __tlb_adjust_range(struct mmu_gather *tlb,  in __tlb_adjust_range()  argument
290 	tlb->start = min(tlb->start, address);  in __tlb_adjust_range()
291 	tlb->end = max(tlb->end, address + range_size);  in __tlb_adjust_range()
294 static inline void __tlb_reset_range(struct mmu_gather *tlb)  in __tlb_reset_range()  argument
296 	if (tlb->fullmm) {  in __tlb_reset_range()
297 		tlb->start = tlb->end = ~0;  in __tlb_reset_range()
299 		tlb->start = TASK_SIZE;  in __tlb_reset_range()
300 		tlb->end = 0;  in __tlb_reset_range()
302 	tlb->freed_tables = 0;  in __tlb_reset_range()
303 	tlb->cleared_ptes = 0;  in __tlb_reset_range()
304 	tlb->cleared_pmds = 0;  in __tlb_reset_range()
305 	tlb->cleared_puds = 0;  in __tlb_reset_range()
306 	tlb->cleared_p4ds = 0;  in __tlb_reset_range()
328 static inline void tlb_flush(struct mmu_gather *tlb)  in tlb_flush()  argument
330 	if (tlb->end)  in tlb_flush()
331 		flush_tlb_mm(tlb->mm);  in tlb_flush()
335 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }  in tlb_update_vma_flags()  argument
338 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) { }  in tlb_end_vma()  argument
353 static inline void tlb_flush(struct mmu_gather *tlb)  in tlb_flush()  argument
355 	if (tlb->fullmm || tlb->need_flush_all) {  in tlb_flush()
356 		flush_tlb_mm(tlb->mm);  in tlb_flush()
357 	} else if (tlb->end) {  in tlb_flush()
359 			.vm_mm = tlb->mm,  in tlb_flush()
360 			.vm_flags = (tlb->vma_exec ? VM_EXEC    : 0) |  in tlb_flush()
361 				    (tlb->vma_huge ? VM_HUGETLB : 0),  in tlb_flush()
364 		flush_tlb_range(&vma, tlb->start, tlb->end);  in tlb_flush()
369 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_update_vma_flags()  argument
382 	tlb->vma_huge = !!(vma->vm_flags & VM_HUGETLB);  in tlb_update_vma_flags()
383 	tlb->vma_exec = !!(vma->vm_flags & VM_EXEC);  in tlb_update_vma_flags()
389 tlb_update_vma_flags(struct mmu_gather *tlb, struct vm_area_struct *vma) { }  in tlb_update_vma_flags()  argument
395 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)  in tlb_flush_mmu_tlbonly()  argument
397 	if (!tlb->end)  in tlb_flush_mmu_tlbonly()
400 	tlb_flush(tlb);  in tlb_flush_mmu_tlbonly()
401 	mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);  in tlb_flush_mmu_tlbonly()
402 	__tlb_reset_range(tlb);  in tlb_flush_mmu_tlbonly()
405 static inline void tlb_remove_page_size(struct mmu_gather *tlb,  in tlb_remove_page_size()  argument
408 	if (__tlb_remove_page_size(tlb, page, page_size))  in tlb_remove_page_size()
409 		tlb_flush_mmu(tlb);  in tlb_remove_page_size()
412 static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)  in __tlb_remove_page()  argument
414 	return __tlb_remove_page_size(tlb, page, PAGE_SIZE);  in __tlb_remove_page()
421 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)  in tlb_remove_page()  argument
423 	return tlb_remove_page_size(tlb, page, PAGE_SIZE);  in tlb_remove_page()
426 static inline void tlb_change_page_size(struct mmu_gather *tlb,  in tlb_change_page_size()  argument
430 	if (tlb->page_size && tlb->page_size != page_size) {  in tlb_change_page_size()
431 		if (!tlb->fullmm)  in tlb_change_page_size()
432 			tlb_flush_mmu(tlb);  in tlb_change_page_size()
435 	tlb->page_size = page_size;  in tlb_change_page_size()
439 static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb)  in tlb_get_unmap_shift()  argument
441 	if (tlb->cleared_ptes)  in tlb_get_unmap_shift()
443 	if (tlb->cleared_pmds)  in tlb_get_unmap_shift()
445 	if (tlb->cleared_puds)  in tlb_get_unmap_shift()
447 	if (tlb->cleared_p4ds)  in tlb_get_unmap_shift()
453 static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb)  in tlb_get_unmap_size()  argument
455 	return 1UL << tlb_get_unmap_shift(tlb);  in tlb_get_unmap_size()
464 static inline void tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_start_vma()  argument
466 	if (tlb->fullmm)  in tlb_start_vma()
469 	tlb_update_vma_flags(tlb, vma);  in tlb_start_vma()
475 static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)  in tlb_end_vma()  argument
477 	if (tlb->fullmm)  in tlb_end_vma()
486 	tlb_flush_mmu_tlbonly(tlb);  in tlb_end_vma()
491 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)  argument
501 #define tlb_remove_tlb_entry(tlb, ptep, address)		\  argument
503 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
504 		tlb->cleared_ptes = 1;				\
505 		__tlb_remove_tlb_entry(tlb, ptep, address);	\
508 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)	\  argument
511 		__tlb_adjust_range(tlb, address, _sz);		\
513 			tlb->cleared_pmds = 1;			\
515 			tlb->cleared_puds = 1;			\
516 		__tlb_remove_tlb_entry(tlb, ptep, address);	\
524 #define __tlb_remove_pmd_tlb_entry(tlb, pmdp, address) do {} while (0)  argument
527 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)			\  argument
529 		__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE);	\
530 		tlb->cleared_pmds = 1;					\
531 		__tlb_remove_pmd_tlb_entry(tlb, pmdp, address);		\
539 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0)  argument
542 #define tlb_remove_pud_tlb_entry(tlb, pudp, address)			\  argument
544 		__tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE);	\
545 		tlb->cleared_puds = 1;					\
546 		__tlb_remove_pud_tlb_entry(tlb, pudp, address);		\
568 #define pte_free_tlb(tlb, ptep, address)			\  argument
570 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
571 		tlb->freed_tables = 1;				\
572 		tlb->cleared_pmds = 1;				\
573 		__pte_free_tlb(tlb, ptep, address);		\
578 #define pmd_free_tlb(tlb, pmdp, address)			\  argument
580 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
581 		tlb->freed_tables = 1;				\
582 		tlb->cleared_puds = 1;				\
583 		__pmd_free_tlb(tlb, pmdp, address);		\
589 #define pud_free_tlb(tlb, pudp, address)			\  argument
591 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
592 		tlb->freed_tables = 1;				\
593 		tlb->cleared_p4ds = 1;				\
594 		__pud_free_tlb(tlb, pudp, address);		\
601 #define p4d_free_tlb(tlb, pudp, address)			\  argument
603 		__tlb_adjust_range(tlb, address, PAGE_SIZE);	\
604 		tlb->freed_tables = 1;				\
605 		__p4d_free_tlb(tlb, pudp, address);		\