85 		struct hmm_range *range;  in notifiers_decrement()  local
87 		list_for_each_entry(range, &hmm->ranges, list) {  in notifiers_decrement()
88 			if (range->valid)  in notifiers_decrement()
90 			range->valid = true;  in notifiers_decrement()
102 	struct hmm_range *range;  in hmm_invalidate_range_start()  local
108 	list_for_each_entry(range, &hmm->ranges, list) {  in hmm_invalidate_range_start()
109 		if (nrange->end < range->start || nrange->start >= range->end)  in hmm_invalidate_range_start()
112 		range->valid = false;  in hmm_invalidate_range_start()
217 	struct hmm_range	*range;  member
228 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_do_fault()  local
251 	*pfn = range->values[HMM_PFN_ERROR];  in hmm_vma_do_fault()
260 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_pfns_bad()  local
261 	uint64_t *pfns = range->pfns;  in hmm_pfns_bad()
264 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_pfns_bad()
266 		pfns[i] = range->values[HMM_PFN_ERROR];  in hmm_pfns_bad()
288 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_hole_()  local
289 	uint64_t *pfns = range->pfns;  in hmm_vma_walk_hole_()
293 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hole_()
299 		pfns[i] = range->values[HMM_PFN_NONE];  in hmm_vma_walk_hole_()
317 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_pte_need_fault()  local
332 	pfns = (pfns & range->pfn_flags_mask) | range->default_flags;  in hmm_pte_need_fault()
335 	if (!(pfns & range->flags[HMM_PFN_VALID]))  in hmm_pte_need_fault()
338 	if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) {  in hmm_pte_need_fault()
340 		if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) {  in hmm_pte_need_fault()
341 			*write_fault = pfns & range->flags[HMM_PFN_WRITE];  in hmm_pte_need_fault()
348 	*fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);  in hmm_pte_need_fault()
350 	if ((pfns & range->flags[HMM_PFN_WRITE]) &&  in hmm_pte_need_fault()
351 	    !(cpu_flags & range->flags[HMM_PFN_WRITE])) {  in hmm_pte_need_fault()
382 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_hole()  local
387 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hole()
389 	pfns = &range->pfns[i];  in hmm_vma_walk_hole()
395 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)  in pmd_to_hmm_pfn_flags()  argument
399 	return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |  in pmd_to_hmm_pfn_flags()
400 				range->flags[HMM_PFN_WRITE] :  in pmd_to_hmm_pfn_flags()
401 				range->flags[HMM_PFN_VALID];  in pmd_to_hmm_pfn_flags()
409 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_pmd()  local
415 	cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);  in hmm_vma_handle_pmd()
430 		pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags;  in hmm_vma_handle_pmd()
445 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)  in pte_to_hmm_pfn_flags()  argument
449 	return pte_write(pte) ? range->flags[HMM_PFN_VALID] |  in pte_to_hmm_pfn_flags()
450 				range->flags[HMM_PFN_WRITE] :  in pte_to_hmm_pfn_flags()
451 				range->flags[HMM_PFN_VALID];  in pte_to_hmm_pfn_flags()
459 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_handle_pte()  local
465 	*pfn = range->values[HMM_PFN_NONE];  in hmm_vma_handle_pte()
480 			cpu_flags = pte_to_hmm_pfn_flags(range, pte);  in hmm_vma_handle_pte()
493 			cpu_flags = range->flags[HMM_PFN_VALID] |  in hmm_vma_handle_pte()
494 				range->flags[HMM_PFN_DEVICE_PRIVATE];  in hmm_vma_handle_pte()
496 				range->flags[HMM_PFN_WRITE] : 0;  in hmm_vma_handle_pte()
501 			*pfn = hmm_device_entry_from_pfn(range,  in hmm_vma_handle_pte()
518 		*pfn = range->values[HMM_PFN_ERROR];  in hmm_vma_handle_pte()
521 		cpu_flags = pte_to_hmm_pfn_flags(range, pte);  in hmm_vma_handle_pte()
535 		*pfn = range->values[HMM_PFN_SPECIAL];  in hmm_vma_handle_pte()
539 	*pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags;  in hmm_vma_handle_pte()
558 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_pmd()  local
559 	uint64_t *pfns = range->pfns;  in hmm_vma_walk_pmd()
574 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
576 		pfns = &range->pfns[i];  in hmm_vma_walk_pmd()
604 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
618 	i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pmd()
647 static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud)  in pud_to_hmm_pfn_flags()  argument
651 	return pud_write(pud) ? range->flags[HMM_PFN_VALID] |  in pud_to_hmm_pfn_flags()
652 				range->flags[HMM_PFN_WRITE] :  in pud_to_hmm_pfn_flags()
653 				range->flags[HMM_PFN_VALID];  in pud_to_hmm_pfn_flags()
660 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_pud()  local
679 		i = (addr - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_pud()
681 		pfns = &range->pfns[i];  in hmm_vma_walk_pud()
683 		cpu_flags = pud_to_hmm_pfn_flags(range, pud);  in hmm_vma_walk_pud()
696 			pfns[i] = hmm_device_entry_from_pfn(range, pfn) |  in hmm_vma_walk_pud()
732 	struct hmm_range *range = hmm_vma_walk->range;  in hmm_vma_walk_hugetlb_entry()  local
743 	i = (start - range->start) >> PAGE_SHIFT;  in hmm_vma_walk_hugetlb_entry()
744 	orig_pfn = range->pfns[i];  in hmm_vma_walk_hugetlb_entry()
745 	range->pfns[i] = range->values[HMM_PFN_NONE];  in hmm_vma_walk_hugetlb_entry()
746 	cpu_flags = pte_to_hmm_pfn_flags(range, entry);  in hmm_vma_walk_hugetlb_entry()
757 		range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) |  in hmm_vma_walk_hugetlb_entry()
773 static void hmm_pfns_clear(struct hmm_range *range,  in hmm_pfns_clear()  argument
779 		*pfns = range->values[HMM_PFN_NONE];  in hmm_pfns_clear()
791 int hmm_range_register(struct hmm_range *range, struct hmm_mirror *mirror)  in hmm_range_register()  argument
796 	range->valid = false;  in hmm_range_register()
797 	range->hmm = NULL;  in hmm_range_register()
799 	if ((range->start & (PAGE_SIZE - 1)) || (range->end & (PAGE_SIZE - 1)))  in hmm_range_register()
801 	if (range->start >= range->end)  in hmm_range_register()
811 	range->hmm = hmm;  in hmm_range_register()
812 	list_add(&range->list, &hmm->ranges);  in hmm_range_register()
819 		range->valid = true;  in hmm_range_register()
833 void hmm_range_unregister(struct hmm_range *range)  in hmm_range_unregister()  argument
835 	struct hmm *hmm = range->hmm;  in hmm_range_unregister()
839 	list_del_init(&range->list);  in hmm_range_unregister()
850 	range->valid = false;  in hmm_range_unregister()
851 	memset(&range->hmm, POISON_INUSE, sizeof(range->hmm));  in hmm_range_unregister()
890 long hmm_range_fault(struct hmm_range *range, unsigned int flags)  in hmm_range_fault()  argument
893 	unsigned long start = range->start, end;  in hmm_range_fault()
895 	struct hmm *hmm = range->hmm;  in hmm_range_fault()
903 		if (!range->valid)  in hmm_range_fault()
916 			hmm_pfns_clear(range, range->pfns,  in hmm_range_fault()
917 				range->start, range->end);  in hmm_range_fault()
924 		hmm_vma_walk.range = range;  in hmm_range_fault()
925 		end = min(range->end, vma->vm_end);  in hmm_range_fault()
936 		} while (ret == -EBUSY && range->valid);  in hmm_range_fault()
941 			i = (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;  in hmm_range_fault()
942 			hmm_pfns_clear(range, &range->pfns[i],  in hmm_range_fault()
943 				hmm_vma_walk.last, range->end);  in hmm_range_fault()
948 	} while (start < range->end);  in hmm_range_fault()
950 	return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;  in hmm_range_fault()
964 long hmm_range_dma_map(struct hmm_range *range, struct device *device,  in hmm_range_dma_map()  argument
970 	ret = hmm_range_fault(range, flags);  in hmm_range_dma_map()
974 	npages = (range->end - range->start) >> PAGE_SHIFT;  in hmm_range_dma_map()
989 		page = hmm_device_entry_to_page(range, range->pfns[i]);  in hmm_range_dma_map()
994 		if (!range->valid) {  in hmm_range_dma_map()
1000 		if (range->pfns[i] & range->flags[HMM_PFN_WRITE])  in hmm_range_dma_map()
1019 		page = hmm_device_entry_to_page(range, range->pfns[i]);  in hmm_range_dma_map()
1027 		if (range->pfns[i] & range->flags[HMM_PFN_WRITE])  in hmm_range_dma_map()
1051 long hmm_range_dma_unmap(struct hmm_range *range,  in hmm_range_dma_unmap()  argument
1060 	if (range->end <= range->start)  in hmm_range_dma_unmap()
1064 	if (!range->pfns)  in hmm_range_dma_unmap()
1067 	npages = (range->end - range->start) >> PAGE_SHIFT;  in hmm_range_dma_unmap()
1072 		page = hmm_device_entry_to_page(range, range->pfns[i]);  in hmm_range_dma_unmap()
1077 		if (range->pfns[i] & range->flags[HMM_PFN_WRITE]) {  in hmm_range_dma_unmap()
1090 		range->pfns[i] = range->values[HMM_PFN_NONE];  in hmm_range_dma_unmap()