160 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)  in dmirror_do_fault()  argument
162 	unsigned long *pfns = range->hmm_pfns;  in dmirror_do_fault()
165 	for (pfn = (range->start >> PAGE_SHIFT);  in dmirror_do_fault()
166 	     pfn < (range->end >> PAGE_SHIFT);  in dmirror_do_fault()
184 		else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))  in dmirror_do_fault()
211 				const struct mmu_notifier_range *range,  in dmirror_interval_invalidate()  argument
220 	if (range->event == MMU_NOTIFY_MIGRATE &&  in dmirror_interval_invalidate()
221 	    range->migrate_pgmap_owner == dmirror->mdevice)  in dmirror_interval_invalidate()
224 	if (mmu_notifier_range_blockable(range))  in dmirror_interval_invalidate()
230 	dmirror_do_update(dmirror, range->start, range->end);  in dmirror_interval_invalidate()
241 				struct hmm_range *range)  in dmirror_range_fault()  argument
254 		range->notifier_seq = mmu_interval_read_begin(range->notifier);  in dmirror_range_fault()
256 		ret = hmm_range_fault(range);  in dmirror_range_fault()
265 		if (mmu_interval_read_retry(range->notifier,  in dmirror_range_fault()
266 					    range->notifier_seq)) {  in dmirror_range_fault()
273 	ret = dmirror_do_fault(dmirror, range);  in dmirror_range_fault()
286 	struct hmm_range range = {  in dmirror_fault()  local
300 	for (addr = start; addr < end; addr = range.end) {  in dmirror_fault()
301 		range.start = addr;  in dmirror_fault()
302 		range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);  in dmirror_fault()
304 		ret = dmirror_range_fault(dmirror, &range);  in dmirror_fault()
472 	devmem->pagemap.range.start = res->start;  in dmirror_allocate_chunk()
473 	devmem->pagemap.range.end = res->end;  in dmirror_allocate_chunk()
500 	pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;  in dmirror_allocate_chunk()
501 	pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);  in dmirror_allocate_chunk()
530 	release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));  in dmirror_allocate_chunk()
740 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,  in dmirror_mkentry()  argument
776 				const struct mmu_notifier_range *range,  in dmirror_snapshot_invalidate()  argument
783 	if (mmu_notifier_range_blockable(range))  in dmirror_snapshot_invalidate()
803 				  struct hmm_range *range,  in dmirror_range_snapshot()  argument
815 	range->notifier = &notifier.notifier;  in dmirror_range_snapshot()
817 	ret = mmu_interval_notifier_insert(range->notifier, mm,  in dmirror_range_snapshot()
818 			range->start, range->end - range->start,  in dmirror_range_snapshot()
829 		range->notifier_seq = mmu_interval_read_begin(range->notifier);  in dmirror_range_snapshot()
832 		ret = hmm_range_fault(range);  in dmirror_range_snapshot()
841 		if (mmu_interval_read_retry(range->notifier,  in dmirror_range_snapshot()
842 					    range->notifier_seq)) {  in dmirror_range_snapshot()
849 	n = (range->end - range->start) >> PAGE_SHIFT;  in dmirror_range_snapshot()
851 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);  in dmirror_range_snapshot()
855 	mmu_interval_notifier_remove(range->notifier);  in dmirror_range_snapshot()
870 	struct hmm_range range = {  in dmirror_snapshot()  local
894 		range.start = addr;  in dmirror_snapshot()
895 		range.end = next;  in dmirror_snapshot()
897 		ret = dmirror_range_snapshot(dmirror, &range, perm);  in dmirror_snapshot()
901 		n = (range.end - range.start) >> PAGE_SHIFT;  in dmirror_snapshot()
1103 			release_mem_region(devmem->pagemap.range.start,  in dmirror_device_remove()
1104 					   range_len(&devmem->pagemap.range));  in dmirror_device_remove()