83 	struct dmirror			*dmirror;  member
90 struct dmirror {  struct
147 dmirror_select_device(struct dmirror *dmirror)  in dmirror_select_device()  argument
149 	return (dmirror->mdevice->zone_device_type ==  in dmirror_select_device()
163 	struct dmirror *dmirror;  in dmirror_fops_open()  local
167 	dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);  in dmirror_fops_open()
168 	if (dmirror == NULL)  in dmirror_fops_open()
171 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
172 	mutex_init(&dmirror->mutex);  in dmirror_fops_open()
173 	xa_init(&dmirror->pt);  in dmirror_fops_open()
175 	ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,  in dmirror_fops_open()
178 		kfree(dmirror);  in dmirror_fops_open()
182 	filp->private_data = dmirror;  in dmirror_fops_open()
188 	struct dmirror *dmirror = filp->private_data;  in dmirror_fops_release()  local
190 	mmu_interval_notifier_remove(&dmirror->notifier);  in dmirror_fops_release()
191 	xa_destroy(&dmirror->pt);  in dmirror_fops_release()
192 	kfree(dmirror);  in dmirror_fops_release()
207 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)  in dmirror_do_fault()  argument
233 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_do_fault()
241 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,  in dmirror_do_update()  argument
252 	xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,  in dmirror_do_update()
254 		xa_erase(&dmirror->pt, pfn);  in dmirror_do_update()
261 	struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);  in dmirror_interval_invalidate()  local
268 	    range->owner == dmirror->mdevice)  in dmirror_interval_invalidate()
272 		mutex_lock(&dmirror->mutex);  in dmirror_interval_invalidate()
273 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_interval_invalidate()
277 	dmirror_do_update(dmirror, range->start, range->end);  in dmirror_interval_invalidate()
279 	mutex_unlock(&dmirror->mutex);  in dmirror_interval_invalidate()
287 static int dmirror_range_fault(struct dmirror *dmirror,  in dmirror_range_fault()  argument
290 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_fault()
311 		mutex_lock(&dmirror->mutex);  in dmirror_range_fault()
314 			mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
320 	ret = dmirror_do_fault(dmirror, range);  in dmirror_range_fault()
322 	mutex_unlock(&dmirror->mutex);  in dmirror_range_fault()
327 static int dmirror_fault(struct dmirror *dmirror, unsigned long start,  in dmirror_fault()  argument
330 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_fault()
334 		.notifier = &dmirror->notifier,  in dmirror_fault()
339 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
351 		ret = dmirror_range_fault(dmirror, &range);  in dmirror_fault()
360 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,  in dmirror_do_read()  argument
373 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_read()
389 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_read()  argument
406 		mutex_lock(&dmirror->mutex);  in dmirror_read()
407 		ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_read()
408 		mutex_unlock(&dmirror->mutex);  in dmirror_read()
413 		ret = dmirror_fault(dmirror, start, end, false);  in dmirror_read()
429 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,  in dmirror_do_write()  argument
442 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_do_write()
458 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)  in dmirror_write()  argument
480 		mutex_lock(&dmirror->mutex);  in dmirror_write()
481 		ret = dmirror_do_write(dmirror, start, end, &bounce);  in dmirror_write()
482 		mutex_unlock(&dmirror->mutex);  in dmirror_write()
487 		ret = dmirror_fault(dmirror, start, end, true);  in dmirror_write()
646 					   struct dmirror *dmirror)  in dmirror_migrate_alloc_and_copy()  argument
648 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_migrate_alloc_and_copy()
688 		rpage->zone_device_data = dmirror;  in dmirror_migrate_alloc_and_copy()
699 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,  in dmirror_check_atomic()  argument
707 		entry = xa_load(&dmirror->pt, pfn);  in dmirror_check_atomic()
716 			      struct page **pages, struct dmirror *dmirror)  in dmirror_atomic_map()  argument
722 	mutex_lock(&dmirror->mutex);  in dmirror_atomic_map()
732 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_atomic_map()
734 			mutex_unlock(&dmirror->mutex);  in dmirror_atomic_map()
741 	mutex_unlock(&dmirror->mutex);  in dmirror_atomic_map()
746 					    struct dmirror *dmirror)  in dmirror_migrate_finalize_and_map()  argument
755 	mutex_lock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
772 		entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);  in dmirror_migrate_finalize_and_map()
774 			mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
779 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate_finalize_and_map()
783 static int dmirror_exclusive(struct dmirror *dmirror,  in dmirror_exclusive()  argument
788 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_exclusive()
820 			mapped = dmirror_atomic_map(addr, next, pages, dmirror);  in dmirror_exclusive()
841 	mutex_lock(&dmirror->mutex);  in dmirror_exclusive()
842 	ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_exclusive()
843 	mutex_unlock(&dmirror->mutex);  in dmirror_exclusive()
856 						      struct dmirror *dmirror)  in dmirror_devmem_fault_alloc_and_copy()  argument
883 		xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);  in dmirror_devmem_fault_alloc_and_copy()
906 static int dmirror_migrate_to_system(struct dmirror *dmirror,  in dmirror_migrate_to_system()  argument
911 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_migrate_to_system()
945 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_system()
946 		args.flags = dmirror_select_device(dmirror);  in dmirror_migrate_to_system()
953 		dmirror_devmem_fault_alloc_and_copy(&args, dmirror);  in dmirror_migrate_to_system()
966 static int dmirror_migrate_to_device(struct dmirror *dmirror,  in dmirror_migrate_to_device()  argument
971 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_migrate_to_device()
1005 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_device()
1012 		dmirror_migrate_alloc_and_copy(&args, dmirror);  in dmirror_migrate_to_device()
1014 		dmirror_migrate_finalize_and_map(&args, dmirror);  in dmirror_migrate_to_device()
1027 	mutex_lock(&dmirror->mutex);  in dmirror_migrate_to_device()
1028 	ret = dmirror_do_read(dmirror, start, end, &bounce);  in dmirror_migrate_to_device()
1029 	mutex_unlock(&dmirror->mutex);  in dmirror_migrate_to_device()
1045 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,  in dmirror_mkentry()  argument
1062 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1068 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1092 	struct dmirror *dmirror = dmi->dmirror;  in dmirror_snapshot_invalidate()  local
1095 		mutex_lock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
1096 	else if (!mutex_trylock(&dmirror->mutex))  in dmirror_snapshot_invalidate()
1105 	mutex_unlock(&dmirror->mutex);  in dmirror_snapshot_invalidate()
1113 static int dmirror_range_snapshot(struct dmirror *dmirror,  in dmirror_range_snapshot()  argument
1117 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_range_snapshot()
1125 	notifier.dmirror = dmirror;  in dmirror_range_snapshot()
1151 		mutex_lock(&dmirror->mutex);  in dmirror_range_snapshot()
1154 			mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
1162 		dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);  in dmirror_range_snapshot()
1164 	mutex_unlock(&dmirror->mutex);  in dmirror_range_snapshot()
1170 static int dmirror_snapshot(struct dmirror *dmirror,  in dmirror_snapshot()  argument
1173 	struct mm_struct *mm = dmirror->notifier.mm;  in dmirror_snapshot()
1183 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
1208 		ret = dmirror_range_snapshot(dmirror, &range, perm);  in dmirror_snapshot()
1311 	struct dmirror *dmirror;  in dmirror_fops_unlocked_ioctl()  local
1314 	dmirror = filp->private_data;  in dmirror_fops_unlocked_ioctl()
1315 	if (!dmirror)  in dmirror_fops_unlocked_ioctl()
1331 		ret = dmirror_read(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1335 		ret = dmirror_write(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1339 		ret = dmirror_migrate_to_device(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1343 		ret = dmirror_migrate_to_system(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1347 		ret = dmirror_exclusive(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1351 		ret = dmirror_check_atomic(dmirror, cmd.addr,  in dmirror_fops_unlocked_ioctl()
1356 		ret = dmirror_snapshot(dmirror, &cmd);  in dmirror_fops_unlocked_ioctl()
1360 		dmirror_device_remove_chunks(dmirror->mdevice);  in dmirror_fops_unlocked_ioctl()
1434 	struct dmirror *dmirror;  in dmirror_devmem_fault()  local
1443 	dmirror = rpage->zone_device_data;  in dmirror_devmem_fault()
1451 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1452 	args.flags = dmirror_select_device(dmirror);  in dmirror_devmem_fault()
1458 	ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);  in dmirror_devmem_fault()