91 	struct dmirror_device		*mdevice;  member
102 	struct dmirror_device	*mdevice;  member
140 static bool dmirror_is_private_zone(struct dmirror_device *mdevice)  in dmirror_is_private_zone()  argument
142 	return (mdevice->zone_device_type ==  in dmirror_is_private_zone()
149 	return (dmirror->mdevice->zone_device_type ==  in dmirror_select_device()
171 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
204 	return dmirror_page_to_chunk(page)->mdevice;  in dmirror_page_to_device()
268 	    range->owner == dmirror->mdevice)  in dmirror_interval_invalidate()
339 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
499 static int dmirror_allocate_chunk(struct dmirror_device *mdevice,  in dmirror_allocate_chunk()  argument
514 	switch (mdevice->zone_device_type) {  in dmirror_allocate_chunk()
525 		devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?  in dmirror_allocate_chunk()
539 	devmem->pagemap.owner = mdevice;  in dmirror_allocate_chunk()
541 	mutex_lock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
543 	if (mdevice->devmem_count == mdevice->devmem_capacity) {  in dmirror_allocate_chunk()
547 		new_capacity = mdevice->devmem_capacity +  in dmirror_allocate_chunk()
549 		new_chunks = krealloc(mdevice->devmem_chunks,  in dmirror_allocate_chunk()
554 		mdevice->devmem_capacity = new_capacity;  in dmirror_allocate_chunk()
555 		mdevice->devmem_chunks = new_chunks;  in dmirror_allocate_chunk()
566 	devmem->mdevice = mdevice;  in dmirror_allocate_chunk()
569 	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;  in dmirror_allocate_chunk()
571 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
575 		mdevice->devmem_count,  in dmirror_allocate_chunk()
576 		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),  in dmirror_allocate_chunk()
579 	spin_lock(&mdevice->lock);  in dmirror_allocate_chunk()
583 		page->zone_device_data = mdevice->free_pages;  in dmirror_allocate_chunk()
584 		mdevice->free_pages = page;  in dmirror_allocate_chunk()
587 		*ppage = mdevice->free_pages;  in dmirror_allocate_chunk()
588 		mdevice->free_pages = (*ppage)->zone_device_data;  in dmirror_allocate_chunk()
589 		mdevice->calloc++;  in dmirror_allocate_chunk()
591 	spin_unlock(&mdevice->lock);  in dmirror_allocate_chunk()
596 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
606 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)  in dmirror_devmem_alloc_page()  argument
617 	if (dmirror_is_private_zone(mdevice)) {  in dmirror_devmem_alloc_page()
622 	spin_lock(&mdevice->lock);  in dmirror_devmem_alloc_page()
624 	if (mdevice->free_pages) {  in dmirror_devmem_alloc_page()
625 		dpage = mdevice->free_pages;  in dmirror_devmem_alloc_page()
626 		mdevice->free_pages = dpage->zone_device_data;  in dmirror_devmem_alloc_page()
627 		mdevice->calloc++;  in dmirror_devmem_alloc_page()
628 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
630 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
631 		if (dmirror_allocate_chunk(mdevice, &dpage))  in dmirror_devmem_alloc_page()
648 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_migrate_alloc_and_copy()  local
672 		dpage = dmirror_devmem_alloc_page(mdevice);  in dmirror_migrate_alloc_and_copy()
945 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_system()
1005 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_device()
1062 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1068 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1183 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
1266 	struct dmirror_device *mdevice = devmem->mdevice;  in dmirror_remove_free_pages()  local
1269 	for (page = mdevice->free_pages; page; page = page->zone_device_data)  in dmirror_remove_free_pages()
1271 			mdevice->free_pages = page->zone_device_data;  in dmirror_remove_free_pages()
1274 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)  in dmirror_device_remove_chunks()  argument
1278 	mutex_lock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1279 	if (mdevice->devmem_chunks) {  in dmirror_device_remove_chunks()
1280 		for (i = 0; i < mdevice->devmem_count; i++) {  in dmirror_device_remove_chunks()
1282 				mdevice->devmem_chunks[i];  in dmirror_device_remove_chunks()
1284 			spin_lock(&mdevice->lock);  in dmirror_device_remove_chunks()
1287 			spin_unlock(&mdevice->lock);  in dmirror_device_remove_chunks()
1296 		mdevice->devmem_count = 0;  in dmirror_device_remove_chunks()
1297 		mdevice->devmem_capacity = 0;  in dmirror_device_remove_chunks()
1298 		mdevice->free_pages = NULL;  in dmirror_device_remove_chunks()
1299 		kfree(mdevice->devmem_chunks);  in dmirror_device_remove_chunks()
1300 		mdevice->devmem_chunks = NULL;  in dmirror_device_remove_chunks()
1302 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1360 		dmirror_device_remove_chunks(dmirror->mdevice);  in dmirror_fops_unlocked_ioctl()
1411 	struct dmirror_device *mdevice;  in dmirror_devmem_free()  local
1416 	mdevice = dmirror_page_to_device(page);  in dmirror_devmem_free()
1417 	spin_lock(&mdevice->lock);  in dmirror_devmem_free()
1421 		mdevice->cfree++;  in dmirror_devmem_free()
1422 		page->zone_device_data = mdevice->free_pages;  in dmirror_devmem_free()
1423 		mdevice->free_pages = page;  in dmirror_devmem_free()
1425 	spin_unlock(&mdevice->lock);  in dmirror_devmem_free()
1451 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1476 static int dmirror_device_init(struct dmirror_device *mdevice, int id)  in dmirror_device_init()  argument
1482 	mutex_init(&mdevice->devmem_lock);  in dmirror_device_init()
1483 	spin_lock_init(&mdevice->lock);  in dmirror_device_init()
1485 	cdev_init(&mdevice->cdevice, &dmirror_fops);  in dmirror_device_init()
1486 	mdevice->cdevice.owner = THIS_MODULE;  in dmirror_device_init()
1487 	device_initialize(&mdevice->device);  in dmirror_device_init()
1488 	mdevice->device.devt = dev;  in dmirror_device_init()
1490 	ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);  in dmirror_device_init()
1494 	ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);  in dmirror_device_init()
1499 	return dmirror_allocate_chunk(mdevice, NULL);  in dmirror_device_init()
1502 static void dmirror_device_remove(struct dmirror_device *mdevice)  in dmirror_device_remove()  argument
1504 	dmirror_device_remove_chunks(mdevice);  in dmirror_device_remove()
1505 	cdev_device_del(&mdevice->cdevice, &mdevice->device);  in dmirror_device_remove()