91 	struct dmirror_device		*mdevice;  member
102 	struct dmirror_device	*mdevice;  member
140 static bool dmirror_is_private_zone(struct dmirror_device *mdevice)  in dmirror_is_private_zone()  argument
142 	return (mdevice->zone_device_type ==  in dmirror_is_private_zone()
149 	return (dmirror->mdevice->zone_device_type ==  in dmirror_select_device()
171 	dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);  in dmirror_fops_open()
204 	return dmirror_page_to_chunk(page)->mdevice;  in dmirror_page_to_device()
268 	    range->owner == dmirror->mdevice)  in dmirror_interval_invalidate()
339 		.dev_private_owner = dmirror->mdevice,  in dmirror_fault()
493 static int dmirror_allocate_chunk(struct dmirror_device *mdevice,  in dmirror_allocate_chunk()  argument
508 	switch (mdevice->zone_device_type) {  in dmirror_allocate_chunk()
519 		devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?  in dmirror_allocate_chunk()
533 	devmem->pagemap.owner = mdevice;  in dmirror_allocate_chunk()
535 	mutex_lock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
537 	if (mdevice->devmem_count == mdevice->devmem_capacity) {  in dmirror_allocate_chunk()
541 		new_capacity = mdevice->devmem_capacity +  in dmirror_allocate_chunk()
543 		new_chunks = krealloc(mdevice->devmem_chunks,  in dmirror_allocate_chunk()
548 		mdevice->devmem_capacity = new_capacity;  in dmirror_allocate_chunk()
549 		mdevice->devmem_chunks = new_chunks;  in dmirror_allocate_chunk()
560 	devmem->mdevice = mdevice;  in dmirror_allocate_chunk()
563 	mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;  in dmirror_allocate_chunk()
565 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
569 		mdevice->devmem_count,  in dmirror_allocate_chunk()
570 		mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),  in dmirror_allocate_chunk()
573 	spin_lock(&mdevice->lock);  in dmirror_allocate_chunk()
577 		page->zone_device_data = mdevice->free_pages;  in dmirror_allocate_chunk()
578 		mdevice->free_pages = page;  in dmirror_allocate_chunk()
581 		*ppage = mdevice->free_pages;  in dmirror_allocate_chunk()
582 		mdevice->free_pages = (*ppage)->zone_device_data;  in dmirror_allocate_chunk()
583 		mdevice->calloc++;  in dmirror_allocate_chunk()
585 	spin_unlock(&mdevice->lock);  in dmirror_allocate_chunk()
590 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_allocate_chunk()
600 static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)  in dmirror_devmem_alloc_page()  argument
611 	if (dmirror_is_private_zone(mdevice)) {  in dmirror_devmem_alloc_page()
616 	spin_lock(&mdevice->lock);  in dmirror_devmem_alloc_page()
618 	if (mdevice->free_pages) {  in dmirror_devmem_alloc_page()
619 		dpage = mdevice->free_pages;  in dmirror_devmem_alloc_page()
620 		mdevice->free_pages = dpage->zone_device_data;  in dmirror_devmem_alloc_page()
621 		mdevice->calloc++;  in dmirror_devmem_alloc_page()
622 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
624 		spin_unlock(&mdevice->lock);  in dmirror_devmem_alloc_page()
625 		if (dmirror_allocate_chunk(mdevice, &dpage))  in dmirror_devmem_alloc_page()
642 	struct dmirror_device *mdevice = dmirror->mdevice;  in dmirror_migrate_alloc_and_copy()  local
666 		dpage = dmirror_devmem_alloc_page(mdevice);  in dmirror_migrate_alloc_and_copy()
939 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_system()
999 		args.pgmap_owner = dmirror->mdevice;  in dmirror_migrate_to_device()
1056 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1062 		if (dmirror->mdevice == dmirror_page_to_device(page))  in dmirror_mkentry()
1177 		.dev_private_owner = dmirror->mdevice,  in dmirror_snapshot()
1260 	struct dmirror_device *mdevice = devmem->mdevice;  in dmirror_remove_free_pages()  local
1263 	for (page = mdevice->free_pages; page; page = page->zone_device_data)  in dmirror_remove_free_pages()
1265 			mdevice->free_pages = page->zone_device_data;  in dmirror_remove_free_pages()
1268 static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)  in dmirror_device_remove_chunks()  argument
1272 	mutex_lock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1273 	if (mdevice->devmem_chunks) {  in dmirror_device_remove_chunks()
1274 		for (i = 0; i < mdevice->devmem_count; i++) {  in dmirror_device_remove_chunks()
1276 				mdevice->devmem_chunks[i];  in dmirror_device_remove_chunks()
1278 			spin_lock(&mdevice->lock);  in dmirror_device_remove_chunks()
1281 			spin_unlock(&mdevice->lock);  in dmirror_device_remove_chunks()
1290 		mdevice->devmem_count = 0;  in dmirror_device_remove_chunks()
1291 		mdevice->devmem_capacity = 0;  in dmirror_device_remove_chunks()
1292 		mdevice->free_pages = NULL;  in dmirror_device_remove_chunks()
1293 		kfree(mdevice->devmem_chunks);  in dmirror_device_remove_chunks()
1294 		mdevice->devmem_chunks = NULL;  in dmirror_device_remove_chunks()
1296 	mutex_unlock(&mdevice->devmem_lock);  in dmirror_device_remove_chunks()
1354 		dmirror_device_remove_chunks(dmirror->mdevice);  in dmirror_fops_unlocked_ioctl()
1405 	struct dmirror_device *mdevice;  in dmirror_devmem_free()  local
1410 	mdevice = dmirror_page_to_device(page);  in dmirror_devmem_free()
1411 	spin_lock(&mdevice->lock);  in dmirror_devmem_free()
1415 		mdevice->cfree++;  in dmirror_devmem_free()
1416 		page->zone_device_data = mdevice->free_pages;  in dmirror_devmem_free()
1417 		mdevice->free_pages = page;  in dmirror_devmem_free()
1419 	spin_unlock(&mdevice->lock);  in dmirror_devmem_free()
1445 	args.pgmap_owner = dmirror->mdevice;  in dmirror_devmem_fault()
1470 static int dmirror_device_init(struct dmirror_device *mdevice, int id)  in dmirror_device_init()  argument
1476 	mutex_init(&mdevice->devmem_lock);  in dmirror_device_init()
1477 	spin_lock_init(&mdevice->lock);  in dmirror_device_init()
1479 	cdev_init(&mdevice->cdevice, &dmirror_fops);  in dmirror_device_init()
1480 	mdevice->cdevice.owner = THIS_MODULE;  in dmirror_device_init()
1481 	device_initialize(&mdevice->device);  in dmirror_device_init()
1482 	mdevice->device.devt = dev;  in dmirror_device_init()
1484 	ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);  in dmirror_device_init()
1488 	ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);  in dmirror_device_init()
1493 	return dmirror_allocate_chunk(mdevice, NULL);  in dmirror_device_init()
1496 static void dmirror_device_remove(struct dmirror_device *mdevice)  in dmirror_device_remove()  argument
1498 	dmirror_device_remove_chunks(mdevice);  in dmirror_device_remove()
1499 	cdev_device_del(&mdevice->cdevice, &mdevice->device);  in dmirror_device_remove()