198 static void null_del_dev(struct nullb *nullb);
333 			null_del_dev(dev->nullb);  in nullb_device_power_store()
461 		null_del_dev(dev->nullb);  in nullb_group_drop_item()
500 static inline int null_cache_active(struct nullb *nullb)  in null_cache_active()  argument
502 	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);  in null_cache_active()
688 static void null_free_sector(struct nullb *nullb, sector_t sector,  in null_free_sector()  argument
696 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in null_free_sector()
709 				nullb->dev->curr_cache -= PAGE_SIZE;  in null_free_sector()
714 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,  in null_radix_tree_insert()  argument
719 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in null_radix_tree_insert()
726 		nullb->dev->curr_cache += PAGE_SIZE;  in null_radix_tree_insert()
760 static struct nullb_page *__null_lookup_page(struct nullb *nullb,  in __null_lookup_page()  argument
771 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;  in __null_lookup_page()
781 static struct nullb_page *null_lookup_page(struct nullb *nullb,  in null_lookup_page()  argument
787 		page = __null_lookup_page(nullb, sector, for_write, true);  in null_lookup_page()
790 	return __null_lookup_page(nullb, sector, for_write, false);  in null_lookup_page()
793 static struct nullb_page *null_insert_page(struct nullb *nullb,  in null_insert_page()  argument
795 	__releases(&nullb->lock)  in null_insert_page()
796 	__acquires(&nullb->lock)  in null_insert_page()
801 	t_page = null_lookup_page(nullb, sector, true, ignore_cache);  in null_insert_page()
805 	spin_unlock_irq(&nullb->lock);  in null_insert_page()
814 	spin_lock_irq(&nullb->lock);  in null_insert_page()
817 	t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);  in null_insert_page()
824 	spin_lock_irq(&nullb->lock);  in null_insert_page()
825 	return null_lookup_page(nullb, sector, true, ignore_cache);  in null_insert_page()
828 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)  in null_flush_cache_page()  argument
838 	t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);  in null_flush_cache_page()
844 			ret = radix_tree_delete_item(&nullb->dev->data,  in null_flush_cache_page()
858 			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {  in null_flush_cache_page()
862 				nullb->dev->blocksize);  in null_flush_cache_page()
870 	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);  in null_flush_cache_page()
872 	nullb->dev->curr_cache -= PAGE_SIZE;  in null_flush_cache_page()
877 static int null_make_cache_space(struct nullb *nullb, unsigned long n)  in null_make_cache_space()  argument
884 	if ((nullb->dev->cache_size * 1024 * 1024) >  in null_make_cache_space()
885 	     nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)  in null_make_cache_space()
888 	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,  in null_make_cache_space()
889 			(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);  in null_make_cache_space()
895 		nullb->cache_flush_pos = c_pages[i]->page->index;  in null_make_cache_space()
910 		err = null_flush_cache_page(nullb, c_pages[i]);  in null_make_cache_space()
919 			nullb->cache_flush_pos = 0;  in null_make_cache_space()
922 			spin_unlock_irq(&nullb->lock);  in null_make_cache_space()
923 			spin_lock_irq(&nullb->lock);  in null_make_cache_space()
930 static int copy_to_nullb(struct nullb *nullb, struct page *source,  in copy_to_nullb()  argument
939 		temp = min_t(size_t, nullb->dev->blocksize, n - count);  in copy_to_nullb()
941 		if (null_cache_active(nullb) && !is_fua)  in copy_to_nullb()
942 			null_make_cache_space(nullb, PAGE_SIZE);  in copy_to_nullb()
945 		t_page = null_insert_page(nullb, sector,  in copy_to_nullb()
946 			!null_cache_active(nullb) || is_fua);  in copy_to_nullb()
959 			null_free_sector(nullb, sector, true);  in copy_to_nullb()
967 static int copy_from_nullb(struct nullb *nullb, struct page *dest,  in copy_from_nullb()  argument
976 		temp = min_t(size_t, nullb->dev->blocksize, n - count);  in copy_from_nullb()
979 		t_page = null_lookup_page(nullb, sector, false,  in copy_from_nullb()
980 			!null_cache_active(nullb));  in copy_from_nullb()
999 static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)  in null_handle_discard()  argument
1003 	spin_lock_irq(&nullb->lock);  in null_handle_discard()
1005 		temp = min_t(size_t, n, nullb->dev->blocksize);  in null_handle_discard()
1006 		null_free_sector(nullb, sector, false);  in null_handle_discard()
1007 		if (null_cache_active(nullb))  in null_handle_discard()
1008 			null_free_sector(nullb, sector, true);  in null_handle_discard()
1012 	spin_unlock_irq(&nullb->lock);  in null_handle_discard()
1015 static int null_handle_flush(struct nullb *nullb)  in null_handle_flush()  argument
1019 	if (!null_cache_active(nullb))  in null_handle_flush()
1022 	spin_lock_irq(&nullb->lock);  in null_handle_flush()
1024 		err = null_make_cache_space(nullb,  in null_handle_flush()
1025 			nullb->dev->cache_size * 1024 * 1024);  in null_handle_flush()
1026 		if (err || nullb->dev->curr_cache == 0)  in null_handle_flush()
1030 	WARN_ON(!radix_tree_empty(&nullb->dev->cache));  in null_handle_flush()
1031 	spin_unlock_irq(&nullb->lock);  in null_handle_flush()
1035 static int null_transfer(struct nullb *nullb, struct page *page,  in null_transfer()  argument
1042 		err = copy_from_nullb(nullb, page, off, sector, len);  in null_transfer()
1046 		err = copy_to_nullb(nullb, page, off, sector, len, is_fua);  in null_transfer()
1055 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_rq()  local
1065 		null_handle_discard(nullb, sector, blk_rq_bytes(rq));  in null_handle_rq()
1069 	spin_lock_irq(&nullb->lock);  in null_handle_rq()
1072 		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,  in null_handle_rq()
1076 			spin_unlock_irq(&nullb->lock);  in null_handle_rq()
1081 	spin_unlock_irq(&nullb->lock);  in null_handle_rq()
1089 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_bio()  local
1099 		null_handle_discard(nullb, sector,  in null_handle_bio()
1104 	spin_lock_irq(&nullb->lock);  in null_handle_bio()
1107 		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,  in null_handle_bio()
1111 			spin_unlock_irq(&nullb->lock);  in null_handle_bio()
1116 	spin_unlock_irq(&nullb->lock);  in null_handle_bio()
1120 static void null_stop_queue(struct nullb *nullb)  in null_stop_queue()  argument
1122 	struct request_queue *q = nullb->q;  in null_stop_queue()
1124 	if (nullb->dev->queue_mode == NULL_Q_MQ)  in null_stop_queue()
1128 static void null_restart_queue_async(struct nullb *nullb)  in null_restart_queue_async()  argument
1130 	struct request_queue *q = nullb->q;  in null_restart_queue_async()
1132 	if (nullb->dev->queue_mode == NULL_Q_MQ)  in null_restart_queue_async()
1139 	struct nullb *nullb = dev->nullb;  in null_handle_throttled()  local
1143 	if (!hrtimer_active(&nullb->bw_timer))  in null_handle_throttled()
1144 		hrtimer_restart(&nullb->bw_timer);  in null_handle_throttled()
1146 	if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {  in null_handle_throttled()
1147 		null_stop_queue(nullb);  in null_handle_throttled()
1149 		if (atomic_long_read(&nullb->cur_bytes) > 0)  in null_handle_throttled()
1150 			null_restart_queue_async(nullb);  in null_handle_throttled()
1215 	struct nullb *nullb = dev->nullb;  in null_handle_cmd()  local
1225 		cmd->error = errno_to_blk_status(null_handle_flush(nullb));  in null_handle_cmd()
1229 	if (nullb->dev->badblocks.shift != -1) {  in null_handle_cmd()
1248 	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);  in nullb_bwtimer_fn()  local
1250 	unsigned int mbps = nullb->dev->mbps;  in nullb_bwtimer_fn()
1252 	if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))  in nullb_bwtimer_fn()
1255 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));  in nullb_bwtimer_fn()
1256 	null_restart_queue_async(nullb);  in nullb_bwtimer_fn()
1258 	hrtimer_forward_now(&nullb->bw_timer, timer_interval);  in nullb_bwtimer_fn()
1263 static void nullb_setup_bwtimer(struct nullb *nullb)  in nullb_setup_bwtimer()  argument
1267 	hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in nullb_setup_bwtimer()
1268 	nullb->bw_timer.function = nullb_bwtimer_fn;  in nullb_setup_bwtimer()
1269 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));  in nullb_setup_bwtimer()
1270 	hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);  in nullb_setup_bwtimer()
1273 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)  in nullb_to_queue()  argument
1277 	if (nullb->nr_queues != 1)  in nullb_to_queue()
1278 		index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);  in nullb_to_queue()
1280 	return &nullb->queues[index];  in nullb_to_queue()
1287 	struct nullb *nullb = q->queuedata;  in null_queue_bio()  local
1288 	struct nullb_queue *nq = nullb_to_queue(nullb);  in null_queue_bio()
1373 static void cleanup_queues(struct nullb *nullb)  in cleanup_queues()  argument
1377 	for (i = 0; i < nullb->nr_queues; i++)  in cleanup_queues()
1378 		cleanup_queue(&nullb->queues[i]);  in cleanup_queues()
1380 	kfree(nullb->queues);  in cleanup_queues()
1383 static void null_del_dev(struct nullb *nullb)  in null_del_dev()  argument
1385 	struct nullb_device *dev = nullb->dev;  in null_del_dev()
1387 	ida_simple_remove(&nullb_indexes, nullb->index);  in null_del_dev()
1389 	list_del_init(&nullb->list);  in null_del_dev()
1391 	del_gendisk(nullb->disk);  in null_del_dev()
1393 	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {  in null_del_dev()
1394 		hrtimer_cancel(&nullb->bw_timer);  in null_del_dev()
1395 		atomic_long_set(&nullb->cur_bytes, LONG_MAX);  in null_del_dev()
1396 		null_restart_queue_async(nullb);  in null_del_dev()
1399 	blk_cleanup_queue(nullb->q);  in null_del_dev()
1401 	    nullb->tag_set == &nullb->__tag_set)  in null_del_dev()
1402 		blk_mq_free_tag_set(nullb->tag_set);  in null_del_dev()
1403 	put_disk(nullb->disk);  in null_del_dev()
1404 	cleanup_queues(nullb);  in null_del_dev()
1405 	if (null_cache_active(nullb))  in null_del_dev()
1406 		null_free_device_storage(nullb->dev, true);  in null_del_dev()
1407 	kfree(nullb);  in null_del_dev()
1408 	dev->nullb = NULL;  in null_del_dev()
1411 static void null_config_discard(struct nullb *nullb)  in null_config_discard()  argument
1413 	if (nullb->dev->discard == false)  in null_config_discard()
1415 	nullb->q->limits.discard_granularity = nullb->dev->blocksize;  in null_config_discard()
1416 	nullb->q->limits.discard_alignment = nullb->dev->blocksize;  in null_config_discard()
1417 	blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);  in null_config_discard()
1418 	blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q);  in null_config_discard()
1437 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)  in null_init_queue()  argument
1439 	BUG_ON(!nullb);  in null_init_queue()
1443 	nq->queue_depth = nullb->queue_depth;  in null_init_queue()
1444 	nq->dev = nullb->dev;  in null_init_queue()
1447 static void null_init_queues(struct nullb *nullb)  in null_init_queues()  argument
1449 	struct request_queue *q = nullb->q;  in null_init_queues()
1457 		nq = &nullb->queues[i];  in null_init_queues()
1459 		null_init_queue(nullb, nq);  in null_init_queues()
1460 		nullb->nr_queues++;  in null_init_queues()
1490 static int setup_queues(struct nullb *nullb)  in setup_queues()  argument
1492 	nullb->queues = kcalloc(nullb->dev->submit_queues,  in setup_queues()
1495 	if (!nullb->queues)  in setup_queues()
1498 	nullb->queue_depth = nullb->dev->hw_queue_depth;  in setup_queues()
1503 static int init_driver_queues(struct nullb *nullb)  in init_driver_queues()  argument
1508 	for (i = 0; i < nullb->dev->submit_queues; i++) {  in init_driver_queues()
1509 		nq = &nullb->queues[i];  in init_driver_queues()
1511 		null_init_queue(nullb, nq);  in init_driver_queues()
1516 		nullb->nr_queues++;  in init_driver_queues()
1521 static int null_gendisk_register(struct nullb *nullb)  in null_gendisk_register()  argument
1526 	disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);  in null_gendisk_register()
1529 	size = (sector_t)nullb->dev->size * 1024 * 1024ULL;  in null_gendisk_register()
1534 	disk->first_minor	= nullb->index;  in null_gendisk_register()
1536 	disk->private_data	= nullb;  in null_gendisk_register()
1537 	disk->queue		= nullb->q;  in null_gendisk_register()
1538 	strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);  in null_gendisk_register()
1540 	if (nullb->dev->zoned) {  in null_gendisk_register()
1551 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)  in null_init_tag_set()  argument
1554 	set->nr_hw_queues = nullb ? nullb->dev->submit_queues :  in null_init_tag_set()
1556 	set->queue_depth = nullb ? nullb->dev->hw_queue_depth :  in null_init_tag_set()
1558 	set->numa_node = nullb ? nullb->dev->home_node : g_home_node;  in null_init_tag_set()
1565 	if ((nullb && nullb->dev->blocking) || g_blocking)  in null_init_tag_set()
1627 	struct nullb *nullb;  in null_add_dev()  local
1632 	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);  in null_add_dev()
1633 	if (!nullb) {  in null_add_dev()
1637 	nullb->dev = dev;  in null_add_dev()
1638 	dev->nullb = nullb;  in null_add_dev()
1640 	spin_lock_init(&nullb->lock);  in null_add_dev()
1642 	rv = setup_queues(nullb);  in null_add_dev()
1648 			nullb->tag_set = &tag_set;  in null_add_dev()
1651 			nullb->tag_set = &nullb->__tag_set;  in null_add_dev()
1652 			rv = null_init_tag_set(nullb, nullb->tag_set);  in null_add_dev()
1661 		nullb->tag_set->timeout = 5 * HZ;  in null_add_dev()
1662 		nullb->q = blk_mq_init_queue(nullb->tag_set);  in null_add_dev()
1663 		if (IS_ERR(nullb->q)) {  in null_add_dev()
1667 		null_init_queues(nullb);  in null_add_dev()
1669 		nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node);  in null_add_dev()
1670 		if (!nullb->q) {  in null_add_dev()
1674 		blk_queue_make_request(nullb->q, null_queue_bio);  in null_add_dev()
1675 		rv = init_driver_queues(nullb);  in null_add_dev()
1682 		nullb_setup_bwtimer(nullb);  in null_add_dev()
1686 		set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);  in null_add_dev()
1687 		blk_queue_write_cache(nullb->q, true, true);  in null_add_dev()
1695 		blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects);  in null_add_dev()
1696 		nullb->q->limits.zoned = BLK_ZONED_HM;  in null_add_dev()
1697 		blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, nullb->q);  in null_add_dev()
1698 		blk_queue_required_elevator_features(nullb->q,  in null_add_dev()
1702 	nullb->q->queuedata = nullb;  in null_add_dev()
1703 	blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);  in null_add_dev()
1704 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);  in null_add_dev()
1707 	nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);  in null_add_dev()
1708 	dev->index = nullb->index;  in null_add_dev()
1711 	blk_queue_logical_block_size(nullb->q, dev->blocksize);  in null_add_dev()
1712 	blk_queue_physical_block_size(nullb->q, dev->blocksize);  in null_add_dev()
1714 	null_config_discard(nullb);  in null_add_dev()
1716 	sprintf(nullb->disk_name, "nullb%d", nullb->index);  in null_add_dev()
1718 	rv = null_gendisk_register(nullb);  in null_add_dev()
1723 	list_add_tail(&nullb->list, &nullb_list);  in null_add_dev()
1731 	blk_cleanup_queue(nullb->q);  in null_add_dev()
1733 	if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)  in null_add_dev()
1734 		blk_mq_free_tag_set(nullb->tag_set);  in null_add_dev()
1736 	cleanup_queues(nullb);  in null_add_dev()
1738 	kfree(nullb);  in null_add_dev()
1747 	struct nullb *nullb;  in null_init()  local
1820 		nullb = list_entry(nullb_list.next, struct nullb, list);  in null_init()
1821 		dev = nullb->dev;  in null_init()
1822 		null_del_dev(nullb);  in null_init()
1836 	struct nullb *nullb;  in null_exit()  local
1846 		nullb = list_entry(nullb_list.next, struct nullb, list);  in null_exit()
1847 		dev = nullb->dev;  in null_exit()
1848 		null_del_dev(nullb);  in null_exit()