Lines Matching refs:nullb
246 static void null_del_dev(struct nullb *nullb);
248 static struct nullb *null_find_dev_by_name(const char *name);
353 if (!dev->nullb) in nullb_update_nr_hw_queues()
378 set = dev->nullb->tag_set; in nullb_update_nr_hw_queues()
462 null_del_dev(dev->nullb); in nullb_device_power_store()
643 null_del_dev(dev->nullb); in nullb_group_drop_item()
689 static inline int null_cache_active(struct nullb *nullb) in null_cache_active() argument
691 return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); in null_cache_active()
896 static void null_free_sector(struct nullb *nullb, sector_t sector, in null_free_sector() argument
904 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; in null_free_sector()
917 nullb->dev->curr_cache -= PAGE_SIZE; in null_free_sector()
922 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx, in null_radix_tree_insert() argument
927 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; in null_radix_tree_insert()
934 nullb->dev->curr_cache += PAGE_SIZE; in null_radix_tree_insert()
968 static struct nullb_page *__null_lookup_page(struct nullb *nullb, in __null_lookup_page() argument
979 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; in __null_lookup_page()
989 static struct nullb_page *null_lookup_page(struct nullb *nullb, in null_lookup_page() argument
995 page = __null_lookup_page(nullb, sector, for_write, true); in null_lookup_page()
998 return __null_lookup_page(nullb, sector, for_write, false); in null_lookup_page()
1001 static struct nullb_page *null_insert_page(struct nullb *nullb, in null_insert_page() argument
1003 __releases(&nullb->lock) in null_insert_page()
1004 __acquires(&nullb->lock) in null_insert_page()
1009 t_page = null_lookup_page(nullb, sector, true, ignore_cache); in null_insert_page()
1013 spin_unlock_irq(&nullb->lock); in null_insert_page()
1022 spin_lock_irq(&nullb->lock); in null_insert_page()
1025 t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache); in null_insert_page()
1032 spin_lock_irq(&nullb->lock); in null_insert_page()
1033 return null_lookup_page(nullb, sector, true, ignore_cache); in null_insert_page()
1036 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page) in null_flush_cache_page() argument
1046 t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true); in null_flush_cache_page()
1052 ret = radix_tree_delete_item(&nullb->dev->data, in null_flush_cache_page()
1066 i += (nullb->dev->blocksize >> SECTOR_SHIFT)) { in null_flush_cache_page()
1070 nullb->dev->blocksize); in null_flush_cache_page()
1078 ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page); in null_flush_cache_page()
1080 nullb->dev->curr_cache -= PAGE_SIZE; in null_flush_cache_page()
1085 static int null_make_cache_space(struct nullb *nullb, unsigned long n) in null_make_cache_space() argument
1092 if ((nullb->dev->cache_size * 1024 * 1024) > in null_make_cache_space()
1093 nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0) in null_make_cache_space()
1096 nr_pages = radix_tree_gang_lookup(&nullb->dev->cache, in null_make_cache_space()
1097 (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH); in null_make_cache_space()
1103 nullb->cache_flush_pos = c_pages[i]->page->index; in null_make_cache_space()
1118 err = null_flush_cache_page(nullb, c_pages[i]); in null_make_cache_space()
1127 nullb->cache_flush_pos = 0; in null_make_cache_space()
1130 spin_unlock_irq(&nullb->lock); in null_make_cache_space()
1131 spin_lock_irq(&nullb->lock); in null_make_cache_space()
1138 static int copy_to_nullb(struct nullb *nullb, struct page *source, in copy_to_nullb() argument
1146 temp = min_t(size_t, nullb->dev->blocksize, n - count); in copy_to_nullb()
1148 if (null_cache_active(nullb) && !is_fua) in copy_to_nullb()
1149 null_make_cache_space(nullb, PAGE_SIZE); in copy_to_nullb()
1152 t_page = null_insert_page(nullb, sector, in copy_to_nullb()
1153 !null_cache_active(nullb) || is_fua); in copy_to_nullb()
1162 null_free_sector(nullb, sector, true); in copy_to_nullb()
1170 static int copy_from_nullb(struct nullb *nullb, struct page *dest, in copy_from_nullb() argument
1178 temp = min_t(size_t, nullb->dev->blocksize, n - count); in copy_from_nullb()
1181 t_page = null_lookup_page(nullb, sector, false, in copy_from_nullb()
1182 !null_cache_active(nullb)); in copy_from_nullb()
1196 static void nullb_fill_pattern(struct nullb *nullb, struct page *page, in nullb_fill_pattern() argument
1205 struct nullb *nullb = dev->nullb; in null_handle_discard() local
1209 spin_lock_irq(&nullb->lock); in null_handle_discard()
1212 null_free_sector(nullb, sector, false); in null_handle_discard()
1213 if (null_cache_active(nullb)) in null_handle_discard()
1214 null_free_sector(nullb, sector, true); in null_handle_discard()
1218 spin_unlock_irq(&nullb->lock); in null_handle_discard()
1223 static int null_handle_flush(struct nullb *nullb) in null_handle_flush() argument
1227 if (!null_cache_active(nullb)) in null_handle_flush()
1230 spin_lock_irq(&nullb->lock); in null_handle_flush()
1232 err = null_make_cache_space(nullb, in null_handle_flush()
1233 nullb->dev->cache_size * 1024 * 1024); in null_handle_flush()
1234 if (err || nullb->dev->curr_cache == 0) in null_handle_flush()
1238 WARN_ON(!radix_tree_empty(&nullb->dev->cache)); in null_handle_flush()
1239 spin_unlock_irq(&nullb->lock); in null_handle_flush()
1243 static int null_transfer(struct nullb *nullb, struct page *page, in null_transfer() argument
1247 struct nullb_device *dev = nullb->dev; in null_transfer()
1253 valid_len = null_zone_valid_read_len(nullb, in null_transfer()
1257 err = copy_from_nullb(nullb, page, off, in null_transfer()
1264 nullb_fill_pattern(nullb, page, len, off); in null_transfer()
1268 err = copy_to_nullb(nullb, page, off, sector, len, is_fua); in null_transfer()
1277 struct nullb *nullb = cmd->nq->dev->nullb; in null_handle_rq() local
1284 spin_lock_irq(&nullb->lock); in null_handle_rq()
1287 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, in null_handle_rq()
1291 spin_unlock_irq(&nullb->lock); in null_handle_rq()
1296 spin_unlock_irq(&nullb->lock); in null_handle_rq()
1304 struct nullb *nullb = cmd->nq->dev->nullb; in null_handle_bio() local
1311 spin_lock_irq(&nullb->lock); in null_handle_bio()
1314 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, in null_handle_bio()
1318 spin_unlock_irq(&nullb->lock); in null_handle_bio()
1323 spin_unlock_irq(&nullb->lock); in null_handle_bio()
1327 static void null_stop_queue(struct nullb *nullb) in null_stop_queue() argument
1329 struct request_queue *q = nullb->q; in null_stop_queue()
1331 if (nullb->dev->queue_mode == NULL_Q_MQ) in null_stop_queue()
1335 static void null_restart_queue_async(struct nullb *nullb) in null_restart_queue_async() argument
1337 struct request_queue *q = nullb->q; in null_restart_queue_async()
1339 if (nullb->dev->queue_mode == NULL_Q_MQ) in null_restart_queue_async()
1346 struct nullb *nullb = dev->nullb; in null_handle_throttled() local
1350 if (!hrtimer_active(&nullb->bw_timer)) in null_handle_throttled()
1351 hrtimer_restart(&nullb->bw_timer); in null_handle_throttled()
1353 if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) { in null_handle_throttled()
1354 null_stop_queue(nullb); in null_handle_throttled()
1356 if (atomic_long_read(&nullb->cur_bytes) > 0) in null_handle_throttled()
1357 null_restart_queue_async(nullb); in null_handle_throttled()
1471 struct nullb *nullb = dev->nullb; in null_handle_cmd() local
1481 cmd->error = errno_to_blk_status(null_handle_flush(nullb)); in null_handle_cmd()
1501 struct nullb *nullb = container_of(timer, struct nullb, bw_timer); in nullb_bwtimer_fn() local
1503 unsigned int mbps = nullb->dev->mbps; in nullb_bwtimer_fn()
1505 if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps)) in nullb_bwtimer_fn()
1508 atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps)); in nullb_bwtimer_fn()
1509 null_restart_queue_async(nullb); in nullb_bwtimer_fn()
1511 hrtimer_forward_now(&nullb->bw_timer, timer_interval); in nullb_bwtimer_fn()
1516 static void nullb_setup_bwtimer(struct nullb *nullb) in nullb_setup_bwtimer() argument
1520 hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in nullb_setup_bwtimer()
1521 nullb->bw_timer.function = nullb_bwtimer_fn; in nullb_setup_bwtimer()
1522 atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps)); in nullb_setup_bwtimer()
1523 hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL); in nullb_setup_bwtimer()
1526 static struct nullb_queue *nullb_to_queue(struct nullb *nullb) in nullb_to_queue() argument
1530 if (nullb->nr_queues != 1) in nullb_to_queue()
1531 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues); in nullb_to_queue()
1533 return &nullb->queues[index]; in nullb_to_queue()
1540 struct nullb *nullb = bio->bi_bdev->bd_disk->private_data; in null_submit_bio() local
1541 struct nullb_queue *nq = nullb_to_queue(nullb); in null_submit_bio()
1590 struct nullb *nullb = set->driver_data; in null_map_queues() local
1595 if (nullb) { in null_map_queues()
1596 struct nullb_device *dev = nullb->dev; in null_map_queues()
1759 static void cleanup_queues(struct nullb *nullb) in cleanup_queues() argument
1763 for (i = 0; i < nullb->nr_queues; i++) in cleanup_queues()
1764 cleanup_queue(&nullb->queues[i]); in cleanup_queues()
1766 kfree(nullb->queues); in cleanup_queues()
1772 struct nullb *nullb = nq->dev->nullb; in null_exit_hctx() local
1774 nullb->nr_queues--; in null_exit_hctx()
1777 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) in null_init_queue() argument
1780 nq->queue_depth = nullb->queue_depth; in null_init_queue()
1781 nq->dev = nullb->dev; in null_init_queue()
1789 struct nullb *nullb = hctx->queue->queuedata; in null_init_hctx() local
1792 if (should_init_hctx_fail(nullb->dev)) in null_init_hctx()
1795 nq = &nullb->queues[hctx_idx]; in null_init_hctx()
1797 null_init_queue(nullb, nq); in null_init_hctx()
1798 nullb->nr_queues++; in null_init_hctx()
1813 static void null_del_dev(struct nullb *nullb) in null_del_dev() argument
1817 if (!nullb) in null_del_dev()
1820 dev = nullb->dev; in null_del_dev()
1822 ida_simple_remove(&nullb_indexes, nullb->index); in null_del_dev()
1824 list_del_init(&nullb->list); in null_del_dev()
1826 del_gendisk(nullb->disk); in null_del_dev()
1828 if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) { in null_del_dev()
1829 hrtimer_cancel(&nullb->bw_timer); in null_del_dev()
1830 atomic_long_set(&nullb->cur_bytes, LONG_MAX); in null_del_dev()
1831 null_restart_queue_async(nullb); in null_del_dev()
1834 put_disk(nullb->disk); in null_del_dev()
1836 nullb->tag_set == &nullb->__tag_set) in null_del_dev()
1837 blk_mq_free_tag_set(nullb->tag_set); in null_del_dev()
1838 cleanup_queues(nullb); in null_del_dev()
1839 if (null_cache_active(nullb)) in null_del_dev()
1840 null_free_device_storage(nullb->dev, true); in null_del_dev()
1841 kfree(nullb); in null_del_dev()
1842 dev->nullb = NULL; in null_del_dev()
1845 static void null_config_discard(struct nullb *nullb) in null_config_discard() argument
1847 if (nullb->dev->discard == false) in null_config_discard()
1850 if (!nullb->dev->memory_backed) { in null_config_discard()
1851 nullb->dev->discard = false; in null_config_discard()
1856 if (nullb->dev->zoned) { in null_config_discard()
1857 nullb->dev->discard = false; in null_config_discard()
1862 nullb->q->limits.discard_granularity = nullb->dev->blocksize; in null_config_discard()
1863 blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9); in null_config_discard()
1900 static int setup_queues(struct nullb *nullb) in setup_queues() argument
1907 nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue), in setup_queues()
1909 if (!nullb->queues) in setup_queues()
1912 nullb->queue_depth = nullb->dev->hw_queue_depth; in setup_queues()
1916 static int init_driver_queues(struct nullb *nullb) in init_driver_queues() argument
1921 for (i = 0; i < nullb->dev->submit_queues; i++) { in init_driver_queues()
1922 nq = &nullb->queues[i]; in init_driver_queues()
1924 null_init_queue(nullb, nq); in init_driver_queues()
1929 nullb->nr_queues++; in init_driver_queues()
1934 static int null_gendisk_register(struct nullb *nullb) in null_gendisk_register() argument
1936 sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT; in null_gendisk_register()
1937 struct gendisk *disk = nullb->disk; in null_gendisk_register()
1942 disk->first_minor = nullb->index; in null_gendisk_register()
1944 if (queue_is_mq(nullb->q)) in null_gendisk_register()
1948 disk->private_data = nullb; in null_gendisk_register()
1949 strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN); in null_gendisk_register()
1951 if (nullb->dev->zoned) { in null_gendisk_register()
1952 int ret = null_register_zoned_dev(nullb); in null_gendisk_register()
1961 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) in null_init_tag_set() argument
1968 if (nullb) { in null_init_tag_set()
1969 hw_queues = nullb->dev->submit_queues; in null_init_tag_set()
1970 poll_queues = nullb->dev->poll_queues; in null_init_tag_set()
1971 queue_depth = nullb->dev->hw_queue_depth; in null_init_tag_set()
1972 numa_node = nullb->dev->home_node; in null_init_tag_set()
1973 if (nullb->dev->no_sched) in null_init_tag_set()
1975 if (nullb->dev->shared_tag_bitmap) in null_init_tag_set()
1977 if (nullb->dev->blocking) in null_init_tag_set()
1995 set->driver_data = nullb; in null_init_tag_set()
2085 struct nullb *nullb; in null_add_dev() local
2092 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node); in null_add_dev()
2093 if (!nullb) { in null_add_dev()
2097 nullb->dev = dev; in null_add_dev()
2098 dev->nullb = nullb; in null_add_dev()
2100 spin_lock_init(&nullb->lock); in null_add_dev()
2102 rv = setup_queues(nullb); in null_add_dev()
2108 nullb->tag_set = &tag_set; in null_add_dev()
2111 nullb->tag_set = &nullb->__tag_set; in null_add_dev()
2112 rv = null_init_tag_set(nullb, nullb->tag_set); in null_add_dev()
2118 nullb->tag_set->timeout = 5 * HZ; in null_add_dev()
2119 nullb->disk = blk_mq_alloc_disk(nullb->tag_set, nullb); in null_add_dev()
2120 if (IS_ERR(nullb->disk)) { in null_add_dev()
2121 rv = PTR_ERR(nullb->disk); in null_add_dev()
2124 nullb->q = nullb->disk->queue; in null_add_dev()
2127 nullb->disk = blk_alloc_disk(nullb->dev->home_node); in null_add_dev()
2128 if (!nullb->disk) in null_add_dev()
2131 nullb->q = nullb->disk->queue; in null_add_dev()
2132 rv = init_driver_queues(nullb); in null_add_dev()
2139 nullb_setup_bwtimer(nullb); in null_add_dev()
2143 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); in null_add_dev()
2144 blk_queue_write_cache(nullb->q, true, true); in null_add_dev()
2148 rv = null_init_zoned_dev(dev, nullb->q); in null_add_dev()
2153 nullb->q->queuedata = nullb; in null_add_dev()
2154 blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q); in null_add_dev()
2162 nullb->index = rv; in null_add_dev()
2166 blk_queue_logical_block_size(nullb->q, dev->blocksize); in null_add_dev()
2167 blk_queue_physical_block_size(nullb->q, dev->blocksize); in null_add_dev()
2169 dev->max_sectors = queue_max_hw_sectors(nullb->q); in null_add_dev()
2171 blk_queue_max_hw_sectors(nullb->q, dev->max_sectors); in null_add_dev()
2174 blk_queue_virt_boundary(nullb->q, PAGE_SIZE - 1); in null_add_dev()
2176 null_config_discard(nullb); in null_add_dev()
2180 snprintf(nullb->disk_name, sizeof(nullb->disk_name), in null_add_dev()
2183 sprintf(nullb->disk_name, "nullb%d", nullb->index); in null_add_dev()
2186 rv = null_gendisk_register(nullb); in null_add_dev()
2191 list_add_tail(&nullb->list, &nullb_list); in null_add_dev()
2194 pr_info("disk %s created\n", nullb->disk_name); in null_add_dev()
2199 ida_free(&nullb_indexes, nullb->index); in null_add_dev()
2203 put_disk(nullb->disk); in null_add_dev()
2205 if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) in null_add_dev()
2206 blk_mq_free_tag_set(nullb->tag_set); in null_add_dev()
2208 cleanup_queues(nullb); in null_add_dev()
2210 kfree(nullb); in null_add_dev()
2211 dev->nullb = NULL; in null_add_dev()
2216 static struct nullb *null_find_dev_by_name(const char *name) in null_find_dev_by_name()
2218 struct nullb *nullb = NULL, *nb; in null_find_dev_by_name() local
2223 nullb = nb; in null_find_dev_by_name()
2229 return nullb; in null_find_dev_by_name()
2250 static void null_destroy_dev(struct nullb *nullb) in null_destroy_dev() argument
2252 struct nullb_device *dev = nullb->dev; in null_destroy_dev()
2254 null_del_dev(nullb); in null_destroy_dev()
2263 struct nullb *nullb; in null_init() local
2334 nullb = list_entry(nullb_list.next, struct nullb, list); in null_init()
2335 null_destroy_dev(nullb); in null_init()
2348 struct nullb *nullb; in null_exit() local
2356 nullb = list_entry(nullb_list.next, struct nullb, list); in null_exit()
2357 null_destroy_dev(nullb); in null_exit()