755 static void put_tag(struct nullb_queue *nq, unsigned int tag)  in put_tag()  argument
757 	clear_bit_unlock(tag, nq->tag_map);  in put_tag()
759 	if (waitqueue_active(&nq->wait))  in put_tag()
760 		wake_up(&nq->wait);  in put_tag()
763 static unsigned int get_tag(struct nullb_queue *nq)  in get_tag()  argument
768 		tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);  in get_tag()
769 		if (tag >= nq->queue_depth)  in get_tag()
771 	} while (test_and_set_bit_lock(tag, nq->tag_map));  in get_tag()
778 	put_tag(cmd->nq, cmd->tag);  in free_cmd()
783 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)  in __alloc_cmd()  argument
788 	tag = get_tag(nq);  in __alloc_cmd()
790 		cmd = &nq->cmds[tag];  in __alloc_cmd()
793 		cmd->nq = nq;  in __alloc_cmd()
794 		if (nq->dev->irqmode == NULL_IRQ_TIMER) {  in __alloc_cmd()
805 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, struct bio *bio)  in alloc_cmd()  argument
815 		cmd = __alloc_cmd(nq);  in alloc_cmd()
820 		prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);  in alloc_cmd()
822 		finish_wait(&nq->wait, &wait);  in alloc_cmd()
828 	int queue_mode = cmd->nq->dev->queue_mode;  in end_cmd()
852 	ktime_t kt = cmd->nq->dev->completion_nsec;  in null_cmd_end_timer()
1277 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_rq()
1304 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_bio()
1345 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_throttled()
1368 	struct badblocks *bb = &cmd->nq->dev->badblocks;  in null_handle_badblocks()
1383 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_memory_backed()
1399 	struct nullb_device *dev = cmd->nq->dev;  in nullb_zero_read_cmd_buffer()
1426 	switch (cmd->nq->dev->irqmode) {  in nullb_complete_cmd()
1428 		switch (cmd->nq->dev->queue_mode) {  in nullb_complete_cmd()
1452 	struct nullb_device *dev = cmd->nq->dev;  in null_process_cmd()
1470 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_cmd()
1541 	struct nullb_queue *nq = nullb_to_queue(nullb);  in null_submit_bio()  local
1543 	null_handle_cmd(alloc_cmd(nq, bio), sector, nr_sectors, bio_op(bio));  in null_submit_bio()
1551 	struct nullb_device *dev = cmd->nq->dev;  in should_timeout_request()
1559 	struct nullb_device *dev = cmd->nq->dev;  in should_requeue_request()
1643 	struct nullb_queue *nq = hctx->driver_data;  in null_poll()  local
1648 	spin_lock(&nq->poll_lock);  in null_poll()
1649 	list_splice_init(&nq->poll_list, &list);  in null_poll()
1652 	spin_unlock(&nq->poll_lock);  in null_poll()
1678 		struct nullb_queue *nq = hctx->driver_data;  in null_timeout_rq()  local
1680 		spin_lock(&nq->poll_lock);  in null_timeout_rq()
1683 			spin_unlock(&nq->poll_lock);  in null_timeout_rq()
1687 		spin_unlock(&nq->poll_lock);  in null_timeout_rq()
1710 	struct nullb_queue *nq = hctx->driver_data;  in null_queue_rq()  local
1717 	if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {  in null_queue_rq()
1723 	cmd->nq = nq;  in null_queue_rq()
1734 		nq->requeue_selection++;  in null_queue_rq()
1735 		if (nq->requeue_selection & 1)  in null_queue_rq()
1742 		spin_lock(&nq->poll_lock);  in null_queue_rq()
1743 		list_add_tail(&rq->queuelist, &nq->poll_list);  in null_queue_rq()
1744 		spin_unlock(&nq->poll_lock);  in null_queue_rq()
1753 static void cleanup_queue(struct nullb_queue *nq)  in cleanup_queue()  argument
1755 	bitmap_free(nq->tag_map);  in cleanup_queue()
1756 	kfree(nq->cmds);  in cleanup_queue()
1771 	struct nullb_queue *nq = hctx->driver_data;  in null_exit_hctx()  local
1772 	struct nullb *nullb = nq->dev->nullb;  in null_exit_hctx()
1777 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)  in null_init_queue()  argument
1779 	init_waitqueue_head(&nq->wait);  in null_init_queue()
1780 	nq->queue_depth = nullb->queue_depth;  in null_init_queue()
1781 	nq->dev = nullb->dev;  in null_init_queue()
1782 	INIT_LIST_HEAD(&nq->poll_list);  in null_init_queue()
1783 	spin_lock_init(&nq->poll_lock);  in null_init_queue()
1790 	struct nullb_queue *nq;  in null_init_hctx()  local
1795 	nq = &nullb->queues[hctx_idx];  in null_init_hctx()
1796 	hctx->driver_data = nq;  in null_init_hctx()
1797 	null_init_queue(nullb, nq);  in null_init_hctx()
1877 static int setup_commands(struct nullb_queue *nq)  in setup_commands()  argument
1882 	nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);  in setup_commands()
1883 	if (!nq->cmds)  in setup_commands()
1886 	nq->tag_map = bitmap_zalloc(nq->queue_depth, GFP_KERNEL);  in setup_commands()
1887 	if (!nq->tag_map) {  in setup_commands()
1888 		kfree(nq->cmds);  in setup_commands()
1892 	for (i = 0; i < nq->queue_depth; i++) {  in setup_commands()
1893 		cmd = &nq->cmds[i];  in setup_commands()
1918 	struct nullb_queue *nq;  in init_driver_queues()  local
1922 		nq = &nullb->queues[i];  in init_driver_queues()
1924 		null_init_queue(nullb, nq);  in init_driver_queues()
1926 		ret = setup_commands(nq);  in init_driver_queues()