536 static void put_tag(struct nullb_queue *nq, unsigned int tag)  in put_tag()  argument
538 	clear_bit_unlock(tag, nq->tag_map);  in put_tag()
540 	if (waitqueue_active(&nq->wait))  in put_tag()
541 		wake_up(&nq->wait);  in put_tag()
544 static unsigned int get_tag(struct nullb_queue *nq)  in get_tag()  argument
549 		tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);  in get_tag()
550 		if (tag >= nq->queue_depth)  in get_tag()
552 	} while (test_and_set_bit_lock(tag, nq->tag_map));  in get_tag()
559 	put_tag(cmd->nq, cmd->tag);  in free_cmd()
564 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)  in __alloc_cmd()  argument
569 	tag = get_tag(nq);  in __alloc_cmd()
571 		cmd = &nq->cmds[tag];  in __alloc_cmd()
573 		cmd->nq = nq;  in __alloc_cmd()
574 		if (nq->dev->irqmode == NULL_IRQ_TIMER) {  in __alloc_cmd()
585 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)  in alloc_cmd()  argument
590 	cmd = __alloc_cmd(nq);  in alloc_cmd()
595 		prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);  in alloc_cmd()
596 		cmd = __alloc_cmd(nq);  in alloc_cmd()
603 	finish_wait(&nq->wait, &wait);  in alloc_cmd()
610 	int queue_mode = cmd->nq->dev->queue_mode;  in end_cmd()
650 	ktime_t kt = cmd->nq->dev->completion_nsec;  in null_cmd_end_timer()
1068 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_rq()
1102 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_bio()
1162 	struct nullb_device *dev = cmd->nq->dev;  in cmd_report_zone()
1181 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_cmd()
1342 	struct nullb_queue *nq = nullb_to_queue(nullb);  in null_queue_bio()  local
1345 	cmd = alloc_cmd(nq, 1);  in null_queue_bio()
1362 	struct nullb_queue *nq = nullb_to_queue(nullb);  in null_rq_prep_fn()  local
1365 	cmd = alloc_cmd(nq, 0);  in null_rq_prep_fn()
1426 	struct nullb_queue *nq = hctx->driver_data;  in null_queue_rq()  local
1430 	if (nq->dev->irqmode == NULL_IRQ_TIMER) {  in null_queue_rq()
1435 	cmd->nq = nq;  in null_queue_rq()
1444 		nq->requeue_selection++;  in null_queue_rq()
1445 		if (nq->requeue_selection & 1)  in null_queue_rq()
1464 static void cleanup_queue(struct nullb_queue *nq)  in cleanup_queue()  argument
1466 	kfree(nq->tag_map);  in cleanup_queue()
1467 	kfree(nq->cmds);  in cleanup_queue()
1533 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)  in null_init_queue()  argument
1536 	BUG_ON(!nq);  in null_init_queue()
1538 	init_waitqueue_head(&nq->wait);  in null_init_queue()
1539 	nq->queue_depth = nullb->queue_depth;  in null_init_queue()
1540 	nq->dev = nullb->dev;  in null_init_queue()
1547 	struct nullb_queue *nq;  in null_init_queues()  local
1553 		nq = &nullb->queues[i];  in null_init_queues()
1554 		hctx->driver_data = nq;  in null_init_queues()
1555 		null_init_queue(nullb, nq);  in null_init_queues()
1560 static int setup_commands(struct nullb_queue *nq)  in setup_commands()  argument
1565 	nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);  in setup_commands()
1566 	if (!nq->cmds)  in setup_commands()
1569 	tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;  in setup_commands()
1570 	nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);  in setup_commands()
1571 	if (!nq->tag_map) {  in setup_commands()
1572 		kfree(nq->cmds);  in setup_commands()
1576 	for (i = 0; i < nq->queue_depth; i++) {  in setup_commands()
1577 		cmd = &nq->cmds[i];  in setup_commands()
1602 	struct nullb_queue *nq;  in init_driver_queues()  local
1606 		nq = &nullb->queues[i];  in init_driver_queues()
1608 		null_init_queue(nullb, nq);  in init_driver_queues()
1610 		ret = setup_commands(nq);  in init_driver_queues()