545 static void put_tag(struct nullb_queue *nq, unsigned int tag)  in put_tag()  argument
547 	clear_bit_unlock(tag, nq->tag_map);  in put_tag()
549 	if (waitqueue_active(&nq->wait))  in put_tag()
550 		wake_up(&nq->wait);  in put_tag()
553 static unsigned int get_tag(struct nullb_queue *nq)  in get_tag()  argument
558 		tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);  in get_tag()
559 		if (tag >= nq->queue_depth)  in get_tag()
561 	} while (test_and_set_bit_lock(tag, nq->tag_map));  in get_tag()
568 	put_tag(cmd->nq, cmd->tag);  in free_cmd()
573 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)  in __alloc_cmd()  argument
578 	tag = get_tag(nq);  in __alloc_cmd()
580 		cmd = &nq->cmds[tag];  in __alloc_cmd()
582 		cmd->nq = nq;  in __alloc_cmd()
583 		if (nq->dev->irqmode == NULL_IRQ_TIMER) {  in __alloc_cmd()
594 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)  in alloc_cmd()  argument
599 	cmd = __alloc_cmd(nq);  in alloc_cmd()
604 		prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);  in alloc_cmd()
605 		cmd = __alloc_cmd(nq);  in alloc_cmd()
612 	finish_wait(&nq->wait, &wait);  in alloc_cmd()
618 	int queue_mode = cmd->nq->dev->queue_mode;  in end_cmd()
642 	ktime_t kt = cmd->nq->dev->completion_nsec;  in null_cmd_end_timer()
1055 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_rq()
1089 	struct nullb *nullb = cmd->nq->dev->nullb;  in null_handle_bio()
1138 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_throttled()
1161 	struct badblocks *bb = &cmd->nq->dev->badblocks;  in null_handle_badblocks()
1174 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_memory_backed()
1188 	switch (cmd->nq->dev->irqmode) {  in nullb_complete_cmd()
1190 		switch (cmd->nq->dev->queue_mode) {  in nullb_complete_cmd()
1214 	struct nullb_device *dev = cmd->nq->dev;  in null_handle_cmd()
1288 	struct nullb_queue *nq = nullb_to_queue(nullb);  in null_queue_bio()  local
1291 	cmd = alloc_cmd(nq, 1);  in null_queue_bio()
1327 	struct nullb_queue *nq = hctx->driver_data;  in null_queue_rq()  local
1333 	if (nq->dev->irqmode == NULL_IRQ_TIMER) {  in null_queue_rq()
1338 	cmd->nq = nq;  in null_queue_rq()
1347 		nq->requeue_selection++;  in null_queue_rq()
1348 		if (nq->requeue_selection & 1)  in null_queue_rq()
1367 static void cleanup_queue(struct nullb_queue *nq)  in cleanup_queue()  argument
1369 	kfree(nq->tag_map);  in cleanup_queue()
1370 	kfree(nq->cmds);  in cleanup_queue()
1437 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)  in null_init_queue()  argument
1440 	BUG_ON(!nq);  in null_init_queue()
1442 	init_waitqueue_head(&nq->wait);  in null_init_queue()
1443 	nq->queue_depth = nullb->queue_depth;  in null_init_queue()
1444 	nq->dev = nullb->dev;  in null_init_queue()
1451 	struct nullb_queue *nq;  in null_init_queues()  local
1457 		nq = &nullb->queues[i];  in null_init_queues()
1458 		hctx->driver_data = nq;  in null_init_queues()
1459 		null_init_queue(nullb, nq);  in null_init_queues()
1464 static int setup_commands(struct nullb_queue *nq)  in setup_commands()  argument
1469 	nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);  in setup_commands()
1470 	if (!nq->cmds)  in setup_commands()
1473 	tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;  in setup_commands()
1474 	nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);  in setup_commands()
1475 	if (!nq->tag_map) {  in setup_commands()
1476 		kfree(nq->cmds);  in setup_commands()
1480 	for (i = 0; i < nq->queue_depth; i++) {  in setup_commands()
1481 		cmd = &nq->cmds[i];  in setup_commands()
1505 	struct nullb_queue *nq;  in init_driver_queues()  local
1509 		nq = &nullb->queues[i];  in init_driver_queues()
1511 		null_init_queue(nullb, nq);  in init_driver_queues()
1513 		ret = setup_commands(nq);  in init_driver_queues()