106 	struct virtio_blk *vblk = hctx->queue->queuedata;  in get_virtio_blk_vq()  local
107 	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];  in get_virtio_blk_vq()
277 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
284 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
287 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
300 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
301 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
306 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_commit_rqs()  local
307 	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];  in virtio_commit_rqs()
319 					struct virtio_blk *vblk,  in virtblk_prep_rq()  argument
325 	status = virtblk_setup_cmd(vblk->vdev, req, vbr);  in virtblk_prep_rq()
343 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
352 	status = virtblk_prep_rq(hctx, vblk, req, vbr);  in virtio_queue_rq()
356 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
357 	err = virtblk_add_req(vblk->vqs[qid].vq, vbr);  in virtio_queue_rq()
359 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
365 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
378 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
380 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
383 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
389 	struct virtio_blk *vblk = req->mq_hctx->queue->queuedata;  in virtblk_prep_rq_batch()  local
394 	return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK;  in virtblk_prep_rq_batch()
459 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
460 	struct request_queue *q = vblk->disk->queue;  in virtblk_get_id()
482 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
485 	mutex_lock(&vblk->vdev_mutex);  in virtblk_getgeo()
487 	if (!vblk->vdev) {  in virtblk_getgeo()
493 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
494 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
496 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
498 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
507 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_getgeo()
513 	struct virtio_blk *vblk = disk->private_data;  in virtblk_free_disk()  local
515 	ida_simple_remove(&vd_index_ida, vblk->index);  in virtblk_free_disk()
516 	mutex_destroy(&vblk->vdev_mutex);  in virtblk_free_disk()
517 	kfree(vblk);  in virtblk_free_disk()
559 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)  in virtblk_update_capacity()  argument
561 	struct virtio_device *vdev = vblk->vdev;  in virtblk_update_capacity()
562 	struct request_queue *q = vblk->disk->queue;  in virtblk_update_capacity()
579 		   vblk->disk->disk_name,  in virtblk_update_capacity()
586 	set_capacity_and_notify(vblk->disk, capacity);  in virtblk_update_capacity()
591 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
594 	virtblk_update_capacity(vblk, true);  in virtblk_config_changed_work()
599 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
601 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
604 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
613 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
633 	vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs;  in init_vq()
634 	vblk->io_queues[HCTX_TYPE_READ] = 0;  in init_vq()
635 	vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs;  in init_vq()
638 				vblk->io_queues[HCTX_TYPE_DEFAULT],  in init_vq()
639 				vblk->io_queues[HCTX_TYPE_READ],  in init_vq()
640 				vblk->io_queues[HCTX_TYPE_POLL]);  in init_vq()
642 	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);  in init_vq()
643 	if (!vblk->vqs)  in init_vq()
656 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
657 		names[i] = vblk->vqs[i].name;  in init_vq()
662 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%d", i);  in init_vq()
663 		names[i] = vblk->vqs[i].name;  in init_vq()
672 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
673 		vblk->vqs[i].vq = vqs[i];  in init_vq()
675 	vblk->num_vqs = num_vqs;  in init_vq()
682 		kfree(vblk->vqs);  in init_vq()
736 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
738 	blk_queue_write_cache(vblk->disk->queue, writeback, false);  in virtblk_update_cache_mode()
750 	struct virtio_blk *vblk = disk->private_data;  in cache_type_store()  local
751 	struct virtio_device *vdev = vblk->vdev;  in cache_type_store()
754 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in cache_type_store()
768 	struct virtio_blk *vblk = disk->private_data;  in cache_type_show()  local
769 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in cache_type_show()
788 	struct virtio_blk *vblk = disk->private_data;  in virtblk_attrs_are_visible()  local
789 	struct virtio_device *vdev = vblk->vdev;  in virtblk_attrs_are_visible()
810 	struct virtio_blk *vblk = set->driver_data;  in virtblk_map_queues()  local
816 		map->nr_queues = vblk->io_queues[i];  in virtblk_map_queues()
831 			blk_mq_virtio_map_queues(&set->map[i], vblk->vdev, 0);  in virtblk_map_queues()
848 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtblk_poll()  local
867 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_poll()
888 	struct virtio_blk *vblk;  in virtblk_probe()  local
923 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
924 	if (!vblk) {  in virtblk_probe()
929 	mutex_init(&vblk->vdev_mutex);  in virtblk_probe()
931 	vblk->vdev = vdev;  in virtblk_probe()
933 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
935 	err = init_vq(vblk);  in virtblk_probe()
941 		queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
949 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
950 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
951 	vblk->tag_set.queue_depth = queue_depth;  in virtblk_probe()
952 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
953 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
954 	vblk->tag_set.cmd_size =  in virtblk_probe()
957 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
958 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
959 	vblk->tag_set.nr_maps = 1;  in virtblk_probe()
960 	if (vblk->io_queues[HCTX_TYPE_POLL])  in virtblk_probe()
961 		vblk->tag_set.nr_maps = 3;  in virtblk_probe()
963 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
967 	vblk->disk = blk_mq_alloc_disk(&vblk->tag_set, vblk);  in virtblk_probe()
968 	if (IS_ERR(vblk->disk)) {  in virtblk_probe()
969 		err = PTR_ERR(vblk->disk);  in virtblk_probe()
972 	q = vblk->disk->queue;  in virtblk_probe()
974 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
976 	vblk->disk->major = major;  in virtblk_probe()
977 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
978 	vblk->disk->minors = 1 << PART_BITS;  in virtblk_probe()
979 	vblk->disk->private_data = vblk;  in virtblk_probe()
980 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
981 	vblk->index = index;  in virtblk_probe()
988 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
1147 	virtblk_update_capacity(vblk, false);  in virtblk_probe()
1150 	err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);  in virtblk_probe()
1157 	put_disk(vblk->disk);  in virtblk_probe()
1159 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
1162 	kfree(vblk->vqs);  in virtblk_probe()
1164 	kfree(vblk);  in virtblk_probe()
1173 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
1176 	flush_work(&vblk->config_work);  in virtblk_remove()
1178 	del_gendisk(vblk->disk);  in virtblk_remove()
1179 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
1181 	mutex_lock(&vblk->vdev_mutex);  in virtblk_remove()
1187 	vblk->vdev = NULL;  in virtblk_remove()
1190 	kfree(vblk->vqs);  in virtblk_remove()
1192 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_remove()
1194 	put_disk(vblk->disk);  in virtblk_remove()
1200 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
1206 	flush_work(&vblk->config_work);  in virtblk_freeze()
1208 	blk_mq_quiesce_queue(vblk->disk->queue);  in virtblk_freeze()
1211 	kfree(vblk->vqs);  in virtblk_freeze()
1218 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
1227 	blk_mq_unquiesce_queue(vblk->disk->queue);  in virtblk_restore()