176 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
183 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
186 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
199 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
200 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
205 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_commit_rqs()  local
206 	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];  in virtio_commit_rqs()
220 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
231 	BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);  in virtio_queue_rq()
256 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);  in virtio_queue_rq()
258 		0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));  in virtio_queue_rq()
259 	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));  in virtio_queue_rq()
272 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);  in virtio_queue_rq()
274 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);  in virtio_queue_rq()
277 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
278 	err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
280 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
286 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
297 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
299 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
302 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
310 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
311 	struct request_queue *q = vblk->disk->queue;  in virtblk_get_id()
323 	blk_execute_rq(vblk->disk, req, false);  in virtblk_get_id()
330 static void virtblk_get(struct virtio_blk *vblk)  in virtblk_get()  argument
332 	refcount_inc(&vblk->refs);  in virtblk_get()
335 static void virtblk_put(struct virtio_blk *vblk)  in virtblk_put()  argument
337 	if (refcount_dec_and_test(&vblk->refs)) {  in virtblk_put()
338 		ida_simple_remove(&vd_index_ida, vblk->index);  in virtblk_put()
339 		mutex_destroy(&vblk->vdev_mutex);  in virtblk_put()
340 		kfree(vblk);  in virtblk_put()
346 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_open()  local
349 	mutex_lock(&vblk->vdev_mutex);  in virtblk_open()
351 	if (vblk->vdev)  in virtblk_open()
352 		virtblk_get(vblk);  in virtblk_open()
356 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_open()
362 	struct virtio_blk *vblk = disk->private_data;  in virtblk_release()  local
364 	virtblk_put(vblk);  in virtblk_release()
370 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
373 	mutex_lock(&vblk->vdev_mutex);  in virtblk_getgeo()
375 	if (!vblk->vdev) {  in virtblk_getgeo()
381 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
382 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
384 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
386 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
395 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_getgeo()
439 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)  in virtblk_update_capacity()  argument
441 	struct virtio_device *vdev = vblk->vdev;  in virtblk_update_capacity()
442 	struct request_queue *q = vblk->disk->queue;  in virtblk_update_capacity()
459 		   vblk->disk->disk_name,  in virtblk_update_capacity()
466 	set_capacity_and_notify(vblk->disk, capacity);  in virtblk_update_capacity()
471 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
474 	virtblk_update_capacity(vblk, true);  in virtblk_config_changed_work()
479 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
481 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
484 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
492 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
503 	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);  in init_vq()
504 	if (!vblk->vqs)  in init_vq()
517 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
518 		names[i] = vblk->vqs[i].name;  in init_vq()
527 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
528 		vblk->vqs[i].vq = vqs[i];  in init_vq()
530 	vblk->num_vqs = num_vqs;  in init_vq()
537 		kfree(vblk->vqs);  in init_vq()
591 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
593 	blk_queue_write_cache(vblk->disk->queue, writeback, false);  in virtblk_update_cache_mode()
605 	struct virtio_blk *vblk = disk->private_data;  in cache_type_store()  local
606 	struct virtio_device *vdev = vblk->vdev;  in cache_type_store()
609 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in cache_type_store()
623 	struct virtio_blk *vblk = disk->private_data;  in cache_type_show()  local
624 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in cache_type_show()
643 	struct virtio_blk *vblk = disk->private_data;  in virtblk_attrs_are_visible()  local
644 	struct virtio_device *vdev = vblk->vdev;  in virtblk_attrs_are_visible()
666 	struct virtio_blk *vblk = set->driver_data;  in virtblk_init_request()  local
669 	sg_init_table(vbr->sg, vblk->sg_elems);  in virtblk_init_request()
675 	struct virtio_blk *vblk = set->driver_data;  in virtblk_map_queues()  local
678 					vblk->vdev, 0);  in virtblk_map_queues()
694 	struct virtio_blk *vblk;  in virtblk_probe()  local
729 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
730 	if (!vblk) {  in virtblk_probe()
736 	refcount_set(&vblk->refs, 1);  in virtblk_probe()
737 	mutex_init(&vblk->vdev_mutex);  in virtblk_probe()
739 	vblk->vdev = vdev;  in virtblk_probe()
740 	vblk->sg_elems = sg_elems;  in virtblk_probe()
742 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
744 	err = init_vq(vblk);  in virtblk_probe()
750 		queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
758 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
759 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
760 	vblk->tag_set.queue_depth = queue_depth;  in virtblk_probe()
761 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
762 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
763 	vblk->tag_set.cmd_size =  in virtblk_probe()
766 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
767 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
769 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
773 	vblk->disk = blk_mq_alloc_disk(&vblk->tag_set, vblk);  in virtblk_probe()
774 	if (IS_ERR(vblk->disk)) {  in virtblk_probe()
775 		err = PTR_ERR(vblk->disk);  in virtblk_probe()
778 	q = vblk->disk->queue;  in virtblk_probe()
780 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
782 	vblk->disk->major = major;  in virtblk_probe()
783 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
784 	vblk->disk->minors = 1 << PART_BITS;  in virtblk_probe()
785 	vblk->disk->private_data = vblk;  in virtblk_probe()
786 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
787 	vblk->disk->flags |= GENHD_FL_EXT_DEVT;  in virtblk_probe()
788 	vblk->index = index;  in virtblk_probe()
795 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
798 	blk_queue_max_segments(q, vblk->sg_elems-2);  in virtblk_probe()
875 	virtblk_update_capacity(vblk, false);  in virtblk_probe()
878 	err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);  in virtblk_probe()
885 	blk_cleanup_disk(vblk->disk);  in virtblk_probe()
887 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
890 	kfree(vblk->vqs);  in virtblk_probe()
892 	kfree(vblk);  in virtblk_probe()
901 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
904 	flush_work(&vblk->config_work);  in virtblk_remove()
906 	del_gendisk(vblk->disk);  in virtblk_remove()
907 	blk_cleanup_disk(vblk->disk);  in virtblk_remove()
908 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
910 	mutex_lock(&vblk->vdev_mutex);  in virtblk_remove()
916 	vblk->vdev = NULL;  in virtblk_remove()
919 	kfree(vblk->vqs);  in virtblk_remove()
921 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_remove()
923 	virtblk_put(vblk);  in virtblk_remove()
929 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
935 	flush_work(&vblk->config_work);  in virtblk_freeze()
937 	blk_mq_quiesce_queue(vblk->disk->queue);  in virtblk_freeze()
940 	kfree(vblk->vqs);  in virtblk_freeze()
947 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
956 	blk_mq_unquiesce_queue(vblk->disk->queue);  in virtblk_restore()