176 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
183 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
186 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
199 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
200 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
205 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_commit_rqs()  local
206 	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];  in virtio_commit_rqs()
220 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
231 	BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);  in virtio_queue_rq()
256 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);  in virtio_queue_rq()
258 		0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));  in virtio_queue_rq()
259 	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));  in virtio_queue_rq()
272 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);  in virtio_queue_rq()
274 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);  in virtio_queue_rq()
277 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
278 	err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
280 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
286 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
297 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
299 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
302 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
310 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
311 	struct request_queue *q = vblk->disk->queue;  in virtblk_get_id()
323 	blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);  in virtblk_get_id()
330 static void virtblk_get(struct virtio_blk *vblk)  in virtblk_get()  argument
332 	refcount_inc(&vblk->refs);  in virtblk_get()
335 static void virtblk_put(struct virtio_blk *vblk)  in virtblk_put()  argument
337 	if (refcount_dec_and_test(&vblk->refs)) {  in virtblk_put()
338 		ida_simple_remove(&vd_index_ida, vblk->index);  in virtblk_put()
339 		mutex_destroy(&vblk->vdev_mutex);  in virtblk_put()
340 		kfree(vblk);  in virtblk_put()
346 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_open()  local
349 	mutex_lock(&vblk->vdev_mutex);  in virtblk_open()
351 	if (vblk->vdev)  in virtblk_open()
352 		virtblk_get(vblk);  in virtblk_open()
356 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_open()
362 	struct virtio_blk *vblk = disk->private_data;  in virtblk_release()  local
364 	virtblk_put(vblk);  in virtblk_release()
370 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
373 	mutex_lock(&vblk->vdev_mutex);  in virtblk_getgeo()
375 	if (!vblk->vdev) {  in virtblk_getgeo()
381 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
382 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
384 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
386 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
395 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_getgeo()
439 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)  in virtblk_update_capacity()  argument
441 	struct virtio_device *vdev = vblk->vdev;  in virtblk_update_capacity()
442 	struct request_queue *q = vblk->disk->queue;  in virtblk_update_capacity()
466 		   vblk->disk->disk_name,  in virtblk_update_capacity()
473 	set_capacity_revalidate_and_notify(vblk->disk, capacity, true);  in virtblk_update_capacity()
478 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
481 	virtblk_update_capacity(vblk, true);  in virtblk_config_changed_work()
486 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
488 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
491 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
499 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
510 	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);  in init_vq()
511 	if (!vblk->vqs)  in init_vq()
524 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
525 		names[i] = vblk->vqs[i].name;  in init_vq()
534 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
535 		vblk->vqs[i].vq = vqs[i];  in init_vq()
537 	vblk->num_vqs = num_vqs;  in init_vq()
544 		kfree(vblk->vqs);  in init_vq()
598 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
600 	blk_queue_write_cache(vblk->disk->queue, writeback, false);  in virtblk_update_cache_mode()
601 	revalidate_disk_size(vblk->disk, true);  in virtblk_update_cache_mode()
613 	struct virtio_blk *vblk = disk->private_data;  in cache_type_store()  local
614 	struct virtio_device *vdev = vblk->vdev;  in cache_type_store()
617 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in cache_type_store()
631 	struct virtio_blk *vblk = disk->private_data;  in cache_type_show()  local
632 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in cache_type_show()
651 	struct virtio_blk *vblk = disk->private_data;  in virtblk_attrs_are_visible()  local
652 	struct virtio_device *vdev = vblk->vdev;  in virtblk_attrs_are_visible()
674 	struct virtio_blk *vblk = set->driver_data;  in virtblk_init_request()  local
677 	sg_init_table(vbr->sg, vblk->sg_elems);  in virtblk_init_request()
683 	struct virtio_blk *vblk = set->driver_data;  in virtblk_map_queues()  local
686 					vblk->vdev, 0);  in virtblk_map_queues()
702 	struct virtio_blk *vblk;  in virtblk_probe()  local
733 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
734 	if (!vblk) {  in virtblk_probe()
740 	refcount_set(&vblk->refs, 1);  in virtblk_probe()
741 	mutex_init(&vblk->vdev_mutex);  in virtblk_probe()
743 	vblk->vdev = vdev;  in virtblk_probe()
744 	vblk->sg_elems = sg_elems;  in virtblk_probe()
746 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
748 	err = init_vq(vblk);  in virtblk_probe()
753 	vblk->disk = alloc_disk(1 << PART_BITS);  in virtblk_probe()
754 	if (!vblk->disk) {  in virtblk_probe()
761 		virtblk_queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
767 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
768 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
769 	vblk->tag_set.queue_depth = virtblk_queue_depth;  in virtblk_probe()
770 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
771 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
772 	vblk->tag_set.cmd_size =  in virtblk_probe()
775 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
776 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
778 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
782 	q = blk_mq_init_queue(&vblk->tag_set);  in virtblk_probe()
787 	vblk->disk->queue = q;  in virtblk_probe()
789 	q->queuedata = vblk;  in virtblk_probe()
791 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
793 	vblk->disk->major = major;  in virtblk_probe()
794 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
795 	vblk->disk->private_data = vblk;  in virtblk_probe()
796 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
797 	vblk->disk->flags |= GENHD_FL_EXT_DEVT;  in virtblk_probe()
798 	vblk->index = index;  in virtblk_probe()
805 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
808 	blk_queue_max_segments(q, vblk->sg_elems-2);  in virtblk_probe()
885 	virtblk_update_capacity(vblk, false);  in virtblk_probe()
888 	device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);  in virtblk_probe()
892 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
894 	put_disk(vblk->disk);  in virtblk_probe()
897 	kfree(vblk->vqs);  in virtblk_probe()
899 	kfree(vblk);  in virtblk_probe()
908 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
911 	flush_work(&vblk->config_work);  in virtblk_remove()
913 	del_gendisk(vblk->disk);  in virtblk_remove()
914 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_remove()
916 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
918 	mutex_lock(&vblk->vdev_mutex);  in virtblk_remove()
924 	vblk->vdev = NULL;  in virtblk_remove()
926 	put_disk(vblk->disk);  in virtblk_remove()
928 	kfree(vblk->vqs);  in virtblk_remove()
930 	mutex_unlock(&vblk->vdev_mutex);  in virtblk_remove()
932 	virtblk_put(vblk);  in virtblk_remove()
938 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
944 	flush_work(&vblk->config_work);  in virtblk_freeze()
946 	blk_mq_quiesce_queue(vblk->disk->queue);  in virtblk_freeze()
954 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
963 	blk_mq_unquiesce_queue(vblk->disk->queue);  in virtblk_restore()