119 	struct virtio_blk *vblk = req->q->queuedata;  in virtblk_scsi_request_done()  local
122 	sreq->resid_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.residual);  in virtblk_scsi_request_done()
123 	sreq->sense_len = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.sense_len);  in virtblk_scsi_request_done()
124 	sreq->result = virtio32_to_cpu(vblk->vdev, vbr->in_hdr.errors);  in virtblk_scsi_request_done()
131 	struct virtio_blk *vblk = disk->private_data;  in virtblk_ioctl()  local
136 	if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))  in virtblk_ioctl()
231 	struct virtio_blk *vblk = vq->vdev->priv;  in virtblk_done()  local
238 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtblk_done()
241 		while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {  in virtblk_done()
253 		blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);  in virtblk_done()
254 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtblk_done()
259 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_commit_rqs()  local
260 	struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];  in virtio_commit_rqs()
274 	struct virtio_blk *vblk = hctx->queue->queuedata;  in virtio_queue_rq()  local
285 	BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);  in virtio_queue_rq()
314 	vbr->out_hdr.type = cpu_to_virtio32(vblk->vdev, type);  in virtio_queue_rq()
316 		0 : cpu_to_virtio64(vblk->vdev, blk_rq_pos(req));  in virtio_queue_rq()
317 	vbr->out_hdr.ioprio = cpu_to_virtio32(vblk->vdev, req_get_ioprio(req));  in virtio_queue_rq()
330 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_OUT);  in virtio_queue_rq()
332 			vbr->out_hdr.type |= cpu_to_virtio32(vblk->vdev, VIRTIO_BLK_T_IN);  in virtio_queue_rq()
335 	spin_lock_irqsave(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
337 		err = virtblk_add_req_scsi(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
339 		err = virtblk_add_req(vblk->vqs[qid].vq, vbr, vbr->sg, num);  in virtio_queue_rq()
341 		virtqueue_kick(vblk->vqs[qid].vq);  in virtio_queue_rq()
343 		spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
351 	if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))  in virtio_queue_rq()
353 	spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);  in virtio_queue_rq()
356 		virtqueue_notify(vblk->vqs[qid].vq);  in virtio_queue_rq()
364 	struct virtio_blk *vblk = disk->private_data;  in virtblk_get_id()  local
365 	struct request_queue *q = vblk->disk->queue;  in virtblk_get_id()
377 	blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);  in virtblk_get_id()
387 	struct virtio_blk *vblk = bd->bd_disk->private_data;  in virtblk_getgeo()  local
390 	if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {  in virtblk_getgeo()
391 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
393 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
395 		virtio_cread(vblk->vdev, struct virtio_blk_config,  in virtblk_getgeo()
445 static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)  in virtblk_update_capacity()  argument
447 	struct virtio_device *vdev = vblk->vdev;  in virtblk_update_capacity()
448 	struct request_queue *q = vblk->disk->queue;  in virtblk_update_capacity()
472 		   vblk->disk->disk_name,  in virtblk_update_capacity()
479 	set_capacity(vblk->disk, capacity);  in virtblk_update_capacity()
484 	struct virtio_blk *vblk =  in virtblk_config_changed_work()  local
488 	virtblk_update_capacity(vblk, true);  in virtblk_config_changed_work()
489 	revalidate_disk(vblk->disk);  in virtblk_config_changed_work()
490 	kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp);  in virtblk_config_changed_work()
495 	struct virtio_blk *vblk = vdev->priv;  in virtblk_config_changed()  local
497 	queue_work(virtblk_wq, &vblk->config_work);  in virtblk_config_changed()
500 static int init_vq(struct virtio_blk *vblk)  in init_vq()  argument
508 	struct virtio_device *vdev = vblk->vdev;  in init_vq()
519 	vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);  in init_vq()
520 	if (!vblk->vqs)  in init_vq()
533 		snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);  in init_vq()
534 		names[i] = vblk->vqs[i].name;  in init_vq()
543 		spin_lock_init(&vblk->vqs[i].lock);  in init_vq()
544 		vblk->vqs[i].vq = vqs[i];  in init_vq()
546 	vblk->num_vqs = num_vqs;  in init_vq()
553 		kfree(vblk->vqs);  in init_vq()
607 	struct virtio_blk *vblk = vdev->priv;  in virtblk_update_cache_mode()  local
609 	blk_queue_write_cache(vblk->disk->queue, writeback, false);  in virtblk_update_cache_mode()
610 	revalidate_disk(vblk->disk);  in virtblk_update_cache_mode()
622 	struct virtio_blk *vblk = disk->private_data;  in cache_type_store()  local
623 	struct virtio_device *vdev = vblk->vdev;  in cache_type_store()
626 	BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));  in cache_type_store()
640 	struct virtio_blk *vblk = disk->private_data;  in cache_type_show()  local
641 	u8 writeback = virtblk_get_cache_mode(vblk->vdev);  in cache_type_show()
660 	struct virtio_blk *vblk = disk->private_data;  in virtblk_attrs_are_visible()  local
661 	struct virtio_device *vdev = vblk->vdev;  in virtblk_attrs_are_visible()
683 	struct virtio_blk *vblk = set->driver_data;  in virtblk_init_request()  local
689 	sg_init_table(vbr->sg, vblk->sg_elems);  in virtblk_init_request()
695 	struct virtio_blk *vblk = set->driver_data;  in virtblk_map_queues()  local
698 					vblk->vdev, 0);  in virtblk_map_queues()
726 	struct virtio_blk *vblk;  in virtblk_probe()  local
757 	vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);  in virtblk_probe()
758 	if (!vblk) {  in virtblk_probe()
763 	vblk->vdev = vdev;  in virtblk_probe()
764 	vblk->sg_elems = sg_elems;  in virtblk_probe()
766 	INIT_WORK(&vblk->config_work, virtblk_config_changed_work);  in virtblk_probe()
768 	err = init_vq(vblk);  in virtblk_probe()
773 	vblk->disk = alloc_disk(1 << PART_BITS);  in virtblk_probe()
774 	if (!vblk->disk) {  in virtblk_probe()
781 		virtblk_queue_depth = vblk->vqs[0].vq->num_free;  in virtblk_probe()
787 	memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));  in virtblk_probe()
788 	vblk->tag_set.ops = &virtio_mq_ops;  in virtblk_probe()
789 	vblk->tag_set.queue_depth = virtblk_queue_depth;  in virtblk_probe()
790 	vblk->tag_set.numa_node = NUMA_NO_NODE;  in virtblk_probe()
791 	vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;  in virtblk_probe()
792 	vblk->tag_set.cmd_size =  in virtblk_probe()
795 	vblk->tag_set.driver_data = vblk;  in virtblk_probe()
796 	vblk->tag_set.nr_hw_queues = vblk->num_vqs;  in virtblk_probe()
798 	err = blk_mq_alloc_tag_set(&vblk->tag_set);  in virtblk_probe()
802 	q = blk_mq_init_queue(&vblk->tag_set);  in virtblk_probe()
807 	vblk->disk->queue = q;  in virtblk_probe()
809 	q->queuedata = vblk;  in virtblk_probe()
811 	virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);  in virtblk_probe()
813 	vblk->disk->major = major;  in virtblk_probe()
814 	vblk->disk->first_minor = index_to_minor(index);  in virtblk_probe()
815 	vblk->disk->private_data = vblk;  in virtblk_probe()
816 	vblk->disk->fops = &virtblk_fops;  in virtblk_probe()
817 	vblk->disk->flags |= GENHD_FL_EXT_DEVT;  in virtblk_probe()
818 	vblk->index = index;  in virtblk_probe()
825 		set_disk_ro(vblk->disk, 1);  in virtblk_probe()
828 	blk_queue_max_segments(q, vblk->sg_elems-2);  in virtblk_probe()
905 	virtblk_update_capacity(vblk, false);  in virtblk_probe()
908 	device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);  in virtblk_probe()
912 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_probe()
914 	put_disk(vblk->disk);  in virtblk_probe()
918 	kfree(vblk);  in virtblk_probe()
927 	struct virtio_blk *vblk = vdev->priv;  in virtblk_remove()  local
928 	int index = vblk->index;  in virtblk_remove()
932 	flush_work(&vblk->config_work);  in virtblk_remove()
934 	del_gendisk(vblk->disk);  in virtblk_remove()
935 	blk_cleanup_queue(vblk->disk->queue);  in virtblk_remove()
937 	blk_mq_free_tag_set(&vblk->tag_set);  in virtblk_remove()
942 	refc = kref_read(&disk_to_dev(vblk->disk)->kobj.kref);  in virtblk_remove()
943 	put_disk(vblk->disk);  in virtblk_remove()
945 	kfree(vblk->vqs);  in virtblk_remove()
946 	kfree(vblk);  in virtblk_remove()
956 	struct virtio_blk *vblk = vdev->priv;  in virtblk_freeze()  local
962 	flush_work(&vblk->config_work);  in virtblk_freeze()
964 	blk_mq_quiesce_queue(vblk->disk->queue);  in virtblk_freeze()
972 	struct virtio_blk *vblk = vdev->priv;  in virtblk_restore()  local
981 	blk_mq_unquiesce_queue(vblk->disk->queue);  in virtblk_restore()